diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 80021f7..9f03838 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -51,54 +51,16 @@ run() { fi } -run "inline comment ban" bash -c ' - mapfile -t files < <(git diff --cached --name-only --diff-filter=ACMR -- "*.rs") - if ((${#files[@]} == 0)); then - exit 0 - fi - if grep -En "^\s*//[^/!]" "${files[@]}" 2>/dev/null; then - echo "" >&2 - echo "pre-commit: inline // comments are forbidden; use /// or //! docstrings only." >&2 - exit 1 - fi - if grep -En "[^:\"'\''\`]\s+//[^/!]" "${files[@]}" 2>/dev/null; then - echo "" >&2 - echo "pre-commit: end-of-line // comments are forbidden; use /// or //! docstrings only." >&2 - exit 1 - fi -' +run "Rust source rules" scripts/check-rust-file-size.sh run "cargo fmt --all -- --check" cargo fmt --all -- --check run "cargo clippy --all-targets -- -D warnings" cargo clippy --all-targets -- -D warnings -run "cargo test --lib --workspace" cargo test --lib --workspace +run "cargo test --lib --workspace" scripts/cargo-test-isolated-home.sh test --lib --workspace -# ── FFI-specific gates ──────────────────────────────────────────────────────── -# Mirrors CI jobs: ffi-codegen-drift, ffi-passthrough, ffi-header-drift. -# Fast-skipped in full when no crates/ffi/** files are staged. -# -# Scope: only the fast structural gates run locally (codegen drift, stub -# passthrough, optional header drift). panic_spike (requires a release-ffi -# example build) and the Python smoke harness (requires a full cdylib build -# plus a Python runtime) are CI-only — jobs ffi-panic-guard and -# ffi-python-smoke — to keep commits fast. +# FFI-specific gates are skipped when no FFI files are staged. Runtime panic +# and Python smoke proofs remain CI-only to keep commits fast. if [ "$HAS_FFI_CHANGES" -eq 1 ]; then - # Gate 1: Codegen drift — build triggers build.rs which regenerates - # generated.rs in-source; diff detects a stale staged copy. - run "FFI codegen build" cargo build --locked -p agent-desktop-ffi --features stub-adapter - if ! git diff --quiet crates/ffi/src/commands/generated.rs; then - printf '\033[1;31m✗ FFI codegen drift\033[0m\n' >&2 - printf '\n' >&2 - printf 'pre-commit: crates/ffi/src/commands/generated.rs is stale.\n' >&2 - printf ' The hook already regenerated it — just stage the updated file:\n' >&2 - printf ' git add crates/ffi/src/commands/generated.rs\n' >&2 - printf '\npre-commit: refusing the commit. Fix the issues above or rerun with SKIP_PRECOMMIT=1 to bypass.\n' >&2 - exit 1 - fi - - # Gate 2: Stub-adapter passthrough tests. run "FFI stub passthrough" cargo test --locked -p agent-desktop-ffi --features stub-adapter --test c_abi_passthrough - # Gate 3: Header drift — graceful skip if cbindgen is absent or not 0.29.4. - # The hard gate is CI job ffi-header-drift; local absence is not a commit blocker. export PATH="${HOME}/.cargo/bin:${PATH}" _cbindgen_cmd="" if command -v cbindgen >/dev/null 2>&1; then diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..715d0d7 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - agent-desktop-e2e diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdd3959..ab3af47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,51 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - run: rustup component add rustfmt - run: cargo fmt --all -- --check + - name: Check E2E scripts without running native automation + run: | + scripts/check-bash3-compat.sh + shellcheck -x -e SC2034,SC2154 tests/e2e/run.sh tests/e2e/electron-live.sh tests/e2e/permission-contract.sh tests/e2e/guard-command.sh scripts/*.sh .githooks/pre-commit + PYTHONPYCACHEPREFIX=/tmp/agent-desktop-pycache python3 -m py_compile tests/e2e/*.py + PYTHONPYCACHEPREFIX=/tmp/agent-desktop-pycache python3 -m unittest discover -s tests/e2e -p 'test_*.py' + - name: Validate GitHub Actions workflows + run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.11 + + msrv: + name: Rust 1.89 MSRV + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - run: rustup toolchain install 1.89.0 --profile minimal + - run: cargo +1.89.0 check --locked -p agent-desktop-core -p agent-desktop-linux -p agent-desktop --all-targets + + platform-check: + name: Native check (${{ matrix.platform }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 25 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: Linux + os: ubuntu-latest + package: agent-desktop-linux + - platform: Windows + os: windows-latest + package: agent-desktop-windows + - platform: macOS + os: macos-latest + package: agent-desktop-macos + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Install pinned Rust toolchain + run: rustup show + - name: Check native adapter and binary + run: cargo check --locked -p agent-desktop-core -p ${{ matrix.package }} -p agent-desktop --all-targets test: name: Test @@ -40,7 +85,12 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install Rust toolchain - run: rustup show + run: | + rustup show + rustup component add clippy + + - name: Verify macOS Bash 3.2 compatibility + run: /bin/bash scripts/check-bash3-compat.sh - name: Cache cargo registry uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -63,7 +113,8 @@ jobs: - name: Check dependency isolation run: | - if cargo tree --locked -p agent-desktop-core 2>/dev/null | grep -E 'agent-desktop-(macos|windows|linux)'; then + TREE=$(cargo tree --locked -p agent-desktop-core --edges normal,build) + if printf '%s\n' "$TREE" | grep -E 'agent-desktop-(macos|windows|linux)'; then echo "FAIL: core crate depends on platform crates" exit 1 fi @@ -72,68 +123,77 @@ jobs: - name: Check release metadata consistency run: scripts/check-release-consistency.sh - - name: Clippy - run: cargo clippy --locked --all-targets -- -D warnings + - name: Enforce Rust source rules + run: scripts/check-rust-file-size.sh - - name: Unit tests - run: cargo test --locked --lib --workspace + - name: Clippy + run: cargo clippy --locked -p agent-desktop-core -p agent-desktop-macos -p agent-desktop -p agent-desktop-ffi --all-targets -- -D warnings + + - name: Core and macOS unit tests + run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-core -p agent-desktop-macos --lib + + - name: Locator benchmark adapter contract + run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-core --example locator_benchmark + + - name: Deterministic permission acceptance contract + run: bash tests/e2e/permission-contract.sh - name: Binary command tests - run: cargo test --locked -p agent-desktop + run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop # The FFI crate ships integration harnesses under crates/ffi/tests/ that # exercise the C-ABI from outside the crate (raw extern "C" decls, enum # fuzzing, out-param zeroing, last-error lifetimes). The workspace --lib # run skips them, so wire them in explicitly so regressions fail PR CI. - name: FFI integration tests - run: cargo test --locked -p agent-desktop-ffi --tests + run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-ffi --tests - # Use the ci profile (no LTO, opt-level 1) — fast compile, still checks the binary builds. - - name: Build binary (ci profile) - run: cargo build --locked --profile ci - - - name: Check binary size + - name: Build stripped release binary run: | - SIZE=$(stat -f%z target/ci/agent-desktop) - LIMIT=$((15 * 1024 * 1024)) - echo "Binary size: $(du -sh target/ci/agent-desktop | cut -f1)" - if [ "$SIZE" -gt "$LIMIT" ]; then - echo "FAIL: binary exceeds 15MB limit (${SIZE} bytes)" + cargo build --locked --release -p agent-desktop + cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper + + - name: Verify shipped binary version flag + run: | + PACKAGE_ID=$(cargo pkgid -p agent-desktop) + EXPECTED_VERSION=${PACKAGE_ID##*@} + EXPECTED_OUTPUT="agent-desktop $EXPECTED_VERSION" + ACTUAL_OUTPUT=$(target/release/agent-desktop --version) + if [ "$ACTUAL_OUTPUT" != "$EXPECTED_OUTPUT" ]; then + echo "FAIL: expected '$EXPECTED_OUTPUT', got '$ACTUAL_OUTPUT'" exit 1 fi - echo "OK: binary within 15MB limit" + + - name: Check shipped binary size + run: | + SIZE=$(stat -f%z target/release/agent-desktop) + HELPER_SIZE=$(stat -f%z target/release/agent-desktop-macos-helper) + LIMIT=$((15 * 1024 * 1024)) + echo "Binary size: $(du -sh target/release/agent-desktop | cut -f1)" + echo "Helper size: $(du -sh target/release/agent-desktop-macos-helper | cut -f1)" + if [ "$SIZE" -gt "$LIMIT" ] || [ "$HELPER_SIZE" -gt "$LIMIT" ]; then + echo "FAIL: shipped executable exceeds 15MB limit (binary=${SIZE}, helper=${HELPER_SIZE} bytes)" + exit 1 + fi + echo "OK: shipped executables are within the 15MB per-file limit" - name: FFI cdylib build (release-ffi profile) run: cargo build --locked --profile release-ffi -p agent-desktop-ffi + - name: FFI dylib-adjacent helper discovery smoke + run: scripts/ci-ffi-helper-discovery-smoke.sh + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '24' - name: NPM package contents - run: node scripts/check-npm-package.js + run: | + node --test tests/npm/*.test.js + node scripts/check-npm-package.js - name: NPM wrapper smoke - run: | - case "$(uname -s)-$(uname -m)" in - Darwin-arm64) NAME=agent-desktop-darwin-arm64 ;; - Darwin-x86_64) NAME=agent-desktop-darwin-x64 ;; - *) - echo "Unsupported smoke-test platform: $(uname -s)-$(uname -m)" - exit 1 - ;; - esac - trap 'rm -f "npm/bin/${NAME}"' EXIT - cp target/ci/agent-desktop "npm/bin/${NAME}" - chmod +x "npm/bin/${NAME}" - node npm/bin/agent-desktop.js version > /tmp/agent-desktop-version.json - node -e " - const out = require('fs').readFileSync('/tmp/agent-desktop-version.json', 'utf8'); - const json = JSON.parse(out); - if (json.ok !== true || !json.data || typeof json.data.version !== 'string') { - throw new Error('agent-desktop npm wrapper did not return version JSON'); - } - " + run: scripts/ci-npm-wrapper-smoke.sh ffi-python-smoke: name: FFI Python Smoke @@ -232,11 +292,9 @@ jobs: # This job proves the contract in two steps: # 1. Assert the Cargo.toml profile setting directly (simpler and more # reliable than a build.rs env check for a dual cdylib+rlib crate). - # 2. Build and run crates/ffi/examples/panic_spike under the release-ffi - # profile: an extern-C fn panics, catch_unwind catches it, and the - # example prints "PANIC CAUGHT OK (code = -1)" and exits 0. If the - # profile were flipped to panic="abort" the process would SIGABRT - # instead, making this step a runtime proof — not just a string check. + # 2. Build the actual feature-gated cdylib panic entrypoint, load it with + # dlopen from a C process, and require the exported ABI boundary to + # return ErrInternal instead of aborting the host. ffi-panic-guard: name: FFI Panic Guard runs-on: macos-latest @@ -268,8 +326,8 @@ jobs: fi echo "OK: release-ffi profile retains panic=\"unwind\"" - - name: Prove catch_unwind survives the release-ffi profile - run: cargo run --locked --profile release-ffi --example panic_spike -p agent-desktop-ffi + - name: Prove the shipped cdylib catches an exported-boundary panic + run: bash crates/ffi/tests/run_cdylib_panic_probe.sh # Gate 3 — Stub-adapter passthrough tests: the Family-B command-backed # entrypoints (ad_snapshot, ad_status, ad_wait, ad_execute_by_ref, @@ -286,10 +344,9 @@ jobs: # documented follow-up. # # Per-target build coverage: - # - PR CI: this job runs on Linux (no main-thread guard off macOS, so the - # stub adapter is reached directly) to assert PLATFORM_NOT_SUPPORTED for - # every covered entrypoint — directly proving the non-macOS parity - # U10 targets. + # - PR CI: this job runs on Linux and reaches the stub adapter directly to + # assert PLATFORM_NOT_SUPPORTED for every covered entrypoint, proving the + # non-macOS parity U10 targets. # - Release: release.yml build-ffi matrix covers macOS×2 + Linux×2 + # Windows×1 for the real cdylib. PR builds do not duplicate that # cross-compile matrix to avoid bloating CI on every push. @@ -317,44 +374,5 @@ jobs: - name: Run stub-adapter passthrough tests run: > - cargo test --locked -p agent-desktop-ffi + scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-ffi --features stub-adapter --test c_abi_passthrough - - # Gate 4 — Codegen drift: the committed src/commands/generated.rs must - # match what build.rs produces from the current templates. A stale committed - # file means the source diverged from the templates — either a hand-edit - # bypassed the generator or a template was changed without rebuilding. - ffi-codegen-drift: - name: FFI Codegen Drift - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Install Rust toolchain - run: rustup show - - - name: Cache cargo registry - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock', 'Cargo.toml', 'crates/**/Cargo.toml', 'src/Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Regenerate FFI command wrappers via build.rs - run: cargo build --locked -p agent-desktop-ffi --features stub-adapter - - - name: Assert no drift in commands/generated.rs - run: | - if ! git diff --exit-code crates/ffi/src/commands/generated.rs; then - echo "" - echo "FAIL: crates/ffi/src/commands/generated.rs is out of sync with build.rs templates." - echo " Run 'cargo build -p agent-desktop-ffi' locally and commit the updated file." - exit 1 - fi - echo "OK: commands/generated.rs matches build.rs output" diff --git a/.github/workflows/native-e2e.yml b/.github/workflows/native-e2e.yml new file mode 100644 index 0000000..c7bf711 --- /dev/null +++ b/.github/workflows/native-e2e.yml @@ -0,0 +1,26 @@ +name: Native desktop E2E + +on: + workflow_dispatch: + +concurrency: + group: agent-desktop-native-e2e + cancel-in-progress: false + +permissions: {} + +jobs: + native-e2e: + name: Native macOS E2E + runs-on: [self-hosted, macOS, agent-desktop-e2e] + timeout-minutes: 45 + permissions: + contents: read + env: + AGENT_DESKTOP_NATIVE_E2E_RUNNER: "1" + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Install pinned Rust toolchain + run: rustup show + - name: Build and run the exclusive native suite + run: scripts/run-native-e2e-ci.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 299d3b5..262fc8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -156,15 +156,19 @@ jobs: restore-keys: ${{ runner.os }}-${{ matrix.target }}-cargo- - name: Build release binary - run: cargo build --locked --release --target ${{ matrix.target }} + run: | + cargo build --locked --release -p agent-desktop --target ${{ matrix.target }} + cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper --target ${{ matrix.target }} - name: Check binary size run: | SIZE=$(stat -f%z target/${{ matrix.target }}/release/agent-desktop) + HELPER_SIZE=$(stat -f%z target/${{ matrix.target }}/release/agent-desktop-macos-helper) LIMIT=$((15 * 1024 * 1024)) echo "Binary size: $(du -sh target/${{ matrix.target }}/release/agent-desktop | cut -f1)" - if [ "$SIZE" -gt "$LIMIT" ]; then - echo "FAIL: binary exceeds 15MB limit (${SIZE} bytes)" + echo "Helper size: $(du -sh target/${{ matrix.target }}/release/agent-desktop-macos-helper | cut -f1)" + if [ "$SIZE" -gt "$LIMIT" ] || [ "$HELPER_SIZE" -gt "$LIMIT" ]; then + echo "FAIL: shipped executable exceeds 15MB limit (binary=${SIZE}, helper=${HELPER_SIZE} bytes)" exit 1 fi @@ -173,7 +177,16 @@ jobs: VERSION: ${{ needs.release-please.outputs.version }} run: | TARBALL="agent-desktop-v${VERSION}-${{ matrix.target }}.tar.gz" - tar -czf "${TARBALL}" -C target/${{ matrix.target }}/release agent-desktop + BINARY="target/${{ matrix.target }}/release/agent-desktop" + HELPER="target/${{ matrix.target }}/release/agent-desktop-macos-helper" + BINARY_SHA=$(shasum -a 256 "$BINARY" | awk '{print $1}') + HELPER_SHA=$(shasum -a 256 "$HELPER" | awk '{print $1}') + tar -czf "${TARBALL}" -C "$(dirname "$BINARY")" agent-desktop agent-desktop-macos-helper + if [ "$BINARY_SHA" != "$(shasum -a 256 "$BINARY" | awk '{print $1}')" ] || \ + [ "$HELPER_SHA" != "$(shasum -a 256 "$HELPER" | awk '{print $1}')" ]; then + echo "Release executable changed while packaging" >&2 + exit 1 + fi shasum -a 256 "${TARBALL}" > "${TARBALL}.sha256" - name: Upload artifact @@ -245,7 +258,7 @@ jobs: shell: bash run: | # Co-occurrence of the release profile token and the stub flag on the same - # line is the discriminator. Legitimate stub uses in this file (codegen regen, + # line is the discriminator. Legitimate stub uses in this file (release gates, # passthrough test) omit the release-ffi profile, so they do not trigger. PROFILE_TOKEN="release-ffi" STUB_TOKEN="stub-adapter" @@ -260,6 +273,10 @@ jobs: - name: Build FFI cdylib (release-ffi profile) run: cargo build --locked --profile release-ffi -p agent-desktop-ffi --target ${{ matrix.target }} + - name: Build macOS helper for FFI consumers + if: runner.os == 'macOS' + run: cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper --target ${{ matrix.target }} + # build.rs bakes install_name=@rpath/... ; a regression here silently # breaks Swift/SPM consumers, so verify before shipping. - name: Verify macOS install_name @@ -282,6 +299,10 @@ jobs: STAGE="agent-desktop-ffi-v${VERSION}-${{ matrix.target }}" mkdir -p "$STAGE/lib" "$STAGE/include" cp "target/${{ matrix.target }}/release-ffi/${{ matrix.lib_name }}" "$STAGE/lib/" + if [ "$RUNNER_OS" = "macOS" ]; then + cp "target/${{ matrix.target }}/release/agent-desktop-macos-helper" "$STAGE/lib/" + chmod +x "$STAGE/lib/agent-desktop-macos-helper" + fi cp crates/ffi/include/agent_desktop.h "$STAGE/include/" cp LICENSE "$STAGE/" cat > "$STAGE/README.md" <> "$STAGE/README.md" + fi + if [ "$RUNNER_OS" != "Windows" ]; then + (cd "$STAGE" && shasum -a 256 lib/* include/* LICENSE > SHA256SUMS) + fi echo "STAGE=$STAGE" >> "$GITHUB_ENV" - name: Create archive (Unix) @@ -304,6 +332,10 @@ jobs: shell: bash run: | tar -czf "${STAGE}.tar.gz" "${STAGE}" + (cd "$STAGE" && shasum -a 256 -c SHA256SUMS) + if [ "$RUNNER_OS" = "macOS" ]; then + tar -tzf "${STAGE}.tar.gz" | grep -Fx "${STAGE}/lib/agent-desktop-macos-helper" + fi shasum -a 256 "${STAGE}.tar.gz" > "${STAGE}.tar.gz.sha256" - name: Create archive (Windows) @@ -325,11 +357,8 @@ jobs: agent-desktop-ffi-v*.zip.sha256 retention-days: 1 - # Re-runs the three FFI integrity gates at release time so a hotfix or a PR - # that bypassed ci.yml cannot ship a dylib with a stale committed header or - # stale generated.rs. Gate 1 (header drift) must run on macos-latest because - # the committed header is macOS-generated and cbindgen may omit macOS-gated - # symbols on Linux. Gates 2 and 3 are consolidated here for a single job. + # Re-runs FFI integrity gates at release time. Header verification runs on + # macOS because cbindgen may omit macOS-gated symbols on Linux. ffi-release-gates: name: FFI Release Gates needs: release-please @@ -359,9 +388,6 @@ jobs: - name: Install pinned cbindgen 0.29.4 run: cargo install cbindgen --version 0.29.4 --locked - # Gate 1 — Header drift: committed crates/ffi/include/agent_desktop.h must - # match what cbindgen 0.29.4 generates from the current source. Exits - # non-zero when there is any diff. - name: Header drift check (cbindgen --verify) run: > cbindgen crates/ffi @@ -369,32 +395,11 @@ jobs: --output crates/ffi/include/agent_desktop.h --verify - # Gate 2 — Codegen drift: build.rs must produce the exact same - # crates/ffi/src/commands/generated.rs that is committed. - - name: Regenerate FFI command wrappers via build.rs - run: cargo build --locked -p agent-desktop-ffi --features stub-adapter - - - name: Codegen drift check (generated.rs) - run: | - if ! git diff --exit-code crates/ffi/src/commands/generated.rs; then - echo "" - echo "FAIL: crates/ffi/src/commands/generated.rs is out of sync with build.rs templates." - echo " Run 'cargo build -p agent-desktop-ffi' locally and commit the updated file." - exit 1 - fi - echo "OK: commands/generated.rs matches build.rs output" - - # Gate 3 — Stub passthrough: command-backed entrypoints must return - # structured PLATFORM_NOT_SUPPORTED envelopes via the stub adapter. - name: Stub passthrough test run: > - cargo test --locked -p agent-desktop-ffi + scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-ffi --features stub-adapter --test c_abi_passthrough - # Gate 4 — Panic-boundary proof: mirrors ffi-panic-guard in ci.yml so a - # tag-only flip to panic=abort cannot slip past release gates. Two steps: - # (a) assert the Cargo.toml profile setting; (b) run the panic_spike example - # which exits 0 if catch_unwind caught the panic, SIGABRTs if panic=abort. - name: Assert release-ffi profile keeps panic=unwind run: | if ! grep -A5 '\[profile\.release-ffi\]' Cargo.toml | grep -q 'panic.*=.*"unwind"'; then @@ -404,8 +409,8 @@ jobs: fi echo "OK: release-ffi profile retains panic=\"unwind\"" - - name: Prove catch_unwind survives the release-ffi profile - run: cargo run --locked --profile release-ffi --example panic_spike -p agent-desktop-ffi + - name: Prove the shipped cdylib catches an exported-boundary panic + run: bash crates/ffi/tests/run_cdylib_panic_probe.sh publish-github: name: Publish to GitHub Release diff --git a/.gitignore b/.gitignore index aa1cd74..d22d5b9 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,10 @@ lcov.info tarpaulin-report.html coverage/ +# Generated review and benchmark evidence +/benchmarks/locator-resolution/results/ +/regression_findings.json + # Binary artifacts *.dylib *.so @@ -75,6 +79,8 @@ skills-lock.json docs/* !docs/architecture.excalidraw !docs/architecture.png +!docs/faq.md +!docs/json-output.md !docs/phases.md !docs/solutions/ !docs/solutions/** @@ -85,3 +91,5 @@ rust_out # trace export artifacts must never land in the repo tree trace-*.html +__pycache__/ +*.py[cod] diff --git a/CLAUDE.md b/CLAUDE.md index 29a1b28..4b79432 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,6 +16,7 @@ cargo fmt --all -- --check # Format check cargo fmt --all # Auto-format cargo tree -p agent-desktop-core # Verify no platform crate leaks (CI enforces) bash tests/e2e/run.sh # E2E: real binary vs fixture app, verify by observation (needs --release + AX permission) +bash scripts/perf-baseline-compare.sh # Perf A/B vs merge-base -> report.html (add --apps "Slack,Google Chrome" for Electron coverage) ``` Run the binary: `./target/release/agent-desktop snapshot --app Finder -i` @@ -160,21 +161,24 @@ Batch is not a second dispatcher. `src/batch/mod.rs` deserializes JSON entries i ### Additive Phase Model -- **Phase 1:** Foundation + macOS MVP (56 commands, core engine, macOS adapter) -- **Phase 2:** Windows + Linux adapters, 10+ new commands — core untouched -- **Phase 3:** MCP server mode via `--mcp` flag — wraps existing commands -- **Phase 4:** Daemon, sessions, enterprise quality gates +Numbering follows `docs/phases.md` (the source of truth for phase scope): -Phases 2–4 add adapters, transports, and production readiness work. Nothing in core is rebuilt. +- **Phase 1 / 1.5 / 1.6 (completed):** Foundation + macOS MVP, FFI cdylib distribution, and the Playwright-grade foundation contract (capability supertraits, auto-wait, occlusion gate, live locator, ProcessState, envelope 2.1) +- **Phase 2:** Windows adapter — delivered as sub-phases 2.0–2.15, beginning with a raw-script platform exploration phase, each a <=2,000-LOC PR into the `feat/windows-adapter` integration branch +- **Phase 3:** Linux adapter — same sub-phase template onto `feat/linux-adapter` +- **Phase 4:** MCP server mode via `--mcp` flag — wraps existing commands +- **Phase 5:** Daemon, sessions, enterprise quality gates + +Later phases add adapters, transports, and production readiness work. Nothing in core is rebuilt; platform work implements into the settled contract. ## Coding Standards ### File Rules -- **400 LOC hard limit per file.** If approaching 400, split by responsibility. No exceptions. _Exception: files bearing an `@generated` marker that are produced by `build.rs` codegen and validated by a CI drift gate are exempt — the limit applies to the hand-written templates and the build script itself, not the generated output. Do not hand-edit these files; fix the generator instead._ +- **400 LOC hard limit per hand-written file.** If approaching 400, split by responsibility. The committed C header at `crates/ffi/include/agent_desktop.h` is generated by cbindgen and verified by the `ffi-header-drift` CI job; change the Rust ABI declarations or cbindgen configuration and regenerate it rather than hand-editing generated declarations. - **No inline comments.** Code must be self-documenting through naming. Only Rust doc-comments (`///`) on public items when the name alone is insufficient. - **One struct/enum per file** for domain types. `node.rs` defines `AccessibilityNode`. `action.rs` defines `Action`. -- **One command per file.** Each CLI command lives in its own file under `commands/`. Filename matches the command name. _This rule scopes to hand-written command files; a single `@generated` wrapper file that consolidates multiple command entrypoints is not a violation._ +- **One command per file.** Each CLI command lives in its own file under `commands/`. Filename matches the command name. - **No God objects.** No struct with more than 7 fields. No function with more than 5 parameters. Use builder patterns or config structs. - **Explicit pub boundaries.** Only `lib.rs` re-exports public items. Internal modules use `pub(crate)`. No wildcard re-exports. @@ -192,7 +196,7 @@ Phases 2–4 add adapters, transports, and production readiness work. Nothing in PERM_DENIED, ELEMENT_NOT_FOUND, APP_NOT_FOUND, ACTION_FAILED, ACTION_NOT_SUPPORTED, STALE_REF, AMBIGUOUS_TARGET, WINDOW_NOT_FOUND, PLATFORM_NOT_SUPPORTED, TIMEOUT, INVALID_ARGS, NOTIFICATION_NOT_FOUND, -SNAPSHOT_NOT_FOUND, POLICY_DENIED, INTERNAL +SNAPSHOT_NOT_FOUND, POLICY_DENIED, APP_UNRESPONSIVE, INTERNAL ``` ### Exit Codes @@ -213,7 +217,7 @@ SNAPSHOT_NOT_FOUND, POLICY_DENIED, INTERNAL | Functions | `snake_case`, verb-first | `build_tree()`, `allocate_refs()` | | Constants | `SCREAMING_SNAKE_CASE` | `MAX_TREE_DEPTH`, `DEFAULT_TIMEOUT_MS` | | CLI flags | kebab-case | `--max-depth`, `--include-bounds` | -| Ref IDs | `@e{n}` sequential | `@e1`, `@e2`, `@e14` | +| Ref IDs | `@:e{n}` qualified output | `@s8f3k2p9:e1`, `@s8f3k2p9:e14` | ### Platform Crate Folder Structure @@ -278,7 +282,7 @@ Every command produces a response envelope: ```json { - "version": "2.0", + "version": "2.1", "ok": true, "command": "snapshot", "data": { @@ -294,18 +298,27 @@ Error responses: ```json { - "version": "2.0", + "version": "2.1", "ok": false, "command": "click", "error": { "code": "STALE_REF", "message": "Element could not be resolved from the requested snapshot", - "suggestion": "Run 'snapshot' to refresh, then retry with updated ref" + "suggestion": "Run 'snapshot' to refresh, then retry with updated ref", + "recovery": { + "strategy": "refresh_snapshot_then_retry_original", + "retryable": true, + "requires_fresh_snapshot": true + }, + "disposition": { + "delivery": "not_delivered", + "retry": "safe" + } } } ``` -The `error` object may also carry an optional `details` object (e.g. the actionability report on an actionability failure, candidate summaries on `AMBIGUOUS_TARGET`, or the last observed state on a `wait` `TIMEOUT`). +The `error` object may also carry optional `details` and `recovery` objects. Version 2.1 removed the 2.0 `retry_command` string without a compatibility alias; consumers must use `recovery.strategy` only when `disposition.retry` is `safe`. ### Serialization Rules @@ -316,7 +329,7 @@ The `error` object may also carry an optional `details` object (e.g. the actiona ## Ref System -- Refs allocated in depth-first document order: `@e1`, `@e2`, etc. +- Refs are allocated in depth-first document order and emitted as snapshot-qualified IDs such as `@s8f3k2p9:e1`. Legacy bare IDs such as `@e1` remain valid input only with an explicit `--snapshot s8f3k2p9`. - An element receives a ref when it is **addressable for an action**: its role is interactive (`button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `switch`, `colorwell`, `menubutton`, `incrementor`, `dockitem`), **or** it advertises an available action regardless of role. Container roles such as `scrollarea` (Scroll) and `disclosure` (Expand/Collapse/Click) are not interactive by role but are genuinely actionable, so they are ref-able — `scroll` / `expand` / `collapse` need a ref to target them - A bare `SetFocus` affordance does not qualify on its own (focusability is not a primary action), so inert focusable containers stay ref-less - Static text and non-actionable groups/containers do NOT get refs (they remain in tree for context) @@ -327,14 +340,14 @@ The `error` object may also carry an optional `details` object (e.g. the actiona - Progressive traversal: `--skeleton` clamps depth to 3, annotates truncated containers with `children_count`. Named/described containers at boundary receive refs as drill-down targets - Drill-down: `--root @ref` starts from a previously-discovered ref with scoped invalidation (only that ref's subtree refs are replaced on re-drill) - RefMap size check: write-side guard prevents >1MB refmap files -- **Sessions:** `session start` creates a manifest-gated session under `~/.agent-desktop/sessions//`, sets `current_session`, and (by default) enables automatic trace segments. Bare `--session ` without a manifest scopes only the snapshot namespace — no surprise trace files -- **Trace:** manifest `trace: on` writes per-process JSONL segments under `/trace/-.jsonl`; `--trace ` overrides to one file; activation resolves `--session` > `AGENT_DESKTOP_SESSION` > `current_session` +- **Sessions:** `session start` creates and returns a manifest-gated session under `~/.agent-desktop/sessions//` and enables automatic trace segments by default. It does not activate that session for later processes. Pass the returned ID through `--session` or `AGENT_DESKTOP_SESSION`. Bare `--session ` without a manifest scopes only the snapshot namespace — no surprise trace files. +- **Trace:** manifest `trace: on` writes per-process JSONL segments under `/trace/-.jsonl`; `--trace ` overrides to one file; activation resolves `--session` > `AGENT_DESKTOP_SESSION` > no session. Snapshot lookup is confined to that selected namespace and never searches other sessions. ## PlatformAdapter Trait Core defines `PlatformAdapter`; platform crates implement it. Methods default to `not_supported()`, so an adapter only implements what it supports. Read the -current signatures in `crates/core/src/adapter.rs` — notably strict resolution +current signatures in `crates/core/src/adapter/mod.rs` and its `actions.rs`, `input.rs`, `observation.rs`, and `system.rs` capability fragments — notably strict resolution (`resolve_element_strict*` → STALE_REF on 0, AMBIGUOUS_TARGET on 2+), live reads for the actionability preflight (`get_live_*`), and `is_protected_process` (keeps platform-specific process names out of core). @@ -369,10 +382,10 @@ for the actionability preflight (`get_live_*`), and `is_protected_process` ## Commands -56 commands spanning App/Window, Observation, Interaction, Scroll, Keyboard, +58 commands spanning App/Window, Observation, Interaction, Scroll, Keyboard, Mouse, Notifications (macOS), Clipboard, Wait, System (including `session`), and Batch. The full surface and per-command reference live in `skills/agent-desktop/`. -All 56 are implemented on macOS (Phase 1); Windows/Linux (Phase 2/3) target the +All 58 are implemented on macOS (Phase 1); Windows/Linux (Phase 2/3) target the same surface. Adding a command: see the Extensibility Pattern above. ## Non-Goals @@ -388,3 +401,7 @@ same surface. Adding a command: see the Extensibility Pattern above. - PRD v2.0: `docs/agent_desktop_prd_v2.pdf` - Architecture Brainstorm: `docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md` - Phase 1 Plan: `docs/plans/2026-02-19-feat-agent-desktop-phase1-foundation-plan.md` + +## Definition of Done: Performance Baseline + +Every substantive change ends with a performance baseline check before merge: run `bash scripts/perf-baseline-compare.sh` (optionally `--apps "Slack,Google Chrome"` for dense Electron/Chromium coverage) and review the generated `report.html` against the merge-base. Latency deltas must be intentional and explainable — never discovered by users. diff --git a/CONCEPTS.md b/CONCEPTS.md index f5e53f6..28512a0 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -11,7 +11,7 @@ A structured representation of an application's user interface exposed by the op An observation of an accessibility tree at a point in time, persisted with the element refs allocated from that observation. ### Snapshot ID -A compact identifier for one persisted snapshot. Explicit snapshot IDs are direct handles; callers do not need the original session when they pass the ID. +A compact identifier for one persisted snapshot. Lookup is confined to the selected session namespace, so an ID created in a session is not a cross-session handle. ### Surface A scoped UI layer that can be observed separately from the whole window, such as an open menu, sheet, popover, alert, or focused area. @@ -24,7 +24,7 @@ A snapshot operation that starts from an existing ref to observe that element's ### Ref A short element identifier assigned by agent-desktop to an actionable or drillable node in a snapshot. -Refs are deterministic inside one snapshot but are not stable across UI changes. Callers either pass the snapshot ID that produced the ref or intentionally use the session's latest snapshot pointer. +Refs are deterministic inside one snapshot but are not stable across UI changes. Snapshot and find output qualify each ref with its snapshot ID. Legacy bare refs require the producing snapshot ID as a separate argument. ### RefMap The persisted mapping from refs to the identity evidence needed to re-identify elements later. @@ -47,9 +47,9 @@ Strict ref resolution rejects missing, stale, and ambiguous matches instead of g ### Session An on-disk container under `~/.agent-desktop/sessions//` that owns snapshot refmaps, an optional trace directory, and a `session.json` manifest. -`session start` writes the manifest (`trace: on` unless `--no-trace`), pre-creates `trace/` when tracing is on, and sets `~/.agent-desktop/current_session`. Activating a session (via pointer, `AGENT_DESKTOP_SESSION`, or `--session`) relocates the latest-snapshot namespace as well as the trace sink. Bare `--session ` without a manifest remains snapshot-namespace-only for backward compatibility. +`session start` writes the manifest (`trace: on` unless `--no-trace`) and pre-creates `trace/` when tracing is on. It returns the new ID but does not activate it for later processes. Explicit `--session` takes precedence over `AGENT_DESKTOP_SESSION`; with neither, commands use the global, non-session namespace. Bare `--session ` without a manifest remains snapshot-namespace-only for backward compatibility. -Use sessions when callers intentionally omit `--snapshot` and want a shared latest observation — typically after `session start` for a coordinated run. Explicit snapshot IDs remain the deterministic path for pinned actions and can be resolved without also passing the session. +Use sessions when callers want a coordinated snapshot namespace and trace sink. Every lookup is confined to its selected namespace, so a snapshot created under a session requires that same `--session` or `AGENT_DESKTOP_SESSION` scope later. Qualified refs remain the deterministic path for pinned actions inside that namespace. ### Session Manifest The `session.json` file describing one session: id, optional name, created/ended timestamps, and `trace: on|off`. @@ -84,7 +84,9 @@ The platform-neutral set of supported action names that core uses to compare com Each adapter maps native primitives into this shared vocabulary before core evaluates actionability. New commands should extend the central vocabulary first, then reuse it from actionability, ref allocation, predicates, FFI tests, and platform adapters. ### Interaction Policy -The side-effect contract attached to an action request, controlling whether the command may steal focus, move the cursor, or use physical input fallbacks. Ref commands expose exactly two modes: **headless** (the default — accessibility-only, no cursor, fails closed when the AX path is unavailable) and **headed** (opt-in via the global `--headed` flag — permits focus stealing and cursor movement so the action chain's physical fallbacks can complete). The AX path is always tried first, so headed never regresses headless-capable elements. The `headed` upgrade applies uniformly to every ref command; each command still declares its own headless base policy (most are pure-AX; `type` uses a focus-fallback base because typing requires focus but never moves the cursor). +The side-effect contract attached to an action request, controlling whether the command may steal focus, move the cursor, or use physical input. Ref commands expose exactly two modes: **headless** (the default — accessibility-only, no cursor, fails closed when the semantic path is unavailable) and **headed** (opt-in via the global `--headed` flag — authorizes the action's declared focus and cursor preconditions). + +Core owns those preconditions through `HeadedRequirement`: `FocusedWindow` for keyboard or focus-sensitive work, and `FocusedWindowAndCursor` for pointer delivery. For ref actions, core focuses the exact source window before dispatch and resolves a verified target point for pointer work; the platform adapter owns the OS-specific focus primitive and delivery mechanism. Raw coordinate input has no ref identity, so it never infers or focuses a window. On macOS, headed `click`, `right-click`, `type`, `clear`, and `scroll` are physical-first; double/triple-click, hover, and drag are physical-only; expand/collapse and the remaining semantic actions stay semantic after the core focus precondition. ### Headless Ref Action A ref-based action that uses semantic accessibility operations without implicit focus stealing, cursor movement, synthetic keyboard input, or pasteboard use. This is the default mode. @@ -92,7 +94,7 @@ A ref-based action that uses semantic accessibility operations without implicit Headless ref actions may still fail when the native accessibility API cannot perform the requested semantic operation; they fail closed with structured actionability or policy errors rather than silently substituting physical input. The broader **headed** policy must be selected explicitly with `--headed`. ### Action Chain -The ordered ladder of strategies a ref action walks to perform one intent — semantic accessibility actions first, then settable attributes, then policy-gated physical input — with each step verified against the element's observed state before it counts as success. +The ordered ladder of strategies a ref action walks to perform one intent, with each step gated by policy and its delivery evidence recorded. The order is action-specific: natural input may put a headed physical step first, while semantic state changes use accessibility actions or settable attributes only. The chain pins one execution deadline at its start (distinct from the Resolver Deadline, which budgets re-identification) and every step observes it. Expiry while a step may have partially mutated the element surfaces as a structured timeout carrying the observed state, never as a plain step failure — the caller must be able to tell "nothing happened" from "something may have happened". @@ -105,11 +107,11 @@ The remaining time budget carried through strict ref resolution so every native ### Coordinate Fallback An explicit opt-in path that uses screen coordinates or physical input when semantic accessibility operations cannot perform the requested action. -Physical input lands on the topmost window at the target point, so the fallback first ensures the target element's own window is frontmost — the app being frontmost is not sufficient when the element lives in a background window of that app. +Ref-targeted physical input lands on the topmost window at the resolved point, so core first ensures the target element's exact window is frontmost — the app being frontmost is not sufficient when the element lives in a background window of that app. Raw `--xy` input carries no window identity and therefore moves/clicks at the requested coordinates without focusing any application. ### FFI Ref-Action Parity The requirement that language bindings using refs follow the same strict resolution, actionability, and interaction-policy semantics as CLI ref commands. ## Relationships -A session owns one latest-snapshot pointer, an optional manifest-gated trace directory, and persisted snapshot refmaps. A snapshot persists a ref map and can be selected directly by snapshot ID. A ref resolves through strict ref resolution into live native evidence, then actionability decides whether a headless ref action can safely dispatch, and the action chain executes that dispatch under its own deadline with the interaction policy gating its physical steps. FFI ref-action parity keeps that same relationship true for language bindings. +A session owns one latest-snapshot pointer, an optional manifest-gated trace directory, and persisted snapshot refmaps. A snapshot persists a ref map and can be selected by ID within that same namespace. A ref resolves through strict ref resolution into live native evidence, then actionability decides whether the action can safely dispatch. In headed mode, core applies the action's focus/cursor requirement before the platform adapter executes the action-specific chain under its own deadline. FFI ref-action parity keeps that same relationship true for language bindings. diff --git a/Cargo.lock b/Cargo.lock index 2d75275..0e53449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,7 @@ dependencies = [ "smallvec", "thiserror", "tracing", + "windows-sys", ] [[package]] @@ -74,6 +75,7 @@ dependencies = [ "core-foundation", "core-foundation-sys", "core-graphics", + "libc", "rustc-hash", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index d3f4d23..5354e97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,11 @@ [workspace] -members = ["crates/core", "crates/macos", "crates/windows", "crates/linux", "crates/ffi", "src"] +members = ["crates/core", "crates/macos", "crates/windows", "crates/linux", "crates/ffi", "src"] +default-members = ["crates/core", "crates/macos", "crates/windows", "crates/linux", "src"] resolver = "2" [workspace.package] edition = "2024" -rust-version = "1.85" +rust-version = "1.89" license = "Apache-2.0" version = "0.4.7" # x-release-please-version @@ -20,6 +21,7 @@ rustc-hash = "2.1" libc = "0.2" smallvec = { version = "1.13", features = ["serde", "union"] } agent-desktop-core = { path = "crates/core" } +windows-sys = { version = "0.61.2" } [workspace.lints.rust] unsafe_op_in_unsafe_fn = "warn" diff --git a/README.md b/README.md index 30a6f8e..9317700 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,9 @@ - **Native Rust CLI**: Fast, single binary, no runtime dependencies - **C-ABI cdylib** (`libagent_desktop_ffi`): Load once from Python / Swift / Go / Ruby / Node / C instead of forking the CLI per call -- **56 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, plus a bundled `skills` doc loader +- **58 command names, 54 operational commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, plus a bundled `skills` doc loader. The four held-input names are reserved for a stateful daemon and fail closed in the stateless CLI. - **Progressive skeleton traversal**: 78–96% token reduction on dense apps via shallow overview + targeted drill-down -- **Snapshot & refs**: AI-optimized workflow using compact snapshot IDs and deterministic element references (`@e1`, `@e2`) +- **Snapshot & refs**: AI-optimized workflow using compact snapshot IDs and qualified element references (`@s8f3k2p9:e1`, `@s8f3k2p9:e2`) - **Headless-by-default interactions**: Ref actions use accessibility APIs and block silent focus, cursor, keyboard, or pasteboard side effects - **Structured JSON output**: Machine-readable responses with error codes and recovery hints - **Works with any app**: Finder, Safari, System Settings, Xcode, Slack — anything with an accessibility tree @@ -71,14 +71,14 @@ cargo build --release cp target/release/agent-desktop /usr/local/bin/ ``` -Requires Rust 1.85+ and macOS 13.0+. +Requires Rust 1.89+ and macOS 13.0+. ### Permissions -macOS requires Accessibility permission. Screenshots also require Screen Recording permission. Grant them in **System Settings > Privacy & Security** by adding the app that launches agent-desktop, or: +macOS requires Accessibility permission. Screenshots also require Screen Recording permission, and the Notification Center opener requires Automation permission for System Events. Plain permission checks never prompt. Request missing permissions in a bounded isolated helper with: ```bash -agent-desktop permissions --request # trigger platform permission request path +agent-desktop permissions --request # request missing permissions in an isolated helper ``` Permission fields are explicit objects, for example: @@ -87,10 +87,12 @@ Permission fields are explicit objects, for example: { "accessibility": { "state": "granted" }, "screen_recording": { "state": "denied", "suggestion": "Grant Screen Recording permission" }, - "automation": { "state": "not_required" } + "automation": { "state": "unknown" } } ``` +Automation reports `granted`, `denied`, or `unknown`; `unknown` means macOS would need to prompt or System Events could not be probed without prompting. + ## Language bindings (FFI) Every GitHub Release ships a prebuilt C-ABI cdylib (`libagent_desktop_ffi`) for macOS, Linux, and Windows alongside the CLI tarballs. `dlopen` it and call the functions declared in `agent_desktop.h` for in-process calls instead of fork-exec per command. @@ -98,9 +100,9 @@ Every GitHub Release ships a prebuilt C-ABI cdylib (`libagent_desktop_ffi`) for ```python import ctypes lib = ctypes.CDLL("./lib/libagent_desktop_ffi.dylib") -lib.ad_init(1) # verify ABI major (AD_ABI_VERSION_MAJOR) before any call +lib.ad_init(3) # verify ABI major (AD_ABI_VERSION_MAJOR) before any call adapter = lib.ad_adapter_create() -# observe -> act: ad_snapshot -> parse an @e ref -> ad_execute_by_ref ... +# observe -> act: ad_snapshot -> parse a qualified ref -> ad_execute_by_ref ... lib.ad_adapter_destroy(adapter) ``` @@ -142,9 +144,10 @@ Agent loop: snapshot → decide → act → snapshot → decide → act → ... ### Trace viewer (read back a session) ```bash -agent-desktop session start --screenshots # opt-in replay artifacts (PNG + refmap copies) -agent-desktop snapshot --app Finder -i # work normally under the active session -agent-desktop click @e5 +session_id=$(agent-desktop session start --screenshots | jq -r '.data.session_id') +export AGENT_DESKTOP_SESSION="$session_id" +agent-desktop snapshot --app Finder -i # work inside the explicit session scope +agent-desktop click @s8f3k2p9:e5 agent-desktop trace show --limit 500 # bounded JSON timeline for agents agent-desktop trace export --out run.html # single-file HTML viewer (works from file://) ``` @@ -153,19 +156,20 @@ agent-desktop trace export --out run.html # single-file HTML viewer (works fr ### Shared sessions for multi-agent workflows -Run `session start` once per agent run to create a trace-enabled session (manifest `trace: on` by default) and set the active pointer. Subsequent commands in that run get automatic JSONL segments under `~/.agent-desktop/sessions//trace/` and share the session's latest-snapshot namespace — no `--trace` on every call. +Run `session start` once per agent run to create a trace-enabled session (manifest `trace: on` by default), then pass the returned ID with global `--session ` or `AGENT_DESKTOP_SESSION=`. Commands in that explicit scope get automatic JSONL segments under `~/.agent-desktop/sessions//trace/` and share the session's latest-snapshot namespace — no `--trace` on every call. -For concurrent **independent** agents, set `AGENT_DESKTOP_SESSION=` per process instead of relying on the global pointer. When multiple agents share one session id, each agent should act on the `snapshot_id` from its own `snapshot` call; implicit latest is a single-agent convenience. +For concurrent **independent** agents, set `AGENT_DESKTOP_SESSION=` per process. When multiple agents share one session ID, each agent should act on the qualified refs from its own `snapshot` call rather than assuming the namespace's latest snapshot is unchanged. -Bare `--session ` without a manifest (no `session start`) still scopes the snapshot namespace only and writes no trace files. Explicit `--snapshot ` resolves cross-session. +Bare `--session ` without a manifest (no `session start`) still scopes the snapshot namespace only and writes no trace files. Snapshot IDs resolve only inside the selected session namespace; they never trigger a cross-session search. ```bash -agent-desktop session start --name release-fix -agent-desktop snapshot --app Xcode -i --compact # uses active session + tracing -agent-desktop wait --element @e9 --predicate actionable --timeout 5000 -agent-desktop click @e9 -agent-desktop click @e9 --snapshot s2 # pin to a specific observation -agent-desktop session end +agent-desktop session start --name release-fix # note data.session_id +export AGENT_DESKTOP_SESSION= +agent-desktop snapshot --app Xcode -i --compact # uses selected session + tracing +agent-desktop wait --element @s8f3k2p9:e9 --predicate actionable --timeout 5000 +agent-desktop click @s8f3k2p9:e9 +agent-desktop click @e9 --snapshot s2 # legacy bare ref, explicitly pinned +agent-desktop session end "$AGENT_DESKTOP_SESSION" agent-desktop session gc ``` @@ -188,25 +192,26 @@ agent-desktop list-surfaces --app Notes # list menus, sheets, popovers, ### Interaction ```bash -agent-desktop click @e3 # semantic AX-first click -agent-desktop double-click @e3 # AXOpen; physical double-click uses --headed mouse-click --count 2 -agent-desktop triple-click @e3 # POLICY_DENIED if physical input is disabled -agent-desktop right-click @e3 # open verified context menu -agent-desktop type @e5 "hello world" # insert text into element -agent-desktop set-value @e5 "new value" # set value directly via AX -agent-desktop clear @e5 # clear element value -agent-desktop focus @e5 # set keyboard focus -agent-desktop select @e9 "Option B" # select verified dropdown/list option -agent-desktop toggle @e12 # flip checkbox or switch -agent-desktop check @e12 # idempotent check -agent-desktop uncheck @e12 # idempotent uncheck -agent-desktop expand @e15 # expand disclosure/tree item -agent-desktop collapse @e15 # collapse disclosure/tree item -agent-desktop scroll @e1 --direction down --amount 3 # scroll (AX-first) -agent-desktop scroll-to @e20 # scroll element into view +agent-desktop click @s8f3k2p9:e3 # strict headless AX click +agent-desktop --headed click @s8f3k2p9:e3 # physical click, focus/cursor allowed +agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click +agent-desktop --headed triple-click @s8f3k2p9:e3 # physical triple-click +agent-desktop right-click @s8f3k2p9:e3 # open context menu; inspect effect before retrying +agent-desktop type @s8f3k2p9:e5 "hello world" # insert text into element +agent-desktop set-value @s8f3k2p9:e5 "new value" # set value directly via AX +agent-desktop clear @s8f3k2p9:e5 # clear element value +agent-desktop focus @s8f3k2p9:e5 # set keyboard focus +agent-desktop select @s8f3k2p9:e9 "Option B" # select verified dropdown/list option +agent-desktop toggle @s8f3k2p9:e12 # flip checkbox or switch +agent-desktop check @s8f3k2p9:e12 # idempotent check +agent-desktop uncheck @s8f3k2p9:e12 # idempotent uncheck +agent-desktop expand @s8f3k2p9:e15 # expand disclosure/tree item +agent-desktop collapse @s8f3k2p9:e15 # collapse disclosure/tree item +agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 # strict headless AX scroll +agent-desktop scroll-to @s8f3k2p9:e20 # scroll element into view ``` -> **(macOS, Phase 1)** Pure cursor gestures have no accessibility equivalent, so `triple-click`, `hover`, and `drag` are always physical; `double-click` is headless via `AXOpen` and only needs `--headed` for gesture-only targets. Windows (UIA) and Linux (AT-SPI) adapters may expose different capabilities. See `skills/agent-desktop/references/commands-interaction.md`. +> **(macOS, Phase 1)** Default ref actions are strict headless semantic operations. In headed mode, core focuses the exact ref window before dispatch; pointer commands additionally require a verified target point, while the adapter owns physical delivery. `click`, `right-click`, `type`, `clear`, and `scroll` are physical-first; double/triple-click, hover, and drag are physical-only; expand/collapse and other semantic commands remain semantic. Raw coordinates never imply a target window and therefore never steal focus. See `skills/agent-desktop/references/commands-interaction.md`. ### Keyboard @@ -214,22 +219,22 @@ agent-desktop scroll-to @e20 # scroll element into view agent-desktop press cmd+s # key combo agent-desktop press cmd+shift+z # multi-modifier agent-desktop press escape # single key -agent-desktop key-down shift # hold key -agent-desktop key-up shift # release key ``` +`key-down` and `key-up` are reserved command names and return `ACTION_NOT_SUPPORTED` until a stateful daemon can own the held-key lifetime. + ### Mouse ```bash -agent-desktop --headed hover @e3 # move cursor to element +agent-desktop --headed hover @s8f3k2p9:e3 # move cursor to element agent-desktop --headed hover --xy 500,300 # move cursor to coordinates -agent-desktop --headed drag --from @e3 --to @e8 # drag between elements +agent-desktop --headed drag --from @s8f3k2p9:e3 --to @s8f3k2p9:e8 # drag between elements agent-desktop --headed drag --from-xy 100,200 --to-xy 400,200 # drag between coordinates agent-desktop --headed mouse-click --xy 500,300 # click at coordinates -agent-desktop --headed mouse-down --xy 500,300 # press at coordinates -agent-desktop --headed mouse-up --xy 500,300 # release at coordinates ``` +`mouse-down` and `mouse-up` are likewise reserved; use the atomic `mouse-click` or `drag` commands. + ### App & Window Management ```bash @@ -240,26 +245,31 @@ agent-desktop close-app Safari --force # force quit (SIGTERM, then SIGKILL if agent-desktop list-apps # list running GUI apps agent-desktop list-windows # list visible windows agent-desktop list-windows --app Finder # windows for specific app -agent-desktop focus-window w-4521 # bring window to front -agent-desktop resize-window w-4521 800 600 # resize -agent-desktop move-window w-4521 100 100 # move -agent-desktop minimize w-4521 # minimize -agent-desktop maximize w-4521 # maximize -agent-desktop restore w-4521 # restore +agent-desktop focus-window --window-id w-4521 # bring exact window to front +agent-desktop resize-window --window-id w-4521 --width 800 --height 600 +agent-desktop move-window --window-id w-4521 --x 100 --y 100 +agent-desktop minimize --window-id w-4521 +agent-desktop maximize --window-id w-4521 +agent-desktop restore --window-id w-4521 ``` ### Notifications *(macOS only)* ```bash -agent-desktop list-notifications # list all notifications -agent-desktop list-notifications --app "Slack" # filter by app -agent-desktop list-notifications --text "deploy" --limit 5 # filter by text -agent-desktop dismiss-notification 1 # dismiss by index -agent-desktop dismiss-all-notifications # dismiss all -agent-desktop dismiss-all-notifications --app "Slack" # dismiss all from app -agent-desktop notification-action 1 --action "Reply" # click action button +agent-desktop --headed list-notifications # open Notification Center if needed, then list +agent-desktop --headed list-notifications --app "Slack" # filter by app +agent-desktop --headed list-notifications --text "deploy" --limit 5 # filter by text +agent-desktop --headed dismiss-notification 1 --expected-app "Slack" --expected-title "Deploy complete" +agent-desktop --headed dismiss-all-notifications # dismiss all +agent-desktop --headed dismiss-all-notifications --app "Slack" # dismiss all from app +agent-desktop --headed notification-action 1 "Reply" --expected-app "Slack" --expected-title "Deploy complete" ``` +Single-notification mutations require an app or title fingerprint from the +same listing. Every mutation requires `--headed` because it opens and focuses +Notification Center. Headless listing can only observe an already-open center; +headed listing may open it and restore the prior frontmost app afterward. + ### Clipboard ```bash @@ -272,9 +282,9 @@ agent-desktop clipboard-clear # clear clipboard ```bash agent-desktop wait 500 # sleep 500ms -agent-desktop wait --element @e3 --timeout 5000 # wait for element -agent-desktop wait --element @e3 --predicate actionable # wait until safe to act -agent-desktop wait --element @e5 --predicate value --value ready +agent-desktop wait --element @s8f3k2p9:e3 --timeout 5000 # wait for element +agent-desktop wait --element @s8f3k2p9:e3 --predicate actionable # wait until safe to act +agent-desktop wait --element @s8f3k2p9:e5 --predicate value --value ready agent-desktop wait --window "Save" --timeout 10000 # wait for window agent-desktop wait --text "Loading complete" --app Safari # wait for text agent-desktop wait --text "Done" --count 1 --app Xcode # wait for exact match count @@ -300,13 +310,13 @@ agent-desktop --session run-a batch '[ ### System ```bash -agent-desktop session start [--name LABEL] [--no-trace] # trace-enabled run (sets active pointer) +agent-desktop session start [--name LABEL] [--no-trace] # create session; pass returned ID explicitly agent-desktop session end [id] agent-desktop session list agent-desktop session gc [--older-than SECS] [--ended] agent-desktop status # platform, permissions, session_id, tracing, latest snapshot agent-desktop permissions # check accessibility/screen-recording/automation -agent-desktop permissions --request # invoke platform request path +agent-desktop permissions --request # request in the bounded isolated helper agent-desktop version # version string agent-desktop skills get desktop --full # bundled agent guidance ``` @@ -332,55 +342,11 @@ agent-desktop snapshot [OPTIONS] ## JSON Output -Every command returns structured JSON: - -```json -{ - "version": "2.0", - "ok": true, - "command": "click", - "data": { "action": "click" } -} -``` - -Errors include machine-readable codes and recovery hints: - -```json -{ - "version": "2.0", - "ok": false, - "command": "click", - "error": { - "code": "STALE_REF", - "message": "Element at @e7 no longer matches the last snapshot", - "suggestion": "Run 'snapshot' to refresh refs, then retry" - } -} -``` - -### Error Codes - -| Code | Meaning | -|------|---------| -| `PERM_DENIED` | Accessibility permission not granted | -| `ELEMENT_NOT_FOUND` | No element matched the ref or query | -| `APP_NOT_FOUND` | Application not running or no windows | -| `STALE_REF` | Ref could not be re-identified in the live UI | -| `AMBIGUOUS_TARGET` | Ref recovery matched multiple plausible targets | -| `SNAPSHOT_NOT_FOUND` | Snapshot ID is missing or expired | -| `POLICY_DENIED` | Physical/headed path blocked by policy | -| `ACTION_FAILED` | The OS rejected the action | -| `PLATFORM_NOT_SUPPORTED` | Adapter method not implemented on this platform | -| `TIMEOUT` | Wait condition expired | -| `INVALID_ARGS` | Invalid argument values | - -### Exit Codes - -`0` success, `1` structured error (JSON on stdout), `2` argument parse error. +See the [versioned JSON envelope, error-code, and exit-code contract](docs/json-output.md). ## Ref System -`snapshot` assigns refs to interactive elements in depth-first order: `@e1`, `@e2`, `@e3`, etc. Refs are scoped to a compact `snapshot_id` such as `s8f3k2p9`. Commands can omit `--snapshot` to use the active session's latest snapshot pointer, but passing the ID is more deterministic in multi-step flows and does not require also passing `--session`. +`snapshot` assigns local positions in depth-first order and emits qualified refs such as `@s8f3k2p9:e1`, `@s8f3k2p9:e2`, and `@s8f3k2p9:e3`. A qualified ref embeds the exact snapshot ID and needs no separate `--snapshot`. Legacy bare refs such as `@e3` remain accepted only with an explicit `--snapshot s8f3k2p9`. Snapshot lookup stays inside the selected session namespace. Interactive roles that receive refs: `button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `incrementor`, `menubutton`, `switch`, `colorwell`, `dockitem`. @@ -388,14 +354,14 @@ Static elements (labels, groups, containers) appear in the tree for context but Reliability contract: -- `session start` creates a manifest-gated session with automatic trace segments and relocates the latest-snapshot namespace. Activation resolves `--session` > `AGENT_DESKTOP_SESSION` > `~/.agent-desktop/current_session` (pointer written only by `session start`). +- `session start` creates and returns a manifest-gated session with automatic trace segments. It does not activate later processes. Activation resolves explicit `--session` first, then `AGENT_DESKTOP_SESSION`; otherwise the command uses the global, non-session namespace. - Bare `--session ` without a manifest scopes snapshots only — no surprise trace files for existing callers. -- Explicit `--snapshot ` resolves the saved snapshot directly, including across sessions. +- Snapshot lookup is confined to the selected namespace. A session-owned snapshot requires the same explicit `--session` or `AGENT_DESKTOP_SESSION` scope. - Ref actions re-identify targets at action time: a moved unique target can proceed, while missing or changed stable identity returns `STALE_REF`. - Mutable value text is not treated as stable identity, so text fields and timers can keep resolving when the saved window, path, role, and bounds evidence still identify the same element. - Multiple plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily. - Actions run an actionability preflight before dispatch: visibility, stability, enabled state, supported action, policy, and editability. -- `wait --element @e3 --predicate actionable` polls until the target can be acted on. +- `wait --element @s8f3k2p9:e3 --predicate actionable` polls until the target can be acted on. - With an active trace-enabled session, JSONL segments land under `sessions//trace/-*.jsonl` automatically. `--trace ` overrides to one file; `--trace-strict` fails on setup and pre-action writes (post-action traces are best-effort). Stale ref recovery: @@ -427,91 +393,7 @@ cargo clippy --all-targets -- -D warnings # lint (must pass with zero warnings) ## FAQ -### What is agent-desktop? - -agent-desktop is a native desktop automation CLI for AI agents. It lets agents observe and control desktop apps through OS accessibility trees, using structured JSON instead of screenshots, pixel matching, or browser-only automation. - -### Does agent-desktop require screenshots or pixel matching? - -No. The core workflow reads native accessibility trees and assigns refs to interactive elements. Screenshots are available as a separate command, but agents do not need screenshots or pixel matching to click buttons, type into fields, inspect menus, or navigate app windows. - -### How does agent-desktop work? - -| Component | Function | -|-----------|----------| -| **Native Rust CLI** | Fast, single binary, no runtime dependencies | -| **C-ABI cdylib** | Load once from Python, Swift, Go, Ruby, Node, or C instead of forking | -| **56 Commands** | Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, and bundled `skills` docs | -| **Snapshot & Refs** | Compact snapshot IDs and deterministic element refs like `@e1`, `@e2` | -| **Structured JSON** | Machine-readable responses with error codes and recovery hints | - -### What makes agent-desktop useful for AI agents? - -| Feature | Benefit | -|---------|---------| -| **Progressive Skeleton Traversal** | 78–96% token reduction on dense apps | -| **Headless-by-Default Actions** | Ref actions use accessibility APIs and block unintended physical side effects | -| **Snapshot Refs** | Agents act on stable refs within a snapshot instead of guessing coordinates | -| **Recovery Hints** | Errors include machine-readable codes and suggestions for the next agent step | -| **Cross-Language FFI** | Python, Swift, Go, Ruby, Node, C, and C++ hosts can call the native library directly | - -### Which platforms are supported? - -| Feature | macOS | Windows | Linux | -|---------|:-----:|:-------:|:-----:| -| Accessibility tree | **Yes** | Planned | Planned | -| Click/type/keyboard | **Yes** | Planned | Planned | -| Mouse input | **Yes** | Planned | Planned | -| Screenshot | **Yes** | Planned | Planned | -| Clipboard | **Yes** | Planned | Planned | -| App/window management | **Yes** | Planned | Planned | -| Notifications | **Yes** | Planned | Planned | - -### How do I install agent-desktop? - -Install the CLI from npm: - -```bash -npm install -g agent-desktop -agent-desktop snapshot --app Safari -``` - -Build the FFI library from source: - -```bash -cargo build --release -# Outputs: libagent_desktop_ffi.dylib/.so/.dll -``` - -### What is the ref system? - -`snapshot` assigns refs to interactive elements in depth-first order: `@e1`, `@e2`, `@e3`, etc. Refs are scoped to a compact `snapshot_id` such as `s8f3k2p9`. Commands can omit `--snapshot` to use the active session's latest snapshot pointer, but explicit snapshot IDs are the deterministic path and do not require also passing `--session`. - -Interactive roles that receive refs: - -`button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `incrementor`, `menubutton`, `switch`, `colorwell`, `dockitem`. - -Stale ref recovery: - -```text -snapshot -> act -> STALE_REF? -> snapshot again -> retry -``` - -### Is agent-desktop free and open source? - -Yes. agent-desktop is Apache-2.0 licensed for personal and commercial use. - -### Where can I get help? - -| Resource | Link | -|----------|------| -| **Repository** | [github.com/lahfir/agent-desktop](https://github.com/lahfir/agent-desktop) | -| **ClawHub Skill** | [clawhub.ai/lahfir/agent-desktop](https://clawhub.ai/lahfir/agent-desktop) | -| **skills.sh Listing** | [skills.sh/lahfir/agent-desktop/agent-desktop](https://skills.sh/lahfir/agent-desktop/agent-desktop) | -| **npm Package** | [npmjs.com/package/agent-desktop](https://www.npmjs.com/package/agent-desktop) | -| **CI Status** | [GitHub Actions](https://github.com/lahfir/agent-desktop/actions/workflows/ci.yml?query=branch%3Amain) | -| **Releases** | [GitHub Releases](https://github.com/lahfir/agent-desktop/releases) | -| **Issues** | [GitHub Issues](https://github.com/lahfir/agent-desktop/issues) | +See the [complete FAQ](docs/faq.md) for architecture, platform support, installation, refs, licensing, and support links. ## License diff --git a/benchmarks/locator-resolution/README.md b/benchmarks/locator-resolution/README.md new file mode 100644 index 0000000..445784b --- /dev/null +++ b/benchmarks/locator-resolution/README.md @@ -0,0 +1,46 @@ +# Locator resolution benchmark + +This deterministic harness compares the legacy full-snapshot locator path with +the handle-free observed-tree evaluator over synthetic Chromium/Electron accessibility +trees. It exercises deep anonymous wrapper chains, duplicate role/name +candidates, moving bounds, simultaneous `AXIdentifier` and `AXDOMIdentifier` +values, containment predicates, and large trees. + +Run it without network access: + +```bash +rtk cargo run -p agent-desktop-core --release --example locator_benchmark \ + > /private/tmp/agent-desktop-locator-synthetic.json +``` + +The JSON report includes 31-run p50/p95 wall-clock latency, candidate nodes, +predicate work, requested synthetic attribute values, cardinality, and +correctness. `live_find_selected_refs` uses the same selected-match-only +materialization contract as the default CLI `find` path. `live_arena_direct` is +reported separately as a lower-overhead direct-target metric, and +`live_count_no_refmap` verifies that count-only requests perform no action or +settable evidence reads. The harness does not claim absolute native AX calls: +role-dependent action and settable probes make that model inaccurate. Native +Slack/Electron IPC measurements require Accessibility permission and belong in +the privileged macOS integration suite. + +This synthetic benchmark JSON is consumed by the repo-wide performance report: +run `bash scripts/perf-baseline-compare.sh` (optionally with `--apps "Slack,Google +Chrome"`) to produce `report.html`, covering the HEAD-vs-main fixture A/B, +optional real-app read-only probes, and this synthetic benchmark. + +For privileged Electron/Chromium runs, the macOS adapter first asks whether the +application root exposes `AXManualAccessibility` as settable, reads its current +value, and sets it at most once when supported. The adapter then waits within +the request's existing absolute deadline until the attribute reports ready, +recording attempted/succeeded/ready activation in locator stats. Unsupported +native applications are left unchanged. This follows +[Electron's official third-party accessibility guidance](https://github.com/electron/electron/blob/main/docs/tutorial/accessibility.md), +which documents setting `AXManualAccessibility` from native assistive software +to expose Chromium's accessibility tree before automatic assistive-technology +detection has enabled it. The +[current Electron macOS implementation](https://github.com/electron/electron/blob/main/shell/browser/mac/electron_application.mm) +advertises the attribute as settable, reports `true` only in complete mode, and +handles `AXEnhancedUserInterface` separately; that source behavior is why the +benchmark methodology requires a post-set readiness read instead of treating a +successful setter as immediate readiness. diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 053f2f0..4e3c101 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -15,5 +15,15 @@ base64.workspace = true libc.workspace = true smallvec.workspace = true +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Security", + "Win32_Security_Authorization", + "Win32_Storage_FileSystem", + "Win32_System_SystemServices", + "Win32_System_Threading", +] } + [lints] workspace = true diff --git a/crates/core/examples/locator_benchmark.rs b/crates/core/examples/locator_benchmark.rs new file mode 100644 index 0000000..c6c4c27 --- /dev/null +++ b/crates/core/examples/locator_benchmark.rs @@ -0,0 +1,283 @@ +#[path = "locator_benchmark/adapter.rs"] +mod adapter; +#[path = "locator_benchmark/fixture.rs"] +mod fixture; +#[path = "locator_benchmark/fixture_builder.rs"] +mod fixture_builder; +#[path = "locator_benchmark/fixture_node.rs"] +mod fixture_node; +#[path = "locator_benchmark/legacy.rs"] +mod legacy; +#[path = "locator_benchmark/live.rs"] +mod live; +#[path = "locator_benchmark/scenario.rs"] +mod scenario; +#[path = "locator_benchmark/scenarios.rs"] +mod scenarios; + +use crate::{ + legacy::run_legacy, + live::{run_live_count, run_live_direct, run_live_find}, + scenario::Scenario, +}; +use serde_json::{Value, json}; +use std::{error::Error, hint::black_box}; + +const WARMUP_RUNS: usize = 5; +const MEASURED_RUNS: usize = 31; + +fn main() -> Result<(), Box> { + let reports = scenarios::all() + .iter() + .map(benchmark_scenario) + .collect::, _>>()?; + let report = json!({ + "schema_version": "1.3", + "benchmark": "locator-resolution-electron-synthetic", + "methodology": { + "warmup_runs": WARMUP_RUNS, + "measured_runs": MEASURED_RUNS, + "timing": "wall clock around fixture-to-result resolution; fixture generation excluded", + "legacy_path": "full AccessibilityNode snapshot, ref allocation, recursive matcher", + "live_direct_path": "handle-free observed tree, strict direct target selection without ref-map materialization, memoized evaluator", + "live_find_path": "CLI-compatible default find selection with selected-match-only ref materialization from the same observed tree", + "live_count_path": "CLI-compatible count selection without ref evidence or ref-map materialization", + "read_accounting": "per-path requested AX attribute slots plus action, child-label, promotion, and settable probe classes from locator stats", + "native_ipc": "not measured; native action and settable probe counts vary by role", + "scope": "deterministic synthetic Chromium/Electron accessibility fixtures; CPU and requested synthetic attributes only", + }, + "scenarios": reports, + }); + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} + +fn benchmark_scenario(scenario: &Scenario) -> Result> { + for run in 0..WARMUP_RUNS { + let fixture = scenario.frame(run); + black_box(run_legacy(fixture, &scenario.query)?); + black_box(run_live_direct(fixture, &scenario.query)?); + black_box(run_live_find(fixture, &scenario.query)?); + black_box(run_live_count(fixture, &scenario.query)?); + } + + let mut legacy_ns = Vec::with_capacity(MEASURED_RUNS); + let mut direct_ns = Vec::with_capacity(MEASURED_RUNS); + let mut find_ns = Vec::with_capacity(MEASURED_RUNS); + let mut count_ns = Vec::with_capacity(MEASURED_RUNS); + let mut legacy_counts = Vec::with_capacity(MEASURED_RUNS); + let mut direct_counts = Vec::with_capacity(MEASURED_RUNS); + let mut find_counts = Vec::with_capacity(MEASURED_RUNS); + let mut count_counts = Vec::with_capacity(MEASURED_RUNS); + let mut legacy_scanned = 0_u64; + let mut direct_scanned = 0_u64; + let mut find_scanned = 0_u64; + let mut legacy_predicate_visits = 0_u64; + let mut direct_predicate_cells = 0_u64; + let mut find_predicate_cells = 0_u64; + let mut direct_dom_matches = 0_u64; + let mut find_dom_matches = 0_u64; + let mut find_ref_count = 0_usize; + let mut count_action_reads = 0_u64; + let mut direct_action_reads = 0_u64; + let mut find_action_reads = 0_u64; + let mut direct_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64); + let mut find_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64); + let mut count_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64); + let mut find_refs_reresolvable = true; + + for run in 0..MEASURED_RUNS { + let fixture = scenario.frame(run); + let (legacy, direct, find, count) = match run % 4 { + 0 => ( + run_legacy(fixture, &scenario.query)?, + run_live_direct(fixture, &scenario.query)?, + run_live_find(fixture, &scenario.query)?, + run_live_count(fixture, &scenario.query)?, + ), + 1 => { + let direct = run_live_direct(fixture, &scenario.query)?; + let find = run_live_find(fixture, &scenario.query)?; + let count = run_live_count(fixture, &scenario.query)?; + let legacy = run_legacy(fixture, &scenario.query)?; + (legacy, direct, find, count) + } + 2 => { + let find = run_live_find(fixture, &scenario.query)?; + let count = run_live_count(fixture, &scenario.query)?; + let legacy = run_legacy(fixture, &scenario.query)?; + let direct = run_live_direct(fixture, &scenario.query)?; + (legacy, direct, find, count) + } + _ => { + let count = run_live_count(fixture, &scenario.query)?; + let legacy = run_legacy(fixture, &scenario.query)?; + let direct = run_live_direct(fixture, &scenario.query)?; + let find = run_live_find(fixture, &scenario.query)?; + (legacy, direct, find, count) + } + }; + legacy_ns.push(legacy.0); + direct_ns.push(direct.elapsed_ns); + find_ns.push(find.elapsed_ns); + count_ns.push(count.elapsed_ns); + legacy_counts.push(legacy.1); + direct_counts.push(direct.correctness.matches); + find_counts.push(find.correctness.matches); + count_counts.push(count.correctness.matches); + legacy_scanned = legacy.2; + legacy_predicate_visits = legacy.3; + direct_scanned = direct.visited; + direct_predicate_cells = direct.memo_cells; + direct_dom_matches = direct.dom_matches; + find_scanned = find.visited; + find_predicate_cells = find.memo_cells; + find_dom_matches = find.dom_matches; + find_ref_count = find.ref_count; + count_action_reads = count.action_reads; + direct_action_reads = direct.action_reads; + find_action_reads = find.action_reads; + direct_reads = read_counts(&direct); + find_reads = read_counts(&find); + count_reads = read_counts(&count); + find_refs_reresolvable &= find.correctness.selected_refs_reresolvable; + } + + let expected = scenario.expected_matches; + let legacy_correct = legacy_counts.iter().all(|count| *count == expected); + let direct_correct = direct_counts.iter().all(|count| *count == expected); + let find_correct = find_counts.iter().all(|count| *count == expected); + let count_correct = count_counts.iter().all(|count| *count == expected); + let legacy_p50 = percentile(&mut legacy_ns, 50); + let legacy_p95 = percentile(&mut legacy_ns, 95); + let direct_p50 = percentile(&mut direct_ns, 50); + let direct_p95 = percentile(&mut direct_ns, 95); + let find_p50 = percentile(&mut find_ns, 50); + let find_p95 = percentile(&mut find_ns, 95); + let count_p50 = percentile(&mut count_ns, 50); + let count_p95 = percentile(&mut count_ns, 95); + let fixture = scenario.frame(0); + let legacy_attributes_requested = fixture.nodes.len() as u64 * 22; + + Ok(json!({ + "name": scenario.name, + "fixture": { + "frames": scenario.frames.len(), + "nodes": fixture.nodes.len(), + "roots": fixture.roots.len(), + "moving_bounds": scenario.moving_bounds_verified(), + "legacy_locator_result_retained_handles": fixture.nodes.len(), + "observed_tree_retained_handles": 0, + }, + "expectation": { + "matches": expected, + "cardinality": cardinality(expected), + }, + "legacy_snapshot": { + "p50_us": ns_to_us(legacy_p50), + "p95_us": ns_to_us(legacy_p95), + "observed_matches": legacy_counts[0], + "correct_all_runs": legacy_correct, + "candidate_nodes_scanned": legacy_scanned, + "predicate_node_visits": legacy_predicate_visits, + "attributes_requested": legacy_attributes_requested, + "attribute_batches": fixture.nodes.len(), + "action_reads": fixture.nodes.len(), + }, + "live_arena_direct": { + "p50_us": ns_to_us(direct_p50), + "p95_us": ns_to_us(direct_p95), + "observed_matches": direct_counts[0], + "correct_all_runs": direct_correct, + "candidate_nodes_scanned": direct_scanned, + "memo_cells_evaluated": direct_predicate_cells, + "native_id_dom_matches": direct_dom_matches, + "attributes_requested": direct_reads.0, + "attribute_batches": direct_reads.1, + "child_label_reads": direct_reads.2, + "promotion_reads": direct_reads.3, + "settable_reads": direct_reads.4, + "action_reads": direct_action_reads, + "peak_handles_owned": direct_reads.5, + }, + "live_find_selected_refs": { + "p50_us": ns_to_us(find_p50), + "p95_us": ns_to_us(find_p95), + "observed_matches": find_counts[0], + "correct_all_runs": find_correct, + "candidate_nodes_scanned": find_scanned, + "memo_cells_evaluated": find_predicate_cells, + "native_id_dom_matches": find_dom_matches, + "ref_count": find_ref_count, + "selected_refs_reresolvable": find_refs_reresolvable, + "attributes_requested": find_reads.0, + "attribute_batches": find_reads.1, + "child_label_reads": find_reads.2, + "promotion_reads": find_reads.3, + "settable_reads": find_reads.4, + "action_reads": find_action_reads, + "peak_handles_owned": find_reads.5, + }, + "live_count_no_refmap": { + "p50_us": ns_to_us(count_p50), + "p95_us": ns_to_us(count_p95), + "observed_matches": count_counts[0], + "correct_all_runs": count_correct, + "action_reads": count_action_reads, + "ref_count": 0, + "attributes_requested": count_reads.0, + "attribute_batches": count_reads.1, + "child_label_reads": count_reads.2, + "promotion_reads": count_reads.3, + "settable_reads": count_reads.4, + "peak_handles_owned": count_reads.5, + }, + "comparison": { + "p50_find_speedup": ratio(legacy_p50, find_p50), + "p95_find_speedup": ratio(legacy_p95, find_p95), + "p50_direct_target_speedup": ratio(legacy_p50, direct_p50), + "p95_direct_target_speedup": ratio(legacy_p95, direct_p95), + "p50_count_speedup": ratio(legacy_p50, count_p50), + "p95_count_speedup": ratio(legacy_p95, count_p95), + "find_correctness_delta": i8::from(find_correct) - i8::from(legacy_correct), + "direct_target_correctness_delta": i8::from(direct_correct) - i8::from(legacy_correct), + "count_correctness_delta": i8::from(count_correct) - i8::from(legacy_correct), + }, + })) +} + +fn read_counts(run: &live::LiveRun) -> (u64, u64, u64, u64, u64, u64) { + ( + run.attributes_requested, + run.attribute_batches, + run.child_label_reads, + run.promotion_reads, + run.settable_reads, + run.peak_handles_owned, + ) +} + +fn percentile(values: &mut [u128], percentile: usize) -> u128 { + values.sort_unstable(); + let index = (values.len() * percentile).div_ceil(100).saturating_sub(1); + values[index] +} + +fn ns_to_us(nanoseconds: u128) -> f64 { + nanoseconds as f64 / 1_000.0 +} + +fn ratio(before: u128, after: u128) -> f64 { + if after == 0 { + return 0.0; + } + before as f64 / after as f64 +} + +fn cardinality(matches: usize) -> &'static str { + match matches { + 0 => "zero", + 1 => "one", + _ => "many", + } +} diff --git a/crates/core/examples/locator_benchmark/adapter.rs b/crates/core/examples/locator_benchmark/adapter.rs new file mode 100644 index 0000000..9d107ad --- /dev/null +++ b/crates/core/examples/locator_benchmark/adapter.rs @@ -0,0 +1,173 @@ +use crate::{ + fixture::Fixture, + fixture_builder::{live_target_tree, live_tree}, +}; +use agent_desktop_core::{ + ActionOps, AdapterError, Deadline, ElementIdentifier, ErrorCode, IdentifierKind, InputOps, + NativeHandle, ObservationOps, ObservationRequest, ObservationRoot, ObservationSource, + ObservedTree, RefEntry, SystemOps, WindowFilter, WindowInfo, +}; + +pub(crate) struct FixtureAdapter<'a> { + pub fixture: &'a Fixture, +} + +impl ActionOps for FixtureAdapter<'_> {} +impl InputOps for FixtureAdapter<'_> {} +impl SystemOps for FixtureAdapter<'_> {} + +impl ObservationOps for FixtureAdapter<'_> { + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: Deadline, + ) -> Result, AdapterError> { + Ok(vec![self.fixture.window.clone()]) + } + + fn observe_tree( + &self, + root: ObservationRoot<'_>, + request: &ObservationRequest, + ) -> Result { + match &root { + ObservationRoot::Window(_) => { + live_tree(self.fixture, request.evidence_for_raw_depth(0)) + } + ObservationRoot::Element { entry, .. } => { + let index = fixture_index_for_path(self.fixture, entry.scope.path.as_slice())?; + live_target_tree( + self.fixture, + index, + ObservationSource::from_root(&root), + request, + ) + } + } + } + + fn resolve_locator_anchor( + &self, + entry: &RefEntry, + deadline: Deadline, + ) -> Result { + if deadline.is_expired() { + return Err(AdapterError::timeout( + "benchmark locator anchor deadline expired", + )); + } + verify_source(self.fixture, entry)?; + let index = fixture_index_for_path(self.fixture, entry.scope.path.as_slice())?; + let node = self + .fixture + .nodes + .get(index as usize) + .ok_or_else(|| stale_anchor("benchmark locator path is invalid"))?; + if node.role != entry.identity.role { + return Err(stale_anchor("benchmark locator role changed")); + } + let identity_matches = entry + .identity + .native_id + .as_ref() + .is_some_and(|identifier| identifier_matches(node, identifier)); + let bounds_match = entry.geometry.bounds_hash.is_some() + && entry.geometry.bounds_hash == node.bounds.bounds_hash(); + if !identity_matches && !bounds_match { + return Err(stale_anchor("benchmark locator anchor changed")); + } + Ok(NativeHandle::null()) + } +} + +fn verify_source(fixture: &Fixture, entry: &RefEntry) -> Result<(), AdapterError> { + let window = &fixture.window; + let process_matches = entry.process.pid == window.pid + && entry.process.process_instance == window.process_instance; + let window_matches = entry.source.source_window_id.as_deref() == Some(window.id.as_str()) + && entry.source.source_app.as_deref() == Some(window.app.as_str()); + if process_matches && window_matches { + Ok(()) + } else { + Err(stale_anchor("benchmark locator source changed")) + } +} + +fn identifier_matches( + node: &crate::fixture_node::FixtureNode, + identifier: &ElementIdentifier, +) -> bool { + let value = identifier.value.as_str(); + match identifier.kind { + IdentifierKind::AxIdentifier => node.identifiers.0.as_deref() == Some(value), + IdentifierKind::AxDomIdentifier => node.identifiers.1.as_deref() == Some(value), + IdentifierKind::Unknown => { + node.identifiers.0.as_deref() == Some(value) + || node.identifiers.1.as_deref() == Some(value) + } + IdentifierKind::AutomationId + | IdentifierKind::RuntimeId + | IdentifierKind::AtspiObjectPath => false, + } +} + +fn stale_anchor(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::StaleRef, message) +} + +fn fixture_index_for_path(fixture: &Fixture, path: &[usize]) -> Result { + let mut index = *fixture + .roots + .first() + .ok_or_else(|| AdapterError::internal("benchmark fixture has no root"))?; + for child_order in path { + let node = fixture + .nodes + .get(index as usize) + .ok_or_else(|| AdapterError::internal("benchmark fixture path is invalid"))?; + index = *node + .children + .get(*child_order) + .ok_or_else(|| AdapterError::internal("benchmark fixture path is invalid"))?; + } + Ok(index) +} + +#[cfg(test)] +mod tests { + #[test] + fn selected_find_hydrates_every_benchmark_frame() { + for scenario in crate::scenarios::all() { + for fixture in &scenario.frames { + let result = crate::live::run_live_find(fixture, &scenario.query) + .expect("selected benchmark find"); + + assert_eq!( + result.correctness.matches, scenario.expected_matches, + "{}", + scenario.name + ); + assert!( + result.correctness.selected_refs_reresolvable, + "{}", + scenario.name + ); + } + } + } + + #[test] + fn selected_find_uses_the_exact_path_anchor() { + let scenario = crate::scenarios::all() + .into_iter() + .find(|scenario| scenario.name == "duplicate_button_role_and_name") + .expect("benchmark scenario"); + + let result = crate::live::run_live_find(scenario.frame(0), &scenario.query) + .expect("selected benchmark find"); + + assert_eq!(result.correctness.matches, scenario.expected_matches); + assert!(result.correctness.selected_refs_reresolvable); + assert_eq!(result.ref_count, 50); + } +} diff --git a/crates/core/examples/locator_benchmark/fixture.rs b/crates/core/examples/locator_benchmark/fixture.rs new file mode 100644 index 0000000..82324b2 --- /dev/null +++ b/crates/core/examples/locator_benchmark/fixture.rs @@ -0,0 +1,9 @@ +use crate::fixture_node::FixtureNode; +use agent_desktop_core::WindowInfo; + +#[derive(Clone)] +pub(crate) struct Fixture { + pub nodes: Vec, + pub roots: Vec, + pub window: WindowInfo, +} diff --git a/crates/core/examples/locator_benchmark/fixture_builder.rs b/crates/core/examples/locator_benchmark/fixture_builder.rs new file mode 100644 index 0000000..c91c31a --- /dev/null +++ b/crates/core/examples/locator_benchmark/fixture_builder.rs @@ -0,0 +1,318 @@ +use crate::{fixture::Fixture, fixture_node::FixtureNode}; +use agent_desktop_core::{ + AdapterError, ElementIdentifier, EvidenceRequirements, IdentifierEvidence, IdentifierKind, + LocatorEvidence, LocatorField, LocatorIdentifierStats, LocatorReadCounts, LocatorReadStats, + LocatorRefEvidence, LocatorSemanticReadStats, LocatorStats, LocatorTraversalStats, + ObservationRequest, ObservationSource, ObservedSubtree, ObservedTree, +}; + +pub(crate) fn live_tree( + fixture: &Fixture, + requirements: EvidenceRequirements, +) -> Result { + live_tree_from_roots( + fixture, + &fixture.roots, + ObservationSource::Window(fixture.window.clone()), + requirements, + ) +} + +pub(crate) fn live_tree_from_roots( + fixture: &Fixture, + root_indices: &[u32], + source: ObservationSource, + requirements: EvidenceRequirements, +) -> Result { + let roots = root_indices + .iter() + .map(|root| live_node(fixture, *root, requirements)) + .collect::, _>>()?; + let observed_indices = reachable_indices(fixture, root_indices)?; + ObservedTree::from_roots( + roots, + source, + stats_for_indices(fixture, &observed_indices, requirements), + true, + ) +} + +pub(crate) fn live_target_tree( + fixture: &Fixture, + index: u32, + source: ObservationSource, + request: &ObservationRequest, +) -> Result { + let mut reads = Vec::new(); + let root = live_target_node(fixture, index, request, 0, &mut reads)?; + ObservedTree::from_roots(vec![root], source, stats_for_reads(fixture, &reads), true) +} + +fn live_target_node( + fixture: &Fixture, + index: u32, + request: &ObservationRequest, + raw_depth: u8, + reads: &mut Vec<(usize, EvidenceRequirements)>, +) -> Result { + let node = fixture + .nodes + .get(index as usize) + .ok_or_else(|| AdapterError::internal("benchmark fixture index is out of bounds"))?; + let requirements = request.evidence_for_raw_depth(raw_depth); + reads.push((index as usize, requirements)); + let at_boundary = raw_depth >= request.max_logical_depth || raw_depth >= request.max_raw_depth; + let children = if at_boundary { + Vec::new() + } else { + node.children + .iter() + .map(|child| { + live_target_node(fixture, *child, request, raw_depth.saturating_add(1), reads) + }) + .collect::, _>>()? + }; + let children_count = at_boundary + .then(|| u32::try_from(node.children.len()).ok()) + .flatten() + .filter(|count| *count > 0); + Ok(ObservedSubtree::new( + node_evidence(node, requirements), + children, + true, + children_count, + )) +} + +fn live_node( + fixture: &Fixture, + index: u32, + requirements: EvidenceRequirements, +) -> Result { + let node = fixture + .nodes + .get(index as usize) + .ok_or_else(|| AdapterError::internal("benchmark fixture index is out of bounds"))?; + let children = node + .children + .iter() + .map(|child| live_node(fixture, *child, requirements)) + .collect::, _>>()?; + Ok(ObservedSubtree::new( + node_evidence(node, requirements), + children, + true, + None, + )) +} + +fn node_evidence(node: &FixtureNode, requirements: EvidenceRequirements) -> LocatorEvidence { + LocatorEvidence { + role: LocatorField::Known(node.role.clone()), + name: requested_field(requirements.name, node.name.clone()), + description: requested_field(requirements.description, None), + value: requested_field(requirements.value, None), + identifiers: if requirements.identifiers { + IdentifierEvidence::typed( + [ + node.identifiers.0.clone().map(|value| ElementIdentifier { + kind: IdentifierKind::AxIdentifier, + value, + }), + node.identifiers.1.clone().map(|value| ElementIdentifier { + kind: IdentifierKind::AxDomIdentifier, + value, + }), + ] + .into_iter() + .flatten(), + Some(0), + true, + ) + } else { + IdentifierEvidence::unknown() + }, + states: if requirements.states { + LocatorField::Known(Vec::new()) + } else { + LocatorField::Unknown + }, + ref_evidence: LocatorRefEvidence { + bounds: if requirements.ref_evidence.bounds { + LocatorField::Known(node.bounds) + } else { + LocatorField::Unknown + }, + available_actions: if requirements.ref_evidence.actions { + LocatorField::Known(actions(node)) + } else { + LocatorField::Unknown + }, + }, + } +} + +fn stats_for_indices( + fixture: &Fixture, + indices: &[usize], + requirements: EvidenceRequirements, +) -> LocatorStats { + let reads = indices + .iter() + .map(|index| (*index, requirements)) + .collect::>(); + stats_for_reads(fixture, &reads) +} + +fn stats_for_reads(fixture: &Fixture, reads: &[(usize, EvidenceRequirements)]) -> LocatorStats { + let count = reads.len() as u64; + let indices = reads.iter().map(|(index, _)| *index).collect::>(); + LocatorStats { + traversal: LocatorTraversalStats { + nodes_visited: count, + peak_handles_owned: 0, + max_raw_depth: 0, + max_logical_depth: 0, + web_wrapper_nodes: indices + .iter() + .filter_map(|index| fixture.nodes.get(*index)) + .filter(|node| node.role == "group" && node.name.is_none()) + .count() as u64, + ..LocatorTraversalStats::default() + }, + reads: LocatorReadStats { + counts: LocatorReadCounts { + attribute_batches: count, + attributes_requested: reads + .iter() + .map(|(_, requirements)| u64::from(attribute_count(*requirements))) + .sum(), + child_reads: count, + action_reads: reads + .iter() + .filter(|(_, requirements)| requirements.ref_evidence.actions) + .count() as u64, + ..LocatorReadCounts::default() + }, + ..LocatorReadStats::default() + }, + semantic_reads: LocatorSemanticReadStats { + settable_reads: reads + .iter() + .filter_map(|(index, requirements)| { + fixture + .nodes + .get(*index) + .map(|node| modeled_settable_read(node, *requirements)) + }) + .sum(), + ..LocatorSemanticReadStats::default() + }, + identifiers: identifier_stats(fixture, &indices), + ..LocatorStats::default() + } +} + +fn requested_field(requested: bool, value: Option) -> LocatorField { + if requested { + value.map_or(LocatorField::Absent, LocatorField::Known) + } else { + LocatorField::Unknown + } +} + +fn attribute_count(requirements: EvidenceRequirements) -> u32 { + let mut mask = 1_u32; + if requirements.name || requirements.description { + for index in [1, 2, 3, 15, 16, 17] { + mask |= 1_u32 << index; + } + } + if requirements.value { + mask |= 1_u32 << 3; + } + if requirements.identifiers { + mask |= (1_u32 << 13) | (1_u32 << 14); + } + if requirements.states { + for index in 3..=12 { + mask |= 1_u32 << index; + } + mask |= (1_u32 << 18) | (1_u32 << 19); + } + if requirements.ref_evidence.bounds { + for index in 18..=19 { + mask |= 1_u32 << index; + } + } + if requirements.ref_evidence.actions { + for index in 20..=21 { + mask |= 1_u32 << index; + } + } + mask.count_ones() +} + +fn modeled_settable_read(node: &FixtureNode, requirements: EvidenceRequirements) -> u64 { + let state_read = u64::from(requirements.states && node.role == "textfield"); + let action_reads = if requirements.ref_evidence.actions { + match node.role.as_str() { + "textfield" => 2, + "button" => 1, + _ => 0, + } + } else { + 0 + }; + state_read + action_reads +} + +fn reachable_indices(fixture: &Fixture, roots: &[u32]) -> Result, AdapterError> { + fn visit(fixture: &Fixture, index: u32, indices: &mut Vec) -> Result<(), AdapterError> { + let node = fixture + .nodes + .get(index as usize) + .ok_or_else(|| AdapterError::internal("benchmark fixture index is out of bounds"))?; + indices.push(index as usize); + for child in &node.children { + visit(fixture, *child, indices)?; + } + Ok(()) + } + + let mut indices = Vec::new(); + for root in roots { + visit(fixture, *root, &mut indices)?; + } + Ok(indices) +} + +fn actions(node: &FixtureNode) -> Vec { + match node.role.as_str() { + "button" | "textfield" => vec!["Click".to_string()], + _ => Vec::new(), + } +} + +fn identifier_stats(fixture: &Fixture, indices: &[usize]) -> LocatorIdentifierStats { + LocatorIdentifierStats { + values_observed: indices + .iter() + .filter_map(|index| fixture.nodes.get(*index)) + .map(|node| { + u64::from(node.identifiers.0.is_some()) + u64::from(node.identifiers.1.is_some()) + }) + .sum(), + nodes_with_identifiers: indices + .iter() + .filter_map(|index| fixture.nodes.get(*index)) + .filter(|node| node.identifiers.0.is_some() || node.identifiers.1.is_some()) + .count() as u64, + nodes_with_multiple_identifiers: indices + .iter() + .filter_map(|index| fixture.nodes.get(*index)) + .filter(|node| node.identifiers.0.is_some() && node.identifiers.1.is_some()) + .count() as u64, + ..LocatorIdentifierStats::default() + } +} diff --git a/crates/core/examples/locator_benchmark/fixture_node.rs b/crates/core/examples/locator_benchmark/fixture_node.rs new file mode 100644 index 0000000..f910f64 --- /dev/null +++ b/crates/core/examples/locator_benchmark/fixture_node.rs @@ -0,0 +1,28 @@ +use agent_desktop_core::Rect; + +#[derive(Clone)] +pub(crate) struct FixtureNode { + pub role: String, + pub name: Option, + pub identifiers: (Option, Option), + pub bounds: Rect, + pub children: Vec, +} + +impl FixtureNode { + pub fn new( + role: &str, + name: Option<&str>, + identifiers: (Option, Option), + bounds: Rect, + children: Vec, + ) -> Self { + Self { + role: role.to_string(), + name: name.map(str::to_string), + identifiers, + bounds, + children, + } + } +} diff --git a/crates/core/examples/locator_benchmark/legacy.rs b/crates/core/examples/locator_benchmark/legacy.rs new file mode 100644 index 0000000..e40fca0 --- /dev/null +++ b/crates/core/examples/locator_benchmark/legacy.rs @@ -0,0 +1,140 @@ +use crate::{adapter::FixtureAdapter, fixture::Fixture}; +use agent_desktop_core::{ + AccessibilityNode, AppError, Deadline, LocatorQuery, accessibility_node_matches, snapshot, +}; +use std::{hint::black_box, time::Instant}; + +pub(crate) fn run_legacy( + fixture: &Fixture, + query: &LocatorQuery, +) -> Result<(u128, usize, u64, u64), AppError> { + let adapter = FixtureAdapter { fixture }; + let started = Instant::now(); + let snapshot = snapshot::build( + &adapter, + &agent_desktop_core::TreeOptions::default(), + Some(&fixture.window.app), + None, + Deadline::after(5_000)?, + )?; + let mut scanned = 0_u64; + let matches = count_matches(&snapshot.tree, query, &mut scanned); + black_box((&snapshot.refmap, matches)); + let elapsed = started.elapsed().as_nanos(); + let mut predicate_visits = 0_u64; + instrument_predicate_visits(&snapshot.tree, query, &mut predicate_visits); + Ok((elapsed, matches, scanned, predicate_visits)) +} + +fn count_matches(node: &AccessibilityNode, query: &LocatorQuery, scanned: &mut u64) -> usize { + *scanned += 1; + usize::from(accessibility_node_matches(node, query)) + + node + .children + .iter() + .map(|child| count_matches(child, query, scanned)) + .sum::() +} + +fn instrument_predicate_visits(node: &AccessibilityNode, query: &LocatorQuery, visits: &mut u64) { + let _ = instrumented_match(node, query, visits); + for child in &node.children { + instrument_predicate_visits(child, query, visits); + } +} + +fn instrumented_match(node: &AccessibilityNode, query: &LocatorQuery, visits: &mut u64) -> bool { + *visits += 1; + if !identity_matches(node, query) { + return false; + } + if let Some(expected) = query.has_text.as_deref() { + if !node_text_matches(node, expected) && !descendant_text_matches(node, expected, visits) { + return false; + } + } + if let Some(has) = query.containment.has.as_deref() { + if !descendant_query_matches(node, has, visits) { + return false; + } + } + if let Some(has_not) = query.containment.has_not.as_deref() { + if descendant_query_matches(node, has_not, visits) { + return false; + } + } + true +} + +fn identity_matches(node: &AccessibilityNode, query: &LocatorQuery) -> bool { + query + .identity + .role + .as_deref() + .is_none_or(|expected| normalized(&node.role) == expected) + && field_matches( + query.identity.name.as_deref(), + node.identity.name.as_deref(), + query.exact, + ) + && query.identity.native_id.as_deref().is_none_or(|expected| { + node.identity + .native_id + .as_ref() + .is_some_and(|actual| actual.value == expected) + }) +} + +fn field_matches(expected: Option<&str>, actual: Option<&str>, exact: bool) -> bool { + let Some(expected) = expected else { + return true; + }; + let Some(actual) = actual else { + return false; + }; + let actual = normalized(actual); + if exact { + actual == expected + } else { + actual.contains(expected) + } +} + +fn descendant_text_matches(node: &AccessibilityNode, expected: &str, visits: &mut u64) -> bool { + for child in &node.children { + *visits += 1; + if node_text_matches(child, expected) || descendant_text_matches(child, expected, visits) { + return true; + } + } + false +} + +fn descendant_query_matches( + node: &AccessibilityNode, + query: &LocatorQuery, + visits: &mut u64, +) -> bool { + node.children.iter().any(|child| { + instrumented_match(child, query, visits) || descendant_query_matches(child, query, visits) + }) +} + +fn node_text_matches(node: &AccessibilityNode, expected: &str) -> bool { + [ + node.identity.name.as_deref(), + node.identity.description.as_deref(), + node.identity.value.as_deref(), + ] + .into_iter() + .flatten() + .any(|value| normalized(value).contains(expected)) +} + +fn normalized(value: &str) -> String { + value + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase() +} diff --git a/crates/core/examples/locator_benchmark/live.rs b/crates/core/examples/locator_benchmark/live.rs new file mode 100644 index 0000000..b91f495 --- /dev/null +++ b/crates/core/examples/locator_benchmark/live.rs @@ -0,0 +1,148 @@ +use crate::{adapter::FixtureAdapter, fixture::Fixture}; +use agent_desktop_core::{ + AppError, Deadline, LocatorMaterialization, LocatorQuery, LocatorResolution, + LocatorResolveRequest, LocatorSelection, ObservationRoot, resolve_query, +}; +use std::{hint::black_box, time::Instant}; + +pub(crate) struct LiveCorrectness { + pub(crate) matches: usize, + pub(crate) selected_refs_reresolvable: bool, +} + +pub(crate) struct LiveRun { + pub(crate) elapsed_ns: u128, + pub(crate) correctness: LiveCorrectness, + pub(crate) visited: u64, + pub(crate) memo_cells: u64, + pub(crate) dom_matches: u64, + pub(crate) ref_count: usize, + pub(crate) action_reads: u64, + pub(crate) attributes_requested: u64, + pub(crate) attribute_batches: u64, + pub(crate) child_label_reads: u64, + pub(crate) promotion_reads: u64, + pub(crate) settable_reads: u64, + pub(crate) peak_handles_owned: u64, +} + +pub(crate) fn run_live_direct( + fixture: &Fixture, + query: &LocatorQuery, +) -> Result { + run_live( + fixture, + query, + LocatorSelection::Strict, + LocatorMaterialization::None, + ) +} + +pub(crate) fn run_live_find(fixture: &Fixture, query: &LocatorQuery) -> Result { + run_live( + fixture, + query, + LocatorSelection::All { limit: Some(50) }, + LocatorMaterialization::SelectedMatches, + ) +} + +pub(crate) fn run_live_count(fixture: &Fixture, query: &LocatorQuery) -> Result { + run_live( + fixture, + query, + LocatorSelection::Count, + LocatorMaterialization::None, + ) +} + +fn run_live( + fixture: &Fixture, + query: &LocatorQuery, + selection: LocatorSelection, + materialization: LocatorMaterialization, +) -> Result { + let request = LocatorResolveRequest { + selection, + deadline: Deadline::after(5_000)?, + max_raw_depth: 50, + materialization, + }; + let adapter = FixtureAdapter { fixture }; + let started = Instant::now(); + let resolution = resolve_query( + &adapter, + query, + ObservationRoot::Window(&fixture.window), + &request, + )?; + black_box((&resolution.refmap, &resolution.matches)); + let elapsed = started.elapsed().as_nanos(); + let ref_count = resolution.refmap.as_ref().map_or(0, |refmap| refmap.len()); + let selected_refs_reresolvable = selected_refs_reresolvable(fixture, &resolution); + Ok(LiveRun { + elapsed_ns: elapsed, + correctness: LiveCorrectness { + matches: resolution.meta.total_matches as usize, + selected_refs_reresolvable, + }, + visited: resolution.stats.traversal.nodes_visited, + memo_cells: resolution.stats.evaluation.memo_cells_evaluated, + dom_matches: resolution.stats.identifiers.fallback_matches, + ref_count, + action_reads: resolution.stats.reads.counts.action_reads, + attributes_requested: resolution.stats.reads.counts.attributes_requested, + attribute_batches: resolution.stats.reads.counts.attribute_batches, + child_label_reads: resolution.stats.semantic_reads.child_label_reads, + promotion_reads: resolution.stats.semantic_reads.promotion_reads, + settable_reads: resolution.stats.semantic_reads.settable_reads, + peak_handles_owned: resolution.stats.traversal.peak_handles_owned, + }) +} + +fn selected_refs_reresolvable(fixture: &Fixture, resolution: &LocatorResolution) -> bool { + let Some(refmap) = resolution.refmap.as_ref() else { + return true; + }; + resolution + .matches + .iter() + .filter_map(|selected| selected.data.ref_id.as_deref()) + .all(|ref_id| { + let Some(entry) = refmap.get(ref_id) else { + return false; + }; + fixture.nodes.iter().any(|node| { + entry.geometry.bounds_hash == node.bounds.bounds_hash() + && node.role == entry.identity.role + && entry.identity.native_id.as_ref().map_or_else( + || entry.identity.name == node.name, + |expected| { + node.identifiers.0.as_deref() == Some(expected.value.as_str()) + || node.identifiers.1.as_deref() == Some(expected.value.as_str()) + }, + ) + }) + }) +} + +#[cfg(test)] +mod tests { + use agent_desktop_core::Rect; + + #[test] + fn duplicate_names_require_the_selected_nodes_bounds() { + let selected = Rect { + x: 10.0, + y: 20.0, + width: 30.0, + height: 40.0, + }; + let sibling = Rect { + x: 50.0, + ..selected + }; + + assert_ne!(selected.bounds_hash(), sibling.bounds_hash()); + } +} diff --git a/crates/core/examples/locator_benchmark/scenario.rs b/crates/core/examples/locator_benchmark/scenario.rs new file mode 100644 index 0000000..f66af1b --- /dev/null +++ b/crates/core/examples/locator_benchmark/scenario.rs @@ -0,0 +1,45 @@ +use crate::fixture::Fixture; +use agent_desktop_core::{LocatorQuery, Rect}; +use std::collections::BTreeSet; + +pub(crate) struct Scenario { + pub name: &'static str, + pub frames: Vec, + pub query: LocatorQuery, + pub expected_matches: usize, +} + +impl Scenario { + pub fn frame(&self, run: usize) -> &Fixture { + &self.frames[run % self.frames.len()] + } + + pub fn moving_bounds_verified(&self) -> bool { + let bounds = self + .frames + .iter() + .flat_map(target_bounds) + .map(rect_key) + .collect::>(); + bounds.len() > 1 + } +} + +fn target_bounds(fixture: &Fixture) -> impl Iterator + '_ { + fixture.nodes.iter().filter_map(|node| { + node.identifiers + .1 + .as_deref() + .is_some_and(|value| value == "composer-send") + .then_some(node.bounds) + }) +} + +fn rect_key(bounds: Rect) -> (i64, i64, i64, i64) { + ( + (bounds.x * 100.0) as i64, + (bounds.y * 100.0) as i64, + (bounds.width * 100.0) as i64, + (bounds.height * 100.0) as i64, + ) +} diff --git a/crates/core/examples/locator_benchmark/scenarios.rs b/crates/core/examples/locator_benchmark/scenarios.rs new file mode 100644 index 0000000..23a590f --- /dev/null +++ b/crates/core/examples/locator_benchmark/scenarios.rs @@ -0,0 +1,239 @@ +use crate::{fixture::Fixture, fixture_node::FixtureNode, scenario::Scenario}; +use agent_desktop_core::{ContainmentPredicate, IdentityPredicate, LocatorQuery, Rect, WindowInfo}; + +const MOVING_FRAMES: usize = 11; + +pub(crate) fn all() -> Vec { + vec![ + deep_anonymous_containment(), + duplicate_role_and_name(), + electron_dual_identifier_moving_bounds(), + large_electron_channel_tree(), + ] +} + +fn deep_anonymous_containment() -> Scenario { + let fixture = chain_fixture(64, 48); + Scenario { + name: "deep_anonymous_has_text", + frames: vec![fixture], + query: LocatorQuery { + identity: IdentityPredicate { + role: Some("group".into()), + name: Some("layer".into()), + ..IdentityPredicate::default() + }, + has_text: Some("needle".into()), + ..LocatorQuery::default() + }, + expected_matches: 64 * 24, + } +} + +fn duplicate_role_and_name() -> Scenario { + let fixture = button_fixture(0, false); + Scenario { + name: "duplicate_button_role_and_name", + frames: vec![fixture], + query: LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + name: Some("send".into()), + ..IdentityPredicate::default() + }, + exact: true, + ..LocatorQuery::default() + }, + expected_matches: 512, + } +} + +fn electron_dual_identifier_moving_bounds() -> Scenario { + let frames = (0..MOVING_FRAMES) + .map(|frame| button_fixture(frame, true)) + .collect(); + Scenario { + name: "electron_dual_identifier_moving_bounds", + frames, + query: LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + native_id: Some("composer-send".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }, + expected_matches: 1, + } +} + +fn large_electron_channel_tree() -> Scenario { + let fixture = channel_fixture(640, 8); + Scenario { + name: "large_nested_channel_has_unread", + frames: vec![fixture], + query: LocatorQuery { + identity: IdentityPredicate { + role: Some("group".into()), + name: Some("channel".into()), + ..IdentityPredicate::default() + }, + containment: ContainmentPredicate { + has: Some(Box::new(LocatorQuery { + identity: IdentityPredicate { + name: Some("unread".into()), + ..IdentityPredicate::default() + }, + exact: true, + ..LocatorQuery::default() + })), + has_not: None, + }, + exact: true, + ..LocatorQuery::default() + }, + expected_matches: 10, + } +} + +fn chain_fixture(chains: usize, depth: usize) -> Fixture { + let mut nodes = vec![node("group", Some("Electron Root"), 0.0, Vec::new())]; + for chain in 0..chains { + let root = push( + &mut nodes, + node("group", Some("Layer"), chain as f64, Vec::new()), + ); + nodes[0].children.push(root); + let mut parent = root; + for level in 1..depth { + let name = (level % 2 == 0).then_some("Layer"); + let child = push( + &mut nodes, + node("group", name, (chain * depth + level) as f64, Vec::new()), + ); + nodes[parent as usize].children.push(child); + parent = child; + } + let needle = push( + &mut nodes, + node("statictext", Some("Needle"), chain as f64, Vec::new()), + ); + nodes[parent as usize].children.push(needle); + } + Fixture { + nodes, + roots: vec![0], + window: window(), + } +} + +fn button_fixture(frame: usize, dual_identifier: bool) -> Fixture { + let mut nodes = vec![node("group", Some("Composer"), 0.0, Vec::new())]; + let target_position = frame * 37 % 512; + for position in 0..512 { + let logical = if position == target_position { + 0 + } else { + position + 1 + }; + let identifiers = if position == target_position && dual_identifier { + ( + Some("AXNode-8842".to_string()), + Some("composer-send".to_string()), + ) + } else { + ( + Some(format!("AXNode-{logical}")), + Some(format!("send-{logical}")), + ) + }; + let bounds = Rect { + x: (position % 32) as f64 * 18.0 + frame as f64 * 0.75, + y: (position / 32) as f64 * 22.0 + frame as f64 * 1.25, + width: 16.0, + height: 20.0, + }; + let child = push( + &mut nodes, + FixtureNode::new("button", Some("Send"), identifiers, bounds, Vec::new()), + ); + nodes[0].children.push(child); + } + Fixture { + nodes, + roots: vec![0], + window: window(), + } +} + +fn channel_fixture(channels: usize, wrapper_depth: usize) -> Fixture { + let mut nodes = vec![node("group", Some("Slack"), 0.0, Vec::new())]; + for channel in 0..channels { + let row = push( + &mut nodes, + node("group", Some("Channel"), channel as f64, Vec::new()), + ); + nodes[0].children.push(row); + let mut parent = row; + for wrapper in 0..wrapper_depth { + let child = push( + &mut nodes, + node( + "group", + None, + (channel * wrapper_depth + wrapper) as f64, + Vec::new(), + ), + ); + nodes[parent as usize].children.push(child); + parent = child; + } + let label = (channel % 64 == 0).then_some("Unread"); + let text = push( + &mut nodes, + node("statictext", label, channel as f64, Vec::new()), + ); + nodes[parent as usize].children.push(text); + } + Fixture { + nodes, + roots: vec![0], + window: window(), + } +} + +fn node(role: &str, name: Option<&str>, offset: f64, children: Vec) -> FixtureNode { + FixtureNode::new( + role, + name, + (None, None), + Rect { + x: offset % 1200.0, + y: offset % 800.0, + width: 100.0, + height: 24.0, + }, + children, + ) +} + +fn push(nodes: &mut Vec, node: FixtureNode) -> u32 { + let index = nodes.len() as u32; + nodes.push(node); + index +} + +fn window() -> WindowInfo { + WindowInfo { + id: "w-electron-benchmark".into(), + title: "Electron Benchmark".into(), + app: "SyntheticElectron".into(), + pid: agent_desktop_core::ProcessId::new(4242), + process_instance: Some("benchmark-4242".into()), + bounds: None, + state: agent_desktop_core::WindowState { + is_focused: true, + ..Default::default() + }, + } +} diff --git a/crates/core/src/accname.rs b/crates/core/src/accname.rs new file mode 100644 index 0000000..6974801 --- /dev/null +++ b/crates/core/src/accname.rs @@ -0,0 +1,39 @@ +use crate::name_evidence::NameEvidence; + +pub fn compute_name(evidence: &NameEvidence) -> Option { + [ + evidence.explicit_label.as_deref(), + evidence.labelled_by_text.as_deref(), + evidence.native_title.as_deref(), + evidence.static_value.as_deref(), + evidence.child_label.as_deref(), + evidence.placeholder.as_deref(), + evidence.description.as_deref(), + ] + .into_iter() + .find_map(non_blank) + .map(str::to_string) +} + +pub fn compute_description(evidence: &NameEvidence) -> Option { + let description = non_blank(evidence.description.as_deref())?; + [ + evidence.explicit_label.as_deref(), + evidence.labelled_by_text.as_deref(), + evidence.native_title.as_deref(), + evidence.static_value.as_deref(), + evidence.child_label.as_deref(), + evidence.placeholder.as_deref(), + ] + .into_iter() + .any(|candidate| non_blank(candidate).is_some()) + .then(|| description.to_string()) +} + +fn non_blank(value: Option<&str>) -> Option<&str> { + value.filter(|value| !value.trim().is_empty()) +} + +#[cfg(test)] +#[path = "accname_tests.rs"] +mod tests; diff --git a/crates/core/src/accname_tests.rs b/crates/core/src/accname_tests.rs new file mode 100644 index 0000000..106104e --- /dev/null +++ b/crates/core/src/accname_tests.rs @@ -0,0 +1,68 @@ +use super::*; + +fn evidence() -> NameEvidence { + NameEvidence { + explicit_label: Some("explicit".into()), + labelled_by_text: Some("labelled".into()), + native_title: Some("title".into()), + static_value: Some("value".into()), + child_label: Some("child".into()), + placeholder: Some("placeholder".into()), + description: Some("description".into()), + } +} + +#[test] +fn name_precedence_is_platform_neutral_and_complete() { + let mut value = evidence(); + assert_eq!(compute_name(&value).as_deref(), Some("explicit")); + value.explicit_label = None; + assert_eq!(compute_name(&value).as_deref(), Some("labelled")); + value.labelled_by_text = None; + assert_eq!(compute_name(&value).as_deref(), Some("title")); + value.native_title = None; + assert_eq!(compute_name(&value).as_deref(), Some("value")); + value.static_value = None; + assert_eq!(compute_name(&value).as_deref(), Some("child")); + value.child_label = None; + assert_eq!(compute_name(&value).as_deref(), Some("placeholder")); + value.placeholder = None; + assert_eq!(compute_name(&value).as_deref(), Some("description")); +} + +#[test] +fn blank_evidence_is_ignored_without_rewriting_content() { + let value = NameEvidence { + explicit_label: Some(" \t ".into()), + native_title: Some(" Save ".into()), + ..NameEvidence::default() + }; + + assert_eq!(compute_name(&value).as_deref(), Some(" Save ")); +} + +#[test] +fn description_is_not_duplicated_when_it_supplies_the_name() { + let value = NameEvidence { + description: Some("Only description".into()), + ..NameEvidence::default() + }; + + assert_eq!(compute_name(&value).as_deref(), Some("Only description")); + assert_eq!(compute_description(&value), None); +} + +#[test] +fn description_remains_separate_when_an_earlier_name_exists() { + let value = NameEvidence { + placeholder: Some("Search".into()), + description: Some("Filters all messages".into()), + ..NameEvidence::default() + }; + + assert_eq!(compute_name(&value).as_deref(), Some("Search")); + assert_eq!( + compute_description(&value).as_deref(), + Some("Filters all messages") + ); +} diff --git a/crates/core/src/action.rs b/crates/core/src/action.rs index aa046ec..5faa923 100644 --- a/crates/core/src/action.rs +++ b/crates/core/src/action.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +use crate::{Direction, DragParams, KeyCombo}; + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Action { Click, @@ -26,6 +28,32 @@ pub enum Action { } impl Action { + pub fn headed_requirement(&self) -> crate::HeadedRequirement { + match self { + Self::Click + | Self::DoubleClick + | Self::RightClick + | Self::TripleClick + | Self::Scroll(_, _) + | Self::Hover + | Self::Drag(_) => crate::HeadedRequirement::FocusedWindowAndCursor, + Self::SetValue(_) + | Self::SetFocus + | Self::Expand + | Self::Collapse + | Self::Select(_) + | Self::Toggle + | Self::Check + | Self::Uncheck + | Self::ScrollTo + | Self::PressKey(_) + | Self::KeyDown(_) + | Self::KeyUp(_) + | Self::TypeText(_) + | Self::Clear => crate::HeadedRequirement::FocusedWindow, + } + } + pub fn name(&self) -> &'static str { match self { Self::Click => "click", @@ -56,17 +84,46 @@ impl Action { matches!(self, Self::Hover | Self::Drag(_)) } + pub fn requires_hit_test(&self) -> bool { + matches!( + self, + Self::Click + | Self::DoubleClick + | Self::RightClick + | Self::TripleClick + | Self::Hover + | Self::Drag(_) + ) + } + + pub fn requires_scroll_into_view(&self) -> bool { + matches!( + self, + Self::Click + | Self::DoubleClick + | Self::RightClick + | Self::TripleClick + | Self::SetValue(_) + | Self::Expand + | Self::Collapse + | Self::Select(_) + | Self::Toggle + | Self::Check + | Self::Uncheck + | Self::TypeText(_) + | Self::Clear + | Self::Hover + | Self::Drag(_) + ) + } + pub fn may_use_focus_fallback(&self) -> bool { matches!(self, Self::TypeText(_) | Self::PressKey(_)) } - /// Returns the minimum `InteractionPolicy` the CLI uses for this action. - /// `TypeText` and `PressKey` require focus to land in the right field, so - /// their base is `focus_fallback`. Everything else is pure-AX and uses - /// `headless`. FFI callers join this base with their caller-supplied policy - /// so they can only elevate, never downgrade below CLI parity. + /// Returns the command's minimum interaction policy. pub fn base_interaction_policy(&self) -> crate::interaction_policy::InteractionPolicy { - if self.may_use_focus_fallback() { + if matches!(self, Self::PressKey(_)) { crate::interaction_policy::InteractionPolicy::focus_fallback() } else { crate::interaction_policy::InteractionPolicy::headless() @@ -74,77 +131,6 @@ impl Action { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum Direction { - Up, - Down, - Left, - Right, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Point { - pub x: f64, - pub y: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum MouseButton { - Left, - Right, - Middle, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DragParams { - pub from: Point, - pub to: Point, - #[serde(skip_serializing_if = "Option::is_none")] - pub duration_ms: Option, - /// Time to hold over the destination before releasing. Some platforms require - /// a minimum dwell before the drop registers; `None` uses the adapter default. - #[serde(skip_serializing_if = "Option::is_none")] - pub drop_delay_ms: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum MouseEventKind { - Move, - Down, - Up, - Click { count: u32 }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MouseEvent { - pub kind: MouseEventKind, - pub point: Point, - pub button: MouseButton, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum WindowOp { - Resize { width: f64, height: f64 }, - Move { x: f64, y: f64 }, - Minimize, - Maximize, - Restore, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct KeyCombo { - pub key: String, - pub modifiers: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum Modifier { - Cmd, - Ctrl, - Alt, - Shift, -} - #[cfg(test)] #[path = "action_tests.rs"] mod tests; diff --git a/crates/core/src/action_request.rs b/crates/core/src/action_request.rs index b5f4155..f476f1f 100644 --- a/crates/core/src/action_request.rs +++ b/crates/core/src/action_request.rs @@ -1,3 +1,4 @@ +use crate::Point; use crate::action::Action; use crate::interaction_policy::InteractionPolicy; use serde::{Deserialize, Serialize}; @@ -6,13 +7,30 @@ use serde::{Deserialize, Serialize}; pub struct ActionRequest { pub action: Action, pub policy: InteractionPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, + #[serde(skip)] + pub(crate) verified_point: Option, + #[serde(skip)] + pub(crate) expected_process: Option, } impl ActionRequest { + pub fn headed_requirement(&self) -> crate::HeadedRequirement { + if self.policy.is_headed() { + self.action.headed_requirement() + } else { + crate::HeadedRequirement::None + } + } + pub fn headless(action: Action) -> Self { Self { action, policy: InteractionPolicy::headless(), + timeout_ms: None, + verified_point: None, + expected_process: None, } } @@ -20,6 +38,9 @@ impl ActionRequest { Self { action, policy: InteractionPolicy::focus_fallback(), + timeout_ms: None, + verified_point: None, + expected_process: None, } } @@ -27,14 +48,40 @@ impl ActionRequest { Self { action, policy: InteractionPolicy::headed(), + timeout_ms: None, + verified_point: None, + expected_process: None, } } + + pub fn with_timeout_ms(mut self, timeout_ms: Option) -> Self { + self.timeout_ms = timeout_ms; + self + } + + pub(crate) fn with_verified_point(mut self, point: Option) -> Self { + self.verified_point = point; + self + } + + pub fn with_expected_process(mut self, process: crate::ProcessIdentity) -> Self { + self.expected_process = Some(process); + self + } + + pub fn verified_point(&self) -> Option<&Point> { + self.verified_point.as_ref() + } + + pub fn expected_process(&self) -> Option<&crate::ProcessIdentity> { + self.expected_process.as_ref() + } } #[cfg(test)] mod tests { use super::*; - use crate::action::{Action, Direction}; + use crate::{Action, Direction}; #[test] fn default_policy_is_headless() { @@ -49,10 +96,67 @@ mod tests { assert_eq!(request.policy, InteractionPolicy::headless()); } + #[test] + fn headed_requirement_is_inert_until_the_caller_selects_headed_mode() { + assert_eq!( + ActionRequest::headless(Action::Click).headed_requirement(), + crate::HeadedRequirement::None + ); + assert_eq!( + ActionRequest::headed(Action::Click).headed_requirement(), + crate::HeadedRequirement::FocusedWindowAndCursor + ); + assert_eq!( + ActionRequest::headed(Action::TypeText("text".into())).headed_requirement(), + crate::HeadedRequirement::FocusedWindow + ); + } + #[test] fn focus_fallback_policy_never_moves_cursor() { let request = ActionRequest::focus_fallback(Action::Scroll(Direction::Down, 1)); assert!(request.policy.allow_focus_steal); assert!(!request.policy.allow_cursor_move); } + + /// Regression coverage: `ActionRequest.timeout_ms` must stay + /// `#[serde(default)]` so a legacy payload recorded before `timeout_ms` + /// existed (or any FFI/batch caller that omits the key) still + /// deserializes instead of erroring out. + #[test] + fn action_request_json_without_timeout_ms_key_deserializes_to_none() { + let request: ActionRequest = serde_json::from_value(serde_json::json!({ + "action": "Click", + "policy": { + "allow_focus_steal": false, + "allow_cursor_move": false, + }, + })) + .unwrap(); + + assert_eq!(request.timeout_ms, None); + } + + #[test] + fn positive_timeout_round_trips_through_the_wire_contract() { + let request = ActionRequest::headless(Action::Click).with_timeout_ms(Some(5_000)); + let json = serde_json::to_value(&request).unwrap(); + + assert_eq!(json["timeout_ms"], 5_000); + let decoded: ActionRequest = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.timeout_ms, Some(5_000)); + } + + #[test] + fn execution_identity_is_runtime_only() { + let request = ActionRequest::headless(Action::Click) + .with_expected_process(crate::ProcessIdentity::new(42, "generation-1")); + + assert_eq!( + request.expected_process(), + Some(&crate::ProcessIdentity::new(42, "generation-1")) + ); + let json = serde_json::to_value(request).unwrap(); + assert!(json.get("expected_process").is_none()); + } } diff --git a/crates/core/src/action_result.rs b/crates/core/src/action_result.rs index 287db09..2372271 100644 --- a/crates/core/src/action_result.rs +++ b/crates/core/src/action_result.rs @@ -1,5 +1,8 @@ -use crate::{action_step::ActionStep, element_state::ElementState}; -use serde::{Deserialize, Serialize}; +use crate::{ + Action, AdapterError, DeliverySemantics, ErrorCode, action_step::ActionStep, + action_step_outcome::ActionStepOutcome, element_state::ElementState, +}; +use serde::{Deserialize, Deserializer, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ActionResult { @@ -8,14 +11,56 @@ pub struct ActionResult { pub post_state: Option, #[serde(skip_serializing_if = "Vec::is_empty", default)] pub steps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub details: Option, + #[serde( + default = "default_action_disposition", + deserialize_with = "deserialize_action_disposition" + )] + disposition: DeliverySemantics, } impl ActionResult { - pub fn new(action: impl Into) -> Self { + pub fn from_execution( + action: &Action, + steps: Vec, + post_state: Option, + ) -> Result { + let label = action.name(); + let (delivered, verified) = delivery_summary(&steps); + if !delivered { + return Ok(Self::satisfied_without_delivery(label).with_steps(steps)); + } + if let Some(state) = post_state.as_ref() { + verify_post_state(action, state)?; + } + let mut result = Self::delivered_unverified(label).with_steps(steps); + if verified { + result = result.with_verified_delivery(); + } + if let Some(state) = post_state { + result = result.with_state(state); + } + Ok(result) + } + + pub fn satisfied_without_delivery(action: impl Into) -> Self { Self { action: action.into(), post_state: None, steps: Vec::new(), + details: None, + disposition: DeliverySemantics::not_delivered(), + } + } + + pub fn delivered_unverified(action: impl Into) -> Self { + Self { + action: action.into(), + post_state: None, + steps: Vec::new(), + details: None, + disposition: default_action_disposition(), } } @@ -28,4 +73,74 @@ impl ActionResult { self.steps = steps; self } + + pub fn with_details(mut self, details: serde_json::Value) -> Self { + self.details = Some(details); + self + } + + pub fn with_verified_delivery(mut self) -> Self { + if self.disposition == DeliverySemantics::delivered_unverified() { + self.disposition = DeliverySemantics::delivered_verified(); + } + self + } + + pub const fn disposition(&self) -> DeliverySemantics { + self.disposition + } } + +fn delivery_summary(steps: &[ActionStep]) -> (bool, bool) { + let mut delivered = false; + let mut verification = None; + for step in steps { + if matches!(step.outcome, ActionStepOutcome::Succeeded) { + delivered = true; + if let Some(verified) = step.verified() { + verification = Some(verification.unwrap_or(true) && verified); + } + } + } + (delivered, verification.unwrap_or(false)) +} + +fn verify_post_state(action: &Action, state: &ElementState) -> Result<(), AdapterError> { + if matches!(action, Action::Clear) + && state + .value + .as_deref() + .is_some_and(|value| !value.is_empty()) + { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "Clear reported success but element value is still non-empty", + ) + .with_suggestion("Retry 'clear', or use 'press cmd+a' then 'press delete'.") + .with_disposition(DeliverySemantics::delivered_unverified())); + } + Ok(()) +} + +const fn default_action_disposition() -> DeliverySemantics { + DeliverySemantics::delivered_unverified() +} + +fn deserialize_action_disposition<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let disposition = DeliverySemantics::deserialize(deserializer)?; + match disposition { + DeliverySemantics::NotDelivered + | DeliverySemantics::DeliveredUnverified + | DeliverySemantics::DeliveredVerified => Ok(disposition), + _ => Err(serde::de::Error::custom( + "successful action result must represent satisfied or delivered work", + )), + } +} + +#[cfg(test)] +#[path = "action_result_tests.rs"] +mod tests; diff --git a/crates/core/src/action_result_tests.rs b/crates/core/src/action_result_tests.rs new file mode 100644 index 0000000..272a767 --- /dev/null +++ b/crates/core/src/action_result_tests.rs @@ -0,0 +1,190 @@ +use super::*; + +fn state(value: Option<&str>) -> ElementState { + ElementState { + role: "textfield".into(), + states: vec![], + value: value.map(str::to_string), + enabled: None, + hidden: None, + offscreen: None, + } +} + +fn execution_result(verifications: &[Option]) -> ActionResult { + let steps = verifications + .iter() + .map(|verified| { + let step = ActionStep::succeeded("Step"); + match verified { + Some(value) => step.with_verified(*value), + None => step, + } + }) + .collect(); + ActionResult::from_execution(&Action::Click, steps, None).expect("execution result") +} + +#[test] +fn delivered_unverified_constructor_is_explicit() { + assert_eq!( + ActionResult::delivered_unverified("click").disposition(), + DeliverySemantics::delivered_unverified() + ); +} + +#[test] +fn satisfied_without_delivery_is_successful_and_safe_to_repeat() { + let result = ActionResult::satisfied_without_delivery("check").with_steps(vec![ + ActionStep::skipped("AlreadyInState").with_verified(true), + ]); + + assert_eq!(result.disposition(), DeliverySemantics::not_delivered()); + let json = serde_json::to_value(result).unwrap(); + assert_eq!(json["disposition"]["delivery"], "not_delivered"); + assert_eq!(json["disposition"]["retry"], "safe"); +} + +#[test] +fn verification_cannot_turn_a_no_op_into_claimed_delivery() { + let result = ActionResult::satisfied_without_delivery("check").with_verified_delivery(); + + assert_eq!(result.disposition(), DeliverySemantics::not_delivered()); +} + +#[test] +fn verified_action_uses_verified_delivery() { + assert_eq!( + ActionResult::delivered_unverified("click") + .with_verified_delivery() + .disposition(), + DeliverySemantics::delivered_verified() + ); +} + +#[test] +fn legacy_json_defaults_to_delivered_unverified() { + let result: ActionResult = serde_json::from_value(serde_json::json!({ + "action": "click" + })) + .expect("legacy action result"); + + assert_eq!( + result.disposition(), + DeliverySemantics::delivered_unverified() + ); +} + +#[test] +fn successful_action_rejects_unknown_and_uncertain_dispositions() { + for delivery in ["unknown", "delivery_uncertain"] { + let retry = if delivery == "unknown" { + "unknown" + } else { + "unsafe" + }; + let value = serde_json::json!({ + "action": "click", + "disposition": { + "delivery": delivery, + "retry": retry + } + }); + + assert!(serde_json::from_value::(value).is_err()); + } +} + +#[test] +fn successful_action_deserializes_satisfied_without_delivery() { + let value = serde_json::json!({ + "action": "check", + "disposition": { + "delivery": "not_delivered", + "retry": "safe" + } + }); + + let result: ActionResult = serde_json::from_value(value).unwrap(); + assert_eq!(result.disposition(), DeliverySemantics::not_delivered()); +} + +#[test] +fn execution_without_succeeded_steps_is_satisfied_without_delivery() { + let steps = vec![ActionStep::skipped("AlreadyInState").with_verified(true)]; + let result = ActionResult::from_execution(&Action::Check, steps, Some(state(Some("1")))) + .expect("no-op result"); + + assert_eq!(result.disposition(), DeliverySemantics::not_delivered()); + assert_eq!(result.steps.len(), 1); + assert!(result.post_state.is_none()); +} + +#[test] +fn execution_derives_disposition_from_succeeded_step_evidence() { + for (verifications, expected) in [ + (vec![Some(true)], DeliverySemantics::delivered_verified()), + (vec![Some(false)], DeliverySemantics::delivered_unverified()), + ( + vec![Some(true), None], + DeliverySemantics::delivered_verified(), + ), + (vec![None], DeliverySemantics::delivered_unverified()), + ( + vec![Some(true), Some(false)], + DeliverySemantics::delivered_unverified(), + ), + ] { + assert_eq!(execution_result(&verifications).disposition(), expected); + } +} + +#[test] +fn execution_attaches_post_state_without_changing_serialization() { + let steps = vec![ActionStep::succeeded("AXValue").with_verified(false)]; + let post_state = state(Some("done")); + let actual = ActionResult::from_execution( + &Action::SetValue("done".into()), + steps.clone(), + Some(post_state.clone()), + ) + .expect("result"); + let expected = ActionResult::delivered_unverified("set-value") + .with_steps(steps) + .with_state(post_state); + + assert_eq!( + serde_json::to_value(actual).unwrap(), + serde_json::to_value(expected).unwrap() + ); +} + +#[test] +fn clear_postcondition_preserves_exact_error_contract() { + for post_state in [state(Some("")), state(None)] { + ActionResult::from_execution( + &Action::Clear, + vec![ActionStep::succeeded("AXValue").with_verified(true)], + Some(post_state), + ) + .expect("clear result"); + } + + let error = ActionResult::from_execution( + &Action::Clear, + vec![ActionStep::succeeded("AXValue").with_verified(true)], + Some(state(Some("still here"))), + ) + .expect_err("non-empty clear must fail"); + + assert_eq!(error.code, ErrorCode::ActionFailed); + assert_eq!( + error.message, + "Clear reported success but element value is still non-empty" + ); + assert_eq!( + error.suggestion.as_deref(), + Some("Retry 'clear', or use 'press cmd+a' then 'press delete'.") + ); + assert_eq!(error.disposition, DeliverySemantics::delivered_unverified()); +} diff --git a/crates/core/src/action_step.rs b/crates/core/src/action_step.rs index f2c2249..6cede83 100644 --- a/crates/core/src/action_step.rs +++ b/crates/core/src/action_step.rs @@ -1,10 +1,15 @@ use crate::action_step_outcome::ActionStepOutcome; +use crate::step_mechanism::StepMechanism; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ActionStep { label: String, pub outcome: ActionStepOutcome, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub mechanism: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub verified: Option, } impl ActionStep { @@ -12,6 +17,8 @@ impl ActionStep { Self { label: label.to_string(), outcome: ActionStepOutcome::Attempted, + mechanism: None, + verified: None, } } @@ -19,6 +26,8 @@ impl ActionStep { Self { label: label.to_string(), outcome: ActionStepOutcome::Skipped, + mechanism: None, + verified: None, } } @@ -26,10 +35,34 @@ impl ActionStep { Self { label: label.to_string(), outcome: ActionStepOutcome::Succeeded, + mechanism: None, + verified: None, } } pub fn label(&self) -> &str { &self.label } + + pub fn mechanism(&self) -> Option { + self.mechanism + } + + pub fn verified(&self) -> Option { + self.verified + } + + pub fn with_mechanism(mut self, mechanism: StepMechanism) -> Self { + self.mechanism = Some(mechanism); + self + } + + pub fn with_verified(mut self, verified: bool) -> Self { + self.verified = Some(verified); + self + } } + +#[cfg(test)] +#[path = "action_step_tests.rs"] +mod tests; diff --git a/crates/core/src/action_step_tests.rs b/crates/core/src/action_step_tests.rs new file mode 100644 index 0000000..e3e843d --- /dev/null +++ b/crates/core/src/action_step_tests.rs @@ -0,0 +1,48 @@ +use super::ActionStep; +use crate::step_mechanism::StepMechanism; +use crate::trace_sanitize::sanitize_trace_value; +use serde_json::json; + +#[test] +fn legacy_action_step_json_round_trips_without_new_fields() { + let legacy = r#"{"label":"AXPress","outcome":"succeeded"}"#; + let step: ActionStep = serde_json::from_str(legacy).unwrap(); + assert_eq!(step.label(), "AXPress"); + assert!(step.mechanism().is_none()); + assert!(step.verified().is_none()); + let round_trip = serde_json::to_value(&step).unwrap(); + let legacy_value: serde_json::Value = serde_json::from_str(legacy).unwrap(); + assert_eq!(round_trip, legacy_value); +} + +#[test] +fn action_step_serializes_mechanism_and_verified() { + let step = ActionStep::succeeded("verified_press") + .with_mechanism(StepMechanism::SemanticApi) + .with_verified(true); + let value = serde_json::to_value(&step).unwrap(); + assert_eq!(value["mechanism"], "semantic_api"); + assert_eq!(value["verified"], true); +} + +#[test] +fn action_step_omits_absent_mechanism_and_verified() { + let step = ActionStep::skipped("AXConfirm"); + let value = serde_json::to_value(&step).unwrap(); + assert!(value.get("mechanism").is_none()); + assert!(value.get("verified").is_none()); +} + +#[test] +fn trace_preserves_mechanism_and_verified() { + let value = sanitize_trace_value(json!({ + "steps": [{ + "label": "AXPress", + "outcome": "succeeded", + "mechanism": "semantic_api", + "verified": true + }] + })); + assert_eq!(value["steps"][0]["mechanism"], "semantic_api"); + assert_eq!(value["steps"][0]["verified"], true); +} diff --git a/crates/core/src/action_tests.rs b/crates/core/src/action_tests.rs index 960b635..bbc6dc6 100644 --- a/crates/core/src/action_tests.rs +++ b/crates/core/src/action_tests.rs @@ -1,5 +1,8 @@ -use super::*; -use crate::interaction_policy::InteractionPolicy; +use super::Action; +use crate::{ + Direction, DragParams, KeyCombo, Modifier, MouseButton, MouseEvent, MouseEventKind, Point, + interaction_policy::InteractionPolicy, +}; fn dummy_key() -> KeyCombo { KeyCombo { @@ -26,7 +29,7 @@ fn action_names_do_not_include_payloads() { ( Action::PressKey(KeyCombo { key: "A".into(), - modifiers: vec![Modifier::Cmd], + modifiers: vec![Modifier::Meta], }), "press", ), @@ -37,6 +40,47 @@ fn action_names_do_not_include_payloads() { } } +#[test] +fn headed_requirements_distinguish_window_and_pointer_preconditions() { + for action in [ + Action::Click, + Action::DoubleClick, + Action::RightClick, + Action::TripleClick, + Action::Scroll(Direction::Down, 1), + Action::Hover, + Action::Drag(dummy_drag()), + ] { + assert_eq!( + action.headed_requirement(), + crate::HeadedRequirement::FocusedWindowAndCursor, + "{} must require focus plus cursor delivery", + action.name() + ); + } + for action in [ + Action::SetValue("v".into()), + Action::SetFocus, + Action::Expand, + Action::Collapse, + Action::Select("v".into()), + Action::Toggle, + Action::Check, + Action::Uncheck, + Action::ScrollTo, + Action::PressKey(dummy_key()), + Action::TypeText("v".into()), + Action::Clear, + ] { + assert_eq!( + action.headed_requirement(), + crate::HeadedRequirement::FocusedWindow, + "{} must require focus without pointer movement", + action.name() + ); + } +} + #[test] fn pure_ax_actions_base_policy_is_headless() { let headless = InteractionPolicy::headless(); @@ -68,21 +112,21 @@ fn pure_ax_actions_base_policy_is_headless() { } #[test] -fn press_key_and_type_text_base_policy_is_focus_fallback() { - let focus = InteractionPolicy::focus_fallback(); +fn type_text_is_headless_but_explicit_press_allows_focus() { + let headless = InteractionPolicy::headless(); assert_eq!( Action::PressKey(KeyCombo { key: "a".into(), - modifiers: vec![Modifier::Cmd], + modifiers: vec![Modifier::Meta], }) .base_interaction_policy(), - focus, - "PressKey must request focus_fallback to land in the right field" + InteractionPolicy::focus_fallback(), + "PressKey is an explicit physical-input command" ); assert_eq!( Action::TypeText("hello".into()).base_interaction_policy(), - focus, - "TypeText must request focus_fallback" + headless, + "TypeText must not gain implicit focus or keyboard permission" ); } @@ -123,3 +167,155 @@ fn hover_and_drag_base_policy_is_headless_independent_of_cursor_requirement() { "Drag.requires_cursor_policy() must still be true" ); } + +#[test] +fn requires_hit_test_covers_ref_targeted_pointer_actions() { + let hit_tested: &[Action] = &[ + Action::Click, + Action::DoubleClick, + Action::RightClick, + Action::TripleClick, + Action::Hover, + Action::Drag(dummy_drag()), + ]; + for action in hit_tested { + assert!( + action.requires_hit_test(), + "{} must require a hit test before dispatch", + action.name() + ); + } + + let not_hit_tested: &[Action] = &[ + Action::SetValue("v".into()), + Action::SetFocus, + Action::Expand, + Action::Collapse, + Action::Select("s".into()), + Action::Toggle, + Action::Check, + Action::Uncheck, + Action::Scroll(Direction::Down, 1), + Action::ScrollTo, + Action::PressKey(dummy_key()), + Action::KeyDown(dummy_key()), + Action::KeyUp(dummy_key()), + Action::TypeText("t".into()), + Action::Clear, + ]; + for action in not_hit_tested { + assert!( + !action.requires_hit_test(), + "{} must not require a hit test", + action.name() + ); + } +} + +#[test] +fn requires_scroll_into_view_covers_all_actions() { + let scroll_into_view: &[Action] = &[ + Action::Click, + Action::DoubleClick, + Action::RightClick, + Action::TripleClick, + Action::SetValue("v".into()), + Action::Expand, + Action::Collapse, + Action::Select("s".into()), + Action::Toggle, + Action::Check, + Action::Uncheck, + Action::TypeText("t".into()), + Action::Clear, + Action::Hover, + Action::Drag(dummy_drag()), + ]; + for action in scroll_into_view { + assert!( + action.requires_scroll_into_view(), + "{} must require scroll-into-view before dispatch", + action.name() + ); + } + + let not_scroll_into_view: &[Action] = &[ + Action::Scroll(Direction::Down, 1), + Action::ScrollTo, + Action::SetFocus, + Action::PressKey(dummy_key()), + Action::KeyDown(dummy_key()), + Action::KeyUp(dummy_key()), + ]; + for action in not_scroll_into_view { + assert!( + !action.requires_scroll_into_view(), + "{} must not require scroll-into-view", + action.name() + ); + } +} + +/// F10 regression coverage: `MouseEvent.modifiers` must stay `#[serde(default)]` +/// so a legacy payload recorded before modifiers existed (or any FFI/batch +/// caller that omits the key) still deserializes instead of erroring out. +#[test] +fn mouse_event_json_without_modifiers_key_deserializes_to_empty() { + let event: MouseEvent = serde_json::from_value(serde_json::json!({ + "kind": { "Click": { "count": 1 } }, + "point": { "x": 1.0, "y": 2.0 }, + "button": "Left", + })) + .unwrap(); + + assert!(event.modifiers.is_empty()); +} + +#[test] +fn mouse_event_json_with_modifiers_round_trips() { + let event = MouseEvent { + kind: MouseEventKind::Click { count: 2 }, + point: Point { x: 1.0, y: 2.0 }, + button: MouseButton::Left, + modifiers: vec![Modifier::Meta, Modifier::Shift], + }; + + let json = serde_json::to_value(&event).unwrap(); + let round_tripped: MouseEvent = serde_json::from_value(json).unwrap(); + + assert_eq!( + round_tripped.modifiers, + vec![Modifier::Meta, Modifier::Shift] + ); +} + +#[test] +fn modifier_meta_serializes_semantically_and_accepts_legacy_cmd() { + assert_eq!(serde_json::to_string(&Modifier::Meta).unwrap(), "\"Meta\""); + assert_eq!( + serde_json::from_str::("\"Cmd\"").unwrap(), + Modifier::Meta + ); +} + +#[test] +fn wheel_event_round_trips_platform_neutral_line_deltas() { + let event = MouseEvent { + kind: MouseEventKind::Wheel { + delta_x: -2.0, + delta_y: 3.0, + }, + point: Point { x: 50.0, y: 60.0 }, + button: MouseButton::Left, + modifiers: vec![Modifier::Shift], + }; + let json = serde_json::to_value(&event).unwrap(); + let decoded: MouseEvent = serde_json::from_value(json).unwrap(); + assert!(matches!( + decoded.kind, + MouseEventKind::Wheel { + delta_x: -2.0, + delta_y: 3.0 + } + )); +} diff --git a/crates/core/src/actionability/check.rs b/crates/core/src/actionability/check.rs index 42b0c50..fe0b0e0 100644 --- a/crates/core/src/actionability/check.rs +++ b/crates/core/src/actionability/check.rs @@ -1,10 +1,19 @@ -use super::ActionabilityStatus; +use super::{occluder::Occluder, status::ActionabilityStatus}; +use crate::ErrorCode; use serde::Serialize; -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct ActionabilityCheck { - pub name: &'static str, - pub status: ActionabilityStatus, +#[derive(Debug, Clone, Serialize, PartialEq)] +pub(crate) struct ActionabilityCheck { + pub(crate) check: &'static str, + pub(crate) status: ActionabilityStatus, #[serde(skip_serializing_if = "Option::is_none")] - pub reason: Option, + pub(crate) reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) occluder: Option, + #[serde(skip)] + pub(crate) terminal_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) hit_test: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) stability: Option, } diff --git a/crates/core/src/actionability/check_result.rs b/crates/core/src/actionability/check_result.rs new file mode 100644 index 0000000..1192e9a --- /dev/null +++ b/crates/core/src/actionability/check_result.rs @@ -0,0 +1,74 @@ +use super::{check::ActionabilityCheck, occluder::Occluder, status::ActionabilityStatus}; +use crate::{ErrorCode, Rect}; + +pub(super) fn pass(check: &'static str) -> ActionabilityCheck { + ActionabilityCheck { + check, + status: ActionabilityStatus::Pass, + reason: None, + occluder: None, + terminal_code: None, + hit_test: None, + stability: None, + } +} + +pub(super) fn fail(check: &'static str, reason: impl Into) -> ActionabilityCheck { + ActionabilityCheck { + check, + status: ActionabilityStatus::Fail, + reason: Some(reason.into()), + occluder: None, + terminal_code: None, + hit_test: None, + stability: None, + } +} + +pub(super) fn fail_terminal( + check: &'static str, + reason: impl Into, + code: ErrorCode, +) -> ActionabilityCheck { + ActionabilityCheck { + check, + status: ActionabilityStatus::Fail, + reason: Some(reason.into()), + occluder: None, + terminal_code: Some(code), + hit_test: None, + stability: None, + } +} + +pub(super) fn unknown(check: &'static str, reason: impl Into) -> ActionabilityCheck { + ActionabilityCheck { + check, + status: ActionabilityStatus::Unknown, + reason: Some(reason.into()), + occluder: None, + terminal_code: None, + hit_test: None, + stability: None, + } +} + +pub(super) fn occluded( + role: Option, + name: Option, + bounds: Option, +) -> ActionabilityCheck { + let reason = role.as_deref().map_or_else( + || "occluded by another element".to_string(), + |role| format!("occluded by {role}"), + ); + ActionabilityCheck { + check: "receives_events", + status: ActionabilityStatus::Fail, + reason: Some(reason), + occluder: Some(Occluder { role, name, bounds }), + terminal_code: None, + hit_test: None, + stability: None, + } +} diff --git a/crates/core/src/actionability/evaluate.rs b/crates/core/src/actionability/evaluate.rs new file mode 100644 index 0000000..c197523 --- /dev/null +++ b/crates/core/src/actionability/evaluate.rs @@ -0,0 +1,159 @@ +use super::{ + evidence::ActionabilityEvidence, gates, receives_events::receives_events_check, + report::ActionabilityReport, requirements::ActionabilityRequirements, + stability::StabilityExpectation, +}; +use crate::{ + AdapterError, ErrorCode, + action_request::ActionRequest, + adapter::{NativeHandle, PlatformAdapter}, +}; +use serde_json::json; + +#[cfg(test)] +use crate::refs::RefEntry; + +#[cfg(test)] +pub(crate) fn check( + entry: &RefEntry, + request: &ActionRequest, +) -> Result { + use crate::{ + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, + hit_test::HitTestResult, + }; + + struct TestHitAdapter; + + impl ObservationOps for TestHitAdapter { + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } + } + + impl ActionOps for TestHitAdapter {} + impl InputOps for TestHitAdapter {} + impl SystemOps for TestHitAdapter {} + + let evidence = ActionabilityEvidence { + state: crate::ElementState { + role: entry.identity.role.clone(), + states: entry.capabilities.states.clone(), + value: entry.identity.value.clone(), + enabled: Some(!crate::state::has_state( + &entry.capabilities.states, + crate::state::DISABLED, + )), + hidden: Some(crate::state::has_state( + &entry.capabilities.states, + crate::state::HIDDEN, + )), + offscreen: Some(crate::state::has_state( + &entry.capabilities.states, + crate::state::OFFSCREEN, + )), + }, + states_complete: true, + bounds: entry.geometry.bounds, + available_actions: entry.capabilities.available_actions.clone(), + }; + let handle = NativeHandle::null(); + let adapter = TestHitAdapter; + check_with_stability( + StabilityExpectation::permissive(entry.geometry.bounds_hash), + &evidence, + request, + Some((&handle, &adapter)), + crate::Deadline::standard()?, + ) +} + +pub(super) fn check_with_stability( + stability: StabilityExpectation, + evidence: &ActionabilityEvidence, + request: &ActionRequest, + hit_test: Option<(&NativeHandle, &dyn PlatformAdapter)>, + deadline: crate::Deadline, +) -> Result { + let requirements = ActionabilityRequirements::for_action(&request.action); + let pointer_delivery = + requirements.pointer_delivery(&request.action, &evidence.available_actions, request.policy); + let mut checks = Vec::new(); + let mut verified_point = None; + if requirements.visible { + checks.push(gates::visibility(evidence)); + } + if requirements.requires_stability(pointer_delivery) { + checks.push(gates::stability(stability, evidence.bounds)); + } + if requirements.enabled { + checks.push(gates::enabled(evidence)); + } + if requirements.editable { + checks.push(gates::editable(evidence, &request.action)); + } + checks.push(gates::action_supported(evidence, request)); + checks.push(gates::policy(request)); + if checks + .iter() + .any(|check| !matches!(check.status, super::status::ActionabilityStatus::Pass)) + { + return finish( + evidence, + ActionabilityReport::from_checks(checks, verified_point, pointer_delivery), + ); + } + if matches!(pointer_delivery, super::PointerDelivery::Physical) { + match hit_test { + Some((handle, adapter)) => { + let (check, point) = + receives_events_check(evidence.bounds, handle, adapter, request, deadline)?; + checks.push(check); + verified_point = point; + } + None => checks.push(super::check_result::unknown( + "receives_events", + "live hit-test context unavailable", + )), + } + } + finish( + evidence, + ActionabilityReport::from_checks(checks, verified_point, pointer_delivery), + ) +} + +fn finish( + evidence: &ActionabilityEvidence, + report: ActionabilityReport, +) -> Result { + if report.actionable { + return Ok(report); + } + let code = report.terminal_code().unwrap_or(ErrorCode::ActionFailed); + let suggestion = if code == ErrorCode::ActionFailed { + "Wait for the target to become actionable, refresh the snapshot, or use an explicit physical/focus command if intended." + } else { + "Waiting will not help: this element cannot satisfy the action as targeted. Target an element that advertises the action (check available_actions in a fresh snapshot) or adjust the interaction policy (e.g. pass --headed)." + }; + let mut details = json!(report); + if let Some(object) = details.as_object_mut() + && let Some(bounds) = evidence.bounds + { + if let Some(bounds_hash) = bounds.bounds_hash() { + object.insert("observed_bounds_hash".into(), json!(bounds_hash)); + } + } + Err(AdapterError::new( + code, + format!("Target is not actionable: {}", report.failure_reasons()), + ) + .with_details(details) + .with_suggestion(suggestion) + .with_disposition(crate::DeliverySemantics::not_delivered())) +} diff --git a/crates/core/src/actionability/evidence.rs b/crates/core/src/actionability/evidence.rs new file mode 100644 index 0000000..9ba0b0f --- /dev/null +++ b/crates/core/src/actionability/evidence.rs @@ -0,0 +1,8 @@ +use crate::{Rect, element_state::ElementState}; + +pub(super) struct ActionabilityEvidence { + pub(super) state: ElementState, + pub(super) states_complete: bool, + pub(super) bounds: Option, + pub(super) available_actions: Vec, +} diff --git a/crates/core/src/actionability/gates.rs b/crates/core/src/actionability/gates.rs new file mode 100644 index 0000000..85a0494 --- /dev/null +++ b/crates/core/src/actionability/gates.rs @@ -0,0 +1,193 @@ +use super::{ + check::ActionabilityCheck, + check_result::{fail, fail_terminal, pass, unknown}, + evidence::ActionabilityEvidence, +}; +use crate::{ErrorCode, Rect, action::Action, action_request::ActionRequest, capability}; + +pub(super) fn visibility(evidence: &ActionabilityEvidence) -> ActionabilityCheck { + match evidence.state.hidden { + Some(true) => return fail("visible", "live hidden state is true"), + None if !evidence.states_complete => { + return unknown("visible", "live hidden state unavailable"); + } + None if crate::state::has_state(&evidence.state.states, crate::state::HIDDEN) => { + return fail("visible", "canonical hidden state is present"); + } + None => {} + Some(false) => {} + } + match evidence.state.offscreen { + Some(true) => return fail("visible", "live offscreen state is true"), + None => return unknown("visible", "live offscreen state unavailable"), + Some(false) => {} + } + let Some(bounds) = evidence.bounds else { + return unknown("visible", "bounds unavailable"); + }; + if !bounds_are_visible(Some(bounds)) { + return fail("visible", "bounds are zero-sized"); + } + pass("visible") +} + +pub(super) fn stability( + expectation: super::stability::StabilityExpectation, + bounds: Option, +) -> ActionabilityCheck { + let evidence = super::stability_evidence::StabilityEvidence { + samples: expectation.samples, + span_ms: expectation.span_ms, + }; + let Some(bounds) = bounds else { + let mut check = unknown("stable", "live bounds unavailable"); + check.stability = Some(evidence); + return check; + }; + let check = if expectation.strict { + if let Some(expected) = expectation.bounds { + if super::stability_sampler::geometry_matches(expected, bounds) { + pass("stable") + } else { + fail("stable", "bounds changed after the stability window") + } + } else if let Some(expected_hash) = expectation.bounds_hash { + if bounds.bounds_hash() == Some(expected_hash) { + pass("stable") + } else { + fail("stable", "bounds changed since the previous observation") + } + } else { + fail("stable", "waiting for stable live bounds observations") + } + } else { + let Some(expected) = expectation.bounds_hash else { + let mut check = unknown("stable", "snapshot bounds hash unavailable"); + check.stability = Some(evidence); + return check; + }; + let Some(observed_hash) = bounds.bounds_hash() else { + let mut check = unknown("stable", "live bounds are invalid"); + check.stability = Some(evidence); + return check; + }; + if observed_hash == expected { + pass("stable") + } else { + unknown("stable", "bounds changed since snapshot") + } + }; + let mut check = check; + check.stability = Some(evidence); + check +} + +pub(super) fn enabled(evidence: &ActionabilityEvidence) -> ActionabilityCheck { + match evidence.state.enabled { + Some(true) => pass("enabled"), + Some(false) => fail("enabled", "live enabled state is false"), + None => unknown("enabled", "live enabled state unavailable"), + } +} + +pub(super) fn action_supported( + evidence: &ActionabilityEvidence, + request: &ActionRequest, +) -> ActionabilityCheck { + if request.action.requires_cursor_policy() { + return pass("supported_action"); + } + if matches!(request.action, Action::ScrollTo) { + return pass("supported_action"); + } + if matches!(request.action, Action::Click | Action::RightClick) { + if capability::supports_direct_semantic_pointer_delivery( + &request.action, + &evidence.available_actions, + ) { + return pass("supported_action"); + } + if request.policy.allow_cursor_move && request.policy.allow_focus_steal { + return pass("supported_action"); + } + return fail_terminal( + "supported_action", + "direct semantic delivery is unavailable and physical fallback is denied", + ErrorCode::PolicyDenied, + ); + } + if matches!(request.action, Action::DoubleClick | Action::TripleClick) { + if request.policy.allow_cursor_move && request.policy.allow_focus_steal { + return pass("supported_action"); + } + return fail_terminal( + "supported_action", + "semantic gesture is unavailable and physical fallback is denied", + ErrorCode::PolicyDenied, + ); + } + if capability::contains_any( + &evidence.available_actions, + capability::for_action(&request.action), + ) { + return pass("supported_action"); + } + if request.action.may_use_focus_fallback() && request.policy.allow_focus_steal { + return pass("supported_action"); + } + if request.action.may_use_focus_fallback() { + return fail_terminal( + "supported_action", + "semantic action is unavailable and focus fallback is denied", + ErrorCode::PolicyDenied, + ); + } + let expected = capability::for_action(&request.action).join(" or "); + fail_terminal( + "supported_action", + format!("{expected} is not available"), + ErrorCode::ActionNotSupported, + ) +} + +pub(super) fn policy(request: &ActionRequest) -> ActionabilityCheck { + if request.action.requires_cursor_policy() && !request.policy.allow_cursor_move { + return fail_terminal( + "policy", + "action requires cursor movement but policy denies it", + ErrorCode::PolicyDenied, + ); + } + pass("policy") +} + +pub(super) fn editable(evidence: &ActionabilityEvidence, action: &Action) -> ActionabilityCheck { + if !matches!( + action, + Action::SetValue(_) | Action::TypeText(_) | Action::Clear + ) { + return pass("editable"); + } + if evidence.state.role == "textfield" || evidence.state.role == "combobox" { + return pass("editable"); + } + if capability::contains(&evidence.available_actions, capability::SET_VALUE) { + return pass("editable"); + } + fail_terminal( + "editable", + format!("role {} is not editable", evidence.state.role), + ErrorCode::ActionNotSupported, + ) +} + +#[cfg(test)] +pub(crate) fn states_are_enabled(states: &[String]) -> bool { + !crate::state::has_state(states, crate::state::DISABLED) +} + +pub(crate) fn bounds_are_visible(bounds: Option) -> bool { + bounds.is_some_and(|bounds| { + bounds.validate().is_ok() && bounds.width > 0.0 && bounds.height > 0.0 + }) +} diff --git a/crates/core/src/actionability/hit_test_evidence.rs b/crates/core/src/actionability/hit_test_evidence.rs new file mode 100644 index 0000000..436689e --- /dev/null +++ b/crates/core/src/actionability/hit_test_evidence.rs @@ -0,0 +1,8 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub(crate) struct HitTestEvidence { + pub(crate) attempted: u8, + pub(crate) unknown: u8, + pub(crate) occluded: u8, +} diff --git a/crates/core/src/actionability/live.rs b/crates/core/src/actionability/live.rs new file mode 100644 index 0000000..fc46afe --- /dev/null +++ b/crates/core/src/actionability/live.rs @@ -0,0 +1,143 @@ +use super::{ + evaluate::check_with_stability, evidence::ActionabilityEvidence, report::ActionabilityReport, + stability::StabilityExpectation, +}; +use crate::{ + AdapterError, DeliverySemantics, ErrorCode, IdentityMatch, LocatorField, + action_request::ActionRequest, + adapter::{LiveElement, NativeHandle, PlatformAdapter}, + ref_identity::{has_meaningful_identity, identity_match}, + refs::RefEntry, +}; + +/// Groups the read-only coordinates of a live element check: which element, +/// which handle to read it through, which adapter to read with, and the +/// deadline that bounds the read. Kept separate from the `request` (what +/// action to check against) and `stability` (how settled the element must +/// be) parameters, which vary independently per call. +pub(crate) struct LiveCheckTarget<'a> { + pub(crate) entry: &'a RefEntry, + pub(crate) handle: &'a NativeHandle, + pub(crate) adapter: &'a dyn PlatformAdapter, + pub(crate) deadline: crate::Deadline, +} + +#[cfg(test)] +pub(crate) fn check_live( + entry: &RefEntry, + handle: &NativeHandle, + adapter: &dyn PlatformAdapter, + request: &ActionRequest, +) -> Result { + let deadline = crate::Deadline::standard()?; + check_live_with_stability( + &LiveCheckTarget { + entry, + handle, + adapter, + deadline, + }, + request, + StabilityExpectation::permissive(entry.geometry.bounds_hash), + ) +} + +pub(crate) fn check_live_with_stability( + target: &LiveCheckTarget<'_>, + request: &ActionRequest, + stability: StabilityExpectation, +) -> Result { + let evidence = observe( + target.entry, + target + .adapter + .get_live_element(target.handle, target.deadline), + )?; + check_with_stability( + stability, + &evidence, + request, + Some((target.handle, target.adapter)), + target.deadline, + ) +} + +fn observe( + entry: &RefEntry, + live: Result, +) -> Result { + let live = match live { + Ok(live) => live, + Err(err) + if matches!( + err.code, + ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported + ) => + { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "Live element evidence is required for actionability checks", + ) + .with_details(serde_json::json!({ "source_code": err.code.as_str() })) + .with_disposition(DeliverySemantics::not_delivered())); + } + Err(err) => return Err(err), + }; + validate_live_identity(entry, &live)?; + Ok(ActionabilityEvidence { + state: live.state, + states_complete: live.states_complete, + bounds: live.bounds, + available_actions: live.available_actions, + }) +} + +fn validate_live_identity(entry: &RefEntry, live: &LiveElement) -> Result<(), AdapterError> { + if live.state.role == "unknown" || live.state.role != entry.identity.role { + return Err(stale_live_element( + "Resolved element changed role before dispatch", + )); + } + if !has_meaningful_identity(entry) { + return Ok(()); + } + let value = live + .state + .value + .clone() + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent); + match identity_match( + entry, + &live.identity.name, + &value, + &live.identity.description, + &live.identity.identifiers, + ) { + IdentityMatch::Match => Ok(()), + IdentityMatch::NoMatch => Err(stale_live_element( + "Resolved element changed identity before dispatch", + )), + IdentityMatch::Unknown => Err(AdapterError::app_unresponsive( + entry + .source + .source_app + .as_deref() + .unwrap_or("target application"), + ) + .with_details(serde_json::json!({ + "kind": "live_identity_evidence", + "complete": false, + "retryable": true, + }))), + } +} + +fn stale_live_element(message: &'static str) -> AdapterError { + AdapterError::new(ErrorCode::StaleRef, message) + .with_suggestion( + "Re-run a snapshot to obtain fresh refs, then retry with the new ref (CLI: snapshot; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)", + ) + .with_details(serde_json::json!({ "retryable": true })) + .with_disposition(DeliverySemantics::not_delivered()) +} diff --git a/crates/core/src/actionability/mod.rs b/crates/core/src/actionability/mod.rs index 414e952..c0d9f66 100644 --- a/crates/core/src/actionability/mod.rs +++ b/crates/core/src/actionability/mod.rs @@ -1,239 +1,36 @@ -use crate::capability; -use crate::{ - action::Action, - action_request::ActionRequest, - adapter::{NativeHandle, PlatformAdapter}, - error::{AdapterError, ErrorCode}, - node::Rect, - refs::RefEntry, -}; -use serde_json::json; - mod check; +mod check_result; +mod evaluate; +mod evidence; +mod gates; +mod hit_test_evidence; +mod live; +mod occluder; +mod pointer_delivery; +mod receives_events; mod report; +mod requirements; +mod stability; +mod stability_evidence; +pub(crate) mod stability_sampler; mod status; -pub use check::ActionabilityCheck; -pub use report::ActionabilityReport; -pub use status::ActionabilityStatus; - #[cfg(test)] -pub fn check( - entry: &RefEntry, - request: &ActionRequest, -) -> Result { - check_with_stability(entry.bounds_hash, entry, request) -} - -pub fn check_live( - entry: &RefEntry, - handle: &NativeHandle, - adapter: &dyn PlatformAdapter, - request: &ActionRequest, -) -> Result { - let mut observed = entry.clone(); - let live = match adapter.get_live_element(handle) { - Ok(live) => live, - Err(err) - if matches!( - err.code, - ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported - ) => - { - return check_with_stability(entry.bounds_hash, &observed, request); - } - Err(err) => return Err(err), - }; - if live_element_is_stale(&live) { - return Err(AdapterError::new( - ErrorCode::StaleRef, - "Resolved element no longer exposes live accessibility state", - ) - .with_suggestion( - "Re-run a snapshot to obtain fresh refs, then retry with the new ref (CLI: snapshot; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)", - )); - } - if let Some(state) = live.state { - observed.role = state.role; - observed.states = state.states; - observed.value = state.value.or(observed.value); - } - observed.bounds = live.bounds; - if let Some(actions) = live.available_actions - && !actions.is_empty() - { - observed.available_actions = actions; - } - check_with_stability(entry.bounds_hash, &observed, request) -} - -fn live_element_is_stale(live: &crate::adapter::LiveElement) -> bool { - let role_unknown = live - .state - .as_ref() - .is_none_or(|state| state.role == "unknown"); - let actions_empty = live.available_actions.as_ref().is_none_or(Vec::is_empty); - role_unknown && live.bounds.is_none() && actions_empty -} - -fn check_with_stability( - expected_bounds_hash: Option, - entry: &RefEntry, - request: &ActionRequest, -) -> Result { - let checks = vec![ - visibility_check(entry), - stability_check(expected_bounds_hash, entry.bounds), - enabled_check(entry), - action_supported_check(entry, request), - policy_check(request), - editable_check(entry, &request.action), - ]; - - let actionable = checks - .iter() - .all(|check| !matches!(check.status, ActionabilityStatus::Fail)); - let report = ActionabilityReport { actionable, checks }; - if report.actionable { - return Ok(report); - } - Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("Target is not actionable: {}", failure_reasons(&report)), - ) - .with_details(json!(report)) - .with_suggestion( - "Wait for the target to become actionable, refresh the snapshot, or use an explicit physical/focus command if intended.", - )) -} - -fn visibility_check(entry: &RefEntry) -> ActionabilityCheck { - let Some(bounds) = entry.bounds else { - return unknown("visible", "bounds unavailable"); - }; - if !bounds_are_visible(Some(bounds)) { - return fail("visible", "bounds are zero-sized"); - } - pass("visible") -} - -fn stability_check(expected_bounds_hash: Option, bounds: Option) -> ActionabilityCheck { - let Some(expected) = expected_bounds_hash else { - return unknown("stable", "snapshot bounds hash unavailable"); - }; - let Some(bounds) = bounds else { - return unknown("stable", "live bounds unavailable"); - }; - if bounds.bounds_hash() != expected { - return unknown("stable", "bounds changed since snapshot"); - } - pass("stable") -} - -fn enabled_check(entry: &RefEntry) -> ActionabilityCheck { - if !states_are_enabled(&entry.states) { - return fail("enabled", "entry state contains disabled"); - } - pass("enabled") -} - -pub fn states_are_enabled(states: &[String]) -> bool { - !states.iter().any(|state| state == "disabled") -} - -pub fn bounds_are_visible(bounds: Option) -> bool { - bounds.is_some_and(|bounds| bounds.width > 0.0 && bounds.height > 0.0) -} - -fn action_supported_check(entry: &RefEntry, request: &ActionRequest) -> ActionabilityCheck { - if request.action.requires_cursor_policy() { - return pass("supported_action"); - } - if capability::contains_any( - &entry.available_actions, - capability::for_action(&request.action), - ) { - return pass("supported_action"); - } - if may_use_fallback(&request.action, request) { - return unknown( - "supported_action", - "semantic action unavailable but fallback policy allows attempt", - ); - } - let expected = capability::for_action(&request.action).join(" or "); - fail("supported_action", format!("{expected} is not available")) -} - -fn policy_check(request: &ActionRequest) -> ActionabilityCheck { - if request.action.requires_cursor_policy() && !request.policy.allow_cursor_move { - return fail( - "policy", - "action requires cursor movement but policy denies it", - ); - } - if request.action.may_use_focus_fallback() && !request.policy.allow_focus_steal { - return fail("policy", "action requires focus but policy denies it"); - } - pass("policy") -} - -fn editable_check(entry: &RefEntry, action: &Action) -> ActionabilityCheck { - if !matches!( - action, - Action::SetValue(_) | Action::TypeText(_) | Action::Clear - ) { - return pass("editable"); - } - if entry.role == "textfield" || entry.role == "combobox" { - return pass("editable"); - } - if capability::contains(&entry.available_actions, capability::SET_VALUE) { - return pass("editable"); - } - fail("editable", format!("role {} is not editable", entry.role)) -} - -fn failure_reasons(report: &ActionabilityReport) -> String { - report - .checks - .iter() - .filter(|check| matches!(check.status, ActionabilityStatus::Fail)) - .map(|check| { - let reason = check.reason.as_deref().unwrap_or("failed"); - format!("{} ({reason})", check.name) - }) - .collect::>() - .join(", ") -} - -fn may_use_fallback(action: &Action, request: &ActionRequest) -> bool { - action.may_use_focus_fallback() && request.policy.allow_focus_steal -} - -fn pass(name: &'static str) -> ActionabilityCheck { - ActionabilityCheck { - name, - status: ActionabilityStatus::Pass, - reason: None, - } -} - -fn fail(name: &'static str, reason: impl Into) -> ActionabilityCheck { - ActionabilityCheck { - name, - status: ActionabilityStatus::Fail, - reason: Some(reason.into()), - } -} - -fn unknown(name: &'static str, reason: impl Into) -> ActionabilityCheck { - ActionabilityCheck { - name, - status: ActionabilityStatus::Unknown, - reason: Some(reason.into()), - } -} +pub(crate) use check::ActionabilityCheck; +#[cfg(test)] +pub(crate) use evaluate::check; +pub(crate) use gates::bounds_are_visible; +#[cfg(test)] +pub(crate) use gates::states_are_enabled; +#[cfg(test)] +pub(crate) use live::check_live; +pub(crate) use live::{LiveCheckTarget, check_live_with_stability}; +pub(crate) use pointer_delivery::PointerDelivery; +pub(crate) use receives_events::require_receives_events; +pub(crate) use requirements::requires_stability; +pub(crate) use stability::StabilityExpectation; +#[cfg(test)] +pub(crate) use status::ActionabilityStatus; #[cfg(test)] #[path = "../actionability_tests.rs"] diff --git a/crates/core/src/actionability/occluder.rs b/crates/core/src/actionability/occluder.rs new file mode 100644 index 0000000..1a1aace --- /dev/null +++ b/crates/core/src/actionability/occluder.rs @@ -0,0 +1,12 @@ +use crate::Rect; +use serde::Serialize; + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub(crate) struct Occluder { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) bounds: Option, +} diff --git a/crates/core/src/actionability/pointer_delivery.rs b/crates/core/src/actionability/pointer_delivery.rs new file mode 100644 index 0000000..59ca39c --- /dev/null +++ b/crates/core/src/actionability/pointer_delivery.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PointerDelivery { + NotApplicable, + Semantic, + Physical, +} diff --git a/crates/core/src/actionability/receives_events.rs b/crates/core/src/actionability/receives_events.rs new file mode 100644 index 0000000..f31f6bd --- /dev/null +++ b/crates/core/src/actionability/receives_events.rs @@ -0,0 +1,114 @@ +use super::{ + PointerDelivery, + check::ActionabilityCheck, + check_result::{occluded, pass, unknown}, + hit_test_evidence::HitTestEvidence, + report::ActionabilityReport, +}; +use crate::{ + AdapterError, ErrorCode, Point, + action_request::ActionRequest, + adapter::{NativeHandle, PlatformAdapter}, + hit_test::HitTestResult, +}; +use serde_json::json; + +pub(super) fn receives_events_check( + bounds: Option, + handle: &NativeHandle, + adapter: &dyn PlatformAdapter, + request: &ActionRequest, + deadline: crate::Deadline, +) -> Result<(ActionabilityCheck, Option), AdapterError> { + if !request.action.requires_hit_test() { + return Ok((pass("receives_events"), None)); + } + let Some(bounds) = bounds else { + return Ok((unknown("receives_events", "bounds unavailable"), None)); + }; + let mut first_occlusion = None; + let mut evidence = HitTestEvidence { + attempted: 0, + unknown: 0, + occluded: 0, + }; + for point in candidate_points(bounds) { + evidence.attempted += 1; + match adapter.hit_test(handle, point.clone(), deadline) { + Ok(HitTestResult::ReachesTarget) => { + let mut check = pass("receives_events"); + check.hit_test = Some(evidence); + return Ok((check, Some(point))); + } + Ok(HitTestResult::InterceptedBy { role, name, bounds }) => { + evidence.occluded += 1; + first_occlusion.get_or_insert_with(|| occluded(role, name, bounds)); + } + Ok(HitTestResult::Unknown) => evidence.unknown += 1, + Err(error) if error.code == ErrorCode::PlatformNotSupported => { + evidence.unknown += 1; + let mut check = unknown("receives_events", "hit testing is not supported"); + check.hit_test = Some(evidence); + return Ok((check, None)); + } + Err(error) => return Err(error), + } + } + let mut check = if evidence.unknown > 0 && evidence.occluded > 0 { + let mut mixed = first_occlusion.unwrap_or_else(|| { + unknown( + "receives_events", + "hit test evidence mixed unknown and occluded outcomes", + ) + }); + mixed.status = super::status::ActionabilityStatus::Unknown; + mixed.reason = Some("hit test evidence mixed unknown and occluded outcomes".into()); + mixed + } else if evidence.unknown > 0 { + unknown("receives_events", "hit test result inconclusive") + } else { + first_occlusion + .unwrap_or_else(|| unknown("receives_events", "hit test result inconclusive")) + }; + check.hit_test = Some(evidence); + Ok((check, None)) +} + +fn candidate_points(bounds: crate::Rect) -> [Point; 5] { + let point = |x: f64, y: f64| Point { + x: bounds.x + bounds.width * x, + y: bounds.y + bounds.height * y, + }; + [ + point(0.5, 0.5), + point(0.25, 0.25), + point(0.75, 0.25), + point(0.25, 0.75), + point(0.75, 0.75), + ] +} + +pub(crate) fn require_receives_events( + handle: &NativeHandle, + point: Point, + adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, +) -> Result<(), AdapterError> { + let check = match adapter.hit_test(handle, point, deadline) { + Ok(HitTestResult::ReachesTarget) => return Ok(()), + Ok(HitTestResult::Unknown) => return Ok(()), + Err(error) if error.code == ErrorCode::PlatformNotSupported => return Ok(()), + Err(error) => return Err(error), + Ok(HitTestResult::InterceptedBy { role, name, bounds }) => occluded(role, name, bounds), + }; + let report = ActionabilityReport::from_checks(vec![check], None, PointerDelivery::Physical); + Err(AdapterError::new( + ErrorCode::ActionFailed, + format!("Target is not actionable: {}", report.failure_reasons()), + ) + .with_details(json!(report)) + .with_suggestion( + "Wait for the target to become actionable, refresh the snapshot, or use an explicit physical/focus command if intended.", + ) + .with_disposition(crate::DeliverySemantics::not_delivered())) +} diff --git a/crates/core/src/actionability/report.rs b/crates/core/src/actionability/report.rs index 2d0df40..2f3a595 100644 --- a/crates/core/src/actionability/report.rs +++ b/crates/core/src/actionability/report.rs @@ -1,8 +1,89 @@ -use super::ActionabilityCheck; +use super::{PointerDelivery, check::ActionabilityCheck, status::ActionabilityStatus}; +use crate::{ErrorCode, Point}; use serde::Serialize; -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct ActionabilityReport { - pub actionable: bool, - pub checks: Vec, +#[derive(Debug, Clone, Serialize, PartialEq)] +pub(crate) struct ActionabilityReport { + pub(crate) actionable: bool, + pub(crate) checks: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) verified_point: Option, + #[serde(skip)] + pub(crate) pointer_delivery: PointerDelivery, +} + +impl ActionabilityReport { + pub(crate) fn from_checks( + checks: Vec, + verified_point: Option, + pointer_delivery: PointerDelivery, + ) -> Self { + let actionable = checks.iter().all(|check| !is_blocking(check)); + Self { + actionable, + checks, + verified_point, + pointer_delivery, + } + } + + pub(crate) fn terminal_code(&self) -> Option { + self.checks + .iter() + .find(|check| is_blocking(check)) + .and_then(|check| check.terminal_code.clone()) + } + + pub(crate) fn failure_reasons(&self) -> String { + self.checks + .iter() + .filter(|check| is_blocking(check)) + .map(|check| { + let reason = check.reason.as_deref().unwrap_or("failed"); + format!("{} ({reason})", check.check) + }) + .collect::>() + .join(", ") + } +} + +fn is_blocking(check: &ActionabilityCheck) -> bool { + matches!(check.status, ActionabilityStatus::Fail) + || matches!(check.status, ActionabilityStatus::Unknown) + && (check.check != "receives_events" + || check + .hit_test + .as_ref() + .is_some_and(|evidence| evidence.occluded > 0)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn failed(check: &'static str, terminal_code: Option) -> ActionabilityCheck { + ActionabilityCheck { + check, + status: ActionabilityStatus::Fail, + reason: None, + occluder: None, + terminal_code, + hit_test: None, + stability: None, + } + } + + #[test] + fn retryable_failure_is_not_overridden_by_a_later_terminal_check() { + let report = ActionabilityReport::from_checks( + vec![ + failed("enabled", None), + failed("supported_action", Some(ErrorCode::PolicyDenied)), + ], + None, + PointerDelivery::NotApplicable, + ); + + assert_eq!(report.terminal_code(), None); + } } diff --git a/crates/core/src/actionability/requirements.rs b/crates/core/src/actionability/requirements.rs new file mode 100644 index 0000000..ef9a33a --- /dev/null +++ b/crates/core/src/actionability/requirements.rs @@ -0,0 +1,85 @@ +use super::PointerDelivery; +use crate::action::Action; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ActionabilityRequirements { + pub(crate) visible: bool, + pub(crate) stable: bool, + pub(crate) enabled: bool, + pub(crate) editable: bool, + pub(crate) receives_events: bool, +} + +impl ActionabilityRequirements { + pub(crate) fn for_action(action: &Action) -> Self { + match action { + Action::Click | Action::DoubleClick | Action::RightClick | Action::TripleClick => { + Self::new(true, true, true, false, true) + } + Action::Hover | Action::Drag(_) => Self::new(true, true, false, false, true), + Action::SetValue(_) | Action::TypeText(_) | Action::Clear => { + Self::new(true, false, true, true, false) + } + Action::Select(_) => Self::new(true, false, true, false, false), + Action::Expand + | Action::Collapse + | Action::Toggle + | Action::Check + | Action::Uncheck + | Action::Scroll(_, _) => Self::new(true, true, true, false, false), + Action::ScrollTo => Self::new(false, true, false, false, false), + Action::SetFocus | Action::PressKey(_) | Action::KeyDown(_) | Action::KeyUp(_) => { + Self::new(false, false, false, false, false) + } + } + } + + pub(crate) fn pointer_delivery( + &self, + action: &Action, + available_actions: &[String], + policy: crate::InteractionPolicy, + ) -> PointerDelivery { + if !self.receives_events { + return PointerDelivery::NotApplicable; + } + if policy.is_headed() && action.headed_requirement().requires_cursor() { + PointerDelivery::Physical + } else if crate::capability::supports_direct_semantic_pointer_delivery( + action, + available_actions, + ) { + PointerDelivery::Semantic + } else { + PointerDelivery::Physical + } + } + + pub(crate) fn requires_stability(&self, pointer_delivery: PointerDelivery) -> bool { + self.stable && !matches!(pointer_delivery, PointerDelivery::Semantic) + } + + const fn new( + visible: bool, + stable: bool, + enabled: bool, + editable: bool, + receives_events: bool, + ) -> Self { + Self { + visible, + stable, + enabled, + editable, + receives_events, + } + } +} + +pub(crate) fn requires_stability(action: &Action) -> bool { + ActionabilityRequirements::for_action(action).stable +} + +#[cfg(test)] +#[path = "requirements_tests.rs"] +mod tests; diff --git a/crates/core/src/actionability/requirements_tests.rs b/crates/core/src/actionability/requirements_tests.rs new file mode 100644 index 0000000..19f3472 --- /dev/null +++ b/crates/core/src/actionability/requirements_tests.rs @@ -0,0 +1,88 @@ +use super::*; +use crate::{DragParams, KeyCombo, Point}; + +fn key() -> KeyCombo { + KeyCombo { + key: "a".into(), + modifiers: Vec::new(), + } +} + +#[test] +fn pointer_actions_require_playwright_style_positional_checks() { + let click = ActionabilityRequirements::for_action(&Action::Click); + assert!(click.visible && click.stable && click.enabled && click.receives_events); + + let hover = ActionabilityRequirements::for_action(&Action::Hover); + assert!(hover.visible && hover.stable && hover.receives_events); + assert!(!hover.enabled); + + let drag = ActionabilityRequirements::for_action(&Action::Drag(DragParams { + from: Point { x: 0.0, y: 0.0 }, + to: Point { x: 1.0, y: 1.0 }, + duration_ms: None, + drop_delay_ms: None, + })); + assert_eq!(drag, hover); +} + +#[test] +fn pointer_delivery_is_selected_from_live_semantic_capability_evidence() { + let requirements = ActionabilityRequirements::for_action(&Action::Click); + + assert_eq!( + requirements.pointer_delivery( + &Action::Click, + &[crate::capability::CLICK.into()], + crate::InteractionPolicy::headless(), + ), + PointerDelivery::Semantic + ); + assert_eq!( + requirements.pointer_delivery(&Action::Click, &[], crate::InteractionPolicy::headless(),), + PointerDelivery::Physical + ); + assert_eq!( + requirements.pointer_delivery( + &Action::DoubleClick, + &[crate::capability::CLICK.into()], + crate::InteractionPolicy::headless(), + ), + PointerDelivery::Physical + ); + assert_eq!( + requirements.pointer_delivery( + &Action::Click, + &[crate::capability::CLICK.into()], + crate::InteractionPolicy::headed(), + ), + PointerDelivery::Physical + ); + + assert!(!requirements.requires_stability(PointerDelivery::Semantic)); + assert!(requirements.requires_stability(PointerDelivery::Physical)); +} + +#[test] +fn editing_actions_do_not_require_positional_stability_or_hit_testing() { + let fill = ActionabilityRequirements::for_action(&Action::SetValue("x".into())); + assert!(fill.visible && fill.enabled && fill.editable); + assert!(!fill.stable && !fill.receives_events); +} + +#[test] +fn keyboard_and_focus_actions_skip_irrelevant_geometry_checks() { + for action in [ + Action::SetFocus, + Action::PressKey(key()), + Action::KeyDown(key()), + Action::KeyUp(key()), + ] { + let requirements = ActionabilityRequirements::for_action(&action); + assert!(!requirements.visible); + assert!(!requirements.stable); + assert!(!requirements.enabled); + assert!(!requirements.editable); + assert!(!requirements.receives_events); + } +} diff --git a/crates/core/src/actionability/stability.rs b/crates/core/src/actionability/stability.rs new file mode 100644 index 0000000..8ac6e91 --- /dev/null +++ b/crates/core/src/actionability/stability.rs @@ -0,0 +1,40 @@ +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct StabilityExpectation { + pub(crate) bounds_hash: Option, + pub(crate) bounds: Option, + pub(crate) strict: bool, + pub(crate) samples: u32, + pub(crate) span_ms: u64, +} + +impl StabilityExpectation { + pub(crate) fn permissive(bounds_hash: Option) -> Self { + Self { + bounds_hash, + bounds: None, + strict: false, + samples: 1, + span_ms: 0, + } + } + + pub(crate) fn strict(bounds: Option, samples: u32, span_ms: u64) -> Self { + Self { + bounds_hash: bounds.and_then(|bounds| bounds.bounds_hash()), + bounds, + strict: true, + samples, + span_ms, + } + } + + pub(crate) fn strict_hash(bounds_hash: Option) -> Self { + Self { + bounds_hash, + bounds: None, + strict: true, + samples: 1, + span_ms: 0, + } + } +} diff --git a/crates/core/src/actionability/stability_evidence.rs b/crates/core/src/actionability/stability_evidence.rs new file mode 100644 index 0000000..89da80d --- /dev/null +++ b/crates/core/src/actionability/stability_evidence.rs @@ -0,0 +1,7 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub(crate) struct StabilityEvidence { + pub(crate) samples: u32, + pub(crate) span_ms: u64, +} diff --git a/crates/core/src/actionability/stability_sampler.rs b/crates/core/src/actionability/stability_sampler.rs new file mode 100644 index 0000000..466aeb6 --- /dev/null +++ b/crates/core/src/actionability/stability_sampler.rs @@ -0,0 +1,69 @@ +use crate::Rect; +use std::time::Duration; + +pub(crate) const STABILITY_SAMPLE_INTERVAL: Duration = Duration::from_millis(17); +pub(crate) const MIN_STABILITY_SPAN: Duration = Duration::from_millis(34); +const GEOMETRY_TOLERANCE: f64 = 0.5; + +pub(crate) struct StabilitySampler { + previous: Option, + consecutive: u32, + stable_since: Duration, +} + +impl StabilitySampler { + pub(crate) fn new() -> Self { + Self { + previous: None, + consecutive: 0, + stable_since: Duration::ZERO, + } + } + + pub(crate) fn observe(&mut self, bounds: Option, elapsed: Duration) -> bool { + let Some(bounds) = bounds else { + self.previous = None; + self.consecutive = 0; + self.stable_since = elapsed; + return false; + }; + if self + .previous + .is_some_and(|previous| geometry_matches(previous, bounds)) + { + self.consecutive += 1; + } else { + self.consecutive = 1; + self.stable_since = elapsed; + } + self.previous = Some(bounds); + self.consecutive >= 3 && elapsed.saturating_sub(self.stable_since) >= MIN_STABILITY_SPAN + } + + pub(crate) fn samples(&self) -> u32 { + self.consecutive + } + + pub(crate) fn stable_span(&self, elapsed: Duration) -> Duration { + elapsed.saturating_sub(self.stable_since) + } + + pub(crate) fn bounds(&self) -> Option { + self.previous + } +} + +pub(crate) fn geometry_matches(left: Rect, right: Rect) -> bool { + [ + (left.x, right.x), + (left.y, right.y), + (left.width, right.width), + (left.height, right.height), + ] + .into_iter() + .all(|(left, right)| (left - right).abs() <= GEOMETRY_TOLERANCE) +} + +#[cfg(test)] +#[path = "stability_sampler_tests.rs"] +mod tests; diff --git a/crates/core/src/actionability/stability_sampler_tests.rs b/crates/core/src/actionability/stability_sampler_tests.rs new file mode 100644 index 0000000..6a03b97 --- /dev/null +++ b/crates/core/src/actionability/stability_sampler_tests.rs @@ -0,0 +1,38 @@ +use super::*; + +fn rect(x: f64) -> Rect { + Rect { + x, + y: 0.0, + width: 100.0, + height: 20.0, + } +} + +#[test] +fn thirty_hertz_change_cannot_pass_on_phase_aligned_equal_samples() { + let mut sampler = StabilitySampler::new(); + assert!(!sampler.observe(Some(rect(0.0)), Duration::ZERO)); + assert!(!sampler.observe(Some(rect(0.0)), Duration::from_millis(17))); + assert!(!sampler.observe(Some(rect(10.0)), Duration::from_millis(34))); + assert!(!sampler.observe(Some(rect(10.0)), Duration::from_millis(51))); + assert!(sampler.observe(Some(rect(10.0)), Duration::from_millis(68))); +} + +#[test] +fn sixty_hertz_motion_resets_the_stability_window() { + let mut sampler = StabilitySampler::new(); + for (index, elapsed) in [0, 17, 34, 51].into_iter().enumerate() { + assert!(!sampler.observe(Some(rect(index as f64)), Duration::from_millis(elapsed))); + } + assert!(!sampler.observe(Some(rect(3.0)), Duration::from_millis(68))); + assert!(sampler.observe(Some(rect(3.0)), Duration::from_millis(85))); +} + +#[test] +fn subpixel_jitter_within_tolerance_is_stable() { + let mut sampler = StabilitySampler::new(); + assert!(!sampler.observe(Some(rect(10.0)), Duration::ZERO)); + assert!(!sampler.observe(Some(rect(10.2)), Duration::from_millis(17))); + assert!(sampler.observe(Some(rect(9.8)), Duration::from_millis(34))); +} diff --git a/crates/core/src/actionability/status.rs b/crates/core/src/actionability/status.rs index 9ff4cf8..5cd1a3a 100644 --- a/crates/core/src/actionability/status.rs +++ b/crates/core/src/actionability/status.rs @@ -2,7 +2,7 @@ use serde::Serialize; #[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] -pub enum ActionabilityStatus { +pub(crate) enum ActionabilityStatus { Pass, Fail, Unknown, diff --git a/crates/core/src/actionability_live_failure_tests.rs b/crates/core/src/actionability_live_failure_tests.rs new file mode 100644 index 0000000..079d488 --- /dev/null +++ b/crates/core/src/actionability_live_failure_tests.rs @@ -0,0 +1,63 @@ +use super::*; + +#[test] +fn empty_live_actions_replace_stale_snapshot_capabilities() { + let stale = entry(); + let adapter = LiveAdapter { + state: None, + bounds: stale.geometry.bounds, + actions: Some(vec![]), + }; + + let err = check_live( + &stale, + &NativeHandle::null(), + &adapter, + &ActionRequest::headless(Action::Click), + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::PolicyDenied); + assert!(err.message.contains("supported_action")); +} + +#[test] +fn unsupported_live_reads_fail_closed() { + let err = check_live( + &entry(), + &NativeHandle::null(), + &UnsupportedLiveAdapter, + &ActionRequest::headless(Action::Click), + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::ActionNotSupported); + assert!(err.message.contains("Live element evidence")); +} + +#[test] +fn empty_live_element_fails_as_stale_before_dispatch() { + let err = check_live( + &entry(), + &NativeHandle::null(), + &DeadLiveElementAdapter, + &ActionRequest::headless(Action::Click), + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::StaleRef); + assert!(err.message.contains("changed role")); +} + +#[test] +fn live_read_errors_are_not_silently_downgraded_to_snapshot_data() { + let err = check_live( + &entry(), + &NativeHandle::null(), + &LiveReadErrorAdapter, + &ActionRequest::headless(Action::Click), + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::PermDenied); +} diff --git a/crates/core/src/actionability_live_tests.rs b/crates/core/src/actionability_live_tests.rs index 5666d29..2dc0d0b 100644 --- a/crates/core/src/actionability_live_tests.rs +++ b/crates/core/src/actionability_live_tests.rs @@ -1,11 +1,12 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ + AdapterError, ErrorCode, Rect, action::Action, action_request::ActionRequest, - adapter::{LiveElement, NativeHandle, PlatformAdapter, SnapshotSurface}, + adapter::{LiveElement, NativeHandle, SnapshotSurface}, capability, element_state::ElementState, - node::Rect, refs::RefEntry, }; @@ -15,87 +16,204 @@ struct LiveAdapter { actions: Option>, } -impl PlatformAdapter for LiveAdapter { - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { +impl ObservationOps for LiveAdapter { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result { + Ok(LiveElement { + identity: crate::adapter::live_identity("OK"), + state: self.state.clone().unwrap_or_else(|| ElementState { + role: "button".into(), + states: Vec::new(), + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }), + states_complete: true, + bounds: self.bounds, + available_actions: self.actions.clone().unwrap_or_default(), + }) + } + + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { Ok(self.state.clone()) } - fn get_element_bounds(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { Ok(self.bounds) } fn get_live_actions( &self, _handle: &NativeHandle, + _deadline: crate::Deadline, ) -> Result>, AdapterError> { Ok(self.actions.clone()) } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } } +impl ActionOps for LiveAdapter {} + +impl InputOps for LiveAdapter {} + +impl SystemOps for LiveAdapter {} + struct CombinedLiveAdapter; -impl PlatformAdapter for CombinedLiveAdapter { - fn get_live_element(&self, _handle: &NativeHandle) -> Result { +impl ObservationOps for CombinedLiveAdapter { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result { Ok(LiveElement { - state: Some(ElementState { + identity: crate::adapter::live_identity("OK"), + state: ElementState { role: "button".into(), states: vec![], value: None, - }), + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, bounds: Some(Rect { x: 1.0, y: 1.0, width: 20.0, height: 20.0, }), - available_actions: Some(vec![capability::CLICK.into()]), + available_actions: vec![capability::CLICK.into()], }) } - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { panic!("check_live should use get_live_element") } - fn get_element_bounds(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { panic!("check_live should use get_live_element") } fn get_live_actions( &self, _handle: &NativeHandle, + _deadline: crate::Deadline, ) -> Result>, AdapterError> { panic!("check_live should use get_live_element") } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } } +impl ActionOps for CombinedLiveAdapter {} + +impl InputOps for CombinedLiveAdapter {} + +impl SystemOps for CombinedLiveAdapter {} + struct LiveReadErrorAdapter; -impl PlatformAdapter for LiveReadErrorAdapter { - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { +impl ObservationOps for LiveReadErrorAdapter { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result { + Err(AdapterError::permission_denied()) + } + + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { Err(AdapterError::permission_denied()) } } +impl ActionOps for LiveReadErrorAdapter {} + +impl InputOps for LiveReadErrorAdapter {} + +impl SystemOps for LiveReadErrorAdapter {} + struct UnsupportedLiveAdapter; -impl PlatformAdapter for UnsupportedLiveAdapter {} +impl ObservationOps for UnsupportedLiveAdapter {} + +impl ActionOps for UnsupportedLiveAdapter {} + +impl InputOps for UnsupportedLiveAdapter {} + +impl SystemOps for UnsupportedLiveAdapter {} struct DeadLiveElementAdapter; -impl PlatformAdapter for DeadLiveElementAdapter { - fn get_live_element(&self, _handle: &NativeHandle) -> Result { +impl ObservationOps for DeadLiveElementAdapter { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result { Ok(LiveElement { - state: Some(ElementState { + identity: crate::adapter::live_identity("OK"), + state: ElementState { role: "unknown".into(), states: vec![], value: None, - }), + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, bounds: None, - available_actions: Some(vec![]), + available_actions: vec![], }) } } +impl ActionOps for DeadLiveElementAdapter {} + +impl InputOps for DeadLiveElementAdapter {} + +impl SystemOps for DeadLiveElementAdapter {} + fn entry() -> RefEntry { let bounds = Rect { x: 1.0, @@ -104,36 +222,54 @@ fn entry() -> RefEntry { height: 20.0, }; RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - description: None, - states: vec![], - bounds: Some(bounds), - bounds_hash: Some(bounds.bounds_hash()), - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: true, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("OK".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: true, + path: smallvec::SmallVec::new(), + }, } } #[test] fn live_actionability_overrides_stale_snapshot_state() { let mut stale = entry(); - stale.states.push("disabled".into()); + stale.capabilities.states.push("disabled".into()); let adapter = LiveAdapter { state: Some(ElementState { role: "button".into(), states: vec![], value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), }), - bounds: stale.bounds, + bounds: stale.geometry.bounds, actions: Some(vec![capability::CLICK.into()]), }; @@ -151,14 +287,14 @@ fn live_actionability_overrides_stale_snapshot_state() { #[test] fn live_actionability_uses_combined_live_element_read() { let mut stale = entry(); - stale.states.push("disabled".into()); - stale.bounds = Some(Rect { + stale.capabilities.states.push("disabled".into()); + stale.geometry.bounds = Some(Rect { x: 1.0, y: 1.0, width: 0.0, height: 20.0, }); - stale.available_actions = vec![]; + stale.capabilities.available_actions = vec![]; let report = check_live( &stale, @@ -174,10 +310,17 @@ fn live_actionability_uses_combined_live_element_read() { #[test] fn live_actionability_uses_actions_gained_after_snapshot() { let mut stale = entry(); - stale.available_actions = vec![]; + stale.capabilities.available_actions = vec![]; let adapter = LiveAdapter { - state: None, - bounds: stale.bounds, + state: Some(ElementState { + role: "button".into(), + states: vec![], + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }), + bounds: stale.geometry.bounds, actions: Some(vec![capability::CLICK.into()]), }; @@ -197,7 +340,7 @@ fn live_actionability_fails_when_action_disappears_after_snapshot() { let stale = entry(); let adapter = LiveAdapter { state: None, - bounds: stale.bounds, + bounds: stale.geometry.bounds, actions: Some(vec![capability::SET_VALUE.into()]), }; @@ -209,12 +352,12 @@ fn live_actionability_fails_when_action_disappears_after_snapshot() { ) .unwrap_err(); - assert_eq!(err.code, ErrorCode::ActionFailed); + assert_eq!(err.code, ErrorCode::PolicyDenied); assert!(err.message.contains("supported_action")); } #[test] -fn live_actionability_allows_identity_resolved_bounds_change() { +fn live_actionability_rejects_unstable_identity_resolved_bounds_change() { let stale = entry(); let adapter = LiveAdapter { state: None, @@ -227,79 +370,17 @@ fn live_actionability_allows_identity_resolved_bounds_change() { actions: Some(vec![capability::CLICK.into()]), }; - let report = check_live( + let err = check_live( &stale, &NativeHandle::null(), &adapter, - &ActionRequest::headless(Action::Click), - ) - .unwrap(); - - assert!(report.actionable); - let stable = report - .checks - .iter() - .find(|check| check.name == "stable") - .unwrap(); - assert_eq!(stable.status, ActionabilityStatus::Unknown); -} - -#[test] -fn empty_live_actions_do_not_erase_snapshot_capabilities() { - let stale = entry(); - let adapter = LiveAdapter { - state: None, - bounds: stale.bounds, - actions: Some(vec![]), - }; - - let report = check_live( - &stale, - &NativeHandle::null(), - &adapter, - &ActionRequest::headless(Action::Click), - ) - .unwrap(); - - assert!(report.actionable); -} - -#[test] -fn unsupported_live_reads_fall_back_to_snapshot_entry() { - let report = check_live( - &entry(), - &NativeHandle::null(), - &UnsupportedLiveAdapter, - &ActionRequest::headless(Action::Click), - ) - .unwrap(); - - assert!(report.actionable); -} - -#[test] -fn empty_live_element_fails_as_stale_before_dispatch() { - let err = check_live( - &entry(), - &NativeHandle::null(), - &DeadLiveElementAdapter, - &ActionRequest::headless(Action::Click), + &ActionRequest::headed(Action::DoubleClick), ) .unwrap_err(); - assert_eq!(err.code, ErrorCode::StaleRef); - assert!(err.message.contains("no longer exposes live")); + assert_eq!(err.code, ErrorCode::ActionFailed); + assert!(err.message.contains("stable")); } -#[test] -fn live_read_errors_are_not_silently_downgraded_to_snapshot_data() { - let err = check_live( - &entry(), - &NativeHandle::null(), - &LiveReadErrorAdapter, - &ActionRequest::headless(Action::Click), - ) - .unwrap_err(); - - assert_eq!(err.code, ErrorCode::PermDenied); -} +#[path = "actionability_live_failure_tests.rs"] +mod failure_tests; diff --git a/crates/core/src/actionability_tests.rs b/crates/core/src/actionability_tests.rs index dd4cad0..e6e1035 100644 --- a/crates/core/src/actionability_tests.rs +++ b/crates/core/src/actionability_tests.rs @@ -1,11 +1,7 @@ use super::*; use crate::{ - action::{Action, Direction}, - action_request::ActionRequest, - adapter::SnapshotSurface, - capability, - node::Rect, - refs::RefEntry, + Action, Direction, ErrorCode, Rect, action_request::ActionRequest, adapter::SnapshotSurface, + capability, refs::RefEntry, }; fn entry() -> RefEntry { @@ -16,25 +12,106 @@ fn entry() -> RefEntry { height: 20.0, }; RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - description: None, - states: vec![], - bounds: Some(bounds), - bounds_hash: Some(bounds.bounds_hash()), - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: true, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("OK".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: true, + path: smallvec::SmallVec::new(), + }, } } +fn visibility_evidence( + hidden: Option, + states: Vec, + states_complete: bool, +) -> evidence::ActionabilityEvidence { + evidence::ActionabilityEvidence { + state: crate::ElementState { + role: "button".into(), + states, + value: None, + enabled: Some(true), + hidden, + offscreen: Some(false), + }, + states_complete, + bounds: Some(Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }), + available_actions: vec![capability::CLICK.into()], + } +} + +#[test] +fn explicit_typed_hidden_state_wins() { + let check = gates::visibility(&visibility_evidence(Some(true), Vec::new(), true)); + + assert_eq!(check.status, ActionabilityStatus::Fail); + assert_eq!(check.reason.as_deref(), Some("live hidden state is true")); +} + +#[test] +fn complete_canonical_hidden_state_fails_when_typed_state_is_absent() { + let check = gates::visibility(&visibility_evidence( + None, + vec![crate::state::HIDDEN.into()], + true, + )); + + assert_eq!(check.status, ActionabilityStatus::Fail); + assert_eq!( + check.reason.as_deref(), + Some("canonical hidden state is present") + ); +} + +#[test] +fn complete_canonical_states_without_hidden_allow_visibility_checks_to_continue() { + let check = gates::visibility(&visibility_evidence(None, Vec::new(), true)); + + assert_eq!(check.status, ActionabilityStatus::Pass); +} + +#[test] +fn incomplete_canonical_states_keep_missing_typed_hidden_unknown() { + let check = gates::visibility(&visibility_evidence(None, Vec::new(), false)); + + assert_eq!(check.status, ActionabilityStatus::Unknown); + assert_eq!( + check.reason.as_deref(), + Some("live hidden state unavailable") + ); +} + #[test] fn click_passes_when_target_is_enabled_visible_and_supported() { let report = check(&entry(), &ActionRequest::headless(Action::Click)).unwrap(); @@ -42,10 +119,27 @@ fn click_passes_when_target_is_enabled_visible_and_supported() { assert!(report.actionable); } +#[test] +fn scroll_to_attempts_adapter_delivery_without_a_native_element_capability() { + let mut target = entry(); + target.capabilities.available_actions.clear(); + + let report = check(&target, &ActionRequest::headless(Action::ScrollTo)).unwrap(); + + assert!(report.actionable); +} + +#[test] +fn states_are_enabled_reads_the_canonical_disabled_token() { + assert!(!states_are_enabled(&[crate::state::DISABLED.to_string()])); + assert!(states_are_enabled(&[])); + assert!(states_are_enabled(&[crate::state::FOCUSED.to_string()])); +} + #[test] fn disabled_entry_fails_before_action_dispatch() { let mut entry = entry(); - entry.states.push("disabled".into()); + entry.capabilities.states.push("disabled".into()); let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err(); @@ -63,14 +157,68 @@ fn zero_sized_bounds_fail_visibility() { width: 0.0, height: 20.0, }; - entry.bounds = Some(bounds); - entry.bounds_hash = Some(bounds.bounds_hash()); + entry.geometry.bounds = Some(bounds); + entry.geometry.bounds_hash = bounds.bounds_hash(); let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err(); assert!(err.message.contains("visible")); } +#[test] +fn hidden_state_fails_visibility_before_action_dispatch() { + let mut entry = entry(); + entry.capabilities.states.push(crate::state::HIDDEN.into()); + + let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err(); + + assert!(err.message.contains("visible")); +} + +#[test] +fn offscreen_state_fails_visibility_before_action_dispatch() { + let mut entry = entry(); + entry + .capabilities + .states + .push(crate::state::OFFSCREEN.into()); + + let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err(); + + assert!(err.message.contains("visible")); +} + +#[test] +fn hidden_entry_fails_visibility_even_when_bounds_are_none() { + let mut entry = entry(); + entry.capabilities.states.push(crate::state::HIDDEN.into()); + entry.geometry.bounds = None; + entry.geometry.bounds_hash = None; + + let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err(); + + assert_eq!(err.code, ErrorCode::ActionFailed); + assert!(err.message.contains("visible")); + assert!(err.message.contains("hidden")); +} + +#[test] +fn offscreen_entry_fails_visibility_even_when_bounds_are_none() { + let mut entry = entry(); + entry + .capabilities + .states + .push(crate::state::OFFSCREEN.into()); + entry.geometry.bounds = None; + entry.geometry.bounds_hash = None; + + let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err(); + + assert_eq!(err.code, ErrorCode::ActionFailed); + assert!(err.message.contains("visible")); + assert!(err.message.contains("offscreen")); +} + #[test] fn text_input_requires_editable_target() { let err = check( @@ -79,6 +227,7 @@ fn text_input_requires_editable_target() { ) .unwrap_err(); + assert_eq!(err.code, ErrorCode::ActionNotSupported); assert!(err.message.contains("editable")); } @@ -86,14 +235,15 @@ fn text_input_requires_editable_target() { fn cursor_movement_requires_physical_policy() { let err = check(&entry(), &ActionRequest::headless(Action::Hover)).unwrap_err(); + assert_eq!(err.code, ErrorCode::PolicyDenied); assert!(err.message.contains("policy")); } #[test] fn headless_type_text_fails_policy_before_dispatch() { let mut target = entry(); - target.role = "textfield".into(); - target.available_actions = vec![capability::SET_VALUE.into()]; + target.identity.role = "textfield".into(); + target.capabilities.available_actions = vec![capability::SET_VALUE.into()]; let err = check( &target, @@ -101,43 +251,59 @@ fn headless_type_text_fails_policy_before_dispatch() { ) .unwrap_err(); - assert!(err.message.contains("policy")); + assert_eq!(err.code, ErrorCode::PolicyDenied); assert!(err.message.contains("focus")); } #[test] -fn right_click_requires_right_click_capability_before_dispatch() { +fn headless_right_click_denies_physical_fallback() { let err = check(&entry(), &ActionRequest::headless(Action::RightClick)).unwrap_err(); - assert_eq!(err.code, ErrorCode::ActionFailed); + assert_eq!(err.code, ErrorCode::PolicyDenied); assert!(err.message.contains("supported_action")); } +#[test] +fn headless_multi_clicks_deny_physical_fallback_before_hit_testing() { + for action in [Action::DoubleClick, Action::TripleClick] { + let err = check(&entry(), &ActionRequest::headless(action)).unwrap_err(); + + assert_eq!(err.code, ErrorCode::PolicyDenied); + assert!(err.message.contains("supported_action")); + } +} + +#[test] +fn headed_click_allows_verified_physical_fallback() { + let mut target = entry(); + target.capabilities.available_actions.clear(); + + assert!(check(&target, &ActionRequest::headed(Action::Click)).is_ok()); +} + +#[test] +fn headed_multi_clicks_require_and_allow_physical_delivery() { + let mut target = entry(); + target.capabilities.available_actions.clear(); + + for action in [Action::DoubleClick, Action::TripleClick] { + assert!(check(&target, &ActionRequest::headed(action)).is_ok()); + } +} + #[test] fn command_aliases_match_platform_capabilities() { let click_entry = entry(); - assert!(check(&click_entry, &ActionRequest::headless(Action::DoubleClick)).is_ok()); - assert!(check(&click_entry, &ActionRequest::headless(Action::TripleClick)).is_ok()); assert!(check(&click_entry, &ActionRequest::headless(Action::Check)).is_ok()); assert!(check(&click_entry, &ActionRequest::headless(Action::Uncheck)).is_ok()); let mut editable = entry(); - editable.role = "textfield".into(); - editable.available_actions = vec![capability::SET_VALUE.into()]; + editable.identity.role = "textfield".into(); + editable.capabilities.available_actions = vec![capability::SET_VALUE.into()]; assert!(check(&editable, &ActionRequest::headless(Action::Clear)).is_ok()); let mut scrollable = entry(); - scrollable.available_actions = vec![capability::SCROLL.into()]; - assert!( - check( - &scrollable, - &ActionRequest::headless(Action::Scroll(Direction::Down, 1)) - ) - .is_ok() - ); - assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_err()); - - scrollable.available_actions = vec![capability::SCROLL_TO.into()]; + scrollable.capabilities.available_actions = vec![capability::SCROLL.into()]; assert!( check( &scrollable, @@ -146,4 +312,14 @@ fn command_aliases_match_platform_capabilities() { .is_ok() ); assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_ok()); + + scrollable.capabilities.available_actions = vec![capability::SCROLL_TO.into()]; + assert!( + check( + &scrollable, + &ActionRequest::headless(Action::Scroll(Direction::Down, 1)) + ) + .is_err() + ); + assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_ok()); } diff --git a/crates/core/src/adapter.rs b/crates/core/src/adapter.rs deleted file mode 100644 index b6a75ba..0000000 --- a/crates/core/src/adapter.rs +++ /dev/null @@ -1,397 +0,0 @@ -use crate::{ - PermissionReport, PermissionState, - action::{DragParams, KeyCombo, MouseEvent, WindowOp}, - action_request::ActionRequest, - action_result::ActionResult, - element_state::ElementState, - error::{AdapterError, ErrorCode}, - node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo}, - notification::{NotificationFilter, NotificationIdentity, NotificationInfo}, - refs::RefEntry, -}; -use std::marker::PhantomData; - -pub struct WindowFilter { - pub focused_only: bool, - pub app: Option, -} - -#[non_exhaustive] -#[derive(Debug, Clone, Copy, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SnapshotSurface { - #[default] - Window, - Focused, - Menu, - Menubar, - Sheet, - Popover, - Alert, -} - -impl SnapshotSurface { - pub fn is_window(surface: &Self) -> bool { - matches!(surface, Self::Window) - } -} - -#[derive(Clone, Copy)] -pub struct TreeOptions { - pub max_depth: u8, - pub include_bounds: bool, - pub interactive_only: bool, - pub compact: bool, - pub surface: SnapshotSurface, - pub skeleton: bool, -} - -impl Default for TreeOptions { - fn default() -> Self { - Self { - max_depth: 10, - include_bounds: false, - interactive_only: false, - compact: false, - surface: SnapshotSurface::Window, - skeleton: false, - } - } -} - -impl TreeOptions { - pub(crate) fn with_ref_identity_bounds(mut self) -> Self { - self.include_bounds = true; - self - } -} - -#[derive(Debug, Clone, Default)] -pub struct LiveElement { - pub state: Option, - pub bounds: Option, - pub available_actions: Option>, -} - -pub(crate) fn optional_live_read( - result: Result, AdapterError>, -) -> Result, AdapterError> { - match result { - Ok(value) => Ok(value), - Err(err) if is_live_read_unsupported(&err) => Ok(None), - Err(err) => Err(err), - } -} - -fn is_live_read_unsupported(err: &AdapterError) -> bool { - matches!( - err.code, - ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported - ) -} - -pub enum ScreenshotTarget { - Screen(usize), - /// Capture the largest visible window owned by this process ID. - Window(i32), - FullScreen, -} - -pub struct NativeHandle { - pub(crate) ptr: *const std::ffi::c_void, - _not_send_sync: PhantomData<*const ()>, -} - -impl NativeHandle { - /// # Safety - /// - /// `ptr` must be a valid platform accessibility handle whose ownership is - /// transferred to the caller. The adapter that creates the handle must - /// document how it is released through [`PlatformAdapter::release_handle`]. - pub unsafe fn from_ptr(ptr: *const std::ffi::c_void) -> Self { - Self { - ptr, - _not_send_sync: PhantomData, - } - } - - pub fn null() -> Self { - Self { - ptr: std::ptr::null(), - _not_send_sync: PhantomData, - } - } -} - -impl NativeHandle { - /// Returns the raw platform pointer. For use by platform adapter crates only. - /// Callers must not retain the pointer beyond the lifetime of this handle. - pub fn as_raw(&self) -> *const std::ffi::c_void { - self.ptr - } -} - -pub struct ImageBuffer { - pub data: Vec, - pub format: ImageFormat, - pub width: u32, - pub height: u32, -} - -pub enum ImageFormat { - Png, - Jpg, -} - -impl ImageFormat { - pub fn as_str(&self) -> &'static str { - match self { - ImageFormat::Png => "png", - ImageFormat::Jpg => "jpg", - } - } -} - -pub trait PlatformAdapter: Send + Sync { - fn list_windows(&self, _filter: &WindowFilter) -> Result, AdapterError> { - Err(AdapterError::not_supported("list_windows")) - } - - fn list_apps(&self) -> Result, AdapterError> { - Err(AdapterError::not_supported("list_apps")) - } - - fn get_tree( - &self, - _win: &WindowInfo, - _opts: &TreeOptions, - ) -> Result { - Err(AdapterError::not_supported("get_tree")) - } - - fn execute_action( - &self, - _handle: &NativeHandle, - _request: ActionRequest, - ) -> Result { - Err(AdapterError::not_supported("execute_action")) - } - - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - Err(AdapterError::not_supported("resolve_element_strict")) - } - - /// Resolves an element under a caller deadline. Defaults to delegating - /// to [`PlatformAdapter::resolve_element_strict`], ignoring the timeout, - /// so adapters that implement only the un-timed variant still support - /// `wait --element`. Override to honor the remaining budget. - fn resolve_element_strict_with_timeout( - &self, - entry: &RefEntry, - timeout: std::time::Duration, - ) -> Result { - tracing::trace!( - ?timeout, - "resolve_element_strict_with_timeout: default impl does not enforce the deadline; override to honor it" - ); - self.resolve_element_strict(entry) - } - - /// Releases a platform handle that an implementation took ownership of during resolve. - /// Adapter methods that receive `&NativeHandle` borrow it only; they must not consume - /// or release it. The default no-op is correct for adapters whose handles are owned - /// or freed elsewhere. - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - Ok(()) - } - - fn permission_report(&self) -> PermissionReport { - PermissionReport { - accessibility: PermissionState::Denied { - suggestion: "Platform adapter not available".into(), - }, - screen_recording: PermissionState::Unknown, - automation: PermissionState::NotRequired, - } - } - - fn unknown_accessibility_means_unsupported(&self) -> bool { - true - } - - fn request_permissions(&self) -> PermissionReport { - self.permission_report() - } - - fn focus_window(&self, _win: &WindowInfo) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("focus_window")) - } - - /// Brings the application owning `pid` to the foreground. Best-effort guard - /// invoked before physical (cursor/keyboard) input that targets a known - /// element, so synthetic events land on the intended window rather than - /// whatever happens to be frontmost. - fn focus_app(&self, _pid: i32) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("focus_app")) - } - - fn launch_app(&self, _id: &str, _timeout_ms: u64) -> Result { - Err(AdapterError::not_supported("launch_app")) - } - - fn close_app(&self, _id: &str, _force: bool) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("close_app")) - } - - /// Reports whether closing `identifier` would terminate a process the OS - /// depends on (window server, login session, shell). The set is - /// inherently platform-specific, so each adapter owns its own list; - /// core only asks. The default denies nothing. - fn is_protected_process(&self, _identifier: &str) -> bool { - false - } - - /// Reports whether `combo` is a platform-dangerous keyboard shortcut that - /// should be refused unless the caller explicitly forces it (for example - /// macOS Cmd+Q quit, Ctrl+Cmd+Q lock, Cmd+Alt+Esc force-quit). Which - /// combos are dangerous — and how key names alias to physical keys — is - /// platform-specific, so each adapter owns its own list; core only asks - /// and lets the caller override via `--force`. The default blocks nothing, - /// leaving the decision entirely to the calling agent. - fn is_blocked_combo(&self, _combo: &KeyCombo) -> bool { - false - } - - fn screenshot(&self, _target: ScreenshotTarget) -> Result { - Err(AdapterError::not_supported("screenshot")) - } - - fn get_clipboard(&self) -> Result { - Err(AdapterError::not_supported("get_clipboard")) - } - - fn set_clipboard(&self, _text: &str) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("set_clipboard")) - } - - fn focused_window(&self) -> Result, AdapterError> { - Err(AdapterError::not_supported("focused_window")) - } - - fn get_live_value(&self, _handle: &NativeHandle) -> Result, AdapterError> { - Err(AdapterError::not_supported("get_live_value")) - } - - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { - Err(AdapterError::not_supported("get_live_state")) - } - - fn get_live_actions( - &self, - _handle: &NativeHandle, - ) -> Result>, AdapterError> { - Err(AdapterError::not_supported("get_live_actions")) - } - - fn get_live_element(&self, handle: &NativeHandle) -> Result { - let live = LiveElement { - state: optional_live_read(self.get_live_state(handle))?, - bounds: optional_live_read(self.get_element_bounds(handle))?, - available_actions: optional_live_read(self.get_live_actions(handle))?, - }; - if live.state.is_none() && live.bounds.is_none() && live.available_actions.is_none() { - return Err(AdapterError::not_supported("get_live_element")); - } - Ok(live) - } - - fn press_key_for_app( - &self, - _app_name: &str, - _combo: &crate::action::KeyCombo, - ) -> Result { - Err(AdapterError::not_supported("press_key_for_app")) - } - - fn wait_for_menu(&self, _pid: i32, _open: bool, _timeout_ms: u64) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("wait_for_menu")) - } - - fn list_surfaces(&self, _pid: i32) -> Result, AdapterError> { - Err(AdapterError::not_supported("list_surfaces")) - } - - fn get_element_bounds(&self, _handle: &NativeHandle) -> Result, AdapterError> { - Err(AdapterError::not_supported("get_element_bounds")) - } - - fn window_op(&self, _win: &WindowInfo, _op: WindowOp) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("window_op")) - } - - fn mouse_event(&self, _event: MouseEvent) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("mouse_event")) - } - - fn key_event(&self, _combo: &KeyCombo, _down: bool) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("key_event")) - } - - fn drag(&self, _params: DragParams) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("drag")) - } - - fn clear_clipboard(&self) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("clear_clipboard")) - } - - fn list_notifications( - &self, - _filter: &NotificationFilter, - ) -> Result, AdapterError> { - Err(AdapterError::not_supported("list_notifications")) - } - - fn dismiss_notification( - &self, - _index: usize, - _app_filter: Option<&str>, - ) -> Result { - Err(AdapterError::not_supported("dismiss_notification")) - } - - fn dismiss_all_notifications( - &self, - _app_filter: Option<&str>, - ) -> Result<(Vec, Vec), AdapterError> { - Err(AdapterError::not_supported("dismiss_all_notifications")) - } - - /// Press a named action button on the notification at `index`. - /// - /// `identity` lets the caller pin the targeted notification to an - /// expected app / title fingerprint. Notification Center reindexes - /// entries between listings, so index-only targeting can press the - /// wrong button if a notification arrives or is dismissed between - /// `list_notifications` and this call. When any identity field is - /// `Some`, implementations must return - /// `ErrorCode::NotificationNotFound` if the row at `index` does not - /// match. Passing an empty identity (or `None`) preserves legacy - /// index-only behavior for callers that reconcile themselves. - fn notification_action( - &self, - _index: usize, - _identity: Option<&NotificationIdentity>, - _action_name: &str, - ) -> Result { - Err(AdapterError::not_supported("notification_action")) - } - - fn get_subtree( - &self, - _handle: &NativeHandle, - _opts: &TreeOptions, - ) -> Result { - Err(AdapterError::not_supported("get_subtree")) - } -} diff --git a/crates/core/src/adapter/actions.rs b/crates/core/src/adapter/actions.rs new file mode 100644 index 0000000..9931856 --- /dev/null +++ b/crates/core/src/adapter/actions.rs @@ -0,0 +1,24 @@ +use crate::{ + AdapterError, InteractionLease, action_request::ActionRequest, action_result::ActionResult, + native_handle::NativeHandle, +}; + +pub trait ActionOps: Send + Sync { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &InteractionLease, + ) -> Result { + Err(AdapterError::not_supported("execute_action")) + } + + fn scroll_into_view( + &self, + handle: &NativeHandle, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + let _ = (handle, lease); + Err(AdapterError::not_supported("scroll_into_view")) + } +} diff --git a/crates/core/src/adapter/input.rs b/crates/core/src/adapter/input.rs new file mode 100644 index 0000000..70464fb --- /dev/null +++ b/crates/core/src/adapter/input.rs @@ -0,0 +1,59 @@ +use crate::{ + AdapterError, ClipboardContent, ClipboardFormat, Deadline, DragParams, InteractionLease, + KeyCombo, MouseEvent, +}; + +/// `get_clipboard`/`set_clipboard` were removed pre-1.0 in favor of +/// `get_clipboard_content`/`set_clipboard_content`; the C ABI +/// (`ad_get_clipboard`/`ad_set_clipboard`) is unaffected. +pub trait InputOps: Send + Sync { + fn mouse_event( + &self, + _event: MouseEvent, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("mouse_event")) + } + + fn key_event( + &self, + _combo: &KeyCombo, + _down: bool, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("key_event")) + } + + fn drag(&self, _params: DragParams, _lease: &InteractionLease) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("drag")) + } + + fn clear_clipboard(&self, _lease: &InteractionLease) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("clear_clipboard")) + } + + /// Reads the requested clipboard representation. Returns `Ok(None)` + /// when the pasteboard has no data of the requested shape (or, for + /// `Auto`, no data at all) — a normal, non-error outcome distinct from + /// `Err(not_supported)`, which means this platform never implements + /// clipboard reads. + fn get_clipboard_content( + &self, + _format: ClipboardFormat, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("get_clipboard_content")) + } + + fn set_clipboard_content( + &self, + _content: &ClipboardContent, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("set_clipboard_content")) + } +} + +#[cfg(test)] +#[path = "input_tests.rs"] +mod tests; diff --git a/crates/core/src/adapter/input_tests.rs b/crates/core/src/adapter/input_tests.rs new file mode 100644 index 0000000..ccd66b9 --- /dev/null +++ b/crates/core/src/adapter/input_tests.rs @@ -0,0 +1,61 @@ +use super::*; +use crate::ErrorCode; + +struct DefaultOnly; +impl InputOps for DefaultOnly {} + +fn lease() -> crate::InteractionLease { + crate::InteractionLease::guarded(crate::Deadline::standard().unwrap(), ()).unwrap() +} + +#[test] +fn default_clear_clipboard_is_not_supported() { + let err = DefaultOnly.clear_clipboard(&lease()).unwrap_err(); + assert_eq!(err.code, ErrorCode::PlatformNotSupported); +} + +#[test] +fn default_get_clipboard_content_is_not_supported() { + let err = DefaultOnly + .get_clipboard_content(ClipboardFormat::Text, crate::Deadline::standard().unwrap()) + .unwrap_err(); + assert_eq!(err.code, ErrorCode::PlatformNotSupported); +} + +#[test] +fn default_set_clipboard_content_is_not_supported() { + let err = DefaultOnly + .set_clipboard_content(&ClipboardContent::Text("x".into()), &lease()) + .unwrap_err(); + assert_eq!(err.code, ErrorCode::PlatformNotSupported); +} + +/// KTD13: `get_clipboard`/`set_clipboard` (string-only) were removed, not +/// wrapped — `get_clipboard_content`/`set_clipboard_content` are the only +/// surface. A regression that re-adds either legacy method would compile +/// (they'd just be unused trait members) so this guard reads the source +/// text directly rather than relying on the type system to catch it. +#[test] +fn legacy_string_clipboard_methods_are_gone() { + let src = include_str!("input.rs"); + assert!( + !src.contains("fn get_clipboard("), + "get_clipboard (string-only) must stay removed; use get_clipboard_content" + ); + assert!( + !src.contains("fn set_clipboard("), + "set_clipboard (string-only) must stay removed; use set_clipboard_content" + ); + assert!( + src.contains("fn clear_clipboard("), + "clear_clipboard must remain (KTD13 keeps it unchanged)" + ); + assert!( + src.contains("fn get_clipboard_content("), + "get_clipboard_content must remain the read surface" + ); + assert!( + src.contains("fn set_clipboard_content("), + "set_clipboard_content must remain the write surface" + ); +} diff --git a/crates/core/src/adapter/mod.rs b/crates/core/src/adapter/mod.rs new file mode 100644 index 0000000..ad93295 --- /dev/null +++ b/crates/core/src/adapter/mod.rs @@ -0,0 +1,28 @@ +pub(crate) mod actions; +pub(crate) mod input; +pub(crate) mod observation; +pub(crate) mod system; +#[cfg(test)] +mod test_support; + +pub(crate) use actions::ActionOps; +pub(crate) use input::InputOps; +pub(crate) use observation::ObservationOps; +pub(crate) use observation::optional_live_read; +pub(crate) use system::SystemOps; +#[cfg(test)] +pub(crate) use test_support::{ + complete_live_observation, exact_window_focus, guarded_interaction_lease, live_identity, + observed_tree, +}; + +pub(crate) use crate::live_element::LiveElement; +pub(crate) use crate::native_handle::NativeHandle; +pub(crate) use crate::screenshot_target::ScreenshotTarget; +pub(crate) use crate::snapshot_surface::SnapshotSurface; +pub(crate) use crate::tree_options::TreeOptions; +pub(crate) use crate::window_filter::WindowFilter; + +pub trait PlatformAdapter: ObservationOps + ActionOps + InputOps + SystemOps {} + +impl PlatformAdapter for T {} diff --git a/crates/core/src/adapter/observation.rs b/crates/core/src/adapter/observation.rs new file mode 100644 index 0000000..f75589f --- /dev/null +++ b/crates/core/src/adapter/observation.rs @@ -0,0 +1,165 @@ +use crate::{ + AccessibilityNode, AdapterError, AppInfo, Deadline, ErrorCode, Rect, SurfaceInfo, WindowInfo, + element_state::ElementState, + live_element::LiveElement, + live_locator::{ObservationRequest, ObservationRoot, ObservedTree}, + native_handle::NativeHandle, + refs::RefEntry, + tree_options::TreeOptions, + window_filter::WindowFilter, +}; + +pub(crate) fn optional_live_read( + result: Result, AdapterError>, +) -> Result, AdapterError> { + match result { + Ok(value) => Ok(value), + Err(err) if is_live_read_unsupported(&err) => Ok(None), + Err(err) => Err(err), + } +} + +fn is_live_read_unsupported(err: &AdapterError) -> bool { + matches!( + err.code, + ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported + ) +} + +pub trait ObservationOps: Send + Sync { + fn observe_tree( + &self, + _root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result { + Err(AdapterError::not_supported("observe_tree")) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("list_windows")) + } + + fn list_apps(&self, _deadline: Deadline) -> Result, AdapterError> { + Err(AdapterError::not_supported("list_apps")) + } + + fn list_apps_scoped( + &self, + name: &str, + bundle_id: Option<&str>, + deadline: Deadline, + ) -> Result, AdapterError> { + Ok(self + .list_apps(deadline)? + .into_iter() + .filter(|app| { + app.name.eq_ignore_ascii_case(name) + && bundle_id.is_none_or(|expected| { + app.bundle_id + .as_deref() + .is_some_and(|actual| actual.eq_ignore_ascii_case(expected)) + }) + }) + .collect()) + } + + fn get_tree( + &self, + _win: &WindowInfo, + _opts: &TreeOptions, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("get_tree")) + } + + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("resolve_element_strict")) + } + + fn resolve_locator_anchor( + &self, + _entry: &RefEntry, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("resolve_locator_anchor")) + } + + fn get_subtree( + &self, + _handle: &NativeHandle, + _opts: &TreeOptions, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("get_subtree")) + } + + fn list_surfaces( + &self, + _process: crate::ProcessIdentity, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("list_surfaces")) + } + + fn get_live_value( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("get_live_value")) + } + + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("get_live_state")) + } + + fn get_live_actions( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result>, AdapterError> { + Err(AdapterError::not_supported("get_live_actions")) + } + + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("get_live_element")) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("get_element_bounds")) + } + + fn hit_test( + &self, + handle: &NativeHandle, + point: crate::Point, + _deadline: Deadline, + ) -> Result { + let _ = (handle, point); + Err(AdapterError::not_supported("hit_test")) + } +} + +#[cfg(test)] +#[path = "observation_tests.rs"] +mod tests; diff --git a/crates/core/src/adapter/observation_tests.rs b/crates/core/src/adapter/observation_tests.rs new file mode 100644 index 0000000..a5f0d97 --- /dev/null +++ b/crates/core/src/adapter/observation_tests.rs @@ -0,0 +1,88 @@ +use super::*; +use crate::{adapter::SnapshotSurface, refs::RefEntry}; +use std::{sync::Mutex, time::Duration}; + +struct TimedResolver { + observed: Mutex>, +} + +impl ObservationOps for TimedResolver { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + deadline: crate::Deadline, + ) -> Result { + *self.observed.lock().expect("capture lock") = Some(deadline.remaining()); + Ok(NativeHandle::null()) + } +} + +#[test] +fn untimed_convenience_delegates_to_the_deadline_aware_primitive() { + let resolver = TimedResolver { + observed: Mutex::new(None), + }; + + resolver + .resolve_element_strict(&entry(), crate::Deadline::standard().unwrap()) + .expect("timed resolver should serve the convenience call"); + + assert!( + resolver + .observed + .lock() + .expect("capture lock") + .is_some_and(|duration| duration <= Duration::from_secs(5)) + ); +} + +#[test] +fn locator_anchor_never_falls_back_to_the_public_strict_resolver() { + let resolver = TimedResolver { + observed: Mutex::new(None), + }; + + let error = resolver + .resolve_locator_anchor(&entry(), crate::Deadline::standard().unwrap()) + .err() + .expect("unsupported exact-path resolution must fail closed"); + + assert_eq!(error.code, ErrorCode::PlatformNotSupported); + assert_eq!(*resolver.observed.lock().expect("capture lock"), None); +} + +fn entry() -> RefEntry { + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: None, + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: Vec::new(), + available_actions: Vec::new(), + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: Default::default(), + }, + } +} diff --git a/crates/core/src/adapter/system.rs b/crates/core/src/adapter/system.rs new file mode 100644 index 0000000..5bbc96c --- /dev/null +++ b/crates/core/src/adapter/system.rs @@ -0,0 +1,233 @@ +use crate::{ + AdapterError, AdapterSession, AppInfo, Deadline, DismissAllNotificationsRequest, + DismissNotificationRequest, ImageBuffer, InteractionLease, InteractionPolicy, KeyCombo, + NotificationActionRequest, NotificationFilter, NotificationInfo, PermissionReport, + PermissionState, ProcessIdentity, SessionAffinity, SignalBaseline, SignalFilter, WindowInfo, + WindowOp, action_result::ActionResult, display_info::DisplayInfo, + screenshot_target::ScreenshotTarget, +}; + +pub trait SystemOps: Send + Sync { + fn acquire_interaction_lease( + &self, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("acquire_interaction_lease")) + } + + fn permission_report(&self, _deadline: Deadline) -> Result { + Ok(PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "Platform adapter not available".into(), + }, + screen_recording: PermissionState::Unknown, + automation: PermissionState::NotRequired, + }) + } + + fn unknown_accessibility_means_unsupported(&self) -> bool { + true + } + + /// Performs one renderer-accessibility activation mutation and returns. + /// Readiness polling belongs to core after the interaction lease is dropped. + fn activate_renderer_accessibility( + &self, + _process: ProcessIdentity, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported( + "activate_renderer_accessibility", + )) + } + + fn request_permissions( + &self, + lease: &InteractionLease, + ) -> Result { + self.permission_report(lease.deadline()) + } + + /// Performs the platform-native focus/raise operation for the exact window. + /// Core decides when focus is required; implementations must return only + /// after that window is confirmed focused, or return an error. + fn focus_window( + &self, + _win: &WindowInfo, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("focus_window")) + } + + fn launch_app( + &self, + _id: &str, + _options: &crate::launch_options::LaunchOptions, + _lease: &InteractionLease, + ) -> Result { + Err(AdapterError::not_supported("launch_app")) + } + + fn process_state( + &self, + _process: ProcessIdentity, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("process_state")) + } + + fn supported_surfaces(&self) -> Vec { + Vec::new() + } + + /// Opens adapter-native connection affinity for a persistent host. + /// + /// The returned session may retain platform connection state, but never a + /// resolved element handle. Stateless command callers do not need to open a + /// session. Adapters without persistent native state return + /// `PLATFORM_NOT_SUPPORTED`. + fn open_session( + &self, + _affinity: &SessionAffinity, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("open_session")) + } + + /// Captures a point-in-time [`SignalBaseline`] snapshot, + /// narrowed by `filter` when the caller already knows which app it cares + /// about. `deadline` is one absolute budget shared by every native read in + /// the capture; an adapter must return `TIMEOUT` rather than publish an + /// observation completed at or after it. `wait --event` calls this once at + /// wait-start and again on every poll, then diffs the two snapshots with + /// `crate::diff_signals` — the adapter never decides what changed, + /// only what the desktop looks like right now. + fn capture_signal_baseline( + &self, + _filter: &SignalFilter, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("capture_signal_baseline")) + } + + fn close_app( + &self, + _app: &AppInfo, + _force: bool, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("close_app")) + } + + /// Reports whether closing `identifier` would terminate a process the OS + /// depends on (window server, login session, shell). The set is + /// inherently platform-specific, so each adapter owns its own list; + /// core only asks. The default denies nothing. + fn is_protected_process(&self, _identifier: &str) -> bool { + false + } + + /// Reports whether `combo` is a platform-dangerous keyboard shortcut that + /// should be refused unless the caller explicitly forces it (for example + /// macOS Cmd+Q quit, Ctrl+Cmd+Q lock, Cmd+Alt+Esc force-quit). Which + /// combos are dangerous — and how key names alias to physical keys — is + /// platform-specific, so each adapter owns its own list; core only asks + /// and lets the caller override via `--force`. The default blocks nothing, + /// leaving the decision entirely to the calling agent. + fn is_blocked_combo(&self, _combo: &KeyCombo) -> bool { + false + } + + fn screenshot( + &self, + _target: ScreenshotTarget, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("screenshot")) + } + + fn list_displays(&self, _deadline: Deadline) -> Result, AdapterError> { + Err(AdapterError::not_supported("list_displays")) + } + + fn focused_window(&self, _deadline: Deadline) -> Result, AdapterError> { + Err(AdapterError::not_supported("focused_window")) + } + + fn press_key_for_app( + &self, + _process: ProcessIdentity, + _combo: &crate::KeyCombo, + _policy: crate::InteractionPolicy, + _lease: &InteractionLease, + ) -> Result { + Err(AdapterError::not_supported("press_key_for_app")) + } + + fn wait_for_menu( + &self, + _process: ProcessIdentity, + _open: bool, + _deadline: Deadline, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("wait_for_menu")) + } + + fn window_op( + &self, + _win: &WindowInfo, + _op: WindowOp, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("window_op")) + } + + /// Resolves a live window by `WindowInfo.id`, corroborating the match against + /// `pid` and, when present, `title`. Opaque ids must not be parsed as numeric + /// outside the adapter; macOS uses `w-`. + fn resolve_window_strict( + &self, + _win: &WindowInfo, + _deadline: Deadline, + ) -> Result { + Err(AdapterError::not_supported("resolve_window_strict")) + } + + fn list_notifications( + &self, + _filter: &NotificationFilter, + _policy: InteractionPolicy, + _deadline: Deadline, + _lease: Option<&InteractionLease>, + ) -> Result, AdapterError> { + Err(AdapterError::not_supported("list_notifications")) + } + + fn dismiss_notification( + &self, + _request: DismissNotificationRequest<'_>, + _lease: &InteractionLease, + ) -> Result { + Err(AdapterError::not_supported("dismiss_notification")) + } + + fn dismiss_all_notifications( + &self, + _request: DismissAllNotificationsRequest<'_>, + _lease: &InteractionLease, + ) -> Result<(Vec, Vec), AdapterError> { + Err(AdapterError::not_supported("dismiss_all_notifications")) + } + + fn notification_action( + &self, + _request: NotificationActionRequest<'_>, + _lease: &InteractionLease, + ) -> Result { + Err(AdapterError::not_supported("notification_action")) + } +} + +#[cfg(test)] +#[path = "system_tests.rs"] +mod tests; diff --git a/crates/core/src/adapter/system_tests.rs b/crates/core/src/adapter/system_tests.rs new file mode 100644 index 0000000..c48899b --- /dev/null +++ b/crates/core/src/adapter/system_tests.rs @@ -0,0 +1,33 @@ +use super::*; + +struct DefaultOnly; +impl SystemOps for DefaultOnly {} + +#[test] +fn default_interaction_lease_is_unsupported_in_test_builds_too() { + let error = match DefaultOnly.acquire_interaction_lease(crate::Deadline::standard().unwrap()) { + Ok(_) => panic!("default lease must fail closed"), + Err(error) => error, + }; + + assert_eq!(error.code, crate::ErrorCode::PlatformNotSupported); +} + +#[test] +fn default_surfaces_fail_closed() { + assert!(DefaultOnly.supported_surfaces().is_empty()); +} + +#[test] +fn default_open_session_is_explicitly_unsupported() { + let affinity = crate::SessionAffinity { + session_id: Some("test-session".into()), + }; + let error = match DefaultOnly.open_session(&affinity, crate::Deadline::standard().unwrap()) { + Ok(_) => panic!("default session affinity must fail closed"), + Err(error) => error, + }; + + assert_eq!(error.code, crate::ErrorCode::PlatformNotSupported); + assert!(error.message.contains("open_session")); +} diff --git a/crates/core/src/adapter/test_support.rs b/crates/core/src/adapter/test_support.rs new file mode 100644 index 0000000..1b53ce5 --- /dev/null +++ b/crates/core/src/adapter/test_support.rs @@ -0,0 +1,168 @@ +macro_rules! complete_live_observation { + ($role:expr, $name:expr, [$($action:expr),* $(,)?]) => { + fn get_live_element( + &self, + _handle: &$crate::adapter::NativeHandle, + _deadline: $crate::Deadline, + ) -> Result<$crate::LiveElement, $crate::AdapterError> { + Ok($crate::LiveElement { + identity: $crate::LiveIdentity { + name: $crate::LocatorField::Known($name.into()), + description: $crate::LocatorField::Absent, + identifiers: $crate::IdentifierEvidence::absent(), + }, + state: $crate::ElementState { + role: $role.into(), + states: Vec::new(), + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, + bounds: Some($crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }), + available_actions: vec![$($action.into()),*], + }) + } + + fn get_live_state( + &self, + _handle: &$crate::adapter::NativeHandle, + _deadline: $crate::Deadline, + ) -> Result, $crate::AdapterError> { + Ok(Some($crate::ElementState { + role: $role.into(), + states: Vec::new(), + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + })) + } + + fn get_element_bounds( + &self, + _handle: &$crate::adapter::NativeHandle, + _deadline: $crate::Deadline, + ) -> Result, $crate::AdapterError> { + Ok(Some($crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + })) + } + + fn get_live_actions( + &self, + _handle: &$crate::adapter::NativeHandle, + _deadline: $crate::Deadline, + ) -> Result>, $crate::AdapterError> { + Ok(Some(vec![$($action.into()),*])) + } + + fn hit_test( + &self, + _handle: &$crate::adapter::NativeHandle, + _point: $crate::Point, + _deadline: $crate::Deadline, + ) -> Result<$crate::hit_test::HitTestResult, $crate::AdapterError> { + Ok($crate::hit_test::HitTestResult::ReachesTarget) + } + }; +} + +pub(crate) use complete_live_observation; + +macro_rules! guarded_interaction_lease { + () => { + fn acquire_interaction_lease( + &self, + deadline: $crate::Deadline, + ) -> Result<$crate::InteractionLease, $crate::AdapterError> { + $crate::InteractionLease::guarded(deadline, ()) + } + }; +} + +pub(crate) use guarded_interaction_lease; + +macro_rules! exact_window_focus { + () => { + fn resolve_window_strict( + &self, + window: &$crate::WindowInfo, + _deadline: $crate::Deadline, + ) -> Result<$crate::WindowInfo, $crate::AdapterError> { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &$crate::WindowInfo, + _lease: &$crate::InteractionLease, + ) -> Result<(), $crate::AdapterError> { + Ok(()) + } + }; +} + +pub(crate) use exact_window_focus; + +pub(crate) fn live_identity(name: &str) -> crate::LiveIdentity { + crate::LiveIdentity { + name: crate::LocatorField::Known(name.into()), + description: crate::LocatorField::Absent, + identifiers: crate::IdentifierEvidence::absent(), + } +} + +pub(crate) fn observed_tree( + root: &crate::live_locator::ObservationRoot<'_>, + node: crate::AccessibilityNode, +) -> Result { + use crate::live_locator::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, LocatorStats, + ObservationSource, ObservedSubtree, ObservedTree, + }; + + fn field(value: Option) -> LocatorField { + value + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent) + } + + fn subtree(node: crate::AccessibilityNode) -> ObservedSubtree { + let native_id = node.identity.native_id.clone(); + let evidence = LocatorEvidence { + role: LocatorField::Known(node.role), + name: field(node.identity.name), + description: field(node.identity.description), + value: field(node.identity.value), + identifiers: IdentifierEvidence::typed(native_id, Some(0), true), + states: LocatorField::Known(node.presentation.states), + ref_evidence: LocatorRefEvidence { + bounds: node + .presentation + .bounds + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent), + available_actions: LocatorField::Known(node.presentation.available_actions), + }, + }; + let children = node.children.into_iter().map(subtree).collect(); + ObservedSubtree::new(evidence, children, true, node.children_count) + } + + ObservedTree::from_roots( + vec![subtree(node)], + ObservationSource::from_root(root), + LocatorStats::default(), + true, + ) +} diff --git a/crates/core/src/adapter_error.rs b/crates/core/src/adapter_error.rs new file mode 100644 index 0000000..b21ce90 --- /dev/null +++ b/crates/core/src/adapter_error.rs @@ -0,0 +1,270 @@ +use serde_json::Value; +use thiserror::Error; + +use crate::{DeliverySemantics, ErrorCode, InteractionPolicy}; + +#[derive(Debug, Error, Clone)] +#[error("{message}")] +pub struct AdapterError { + pub code: ErrorCode, + pub message: String, + pub suggestion: Option, + pub platform_detail: Option, + pub details: Option, + pub disposition: DeliverySemantics, + retryability: crate::retryability::Retryability, +} + +impl AdapterError { + pub fn new(code: ErrorCode, message: impl Into) -> Self { + Self { + code, + message: message.into(), + suggestion: None, + platform_detail: None, + details: None, + disposition: DeliverySemantics::unknown(), + retryability: crate::retryability::Retryability::Unspecified, + } + } + + pub fn with_suggestion(mut self, suggestion: impl Into) -> Self { + self.suggestion = Some(suggestion.into()); + self + } + + pub fn with_platform_detail(mut self, detail: impl Into) -> Self { + self.platform_detail = Some(detail.into()); + self + } + + pub fn with_details(mut self, details: Value) -> Self { + let retryability = crate::retryability::Retryability::from_details(&details); + if retryability != crate::retryability::Retryability::Unspecified { + self.retryability = retryability; + } + self.details = Some(details); + self + } + + pub fn with_disposition(mut self, disposition: DeliverySemantics) -> Self { + self.disposition = disposition; + self + } + + pub fn is_explicitly_retryable(&self) -> bool { + self.retryability == crate::retryability::Retryability::Retryable + } + + pub(crate) fn is_retryable_resolution_failure(&self) -> bool { + matches!( + self.code, + ErrorCode::StaleRef + | ErrorCode::AmbiguousTarget + | ErrorCode::Timeout + | ErrorCode::AppUnresponsive + ) && self.is_explicitly_retryable() + } + + pub fn permits_retry_by_default(&self) -> bool { + self.retryability != crate::retryability::Retryability::NonRetryable + } + + pub fn stale_ref(ref_id: &str) -> Self { + Self::new( + ErrorCode::StaleRef, + format!("{ref_id} not found in current RefMap"), + ) + .with_suggestion( + "Re-run a snapshot to obtain fresh refs, then retry with the new ref \ + (CLI: snapshot [--skeleton]; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn ambiguous_target(message: impl Into) -> Self { + Self::new(ErrorCode::AmbiguousTarget, message) + .with_suggestion( + "Re-run a snapshot to refresh refs, then retry with a more specific ref", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn not_supported(method: &str) -> Self { + Self::new( + ErrorCode::PlatformNotSupported, + format!("{method} is not supported on this platform"), + ) + .with_suggestion( + "Use a platform/build that advertises this capability or choose a supported command", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn element_not_found(ref_id: &str) -> Self { + Self::new( + ErrorCode::ElementNotFound, + format!("Element {ref_id} could not be resolved"), + ) + .with_suggestion("Re-run a snapshot to obtain fresh refs, then retry with the new ref") + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn timeout(message: impl Into) -> Self { + Self::new(ErrorCode::Timeout, message) + .with_suggestion("The target application may be busy or unresponsive") + } + + pub fn notification_not_found(index: usize) -> Self { + Self::new( + ErrorCode::NotificationNotFound, + format!("Notification at index {index} not found"), + ) + .with_suggestion( + "Notification may have been dismissed or expired. \ + Re-run a notification list to see current notifications \ + (CLI: list-notifications; FFI: ad_list_notifications)", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn app_unresponsive(app: &str) -> Self { + Self::new( + ErrorCode::AppUnresponsive, + format!("Application '{app}' is not responding"), + ) + .with_suggestion("Wait for the app to recover or force-quit it before retrying automation") + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn internal(message: impl Into) -> Self { + Self::new(ErrorCode::Internal, message) + } + + pub fn permission_denied() -> Self { + Self::new( + ErrorCode::PermDenied, + "Accessibility permission not granted", + ) + .with_suggestion( + "Open System Settings > Privacy & Security > Accessibility and add the app that launches agent-desktop", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn snapshot_not_found(snapshot_id: &str) -> Self { + Self::new( + ErrorCode::SnapshotNotFound, + format!("Snapshot '{snapshot_id}' not found"), + ) + .with_suggestion( + "Re-run a snapshot and retry with the returned snapshot_id \ + (CLI: snapshot, then pass --snapshot ; FFI: ad_snapshot then supply snapshot_id to ad_execute_by_ref)", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn policy_denied(message: impl Into) -> Self { + Self::new(ErrorCode::PolicyDenied, message) + .with_suggestion( + "Use an explicit mouse/focus command if physical interaction is intended", + ) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn policy_denied_for_policy(message: impl Into, policy: InteractionPolicy) -> Self { + Self::new(ErrorCode::PolicyDenied, message) + .with_suggestion(policy_denied_suggestion(policy)) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn renderer_accessibility_activation_required(message: impl Into) -> Self { + Self::new(ErrorCode::ActionNotSupported, message) + .with_details(serde_json::json!({ + "renderer_accessibility_activation_required": true, + })) + .with_disposition(DeliverySemantics::not_delivered()) + } + + pub fn requires_renderer_accessibility_activation(&self) -> bool { + self.details + .as_ref() + .and_then(|details| details.get("renderer_accessibility_activation_required")) + .and_then(Value::as_bool) + == Some(true) + } +} + +fn policy_denied_suggestion(policy: InteractionPolicy) -> &'static str { + if policy.allow_focus_steal && !policy.allow_cursor_move { + "Enable cursor movement in the interaction policy to permit cursor-driven actions \ + (CLI: --headed; FFI: set allow_cursor_move in the policy), \ + or use an explicit mouse command if physical input is intended" + } else if !policy.allow_focus_steal && !policy.allow_cursor_move { + "Headless mode allows only accessibility-backed actions; \ + enable physical interaction in the policy (CLI: --headed) only if cursor/focus movement is intended, \ + otherwise refresh the snapshot or target an element with the needed semantic action" + } else { + "Use an explicit mouse/focus command if physical interaction is intended" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retryability_is_typed_once_and_survives_diagnostic_enrichment() { + let error = AdapterError::new(ErrorCode::StaleRef, "stale") + .with_details(serde_json::json!({ "retryable": true })) + .with_details(serde_json::json!({ "phase": "resolve" })); + + assert!(error.is_explicitly_retryable()); + assert!(error.permits_retry_by_default()); + } + + #[test] + fn explicit_non_retryable_evidence_overrides_code_defaults() { + let error = AdapterError::new(ErrorCode::StaleRef, "stale") + .with_details(serde_json::json!({ "retryable": false })); + + assert!(!error.is_explicitly_retryable()); + assert!(!error.permits_retry_by_default()); + } + + #[test] + fn retryable_resolution_failure_characterizes_every_error_code() { + let all = [ + ErrorCode::PermDenied, + ErrorCode::ElementNotFound, + ErrorCode::AppNotFound, + ErrorCode::ActionFailed, + ErrorCode::ActionNotSupported, + ErrorCode::StaleRef, + ErrorCode::AmbiguousTarget, + ErrorCode::WindowNotFound, + ErrorCode::PlatformNotSupported, + ErrorCode::Timeout, + ErrorCode::InvalidArgs, + ErrorCode::NotificationNotFound, + ErrorCode::SnapshotNotFound, + ErrorCode::PolicyDenied, + ErrorCode::AppUnresponsive, + ErrorCode::Internal, + ]; + let retryable = [ + ErrorCode::StaleRef, + ErrorCode::AmbiguousTarget, + ErrorCode::Timeout, + ErrorCode::AppUnresponsive, + ]; + + for code in all { + let expected = retryable.contains(&code); + let error = AdapterError::new(code.clone(), "failure") + .with_details(serde_json::json!({ "retryable": true })); + assert_eq!(error.is_retryable_resolution_failure(), expected); + assert!(!AdapterError::new(code, "failure").is_retryable_resolution_failure()); + } + } +} diff --git a/crates/core/src/adapter_session.rs b/crates/core/src/adapter_session.rs new file mode 100644 index 0000000..e251efc --- /dev/null +++ b/crates/core/src/adapter_session.rs @@ -0,0 +1,15 @@ +use crate::AdapterError; + +/// A live adapter-native connection owned by a persistent host. +/// +/// The session may retain platform connection state, such as a Windows COM +/// apartment or Linux D-Bus connection, but must not retain resolved element +/// handles. Elements remain command-scoped so stale identity cannot escape the +/// resolve-and-release boundary. +pub trait AdapterSession: Send + Sync { + fn close(self: Box) -> Result<(), AdapterError>; +} + +#[cfg(test)] +#[path = "adapter_session_tests.rs"] +mod tests; diff --git a/crates/core/src/adapter_session_tests.rs b/crates/core/src/adapter_session_tests.rs new file mode 100644 index 0000000..0d33297 --- /dev/null +++ b/crates/core/src/adapter_session_tests.rs @@ -0,0 +1,36 @@ +use super::AdapterSession; +use crate::AdapterError; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +struct FlagSession { + closed: Arc, +} + +impl AdapterSession for FlagSession { + fn close(self: Box) -> Result<(), AdapterError> { + self.closed.store(true, Ordering::SeqCst); + Ok(()) + } +} + +#[test] +fn boxed_adapter_session_close_runs_through_dynamic_dispatch() { + let closed = Arc::new(AtomicBool::new(false)); + let session: Box = Box::new(FlagSession { + closed: Arc::clone(&closed), + }); + + session.close().unwrap(); + + assert!(closed.load(Ordering::SeqCst)); +} + +fn assert_send_sync() {} + +#[test] +fn boxed_adapter_session_is_send_and_sync() { + assert_send_sync::>(); +} diff --git a/crates/core/src/app_error.rs b/crates/core/src/app_error.rs new file mode 100644 index 0000000..9d8f0ff --- /dev/null +++ b/crates/core/src/app_error.rs @@ -0,0 +1,57 @@ +use thiserror::Error; + +use crate::{AdapterError, ErrorCode}; + +#[derive(Debug, Error)] +pub enum AppError { + #[error(transparent)] + Adapter(#[from] AdapterError), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + #[error("{0}")] + Internal(String), +} + +impl AppError { + pub fn code(&self) -> &str { + match self { + Self::Adapter(error) => error.code.as_str(), + Self::Io(_) | Self::Json(_) | Self::Internal(_) => "INTERNAL", + } + } + + pub fn suggestion(&self) -> Option<&str> { + match self { + Self::Adapter(error) => error.suggestion.as_deref(), + _ => None, + } + } + + pub fn stale_ref(ref_id: &str) -> Self { + Self::Adapter(AdapterError::stale_ref(ref_id)) + } + + pub fn invalid_input(message: impl Into) -> Self { + Self::Adapter( + AdapterError::new(ErrorCode::InvalidArgs, message) + .with_disposition(crate::DeliverySemantics::not_delivered()), + ) + } + + pub fn invalid_input_with_suggestion( + message: impl Into, + suggestion: impl Into, + ) -> Self { + Self::Adapter( + AdapterError::new(ErrorCode::InvalidArgs, message) + .with_suggestion(suggestion) + .with_disposition(crate::DeliverySemantics::not_delivered()), + ) + } +} + +#[cfg(test)] +#[path = "error_tests.rs"] +mod tests; diff --git a/crates/core/src/app_info.rs b/crates/core/src/app_info.rs new file mode 100644 index 0000000..c452c17 --- /dev/null +++ b/crates/core/src/app_info.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +use crate::ProcessId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AppInfo { + pub name: String, + pub pid: ProcessId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bundle_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub process_instance: Option, +} diff --git a/crates/core/src/app_lookup.rs b/crates/core/src/app_lookup.rs new file mode 100644 index 0000000..a4d88a3 --- /dev/null +++ b/crates/core/src/app_lookup.rs @@ -0,0 +1,178 @@ +use serde_json::json; + +use crate::{ + AdapterError, AppError, AppInfo, Deadline, ErrorCode, ProcessIdentity, + adapter::{PlatformAdapter, WindowFilter}, +}; + +pub(crate) fn resolve_app( + app: Option<&str>, + adapter: &dyn PlatformAdapter, + deadline: Deadline, +) -> Result { + if let Some(name) = app { + let candidates = adapter.list_apps_scoped(name, None, deadline)?; + return select_unique_app(candidates, name); + } + let focused = adapter.list_windows( + &WindowFilter { + focused_only: true, + app: None, + }, + deadline, + )?; + let window = match focused.as_slice() { + [] => { + return Err(AdapterError::new( + ErrorCode::AppNotFound, + "No focused application was found when --app was omitted", + ) + .into()); + } + [window] => window, + _ => { + return Err(AdapterError::ambiguous_target( + "A unique focused window is required when --app is omitted", + ) + .with_details(json!({ "focused_window_count": focused.len() })) + .into()); + } + }; + let instance = window + .process_instance + .as_deref() + .filter(|instance| !instance.is_empty()) + .ok_or_else(|| { + AdapterError::new( + ErrorCode::ActionNotSupported, + "Focused window has no process-instance identity", + ) + })?; + let same_pid = adapter + .list_apps_scoped(&window.app, None, deadline)? + .into_iter() + .filter(|candidate| candidate.pid == window.pid) + .collect::>(); + reject_incomplete_app_identity(&same_pid, &window.app)?; + let candidates = same_pid + .into_iter() + .filter(|candidate| candidate.process_instance.as_deref() == Some(instance)) + .collect(); + select_unique_app(candidates, &window.app) +} + +fn select_unique_app(mut candidates: Vec, label: &str) -> Result { + reject_incomplete_app_identity(&candidates, label)?; + match candidates.len() { + 0 => Err(AdapterError::new( + ErrorCode::AppNotFound, + format!("Application '{label}' was not found with exact process identity"), + ) + .into()), + 1 => Ok(candidates.swap_remove(0)), + _ => { + let summaries = candidates + .iter() + .take(10) + .map(|candidate| { + json!({ + "name": candidate.name, + "pid": candidate.pid, + "process_instance": candidate.process_instance, + }) + }) + .collect::>(); + Err(AdapterError::ambiguous_target(format!( + "Multiple application instances matched '{label}'" + )) + .with_details(json!({ + "candidate_count": candidates.len(), + "candidate_summaries_truncated": candidates.len() > summaries.len(), + "candidates": summaries, + })) + .into()) + } + } +} + +fn reject_incomplete_app_identity(candidates: &[AppInfo], label: &str) -> Result<(), AppError> { + let incomplete_count = candidates + .iter() + .filter(|candidate| { + candidate + .process_instance + .as_deref() + .is_none_or(str::is_empty) + }) + .count(); + if incomplete_count == 0 { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!("Application inventory for '{label}' has incomplete process identity"), + ) + .with_details(json!({ + "candidate_count": candidates.len(), + "incomplete_identity_count": incomplete_count, + })) + .into()) +} + +pub(crate) fn process_identity(app: &AppInfo) -> Result { + let instance = app + .process_instance + .as_deref() + .filter(|instance| !instance.is_empty()) + .ok_or_else(|| { + AdapterError::new( + ErrorCode::ActionNotSupported, + "Application has no process-instance identity", + ) + })?; + Ok(ProcessIdentity::new(app.pid, instance)) +} + +pub(crate) fn revalidate_app_for_mutation( + adapter: &dyn PlatformAdapter, + expected: &AppInfo, + deadline: Deadline, +) -> Result { + let expected_identity = process_identity(expected)?; + let same_pid = adapter + .list_apps_scoped(&expected.name, expected.bundle_id.as_deref(), deadline)? + .into_iter() + .filter(|candidate| candidate.pid == expected.pid) + .collect::>(); + reject_incomplete_app_identity(&same_pid, &expected.name)?; + let mut exact = same_pid + .into_iter() + .filter(|candidate| { + candidate.process_instance.as_deref() == Some(expected_identity.instance.as_str()) + && candidate.name.eq_ignore_ascii_case(&expected.name) + && expected.bundle_id.as_deref().is_none_or(|expected_bundle| { + candidate + .bundle_id + .as_deref() + .is_some_and(|actual| actual.eq_ignore_ascii_case(expected_bundle)) + }) + }) + .collect::>(); + match exact.len() { + 1 => Ok(exact.swap_remove(0)), + 0 => Err(AdapterError::new( + ErrorCode::StaleRef, + "Application identity changed before mutation", + ) + .into()), + _ => Err(AdapterError::ambiguous_target( + "Multiple exact application identities remained before mutation", + ) + .with_details(json!({ "candidate_count": exact.len() })) + .into()), + } +} + +#[cfg(test)] +#[path = "app_lookup_tests.rs"] +mod tests; diff --git a/crates/core/src/app_lookup_tests.rs b/crates/core/src/app_lookup_tests.rs new file mode 100644 index 0000000..59c346b --- /dev/null +++ b/crates/core/src/app_lookup_tests.rs @@ -0,0 +1,250 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +struct InventoryAdapter { + apps: Vec, + windows: Vec, +} + +impl ObservationOps for InventoryAdapter { + fn list_apps(&self, _deadline: Deadline) -> Result, AdapterError> { + Ok(self.apps.clone()) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: Deadline, + ) -> Result, AdapterError> { + Ok(self.windows.clone()) + } +} + +impl ActionOps for InventoryAdapter {} +impl InputOps for InventoryAdapter {} +impl SystemOps for InventoryAdapter {} + +fn app(instance: Option<&str>) -> AppInfo { + AppInfo { + name: "Example".into(), + pid: crate::ProcessId::new(42), + bundle_id: Some("com.example.app".into()), + process_instance: instance.map(str::to_string), + } +} + +fn app_with_pid(pid: u32, instance: Option<&str>) -> AppInfo { + AppInfo { + pid: crate::ProcessId::new(pid), + ..app(instance) + } +} + +struct ScopedInventorySpy { + apps: Vec, + global_calls: AtomicUsize, + scopes: Mutex)>>, +} + +impl ScopedInventorySpy { + fn new(apps: Vec) -> Self { + Self { + apps, + global_calls: AtomicUsize::new(0), + scopes: Mutex::new(Vec::new()), + } + } +} + +impl ObservationOps for ScopedInventorySpy { + fn list_apps(&self, _deadline: Deadline) -> Result, AdapterError> { + self.global_calls.fetch_add(1, Ordering::Relaxed); + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "unrelated global inventory failure", + )) + } + + fn list_apps_scoped( + &self, + name: &str, + bundle_id: Option<&str>, + _deadline: Deadline, + ) -> Result, AdapterError> { + self.scopes + .lock() + .unwrap() + .push((name.to_string(), bundle_id.map(str::to_string))); + Ok(self.apps.clone()) + } +} + +impl ActionOps for ScopedInventorySpy {} +impl InputOps for ScopedInventorySpy {} +impl SystemOps for ScopedInventorySpy {} + +struct ScopedPermissionAdapter; + +impl ObservationOps for ScopedPermissionAdapter { + fn list_apps(&self, _deadline: Deadline) -> Result, AdapterError> { + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "unrelated global inventory failure", + )) + } + + fn list_apps_scoped( + &self, + _name: &str, + _bundle_id: Option<&str>, + _deadline: Deadline, + ) -> Result, AdapterError> { + Err(AdapterError::new( + ErrorCode::PermDenied, + "permission denied reading requested process identity", + )) + } +} + +impl ActionOps for ScopedPermissionAdapter {} +impl InputOps for ScopedPermissionAdapter {} +impl SystemOps for ScopedPermissionAdapter {} + +#[test] +fn named_resolution_uses_scoped_inventory_when_global_inventory_fails() { + let adapter = ScopedInventorySpy::new(vec![app(Some("generation-a"))]); + + let resolved = resolve_app(Some("example"), &adapter, Deadline::standard().unwrap()).unwrap(); + + assert_eq!(resolved.pid, 42); + assert_eq!(adapter.global_calls.load(Ordering::Relaxed), 0); + assert_eq!( + *adapter.scopes.lock().unwrap(), + vec![("example".to_string(), None)] + ); +} + +#[test] +fn mutation_revalidation_uses_name_and_bundle_scope() { + let expected = app(Some("generation-a")); + let mut differently_cased = expected.clone(); + differently_cased.name = "example".into(); + differently_cased.bundle_id = Some("COM.EXAMPLE.APP".into()); + let adapter = ScopedInventorySpy::new(vec![differently_cased]); + + let resolved = + revalidate_app_for_mutation(&adapter, &expected, Deadline::standard().unwrap()).unwrap(); + + assert_eq!(resolved.pid, 42); + assert_eq!(adapter.global_calls.load(Ordering::Relaxed), 0); + assert_eq!( + *adapter.scopes.lock().unwrap(), + vec![("Example".to_string(), Some("com.example.app".to_string()))] + ); +} + +#[test] +fn scoped_target_permission_failure_is_preserved() { + let error = resolve_app( + Some("Example"), + &ScopedPermissionAdapter, + Deadline::standard().unwrap(), + ) + .unwrap_err(); + + assert_eq!(error.code(), "PERM_DENIED"); +} + +#[test] +fn default_scoped_inventory_filters_exact_name_and_bundle_case_insensitively() { + let mut wrong_name = app_with_pid(43, Some("generation-b")); + wrong_name.name = "Example Helper".into(); + let mut wrong_bundle = app_with_pid(44, Some("generation-c")); + wrong_bundle.bundle_id = Some("com.example.other".into()); + let adapter = InventoryAdapter { + apps: vec![app(Some("generation-a")), wrong_name, wrong_bundle], + windows: Vec::new(), + }; + + let scoped = adapter + .list_apps_scoped( + "example", + Some("COM.EXAMPLE.APP"), + Deadline::standard().unwrap(), + ) + .unwrap(); + + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].pid, 42); +} + +#[test] +fn named_resolution_rejects_any_matching_candidate_without_generation() { + let adapter = InventoryAdapter { + apps: vec![app(Some("generation-a")), app(None)], + windows: Vec::new(), + }; + + let error = resolve_app(Some("Example"), &adapter, Deadline::standard().unwrap()).unwrap_err(); + + assert_eq!(error.code(), "ACTION_NOT_SUPPORTED"); +} + +#[test] +fn named_resolution_keeps_same_name_multiple_pids_ambiguous() { + let adapter = InventoryAdapter { + apps: vec![ + app_with_pid(42, Some("generation-a")), + app_with_pid(43, Some("generation-b")), + ], + windows: Vec::new(), + }; + + let error = resolve_app(Some("Example"), &adapter, Deadline::standard().unwrap()).unwrap_err(); + + assert_eq!(error.code(), "AMBIGUOUS_TARGET"); +} + +#[test] +fn omitted_app_with_no_focused_window_is_not_found() { + let adapter = InventoryAdapter { + apps: vec![app(Some("generation-a"))], + windows: Vec::new(), + }; + + let error = resolve_app(None, &adapter, Deadline::standard().unwrap()).unwrap_err(); + + assert_eq!(error.code(), "APP_NOT_FOUND"); +} + +#[test] +fn mutation_revalidation_rejects_pid_reuse() { + let expected = app(Some("generation-a")); + let adapter = InventoryAdapter { + apps: vec![app(Some("generation-b"))], + windows: Vec::new(), + }; + + let error = revalidate_app_for_mutation(&adapter, &expected, Deadline::standard().unwrap()) + .unwrap_err(); + + assert_eq!(error.code(), "STALE_REF"); +} + +#[test] +fn mutation_revalidation_rejects_missing_generation() { + let expected = app(Some("generation-a")); + let adapter = InventoryAdapter { + apps: vec![app(None)], + windows: Vec::new(), + }; + + let error = revalidate_app_for_mutation(&adapter, &expected, Deadline::standard().unwrap()) + .unwrap_err(); + + assert_eq!(error.code(), "ACTION_NOT_SUPPORTED"); +} diff --git a/crates/core/src/capability.rs b/crates/core/src/capability.rs index a1ad11e..5a82f64 100644 --- a/crates/core/src/capability.rs +++ b/crates/core/src/capability.rs @@ -33,34 +33,17 @@ pub fn for_action(action: &Action) -> &'static [&'static str] { Action::Select(_) => &[SELECT, CLICK], Action::Toggle => &[TOGGLE, CLICK], Action::Check | Action::Uncheck => &[TOGGLE, CLICK], - Action::Scroll(_, _) => &[SCROLL, SCROLL_TO], + Action::Scroll(_, _) => &[SCROLL], Action::ScrollTo => &[SCROLL_TO], Action::PressKey(_) => &[PRESS_KEY], Action::KeyDown(_) => &[KEY_DOWN], Action::KeyUp(_) => &[KEY_UP], - Action::TypeText(_) => &[TYPE_TEXT, SET_VALUE], + Action::TypeText(_) => &[TYPE_TEXT], Action::Hover => &[HOVER], Action::Drag(_) => &[DRAG], } } -pub fn defaults_for_role(role: &str) -> Vec { - let capabilities: &[&str] = match role { - "button" | "link" | "menuitem" | "tab" | "radiobutton" => &[CLICK], - "textfield" | "incrementor" => &[CLICK, SET_VALUE, SET_FOCUS], - "checkbox" => &[CLICK, TOGGLE], - "combobox" => &[CLICK, SELECT], - "treeitem" => &[CLICK, EXPAND, COLLAPSE], - "slider" => &[SET_VALUE], - "cell" => &[CLICK], - _ => &[CLICK], - }; - capabilities - .iter() - .map(|capability| (*capability).to_string()) - .collect() -} - pub fn contains(actions: &[String], capability: &str) -> bool { actions.iter().any(|action| action == capability) } @@ -71,10 +54,22 @@ pub fn contains_any(actions: &[String], capabilities: &[&str]) -> bool { .any(|capability| contains(actions, capability)) } +pub(crate) fn supports_direct_semantic_pointer_delivery( + action: &Action, + available_actions: &[String], +) -> bool { + let capability = match action { + Action::Click => CLICK, + Action::RightClick => RIGHT_CLICK, + _ => return false, + }; + contains(available_actions, capability) +} + #[cfg(test)] mod tests { use super::*; - use crate::action::{Direction, KeyCombo}; + use crate::{Direction, KeyCombo}; #[test] fn action_capabilities_are_declared_in_one_place() { @@ -82,10 +77,7 @@ mod tests { assert_eq!(for_action(&Action::RightClick), &[RIGHT_CLICK]); assert_eq!(for_action(&Action::SetValue("x".into())), &[SET_VALUE]); assert_eq!(for_action(&Action::Clear), &[SET_VALUE]); - assert_eq!( - for_action(&Action::Scroll(Direction::Down, 1)), - &[SCROLL, SCROLL_TO] - ); + assert_eq!(for_action(&Action::Scroll(Direction::Down, 1)), &[SCROLL]); assert_eq!( for_action(&Action::PressKey(KeyCombo { key: "A".into(), @@ -96,19 +88,25 @@ mod tests { } #[test] - fn role_defaults_are_declared_in_one_place() { - assert_eq!(defaults_for_role("button"), strings(&[CLICK])); - assert_eq!( - defaults_for_role("textfield"), - strings(&[CLICK, SET_VALUE, SET_FOCUS]) - ); - assert_eq!( - defaults_for_role("treeitem"), - strings(&[CLICK, EXPAND, COLLAPSE]) - ); - } + fn direct_semantic_pointer_delivery_requires_an_exact_capability() { + let click = vec![CLICK.to_string()]; + let right_click = vec![RIGHT_CLICK.to_string()]; - fn strings(values: &[&str]) -> Vec { - values.iter().map(|value| (*value).to_string()).collect() + assert!(supports_direct_semantic_pointer_delivery( + &Action::Click, + &click + )); + assert!(supports_direct_semantic_pointer_delivery( + &Action::RightClick, + &right_click + )); + assert!(!supports_direct_semantic_pointer_delivery( + &Action::DoubleClick, + &click + )); + assert!(!supports_direct_semantic_pointer_delivery( + &Action::TripleClick, + &click + )); } } diff --git a/crates/core/src/clipboard_content.rs b/crates/core/src/clipboard_content.rs new file mode 100644 index 0000000..6163a82 --- /dev/null +++ b/crates/core/src/clipboard_content.rs @@ -0,0 +1,22 @@ +use crate::{ClipboardFormat, ImageBuffer}; + +#[derive(Debug)] +pub enum ClipboardContent { + Text(String), + Image(ImageBuffer), + FileUrls(Vec), +} + +impl ClipboardContent { + pub fn format(&self) -> ClipboardFormat { + match self { + ClipboardContent::Text(_) => ClipboardFormat::Text, + ClipboardContent::Image(_) => ClipboardFormat::Image, + ClipboardContent::FileUrls(_) => ClipboardFormat::FileUrls, + } + } +} + +#[cfg(test)] +#[path = "clipboard_content_tests.rs"] +mod tests; diff --git a/crates/core/src/clipboard_content_tests.rs b/crates/core/src/clipboard_content_tests.rs new file mode 100644 index 0000000..53a4739 --- /dev/null +++ b/crates/core/src/clipboard_content_tests.rs @@ -0,0 +1,34 @@ +use super::*; + +#[test] +fn text_content_reports_text_format() { + let content = ClipboardContent::Text("hello".into()); + assert_eq!(content.format(), ClipboardFormat::Text); + assert_eq!(ClipboardFormat::Text.as_str(), "text"); +} + +#[test] +fn image_content_reports_image_format() { + let content = ClipboardContent::Image(ImageBuffer { + data: vec![0u8; 4], + format: crate::ImageFormat::Png, + width: 2, + height: 2, + scale_factor: 1.0, + }); + assert_eq!(content.format(), ClipboardFormat::Image); + assert_eq!(ClipboardFormat::Image.as_str(), "image"); +} + +#[test] +fn file_urls_content_reports_file_urls_format() { + let content = ClipboardContent::FileUrls(vec!["/tmp/a.txt".into()]); + assert_eq!(content.format(), ClipboardFormat::FileUrls); + assert_eq!(ClipboardFormat::FileUrls.as_str(), "file_urls"); +} + +#[test] +fn auto_format_tag_is_distinct_from_content_formats() { + assert_eq!(ClipboardFormat::Auto.as_str(), "auto"); + assert_ne!(ClipboardFormat::Auto, ClipboardFormat::Text); +} diff --git a/crates/core/src/clipboard_format.rs b/crates/core/src/clipboard_format.rs new file mode 100644 index 0000000..74cb904 --- /dev/null +++ b/crates/core/src/clipboard_format.rs @@ -0,0 +1,18 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClipboardFormat { + Auto, + Text, + Image, + FileUrls, +} + +impl ClipboardFormat { + pub fn as_str(&self) -> &'static str { + match self { + ClipboardFormat::Auto => "auto", + ClipboardFormat::Text => "text", + ClipboardFormat::Image => "image", + ClipboardFormat::FileUrls => "file_urls", + } + } +} diff --git a/crates/core/src/commands/batch.rs b/crates/core/src/commands/batch.rs index 080d087..e3fad28 100644 --- a/crates/core/src/commands/batch.rs +++ b/crates/core/src/commands/batch.rs @@ -1,4 +1,4 @@ -use crate::error::AppError; +use crate::AppError; use serde::Deserialize; use serde_json::Value; diff --git a/crates/core/src/commands/check.rs b/crates/core/src/commands/check.rs index 0ba6cf0..4516479 100644 --- a/crates/core/src/commands/check.rs +++ b/crates/core/src/commands/check.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/clear.rs b/crates/core/src/commands/clear.rs index 4f4a676..8a57e31 100644 --- a/crates/core/src/commands/clear.rs +++ b/crates/core/src/commands/clear.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/click.rs b/crates/core/src/commands/click.rs index 6dd9778..06338a7 100644 --- a/crates/core/src/commands/click.rs +++ b/crates/core/src/commands/click.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/clipboard_clear.rs b/crates/core/src/commands/clipboard_clear.rs index 1e59be9..7155143 100644 --- a/crates/core/src/commands/clipboard_clear.rs +++ b/crates/core/src/commands/clipboard_clear.rs @@ -1,7 +1,8 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{AppError, adapter::PlatformAdapter}; use serde_json::{Value, json}; pub fn execute(adapter: &dyn PlatformAdapter) -> Result { - adapter.clear_clipboard()?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + adapter.clear_clipboard(&lease)?; Ok(json!({ "cleared": true })) } diff --git a/crates/core/src/commands/clipboard_get.rs b/crates/core/src/commands/clipboard_get.rs index d7b6a89..1712f5c 100644 --- a/crates/core/src/commands/clipboard_get.rs +++ b/crates/core/src/commands/clipboard_get.rs @@ -1,7 +1,101 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{ + AppError, ClipboardContent, ClipboardFormat, ImageBuffer, + adapter::PlatformAdapter, + context::CommandContext, + refs::{write_private_file, write_user_file}, + refs_store::{ + STALE_TMP_MAX_AGE, + prune::{is_orphaned_tmp_file, remove_stale_files_in_dir}, + }, + session, +}; use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; -pub fn execute(adapter: &dyn PlatformAdapter) -> Result { - let text = adapter.get_clipboard()?; - Ok(json!({ "text": text })) +pub struct ClipboardGetArgs { + pub format: Option, + pub out: Option, } + +static IMAGE_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +const CLIPBOARD_IMAGE_MAX_AGE: Duration = Duration::from_secs(60 * 60); + +pub fn execute( + args: ClipboardGetArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let format = args.format.unwrap_or(ClipboardFormat::Text); + let Some(content) = adapter.get_clipboard_content(format, crate::Deadline::standard()?)? else { + return Ok(json!({ "type": format.as_str(), "found": false })); + }; + match content { + ClipboardContent::Text(text) => Ok(json!({ "type": "text", "text": text })), + ClipboardContent::FileUrls(file_urls) => { + Ok(json!({ "type": "file_urls", "file_urls": file_urls })) + } + ClipboardContent::Image(image) => write_image(image, args.out, context), + } +} + +fn write_image( + image: ImageBuffer, + out: Option, + context: &CommandContext, +) -> Result { + let path = match out { + Some(path) => { + write_user_file(&path, &image.data)?; + path + } + None => { + let path = default_clipboard_image_path(context)?; + write_private_file(&path, &image.data)?; + path + } + }; + Ok(json!({ + "type": "image", + "path": path.to_string_lossy(), + "width": image.width, + "height": image.height, + })) +} + +fn default_clipboard_image_path(context: &CommandContext) -> Result { + let dir = match context.session_id() { + Some(id) => session::session_dir(id)?.join("clipboard"), + None => { + let dir = session::agent_desktop_dir()?.join("tmp"); + prune_sessionless_clipboard_tmp_dir(&dir); + dir + } + }; + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let seq = IMAGE_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + Ok(dir.join(format!( + "clipboard-{}-{nanos}-{seq}.png", + std::process::id() + ))) +} + +fn prune_sessionless_clipboard_tmp_dir(dir: &Path) { + remove_stale_files_in_dir(dir, STALE_TMP_MAX_AGE, is_orphaned_tmp_file); + remove_stale_files_in_dir(dir, CLIPBOARD_IMAGE_MAX_AGE, is_clipboard_image_file); +} + +fn is_clipboard_image_file(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("clipboard-") && name.ends_with(".png")) +} + +#[cfg(test)] +#[path = "clipboard_get_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/clipboard_get_tests.rs b/crates/core/src/commands/clipboard_get_tests.rs new file mode 100644 index 0000000..a823749 --- /dev/null +++ b/crates/core/src/commands/clipboard_get_tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::AdapterError; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::refs_test_support::HomeGuard; +use std::sync::Mutex; + +struct LocalDouble { + response: Mutex, AdapterError>>>, + seen_format: Mutex>, +} + +impl LocalDouble { + fn returning(result: Result, AdapterError>) -> Self { + Self { + response: Mutex::new(Some(result)), + seen_format: Mutex::new(None), + } + } +} + +impl ObservationOps for LocalDouble {} +impl ActionOps for LocalDouble {} +impl SystemOps for LocalDouble {} + +impl InputOps for LocalDouble { + fn get_clipboard_content( + &self, + format: ClipboardFormat, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + *self.seen_format.lock().unwrap() = Some(format); + self.response + .lock() + .unwrap() + .take() + .expect("double invoked more than once in a single test") + } +} + +fn no_session_context() -> CommandContext { + CommandContext::default() +} + +#[test] +fn omitted_format_defaults_to_text_not_auto() { + let double = LocalDouble::returning(Ok(Some(ClipboardContent::Text("hi there".into())))); + let out = execute( + ClipboardGetArgs { + format: None, + out: None, + }, + &double, + &no_session_context(), + ) + .unwrap(); + assert_eq!( + *double.seen_format.lock().unwrap(), + Some(ClipboardFormat::Text) + ); + assert_eq!(out["type"], "text"); + assert_eq!(out["text"], "hi there"); +} + +#[test] +fn missing_content_for_requested_format_is_structured_not_found_not_an_error() { + let double = LocalDouble::returning(Ok(None)); + let out = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Image), + out: None, + }, + &double, + &no_session_context(), + ) + .expect("must return Ok(..) not panic or bubble a hard error"); + assert_eq!(out["type"], "image"); + assert_eq!(out["found"], false); +} + +#[test] +fn file_urls_variant_serializes_list() { + let double = LocalDouble::returning(Ok(Some(ClipboardContent::FileUrls(vec![ + "/tmp/a.txt".into(), + "/tmp/b.txt".into(), + ])))); + let out = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::FileUrls), + out: None, + }, + &double, + &no_session_context(), + ) + .unwrap(); + assert_eq!(out["type"], "file_urls"); + assert_eq!( + out["file_urls"], + serde_json::json!(["/tmp/a.txt", "/tmp/b.txt"]) + ); +} + +#[test] +fn auto_format_request_is_relayed_to_adapter_unmodified() { + let double = LocalDouble::returning(Ok(Some(ClipboardContent::Text("auto text".into())))); + let _ = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Auto), + out: None, + }, + &double, + &no_session_context(), + ) + .unwrap(); + assert_eq!( + *double.seen_format.lock().unwrap(), + Some(ClipboardFormat::Auto) + ); +} + +#[test] +fn image_variant_with_explicit_out_writes_that_path() { + let _home = HomeGuard::new(); + let dir = std::env::temp_dir().join(format!( + "agent-desktop-clipboard-get-test-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let out_path = dir.join("explicit.png"); + + let double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer { + data: vec![1, 2, 3, 4], + format: crate::ImageFormat::Png, + width: 10, + height: 5, + scale_factor: 1.0, + })))); + let out = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Image), + out: Some(out_path.clone()), + }, + &double, + &no_session_context(), + ) + .unwrap(); + + assert_eq!(out["type"], "image"); + assert_eq!(out["path"], out_path.to_string_lossy().into_owned()); + assert_eq!(out["width"], 10); + assert_eq!(out["height"], 5); + assert_eq!(std::fs::read(&out_path).unwrap(), vec![1, 2, 3, 4]); + + let _ = std::fs::remove_dir_all(&dir); +} + +fn write_with_age(path: &std::path::Path, contents: &[u8], age: Duration) { + std::fs::write(path, contents).unwrap(); + let file = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + file.set_modified(SystemTime::now() - age).unwrap(); +} + +#[test] +fn clipboard_image_predicate_matches_only_the_generated_shape() { + assert!(is_clipboard_image_file(Path::new( + "clipboard-123-456-0.png" + ))); + assert!(!is_clipboard_image_file(Path::new( + "clipboard-123-456-0.png.tmp" + ))); + assert!(!is_clipboard_image_file(Path::new("other.png"))); + assert!(!is_clipboard_image_file(Path::new("clipboard-123.txt"))); +} + +#[test] +fn sessionless_prune_removes_stale_matches_keeps_fresh_and_unrelated() { + let _home = HomeGuard::new(); + let dir = session::agent_desktop_dir().unwrap().join("tmp"); + std::fs::create_dir_all(&dir).unwrap(); + + let stale_png = dir.join("clipboard-1-1-0.png"); + let stale_tmp = dir.join("orphan-write.tmp"); + let fresh_png = dir.join("clipboard-2-2-0.png"); + let unrelated = dir.join("notes.txt"); + write_with_age( + &stale_png, + b"old", + CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1), + ); + write_with_age( + &stale_tmp, + b"old", + STALE_TMP_MAX_AGE + Duration::from_secs(1), + ); + write_with_age(&fresh_png, b"new", Duration::from_secs(1)); + write_with_age( + &unrelated, + b"keep me", + CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1), + ); + + prune_sessionless_clipboard_tmp_dir(&dir); + + assert!( + !stale_png.exists(), + "clipboard png older than its TTL must be pruned" + ); + assert!( + !stale_tmp.exists(), + "orphaned tmp file older than its TTL must be pruned" + ); + assert!( + fresh_png.exists(), + "clipboard png within its TTL must survive the sweep" + ); + assert!( + unrelated.exists(), + "a file outside the clipboard-*.png / *.tmp shapes must never be touched" + ); +} + +#[test] +fn sweep_never_removes_a_directory_even_if_it_matches_the_shape() { + let _home = HomeGuard::new(); + let dir = session::agent_desktop_dir().unwrap().join("tmp"); + let fake_dir = dir.join("clipboard-0-0-0.png"); + std::fs::create_dir_all(&fake_dir).unwrap(); + + remove_stale_files_in_dir(&dir, Duration::ZERO, is_clipboard_image_file); + + assert!( + fake_dir.is_dir(), + "the sweep must never remove a directory, even one matching the file shape" + ); +} + +#[test] +fn consecutive_sessionless_image_captures_keep_the_prior_capture() { + let _home = HomeGuard::new(); + let first_double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer { + data: vec![1], + format: crate::ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + })))); + let first = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Image), + out: None, + }, + &first_double, + &no_session_context(), + ) + .unwrap(); + let first_path = PathBuf::from(first["path"].as_str().unwrap()); + + let second_double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer { + data: vec![2], + format: crate::ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + })))); + let _ = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Image), + out: None, + }, + &second_double, + &no_session_context(), + ) + .unwrap(); + + assert!( + first_path.exists(), + "a clipboard image captured moments ago must not be pruned by the very next capture" + ); +} + +#[test] +fn image_variant_without_out_prunes_stale_sessionless_artifacts() { + let _home = HomeGuard::new(); + let dir = session::agent_desktop_dir().unwrap().join("tmp"); + std::fs::create_dir_all(&dir).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for path in [dir.parent().unwrap(), dir.as_path()] { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + } + + let stale_png = dir.join("clipboard-9-9-0.png"); + let stale_tmp = dir.join("orphan.tmp"); + let unrelated = dir.join("keep.txt"); + write_with_age( + &stale_png, + b"old capture", + CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1), + ); + write_with_age( + &stale_tmp, + b"old", + STALE_TMP_MAX_AGE + Duration::from_secs(1), + ); + write_with_age( + &unrelated, + b"keep", + CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1), + ); + + let double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer { + data: vec![7, 7, 7], + format: crate::ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + })))); + let out = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Image), + out: None, + }, + &double, + &no_session_context(), + ) + .unwrap(); + let new_path = PathBuf::from(out["path"].as_str().unwrap()); + + assert!( + new_path.exists(), + "the image just written by this call must survive its own write" + ); + assert!( + !stale_png.exists(), + "a stale clipboard png must be pruned by the next capture" + ); + assert!( + !stale_tmp.exists(), + "a stale orphaned tmp file must be pruned by the next capture" + ); + assert!( + unrelated.exists(), + "a non-matching file must never be touched" + ); +} + +#[cfg(unix)] +#[test] +fn image_variant_without_out_writes_private_0600_file_under_session_dir() { + use std::os::unix::fs::PermissionsExt; + + let _home = HomeGuard::new(); + let context = CommandContext::new(Some("clip-get-test-session".into()), None, false).unwrap(); + + let double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer { + data: vec![9, 9, 9], + format: crate::ImageFormat::Png, + width: 3, + height: 3, + scale_factor: 1.0, + })))); + let out = execute( + ClipboardGetArgs { + format: Some(ClipboardFormat::Image), + out: None, + }, + &double, + &context, + ) + .unwrap(); + + let path = out["path"].as_str().expect("path field").to_string(); + let session_dir = session::session_dir("clip-get-test-session") + .unwrap() + .join("clipboard"); + assert!( + std::path::Path::new(&path).starts_with(&session_dir), + "default image path {path} must live under the session's clipboard dir {}", + session_dir.display() + ); + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, 0o600, + "clipboard image temp file must be 0600, got {mode:o}" + ); +} diff --git a/crates/core/src/commands/clipboard_set.rs b/crates/core/src/commands/clipboard_set.rs index f25ab94..01197cf 100644 --- a/crates/core/src/commands/clipboard_set.rs +++ b/crates/core/src/commands/clipboard_set.rs @@ -1,7 +1,83 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{ + AppError, ClipboardContent, ImageBuffer, ImageFormat, adapter::PlatformAdapter, + parse_png_dimensions, +}; use serde_json::{Value, json}; +use std::path::PathBuf; -pub fn execute(text: String, adapter: &dyn PlatformAdapter) -> Result { - adapter.set_clipboard(&text)?; - Ok(json!({ "ok": true })) +pub struct ClipboardSetArgs { + pub text: Option, + pub image: Option, + pub file_urls: Vec, } + +pub fn execute(args: ClipboardSetArgs, adapter: &dyn PlatformAdapter) -> Result { + let content = build_content(args)?; + let format = content.format(); + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + adapter.set_clipboard_content(&content, &lease)?; + Ok(json!({ "ok": true, "type": format.as_str() })) +} + +fn build_content(args: ClipboardSetArgs) -> Result { + if !args.file_urls.is_empty() { + return Ok(ClipboardContent::FileUrls(validate_file_urls( + &args.file_urls, + )?)); + } + if let Some(path) = args.image { + let data = + crate::private_file::read_regular_bounded(&path, crate::MAX_PNG_INPUT_BYTES as u64) + .map_err(image_read_error)?; + let Some((width, height)) = parse_png_dimensions(&data) else { + return Err(AppError::invalid_input("--image file is not a valid PNG")); + }; + return Ok(ClipboardContent::Image(ImageBuffer { + data, + format: ImageFormat::Png, + width, + height, + scale_factor: 1.0, + })); + } + Ok(ClipboardContent::Text(args.text.unwrap_or_default())) +} + +fn image_read_error(error: std::io::Error) -> AppError { + if matches!( + error.kind(), + std::io::ErrorKind::InvalidData + | std::io::ErrorKind::NotFound + | std::io::ErrorKind::PermissionDenied + ) { + return AppError::invalid_input(format!( + "--image must be a local regular PNG no larger than {} MiB", + crate::MAX_PNG_INPUT_BYTES / (1024 * 1024) + )); + } + AppError::Io(error) +} + +/// Reports missing `--file-url` entries by count and index only — the +/// entries themselves may be sensitive paths and this error can reach a +/// trace file. +fn validate_file_urls(urls: &[String]) -> Result, AppError> { + let missing: Vec = urls + .iter() + .enumerate() + .filter(|(_, path)| !std::path::Path::new(path).exists()) + .map(|(index, _)| index) + .collect(); + if !missing.is_empty() { + return Err(AppError::invalid_input(format!( + "{} of {} --file-url entries do not exist on disk (indexes: {missing:?})", + missing.len(), + urls.len() + ))); + } + Ok(urls.to_vec()) +} + +#[cfg(test)] +#[path = "clipboard_set_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/clipboard_set_tests.rs b/crates/core/src/commands/clipboard_set_tests.rs new file mode 100644 index 0000000..0059eff --- /dev/null +++ b/crates/core/src/commands/clipboard_set_tests.rs @@ -0,0 +1,246 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{AdapterError, ErrorCode}; +use std::sync::Mutex; + +struct LocalDouble { + seen: Mutex>, +} + +impl LocalDouble { + fn new() -> Self { + Self { + seen: Mutex::new(None), + } + } +} + +impl ObservationOps for LocalDouble {} +impl ActionOps for LocalDouble {} +impl SystemOps for LocalDouble { + crate::adapter::guarded_interaction_lease!(); +} + +impl InputOps for LocalDouble { + fn set_clipboard_content( + &self, + content: &ClipboardContent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + let stored = match content { + ClipboardContent::Text(text) => ClipboardContent::Text(text.clone()), + ClipboardContent::Image(image) => ClipboardContent::Image(ImageBuffer { + data: image.data.clone(), + format: crate::ImageFormat::Png, + width: image.width, + height: image.height, + scale_factor: image.scale_factor, + }), + ClipboardContent::FileUrls(urls) => ClipboardContent::FileUrls(urls.clone()), + }; + *self.seen.lock().unwrap() = Some(stored); + Ok(()) + } +} + +#[test] +fn default_with_only_text_writes_text_content() { + let double = LocalDouble::new(); + let out = execute( + ClipboardSetArgs { + text: Some("hello clipboard".into()), + image: None, + file_urls: vec![], + }, + &double, + ) + .unwrap(); + assert_eq!(out["type"], "text"); + match double.seen.lock().unwrap().as_ref().unwrap() { + ClipboardContent::Text(text) => assert_eq!(text, "hello clipboard"), + _ => panic!("expected Text content"), + } +} + +#[test] +fn omitted_text_defaults_to_empty_string_not_error() { + let double = LocalDouble::new(); + let out = execute( + ClipboardSetArgs { + text: None, + image: None, + file_urls: vec![], + }, + &double, + ) + .unwrap(); + assert_eq!(out["type"], "text"); + match double.seen.lock().unwrap().as_ref().unwrap() { + ClipboardContent::Text(text) => assert_eq!(text, ""), + _ => panic!("expected Text content"), + } +} + +#[test] +fn missing_file_url_reports_invalid_args_with_count_and_index_only() { + let existing = std::env::temp_dir().join(format!( + "agent-desktop-clipboard-set-test-{}.txt", + std::process::id() + )); + std::fs::write(&existing, b"present").unwrap(); + + let double = LocalDouble::new(); + let err = execute( + ClipboardSetArgs { + text: None, + image: None, + file_urls: vec![ + existing.to_string_lossy().into_owned(), + "/definitely/does/not/exist/secret-project-name.txt".into(), + ], + }, + &double, + ) + .unwrap_err(); + + let _ = std::fs::remove_file(&existing); + + assert_eq!(err.code(), ErrorCode::InvalidArgs.as_str()); + let message = err.to_string(); + assert!( + message.contains("1 of 2"), + "message should report count, got: {message}" + ); + assert!( + message.contains("[1]"), + "message should report the missing entry's index (1), got: {message}" + ); + assert!( + !message.contains("secret-project-name"), + "message must not leak the missing path's content, got: {message}" + ); + assert!( + double.seen.lock().unwrap().is_none(), + "adapter must not be called on validation failure" + ); +} + +#[test] +fn image_flag_reads_bytes_and_dimensions_from_file() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-clipboard-set-image-{}.png", + std::process::id() + )); + let bytes = vec![ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 4, + 0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 100, 248, 15, 0, 1, 5, + 1, 1, 39, 24, 227, 102, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ]; + std::fs::write(&path, &bytes).unwrap(); + + let double = LocalDouble::new(); + let out = execute( + ClipboardSetArgs { + text: None, + image: Some(path.clone()), + file_urls: vec![], + }, + &double, + ) + .unwrap(); + + let _ = std::fs::remove_file(&path); + + assert_eq!(out["type"], "image"); + match double.seen.lock().unwrap().as_ref().unwrap() { + ClipboardContent::Image(image) => { + assert_eq!(image.width, 1); + assert_eq!(image.height, 1); + assert_eq!(image.data, bytes); + } + _ => panic!("expected Image content"), + } +} + +#[test] +fn missing_image_path_reports_invalid_args() { + let missing = std::env::temp_dir().join(format!( + "agent-desktop-missing-clipboard-image-{}.png", + crate::refs::new_snapshot_id() + )); + + let error = build_content(ClipboardSetArgs { + text: None, + image: Some(missing), + file_urls: Vec::new(), + }) + .unwrap_err(); + + assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str()); +} + +#[test] +fn file_urls_take_priority_over_text_and_image() { + let existing = std::env::temp_dir().join(format!( + "agent-desktop-clipboard-set-priority-{}.txt", + std::process::id() + )); + std::fs::write(&existing, b"present").unwrap(); + + let double = LocalDouble::new(); + let out = execute( + ClipboardSetArgs { + text: Some("ignored".into()), + image: None, + file_urls: vec![existing.to_string_lossy().into_owned()], + }, + &double, + ) + .unwrap(); + + let _ = std::fs::remove_file(&existing); + + assert_eq!(out["type"], "file_urls"); + match double.seen.lock().unwrap().as_ref().unwrap() { + ClipboardContent::FileUrls(urls) => assert_eq!(urls.len(), 1), + _ => panic!("expected FileUrls content"), + } +} + +#[cfg(unix)] +#[test] +fn image_rejects_fifo_without_blocking() { + use std::ffi::CString; + + let path = std::env::temp_dir().join(format!( + "agent-desktop-clipboard-set-fifo-{}", + crate::refs::new_snapshot_id() + )); + let c_path = CString::new(path.to_string_lossy().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + let started = std::time::Instant::now(); + + let error = build_content(ClipboardSetArgs { + text: None, + image: Some(path.clone()), + file_urls: Vec::new(), + }) + .unwrap_err(); + + assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str()); + assert!(started.elapsed() < std::time::Duration::from_secs(1)); + std::fs::remove_file(path).unwrap(); +} + +#[cfg(unix)] +#[test] +fn image_rejects_device_files() { + let error = build_content(ClipboardSetArgs { + text: None, + image: Some(PathBuf::from("/dev/null")), + file_urls: Vec::new(), + }) + .unwrap_err(); + + assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str()); +} diff --git a/crates/core/src/commands/close_app.rs b/crates/core/src/commands/close_app.rs index 3d6cdd7..af17951 100644 --- a/crates/core/src/commands/close_app.rs +++ b/crates/core/src/commands/close_app.rs @@ -1,4 +1,4 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{AppError, adapter::PlatformAdapter}; use serde_json::{Value, json}; pub struct CloseAppArgs { @@ -8,23 +8,41 @@ pub struct CloseAppArgs { pub fn execute(args: CloseAppArgs, adapter: &dyn PlatformAdapter) -> Result { if adapter.is_protected_process(&args.app) { - return Err(AppError::invalid_input_with_suggestion( - format!( - "'{}' is a protected system process and cannot be closed", - args.app - ), - "Target a regular application; session-critical processes (loginwindow, WindowServer, Dock, Finder, launchd) are never closed.", - )); + return Err(protected_process_error(&args.app)); } - adapter.close_app(&args.app, args.force)?; + let deadline = crate::Deadline::standard()?; + let expected = crate::commands::helpers::resolve_app(Some(&args.app), adapter, deadline)?; + let protected = adapter.is_protected_process(&args.app) + || adapter.is_protected_process(&expected.name) + || expected + .bundle_id + .as_deref() + .is_some_and(|bundle| adapter.is_protected_process(bundle)); + if protected { + return Err(protected_process_error(&args.app)); + } + let lease = adapter.acquire_interaction_lease(deadline)?; + let live = crate::commands::helpers::revalidate_app_for_mutation( + adapter, + &expected, + lease.deadline(), + )?; + adapter.close_app(&live, args.force, &lease)?; Ok(json!({ "app": args.app, "method": if args.force { "force" } else { "graceful" }, "requested": true, - "closed": args.force + "closed": true })) } +fn protected_process_error(app: &str) -> AppError { + AppError::invalid_input_with_suggestion( + format!("'{app}' is a protected system process and cannot be closed"), + "Target a regular application; session-critical processes (loginwindow, WindowServer, Dock, Finder, launchd) are never closed.", + ) +} + #[cfg(test)] #[path = "close_app_tests.rs"] mod tests; diff --git a/crates/core/src/commands/close_app_tests.rs b/crates/core/src/commands/close_app_tests.rs index d9ae2b7..d99f39b 100644 --- a/crates/core/src/commands/close_app_tests.rs +++ b/crates/core/src/commands/close_app_tests.rs @@ -1,23 +1,67 @@ use super::*; -use crate::adapter::PlatformAdapter; -use crate::error::{AdapterError, ErrorCode}; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{AdapterError, ErrorCode}; struct ProtectiveAdapter; -impl PlatformAdapter for ProtectiveAdapter { +impl ObservationOps for ProtectiveAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(vec![crate::AppInfo { + name: "TextEdit".into(), + pid: crate::ProcessId::new(42), + bundle_id: Some("com.apple.TextEdit".into()), + process_instance: Some("textedit-instance".into()), + }]) + } +} + +impl ActionOps for ProtectiveAdapter {} + +impl InputOps for ProtectiveAdapter {} + +impl SystemOps for ProtectiveAdapter { + crate::adapter::guarded_interaction_lease!(); + fn is_protected_process(&self, identifier: &str) -> bool { identifier.eq_ignore_ascii_case("CriticalThing") } - fn close_app(&self, _id: &str, _force: bool) -> Result<(), crate::error::AdapterError> { + fn close_app( + &self, + _app: &crate::AppInfo, + _force: bool, + _lease: &crate::InteractionLease, + ) -> Result<(), crate::AdapterError> { Ok(()) } } struct FailingAdapter; -impl PlatformAdapter for FailingAdapter { - fn close_app(&self, _id: &str, _force: bool) -> Result<(), AdapterError> { +impl ObservationOps for FailingAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(vec![crate::AppInfo { + name: "Ghost".into(), + pid: crate::ProcessId::new(77), + bundle_id: None, + process_instance: Some("ghost-instance".into()), + }]) + } +} + +impl ActionOps for FailingAdapter {} + +impl InputOps for FailingAdapter {} + +impl SystemOps for FailingAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn close_app( + &self, + _app: &crate::AppInfo, + _force: bool, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { Err(AdapterError::new(ErrorCode::AppNotFound, "no such app")) } } @@ -45,7 +89,7 @@ fn close_app_blocks_adapter_protected_process() { } #[test] -fn graceful_close_reports_request_without_claiming_termination() { +fn graceful_close_reports_verified_termination() { let value = execute( CloseAppArgs { app: "TextEdit".into(), @@ -58,10 +102,7 @@ fn graceful_close_reports_request_without_claiming_termination() { assert_eq!(value["app"], "TextEdit"); assert_eq!(value["method"], "graceful"); assert_eq!(value["requested"], true); - assert_eq!( - value["closed"], false, - "graceful close must not claim a termination it cannot confirm" - ); + assert_eq!(value["closed"], true); } #[test] @@ -69,7 +110,7 @@ fn close_app_propagates_adapter_errors() { let err = execute( CloseAppArgs { app: "Ghost".into(), - force: true, + force: false, }, &FailingAdapter, ) diff --git a/crates/core/src/commands/collapse.rs b/crates/core/src/commands/collapse.rs index b100bff..acf5ea3 100644 --- a/crates/core/src/commands/collapse.rs +++ b/crates/core/src/commands/collapse.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/combo.rs b/crates/core/src/commands/combo.rs index e2bc3cd..0488ca9 100644 --- a/crates/core/src/commands/combo.rs +++ b/crates/core/src/commands/combo.rs @@ -1,8 +1,4 @@ -use crate::{ - action::{KeyCombo, Modifier}, - adapter::PlatformAdapter, - error::{AdapterError, AppError}, -}; +use crate::{AdapterError, AppError, KeyCombo, Modifier, adapter::PlatformAdapter}; /// Refuses `combo` when the platform adapter reports it as dangerous, unless /// the caller forced it. Whether a combo is dangerous is the adapter's @@ -39,7 +35,7 @@ pub fn parse_combo(s: &str) -> Result { for &part in &parts[..parts.len() - 1] { let modifier = match part { - "cmd" | "command" => Modifier::Cmd, + "meta" | "cmd" | "command" => Modifier::Meta, "ctrl" | "control" => Modifier::Ctrl, "alt" | "option" => Modifier::Alt, "shift" => Modifier::Shift, diff --git a/crates/core/src/commands/combo_tests.rs b/crates/core/src/commands/combo_tests.rs index 371fff3..c7cac5c 100644 --- a/crates/core/src/commands/combo_tests.rs +++ b/crates/core/src/commands/combo_tests.rs @@ -1,18 +1,18 @@ use super::parse_combo; -use crate::action::Modifier; +use crate::Modifier; #[test] fn parse_combo_single_modifier_and_key() { let combo = parse_combo("cmd+k").expect("cmd+k is valid"); assert_eq!(combo.key, "k"); - assert_eq!(combo.modifiers, vec![Modifier::Cmd]); + assert_eq!(combo.modifiers, vec![Modifier::Meta]); } #[test] fn parse_combo_two_modifiers_preserved_in_declaration_order() { let combo = parse_combo("cmd+shift+t").expect("cmd+shift+t is valid"); assert_eq!(combo.key, "t"); - assert_eq!(combo.modifiers, vec![Modifier::Cmd, Modifier::Shift]); + assert_eq!(combo.modifiers, vec![Modifier::Meta, Modifier::Shift]); } #[test] @@ -25,7 +25,10 @@ fn parse_combo_bare_key_yields_empty_modifier_list() { #[test] fn parse_combo_accepts_long_form_modifier_aliases() { let cmd = parse_combo("command+a").expect("command alias"); - assert_eq!(cmd.modifiers, vec![Modifier::Cmd]); + assert_eq!(cmd.modifiers, vec![Modifier::Meta]); + + let meta = parse_combo("meta+a").expect("portable meta name"); + assert_eq!(meta.modifiers, vec![Modifier::Meta]); let alt = parse_combo("option+x").expect("option alias"); assert_eq!(alt.modifiers, vec![Modifier::Alt]); @@ -58,5 +61,5 @@ fn parse_combo_key_is_preserved_verbatim_without_lowercasing() { combo.key, "K", "parse_combo must NOT lowercase the key — normalization is the caller's responsibility" ); - assert_eq!(combo.modifiers, vec![Modifier::Cmd]); + assert_eq!(combo.modifiers, vec![Modifier::Meta]); } diff --git a/crates/core/src/commands/dismiss_all_notifications.rs b/crates/core/src/commands/dismiss_all_notifications.rs index c978d2b..55b5d50 100644 --- a/crates/core/src/commands/dismiss_all_notifications.rs +++ b/crates/core/src/commands/dismiss_all_notifications.rs @@ -1,4 +1,4 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{AppError, adapter::PlatformAdapter, context::CommandContext}; use serde_json::{Value, json}; pub struct DismissAllNotificationsArgs { @@ -8,8 +8,17 @@ pub struct DismissAllNotificationsArgs { pub fn execute( args: DismissAllNotificationsArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { - let (dismissed, failures) = adapter.dismiss_all_notifications(args.app.as_deref())?; + let policy = super::notification_policy::mutation_policy(context)?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let (dismissed, failures) = adapter.dismiss_all_notifications( + crate::DismissAllNotificationsRequest { + app_filter: args.app.as_deref(), + policy, + }, + &lease, + )?; let mut result = json!({ "dismissed_count": dismissed.len(), }); diff --git a/crates/core/src/commands/dismiss_notification.rs b/crates/core/src/commands/dismiss_notification.rs index 3a503a1..e410f74 100644 --- a/crates/core/src/commands/dismiss_notification.rs +++ b/crates/core/src/commands/dismiss_notification.rs @@ -1,16 +1,31 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{AppError, adapter::PlatformAdapter, context::CommandContext}; use serde_json::{Value, json}; pub struct DismissNotificationArgs { pub index: usize, pub app: Option, + pub expected_app: Option, + pub expected_title: Option, } pub fn execute( args: DismissNotificationArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { - let dismissed = adapter.dismiss_notification(args.index, args.app.as_deref())?; + let identity = + super::notification_identity::required_identity(args.expected_app, args.expected_title)?; + let policy = super::notification_policy::mutation_policy(context)?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let dismissed = adapter.dismiss_notification( + crate::DismissNotificationRequest { + index: args.index, + app_filter: args.app.as_deref(), + identity: &identity, + policy, + }, + &lease, + )?; Ok(json!({ "dismissed": dismissed, })) diff --git a/crates/core/src/commands/double_click.rs b/crates/core/src/commands/double_click.rs index 46fd3c7..da4801f 100644 --- a/crates/core/src/commands/double_click.rs +++ b/crates/core/src/commands/double_click.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/drag.rs b/crates/core/src/commands/drag.rs index 3e6a7df..9b3eb46 100644 --- a/crates/core/src/commands/drag.rs +++ b/crates/core/src/commands/drag.rs @@ -1,23 +1,30 @@ use crate::{ - action::DragParams, + AppError, DragParams, adapter::PlatformAdapter, - commands::point_resolve::{ - PointResolveArgs, focus_for_physical_input, require_cursor_policy, - resolve_point_from_ref_or_xy_with_context, + commands::{ + helpers::{apply_post_action_wait, validate_post_action_wait}, + point_resolve::{PointResolveArgs, require_cursor_policy}, + pointer_action::{ + PointResolveAttempt, ensure_point_deadline, focus_point_under_lease, point_deadline, + resolve_point_under_lease, retry_leased_point_phase, wait_for_point_with_deadline, + }, }, context::CommandContext, - error::AppError, }; use serde_json::{Value, json}; +pub struct DragEndpoint { + pub ref_id: Option, + pub xy: Option<(f64, f64)>, +} + pub struct DragArgs { - pub from_ref: Option, - pub from_xy: Option<(f64, f64)>, - pub to_ref: Option, - pub to_xy: Option<(f64, f64)>, + pub from: DragEndpoint, + pub to: DragEndpoint, pub snapshot_id: Option, pub duration_ms: Option, pub drop_delay_ms: Option, + pub timeout_ms: Option, } pub fn execute( @@ -26,34 +33,93 @@ pub fn execute( context: &CommandContext, ) -> Result { require_cursor_policy(context, "drag")?; - let from = resolve_point_from_ref_or_xy_with_context( - PointResolveArgs { - ref_id: args.from_ref.as_deref(), - xy: args.from_xy, - snapshot_id: args.snapshot_id.as_deref(), - missing_input_message: "Provide --from or --from-xy x,y", - }, - adapter, - context, + validate_post_action_wait(context)?; + let deadline = point_deadline(args.timeout_ms)?; + let from_args = PointResolveArgs { + ref_id: args.from.ref_id.as_deref(), + xy: args.from.xy, + snapshot_id: args.snapshot_id.as_deref(), + missing_input_message: "Provide --from or --from-xy x,y", + headed_requirement: crate::HeadedRequirement::FocusedWindowAndCursor, + }; + let to_args = PointResolveArgs { + ref_id: args.to.ref_id.as_deref(), + xy: args.to.xy, + snapshot_id: args.snapshot_id.as_deref(), + missing_input_message: "Provide --to or --to-xy x,y", + headed_requirement: crate::HeadedRequirement::None, + }; + let auto_wait = args.timeout_ms.is_some_and(|timeout_ms| timeout_ms > 0); + if auto_wait { + wait_for_point_with_deadline(from_args, deadline, adapter, context)?; + wait_for_point_with_deadline(to_args, deadline, adapter, context)?; + } + let (from, to, lease) = retry_leased_point_phase(args.timeout_ms, deadline, || { + let lease = adapter.acquire_interaction_lease(deadline)?; + let focused = focus_point_under_lease(from_args, &lease, adapter, context)?; + let from = resolve_point_under_lease( + PointResolveAttempt { + args: from_args, + stability: None, + allow_scroll: !auto_wait, + }, + deadline, + &lease, + adapter, + context, + )?; + let to = resolve_point_under_lease( + PointResolveAttempt { + args: to_args, + stability: None, + allow_scroll: !auto_wait, + }, + deadline, + &lease, + adapter, + context, + )?; + let mut from = resolve_point_under_lease( + PointResolveAttempt { + args: from_args, + stability: Some(from.bounds_hash), + allow_scroll: false, + }, + deadline, + &lease, + adapter, + context, + )?; + let to = resolve_point_under_lease( + PointResolveAttempt { + args: to_args, + stability: Some(to.bounds_hash), + allow_scroll: false, + }, + deadline, + &lease, + adapter, + context, + )?; + from.focused = focused; + Ok((from, to, lease)) + })?; + ensure_point_deadline( + deadline, + Some(json!({ + "status": "actionable", + "from": { "x": from.point.x, "y": from.point.y }, + "to": { "x": to.point.x, "y": to.point.y } + })), )?; - let to = resolve_point_from_ref_or_xy_with_context( - PointResolveArgs { - ref_id: args.to_ref.as_deref(), - xy: args.to_xy, - snapshot_id: args.snapshot_id.as_deref(), - missing_input_message: "Provide --to or --to-xy x,y", - }, - adapter, - context, - )?; - let focused = focus_for_physical_input(from.pid, adapter, context)?; let params = DragParams { from: from.point.clone(), to: to.point.clone(), duration_ms: args.duration_ms, drop_delay_ms: args.drop_delay_ms, }; - adapter.drag(params)?; + params.validate(deadline)?; + adapter.drag(params, &lease)?; let mut response = json!({ "dragged": true, "from": { "x": from.point.x, "y": from.point.y }, @@ -62,10 +128,11 @@ pub fn execute( if let Some(drop_delay_ms) = args.drop_delay_ms { response["drop_delay_ms"] = json!(drop_delay_ms); } - if focused { + if from.focused { response["focused"] = json!(true); } - Ok(response) + drop(lease); + apply_post_action_wait(response, from.source_entry.as_ref(), adapter, context) } #[cfg(test)] diff --git a/crates/core/src/commands/drag_occlusion_retry_tests.rs b/crates/core/src/commands/drag_occlusion_retry_tests.rs new file mode 100644 index 0000000..92efa98 --- /dev/null +++ b/crates/core/src/commands/drag_occlusion_retry_tests.rs @@ -0,0 +1,143 @@ +use super::*; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +struct TransientOcclusionAdapter { + hit_tests: AtomicU32, + lease_acquisitions: AtomicU32, + lease_held: Arc, + captured: Mutex>, +} + +struct LeaseFlag(Arc); + +impl Drop for LeaseFlag { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +impl TransientOcclusionAdapter { + fn new() -> Self { + Self { + hit_tests: AtomicU32::new(0), + lease_acquisitions: AtomicU32::new(0), + lease_held: Arc::new(AtomicBool::new(false)), + captured: Mutex::new(None), + } + } +} + +impl ObservationOps for TransientOcclusionAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 10.0, + y: 20.0, + width: 40.0, + height: 60.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + if self.hit_tests.fetch_add(1, Ordering::SeqCst) == 0 { + Ok(HitTestResult::InterceptedBy { + role: Some("AXSheet".into()), + name: Some("Transient sheet".into()), + bounds: None, + }) + } else { + Ok(HitTestResult::ReachesTarget) + } + } +} + +impl ActionOps for TransientOcclusionAdapter {} + +impl InputOps for TransientOcclusionAdapter { + fn drag( + &self, + params: DragParams, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.captured.lock().unwrap() = Some(params); + Ok(()) + } +} + +impl SystemOps for TransientOcclusionAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + assert!(!self.lease_held.swap(true, Ordering::SeqCst)); + self.lease_acquisitions.fetch_add(1, Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, LeaseFlag(self.lease_held.clone())) + } + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +#[test] +fn drag_retries_transient_post_focus_occlusion_without_holding_lease() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = TransientOcclusionAdapter::new(); + let mut args = cross_app_args(snapshot_id); + args.timeout_ms = Some(500); + + let value = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap(); + + assert_eq!(value["dragged"], true); + assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 2); + assert!(!adapter.lease_held.load(Ordering::SeqCst)); + assert!(adapter.captured.lock().unwrap().is_some()); +} + +#[test] +fn drag_single_shot_does_not_retry_post_focus_occlusion() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = TransientOcclusionAdapter::new(); + + let error = execute( + cross_app_args(snapshot_id), + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(error.code(), "ACTION_FAILED"); + assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 1); + assert!(adapter.captured.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/drag_retry_tests.rs b/crates/core/src/commands/drag_retry_tests.rs new file mode 100644 index 0000000..05283b1 --- /dev/null +++ b/crates/core/src/commands/drag_retry_tests.rs @@ -0,0 +1,378 @@ +use super::*; + +struct StaleThenOkAdapter { + retry: StaleRetryCounter, + minimum_resolves_before_lease: u32, + lease_acquisitions: AtomicU32, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + retry: StaleRetryCounter::new(fail_until), + minimum_resolves_before_lease: 0, + lease_acquisitions: AtomicU32::new(0), + } + } + + fn expect_poll_before_lease(mut self, minimum: u32) -> Self { + self.minimum_resolves_before_lease = minimum; + self + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.retry.attempt() + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 10.0, + y: 20.0, + width: 40.0, + height: 60.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } +} + +impl ActionOps for StaleThenOkAdapter {} + +impl InputOps for StaleThenOkAdapter { + fn drag( + &self, + _params: DragParams, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +impl SystemOps for StaleThenOkAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + assert!(self.retry.calls() >= self.minimum_resolves_before_lease); + self.lease_acquisitions.fetch_add(1, Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, ()) + } + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +#[test] +fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = StaleThenOkAdapter::new(2).expect_poll_before_lease(3); + + let value = execute( + DragArgs { + from: DragEndpoint { + ref_id: Some("@e1".into()), + xy: None, + }, + to: DragEndpoint { + ref_id: Some("@e2".into()), + xy: None, + }, + snapshot_id: Some(snapshot_id), + duration_ms: None, + drop_delay_ms: None, + timeout_ms: Some(5_000), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert_eq!(value["dragged"], true); + assert!(adapter.retry.calls() >= 3); + assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 1); +} + +struct OccludedFromAdapter { + captured: Mutex>, +} + +impl ObservationOps for OccludedFromAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 10.0, + y: 20.0, + width: 40.0, + height: 60.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::InterceptedBy { + role: Some("AXSheet".into()), + name: Some("Save changes?".into()), + bounds: None, + }) + } +} + +impl ActionOps for OccludedFromAdapter {} + +impl InputOps for OccludedFromAdapter { + fn drag( + &self, + params: DragParams, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.captured.lock().unwrap() = Some(params); + Ok(()) + } +} + +impl SystemOps for OccludedFromAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +#[test] +fn drag_from_occluded_ref_fails_preflight_before_dispatch() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = OccludedFromAdapter { + captured: Mutex::new(None), + }; + + let err = execute( + cross_app_args(snapshot_id), + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_FAILED"); + let AppError::Adapter(adapter_error) = &err else { + panic!("expected adapter actionability failure") + }; + assert!( + adapter_error + .details + .as_ref() + .is_some_and(|details| details.to_string().contains("AXSheet")) + ); + assert!(adapter.captured.lock().unwrap().is_none()); +} + +struct MutuallyOffscreenAdapter { + visible_pid: AtomicU32, + dispatched: AtomicU32, +} + +impl ObservationOps for MutuallyOffscreenAdapter { + fn resolve_element_strict( + &self, + entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::new(entry.process.pid.get())) + } + + fn get_element_bounds( + &self, + handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + let pid = *handle.downcast_ref::().unwrap(); + if self.visible_pid.load(Ordering::SeqCst) != pid { + return Ok(None); + } + Ok(Some(Rect { + x: f64::from(pid) * 100.0, + y: 20.0, + width: 40.0, + height: 60.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } +} + +impl ActionOps for MutuallyOffscreenAdapter { + fn scroll_into_view( + &self, + handle: &NativeHandle, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.visible_pid + .store(*handle.downcast_ref::().unwrap(), Ordering::SeqCst); + Ok(()) + } +} + +impl InputOps for MutuallyOffscreenAdapter { + fn drag( + &self, + _params: DragParams, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.dispatched.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +impl SystemOps for MutuallyOffscreenAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +#[test] +fn drag_does_not_dispatch_when_endpoints_cannot_remain_visible_together() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = MutuallyOffscreenAdapter { + visible_pid: AtomicU32::new(1), + dispatched: AtomicU32::new(0), + }; + let mut args = cross_app_args(snapshot_id); + args.timeout_ms = Some(500); + + let error = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap_err(); + + assert_eq!(error.code(), "TIMEOUT"); + assert_eq!(adapter.dispatched.load(Ordering::SeqCst), 0); +} + +#[test] +fn timeout_none_is_single_shot() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = StaleThenOkAdapter::new(1); + + let error = execute( + DragArgs { + from: DragEndpoint { + ref_id: Some("@e1".into()), + xy: None, + }, + to: DragEndpoint { + ref_id: Some("@e2".into()), + xy: None, + }, + snapshot_id: Some(snapshot_id), + duration_ms: None, + drop_delay_ms: None, + timeout_ms: None, + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(error.code(), "STALE_REF"); + assert_eq!(adapter.retry.calls(), 1); +} + +#[test] +fn expired_xy_budget_never_dispatches_drag() { + let adapter = DragCaptureAdapter::new(); + let mut args = xy_args(None); + args.timeout_ms = Some(0); + + let err = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap_err(); + + assert_eq!(err.code(), "TIMEOUT"); + let AppError::Adapter(adapter_error) = &err else { + panic!("expected adapter timeout") + }; + assert_eq!( + adapter_error + .details + .as_ref() + .and_then(|details| details.get("kind")), + Some(&serde_json::json!("actionability_timeout")) + ); + assert!(adapter.captured.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/drag_tests.rs b/crates/core/src/commands/drag_tests.rs index 70ff9e2..eb21690 100644 --- a/crates/core/src/commands/drag_tests.rs +++ b/crates/core/src/commands/drag_tests.rs @@ -1,19 +1,23 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - action::DragParams, - adapter::{NativeHandle, PlatformAdapter}, + AdapterError, DragParams, Rect, + adapter::NativeHandle, capability, - error::AdapterError, - node::Rect, + commands::stale_retry_test_support::StaleRetryCounter, + hit_test::HitTestResult, refs::{RefEntry, RefMap}, refs_store::RefStore, refs_test_support::HomeGuard, }; use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; struct DragCaptureAdapter { captured: Mutex>, - focused_pids: Mutex>, + focused_pids: Mutex>, + resolve_calls: AtomicU32, + focused_bounds: Option, } impl DragCaptureAdapter { @@ -21,16 +25,37 @@ impl DragCaptureAdapter { Self { captured: Mutex::new(None), focused_pids: Mutex::new(Vec::new()), + resolve_calls: AtomicU32::new(0), + focused_bounds: None, } } + + fn with_focused_bounds(mut self, bounds: Rect) -> Self { + self.focused_bounds = Some(bounds); + self + } } -impl PlatformAdapter for DragCaptureAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { +impl ObservationOps for DragCaptureAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.resolve_calls.fetch_add(1, Ordering::SeqCst); Ok(NativeHandle::null()) } - fn get_element_bounds(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + if !self.focused_pids.lock().unwrap().is_empty() + && let Some(bounds) = self.focused_bounds + { + return Ok(Some(bounds)); + } Ok(Some(Rect { x: 10.0, y: 20.0, @@ -39,26 +64,64 @@ impl PlatformAdapter for DragCaptureAdapter { })) } - fn focus_app(&self, pid: i32) -> Result<(), AdapterError> { - self.focused_pids.lock().unwrap().push(pid); + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } +} + +impl ActionOps for DragCaptureAdapter {} + +impl InputOps for DragCaptureAdapter { + fn drag( + &self, + params: DragParams, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.captured.lock().unwrap() = Some(params); Ok(()) } +} - fn drag(&self, params: DragParams) -> Result<(), AdapterError> { - *self.captured.lock().unwrap() = Some(params); +impl SystemOps for DragCaptureAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.focused_pids.lock().unwrap().push(window.pid); Ok(()) } } fn xy_args(drop_delay_ms: Option) -> DragArgs { DragArgs { - from_ref: None, - from_xy: Some((1.0, 2.0)), - to_ref: None, - to_xy: Some((3.0, 4.0)), + from: DragEndpoint { + ref_id: None, + xy: Some((1.0, 2.0)), + }, + to: DragEndpoint { + ref_id: None, + xy: Some((3.0, 4.0)), + }, snapshot_id: None, duration_ms: None, drop_delay_ms, + timeout_ms: None, } } @@ -97,24 +160,39 @@ fn drop_delay_omitted_uses_adapter_default_and_no_response_field() { assert_eq!(captured.drop_delay_ms, None); } -fn ref_entry(pid: i32) -> RefEntry { +fn ref_entry(pid: u32) -> RefEntry { RefEntry { - pid, - role: "button".into(), - name: Some("Item".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(pid), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Item".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some(format!("App {pid}")), + source_window_id: Some(format!("w-{pid}")), + source_window_title: Some(format!("Window {pid}")), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } @@ -128,40 +206,21 @@ fn cross_app_snapshot() -> String { fn cross_app_args(snapshot_id: String) -> DragArgs { DragArgs { - from_ref: Some("@e1".into()), - from_xy: None, - to_ref: Some("@e2".into()), - to_xy: None, + from: DragEndpoint { + ref_id: Some("@e1".into()), + xy: None, + }, + to: DragEndpoint { + ref_id: Some("@e2".into()), + xy: None, + }, snapshot_id: Some(snapshot_id), duration_ms: None, drop_delay_ms: None, + timeout_ms: None, } } -#[test] -fn drag_resolves_ref_bounds_to_center_point() { - let _guard = HomeGuard::new(); - let store = RefStore::new().unwrap(); - let mut refmap = RefMap::new(); - refmap.allocate(ref_entry(1)); - let snapshot_id = store.save_new_snapshot(&refmap).unwrap(); - let adapter = DragCaptureAdapter::new(); - - let args = DragArgs { - from_ref: Some("@e1".into()), - from_xy: None, - to_ref: None, - to_xy: Some((100.0, 200.0)), - snapshot_id: Some(snapshot_id), - duration_ms: None, - drop_delay_ms: Some(300), - }; - execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap(); - - let captured = adapter.captured.lock().unwrap().clone().unwrap(); - assert_eq!((captured.from.x, captured.from.y), (30.0, 50.0)); -} - #[test] fn headless_ref_drag_is_policy_denied_before_cursor_move() { let _guard = HomeGuard::new(); @@ -184,7 +243,12 @@ fn headless_ref_drag_is_policy_denied_before_cursor_move() { fn headed_ref_drag_focuses_only_the_from_app_once() { let _guard = HomeGuard::new(); let snapshot_id = cross_app_snapshot(); - let adapter = DragCaptureAdapter::new(); + let adapter = DragCaptureAdapter::new().with_focused_bounds(Rect { + x: 100.0, + y: 200.0, + width: 40.0, + height: 60.0, + }); let value = execute( cross_app_args(snapshot_id), @@ -194,9 +258,32 @@ fn headed_ref_drag_focuses_only_the_from_app_once() { .unwrap(); assert_eq!(*adapter.focused_pids.lock().unwrap(), vec![1]); + assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 4); + let captured = adapter.captured.lock().unwrap().clone().unwrap(); + assert_eq!((captured.from.x, captured.from.y), (120.0, 230.0)); assert_eq!(value["focused"], true); } +#[test] +fn positive_timeout_uses_stable_post_focus_drag_bounds() { + let _guard = HomeGuard::new(); + let snapshot_id = cross_app_snapshot(); + let adapter = DragCaptureAdapter::new().with_focused_bounds(Rect { + x: 100.0, + y: 200.0, + width: 40.0, + height: 60.0, + }); + let mut args = cross_app_args(snapshot_id); + args.timeout_ms = Some(500); + + let value = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap(); + + assert_eq!(*adapter.focused_pids.lock().unwrap(), vec![1]); + assert_eq!(value["from"], json!({ "x": 120.0, "y": 230.0 })); + assert_eq!(value["to"], json!({ "x": 120.0, "y": 230.0 })); +} + #[test] fn headed_xy_drag_never_steals_focus() { let adapter = DragCaptureAdapter::new(); @@ -211,3 +298,9 @@ fn headed_xy_drag_never_steals_focus() { assert!(adapter.focused_pids.lock().unwrap().is_empty()); assert!(value.get("focused").is_none()); } + +#[path = "drag_retry_tests.rs"] +mod retry_tests; + +#[path = "drag_occlusion_retry_tests.rs"] +mod occlusion_retry_tests; diff --git a/crates/core/src/commands/execute_by_ref.rs b/crates/core/src/commands/execute_by_ref.rs index 1c7990f..3893859 100644 --- a/crates/core/src/commands/execute_by_ref.rs +++ b/crates/core/src/commands/execute_by_ref.rs @@ -1,50 +1,81 @@ use crate::{ + AppError, action::Action, action_request::ActionRequest, adapter::PlatformAdapter, - commands::helpers::{RefArgs, execute_ref_action_with_context}, + commands::helpers::{RefArgs, execute_ref_action_with_context, normalize_action_timeout_ms}, context::CommandContext, - error::AppError, interaction_policy::InteractionPolicy, }; use serde_json::Value; +pub const DEFAULT_ACTION_TIMEOUT_MS: u64 = 5000; + +/// Config struct bundling the caller-supplied inputs to `execute` / +/// `execute_with_timeout`, keeping both functions within the 5-parameter +/// limit (mirrors the `RefArgs`/`ActionRequest` config-struct pattern used +/// elsewhere in this module). +pub struct ExecuteByRefArgs<'a> { + pub ref_id: &'a str, + pub snapshot_id: Option<&'a str>, + pub action: Action, + pub caller_policy: InteractionPolicy, +} + /// Executes an action addressed by a snapshot ref through the canonical /// ref-action pipeline: `RefStore` load → `RefMap` lookup → strict element /// resolution → live actionability preflight → dispatch. /// -/// `snapshot_id` follows CLI `--snapshot` semantics: `None` pins to the -/// latest snapshot for the session; `Some(id)` pins to that specific snapshot. +/// A qualified `ref_id` embeds its snapshot and accepts `snapshot_id: None`. +/// A legacy bare `@eN` ref requires `Some(id)`. When both are supplied, the +/// explicit ID must match the qualified ref. /// /// The effective `InteractionPolicy` is the join of `caller_policy` and the -/// action's CLI base policy, ensuring the result is always at least as -/// permissive as what the CLI would use for the same action, while allowing -/// FFI callers to opt in to higher-permission policies such as `headed`. -/// -/// Note on PressKey: its base policy is `focus_fallback` (derived from -/// `Action::base_interaction_policy`, shared with `TypeText`) because a -/// ref-targeted key press may need the target focused for keystrokes to land. +/// action's base policy. Semantic actions, including `TypeText`, are strict +/// headless; explicit `PressKey` permits focus without cursor movement. FFI +/// callers may opt in to `focus_fallback` or `headed`. pub fn execute( - ref_id: &str, - snapshot_id: Option<&str>, - action: Action, - caller_policy: InteractionPolicy, + args: ExecuteByRefArgs<'_>, adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { + execute_with_timeout(args, DEFAULT_ACTION_TIMEOUT_MS, adapter, context) +} + +pub fn execute_with_timeout( + args: ExecuteByRefArgs<'_>, + timeout_ms: u64, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let ExecuteByRefArgs { + ref_id, + snapshot_id, + action, + caller_policy, + } = args; + let timeout_ms = normalize_action_timeout_ms(timeout_ms); let base = action.base_interaction_policy(); let effective = base.join(caller_policy); let request = ActionRequest { action, policy: effective, + timeout_ms, + verified_point: None, + expected_process: None, }; execute_ref_action_with_context( RefArgs { ref_id: ref_id.to_owned(), snapshot_id: snapshot_id.map(ToOwned::to_owned), + timeout_ms, }, adapter, request, context, ) } + +#[cfg(test)] +#[path = "execute_by_ref_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/execute_by_ref_tests.rs b/crates/core/src/commands/execute_by_ref_tests.rs new file mode 100644 index 0000000..45773da --- /dev/null +++ b/crates/core/src/commands/execute_by_ref_tests.rs @@ -0,0 +1,273 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{ + Action, AdapterError, ErrorCode, KeyCombo, + action_request::ActionRequest, + action_result::ActionResult, + adapter::NativeHandle, + refs::{RefEntry, RefMap}, + refs_store::RefStore, + refs_test_support::HomeGuard, +}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; + +fn snapshot_with_ref(role: &str, available_actions: &[&str]) -> String { + let mut refmap = RefMap::new(); + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; + refmap.allocate(RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: role.into(), + name: Some("Target".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: available_actions.iter().map(|a| (*a).to_string()).collect(), + }, + source: crate::RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-1".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + }); + RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() +} + +struct StaleThenOkAdapter { + resolve_calls: AtomicU32, + fail_until: u32, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + resolve_calls: AtomicU32::new(0), + fail_until, + } + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + let n = self.resolve_calls.fetch_add(1, Ordering::SeqCst) + 1; + if n <= self.fail_until { + return Err(AdapterError::new(ErrorCode::StaleRef, "not yet resolvable") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!( + "textfield", + "Target", + [ + crate::capability::CLICK, + crate::capability::PRESS_KEY, + crate::capability::SET_VALUE + ] + ); +} + +impl ActionOps for StaleThenOkAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("click")) + } +} + +impl InputOps for StaleThenOkAdapter {} +impl SystemOps for StaleThenOkAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +struct PolicyCaptureAdapter { + captured: Mutex>, +} + +impl PolicyCaptureAdapter { + fn new() -> Self { + Self { + captured: Mutex::new(None), + } + } +} + +impl ObservationOps for PolicyCaptureAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!( + "textfield", + "Target", + [ + crate::capability::CLICK, + crate::capability::PRESS_KEY, + crate::capability::SET_VALUE + ] + ); +} + +impl ActionOps for PolicyCaptureAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + let name = request.action.name().to_string(); + *self.captured.lock().unwrap() = Some(request); + Ok(ActionResult::delivered_unverified(name)) + } +} + +impl InputOps for PolicyCaptureAdapter {} +impl SystemOps for PolicyCaptureAdapter { + crate::adapter::guarded_interaction_lease!(); + crate::adapter::exact_window_focus!(); +} + +#[test] +fn default_action_timeout_ms_is_five_seconds() { + assert_eq!(DEFAULT_ACTION_TIMEOUT_MS, 5000); +} + +/// `execute` must forward `DEFAULT_ACTION_TIMEOUT_MS` (not `None` or `0`) into +/// the ref-action retry budget, exactly as a direct +/// `execute_with_timeout(args, DEFAULT_ACTION_TIMEOUT_MS, ..)` call would. +/// A transient `STALE_REF` that clears within the budget must therefore be +/// retried until it resolves rather than surfacing on the first attempt. +#[test] +fn execute_forwards_default_timeout_and_retries_transient_stale_ref() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_with_ref("textfield", &["Click"]); + let adapter = StaleThenOkAdapter::new(2); + + let value = execute( + ExecuteByRefArgs { + ref_id: "@e1", + snapshot_id: Some(&snapshot_id), + action: Action::Click, + caller_policy: InteractionPolicy::headless(), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["action"], "click"); + assert!(adapter.resolve_calls.load(Ordering::SeqCst) >= 3); +} + +/// `execute_with_timeout` must run `normalize_action_timeout_ms` on the raw +/// timeout it is given, so a `0` (the CLI's "no retry budget" sentinel) +/// collapses to `None` and the ref-action pipeline makes exactly one resolve +/// attempt instead of silently retrying anyway. +#[test] +fn execute_with_timeout_zero_normalizes_to_single_attempt() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_with_ref("textfield", &["Click"]); + let adapter = StaleThenOkAdapter::new(1); + + let err = execute_with_timeout( + ExecuteByRefArgs { + ref_id: "@e1", + snapshot_id: Some(&snapshot_id), + action: Action::Click, + caller_policy: InteractionPolicy::headless(), + }, + 0, + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "STALE_REF"); + assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn explicit_press_key_keeps_its_focus_fallback_policy() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_with_ref("textfield", &["PressKey"]); + let adapter = PolicyCaptureAdapter::new(); + + execute( + ExecuteByRefArgs { + ref_id: "@e1", + snapshot_id: Some(&snapshot_id), + action: Action::PressKey(KeyCombo { + key: "A".into(), + modifiers: vec![], + }), + caller_policy: InteractionPolicy::headless(), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + let captured = adapter.captured.lock().unwrap(); + let policy = captured.as_ref().unwrap().policy; + assert!(policy.allow_focus_steal); + assert!(!policy.allow_cursor_move); +} + +#[test] +fn effective_policy_honors_caller_policy_above_action_base() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_with_ref("textfield", &["SetValue"]); + let adapter = PolicyCaptureAdapter::new(); + + execute( + ExecuteByRefArgs { + ref_id: "@e1", + snapshot_id: Some(&snapshot_id), + action: Action::SetValue("value".into()), + caller_policy: InteractionPolicy::headed(), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + let captured = adapter.captured.lock().unwrap(); + let policy = captured.as_ref().unwrap().policy; + assert!(policy.allow_focus_steal); + assert!(policy.allow_cursor_move); +} diff --git a/crates/core/src/commands/expand.rs b/crates/core/src/commands/expand.rs index a0f5eef..313f035 100644 --- a/crates/core/src/commands/expand.rs +++ b/crates/core/src/commands/expand.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/find.rs b/crates/core/src/commands/find.rs index e4036b1..0d92815 100644 --- a/crates/core/src/commands/find.rs +++ b/crates/core/src/commands/find.rs @@ -1,20 +1,34 @@ +#[cfg(test)] +use crate::AccessibilityNode; +#[cfg(test)] +use crate::commands::query; use crate::{ - adapter::PlatformAdapter, commands::query, context::CommandContext, error::AppError, - node::AccessibilityNode, roles, search_text, snapshot, + AppError, IdentityPredicate, LocatorQuery, StatePredicate, adapter::PlatformAdapter, + context::CommandContext, search_text, }; -use serde_json::{Value, json}; +use serde_json::Value; +#[cfg(test)] +use serde_json::json; +#[cfg(test)] use std::collections::BTreeSet; const DEFAULT_LIMIT: usize = 50; -pub use query::FindQuery; - -pub struct FindArgs { - pub app: Option, +/// Match-criteria fields: how a candidate element is identified. Grouped out +/// of [`FindArgs`] to keep it under the repo's field-count limit. +pub struct FindFilterArgs { pub role: Option, pub name: Option, + pub description: Option, + pub native_id: Option, pub value: Option, pub text: Option, + pub exact: bool, +} + +/// Result-shaping fields: which of the matches to return. Mutually exclusive +/// at the CLI/batch layer (enforced by [`validate_find_mode`]). +pub struct FindSelectionArgs { pub count: bool, pub first: bool, pub last: bool, @@ -22,75 +36,77 @@ pub struct FindArgs { pub limit: Option, } +pub struct FindArgs { + pub app: Option, + pub window_id: Option, + pub filter: FindFilterArgs, + pub states: Vec, + pub selection: FindSelectionArgs, +} + pub fn execute( args: FindArgs, adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { validate_find_mode(&args)?; - let query = FindQuery::from_args(&args); - let opts = crate::adapter::TreeOptions::default(); - let result = if args.count { - snapshot::build(adapter, &opts, args.app.as_deref(), None)? - } else { - snapshot::run_with_context(adapter, &opts, args.app.as_deref(), None, context)? - }; + let query = locator_query_from_args(&args)?; + query.validate_states().map_err(AppError::Adapter)?; - if args.count { - return Ok(json!({ "count": count_matches(&result.tree, &query) })); - } - - let mut matches = Vec::new(); - let max_matches = max_matches_for_args(&args); - search_tree( - &result.tree, - &query, - &mut Vec::new(), - &mut matches, - max_matches, - ); - - if args.first { - return Ok(single_match_response( - matches.into_iter().next(), - &query, - &result.tree, - )); - } - - if args.last { - return Ok(single_match_response( - matches.into_iter().last(), - &query, - &result.tree, - )); - } - - if let Some(n) = args.nth { - return Ok(single_match_response( - matches.into_iter().nth(n), - &query, - &result.tree, - )); - } - - let mut response = json!({ "matches": matches }); - attach_roles_present_hint(&mut response, matches.is_empty(), &query, &result.tree); - Ok(response) + live::execute(&args, &query, adapter, context) } -/// When a role-filtered search returns nothing, the caller cannot tell -/// "no elements of this role are on screen" from "this role name does not -/// exist." Listing the roles actually present in the searched tree answers -/// that from live data — no hardcoded vocabulary, so a role any adapter -/// newly emits shows up here automatically. +pub fn parse_state_flag(raw: &str) -> Result { + let (token, expected) = match raw.split_once('=') { + Some((token, value)) => { + let expected = match value { + "true" | "1" => Some(true), + "false" | "0" => Some(false), + _ => { + return Err(AppError::invalid_input_with_suggestion( + format!("Invalid state flag value '{value}' in '{raw}'"), + "Use TOKEN, TOKEN=true, or TOKEN=false.", + )); + } + }; + (token, expected) + } + None => (raw, None), + }; + Ok(StatePredicate { + token: token.to_string(), + expected, + }) +} + +fn locator_query_from_args(args: &FindArgs) -> Result { + Ok(LocatorQuery { + identity: IdentityPredicate { + role: args.filter.role.clone(), + name: args.filter.name.as_deref().map(search_text::normalize), + description: args + .filter + .description + .as_deref() + .map(search_text::normalize), + native_id: args.filter.native_id.clone(), + value: args.filter.value.as_deref().map(search_text::normalize), + }, + has_text: args.filter.text.as_deref().map(search_text::normalize), + exact: args.filter.exact, + states: args.states.clone(), + ..LocatorQuery::default() + }) +} + +#[cfg(test)] fn attach_roles_present_hint( response: &mut Value, is_empty: bool, - query: &FindQuery, + query: &LocatorQuery, tree: &AccessibilityNode, ) { - if !is_empty || query.role.is_none() { + if !is_empty || query.identity.role.is_none() { return; } let mut present = BTreeSet::new(); @@ -103,9 +119,10 @@ fn attach_roles_present_hint( } } +#[cfg(test)] fn single_match_response( found: Option, - query: &FindQuery, + query: &LocatorQuery, tree: &AccessibilityNode, ) -> Value { let is_empty = found.is_none(); @@ -114,6 +131,7 @@ fn single_match_response( response } +#[cfg(test)] fn collect_roles(node: &AccessibilityNode, roles: &mut BTreeSet) { roles.insert(node.role.clone()); for child in &node.children { @@ -121,28 +139,17 @@ fn collect_roles(node: &AccessibilityNode, roles: &mut BTreeSet) { } } -fn max_matches_for_args(args: &FindArgs) -> Option { - if args.count || args.last { - return None; - } - if args.first { - return Some(1); - } - if let Some(n) = args.nth { - return Some(n.saturating_add(1)); - } - match args.limit.unwrap_or(DEFAULT_LIMIT) { - 0 => None, - limit => Some(limit), - } -} - fn validate_find_mode(args: &FindArgs) -> Result<(), AppError> { - let selector_count = [args.count, args.first, args.last, args.nth.is_some()] - .into_iter() - .filter(|selected| *selected) - .count(); - if selector_count > 1 || (selector_count == 1 && args.limit.is_some()) { + let selector_count = [ + args.selection.count, + args.selection.first, + args.selection.last, + args.selection.nth.is_some(), + ] + .into_iter() + .filter(|selected| *selected) + .count(); + if selector_count > 1 || (selector_count == 1 && args.selection.limit.is_some()) { return Err(AppError::invalid_input_with_suggestion( "find accepts only one result-shaping mode", "Use one of --count, --first, --last, --nth, or --limit.", @@ -151,20 +158,10 @@ fn validate_find_mode(args: &FindArgs) -> Result<(), AppError> { Ok(()) } -impl FindQuery { - fn from_args(args: &FindArgs) -> Self { - Self { - role: args.role.as_deref().map(roles::normalize_role_query), - name: args.name.as_deref().map(search_text::normalize), - value: args.value.as_deref().map(search_text::normalize), - text: args.text.as_deref().map(search_text::normalize), - } - } -} - -fn search_tree( +#[cfg(test)] +fn collect_snapshot_matches( node: &AccessibilityNode, - query: &FindQuery, + query: &LocatorQuery, path: &mut Vec, matches: &mut Vec, max_matches: Option, @@ -175,18 +172,19 @@ fn search_tree( if query::node_matches(node, query) { let interactive = node.ref_id.is_some(); let display_name = node + .identity .name .as_deref() - .or(node.value.as_deref()) - .or(node.description.as_deref()) + .or(node.identity.value.as_deref()) + .or(node.identity.description.as_deref()) .map(String::from) .unwrap_or_else(|| format!("(unnamed {})", node.role)); matches.push(json!({ "ref_id": node.ref_id, "role": node.role, "name": display_name, - "value": node.value, - "states": node.states, + "value": node.identity.value, + "states": node.presentation.states, "interactive": interactive, "path": path.clone() })); @@ -196,15 +194,16 @@ fn search_tree( } let label = node + .identity .name .as_deref() - .or(node.value.as_deref()) + .or(node.identity.value.as_deref()) .map(|label| format!("{}:{label}", node.role)) .unwrap_or_else(|| node.role.clone()); path.push(label); for child in &node.children { - if search_tree(child, query, path, matches, max_matches) { + if collect_snapshot_matches(child, query, path, matches, max_matches) { path.pop(); return true; } @@ -214,7 +213,8 @@ fn search_tree( false } -fn count_matches(node: &AccessibilityNode, query: &FindQuery) -> usize { +#[cfg(test)] +fn count_matches(node: &AccessibilityNode, query: &LocatorQuery) -> usize { usize::from(query::node_matches(node, query)) + node .children @@ -223,6 +223,13 @@ fn count_matches(node: &AccessibilityNode, query: &FindQuery) -> usize { .sum::() } +#[path = "find_live.rs"] +mod live; + #[cfg(test)] #[path = "find_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "find_live_tests.rs"] +mod live_tests; diff --git a/crates/core/src/commands/find_live.rs b/crates/core/src/commands/find_live.rs new file mode 100644 index 0000000..8cd2b52 --- /dev/null +++ b/crates/core/src/commands/find_live.rs @@ -0,0 +1,243 @@ +use super::{DEFAULT_LIMIT, FindArgs}; +use crate::{ + AdapterError, AppError, LocatorQuery, + adapter::PlatformAdapter, + context::CommandContext, + live_locator::{ + LocatorMaterialization, LocatorResolution, LocatorResolveRequest, LocatorSelection, + ObservationRoot, resolve_query, + }, + refs::RefMap, + refs_store::RefStore, + snapshot, trace_artifacts, +}; +use serde_json::{Value, json}; +use std::time::Duration; + +const LOCATOR_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_RAW_DEPTH: u8 = 50; + +pub(super) fn execute( + args: &FindArgs, + query: &LocatorQuery, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let deadline = crate::Deadline::from_duration(LOCATOR_TIMEOUT)?; + let window = snapshot::resolve_window( + adapter, + args.app.as_deref(), + args.window_id.as_deref(), + deadline, + )?; + let request = resolve_request(args, deadline); + let mut resolution = resolve_query(adapter, query, ObservationRoot::Window(&window), &request)?; + require_complete(&resolution)?; + let ref_count = resolution.refmap.as_ref().map(RefMap::len); + let snapshot_id = match resolution.refmap.take() { + Some(refmap) => Some(persist_refmap(context, &refmap)?), + None => None, + }; + if let Some(snapshot_id) = snapshot_id.as_deref() { + for found in &mut resolution.matches { + if let Some(local_ref) = found.data.ref_id.as_deref() { + found.data.ref_id = Some(crate::ref_token::qualify_ref_id(snapshot_id, local_ref)); + } + } + } + emit_resolution(context, &resolution, snapshot_id.as_deref(), ref_count)?; + format_response(args, query, resolution, snapshot_id.as_deref()) +} + +fn resolve_request(args: &FindArgs, deadline: crate::Deadline) -> LocatorResolveRequest { + let selection = if args.selection.count { + LocatorSelection::Count + } else if args.selection.first { + LocatorSelection::First + } else if args.selection.last { + LocatorSelection::Last + } else if let Some(index) = args.selection.nth { + LocatorSelection::Nth(u32::try_from(index).unwrap_or(u32::MAX)) + } else { + LocatorSelection::All { + limit: args + .selection + .limit + .map_or(Some(DEFAULT_LIMIT as u32), |limit| { + (limit != 0).then(|| u32::try_from(limit).unwrap_or(u32::MAX)) + }), + } + }; + LocatorResolveRequest { + selection, + deadline, + max_raw_depth: MAX_RAW_DEPTH, + materialization: if args.selection.count { + LocatorMaterialization::None + } else { + LocatorMaterialization::SelectedMatches + }, + } +} + +fn require_complete(resolution: &LocatorResolution) -> Result<(), AppError> { + if resolution.meta.selection_complete { + return Ok(()); + } + Err( + AdapterError::timeout("Locator traversal did not produce an authoritative result") + .with_details(json!({ + "kind": "locator_incomplete", + "observed_matches": resolution.meta.total_matches, + "query_stats": resolution.stats, + "roles_present": resolution.meta.roles_present, + })) + .into(), + ) +} + +fn persist_refmap(context: &CommandContext, refmap: &RefMap) -> Result { + let store = RefStore::for_session(context.session_id())?; + let snapshot_id = store.save_new_snapshot(refmap)?; + trace_artifacts::copy_refmap_if_full(context, &store, &snapshot_id, refmap)?; + Ok(snapshot_id) +} + +fn emit_resolution( + context: &CommandContext, + resolution: &LocatorResolution, + snapshot_id: Option<&str>, + ref_count: Option, +) -> Result<(), AppError> { + context.trace_lazy("locator.resolve", || { + json!({ + "complete": resolution.meta.complete, + "match_count": resolution.meta.total_matches, + "query_stats": resolution.stats, + "ref_count": ref_count, + "snapshot_id": snapshot_id, + "truncated": resolution.meta.truncated, + }) + }) +} + +fn format_response( + args: &FindArgs, + query: &LocatorQuery, + resolution: LocatorResolution, + snapshot_id: Option<&str>, +) -> Result { + if args.selection.count { + return Ok(json!({ "count": resolution.meta.total_matches })); + } + let total_matches = resolution.meta.total_matches; + let truncated = resolution.meta.truncated; + let roles_present = resolution.meta.roles_present; + let matches = resolution + .matches + .into_iter() + .map(|found| serde_json::to_value(found.data)) + .collect::, _>>()?; + if args.selection.first || args.selection.last || args.selection.nth.is_some() { + let mut response = single_match_response(matches.into_iter().next(), query, roles_present); + attach_snapshot_id(&mut response, snapshot_id); + return Ok(response); + } + let mut response = json!({ + "matches": matches, + "total_matches": total_matches, + "truncated": truncated, + }); + let is_empty = response["matches"].as_array().is_some_and(Vec::is_empty); + attach_roles_present(&mut response, is_empty, query, roles_present); + attach_snapshot_id(&mut response, snapshot_id); + Ok(response) +} + +fn attach_snapshot_id(response: &mut Value, snapshot_id: Option<&str>) { + if let (Some(object), Some(snapshot_id)) = (response.as_object_mut(), snapshot_id) { + object.insert("snapshot_id".into(), json!(snapshot_id)); + } +} + +fn single_match_response( + found: Option, + query: &LocatorQuery, + roles_present: Vec, +) -> Value { + let is_empty = found.is_none(); + let mut response = json!({ "match": found }); + attach_roles_present(&mut response, is_empty, query, roles_present); + response +} + +fn attach_roles_present( + response: &mut Value, + is_empty: bool, + query: &LocatorQuery, + roles_present: Vec, +) { + if !is_empty || query.identity.role.is_none() { + return; + } + if let Some(object) = response.as_object_mut() { + object.insert("roles_present".into(), json!(roles_present)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::live_locator::{LocatorResolutionMeta, LocatorStats}; + + fn args(limit: Option) -> FindArgs { + FindArgs { + app: None, + window_id: None, + filter: crate::commands::find::FindFilterArgs { + role: None, + name: None, + description: None, + native_id: None, + value: None, + text: None, + exact: false, + }, + states: Vec::new(), + selection: crate::commands::find::FindSelectionArgs { + count: false, + first: false, + last: false, + nth: None, + limit, + }, + } + } + + #[test] + fn zero_limit_requests_all_matches() { + let request = resolve_request(&args(Some(0)), crate::Deadline::standard().unwrap()); + assert_eq!(request.selection, LocatorSelection::All { limit: None }); + } + + #[test] + fn all_response_exposes_total_and_truncation() { + let resolution = LocatorResolution { + matches: Vec::new(), + refmap: None, + stats: LocatorStats::default(), + meta: LocatorResolutionMeta { + total_matches: 72, + complete: true, + selection_complete: true, + truncated: true, + roles_present: vec!["button".into()], + }, + }; + let response = + format_response(&args(Some(50)), &LocatorQuery::default(), resolution, None).unwrap(); + + assert_eq!(response["total_matches"], 72); + assert_eq!(response["truncated"], true); + } +} diff --git a/crates/core/src/commands/find_live_tests.rs b/crates/core/src/commands/find_live_tests.rs new file mode 100644 index 0000000..25a8ad5 --- /dev/null +++ b/crates/core/src/commands/find_live_tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::{ + AdapterError, WindowInfo, + adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps, WindowFilter}, + live_locator::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorMaterialization, + LocatorRefEvidence, LocatorResolveRequest, LocatorSelection, LocatorStats, + ObservationRequest, ObservationRoot, ObservationSource, ObservedSubtree, ObservedTree, + require_unique, resolve_query, + }, + refs_store::RefStore, + refs_test_support::HomeGuard, +}; +pub(super) struct LiveFindAdapter { + structurally_complete: bool, +} + +impl LiveFindAdapter { + pub(super) fn complete() -> Self { + Self { + structurally_complete: true, + } + } + + fn incomplete() -> Self { + Self { + structurally_complete: false, + } + } + + fn evidence(role: &str, name: Option<&str>) -> LocatorEvidence { + LocatorEvidence { + role: LocatorField::Known(role.into()), + name: name + .map(|value| LocatorField::Known(value.into())) + .unwrap_or(LocatorField::Absent), + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::absent(), + states: LocatorField::Known(Vec::new()), + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Known(Vec::new()), + }, + } + } + + fn node(&self, evidence: LocatorEvidence, children: Vec) -> ObservedSubtree { + ObservedSubtree::new(evidence, children, self.structurally_complete, None) + } +} + +impl ObservationOps for LiveFindAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result { + if let ObservationRoot::Element { entry, .. } = &root { + return ObservedTree::from_roots( + vec![self.node( + Self::evidence(&entry.identity.role, entry.identity.name.as_deref()), + Vec::new(), + )], + ObservationSource::from_root(&root), + LocatorStats::default(), + self.structurally_complete, + ); + } + let ObservationRoot::Window(window) = &root else { + return Err(AdapterError::internal("expected locator root")); + }; + let window = *window; + let marker = if window.id == "w-2" { + "OnlyInWindowTwo" + } else { + "OnlyInWindowOne" + }; + let child = self.node(Self::evidence("button", Some(marker)), Vec::new()); + let root_node = self.node(Self::evidence("window", Some(&window.title)), vec![child]); + ObservedTree::from_roots( + vec![root_node], + ObservationSource::from_root(&root), + LocatorStats::default(), + self.structurally_complete, + ) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(vec![ + WindowInfo { + id: "w-1".into(), + title: "First".into(), + app: "FixtureApp".into(), + pid: crate::ProcessId::new(101), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }, + WindowInfo { + id: "w-2".into(), + title: "Second".into(), + app: "FixtureApp".into(), + pid: crate::ProcessId::new(102), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: false, + ..Default::default() + }, + }, + ]) + } + + fn resolve_locator_anchor( + &self, + _entry: &crate::refs::RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } +} + +impl ActionOps for LiveFindAdapter {} + +impl InputOps for LiveFindAdapter {} +impl SystemOps for LiveFindAdapter {} + +fn named_find(window_id: &str, name: &str) -> FindArgs { + FindArgs { + app: None, + window_id: Some(window_id.into()), + filter: FindFilterArgs { + name: Some(name.into()), + role: None, + description: None, + native_id: None, + value: None, + text: None, + exact: false, + }, + states: Vec::new(), + selection: FindSelectionArgs { + count: false, + first: false, + last: false, + nth: None, + limit: None, + }, + } +} + +fn unfiltered_find(window_id: &str, selection: FindSelectionArgs) -> FindArgs { + FindArgs { + app: None, + window_id: Some(window_id.into()), + filter: FindFilterArgs { + role: None, + name: None, + description: None, + native_id: None, + value: None, + text: None, + exact: false, + }, + states: Vec::new(), + selection, + } +} + +#[test] +fn execute_uses_live_locator_tree_and_persists_its_full_refmap() { + let _guard = HomeGuard::new(); + let adapter = LiveFindAdapter::complete(); + + let response = execute( + named_find("w-2", "OnlyInWindowTwo"), + &adapter, + &CommandContext::default(), + ) + .expect("live find should succeed"); + + let matches = response["matches"].as_array().expect("matches array"); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0]["name"], "OnlyInWindowTwo"); + let snapshot_id = response["snapshot_id"] + .as_str() + .expect("find refs must carry their snapshot namespace"); + assert_eq!(matches[0]["ref_id"], format!("@{snapshot_id}:e1")); + assert_eq!( + RefStore::new() + .unwrap() + .load_snapshot(snapshot_id) + .unwrap() + .len(), + 1 + ); + assert_eq!(RefStore::new().unwrap().load_latest().unwrap().len(), 1); +} + +#[test] +fn sequential_window_finds_return_distinct_ref_namespaces() { + let _guard = HomeGuard::new(); + let adapter = LiveFindAdapter::complete(); + + let first = execute( + named_find("w-1", "OnlyInWindowOne"), + &adapter, + &CommandContext::default(), + ) + .unwrap(); + let second = execute( + named_find("w-2", "OnlyInWindowTwo"), + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + let first_id = first["snapshot_id"].as_str().unwrap(); + let second_id = second["snapshot_id"].as_str().unwrap(); + assert_ne!(first_id, second_id); + let store = RefStore::new().unwrap(); + assert_eq!( + store + .load_snapshot(first_id) + .unwrap() + .get("@e1") + .unwrap() + .process + .pid, + 101 + ); + assert_eq!( + store + .load_snapshot(second_id) + .unwrap() + .get("@e1") + .unwrap() + .process + .pid, + 102 + ); +} + +#[test] +fn execute_rejects_an_incomplete_live_traversal_with_structured_evidence() { + let _guard = HomeGuard::new(); + let adapter = LiveFindAdapter::incomplete(); + + let error = execute( + named_find("w-1", "missing"), + &adapter, + &CommandContext::default(), + ) + .expect_err("an incomplete zero-match result is not authoritative"); + + assert_eq!(error.code(), "TIMEOUT"); + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!( + error.details.as_ref().unwrap()["kind"], + "locator_incomplete" + ); + assert_eq!(error.details.as_ref().unwrap()["observed_matches"], 0); +} + +#[test] +fn count_uses_live_evidence_without_creating_a_snapshot() { + let _guard = HomeGuard::new(); + let adapter = LiveFindAdapter::complete(); + let selection = FindSelectionArgs { + count: true, + first: false, + last: false, + nth: None, + limit: None, + }; + + let response = execute( + unfiltered_find("w-2", selection), + &adapter, + &CommandContext::default(), + ) + .expect("live count should succeed"); + + assert_eq!(response["count"], 2); + assert!(response.get("snapshot_id").is_none()); + assert_eq!( + RefStore::new().unwrap().load_latest().unwrap_err().code(), + "SNAPSHOT_NOT_FOUND" + ); +} + +#[test] +fn empty_live_role_result_reports_observed_roles() { + let _guard = HomeGuard::new(); + let adapter = LiveFindAdapter::complete(); + let mut args = named_find("w-1", "missing"); + args.filter.name = None; + args.filter.role = Some("toolbar".into()); + + let response = execute(args, &adapter, &CommandContext::default()) + .expect("complete empty role query should succeed"); + + let roles = response["roles_present"] + .as_array() + .expect("empty role result should include observed roles"); + assert!(roles.iter().any(|role| role == "button")); + assert!(roles.iter().any(|role| role == "window")); +} + +#[test] +fn ordinal_modes_preserve_live_document_order() { + let _guard = HomeGuard::new(); + let adapter = LiveFindAdapter::complete(); + let first = FindSelectionArgs { + count: false, + first: true, + last: false, + nth: None, + limit: None, + }; + let last = FindSelectionArgs { + count: false, + first: false, + last: true, + nth: None, + limit: None, + }; + + let first_response = execute( + unfiltered_find("w-2", first), + &adapter, + &CommandContext::default(), + ) + .expect("first should succeed"); + let last_response = execute( + unfiltered_find("w-2", last), + &adapter, + &CommandContext::default(), + ) + .expect("last should succeed"); + + assert_eq!(first_response["match"]["name"], "Second"); + assert_eq!(last_response["match"]["name"], "OnlyInWindowTwo"); +} + +#[test] +fn strict_live_resolution_reports_bounded_ambiguous_candidates() { + let adapter = LiveFindAdapter::complete(); + let windows = adapter + .list_windows( + &WindowFilter::default(), + crate::Deadline::standard().unwrap(), + ) + .unwrap(); + let request = LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(1)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::FullRefMap, + }; + let resolution = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&windows[1]), + &request, + ) + .expect("live resolution should succeed"); + + let error = require_unique(resolution) + .err() + .expect("two matches must be ambiguous"); + + assert_eq!(error.code(), "AMBIGUOUS_TARGET"); + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!(error.details.as_ref().unwrap()["candidate_count"], 2); + assert_eq!( + error.details.as_ref().unwrap()["candidate_count_exact"], + true + ); +} diff --git a/crates/core/src/commands/find_tests.rs b/crates/core/src/commands/find_tests.rs index a0930e2..4d488bc 100644 --- a/crates/core/src/commands/find_tests.rs +++ b/crates/core/src/commands/find_tests.rs @@ -1,30 +1,64 @@ use super::*; +use crate::LocatorQuery; +use crate::context::CommandContext; +use crate::refs_test_support::HomeGuard; fn node(name: Option<&str>, value: Option<&str>, description: Option<&str>) -> AccessibilityNode { AccessibilityNode { ref_id: Some("@e1".into()), role: "textfield".into(), - name: name.map(String::from), - value: value.map(String::from), - description: description.map(String::from), - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: name.map(String::from), + value: value.map(String::from), + description: description.map(String::from), + native_id: None, + }, + presentation: Default::default(), children_count: None, children: vec![], } } +fn search_tree( + root: &AccessibilityNode, + query: &LocatorQuery, + path: &mut Vec, + matches: &mut Vec, + max_matches: Option, +) { + collect_snapshot_matches(root, query, path, matches, max_matches); +} + +fn query_from_args(args: &FindArgs) -> LocatorQuery { + locator_query_from_args(args).unwrap() +} + +fn no_filter() -> FindFilterArgs { + FindFilterArgs { + role: None, + name: None, + description: None, + native_id: None, + value: None, + text: None, + exact: false, + } +} + +fn no_selection() -> FindSelectionArgs { + FindSelectionArgs { + count: false, + first: false, + last: false, + nth: None, + limit: None, + } +} + #[test] fn display_name_prefers_value_before_description() { let root = node(None, Some("current value"), Some("help text")); - let query = FindQuery { - role: None, - name: None, - value: None, - text: None, - }; + let query = LocatorQuery::default(); let mut matches = Vec::new(); search_tree(&root, &query, &mut Vec::new(), &mut matches, None); @@ -35,13 +69,8 @@ fn display_name_prefers_value_before_description() { #[test] fn search_tree_match_uses_ref_id_contract_and_includes_states() { let mut root = node(Some("Save"), None, None); - root.states = vec!["enabled".into()]; - let query = FindQuery { - role: None, - name: None, - value: None, - text: None, - }; + root.presentation.states = vec!["enabled".into()]; + let query = LocatorQuery::default(); let mut matches = Vec::new(); search_tree(&root, &query, &mut Vec::new(), &mut matches, None); @@ -54,11 +83,9 @@ fn search_tree_match_uses_ref_id_contract_and_includes_states() { #[test] fn search_tree_matches_text_across_fields() { let root = node(None, Some("Primary"), Some("Secondary")); - let query = FindQuery { - role: None, - name: None, - value: None, - text: Some(search_text::normalize("secondary")), + let query = LocatorQuery { + has_text: Some(search_text::normalize("secondary")), + ..LocatorQuery::default() }; let mut matches = Vec::new(); @@ -72,23 +99,16 @@ fn default_limit_caps_materialized_matches() { let root = AccessibilityNode { ref_id: None, role: "window".into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: (0..60) .map(|i| node(Some(&format!("Button {i}")), None, None)) .collect(), }; - let query = FindQuery { - role: None, - name: None, - value: None, - text: Some(search_text::normalize("button")), + let query = LocatorQuery { + has_text: Some(search_text::normalize("button")), + ..LocatorQuery::default() }; let mut matches = Vec::new(); @@ -107,15 +127,14 @@ fn default_limit_caps_materialized_matches() { fn limit_conflicts_with_single_result_modes_for_batch_too() { let err = validate_find_mode(&FindArgs { app: None, - role: None, - name: None, - value: None, - text: None, - count: false, - first: true, - last: false, - nth: None, - limit: Some(10), + window_id: None, + filter: no_filter(), + states: vec![], + selection: FindSelectionArgs { + first: true, + limit: Some(10), + ..no_selection() + }, }) .unwrap_err(); @@ -127,27 +146,20 @@ fn count_matches_does_not_build_result_json() { let root = AccessibilityNode { ref_id: None, role: "window".into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: vec![ node(Some("Save"), None, None), node(Some("Cancel"), None, None), ], }; - let query = FindQuery { - role: None, - name: None, - value: None, - text: Some(search_text::normalize("a")), + let query = LocatorQuery { + has_text: Some(search_text::normalize("a")), + ..LocatorQuery::default() }; - assert_eq!(count_matches(&root, &query), 2); + assert_eq!(count_matches(&root, &query), 3); } fn role_node(role: &str, name: Option<&str>) -> AccessibilityNode { @@ -157,21 +169,19 @@ fn role_node(role: &str, name: Option<&str>) -> AccessibilityNode { } #[test] -fn textarea_alias_resolves_to_textfield_query() { - let query = FindQuery::from_args(&FindArgs { +fn role_alias_is_preserved_until_live_validation() { + let query = query_from_args(&FindArgs { app: None, - role: Some("textarea".into()), - name: None, - value: None, - text: None, - count: false, - first: false, - last: false, - nth: None, - limit: None, + window_id: None, + filter: FindFilterArgs { + role: Some("textarea".into()), + ..no_filter() + }, + states: vec![], + selection: no_selection(), }); - assert_eq!(query.role.as_deref(), Some("textfield")); + assert_eq!(query.identity.role.as_deref(), Some("textarea")); let root = node(None, Some("doc body"), None); let mut matches = Vec::new(); @@ -180,21 +190,19 @@ fn textarea_alias_resolves_to_textfield_query() { } #[test] -fn unknown_role_passes_through_and_matches_nothing() { - let query = FindQuery::from_args(&FindArgs { +fn unknown_role_is_preserved_until_validation() { + let query = query_from_args(&FindArgs { app: None, - role: Some("navbar".into()), - name: None, - value: None, - text: None, - count: false, - first: false, - last: false, - nth: None, - limit: None, + window_id: None, + filter: FindFilterArgs { + role: Some("navbar".into()), + ..no_filter() + }, + states: vec![], + selection: no_selection(), }); - assert_eq!(query.role.as_deref(), Some("navbar")); + assert_eq!(query.identity.role.as_deref(), Some("navbar")); let root = role_node("textfield", Some("body")); let mut matches = Vec::new(); @@ -210,17 +218,15 @@ fn empty_role_filtered_result_reports_roles_present_from_tree() { role_node("textfield", None), ]; - let query = FindQuery::from_args(&FindArgs { + let query = query_from_args(&FindArgs { app: None, - role: Some("navbar".into()), - name: None, - value: None, - text: None, - count: false, - first: false, - last: false, - nth: None, - limit: None, + window_id: None, + filter: FindFilterArgs { + role: Some("navbar".into()), + ..no_filter() + }, + states: vec![], + selection: no_selection(), }); let response = single_match_response(None, &query, &root); @@ -235,17 +241,15 @@ fn empty_role_filtered_result_reports_roles_present_from_tree() { #[test] fn roles_present_hint_is_omitted_when_a_match_is_found() { let root = role_node("textfield", Some("body")); - let query = FindQuery::from_args(&FindArgs { + let query = query_from_args(&FindArgs { app: None, - role: Some("textfield".into()), - name: None, - value: None, - text: None, - count: false, - first: false, - last: false, - nth: None, - limit: None, + window_id: None, + filter: FindFilterArgs { + role: Some("textfield".into()), + ..no_filter() + }, + states: vec![], + selection: no_selection(), }); let mut matches = Vec::new(); @@ -256,3 +260,46 @@ fn roles_present_hint_is_omitted_when_a_match_is_found() { assert!(response.get("roles_present").is_none()); } + +fn find_args_scoped_to_window(window_id: &str) -> FindArgs { + FindArgs { + app: None, + window_id: Some(window_id.into()), + filter: FindFilterArgs { + name: Some("OnlyInWindowTwo".into()), + ..no_filter() + }, + states: vec![], + selection: no_selection(), + } +} + +#[test] +fn find_scopes_matches_to_requested_window_id() { + let _guard = HomeGuard::new(); + let context = CommandContext::default(); + let adapter = super::live_tests::LiveFindAdapter::complete(); + + let from_window_two = execute(find_args_scoped_to_window("w-2"), &adapter, &context) + .expect("find scoped to w-2 should succeed"); + let hits = from_window_two["matches"] + .as_array() + .expect("matches must be an array"); + assert_eq!( + hits.len(), + 1, + "window w-2's tree owns the only match, got: {from_window_two}" + ); + assert_eq!(hits[0]["name"], "OnlyInWindowTwo"); + + let from_window_one = execute(find_args_scoped_to_window("w-1"), &adapter, &context) + .expect("find scoped to w-1 should succeed"); + let misses = from_window_one["matches"] + .as_array() + .expect("matches must be an array"); + assert!( + misses.is_empty(), + "window w-1 must not surface window w-2's element \ + (window_id must not be ignored or swapped with app), got: {from_window_one}" + ); +} diff --git a/crates/core/src/commands/focus.rs b/crates/core/src/commands/focus.rs index 4ac69ac..4084f33 100644 --- a/crates/core/src/commands/focus.rs +++ b/crates/core/src/commands/focus.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/focus_window.rs b/crates/core/src/commands/focus_window.rs index 5dbfc67..471a0d5 100644 --- a/crates/core/src/commands/focus_window.rs +++ b/crates/core/src/commands/focus_window.rs @@ -1,15 +1,8 @@ use crate::{ + AppError, adapter::{PlatformAdapter, WindowFilter}, - error::{AdapterError, AppError, ErrorCode}, - node::WindowInfo, }; use serde_json::{Value, json}; -use std::time::{Duration, Instant}; - -const FOCUS_SETTLE_TIMEOUT_MS: u64 = 750; -const FOCUS_POLL_INTERVAL_MS: u64 = 50; -const FOCUS_CONFIRMATIONS: u8 = 2; - pub struct FocusWindowArgs { pub window_id: Option, pub app: Option, @@ -17,11 +10,12 @@ pub struct FocusWindowArgs { } pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result { + let deadline = crate::Deadline::standard()?; let filter = WindowFilter { focused_only: false, app: args.app.clone(), }; - let windows = adapter.list_windows(&filter)?; + let windows = adapter.list_windows(&filter, deadline)?; let window = if let Some(id) = &args.window_id { windows.into_iter().find(|w| &w.id == id) @@ -41,87 +35,18 @@ pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result, -) -> Result { - wait_for_focused_window_with_poll_interval( - adapter, - window_id, - app.as_deref(), - Duration::from_millis(FOCUS_POLL_INTERVAL_MS), - ) -} - -fn wait_for_focused_window_with_poll_interval( - adapter: &dyn PlatformAdapter, - window_id: &str, - app: Option<&str>, - poll_interval: Duration, -) -> Result { - let deadline = Instant::now() + Duration::from_millis(FOCUS_SETTLE_TIMEOUT_MS); - let mut confirmations = 0; - loop { - match observed_focused_window(adapter, app)? { - Some(window) if window.id == window_id => { - confirmations += 1; - if confirmations >= FOCUS_CONFIRMATIONS { - return Ok(window); - } - } - _ => { - confirmations = 0; - } - } - - if Instant::now() >= deadline { - return Err(AppError::Adapter( - AdapterError::new( - ErrorCode::ActionFailed, - "Window focus did not settle on the requested window", - ) - .with_suggestion("Run 'list-windows' to refresh window IDs, then retry."), - )); - } - - if !poll_interval.is_zero() { - std::thread::sleep(poll_interval); - } - } -} - -fn observed_focused_window( - adapter: &dyn PlatformAdapter, - app: Option<&str>, -) -> Result, AppError> { - match adapter.focused_window() { - Ok(window) => Ok(window), - Err(err) if err.code == ErrorCode::PlatformNotSupported => adapter - .list_windows(&WindowFilter { - focused_only: true, - app: app.map(str::to_string), - }) - .map(|windows| windows.into_iter().next()) - .map_err(AppError::Adapter), - Err(err) => Err(AppError::Adapter(err)), - } -} - #[cfg(test)] #[path = "focus_window_tests.rs"] mod tests; diff --git a/crates/core/src/commands/focus_window_tests.rs b/crates/core/src/commands/focus_window_tests.rs index ca2350b..1118c7a 100644 --- a/crates/core/src/commands/focus_window_tests.rs +++ b/crates/core/src/commands/focus_window_tests.rs @@ -1,5 +1,8 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{AdapterError, WindowInfo}; use std::sync::Mutex; +use std::time::Duration; struct FocusAdapter { windows: Vec, @@ -8,20 +11,39 @@ struct FocusAdapter { focused_window_supported: bool, } -impl PlatformAdapter for FocusAdapter { - fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError> { +impl ObservationOps for FocusAdapter { + fn list_windows( + &self, + filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { if filter.focused_only { Ok(self.focused_windows.lock().unwrap().clone()) } else { Ok(self.windows.clone()) } } +} - fn focus_window(&self, _win: &WindowInfo) -> Result<(), AdapterError> { +impl ActionOps for FocusAdapter {} + +impl InputOps for FocusAdapter {} + +impl SystemOps for FocusAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn focus_window( + &self, + _win: &WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { Ok(()) } - fn focused_window(&self) -> Result, AdapterError> { + fn focused_window( + &self, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { *self.focused_window_calls.lock().unwrap() += 1; if !self.focused_window_supported { return Err(AdapterError::not_supported("focused_window")); @@ -40,9 +62,13 @@ fn window(id: &str, focused: bool) -> WindowInfo { id: id.into(), title: "Main".into(), app: "TextEdit".into(), - pid: 42, + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), bounds: None, - is_focused: focused, + state: crate::WindowState { + is_focused: focused, + ..Default::default() + }, } } @@ -133,9 +159,14 @@ fn focus_confirmation_resets_after_transient_wrong_window() { focused_window_supported: true, }; - let value = - wait_for_focused_window_with_poll_interval(&adapter, &target.id, None, Duration::ZERO) - .unwrap(); + let value = crate::window_focus::wait_for_focused_window_with_poll_interval( + &adapter, + &target.id, + None, + Duration::ZERO, + crate::Deadline::standard().unwrap(), + ) + .unwrap(); assert_eq!(value.id, "w1"); assert_eq!(*adapter.focused_window_calls.lock().unwrap(), 4); diff --git a/crates/core/src/commands/get.rs b/crates/core/src/commands/get.rs index d5f02b1..62ceb6a 100644 --- a/crates/core/src/commands/get.rs +++ b/crates/core/src/commands/get.rs @@ -1,8 +1,8 @@ use crate::{ + AppError, adapter::{PlatformAdapter, optional_live_read}, commands::helpers::resolve_ref_with_context, context::CommandContext, - error::AppError, }; use serde_json::{Value, json}; @@ -28,20 +28,21 @@ pub fn execute( ) -> Result { let (entry, handle) = resolve_ref_with_context(&args.ref_id, args.snapshot_id.as_deref(), adapter, context)?; + let deadline = crate::Deadline::standard()?; let (prop_name, value) = match args.property { - GetProperty::Role => ("role", json!(entry.role)), - GetProperty::Title => ("title", json!(entry.name)), + GetProperty::Role => ("role", json!(entry.identity.role)), + GetProperty::Title => ("title", json!(entry.identity.name)), GetProperty::Text => { - let live = optional_live_read(adapter.get_live_value(handle.handle()))?; - ("text", json!(live.or(entry.value))) + let live = optional_live_read(adapter.get_live_value(&handle, deadline))?; + ("text", json!(live.or(entry.identity.value))) } GetProperty::Value => { - let live = optional_live_read(adapter.get_live_value(handle.handle()))?; - ("value", json!(live.or(entry.value))) + let live = optional_live_read(adapter.get_live_value(&handle, deadline))?; + ("value", json!(live.or(entry.identity.value))) } - GetProperty::Bounds => ("bounds", json!(entry.bounds)), - GetProperty::States => ("states", json!(entry.states)), + GetProperty::Bounds => ("bounds", json!(entry.geometry.bounds)), + GetProperty::States => ("states", json!(entry.capabilities.states)), }; Ok(json!({ "property": prop_name, "ref": args.ref_id, "value": value })) diff --git a/crates/core/src/commands/helpers.rs b/crates/core/src/commands/helpers.rs index 45a7e5c..04e77a6 100644 --- a/crates/core/src/commands/helpers.rs +++ b/crates/core/src/commands/helpers.rs @@ -1,81 +1,71 @@ use crate::{ - action::WindowOp, + AppError, action_request::ActionRequest, - action_result::ActionResult, - adapter::{PlatformAdapter, TreeOptions, WindowFilter}, + adapter::{PlatformAdapter, TreeOptions}, commands::{wait_selector, wait_selector::WaitSelectorInput}, context::CommandContext, - error::AppError, - node::WindowInfo, - refs::{RefEntry, validate_ref_id}, + ref_action_wait_context::RefActionWaitContext, + ref_resolve_deadline::resolve_within_deadline, + refs::RefEntry, refs_store::RefStore, - resolved_element::ResolvedElement, + resolve_attempt_outcome::ResolveAttemptOutcome, window_lookup, }; use serde_json::{Value, json}; -pub struct AppArgs { - pub app: Option, -} +pub(crate) use crate::app_lookup::{process_identity, resolve_app, revalidate_app_for_mutation}; + +pub use super::window_target::AppArgs; +pub(crate) use super::window_target::{ + resolve_window_for_app, revalidate_window_for_mutation, window_op_command, +}; pub struct RefArgs { pub ref_id: String, pub snapshot_id: Option, + pub timeout_ms: Option, } -pub(crate) fn resolve_ref_with_context<'a>( +pub(crate) fn acquire_interaction_lease( + adapter: &dyn PlatformAdapter, +) -> Result { + Ok(adapter.acquire_interaction_lease(crate::Deadline::standard()?)?) +} + +pub fn normalize_action_timeout_ms(raw: u64) -> Option { + if raw == 0 { None } else { Some(raw) } +} + +pub(crate) fn resolve_ref_with_context( ref_id: &str, snapshot_id: Option<&str>, - adapter: &'a dyn PlatformAdapter, + adapter: &dyn PlatformAdapter, context: &CommandContext, -) -> Result<(RefEntry, ResolvedElement<'a>), AppError> { - validate_ref_id(ref_id)?; - let store = RefStore::for_session(context.session_id())?; - context.trace_lazy( - "ref.resolve.start", - || json!({ "ref": ref_id, "snapshot_id": snapshot_id }), - )?; - let refmap = store.load(snapshot_id).inspect_err(|e| { - tracing::debug!("refmap load failed: {e}"); - let _ = context.trace_lazy("ref.resolve.error", || { - json!({ - "ref": ref_id, - "snapshot_id": snapshot_id, - "code": e.code(), - "message": e.to_string() - }) - }); - })?; - let entry = match refmap.get(ref_id) { - Some(entry) => entry.clone(), - None => { - context.trace_lazy("ref.resolve.error", || { - json!({ - "ref": ref_id, - "snapshot_id": snapshot_id, - "code": "STALE_REF", - "message": "ref not found in current RefMap" - }) - })?; - return Err(AppError::stale_ref(ref_id)); - } - }; - tracing::debug!( - "resolve: {} -> pid={} role={} name_chars={:?}", +) -> Result<(RefEntry, crate::adapter::NativeHandle), AppError> { + resolve_ref_within_deadline( ref_id, - entry.pid, - entry.role, - entry.name.as_deref().map(|name| name.chars().count()) - ); - context.trace_lazy("ref.resolve.entry", || { - json!({ - "ref": ref_id, - "pid": entry.pid, - "role": entry.role, - "name": entry.name - }) - })?; - let handle = adapter.resolve_element_strict(&entry).inspect_err(|err| { + snapshot_id, + crate::Deadline::standard()?, + adapter, + context, + ) +} + +/// Resolves a ref to a live element handle, capping the strict resolve to +/// `deadline` when supplied (the `hover`/`drag` wait path) or resolving +/// uncapped otherwise (`get`/`is`). Delegates entry loading and its +/// `ref.resolve.start/entry/error` tracing to [`load_ref_entry`], then adds +/// handle resolution and the `ref.resolve.ok` event, so budgeted and +/// single-shot resolution trace identically. +fn resolve_ref_within_deadline( + ref_id: &str, + snapshot_id: Option<&str>, + deadline: crate::Deadline, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result<(RefEntry, crate::adapter::NativeHandle), AppError> { + let entry = load_ref_entry(ref_id, snapshot_id, context)?; + let handle = resolve_handle_within_deadline(adapter, &entry, deadline).inspect_err(|err| { let _ = context.trace_lazy("ref.resolve.error", || { json!({ "ref": ref_id, @@ -88,29 +78,23 @@ pub(crate) fn resolve_ref_with_context<'a>( })?; tracing::debug!("resolve: {} resolved successfully", ref_id); context.trace_lazy("ref.resolve.ok", || json!({ "ref": ref_id }))?; - Ok((entry, ResolvedElement::new(adapter, handle))) + Ok((entry, handle)) } -pub(crate) fn resolve_app_pid( - app: Option<&str>, +/// Performs the strict resolve for [`resolve_ref_within_deadline`], capping the +/// attempt to `deadline` when one is supplied and surfacing an exhausted budget +/// as a `TIMEOUT`, or resolving uncapped when it is not. +pub(crate) fn resolve_handle_within_deadline( adapter: &dyn PlatformAdapter, -) -> Result { - if let Some(name) = app { - let apps = adapter.list_apps()?; - apps.into_iter() - .find(|a| a.name.eq_ignore_ascii_case(name)) - .map(|a| a.pid) - .ok_or_else(|| AppError::invalid_input(format!("App '{name}' not found"))) - } else { - let filter = WindowFilter { - focused_only: true, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - windows - .first() - .map(|w| w.pid) - .ok_or_else(|| AppError::invalid_input("No focused window. Use --app to specify.")) + entry: &RefEntry, + deadline: crate::Deadline, +) -> Result { + match resolve_within_deadline(adapter, entry, deadline) { + ResolveAttemptOutcome::Resolved(handle) => Ok(handle), + ResolveAttemptOutcome::Failed(err) => Err(err), + ResolveAttemptOutcome::DeadlinePassed => Err(crate::AdapterError::timeout( + "Target did not resolve within the wait budget", + )), } } @@ -120,32 +104,59 @@ pub(crate) fn execute_ref_action_with_context( request: ActionRequest, context: &CommandContext, ) -> Result { - let (entry, result) = execute_ref_action_result_with_context( - &args.ref_id, - args.snapshot_id.as_deref(), - adapter, - request, - context, - )?; - apply_post_action_wait(serde_json::to_value(result)?, &entry, adapter, context) + let request = request.with_timeout_ms(args.timeout_ms); + validate_post_action_wait(context)?; + let entry = load_ref_entry(&args.ref_id, args.snapshot_id.as_deref(), context)?; + let (result, lease, pre, deadline, lease_started) = + crate::ref_action_wait::execute_with_auto_wait_and_lease( + RefActionWaitContext { + adapter, + entry: &entry, + ref_id: &args.ref_id, + context, + }, + request, + crate::ref_action::dispatch_resolved, + ) + .map_err(AppError::Adapter)?; + let value = serde_json::to_value(result).map_err(|error| { + post_delivery_error(AppError::Json(error), json!({ "action": "delivered" })) + })?; + let lease_hold_ms = u64::try_from(lease_started.elapsed().as_millis()).unwrap_or(u64::MAX); + drop(lease); + let mut outcome = apply_post_action_wait(value, Some(&entry), adapter, context); + update_lease_hold_ms(&mut outcome, lease_hold_ms); + crate::ref_action::finish_artifacts( + crate::ref_action_context::RefActionContext::new( + RefActionWaitContext { + adapter, + entry: &entry, + ref_id: &args.ref_id, + context, + }, + deadline, + ), + &pre, + ); + outcome } -/// Resolves the app name a ref belongs to for post-action polling. Normal -/// refmaps always carry `source_app`; the pid lookup is a fallback for legacy -/// or partially-populated entries so the wait never silently polls the focused -/// window instead of the acted-on app. pub(crate) fn probe_app_name(adapter: &dyn PlatformAdapter, entry: &RefEntry) -> Option { - if entry.source_app.is_some() { - return entry.source_app.clone(); + if entry.source.source_app.is_some() { + return entry.source.source_app.clone(); } - window_lookup::find_window_for_pid(entry.pid, adapter) + let identity = crate::ProcessIdentity::new( + entry.process.pid, + entry.process.process_instance.as_deref()?, + ); + window_lookup::find_window_for_process(identity, adapter, crate::Deadline::standard().ok()?) .ok() .map(|window| window.app) } pub(crate) fn apply_post_action_wait( result: Value, - entry: &RefEntry, + entry: Option<&RefEntry>, adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { @@ -156,8 +167,8 @@ pub(crate) fn apply_post_action_wait( WaitSelectorInput { query_raw: wait.query_raw.clone(), gone: wait.gone, - app: probe_app_name(adapter, entry), - window_id: entry.source_window_id.clone(), + app: entry.and_then(|entry| probe_app_name(adapter, entry)), + window_id: entry.and_then(|entry| entry.source.source_window_id.clone()), opts: TreeOptions::default(), timeout_ms: wait.timeout_ms, }, @@ -175,72 +186,171 @@ pub(crate) fn apply_post_action_wait( if let Some(obj) = details.as_object_mut() { obj.insert("after_action".into(), result); } - Err(AppError::Adapter(adapter_err.with_details(details))) + Err(AppError::Adapter( + adapter_err + .with_details(details) + .with_disposition(crate::DeliverySemantics::delivered_unverified()), + )) } - Err(err) => Err(err), + Err(err) => Err(post_delivery_error(err, result)), } } +fn update_lease_hold_ms(result: &mut Result, lease_hold_ms: u64) { + fn update(value: &mut Value, lease_hold_ms: u64) { + if let Some(object) = value.as_object_mut() { + if let Some(auto_wait) = object.get_mut("auto_wait").and_then(Value::as_object_mut) { + auto_wait.insert("lease_hold_ms".into(), json!(lease_hold_ms)); + } + for value in object.values_mut() { + update(value, lease_hold_ms); + } + } else if let Some(values) = value.as_array_mut() { + for value in values { + update(value, lease_hold_ms); + } + } + } + + match result { + Ok(value) => update(value, lease_hold_ms), + Err(AppError::Adapter(error)) => { + if let Some(details) = &mut error.details { + update(details, lease_hold_ms); + } + } + Err(_) => {} + } +} + +pub(crate) fn validate_post_action_wait(context: &CommandContext) -> Result<(), AppError> { + let Some(wait) = context.wait_selector() else { + return Ok(()); + }; + crate::commands::query::validate_selector(&wait.query_raw)?; + crate::Deadline::after(wait.timeout_ms)?; + Ok(()) +} + +fn post_delivery_error(error: AppError, result: Value) -> AppError { + let mut adapter_error = match error { + AppError::Adapter(error) => error, + other => crate::AdapterError::internal(other.to_string()), + }; + let mut details = adapter_error.details.take().unwrap_or_else(|| json!({})); + if let Some(object) = details.as_object_mut() { + object.insert("after_action".into(), result); + } + AppError::Adapter( + adapter_error + .with_details(details) + .with_disposition(crate::DeliverySemantics::delivered_unverified()), + ) +} + +#[cfg(test)] pub(crate) fn execute_ref_action_result_with_context( ref_id: &str, snapshot_id: Option<&str>, adapter: &dyn PlatformAdapter, request: ActionRequest, context: &CommandContext, -) -> Result<(RefEntry, ActionResult), AppError> { - let (entry, handle) = resolve_ref_with_context(ref_id, snapshot_id, adapter, context)?; - let result = crate::ref_action::execute_resolved( - crate::ref_action::ResolvedRefAction { +) -> Result<(RefEntry, crate::ActionResult), AppError> { + let entry = load_ref_entry(ref_id, snapshot_id, context)?; + let result = crate::ref_action_wait::execute_with_auto_wait( + RefActionWaitContext { adapter, entry: &entry, - handle: handle.handle(), ref_id, context, }, request, - )?; + crate::ref_action::dispatch_resolved, + ) + .map_err(AppError::Adapter)?; Ok((entry, result)) } -pub(crate) fn window_op_command( - args: AppArgs, - adapter: &dyn PlatformAdapter, - op: WindowOp, - response_key: &'static str, -) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let win = match window_lookup::find_window_for_pid(pid, adapter) { - Ok(win) => win, - Err(_) if matches!(op, WindowOp::Restore) => WindowInfo { - id: String::new(), - title: String::new(), - app: args.app.unwrap_or_default(), - pid, - bounds: None, - is_focused: false, - }, - Err(err) => return Err(err), +/// Shared owner of ref-entry loading and its `ref.resolve.start/entry/error` +/// tracing, used by both the ref-action path +/// ([`execute_ref_action_result_with_context`]) and, via +/// [`resolve_ref_within_deadline`], the pointer/get/is resolve path, so a +/// stale ref emits identical telemetry regardless of caller. +/// [`resolve_ref_within_deadline`] builds handle resolution and the +/// `ref.resolve.ok` event on top of the entry this returns. +pub(crate) fn load_ref_entry( + ref_id: &str, + snapshot_id: Option<&str>, + context: &CommandContext, +) -> Result { + let (resolved_snapshot_id, local_ref) = + crate::ref_token::resolve_ref_target(ref_id, snapshot_id)?; + let store = RefStore::for_session(context.session_id())?; + context.trace_lazy( + "ref.resolve.start", + || json!({ "ref": ref_id, "snapshot_id": resolved_snapshot_id }), + )?; + let refmap = store + .load_snapshot(&resolved_snapshot_id) + .inspect_err(|e| { + tracing::debug!("refmap load failed: {e}"); + let _ = context.trace_lazy("ref.resolve.error", || { + json!({ + "ref": ref_id, + "snapshot_id": resolved_snapshot_id, + "code": e.code(), + "message": e.to_string() + }) + }); + })?; + let entry = match refmap.get(&local_ref) { + Some(entry) => entry.clone(), + None => { + context.trace_lazy("ref.resolve.error", || { + json!({ + "ref": ref_id, + "snapshot_id": resolved_snapshot_id, + "code": "STALE_REF", + "message": "ref not found in current RefMap" + }) + })?; + return Err(AppError::stale_ref(ref_id)); + } }; - adapter.window_op(&win, op)?; - Ok(json!({ response_key: true })) -} - -pub(crate) fn resolve_window_for_app( - app: Option<&str>, - adapter: &dyn PlatformAdapter, -) -> Result { - let pid = resolve_app_pid(app, adapter)?; - window_lookup::find_window_for_pid(pid, adapter) + tracing::debug!( + "resolve: {} -> pid={} role={} name_chars={:?}", + ref_id, + entry.process.pid, + entry.identity.role, + entry + .identity + .name + .as_deref() + .map(|name| name.chars().count()) + ); + context.trace_lazy("ref.resolve.entry", || { + json!({ + "ref": ref_id, + "pid": entry.process.pid, + "role": entry.identity.role, + "name": entry.identity.name + }) + })?; + Ok(entry) } #[cfg(test)] #[path = "helpers_test_support.rs"] -mod test_support; +pub(super) mod test_support; #[cfg(test)] #[path = "helpers_tests.rs"] mod tests; #[cfg(test)] -#[path = "helpers_ref_action_tests.rs"] -mod ref_action_tests; +#[path = "helpers_ref_action_dispatch_tests.rs"] +mod ref_action_dispatch_tests; + +#[cfg(test)] +#[path = "helpers_ref_action_wait_tests.rs"] +mod ref_action_wait_tests; diff --git a/crates/core/src/commands/helpers_ref_action_dispatch_tests.rs b/crates/core/src/commands/helpers_ref_action_dispatch_tests.rs new file mode 100644 index 0000000..32c1704 --- /dev/null +++ b/crates/core/src/commands/helpers_ref_action_dispatch_tests.rs @@ -0,0 +1,237 @@ +use super::test_support::{entry, text_entry}; +use super::*; +use crate::AdapterError; +use crate::adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}; +use crate::refs::RefMap; +use crate::refs_test_support::HomeGuard; +use crate::{ + action::Action, action_result::ActionResult, action_step::ActionStep, + element_state::ElementState, interaction_policy::InteractionPolicy, +}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; + +struct RecordingAdapter { + request: Mutex>, +} + +impl ObservationOps for RecordingAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!( + "textfield", + "OK", + [crate::capability::CLICK, crate::capability::TYPE_TEXT] + ); +} + +impl ActionOps for RecordingAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + *self.request.lock().unwrap() = Some(request); + Ok(ActionResult::delivered_unverified("ok") + .with_state(ElementState { + role: "textfield".into(), + states: vec!["focused".into()], + value: Some("updated".into()), + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }) + .with_steps(vec![ActionStep::succeeded("AXPress")])) + } +} + +impl InputOps for RecordingAdapter {} + +impl SystemOps for RecordingAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +struct AmbiguousAdapter { + executed: AtomicU32, +} + +impl ObservationOps for AmbiguousAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Err( + AdapterError::ambiguous_target("2 candidates matched").with_details( + serde_json::json!({ + "candidate_count": 2, + "candidates": [{ "name": "Private OK" }] + }), + ), + ) + } +} + +impl ActionOps for AmbiguousAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + self.executed.fetch_add(1, Ordering::SeqCst); + Ok(ActionResult::delivered_unverified("unexpected")) + } +} + +impl InputOps for AmbiguousAdapter {} + +impl SystemOps for AmbiguousAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +#[test] +fn execute_ref_action_preserves_action_and_policy() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + let mut target = text_entry(); + target + .capabilities + .available_actions + .push(crate::capability::CLICK.into()); + refmap.allocate(target); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = RecordingAdapter { + request: Mutex::new(None), + }; + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + execute_ref_action_with_context( + args, + &adapter, + ActionRequest::headless(Action::Click), + &CommandContext::default(), + ) + .unwrap(); + + let request = adapter.request.lock().unwrap().clone().unwrap(); + assert!(matches!(request.action, Action::Click)); + assert_eq!(request.policy, InteractionPolicy::headless()); +} + +#[test] +fn execute_ref_action_does_not_dispatch_ambiguous_target() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + refmap.allocate(entry()); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = AmbiguousAdapter { + executed: AtomicU32::new(0), + }; + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + let err = execute_ref_action_with_context( + args, + &adapter, + ActionRequest::headless(Action::Click), + &CommandContext::default(), + ) + .expect_err("ambiguous targets fail before action dispatch"); + + assert_eq!(err.code(), "AMBIGUOUS_TARGET"); + assert_eq!(adapter.executed.load(Ordering::SeqCst), 0); +} + +#[test] +fn ref_action_trace_includes_ambiguous_details_without_candidate_names() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + refmap.allocate(entry()); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = AmbiguousAdapter { + executed: AtomicU32::new(0), + }; + let trace_path = std::env::temp_dir().join(format!( + "agent-desktop-ambiguous-trace-{}.jsonl", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let context = CommandContext::new(None, Some(trace_path.clone()), true).unwrap(); + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + let err = execute_ref_action_with_context( + args, + &adapter, + ActionRequest::headless(Action::Click), + &context, + ) + .expect_err("ambiguous targets fail before action dispatch"); + + assert_eq!(err.code(), "AMBIGUOUS_TARGET"); + let trace = std::fs::read_to_string(&trace_path).unwrap(); + assert!(trace.contains("\"candidate_count\":2")); + assert!(!trace.contains("Private OK")); + let _ = std::fs::remove_file(trace_path); +} + +#[test] +fn ref_action_trace_does_not_include_typed_text_payload() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + refmap.allocate(text_entry()); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = RecordingAdapter { + request: Mutex::new(None), + }; + let trace_path = std::env::temp_dir().join(format!( + "agent-desktop-type-trace-{}.jsonl", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let context = CommandContext::new(None, Some(trace_path.clone()), true).unwrap(); + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + execute_ref_action_with_context( + args, + &adapter, + ActionRequest::focus_fallback(Action::TypeText("super-secret".into())), + &context, + ) + .unwrap(); + + let trace = std::fs::read_to_string(&trace_path).unwrap(); + assert!(trace.contains("\"action\":\"type\"")); + assert!(trace.contains("\"event\":\"action.dispatch.start\"")); + assert!(trace.contains("\"event\":\"action.dispatch.ok\"")); + assert!(trace.contains("\"post_state\"")); + assert!(trace.contains("\"steps\"")); + assert!(!trace.contains("super-secret")); + let _ = std::fs::remove_file(trace_path); +} diff --git a/crates/core/src/commands/helpers_ref_action_tests.rs b/crates/core/src/commands/helpers_ref_action_tests.rs deleted file mode 100644 index 31c3267..0000000 --- a/crates/core/src/commands/helpers_ref_action_tests.rs +++ /dev/null @@ -1,454 +0,0 @@ -use super::test_support::{entry, text_entry}; -use super::*; -use crate::adapter::{NativeHandle, WindowFilter}; -use crate::context::WaitSelector; -use crate::error::AdapterError; -use crate::node::{AccessibilityNode, WindowInfo}; -use crate::refs::RefMap; -use crate::refs_test_support::HomeGuard; -use crate::{ - action::Action, action_result::ActionResult, action_step::ActionStep, - element_state::ElementState, interaction_policy::InteractionPolicy, -}; -use std::sync::Mutex; -use std::sync::atomic::{AtomicU32, Ordering}; - -struct RecordingAdapter { - request: Mutex>, -} - -impl PlatformAdapter for RecordingAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - Ok(NativeHandle::null()) - } - - fn execute_action( - &self, - _handle: &NativeHandle, - request: ActionRequest, - ) -> Result { - *self.request.lock().unwrap() = Some(request); - Ok(ActionResult::new("ok") - .with_state(ElementState { - role: "textfield".into(), - states: vec!["focused".into()], - value: Some("updated".into()), - }) - .with_steps(vec![ActionStep::succeeded("AXPress")])) - } -} - -struct AmbiguousAdapter { - executed: AtomicU32, -} - -impl PlatformAdapter for AmbiguousAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - Err( - AdapterError::ambiguous_target("2 candidates matched").with_details( - serde_json::json!({ - "candidate_count": 2, - "candidates": [{ "name": "Private OK" }] - }), - ), - ) - } - - fn execute_action( - &self, - _handle: &NativeHandle, - _request: ActionRequest, - ) -> Result { - self.executed.fetch_add(1, Ordering::SeqCst); - Ok(ActionResult::new("unexpected")) - } -} - -#[test] -fn execute_ref_action_preserves_action_and_policy() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - refmap.allocate(entry()); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = RecordingAdapter { - request: Mutex::new(None), - }; - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - execute_ref_action_with_context( - args, - &adapter, - ActionRequest::headless(Action::Click), - &CommandContext::default(), - ) - .unwrap(); - - let request = adapter.request.lock().unwrap().clone().unwrap(); - assert!(matches!(request.action, Action::Click)); - assert_eq!(request.policy, InteractionPolicy::headless()); -} - -#[test] -fn execute_ref_action_does_not_dispatch_ambiguous_target() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - refmap.allocate(entry()); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = AmbiguousAdapter { - executed: AtomicU32::new(0), - }; - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - let err = execute_ref_action_with_context( - args, - &adapter, - ActionRequest::headless(Action::Click), - &CommandContext::default(), - ) - .expect_err("ambiguous targets fail before action dispatch"); - - assert_eq!(err.code(), "AMBIGUOUS_TARGET"); - assert_eq!(adapter.executed.load(Ordering::SeqCst), 0); -} - -#[test] -fn ref_action_trace_includes_ambiguous_details_without_candidate_names() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - refmap.allocate(entry()); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = AmbiguousAdapter { - executed: AtomicU32::new(0), - }; - let trace_path = std::env::temp_dir().join(format!( - "agent-desktop-ambiguous-trace-{}.jsonl", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let context = CommandContext::new(None, Some(trace_path.clone()), true).unwrap(); - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - let err = execute_ref_action_with_context( - args, - &adapter, - ActionRequest::headless(Action::Click), - &context, - ) - .expect_err("ambiguous targets fail before action dispatch"); - - assert_eq!(err.code(), "AMBIGUOUS_TARGET"); - let trace = std::fs::read_to_string(&trace_path).unwrap(); - assert!(trace.contains("\"candidate_count\":2")); - assert!(!trace.contains("Private OK")); - let _ = std::fs::remove_file(trace_path); -} - -#[test] -fn ref_action_trace_does_not_include_typed_text_payload() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - refmap.allocate(text_entry()); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = RecordingAdapter { - request: Mutex::new(None), - }; - let trace_path = std::env::temp_dir().join(format!( - "agent-desktop-type-trace-{}.jsonl", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let context = CommandContext::new(None, Some(trace_path.clone()), true).unwrap(); - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - execute_ref_action_with_context( - args, - &adapter, - ActionRequest::focus_fallback(Action::TypeText("super-secret".into())), - &context, - ) - .unwrap(); - - let trace = std::fs::read_to_string(&trace_path).unwrap(); - assert!(trace.contains("\"action\":\"type\"")); - assert!(trace.contains("\"event\":\"action.dispatch.start\"")); - assert!(trace.contains("\"event\":\"action.dispatch.ok\"")); - assert!(trace.contains("\"post_state\"")); - assert!(trace.contains("\"steps\"")); - assert!(!trace.contains("super-secret")); - let _ = std::fs::remove_file(trace_path); -} - -struct ScopedWaitAdapter { - request: Mutex>, - polled_app: Mutex>, -} - -impl PlatformAdapter for ScopedWaitAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - Ok(NativeHandle::null()) - } - - fn execute_action( - &self, - _handle: &NativeHandle, - request: ActionRequest, - ) -> Result { - *self.request.lock().unwrap() = Some(request); - Ok(ActionResult::new("ok")) - } - - fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError> { - *self.polled_app.lock().unwrap() = filter.app.clone(); - Ok(vec![WindowInfo { - id: "w-1".into(), - title: "Doc".into(), - app: filter.app.clone().unwrap_or_else(|| "TargetApp".into()), - pid: 1, - bounds: None, - is_focused: true, - }]) - } - - fn get_tree( - &self, - _win: &WindowInfo, - _opts: &crate::adapter::TreeOptions, - ) -> Result { - Ok(AccessibilityNode { - ref_id: None, - role: "window".into(), - name: Some("Saved!".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, - children_count: None, - children: vec![], - }) - } -} - -#[test] -fn post_action_wait_scopes_to_source_app_and_merges_action_result() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - let mut entry = entry(); - entry.source_app = Some("TargetApp".into()); - refmap.allocate(entry); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = ScopedWaitAdapter { - request: Mutex::new(None), - polled_app: Mutex::new(None), - }; - let context = CommandContext::default().with_wait_selector(Some(WaitSelector { - query_raw: ":saved!".into(), - gone: false, - timeout_ms: 5_000, - })); - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - let value = execute_ref_action_with_context( - args, - &adapter, - ActionRequest::headless(Action::Click), - &context, - ) - .unwrap(); - - assert_eq!( - adapter.polled_app.lock().unwrap().as_deref(), - Some("TargetApp") - ); - assert_eq!(value["after_action"]["action"], "ok"); - assert_eq!(value["matched_selector"], ":saved!"); -} - -struct MultiWindowAdapter; - -impl PlatformAdapter for MultiWindowAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - Ok(NativeHandle::null()) - } - - fn execute_action( - &self, - _handle: &NativeHandle, - _request: ActionRequest, - ) -> Result { - Ok(ActionResult::new("ok")) - } - - fn list_windows(&self, _filter: &WindowFilter) -> Result, AdapterError> { - Ok(vec![ - WindowInfo { - id: "w-other".into(), - title: "Other".into(), - app: "App".into(), - pid: 1, - bounds: None, - is_focused: true, - }, - WindowInfo { - id: "w-target".into(), - title: "Target".into(), - app: "App".into(), - pid: 1, - bounds: None, - is_focused: false, - }, - ]) - } - - fn get_tree( - &self, - win: &WindowInfo, - _opts: &crate::adapter::TreeOptions, - ) -> Result { - let children = if win.id == "w-target" { - vec![AccessibilityNode { - ref_id: None, - role: "button".into(), - name: Some("Saved!".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, - children_count: None, - children: vec![], - }] - } else { - vec![] - }; - Ok(AccessibilityNode { - ref_id: None, - role: "window".into(), - name: Some(win.title.clone()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, - children_count: None, - children, - }) - } -} - -#[test] -fn post_action_wait_polls_acted_on_window_not_focused_window() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - let mut entry = entry(); - entry.source_app = Some("App".into()); - entry.source_window_id = Some("w-target".into()); - refmap.allocate(entry); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let context = CommandContext::default().with_wait_selector(Some(WaitSelector { - query_raw: ":saved!".into(), - gone: false, - timeout_ms: 500, - })); - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - let value = execute_ref_action_with_context( - args, - &MultiWindowAdapter, - ActionRequest::headless(Action::Click), - &context, - ) - .expect("wait must match in the acted-on window, not the focused empty window"); - assert_eq!(value["matched_selector"], ":saved!"); - assert_eq!(value["window"]["id"], "w-target"); -} - -#[test] -fn post_action_wait_without_flag_returns_action_only() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - refmap.allocate(entry()); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = RecordingAdapter { - request: Mutex::new(None), - }; - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - let value = execute_ref_action_with_context( - args, - &adapter, - ActionRequest::headless(Action::Click), - &CommandContext::default(), - ) - .unwrap(); - - assert_eq!(value["action"], "ok"); - assert!(value.get("after_action").is_none()); -} - -#[test] -fn post_action_wait_timeout_embeds_action_result_in_details() { - let _guard = HomeGuard::new(); - let mut refmap = RefMap::new(); - let mut entry = entry(); - entry.source_app = Some("TargetApp".into()); - refmap.allocate(entry); - let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = ScopedWaitAdapter { - request: Mutex::new(None), - polled_app: Mutex::new(None), - }; - let context = CommandContext::default().with_wait_selector(Some(WaitSelector { - query_raw: ":never-appears".into(), - gone: false, - timeout_ms: 50, - })); - let args = RefArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id), - }; - - let err = execute_ref_action_with_context( - args, - &adapter, - ActionRequest::headless(Action::Click), - &context, - ) - .unwrap_err(); - - assert_eq!(err.code(), "TIMEOUT"); - let details = match err { - crate::error::AppError::Adapter(adapter_err) => adapter_err.details.expect("details"), - other => panic!("expected adapter timeout, got {other:?}"), - }; - assert_eq!(details["kind"], "wait_timeout"); - assert_eq!(details["after_action"]["action"], "ok"); -} diff --git a/crates/core/src/commands/helpers_ref_action_wait_result_tests.rs b/crates/core/src/commands/helpers_ref_action_wait_result_tests.rs new file mode 100644 index 0000000..5e876c5 --- /dev/null +++ b/crates/core/src/commands/helpers_ref_action_wait_result_tests.rs @@ -0,0 +1,63 @@ +use super::*; + +#[test] +fn post_action_wait_without_flag_returns_action_only() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + refmap.allocate(entry()); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = ScopedWaitAdapter::new(); + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + let value = execute_ref_action_with_context( + args, + &adapter, + ActionRequest::headless(Action::Click), + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["action"], "ok"); + assert!(value.get("after_action").is_none()); +} + +#[test] +fn post_action_wait_timeout_embeds_action_result_in_details() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + let mut entry = entry(); + entry.source.source_app = Some("TargetApp".into()); + refmap.allocate(entry); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = ScopedWaitAdapter::new(); + let context = CommandContext::default().with_wait_selector(Some(WaitSelector { + query_raw: ":never-appears".into(), + gone: false, + timeout_ms: 50, + })); + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + let err = execute_ref_action_with_context( + args, + &adapter, + ActionRequest::headless(Action::Click), + &context, + ) + .unwrap_err(); + + assert_eq!(err.code(), "TIMEOUT"); + let details = match err { + crate::AppError::Adapter(adapter_err) => adapter_err.details.expect("details"), + other => panic!("expected adapter timeout, got {other:?}"), + }; + assert_eq!(details["kind"], "wait_timeout"); + assert_eq!(details["after_action"]["action"], "ok"); +} diff --git a/crates/core/src/commands/helpers_ref_action_wait_tests.rs b/crates/core/src/commands/helpers_ref_action_wait_tests.rs new file mode 100644 index 0000000..3a66b0b --- /dev/null +++ b/crates/core/src/commands/helpers_ref_action_wait_tests.rs @@ -0,0 +1,352 @@ +use super::test_support::entry; +use super::*; +use crate::AdapterError; +use crate::adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps, WindowFilter}; +use crate::context::WaitSelector; +use crate::refs::RefMap; +use crate::refs_test_support::HomeGuard; +use crate::{AccessibilityNode, WindowInfo}; +use crate::{action::Action, action_result::ActionResult}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU32, Ordering}, +}; + +struct LeaseGuard(Arc); + +impl Drop for LeaseGuard { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +struct ScopedWaitAdapter { + request: Mutex>, + polled_app: Mutex>, + lease_held: Arc, + lease_free_polls: AtomicU32, +} + +impl ScopedWaitAdapter { + fn new() -> Self { + Self { + request: Mutex::new(None), + polled_app: Mutex::new(None), + lease_held: Arc::new(AtomicBool::new(false)), + lease_free_polls: AtomicU32::new(0), + } + } +} + +impl ObservationOps for ScopedWaitAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result { + if self.lease_held.load(Ordering::SeqCst) { + return Err(AdapterError::internal( + "post-action wait must poll with the interaction lease released", + )); + } + self.lease_free_polls.fetch_add(1, Ordering::SeqCst); + crate::adapter::observed_tree( + &root, + AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some("Saved!".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }, + ) + } + + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn list_windows( + &self, + filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + *self.polled_app.lock().unwrap() = filter.app.clone(); + Ok(vec![WindowInfo { + id: "w-1".into(), + title: "Doc".into(), + app: filter.app.clone().unwrap_or_else(|| "TargetApp".into()), + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }]) + } + + fn get_tree( + &self, + _win: &WindowInfo, + _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, + ) -> Result { + Ok(AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some("Saved!".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }) + } + + crate::adapter::complete_live_observation!("button", "OK", [crate::capability::CLICK]); +} + +impl ActionOps for ScopedWaitAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + assert!(self.lease_held.load(Ordering::SeqCst)); + *self.request.lock().unwrap() = Some(request); + Ok(ActionResult::delivered_unverified("ok")) + } +} + +impl InputOps for ScopedWaitAdapter {} + +impl SystemOps for ScopedWaitAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + self.lease_held + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .map_err(|_| AdapterError::internal("test lease already held"))?; + crate::InteractionLease::guarded(deadline, LeaseGuard(Arc::clone(&self.lease_held))) + } +} + +#[test] +fn post_action_wait_scopes_to_source_app_and_merges_action_result() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + let mut entry = entry(); + entry.source.source_app = Some("TargetApp".into()); + refmap.allocate(entry); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = ScopedWaitAdapter::new(); + let context = CommandContext::default().with_wait_selector(Some(WaitSelector { + query_raw: ":saved!".into(), + gone: false, + timeout_ms: 5_000, + })); + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + let value = execute_ref_action_with_context( + args, + &adapter, + ActionRequest::headless(Action::Click), + &context, + ) + .unwrap(); + + assert_eq!( + adapter.polled_app.lock().unwrap().as_deref(), + Some("TargetApp") + ); + assert!(!adapter.lease_held.load(Ordering::SeqCst)); + assert!(adapter.lease_free_polls.load(Ordering::SeqCst) >= 1); + assert_eq!(value["after_action"]["action"], "ok"); + assert_eq!(value["matched_selector"], ":saved!"); +} + +struct MultiWindowAdapter; + +impl ObservationOps for MultiWindowAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result { + let crate::live_locator::ObservationRoot::Window(window) = root else { + return Err(AdapterError::internal("expected window root")); + }; + let children = (window.id == "w-target") + .then(|| AccessibilityNode { + ref_id: None, + role: "button".into(), + identity: crate::NodeIdentity { + name: Some("Saved!".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }) + .into_iter() + .collect(); + crate::adapter::observed_tree( + &crate::live_locator::ObservationRoot::Window(window), + AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some(window.title.clone()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children, + }, + ) + } + + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(vec![ + WindowInfo { + id: "w-other".into(), + title: "Other".into(), + app: "App".into(), + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }, + WindowInfo { + id: "w-target".into(), + title: "Target".into(), + app: "App".into(), + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: false, + ..Default::default() + }, + }, + ]) + } + + fn get_tree( + &self, + win: &WindowInfo, + _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, + ) -> Result { + let children = if win.id == "w-target" { + vec![AccessibilityNode { + ref_id: None, + role: "button".into(), + identity: crate::NodeIdentity { + name: Some("Saved!".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }] + } else { + vec![] + }; + Ok(AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some(win.title.clone()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children, + }) + } + + crate::adapter::complete_live_observation!("button", "OK", [crate::capability::CLICK]); +} + +impl ActionOps for MultiWindowAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("ok")) + } +} + +impl InputOps for MultiWindowAdapter {} + +impl SystemOps for MultiWindowAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +#[test] +fn post_action_wait_polls_acted_on_window_not_focused_window() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + let mut entry = entry(); + entry.source.source_app = Some("App".into()); + entry.source.source_window_id = Some("w-target".into()); + refmap.allocate(entry); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let context = CommandContext::default().with_wait_selector(Some(WaitSelector { + query_raw: ":saved!".into(), + gone: false, + timeout_ms: 500, + })); + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + timeout_ms: None, + }; + + let value = execute_ref_action_with_context( + args, + &MultiWindowAdapter, + ActionRequest::headless(Action::Click), + &context, + ) + .expect("wait must match in the acted-on window, not the focused empty window"); + assert_eq!(value["matched_selector"], ":saved!"); + assert_eq!(value["window"]["id"], "w-target"); +} + +#[path = "helpers_ref_action_wait_result_tests.rs"] +mod result_tests; diff --git a/crates/core/src/commands/helpers_test_support.rs b/crates/core/src/commands/helpers_test_support.rs index dec4e52..d8c4ba1 100644 --- a/crates/core/src/commands/helpers_test_support.rs +++ b/crates/core/src/commands/helpers_test_support.rs @@ -1,29 +1,63 @@ use crate::refs::RefEntry; pub(super) fn entry() -> RefEntry { + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Clear".into(), "Click".into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("OK".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec!["Clear".into(), "Click".into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } pub(super) fn text_entry() -> RefEntry { let mut entry = entry(); - entry.role = "textfield".into(); - entry.available_actions = vec!["SetValue".into()]; + entry.identity.role = "textfield".into(); + entry.capabilities.available_actions = vec!["SetValue".into()]; entry } + +pub(in crate::commands) fn save_one_ref_snapshot(role: &str, available_action: &str) -> String { + let mut entry = entry(); + entry.identity.role = role.into(); + entry.identity.name = Some("Target".into()); + entry.capabilities.available_actions = vec![available_action.into()]; + let mut refmap = crate::refs::RefMap::new(); + refmap.allocate(entry); + crate::refs_store::RefStore::new() + .unwrap() + .save_new_snapshot(&refmap) + .unwrap() +} diff --git a/crates/core/src/commands/helpers_tests.rs b/crates/core/src/commands/helpers_tests.rs index 894e9d4..287d3a3 100644 --- a/crates/core/src/commands/helpers_tests.rs +++ b/crates/core/src/commands/helpers_tests.rs @@ -1,50 +1,138 @@ use super::test_support::entry; use super::*; +use crate::AppInfo; use crate::action::Action; use crate::adapter::NativeHandle; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::capability; -use crate::error::{AdapterError, ErrorCode}; -use crate::node::AppInfo; use crate::refs::RefMap; use crate::refs_test_support::HomeGuard; -use std::sync::atomic::{AtomicU32, Ordering}; +use crate::{AdapterError, ErrorCode, WindowInfo, WindowOp}; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; -struct ReleaseCountingAdapter { - releases: AtomicU32, -} +struct DropProbe(Arc); -impl PlatformAdapter for ReleaseCountingAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - Ok(NativeHandle::null()) - } - - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - self.releases.fetch_add(1, Ordering::SeqCst); - Ok(()) +impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); } } +struct DropCountingAdapter { + drops: Arc, +} + +impl ObservationOps for DropCountingAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::new(DropProbe(Arc::clone(&self.drops)))) + } +} + +impl ActionOps for DropCountingAdapter {} + +impl InputOps for DropCountingAdapter {} + +impl SystemOps for DropCountingAdapter {} + struct RestoreWithoutWindowAdapter { op_count: AtomicU32, } -impl PlatformAdapter for RestoreWithoutWindowAdapter { - fn list_apps(&self) -> Result, AdapterError> { +struct TargetedWindowAdapter { + windows: Vec, + operated: AtomicU32, +} + +impl ObservationOps for TargetedWindowAdapter { + fn list_windows( + &self, + _filter: &crate::adapter::WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(self.windows.clone()) + } +} + +impl ActionOps for TargetedWindowAdapter {} + +impl InputOps for TargetedWindowAdapter {} + +impl SystemOps for TargetedWindowAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn resolve_window_strict( + &self, + win: &WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(win.clone()) + } + + fn window_op( + &self, + win: &WindowInfo, + op: WindowOp, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + assert_eq!(win.id, "w-real"); + assert!(matches!(op, WindowOp::Restore)); + self.operated.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +fn targeted_window(id: &str, visible: bool) -> WindowInfo { + WindowInfo { + id: id.into(), + title: "Fixture".into(), + app: "Fixture".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + visible: Some(visible), + ..Default::default() + }, + } +} + +impl ObservationOps for RestoreWithoutWindowAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { Ok(vec![AppInfo { name: "TextEdit".into(), - pid: 42, + pid: crate::ProcessId::new(42), bundle_id: None, + process_instance: Some("test-instance".into()), }]) } fn list_windows( &self, _filter: &crate::adapter::WindowFilter, + _deadline: crate::Deadline, ) -> Result, AdapterError> { Err(AdapterError::new(ErrorCode::WindowNotFound, "no windows")) } +} - fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> { +impl ActionOps for RestoreWithoutWindowAdapter {} + +impl InputOps for RestoreWithoutWindowAdapter {} + +impl SystemOps for RestoreWithoutWindowAdapter { + fn window_op( + &self, + win: &WindowInfo, + op: WindowOp, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { assert_eq!(win.pid, 42); assert!(matches!(op, WindowOp::Restore)); self.op_count.fetch_add(1, Ordering::SeqCst); @@ -53,32 +141,38 @@ impl PlatformAdapter for RestoreWithoutWindowAdapter { } #[test] -fn resolved_element_releases_handle_once_on_drop() { +fn normalize_action_timeout_ms_treats_zero_as_disabled() { + assert_eq!(normalize_action_timeout_ms(0), None); + assert_eq!(normalize_action_timeout_ms(1), Some(1)); +} + +#[test] +fn resolved_native_handle_drops_payload_once() { let _guard = HomeGuard::new(); let mut refmap = RefMap::new(); refmap.allocate(entry()); let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); - let adapter = ReleaseCountingAdapter { - releases: AtomicU32::new(0), + let adapter = DropCountingAdapter { + drops: Arc::new(AtomicU32::new(0)), }; { - let (_entry, resolved) = resolve_ref_with_context( + let (_entry, handle) = resolve_ref_with_context( "@e1", Some(&snapshot_id), &adapter, &CommandContext::default(), ) .unwrap(); - let _handle = resolved.handle(); - assert_eq!(adapter.releases.load(Ordering::SeqCst), 0); + assert!(!handle.is_null()); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 0); } - assert_eq!(adapter.releases.load(Ordering::SeqCst), 1); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); } #[test] -fn explicit_session_snapshot_resolves_without_session_context() { +fn explicit_session_snapshot_resolves_with_matching_session_context() { let _guard = HomeGuard::new(); let mut refmap = RefMap::new(); refmap.allocate(entry()); @@ -86,27 +180,27 @@ fn explicit_session_snapshot_resolves_without_session_context() { .unwrap() .save_new_snapshot(&refmap) .unwrap(); - let adapter = ReleaseCountingAdapter { - releases: AtomicU32::new(0), + let adapter = DropCountingAdapter { + drops: Arc::new(AtomicU32::new(0)), }; - let (_entry, resolved) = resolve_ref_with_context( + let (_entry, handle) = resolve_ref_with_context( "@e1", Some(&snapshot_id), &adapter, - &CommandContext::default(), + &CommandContext::new(Some("agent-a".into()), None, false).unwrap(), ) .unwrap(); - let _handle = resolved.handle(); + assert!(!handle.is_null()); - assert_eq!(adapter.releases.load(Ordering::SeqCst), 0); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 0); } #[test] fn missing_snapshot_keeps_snapshot_not_found_error() { let _guard = HomeGuard::new(); - let adapter = ReleaseCountingAdapter { - releases: AtomicU32::new(0), + let adapter = DropCountingAdapter { + drops: Arc::new(AtomicU32::new(0)), }; let err = match resolve_ref_with_context( @@ -124,14 +218,40 @@ fn missing_snapshot_keeps_snapshot_not_found_error() { } #[test] -fn restore_can_run_when_no_window_is_currently_listed() { +fn restore_fails_closed_when_no_window_is_currently_listed() { let adapter = RestoreWithoutWindowAdapter { op_count: AtomicU32::new(0), }; - let value = window_op_command( + let err = window_op_command( AppArgs { app: Some("TextEdit".into()), + window_id: None, + }, + &adapter, + WindowOp::Restore, + "restored", + ) + .unwrap_err(); + + assert_eq!(err.code(), "WINDOW_NOT_FOUND"); + assert_eq!(adapter.op_count.load(Ordering::SeqCst), 0); +} + +#[test] +fn window_id_targets_one_window_in_an_ambiguous_app_inventory() { + let adapter = TargetedWindowAdapter { + windows: vec![ + targeted_window("w-phantom", false), + targeted_window("w-real", true), + ], + operated: AtomicU32::new(0), + }; + + let value = window_op_command( + AppArgs { + app: Some("Fixture".into()), + window_id: Some("w-real".into()), }, &adapter, WindowOp::Restore, @@ -140,14 +260,14 @@ fn restore_can_run_when_no_window_is_currently_listed() { .unwrap(); assert_eq!(value["restored"], true); - assert_eq!(adapter.op_count.load(Ordering::SeqCst), 1); + assert_eq!(adapter.operated.load(Ordering::SeqCst), 1); } struct CountingPipelineAdapter { resolves: AtomicU32, live_reads: AtomicU32, executes: AtomicU32, - releases: AtomicU32, + drops: Arc, } impl CountingPipelineAdapter { @@ -156,50 +276,93 @@ impl CountingPipelineAdapter { resolves: AtomicU32::new(0), live_reads: AtomicU32::new(0), executes: AtomicU32::new(0), - releases: AtomicU32::new(0), + drops: Arc::new(AtomicU32::new(0)), } } } -impl PlatformAdapter for CountingPipelineAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { +impl ObservationOps for CountingPipelineAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { self.resolves.fetch_add(1, Ordering::SeqCst); - Ok(NativeHandle::null()) + Ok(NativeHandle::new(DropProbe(Arc::clone(&self.drops)))) } fn get_live_element( &self, _handle: &NativeHandle, + _deadline: crate::Deadline, ) -> Result { self.live_reads.fetch_add(1, Ordering::SeqCst); Ok(crate::adapter::LiveElement { - state: Some(crate::element_state::ElementState { + identity: crate::adapter::live_identity("OK"), + state: crate::element_state::ElementState { role: "button".into(), states: vec![], value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, + bounds: Some(crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, }), - bounds: None, - available_actions: Some(vec![capability::CLICK.into()]), + available_actions: vec![capability::CLICK.into()], }) } + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } +} + +impl ActionOps for CountingPipelineAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result { self.executes.fetch_add(1, Ordering::SeqCst); - Ok(crate::action_result::ActionResult::new("click")) - } - - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - self.releases.fetch_add(1, Ordering::SeqCst); - Ok(()) + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) } } +impl InputOps for CountingPipelineAdapter {} + +impl SystemOps for CountingPipelineAdapter { + crate::adapter::guarded_interaction_lease!(); +} + #[test] -fn ref_action_pipeline_makes_one_resolve_one_preflight_one_dispatch() { +fn ref_action_pipeline_resolves_under_lease_before_dispatch() { let _guard = HomeGuard::new(); let store = crate::refs_store::RefStore::new().unwrap(); let mut refmap = RefMap::new(); @@ -219,5 +382,5 @@ fn ref_action_pipeline_makes_one_resolve_one_preflight_one_dispatch() { assert_eq!(adapter.resolves.load(Ordering::SeqCst), 1); assert_eq!(adapter.live_reads.load(Ordering::SeqCst), 1); assert_eq!(adapter.executes.load(Ordering::SeqCst), 1); - assert_eq!(adapter.releases.load(Ordering::SeqCst), 1); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); } diff --git a/crates/core/src/commands/hover.rs b/crates/core/src/commands/hover.rs index 7d11c50..cc94d60 100644 --- a/crates/core/src/commands/hover.rs +++ b/crates/core/src/commands/hover.rs @@ -1,12 +1,15 @@ use crate::{ - action::{MouseButton, MouseEvent, MouseEventKind}, + AppError, MouseButton, MouseEvent, MouseEventKind, adapter::PlatformAdapter, - commands::point_resolve::{ - PointResolveArgs, focus_for_physical_input, require_cursor_policy, - resolve_point_from_ref_or_xy_with_context, + commands::{ + helpers::{apply_post_action_wait, validate_post_action_wait}, + point_resolve::{PointResolveArgs, require_cursor_policy}, + pointer_action::{ + PointResolveAttempt, ensure_point_deadline, focus_point_under_lease, point_deadline, + resolve_point_under_lease, retry_leased_point_phase, wait_for_point_with_deadline, + }, }, context::CommandContext, - error::AppError, }; use serde_json::{Value, json}; @@ -15,6 +18,7 @@ pub struct HoverArgs { pub snapshot_id: Option, pub xy: Option<(f64, f64)>, pub duration_ms: Option, + pub timeout_ms: Option, } pub fn execute( @@ -22,31 +26,76 @@ pub fn execute( adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { - require_cursor_policy(context, "hover")?; - let resolved = resolve_point_from_ref_or_xy_with_context( - PointResolveArgs { - ref_id: args.ref_id.as_deref(), - xy: args.xy, - snapshot_id: args.snapshot_id.as_deref(), - missing_input_message: "Provide a ref (@e1) or --xy x,y", - }, - adapter, - context, - )?; - let focused = focus_for_physical_input(resolved.pid, adapter, context)?; - adapter.mouse_event(MouseEvent { - kind: MouseEventKind::Move, - point: resolved.point.clone(), - button: MouseButton::Left, - })?; - if let Some(ms) = args.duration_ms { - std::thread::sleep(std::time::Duration::from_millis(ms)); + if args.duration_ms.is_some_and(|duration| duration > 0) { + return Err(AppError::invalid_input_with_suggestion( + "Hover duration is unavailable because a stateless process cannot guarantee cursor ownership during a dwell", + "Run hover without --duration, then use `wait ` for an explicit post-hover pause.", + )); } + require_cursor_policy(context, "hover")?; + validate_post_action_wait(context)?; + let deadline = point_deadline(args.timeout_ms)?; + let point_args = PointResolveArgs { + ref_id: args.ref_id.as_deref(), + xy: args.xy, + snapshot_id: args.snapshot_id.as_deref(), + missing_input_message: "Provide a ref (@e1) or --xy x,y", + headed_requirement: crate::HeadedRequirement::FocusedWindowAndCursor, + }; + let auto_wait = args.timeout_ms.is_some_and(|timeout_ms| timeout_ms > 0); + if auto_wait { + wait_for_point_with_deadline(point_args, deadline, adapter, context)?; + } + let (resolved, lease) = retry_leased_point_phase(args.timeout_ms, deadline, || { + let lease = adapter.acquire_interaction_lease(deadline)?; + let focused = focus_point_under_lease(point_args, &lease, adapter, context)?; + let first = resolve_point_under_lease( + PointResolveAttempt { + args: point_args, + stability: None, + allow_scroll: !auto_wait, + }, + deadline, + &lease, + adapter, + context, + )?; + let mut resolved = resolve_point_under_lease( + PointResolveAttempt { + args: point_args, + stability: Some(first.bounds_hash), + allow_scroll: false, + }, + deadline, + &lease, + adapter, + context, + )?; + resolved.focused = focused; + Ok((resolved, lease)) + })?; + ensure_point_deadline( + deadline, + Some(json!({ + "status": "actionable", + "point": { "x": resolved.point.x, "y": resolved.point.y } + })), + )?; + adapter.mouse_event( + MouseEvent { + kind: MouseEventKind::Move, + point: resolved.point.clone(), + button: MouseButton::Left, + modifiers: Vec::new(), + }, + &lease, + )?; let mut response = json!({ "hovered": true, "x": resolved.point.x, "y": resolved.point.y }); - if focused { + if resolved.focused { response["focused"] = json!(true); } - Ok(response) + drop(lease); + apply_post_action_wait(response, resolved.source_entry.as_ref(), adapter, context) } #[cfg(test)] diff --git a/crates/core/src/commands/hover_preflight_tests.rs b/crates/core/src/commands/hover_preflight_tests.rs new file mode 100644 index 0000000..6f7749e --- /dev/null +++ b/crates/core/src/commands/hover_preflight_tests.rs @@ -0,0 +1,342 @@ +use super::*; + +struct OccludedTargetAdapter { + moved_to: Mutex>, + occlusions_before_success: usize, + hit_tests: std::sync::atomic::AtomicUsize, + lease_acquisitions: std::sync::atomic::AtomicUsize, + lease_held: std::sync::Arc, +} + +struct LeaseFlag(std::sync::Arc); + +impl Drop for LeaseFlag { + fn drop(&mut self) { + self.0.store(false, std::sync::atomic::Ordering::SeqCst); + } +} + +impl ObservationOps for OccludedTargetAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 100.0, + y: 200.0, + width: 20.0, + height: 10.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + let attempt = self + .hit_tests + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if attempt >= self.occlusions_before_success { + return Ok(HitTestResult::ReachesTarget); + } + Ok(HitTestResult::InterceptedBy { + role: Some("AXSheet".into()), + name: Some("Save changes?".into()), + bounds: None, + }) + } +} + +impl ActionOps for OccludedTargetAdapter {} + +impl InputOps for OccludedTargetAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.moved_to.lock().unwrap() = Some(event); + Ok(()) + } +} + +impl SystemOps for OccludedTargetAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + assert!( + !self + .lease_held + .swap(true, std::sync::atomic::Ordering::SeqCst) + ); + self.lease_acquisitions + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, LeaseFlag(self.lease_held.clone())) + } + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +/// F27 regression: `hover --ref` previously resolved the ref's bounds to a +/// point and dispatched the mouse move without ever consulting `hit_test`, +/// so an occluded target (e.g. a modal sheet over it) was hovered blind. +/// This proves the preflight now fails before any mouse event is sent. +#[test] +fn hover_on_occluded_ref_fails_preflight_before_dispatch() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = OccludedTargetAdapter { + moved_to: Mutex::new(None), + occlusions_before_success: usize::MAX, + hit_tests: std::sync::atomic::AtomicUsize::new(0), + lease_acquisitions: std::sync::atomic::AtomicUsize::new(0), + lease_held: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let err = execute( + ref_args(snapshot_id), + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_FAILED"); + let AppError::Adapter(adapter_error) = &err else { + panic!("expected adapter actionability failure") + }; + assert!( + adapter_error + .details + .as_ref() + .is_some_and(|details| details.to_string().contains("AXSheet")) + ); + assert!(adapter.moved_to.lock().unwrap().is_none()); + assert_eq!( + adapter.hit_tests.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); + assert_eq!( + adapter + .lease_acquisitions + .load(std::sync::atomic::Ordering::SeqCst), + 1 + ); +} + +#[test] +fn headed_hover_retries_transient_post_focus_occlusion_without_holding_lease() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = OccludedTargetAdapter { + moved_to: Mutex::new(None), + occlusions_before_success: 1, + hit_tests: std::sync::atomic::AtomicUsize::new(0), + lease_acquisitions: std::sync::atomic::AtomicUsize::new(0), + lease_held: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + let mut args = ref_args(snapshot_id); + args.timeout_ms = Some(500); + + let value = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap(); + + assert_eq!(value["hovered"], true); + assert_eq!( + adapter.hit_tests.load(std::sync::atomic::Ordering::SeqCst), + 3 + ); + assert_eq!( + adapter + .lease_acquisitions + .load(std::sync::atomic::Ordering::SeqCst), + 2 + ); + assert!(!adapter.lease_held.load(std::sync::atomic::Ordering::SeqCst)); +} + +struct BackgroundTargetAdapter { + focused: std::sync::atomic::AtomicBool, + hit_tests: std::sync::atomic::AtomicUsize, + moved_to: Mutex>, +} + +impl ObservationOps for BackgroundTargetAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 100.0, + y: 200.0, + width: 20.0, + height: 10.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + self.hit_tests + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.focused.load(std::sync::atomic::Ordering::SeqCst) { + Ok(HitTestResult::ReachesTarget) + } else { + Ok(HitTestResult::InterceptedBy { + role: Some("window".into()), + name: Some("Foreground app".into()), + bounds: None, + }) + } + } +} + +impl ActionOps for BackgroundTargetAdapter {} + +impl InputOps for BackgroundTargetAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.moved_to.lock().unwrap() = Some(event); + Ok(()) + } +} + +impl SystemOps for BackgroundTargetAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.focused + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } +} + +#[test] +fn headed_hover_defers_hit_testing_until_after_focus() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = BackgroundTargetAdapter { + focused: std::sync::atomic::AtomicBool::new(false), + hit_tests: std::sync::atomic::AtomicUsize::new(0), + moved_to: Mutex::new(None), + }; + + execute( + ref_args(snapshot_id), + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert!(adapter.focused.load(std::sync::atomic::Ordering::SeqCst)); + assert_eq!( + adapter.hit_tests.load(std::sync::atomic::Ordering::SeqCst), + 2 + ); + assert!(adapter.moved_to.lock().unwrap().is_some()); +} + +#[test] +fn timeout_none_is_single_shot() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = StaleThenOkAdapter::new(1); + + let error = execute( + HoverArgs { + ref_id: Some("@e1".into()), + snapshot_id: Some(snapshot_id), + xy: None, + duration_ms: None, + timeout_ms: None, + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(error.code(), "STALE_REF"); + assert_eq!(adapter.retry.calls(), 1); +} + +#[test] +fn expired_xy_budget_never_dispatches_mouse_move() { + let adapter = HoverCaptureAdapter::new(); + + let err = execute( + HoverArgs { + ref_id: None, + snapshot_id: None, + xy: Some((5.0, 6.0)), + duration_ms: None, + timeout_ms: Some(0), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "TIMEOUT"); + let AppError::Adapter(adapter_error) = &err else { + panic!("expected adapter timeout") + }; + assert_eq!( + adapter_error + .details + .as_ref() + .and_then(|details| details.get("kind")), + Some(&serde_json::json!("actionability_timeout")) + ); + assert!(adapter.moved_to.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/hover_tests.rs b/crates/core/src/commands/hover_tests.rs index 0331666..1dedfd7 100644 --- a/crates/core/src/commands/hover_tests.rs +++ b/crates/core/src/commands/hover_tests.rs @@ -1,18 +1,24 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ + AdapterError, Rect, adapter::NativeHandle, capability, - error::AdapterError, - node::Rect, + commands::stale_retry_test_support::StaleRetryCounter, + hit_test::HitTestResult, refs::{RefEntry, RefMap}, refs_store::RefStore, refs_test_support::HomeGuard, }; -use std::sync::Mutex; +use std::sync::{ + Mutex, + atomic::{AtomicU32, Ordering}, +}; struct HoverCaptureAdapter { moved_to: Mutex>, - focused_pids: Mutex>, + focused_pids: Mutex>, + focused_bounds: Option, } impl HoverCaptureAdapter { @@ -20,16 +26,35 @@ impl HoverCaptureAdapter { Self { moved_to: Mutex::new(None), focused_pids: Mutex::new(Vec::new()), + focused_bounds: None, } } + + fn with_focused_bounds(mut self, bounds: Rect) -> Self { + self.focused_bounds = Some(bounds); + self + } } -impl PlatformAdapter for HoverCaptureAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { +impl ObservationOps for HoverCaptureAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { Ok(NativeHandle::null()) } - fn get_element_bounds(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + if !self.focused_pids.lock().unwrap().is_empty() + && let Some(bounds) = self.focused_bounds + { + return Ok(Some(bounds)); + } Ok(Some(Rect { x: 100.0, y: 200.0, @@ -38,37 +63,85 @@ impl PlatformAdapter for HoverCaptureAdapter { })) } - fn focus_app(&self, pid: i32) -> Result<(), AdapterError> { - self.focused_pids.lock().unwrap().push(pid); - Ok(()) + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) } +} - fn mouse_event(&self, event: MouseEvent) -> Result<(), AdapterError> { +impl ActionOps for HoverCaptureAdapter {} + +impl InputOps for HoverCaptureAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { *self.moved_to.lock().unwrap() = Some(event); Ok(()) } } -fn ref_snapshot(pid: i32) -> String { +impl SystemOps for HoverCaptureAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.focused_pids.lock().unwrap().push(window.pid); + Ok(()) + } +} + +fn ref_snapshot(pid: u32) -> String { let store = RefStore::new().unwrap(); let mut refmap = RefMap::new(); refmap.allocate(RefEntry { - pid, - role: "button".into(), - name: Some("Target".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(pid), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Target".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some(format!("App {pid}")), + source_window_id: Some(format!("w-{pid}")), + source_window_title: Some(format!("Window {pid}")), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); store.save_new_snapshot(&refmap).unwrap() } @@ -79,6 +152,7 @@ fn ref_args(snapshot_id: String) -> HoverArgs { snapshot_id: Some(snapshot_id), xy: None, duration_ms: None, + timeout_ms: None, } } @@ -114,6 +188,26 @@ fn headed_ref_hover_focuses_target_app_once() { assert_eq!(value["y"], 205.0); } +#[test] +fn positive_timeout_uses_stable_post_focus_hover_bounds() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = HoverCaptureAdapter::new().with_focused_bounds(Rect { + x: 300.0, + y: 400.0, + width: 20.0, + height: 10.0, + }); + let mut args = ref_args(snapshot_id); + args.timeout_ms = Some(500); + + let value = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap(); + + assert_eq!(*adapter.focused_pids.lock().unwrap(), vec![42]); + assert_eq!(value["x"], 310.0); + assert_eq!(value["y"], 405.0); +} + #[test] fn headed_xy_hover_never_steals_focus() { let adapter = HoverCaptureAdapter::new(); @@ -124,6 +218,7 @@ fn headed_xy_hover_never_steals_focus() { snapshot_id: None, xy: Some((5.0, 6.0)), duration_ms: None, + timeout_ms: None, }, &adapter, &CommandContext::default().with_headed(true), @@ -133,3 +228,126 @@ fn headed_xy_hover_never_steals_focus() { assert!(adapter.focused_pids.lock().unwrap().is_empty()); assert!(value.get("focused").is_none()); } + +struct StaleThenOkAdapter { + retry: StaleRetryCounter, + minimum_resolves_before_lease: u32, + lease_acquisitions: AtomicU32, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + retry: StaleRetryCounter::new(fail_until), + minimum_resolves_before_lease: 0, + lease_acquisitions: AtomicU32::new(0), + } + } + + fn expect_poll_before_lease(mut self, minimum: u32) -> Self { + self.minimum_resolves_before_lease = minimum; + self + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.retry.attempt() + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 100.0, + y: 200.0, + width: 20.0, + height: 10.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } +} + +impl ActionOps for StaleThenOkAdapter {} + +impl InputOps for StaleThenOkAdapter { + fn mouse_event( + &self, + _event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +impl SystemOps for StaleThenOkAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + assert!(self.retry.calls() >= self.minimum_resolves_before_lease); + self.lease_acquisitions.fetch_add(1, Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, ()) + } + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +/// Regression for the F2 fix: `hover --ref` previously had no `--timeout-ms` +/// at all, so a transient `STALE_REF` on the resolved ref failed the command +/// outright. This proves the wired budget retries through the real +/// `hover::execute` path. +#[test] +fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = StaleThenOkAdapter::new(2).expect_poll_before_lease(3); + + let value = execute( + HoverArgs { + ref_id: Some("@e1".into()), + snapshot_id: Some(snapshot_id), + xy: None, + duration_ms: None, + timeout_ms: Some(5_000), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert_eq!(value["hovered"], true); + assert!(adapter.retry.calls() >= 3); + assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 1); +} + +#[path = "hover_preflight_tests.rs"] +mod preflight_tests; diff --git a/crates/core/src/commands/input_hold_policy.rs b/crates/core/src/commands/input_hold_policy.rs new file mode 100644 index 0000000..f379109 --- /dev/null +++ b/crates/core/src/commands/input_hold_policy.rs @@ -0,0 +1,12 @@ +use crate::{AdapterError, AppError, ErrorCode}; + +pub(crate) fn reject(command: &str, replacement: &str) -> AppError { + AdapterError::new( + ErrorCode::ActionNotSupported, + format!("{command} is unavailable in stateless mode"), + ) + .with_suggestion(format!( + "Use the atomic {replacement} command; held input requires a daemon-owned transaction that can guarantee release" + )) + .into() +} diff --git a/crates/core/src/commands/is_check.rs b/crates/core/src/commands/is_check.rs index 55d4496..1aaf2de 100644 --- a/crates/core/src/commands/is_check.rs +++ b/crates/core/src/commands/is_check.rs @@ -1,10 +1,11 @@ use crate::{ + AppError, adapter::{PlatformAdapter, optional_live_read}, commands::helpers::resolve_ref_with_context, context::CommandContext, element_state::ElementState, - error::AppError, refs::RefEntry, + state::{self, CHECKED, DISABLED, EXPANDED, FOCUSED, VisibilityEvidence}, }; use serde_json::{Value, json}; @@ -22,7 +23,6 @@ pub enum IsProperty { Expanded, } -/// State is read live when the platform supports it, then falls back to snapshot state. pub fn execute( args: IsArgs, adapter: &dyn PlatformAdapter, @@ -30,8 +30,6 @@ pub fn execute( ) -> Result { let (entry, handle) = resolve_ref_with_context(&args.ref_id, args.snapshot_id.as_deref(), adapter, context)?; - let state = optional_live_read(adapter.get_live_state(handle.handle()))? - .unwrap_or_else(|| state_from_ref_entry(&entry)); let prop_name = match args.property { IsProperty::Visible => "visible", @@ -41,14 +39,47 @@ pub fn execute( IsProperty::Expanded => "expanded", }; - let applicable = is_applicable(&args.property, &entry, &state); + let deadline = crate::Deadline::standard()?; + let live_state = optional_live_read(adapter.get_live_state(&handle, deadline))?; + let state = live_state + .clone() + .unwrap_or_else(|| state_from_ref_entry(&entry)); + let states_from_live = live_state.is_some(); + let live_bounds = optional_live_read(adapter.get_element_bounds(&handle, deadline))?; + let visibility = VisibilityEvidence { + bounds: live_bounds.or(entry.geometry.bounds), + states: state.states.clone(), + bounds_from_live: live_bounds.is_some(), + states_from_live, + }; + + let applicable = match args.property { + IsProperty::Visible => visibility.applicable(), + IsProperty::Enabled | IsProperty::Focused => true, + IsProperty::Checked => { + crate::roles::is_toggleable_role(&entry.identity.role) + || state::has_state(&state.states, CHECKED) + || crate::capability::contains_any( + &entry.capabilities.available_actions, + crate::capability::CHECKED_APPLICABILITY, + ) + } + IsProperty::Expanded => { + crate::roles::is_expandable_role(&entry.identity.role) + || state::has_state(&state.states, EXPANDED) + || crate::capability::contains_any( + &entry.capabilities.available_actions, + crate::capability::EXPANDED_APPLICABILITY, + ) + } + }; let result = match args.property { - IsProperty::Visible => !has_state(&state, "hidden"), - IsProperty::Enabled => !has_state(&state, "disabled"), - IsProperty::Checked => has_state(&state, "checked"), - IsProperty::Focused => has_state(&state, "focused"), - IsProperty::Expanded => has_state(&state, "expanded"), + IsProperty::Visible => visibility.result(), + IsProperty::Enabled => !state::has_state(&state.states, DISABLED), + IsProperty::Checked => state::has_state(&state.states, CHECKED), + IsProperty::Focused => state::has_state(&state.states, FOCUSED), + IsProperty::Expanded => state::has_state(&state.states, EXPANDED), }; Ok( @@ -58,35 +89,12 @@ pub fn execute( fn state_from_ref_entry(entry: &RefEntry) -> ElementState { ElementState { - role: entry.role.clone(), - states: entry.states.clone(), - value: entry.value.clone(), - } -} - -fn has_state(state: &ElementState, name: &str) -> bool { - state.states.iter().any(|s| s == name) -} - -fn is_applicable(property: &IsProperty, entry: &RefEntry, state: &ElementState) -> bool { - match property { - IsProperty::Visible | IsProperty::Enabled | IsProperty::Focused => true, - IsProperty::Checked => { - crate::roles::is_toggleable_role(&entry.role) - || has_state(state, "checked") - || crate::capability::contains_any( - &entry.available_actions, - crate::capability::CHECKED_APPLICABILITY, - ) - } - IsProperty::Expanded => { - crate::roles::is_expandable_role(&entry.role) - || has_state(state, "expanded") - || crate::capability::contains_any( - &entry.available_actions, - crate::capability::EXPANDED_APPLICABILITY, - ) - } + role: entry.identity.role.clone(), + states: entry.capabilities.states.clone(), + value: entry.identity.value.clone(), + enabled: None, + hidden: None, + offscreen: None, } } diff --git a/crates/core/src/commands/is_check_applicability_tests.rs b/crates/core/src/commands/is_check_applicability_tests.rs new file mode 100644 index 0000000..5b08afb --- /dev/null +++ b/crates/core/src/commands/is_check_applicability_tests.rs @@ -0,0 +1,60 @@ +use super::*; + +#[test] +fn action_availability_makes_toggle_and_expand_applicable() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "cell".into(), + name: Some("Disclosure".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec!["Check".into(), "Expand".into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + }); + let adapter = LiveStateAdapter { + state: Mutex::new(None), + bounds: Mutex::new(None), + bounds_supported: false, + state_supported: true, + }; + + for property in [IsProperty::Checked, IsProperty::Expanded] { + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.clone()), + property, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(result["applicable"], true); + } +} diff --git a/crates/core/src/commands/is_check_tests.rs b/crates/core/src/commands/is_check_tests.rs index 22220bb..c63df57 100644 --- a/crates/core/src/commands/is_check_tests.rs +++ b/crates/core/src/commands/is_check_tests.rs @@ -1,24 +1,83 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - adapter::NativeHandle, error::AdapterError, refs::RefMap, refs_store::RefStore, - refs_test_support::HomeGuard, + AdapterError, Rect, adapter::NativeHandle, refs::RefMap, refs_store::RefStore, + refs_test_support::HomeGuard, state, }; use std::sync::Mutex; struct LiveStateAdapter { state: Mutex>, + bounds: Mutex>, + bounds_supported: bool, + state_supported: bool, } -impl PlatformAdapter for LiveStateAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { +impl LiveStateAdapter { + fn with_live(bounds: Rect, states: Vec) -> Self { + Self { + state: Mutex::new(Some(ElementState { + role: "button".into(), + states, + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + })), + bounds: Mutex::new(Some(bounds)), + bounds_supported: true, + state_supported: true, + } + } + + fn without_live_support() -> Self { + Self { + state: Mutex::new(None), + bounds: Mutex::new(None), + bounds_supported: false, + state_supported: false, + } + } +} + +impl ObservationOps for LiveStateAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { Ok(NativeHandle::null()) } - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + if !self.state_supported { + return Err(AdapterError::not_supported("get_live_state")); + } Ok(self.state.lock().unwrap().clone()) } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + if !self.bounds_supported { + return Err(AdapterError::not_supported("get_element_bounds")); + } + Ok(*self.bounds.lock().unwrap()) + } } +impl ActionOps for LiveStateAdapter {} + +impl InputOps for LiveStateAdapter {} + +impl SystemOps for LiveStateAdapter {} + fn save_entry(entry: RefEntry) -> String { let mut refmap = RefMap::new(); refmap.allocate(entry); @@ -27,25 +86,162 @@ fn save_entry(entry: RefEntry) -> String { fn entry(states: Vec, value: Option<&str>, actions: Vec<&str>) -> RefEntry { RefEntry { - pid: 1, - role: "checkbox".into(), - name: Some("Target".into()), - value: value.map(str::to_string), - description: None, - states, - bounds: None, - bounds_hash: None, - available_actions: actions.into_iter().map(str::to_string).collect(), - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "checkbox".into(), + name: Some("Target".into()), + value: value.map(str::to_string), + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states, + available_actions: actions.into_iter().map(str::to_string).collect(), + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } +fn visible_bounds() -> Rect { + Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + } +} + +#[test] +fn hidden_element_reports_not_visible() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec![])); + let adapter = LiveStateAdapter::with_live(visible_bounds(), vec![state::HIDDEN.into()]); + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Visible, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(result["result"], false); + assert_eq!(result["applicable"], true); +} + +#[test] +fn zero_sized_bounds_report_not_visible() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec![])); + let adapter = LiveStateAdapter::with_live( + Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 10.0, + }, + vec![], + ); + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Visible, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(result["result"], false); + assert_eq!(result["applicable"], true); +} + +#[test] +fn offscreen_element_reports_not_visible() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec![])); + let adapter = LiveStateAdapter::with_live(visible_bounds(), vec![state::OFFSCREEN.into()]); + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Visible, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(result["result"], false); + assert_eq!(result["applicable"], true); +} + +#[test] +fn visible_element_with_live_evidence_reports_true() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec![])); + let adapter = LiveStateAdapter::with_live(visible_bounds(), vec![]); + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Visible, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(result["result"], true); + assert_eq!(result["applicable"], true); +} + +#[test] +fn visible_degrades_applicability_when_live_reads_unsupported() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec![])); + let adapter = LiveStateAdapter::without_live_support(); + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Visible, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(result["result"], false); + assert_eq!(result["applicable"], false); +} + #[test] fn checked_uses_live_canonical_state() { let _guard = HomeGuard::new(); @@ -55,7 +251,13 @@ fn checked_uses_live_canonical_state() { role: "checkbox".into(), states: vec!["checked".into()], value: Some("1".into()), + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), })), + bounds: Mutex::new(None), + bounds_supported: false, + state_supported: true, }; let result = execute( @@ -79,6 +281,9 @@ fn checked_does_not_infer_platform_values_in_core() { let snapshot_id = save_entry(entry(vec![], Some("1"), vec!["Toggle"])); let adapter = LiveStateAdapter { state: Mutex::new(None), + bounds: Mutex::new(None), + bounds_supported: false, + state_supported: true, }; let result = execute( @@ -102,6 +307,9 @@ fn checked_falls_back_to_snapshot_state_when_live_state_is_missing() { let snapshot_id = save_entry(entry(vec!["checked".into()], None, vec!["Toggle"])); let adapter = LiveStateAdapter { state: Mutex::new(None), + bounds: Mutex::new(None), + bounds_supported: false, + state_supported: true, }; let result = execute( @@ -123,16 +331,10 @@ fn checked_falls_back_to_snapshot_state_when_live_state_is_missing() { fn basic_state_properties_use_live_state() { let _guard = HomeGuard::new(); let snapshot_id = save_entry(entry(vec![], None, vec![])); - let adapter = LiveStateAdapter { - state: Mutex::new(Some(ElementState { - role: "button".into(), - states: vec!["focused".into(), "expanded".into()], - value: None, - })), - }; + let adapter = + LiveStateAdapter::with_live(visible_bounds(), vec!["focused".into(), "expanded".into()]); for (property, expected) in [ - (IsProperty::Visible, true), (IsProperty::Enabled, true), (IsProperty::Focused, true), (IsProperty::Expanded, true), @@ -153,43 +355,8 @@ fn basic_state_properties_use_live_state() { } } -#[test] -fn action_availability_makes_toggle_and_expand_applicable() { - let _guard = HomeGuard::new(); - let snapshot_id = save_entry(RefEntry { - pid: 1, - role: "cell".into(), - name: Some("Disclosure".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Check".into(), "Expand".into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), - }); - let adapter = LiveStateAdapter { - state: Mutex::new(None), - }; +#[path = "is_check_applicability_tests.rs"] +mod applicability_tests; - for property in [IsProperty::Checked, IsProperty::Expanded] { - let result = execute( - IsArgs { - ref_id: "@e1".into(), - snapshot_id: Some(snapshot_id.clone()), - property, - }, - &adapter, - &CommandContext::default(), - ) - .unwrap(); - - assert_eq!(result["applicable"], true); - } -} +#[path = "is_check_vocabulary_tests.rs"] +mod vocabulary_tests; diff --git a/crates/core/src/commands/is_check_vocabulary_tests.rs b/crates/core/src/commands/is_check_vocabulary_tests.rs new file mode 100644 index 0000000..50918a8 --- /dev/null +++ b/crates/core/src/commands/is_check_vocabulary_tests.rs @@ -0,0 +1,72 @@ +use super::state; + +/// Real drift guard for U1/KTD2: scans the core files that actually compare +/// against state tokens for a bare string literal bypassing the `state::` +/// module (`has_state(_, "literal")` or a `state == "literal"` closure +/// comparison), then asserts every literal found is a member of +/// `STATE_VOCABULARY`. Unlike the guard this replaces — which rebuilt its +/// expected set from the same `state::` constants it checked, so it could +/// never fail — this reads the actual source text, so a literal that +/// bypasses `state::` and drifts from the vocabulary is caught here. +#[test] +fn state_literal_call_sites_use_vocabulary_constants() { + let src_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let scanned = [ + "actionability/mod.rs", + "locator.rs", + "ref_action_wait.rs", + "commands/wait_predicate.rs", + "commands/is_check.rs", + ]; + for rel in scanned { + let path = src_root.join(rel); + let source = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("{} unreadable: {e}", path.display())); + for token in bare_state_literals(&source) { + assert!( + state::STATE_VOCABULARY.contains(&token.as_str()), + "{} references bare state literal \"{token}\" that bypasses state::; \ + use a state:: constant so vocabulary drift is caught at compile time", + path.display() + ); + } + } +} + +/// Proves the scanner in [`state_literal_call_sites_use_vocabulary_constants`] +/// actually detects a drifted literal instead of always returning an empty +/// (vacuously passing) list. +#[test] +fn bare_state_literal_scan_flags_bogus_tokens() { + let source = r#" + fn f(states: &[String]) -> bool { + states.iter().any(|state| state == "zzz_not_a_real_state") + } + "#; + let found = bare_state_literals(source); + assert_eq!(found, vec!["zzz_not_a_real_state".to_string()]); + assert!(!state::STATE_VOCABULARY.contains(&found[0].as_str())); +} + +fn bare_state_literals(source: &str) -> Vec { + let markers = ["has_state(", "state == "]; + let mut tokens = Vec::new(); + for marker in markers { + let mut cursor = 0usize; + while let Some(relative) = source[cursor..].find(marker) { + let after_marker = cursor + relative + marker.len(); + let statement_end = source[after_marker..] + .find([')', ';', '\n']) + .map_or(source.len(), |offset| after_marker + offset); + let statement = &source[after_marker..statement_end]; + if let Some(open_quote) = statement.find('"') { + let rest = &statement[open_quote + 1..]; + if let Some(close_quote) = rest.find('"') { + tokens.push(rest[..close_quote].to_string()); + } + } + cursor = after_marker; + } + } + tokens +} diff --git a/crates/core/src/commands/key_down.rs b/crates/core/src/commands/key_down.rs index 25a4bbe..f8c8c69 100644 --- a/crates/core/src/commands/key_down.rs +++ b/crates/core/src/commands/key_down.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, adapter::PlatformAdapter, commands::combo::{ensure_combo_allowed, parse_combo_normalized}, - error::AppError, }; -use serde_json::{Value, json}; +use serde_json::Value; pub struct KeyDownArgs { pub combo: String, @@ -13,6 +13,7 @@ pub struct KeyDownArgs { pub fn execute(args: KeyDownArgs, adapter: &dyn PlatformAdapter) -> Result { let combo = parse_combo_normalized(&args.combo)?; ensure_combo_allowed(&combo, &args.combo, args.force, adapter)?; - adapter.key_event(&combo, true)?; - Ok(json!({ "key_down": args.combo })) + Err(crate::commands::input_hold_policy::reject( + "key-down", "press", + )) } diff --git a/crates/core/src/commands/key_up.rs b/crates/core/src/commands/key_up.rs index 377fd65..6128931 100644 --- a/crates/core/src/commands/key_up.rs +++ b/crates/core/src/commands/key_up.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, adapter::PlatformAdapter, commands::combo::{ensure_combo_allowed, parse_combo_normalized}, - error::AppError, }; -use serde_json::{Value, json}; +use serde_json::Value; pub struct KeyUpArgs { pub combo: String, @@ -13,6 +13,7 @@ pub struct KeyUpArgs { pub fn execute(args: KeyUpArgs, adapter: &dyn PlatformAdapter) -> Result { let combo = parse_combo_normalized(&args.combo)?; ensure_combo_allowed(&combo, &args.combo, args.force, adapter)?; - adapter.key_event(&combo, false)?; - Ok(json!({ "key_up": args.combo })) + Err(crate::commands::input_hold_policy::reject( + "key-up", "press", + )) } diff --git a/crates/core/src/commands/launch.rs b/crates/core/src/commands/launch.rs index c2c9549..7307385 100644 --- a/crates/core/src/commands/launch.rs +++ b/crates/core/src/commands/launch.rs @@ -1,12 +1,23 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{AppError, adapter::PlatformAdapter, launch_options::LaunchOptions}; use serde_json::Value; pub struct LaunchArgs { pub app: String, - pub timeout_ms: u64, + pub options: LaunchOptions, } pub fn execute(args: LaunchArgs, adapter: &dyn PlatformAdapter) -> Result { - let window = adapter.launch_app(&args.app, args.timeout_ms)?; + crate::wait_timeout_duration(args.options.timeout_ms)?; + let deadline = if args.options.timeout_ms == 0 { + crate::Deadline::standard()? + } else { + crate::Deadline::after(args.options.timeout_ms)? + }; + let lease = adapter.acquire_interaction_lease(deadline)?; + let window = adapter.launch_app(&args.app, &args.options, &lease)?; Ok(serde_json::to_value(window)?) } + +#[cfg(test)] +#[path = "launch_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/launch_tests.rs b/crates/core/src/commands/launch_tests.rs new file mode 100644 index 0000000..cd11e20 --- /dev/null +++ b/crates/core/src/commands/launch_tests.rs @@ -0,0 +1,87 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::launch_options::LaunchOptions; +use crate::{AdapterError, InteractionLease, ProcessId, WindowInfo, WindowState}; +use std::sync::atomic::{AtomicU64, Ordering}; + +struct LaunchAdapter { + lease_timeout_ms: AtomicU64, +} + +impl ObservationOps for LaunchAdapter {} +impl ActionOps for LaunchAdapter {} +impl InputOps for LaunchAdapter {} + +impl SystemOps for LaunchAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + self.lease_timeout_ms + .store(deadline.timeout_ms(), Ordering::SeqCst); + InteractionLease::guarded(deadline, ()) + } + + fn launch_app( + &self, + _id: &str, + _options: &LaunchOptions, + _lease: &InteractionLease, + ) -> Result { + Ok(WindowInfo { + id: "w-1".into(), + title: "Fixture".into(), + app: "Fixture".into(), + pid: ProcessId::new(42), + process_instance: Some("42:1".into()), + bounds: None, + state: WindowState::default(), + }) + } +} + +#[test] +fn launch_lease_uses_the_requested_timeout() { + let adapter = LaunchAdapter { + lease_timeout_ms: AtomicU64::new(0), + }; + let options = LaunchOptions { + timeout_ms: 30_000, + ..Default::default() + }; + + execute( + LaunchArgs { + app: "Fixture".into(), + options, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(adapter.lease_timeout_ms.load(Ordering::SeqCst), 30_000); +} + +#[test] +fn zero_launch_timeout_keeps_a_bounded_lease_for_the_single_attempt() { + let adapter = LaunchAdapter { + lease_timeout_ms: AtomicU64::new(0), + }; + + execute( + LaunchArgs { + app: "Fixture".into(), + options: LaunchOptions { + timeout_ms: 0, + ..Default::default() + }, + }, + &adapter, + ) + .unwrap(); + + assert_eq!( + adapter.lease_timeout_ms.load(Ordering::SeqCst), + crate::Deadline::standard().unwrap().timeout_ms() + ); +} diff --git a/crates/core/src/commands/list_apps.rs b/crates/core/src/commands/list_apps.rs index d7bb515..5d88b17 100644 --- a/crates/core/src/commands/list_apps.rs +++ b/crates/core/src/commands/list_apps.rs @@ -1,4 +1,4 @@ -use crate::{adapter::PlatformAdapter, error::AppError, search_text}; +use crate::{AppError, adapter::PlatformAdapter, search_text}; use serde_json::{Value, json}; pub struct ListAppsArgs { @@ -6,7 +6,7 @@ pub struct ListAppsArgs { } pub fn execute(args: ListAppsArgs, adapter: &dyn PlatformAdapter) -> Result { - let mut apps = adapter.list_apps()?; + let mut apps = adapter.list_apps(crate::Deadline::standard()?)?; if let Some(app) = args.app { let needle = search_text::normalize(&app); apps.retain(|candidate| search_text::contains(&candidate.name, &needle)); diff --git a/crates/core/src/commands/list_apps_tests.rs b/crates/core/src/commands/list_apps_tests.rs index e6101c1..5718b75 100644 --- a/crates/core/src/commands/list_apps_tests.rs +++ b/crates/core/src/commands/list_apps_tests.rs @@ -1,25 +1,34 @@ use super::*; -use crate::{adapter::PlatformAdapter, error::AdapterError, node::AppInfo}; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{AdapterError, AppInfo}; struct AppsAdapter; -impl PlatformAdapter for AppsAdapter { - fn list_apps(&self) -> Result, AdapterError> { +impl ObservationOps for AppsAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { Ok(vec![ AppInfo { name: "Finder".into(), - pid: 1, + pid: crate::ProcessId::new(1), bundle_id: Some("com.apple.finder".into()), + process_instance: Some("test-instance".into()), }, AppInfo { name: "TextEdit".into(), - pid: 2, + pid: crate::ProcessId::new(2), bundle_id: Some("com.apple.TextEdit".into()), + process_instance: Some("test-instance".into()), }, ]) } } +impl ActionOps for AppsAdapter {} + +impl InputOps for AppsAdapter {} + +impl SystemOps for AppsAdapter {} + #[test] fn app_filter_matches_by_name_case_insensitively() { let value = execute( diff --git a/crates/core/src/commands/list_displays.rs b/crates/core/src/commands/list_displays.rs new file mode 100644 index 0000000..86a9062 --- /dev/null +++ b/crates/core/src/commands/list_displays.rs @@ -0,0 +1,7 @@ +use crate::{AppError, adapter::PlatformAdapter}; +use serde_json::Value; + +pub fn execute(adapter: &dyn PlatformAdapter) -> Result { + let displays = adapter.list_displays(crate::Deadline::standard()?)?; + Ok(serde_json::to_value(displays)?) +} diff --git a/crates/core/src/commands/list_displays_tests.rs b/crates/core/src/commands/list_displays_tests.rs new file mode 100644 index 0000000..d39cf88 --- /dev/null +++ b/crates/core/src/commands/list_displays_tests.rs @@ -0,0 +1,54 @@ +use crate::{ + AdapterError, Rect, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, + commands::list_displays, + display_info::DisplayInfo, +}; + +struct DisplayAdapter { + displays: Vec, +} + +impl ObservationOps for DisplayAdapter {} +impl ActionOps for DisplayAdapter {} +impl InputOps for DisplayAdapter {} + +impl SystemOps for DisplayAdapter { + fn list_displays(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(self.displays.clone()) + } +} + +#[test] +fn list_displays_returns_adapter_displays() { + let adapter = DisplayAdapter { + displays: vec![DisplayInfo { + id: "1".into(), + bounds: Rect { + x: 0.0, + y: 0.0, + width: 1440.0, + height: 900.0, + }, + is_primary: true, + scale: 2.0, + }], + }; + let value = list_displays::execute(&adapter).expect("list-displays"); + let displays: Vec = serde_json::from_value(value).expect("deserialize"); + assert_eq!(displays.len(), 1); + assert!(displays[0].is_primary); + assert_eq!(displays[0].scale, 2.0); +} + +#[test] +fn default_trait_impl_returns_platform_not_supported() { + struct Stub; + impl ObservationOps for Stub {} + impl ActionOps for Stub {} + impl InputOps for Stub {} + impl SystemOps for Stub {} + + let err = list_displays::execute(&Stub).expect_err("stub has no displays"); + assert_eq!(err.code(), "PLATFORM_NOT_SUPPORTED"); +} diff --git a/crates/core/src/commands/list_notifications.rs b/crates/core/src/commands/list_notifications.rs index c59fccc..eaf55b5 100644 --- a/crates/core/src/commands/list_notifications.rs +++ b/crates/core/src/commands/list_notifications.rs @@ -1,4 +1,4 @@ -use crate::{adapter::PlatformAdapter, error::AppError, notification::NotificationFilter}; +use crate::{AppError, NotificationFilter, adapter::PlatformAdapter, context::CommandContext}; use serde_json::{Value, json}; pub struct ListNotificationsArgs { @@ -10,13 +10,19 @@ pub struct ListNotificationsArgs { pub fn execute( args: ListNotificationsArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { let filter = NotificationFilter { app: args.app, text: args.text, limit: args.limit, }; - let notifications = adapter.list_notifications(&filter)?; + let notifications = super::notification_policy::list_with_foreground_lease( + &filter, + crate::Deadline::standard()?, + adapter, + context, + )?; Ok(json!({ "count": notifications.len(), "notifications": notifications, diff --git a/crates/core/src/commands/list_surfaces.rs b/crates/core/src/commands/list_surfaces.rs index 3aedcc3..f3a7b54 100644 --- a/crates/core/src/commands/list_surfaces.rs +++ b/crates/core/src/commands/list_surfaces.rs @@ -1,4 +1,4 @@ -use crate::{adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError}; +use crate::{AppError, adapter::PlatformAdapter, commands::helpers::resolve_app}; use serde_json::{Value, json}; pub struct ListSurfacesArgs { @@ -6,7 +6,10 @@ pub struct ListSurfacesArgs { } pub fn execute(args: ListSurfacesArgs, adapter: &dyn PlatformAdapter) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let surfaces = adapter.list_surfaces(pid).map_err(AppError::Adapter)?; - Ok(json!({ "pid": pid, "surfaces": surfaces })) + let deadline = crate::Deadline::standard()?; + let app = resolve_app(args.app.as_deref(), adapter, deadline)?; + let surfaces = adapter + .list_surfaces(crate::commands::helpers::process_identity(&app)?, deadline) + .map_err(AppError::Adapter)?; + Ok(json!({ "pid": app.pid, "surfaces": surfaces })) } diff --git a/crates/core/src/commands/list_windows.rs b/crates/core/src/commands/list_windows.rs index 5b0a3d3..17d988d 100644 --- a/crates/core/src/commands/list_windows.rs +++ b/crates/core/src/commands/list_windows.rs @@ -1,6 +1,6 @@ use crate::{ + AppError, adapter::{PlatformAdapter, WindowFilter}, - error::AppError, }; use serde_json::Value; @@ -13,6 +13,6 @@ pub fn execute(args: ListWindowsArgs, adapter: &dyn PlatformAdapter) -> Result, } pub fn execute( @@ -20,13 +18,25 @@ pub fn execute( context: &CommandContext, ) -> Result { require_cursor_policy(context, "mouse-click")?; - adapter.mouse_event(MouseEvent { - kind: MouseEventKind::Click { count: args.count }, - point: Point { - x: args.x, - y: args.y, + crate::validate_mouse_click_count(args.count)?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let point = Point { + x: args.x, + y: args.y, + }; + point.validate()?; + adapter.mouse_event( + MouseEvent { + kind: MouseEventKind::Click { count: args.count }, + point, + button: args.button, + modifiers: args.modifiers, }, - button: args.button, - })?; + &lease, + )?; Ok(json!({ "clicked": true, "x": args.x, "y": args.y, "count": args.count })) } + +#[cfg(test)] +#[path = "mouse_click_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/mouse_click_tests.rs b/crates/core/src/commands/mouse_click_tests.rs new file mode 100644 index 0000000..693fb15 --- /dev/null +++ b/crates/core/src/commands/mouse_click_tests.rs @@ -0,0 +1,124 @@ +use super::*; +use crate::AdapterError; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use std::sync::Mutex; + +struct ModifierCaptureAdapter { + captured: Mutex>, +} + +impl ModifierCaptureAdapter { + fn new() -> Self { + Self { + captured: Mutex::new(None), + } + } +} + +impl ObservationOps for ModifierCaptureAdapter {} +impl ActionOps for ModifierCaptureAdapter {} +impl SystemOps for ModifierCaptureAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +impl InputOps for ModifierCaptureAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.captured.lock().unwrap() = Some(event); + Ok(()) + } +} + +/// F10 regression: `mouse-click` previously hardcoded `modifiers: Vec::new()` +/// in this command, so `--modifiers cmd,shift` was accepted (once parsed) +/// but silently discarded before reaching the adapter. This proves the +/// requested chord survives unchanged into the dispatched `MouseEvent`. +#[test] +fn requested_modifiers_reach_the_dispatched_mouse_event() { + let adapter = ModifierCaptureAdapter::new(); + + execute( + MouseClickArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + count: 1, + modifiers: vec![Modifier::Meta, Modifier::Shift], + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + let captured = adapter.captured.lock().unwrap(); + let event = captured + .as_ref() + .expect("mouse_event must have been called"); + assert_eq!(event.modifiers, vec![Modifier::Meta, Modifier::Shift]); +} + +#[test] +fn no_modifiers_requested_dispatches_empty_modifiers() { + let adapter = ModifierCaptureAdapter::new(); + + execute( + MouseClickArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + count: 1, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + let captured = adapter.captured.lock().unwrap(); + assert!(captured.as_ref().unwrap().modifiers.is_empty()); +} + +#[test] +fn zero_clicks_fail_before_dispatch() { + let adapter = ModifierCaptureAdapter::new(); + + let err = execute( + MouseClickArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + count: 0, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert!(adapter.captured.lock().unwrap().is_none()); +} + +#[test] +fn excessive_click_count_fails_before_dispatch() { + let adapter = ModifierCaptureAdapter::new(); + + let err = execute( + MouseClickArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + count: crate::MAX_MOUSE_CLICK_COUNT + 1, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert!(adapter.captured.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/mouse_down.rs b/crates/core/src/commands/mouse_down.rs index 8e8607a..ac17baf 100644 --- a/crates/core/src/commands/mouse_down.rs +++ b/crates/core/src/commands/mouse_down.rs @@ -1,31 +1,28 @@ use crate::{ - action::{MouseButton, MouseEvent, MouseEventKind, Point}, - adapter::PlatformAdapter, - commands::point_resolve::require_cursor_policy, - context::CommandContext, - error::AppError, + AppError, Modifier, MouseButton, adapter::PlatformAdapter, + commands::point_resolve::require_cursor_policy, context::CommandContext, }; -use serde_json::{Value, json}; +use serde_json::Value; pub struct MouseDownArgs { pub x: f64, pub y: f64, pub button: MouseButton, + pub modifiers: Vec, } pub fn execute( - args: MouseDownArgs, - adapter: &dyn PlatformAdapter, + _args: MouseDownArgs, + _adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { require_cursor_policy(context, "mouse-down")?; - adapter.mouse_event(MouseEvent { - kind: MouseEventKind::Down, - point: Point { - x: args.x, - y: args.y, - }, - button: args.button, - })?; - Ok(json!({ "pressed": true, "x": args.x, "y": args.y })) + Err(crate::commands::input_hold_policy::reject( + "mouse-down", + "mouse-click or drag", + )) } + +#[cfg(test)] +#[path = "mouse_down_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/mouse_down_tests.rs b/crates/core/src/commands/mouse_down_tests.rs new file mode 100644 index 0000000..62dfd13 --- /dev/null +++ b/crates/core/src/commands/mouse_down_tests.rs @@ -0,0 +1,72 @@ +use super::*; +use crate::AdapterError; +use crate::MouseEvent; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use std::sync::Mutex; + +struct ModifierCaptureAdapter { + captured: Mutex>, +} + +impl ModifierCaptureAdapter { + fn new() -> Self { + Self { + captured: Mutex::new(None), + } + } +} + +impl ObservationOps for ModifierCaptureAdapter {} +impl ActionOps for ModifierCaptureAdapter {} +impl SystemOps for ModifierCaptureAdapter {} + +impl InputOps for ModifierCaptureAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.captured.lock().unwrap() = Some(event); + Ok(()) + } +} + +#[test] +fn requested_modifiers_do_not_bypass_stateless_rejection() { + let adapter = ModifierCaptureAdapter::new(); + + let err = execute( + MouseDownArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + modifiers: vec![Modifier::Ctrl], + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_NOT_SUPPORTED"); + assert!(adapter.captured.lock().unwrap().is_none()); +} + +#[test] +fn no_modifiers_still_requires_a_daemon_owned_transaction() { + let adapter = ModifierCaptureAdapter::new(); + + let err = execute( + MouseDownArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_NOT_SUPPORTED"); + assert!(adapter.captured.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/mouse_move.rs b/crates/core/src/commands/mouse_move.rs index bccb010..d4699d1 100644 --- a/crates/core/src/commands/mouse_move.rs +++ b/crates/core/src/commands/mouse_move.rs @@ -1,9 +1,6 @@ use crate::{ - action::{MouseButton, MouseEvent, MouseEventKind, Point}, - adapter::PlatformAdapter, - commands::point_resolve::require_cursor_policy, - context::CommandContext, - error::AppError, + AppError, MouseButton, MouseEvent, MouseEventKind, Point, adapter::PlatformAdapter, + commands::point_resolve::require_cursor_policy, context::CommandContext, }; use serde_json::{Value, json}; @@ -18,13 +15,20 @@ pub fn execute( context: &CommandContext, ) -> Result { require_cursor_policy(context, "mouse-move")?; - adapter.mouse_event(MouseEvent { - kind: MouseEventKind::Move, - point: Point { - x: args.x, - y: args.y, + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let point = Point { + x: args.x, + y: args.y, + }; + point.validate()?; + adapter.mouse_event( + MouseEvent { + kind: MouseEventKind::Move, + point, + button: MouseButton::Left, + modifiers: Vec::new(), }, - button: MouseButton::Left, - })?; + &lease, + )?; Ok(json!({ "moved": true, "x": args.x, "y": args.y })) } diff --git a/crates/core/src/commands/mouse_up.rs b/crates/core/src/commands/mouse_up.rs index 4a9c5ed..d7748e3 100644 --- a/crates/core/src/commands/mouse_up.rs +++ b/crates/core/src/commands/mouse_up.rs @@ -1,31 +1,28 @@ use crate::{ - action::{MouseButton, MouseEvent, MouseEventKind, Point}, - adapter::PlatformAdapter, - commands::point_resolve::require_cursor_policy, - context::CommandContext, - error::AppError, + AppError, Modifier, MouseButton, adapter::PlatformAdapter, + commands::point_resolve::require_cursor_policy, context::CommandContext, }; -use serde_json::{Value, json}; +use serde_json::Value; pub struct MouseUpArgs { pub x: f64, pub y: f64, pub button: MouseButton, + pub modifiers: Vec, } pub fn execute( - args: MouseUpArgs, - adapter: &dyn PlatformAdapter, + _args: MouseUpArgs, + _adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { require_cursor_policy(context, "mouse-up")?; - adapter.mouse_event(MouseEvent { - kind: MouseEventKind::Up, - point: Point { - x: args.x, - y: args.y, - }, - button: args.button, - })?; - Ok(json!({ "released": true, "x": args.x, "y": args.y })) + Err(crate::commands::input_hold_policy::reject( + "mouse-up", + "mouse-click or drag", + )) } + +#[cfg(test)] +#[path = "mouse_up_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/mouse_up_tests.rs b/crates/core/src/commands/mouse_up_tests.rs new file mode 100644 index 0000000..13fcc2a --- /dev/null +++ b/crates/core/src/commands/mouse_up_tests.rs @@ -0,0 +1,72 @@ +use super::*; +use crate::AdapterError; +use crate::MouseEvent; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use std::sync::Mutex; + +struct ModifierCaptureAdapter { + captured: Mutex>, +} + +impl ModifierCaptureAdapter { + fn new() -> Self { + Self { + captured: Mutex::new(None), + } + } +} + +impl ObservationOps for ModifierCaptureAdapter {} +impl ActionOps for ModifierCaptureAdapter {} +impl SystemOps for ModifierCaptureAdapter {} + +impl InputOps for ModifierCaptureAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + *self.captured.lock().unwrap() = Some(event); + Ok(()) + } +} + +#[test] +fn requested_modifiers_do_not_bypass_stateless_rejection() { + let adapter = ModifierCaptureAdapter::new(); + + let err = execute( + MouseUpArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + modifiers: vec![Modifier::Alt, Modifier::Shift], + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_NOT_SUPPORTED"); + assert!(adapter.captured.lock().unwrap().is_none()); +} + +#[test] +fn no_modifiers_still_requires_a_daemon_owned_transaction() { + let adapter = ModifierCaptureAdapter::new(); + + let err = execute( + MouseUpArgs { + x: 10.0, + y: 20.0, + button: MouseButton::Left, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_NOT_SUPPORTED"); + assert!(adapter.captured.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/mouse_wheel.rs b/crates/core/src/commands/mouse_wheel.rs new file mode 100644 index 0000000..365e448 --- /dev/null +++ b/crates/core/src/commands/mouse_wheel.rs @@ -0,0 +1,49 @@ +use crate::{ + AppError, MouseButton, MouseEvent, MouseEventKind, Point, adapter::PlatformAdapter, + context::CommandContext, +}; +use serde_json::{Value, json}; + +pub struct MouseWheelArgs { + pub x: f64, + pub y: f64, + pub dy: f64, + pub dx: f64, + pub modifiers: Vec, +} + +pub fn execute( + args: MouseWheelArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + crate::commands::point_resolve::require_cursor_policy(context, "mouse-wheel")?; + if !args.x.is_finite() || !args.y.is_finite() || !args.dx.is_finite() || !args.dy.is_finite() { + return Err(AppError::invalid_input( + "mouse-wheel coordinates and line deltas must be finite numbers", + )); + } + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let point = Point { + x: args.x, + y: args.y, + }; + point.validate()?; + adapter.mouse_event( + MouseEvent { + kind: MouseEventKind::Wheel { + delta_x: args.dx, + delta_y: args.dy, + }, + point, + button: MouseButton::Left, + modifiers: args.modifiers, + }, + &lease, + )?; + Ok(json!({ "scrolled": true, "dy": args.dy, "dx": args.dx })) +} + +#[cfg(test)] +#[path = "mouse_wheel_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/mouse_wheel_tests.rs b/crates/core/src/commands/mouse_wheel_tests.rs new file mode 100644 index 0000000..4b989e8 --- /dev/null +++ b/crates/core/src/commands/mouse_wheel_tests.rs @@ -0,0 +1,155 @@ +use super::*; +use crate::AdapterError; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{Modifier, MouseEvent, MouseEventKind}; +use std::sync::Mutex; + +#[derive(Debug, PartialEq)] +struct WheelCall { + x: f64, + y: f64, + dy: f64, + dx: f64, + modifiers: Vec, +} + +struct WheelCaptureAdapter { + captured: Mutex>, + fail: bool, +} + +impl WheelCaptureAdapter { + fn recording() -> Self { + Self { + captured: Mutex::new(None), + fail: false, + } + } + + fn failing() -> Self { + Self { + captured: Mutex::new(None), + fail: true, + } + } +} + +impl ObservationOps for WheelCaptureAdapter {} +impl ActionOps for WheelCaptureAdapter {} +impl SystemOps for WheelCaptureAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +impl InputOps for WheelCaptureAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + let MouseEventKind::Wheel { delta_x, delta_y } = event.kind else { + return Err(AdapterError::not_supported("non-wheel mouse event")); + }; + *self.captured.lock().unwrap() = Some(WheelCall { + x: event.point.x, + y: event.point.y, + dy: delta_y, + dx: delta_x, + modifiers: event.modifiers, + }); + if self.fail { + return Err(AdapterError::not_supported("mouse_wheel")); + } + Ok(()) + } +} + +#[test] +fn requested_wheel_args_reach_the_adapter_unchanged() { + let adapter = WheelCaptureAdapter::recording(); + + execute( + MouseWheelArgs { + x: 10.0, + y: 20.0, + dy: -3.0, + dx: 5.0, + modifiers: vec![Modifier::Shift, Modifier::Alt], + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + let captured = adapter.captured.lock().unwrap(); + let call = captured + .as_ref() + .expect("mouse_wheel must have been called"); + assert_eq!( + *call, + WheelCall { + x: 10.0, + y: 20.0, + dy: -3.0, + dx: 5.0, + modifiers: vec![Modifier::Shift, Modifier::Alt], + } + ); +} + +#[test] +fn returns_scrolled_envelope_with_requested_deltas() { + let adapter = WheelCaptureAdapter::recording(); + + let value = execute( + MouseWheelArgs { + x: 0.0, + y: 0.0, + dy: 7.0, + dx: -2.0, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert_eq!(value, json!({ "scrolled": true, "dy": 7.0, "dx": -2.0 })); +} + +#[test] +fn adapter_error_propagates_as_err() { + let adapter = WheelCaptureAdapter::failing(); + + let result = execute( + MouseWheelArgs { + x: 1.0, + y: 2.0, + dy: 1.0, + dx: 0.0, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default().with_headed(true), + ); + + assert!(result.is_err()); +} + +#[test] +fn headless_policy_rejects_wheel_before_adapter_dispatch() { + let adapter = WheelCaptureAdapter::recording(); + let err = execute( + MouseWheelArgs { + x: 1.0, + y: 2.0, + dy: -3.0, + dx: 0.0, + modifiers: Vec::new(), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + assert_eq!(err.code(), "POLICY_DENIED"); + assert!(adapter.captured.lock().unwrap().is_none()); +} diff --git a/crates/core/src/commands/move_window.rs b/crates/core/src/commands/move_window.rs index efad10c..55e22a1 100644 --- a/crates/core/src/commands/move_window.rs +++ b/crates/core/src/commands/move_window.rs @@ -1,23 +1,33 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_window_for_app, - error::AppError, + AppError, WindowOp, + adapter::PlatformAdapter, + commands::helpers::{resolve_window_for_app, revalidate_window_for_mutation}, }; use serde_json::{Value, json}; pub struct MoveWindowArgs { pub app: Option, + pub window_id: Option, pub x: f64, pub y: f64, } pub fn execute(args: MoveWindowArgs, adapter: &dyn PlatformAdapter) -> Result { - let win = resolve_window_for_app(args.app.as_deref(), adapter)?; + crate::Point { + x: args.x, + y: args.y, + } + .validate()?; + let win = resolve_window_for_app(args.app.as_deref(), args.window_id.as_deref(), adapter)?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let live = revalidate_window_for_mutation(adapter, &win, &lease)?; adapter.window_op( - &win, + &live, WindowOp::Move { x: args.x, y: args.y, }, + &lease, )?; Ok(json!({ "moved": true, "x": args.x, "y": args.y })) } diff --git a/crates/core/src/commands/notification_action.rs b/crates/core/src/commands/notification_action.rs index 1a4ac01..5741f49 100644 --- a/crates/core/src/commands/notification_action.rs +++ b/crates/core/src/commands/notification_action.rs @@ -1,5 +1,5 @@ -use crate::{adapter::PlatformAdapter, error::AppError, notification::NotificationIdentity}; -use serde_json::{Value, json}; +use crate::{AppError, CommandContext, adapter::PlatformAdapter}; +use serde_json::Value; pub struct NotificationActionArgs { pub index: usize, @@ -11,17 +11,20 @@ pub struct NotificationActionArgs { pub fn execute( args: NotificationActionArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { - let identity = if args.expected_app.is_some() || args.expected_title.is_some() { - Some(NotificationIdentity { - expected_app: args.expected_app, - expected_title: args.expected_title, - }) - } else { - None - }; - let result = adapter.notification_action(args.index, identity.as_ref(), &args.action)?; - Ok(json!({ - "action": result.action, - })) + let identity = + super::notification_identity::required_identity(args.expected_app, args.expected_title)?; + let policy = super::notification_policy::mutation_policy(context)?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let result = adapter.notification_action( + crate::NotificationActionRequest { + index: args.index, + identity: &identity, + action_name: &args.action, + policy, + }, + &lease, + )?; + Ok(serde_json::to_value(result)?) } diff --git a/crates/core/src/commands/notification_identity.rs b/crates/core/src/commands/notification_identity.rs new file mode 100644 index 0000000..d7af4b9 --- /dev/null +++ b/crates/core/src/commands/notification_identity.rs @@ -0,0 +1,22 @@ +use crate::{AppError, NotificationIdentity}; + +pub(crate) fn required_identity( + expected_app: Option, + expected_title: Option, +) -> Result { + let identity = NotificationIdentity { + expected_app, + expected_title, + }; + if identity.is_empty() { + return Err(AppError::invalid_input_with_suggestion( + "Notification mutations require --expected-app or --expected-title", + "Pass identity fields from the same list-notifications result to prevent acting on a reordered notification.", + )); + } + Ok(identity) +} + +#[cfg(test)] +#[path = "notification_identity_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/notification_identity_tests.rs b/crates/core/src/commands/notification_identity_tests.rs new file mode 100644 index 0000000..785eae1 --- /dev/null +++ b/crates/core/src/commands/notification_identity_tests.rs @@ -0,0 +1,21 @@ +use super::required_identity; + +#[test] +fn empty_identity_is_rejected() { + assert_eq!( + required_identity(None, None).unwrap_err().code(), + "INVALID_ARGS" + ); + assert_eq!( + required_identity(Some(String::new()), None) + .unwrap_err() + .code(), + "INVALID_ARGS" + ); +} + +#[test] +fn one_identity_field_is_sufficient() { + assert!(required_identity(Some("Slack".into()), None).is_ok()); + assert!(required_identity(None, Some("Build finished".into())).is_ok()); +} diff --git a/crates/core/src/commands/notification_policy.rs b/crates/core/src/commands/notification_policy.rs new file mode 100644 index 0000000..b2315e1 --- /dev/null +++ b/crates/core/src/commands/notification_policy.rs @@ -0,0 +1,30 @@ +use crate::{ + AdapterError, AppError, CommandContext, InteractionPolicy, NotificationFilter, + NotificationInfo, adapter::PlatformAdapter, +}; + +pub(crate) fn mutation_policy(context: &CommandContext) -> Result { + let policy = context.physical_input_policy(); + if !policy.allow_focus_steal { + return Err(AdapterError::policy_denied_for_policy( + "Notification mutations open and focus the operating system notification surface", + policy, + ) + .into()); + } + Ok(policy) +} + +pub(crate) fn list_with_foreground_lease( + filter: &NotificationFilter, + deadline: crate::Deadline, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result, AppError> { + let policy = context.physical_input_policy(); + let lease = policy + .allow_focus_steal + .then(|| adapter.acquire_interaction_lease(deadline)) + .transpose()?; + Ok(adapter.list_notifications(filter, policy, deadline, lease.as_ref())?) +} diff --git a/crates/core/src/commands/permissions.rs b/crates/core/src/commands/permissions.rs index 07f572b..3fd32a5 100644 --- a/crates/core/src/commands/permissions.rs +++ b/crates/core/src/commands/permissions.rs @@ -1,4 +1,4 @@ -use crate::{PermissionReport, adapter::PlatformAdapter, error::AppError}; +use crate::{AppError, PermissionReport, adapter::PlatformAdapter}; use serde_json::{Value, json}; pub struct PermissionsArgs { @@ -11,7 +11,8 @@ pub fn execute_with_report( report: &PermissionReport, ) -> Result { let report = if args.request { - adapter.request_permissions() + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + adapter.request_permissions(&lease)? } else { report.clone() }; diff --git a/crates/core/src/commands/point_resolve.rs b/crates/core/src/commands/point_resolve.rs index 99f2108..ebe715b 100644 --- a/crates/core/src/commands/point_resolve.rs +++ b/crates/core/src/commands/point_resolve.rs @@ -1,22 +1,22 @@ use crate::{ - action::Point, - adapter::PlatformAdapter, - commands::helpers::resolve_ref_with_context, - context::CommandContext, - error::{AdapterError, AppError}, + AdapterError, AppError, Point, actionability, adapter::PlatformAdapter, + commands::helpers::resolve_ref_with_context, context::CommandContext, }; -use serde_json::json; +#[derive(Clone, Copy)] pub(crate) struct PointResolveArgs<'a> { pub ref_id: Option<&'a str>, pub xy: Option<(f64, f64)>, pub snapshot_id: Option<&'a str>, pub missing_input_message: &'a str, + pub headed_requirement: crate::HeadedRequirement, } pub(crate) struct ResolvedPoint { pub point: Point, - pub pid: Option, + pub focused: bool, + pub source_entry: Option, + pub bounds_hash: Option, } pub(crate) fn require_cursor_policy( @@ -38,67 +38,51 @@ pub(crate) fn resolve_point_from_ref_or_xy_with_context( args: PointResolveArgs<'_>, adapter: &dyn PlatformAdapter, context: &CommandContext, + deadline: crate::Deadline, + _lease: &crate::InteractionLease, ) -> Result { if let Some(ref_id) = args.ref_id { let (entry, handle) = resolve_ref_with_context(ref_id, args.snapshot_id, adapter, context)?; let bounds = adapter - .get_element_bounds(handle.handle())? + .get_element_bounds(&handle, deadline)? .ok_or_else(|| AppError::invalid_input(format!("Element {ref_id} has no bounds")))?; + let point = Point { + x: bounds.x + bounds.width / 2.0, + y: bounds.y + bounds.height / 2.0, + }; + actionability::require_receives_events(&handle, point.clone(), adapter, deadline)?; return Ok(ResolvedPoint { - point: Point { - x: bounds.x + bounds.width / 2.0, - y: bounds.y + bounds.height / 2.0, - }, - pid: Some(entry.pid), + point, + focused: false, + source_entry: Some(entry), + bounds_hash: bounds.bounds_hash(), }); } if let Some((x, y)) = args.xy { return Ok(ResolvedPoint { point: Point { x, y }, - pid: None, + focused: false, + source_entry: None, + bounds_hash: None, }); } Err(AppError::invalid_input(args.missing_input_message)) } -/// Ensures the app that owns a ref-resolved point is frontmost before -/// physical input is synthesized, so the events land on its window instead -/// of whatever happens to be frontmost. Headless never steals focus -/// (`--headed` opts in), and a raise failure downgrades to un-focused input -/// rather than erroring. pub(crate) fn focus_for_physical_input( - pid: Option, + entry: Option<&crate::RefEntry>, adapter: &dyn PlatformAdapter, context: &CommandContext, + lease: &crate::InteractionLease, ) -> Result { - let Some(pid) = pid else { return Ok(false) }; + let Some(entry) = entry else { return Ok(false) }; if !context.physical_input_policy().allow_focus_steal { return Ok(false); } - let focused = match adapter.focus_app(pid) { - Ok(()) => true, - Err(err) => { - tracing::debug!("focus before physical input failed for pid {pid}: {err}"); - false - } - }; - context.trace_lazy("input.focus_app", || json!({ "pid": pid, "ok": focused }))?; - Ok(focused) + crate::headed_focus::focus_entry_window(entry, adapter, context, lease)?; + Ok(true) } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn physical_input_requires_headed_context() { - let err = require_cursor_policy(&CommandContext::default(), "mouse-move").unwrap_err(); - - assert_eq!(err.code(), "POLICY_DENIED"); - } - - #[test] - fn headed_context_allows_physical_input() { - require_cursor_policy(&CommandContext::default().with_headed(true), "mouse-move").unwrap(); - } -} +#[path = "point_resolve_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/point_resolve_tests.rs b/crates/core/src/commands/point_resolve_tests.rs new file mode 100644 index 0000000..2f367e2 --- /dev/null +++ b/crates/core/src/commands/point_resolve_tests.rs @@ -0,0 +1,378 @@ +use super::*; + +fn resolve_test_point( + args: PointResolveArgs<'_>, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let deadline = crate::Deadline::standard().unwrap(); + let lease = crate::InteractionLease::guarded(deadline, ()).unwrap(); + resolve_point_from_ref_or_xy_with_context(args, adapter, context, deadline, &lease) +} +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{ + ErrorCode, Rect, + adapter::NativeHandle, + capability, + hit_test::HitTestResult, + refs::{RefEntry, RefMap}, + refs_store::RefStore, + refs_test_support::HomeGuard, +}; +use std::sync::atomic::{AtomicU32, Ordering}; + +#[test] +fn physical_input_requires_headed_context() { + let err = require_cursor_policy(&CommandContext::default(), "mouse-move").unwrap_err(); + + assert_eq!(err.code(), "POLICY_DENIED"); +} + +#[test] +fn headed_context_allows_physical_input() { + require_cursor_policy(&CommandContext::default().with_headed(true), "mouse-move").unwrap(); +} + +/// An adapter whose `hit_test` outcome is fixed per test, so each occlusion +/// scenario (F27) exercises the real `resolve_point_from_ref_or_xy_with_context` +/// path rather than a mock that always echoes success. +struct HitTestOutcomeAdapter { + outcome: Result, +} + +impl ObservationOps for HitTestOutcomeAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 100.0, + y: 200.0, + width: 20.0, + height: 10.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: Point, + _deadline: crate::Deadline, + ) -> Result { + self.outcome.clone() + } +} + +impl ActionOps for HitTestOutcomeAdapter {} +impl InputOps for HitTestOutcomeAdapter {} +impl SystemOps for HitTestOutcomeAdapter {} + +fn ref_snapshot(pid: u32) -> String { + let store = RefStore::new().unwrap(); + let mut refmap = RefMap::new(); + refmap.allocate(RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(pid), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Target".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + }); + store.save_new_snapshot(&refmap).unwrap() +} + +fn ref_args(snapshot_id: &str) -> PointResolveArgs<'_> { + PointResolveArgs { + ref_id: Some("@e1"), + xy: None, + snapshot_id: Some(snapshot_id), + missing_input_message: "Provide a ref (@e1) or --xy x,y", + headed_requirement: crate::HeadedRequirement::None, + } +} + +/// F27 regression: previously the ref-targeted path never called +/// `adapter.hit_test` at all, so an occluded target resolved to a point and +/// dispatch proceeded blind. This proves `InterceptedBy` now blocks +/// resolution and names the occluder. +#[test] +fn intercepted_by_blocks_ref_targeted_point_resolution() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = HitTestOutcomeAdapter { + outcome: Ok(HitTestResult::InterceptedBy { + role: Some("AXSheet".into()), + name: Some("Save changes?".into()), + bounds: None, + }), + }; + + let result = resolve_test_point(ref_args(&snapshot_id), &adapter, &CommandContext::default()); + let err = match result { + Ok(_) => panic!("occluded ref target must not resolve to a point"), + Err(err) => err, + }; + + assert_eq!(err.code(), ErrorCode::ActionFailed.as_str()); + let message = err.to_string(); + assert!(message.contains("AXSheet")); +} + +#[test] +fn reaches_target_allows_ref_targeted_point_resolution() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = HitTestOutcomeAdapter { + outcome: Ok(HitTestResult::ReachesTarget), + }; + + let resolved = + resolve_test_point(ref_args(&snapshot_id), &adapter, &CommandContext::default()).unwrap(); + + assert_eq!(resolved.point.x, 110.0); + assert_eq!(resolved.point.y, 205.0); +} + +#[test] +fn unknown_hit_test_result_allows_ref_targeted_resolution() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = HitTestOutcomeAdapter { + outcome: Ok(HitTestResult::Unknown), + }; + + let resolved = + resolve_test_point(ref_args(&snapshot_id), &adapter, &CommandContext::default()).unwrap(); + assert_eq!((resolved.point.x, resolved.point.y), (110.0, 205.0)); +} + +#[test] +fn unsupported_hit_test_allows_ref_targeted_resolution() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = HitTestOutcomeAdapter { + outcome: Err(AdapterError::not_supported("hit_test")), + }; + + let resolved = + resolve_test_point(ref_args(&snapshot_id), &adapter, &CommandContext::default()).unwrap(); + assert_eq!((resolved.point.x, resolved.point.y), (110.0, 205.0)); +} + +#[test] +fn hit_test_probe_error_is_preserved_for_ref_targeted_resolution() { + let _guard = HomeGuard::new(); + let snapshot_id = ref_snapshot(42); + let adapter = HitTestOutcomeAdapter { + outcome: Err(AdapterError::internal( + "AXUIElementCopyElementAtPosition failed", + )), + }; + + let err = match resolve_test_point(ref_args(&snapshot_id), &adapter, &CommandContext::default()) + { + Ok(_) => panic!("failed hit-test evidence must fail closed"), + Err(err) => err, + }; + + assert_eq!(err.code(), "INTERNAL"); +} + +/// Raw `--xy` input stays raw by design (KTD4): no ref means no occlusion +/// check, even against an adapter that would otherwise report occlusion. +#[test] +fn raw_xy_input_never_calls_hit_test() { + let adapter = HitTestOutcomeAdapter { + outcome: Ok(HitTestResult::InterceptedBy { + role: Some("AXSheet".into()), + name: None, + bounds: None, + }), + }; + + let resolved = resolve_test_point( + PointResolveArgs { + ref_id: None, + xy: Some((5.0, 6.0)), + snapshot_id: None, + missing_input_message: "Provide a ref (@e1) or --xy x,y", + headed_requirement: crate::HeadedRequirement::None, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!((resolved.point.x, resolved.point.y), (5.0, 6.0)); +} + +fn focus_ref_entry() -> RefEntry { + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(42), + process_instance: Some("instance-42".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Target".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-42".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + } +} + +struct FocusFailureAdapter { + resolve_error: Option, + focus_error: Option, + focus_calls: AtomicU32, +} + +impl ObservationOps for FocusFailureAdapter {} +impl ActionOps for FocusFailureAdapter {} +impl InputOps for FocusFailureAdapter {} + +impl SystemOps for FocusFailureAdapter { + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + match &self.resolve_error { + Some(error) => Err(error.clone()), + None => Ok(window.clone()), + } + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.focus_calls.fetch_add(1, Ordering::SeqCst); + match &self.focus_error { + Some(error) => Err(error.clone()), + None => Ok(()), + } + } +} + +fn focus_test_lease() -> crate::InteractionLease { + crate::InteractionLease::guarded(crate::Deadline::standard().unwrap(), ()).unwrap() +} + +#[test] +fn transient_window_resolution_failure_blocks_headed_input() { + let adapter = FocusFailureAdapter { + resolve_error: Some(AdapterError::app_unresponsive("Fixture")), + focus_error: None, + focus_calls: AtomicU32::new(0), + }; + + let error = focus_for_physical_input( + Some(&focus_ref_entry()), + &adapter, + &CommandContext::default().with_headed(true), + &focus_test_lease(), + ) + .unwrap_err(); + + assert_eq!(error.code(), "APP_UNRESPONSIVE"); + assert_eq!(adapter.focus_calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn transient_focus_failure_blocks_headed_input() { + let adapter = FocusFailureAdapter { + resolve_error: None, + focus_error: Some(AdapterError::new( + ErrorCode::ActionFailed, + "window could not be raised", + )), + focus_calls: AtomicU32::new(0), + }; + + let error = focus_for_physical_input( + Some(&focus_ref_entry()), + &adapter, + &CommandContext::default().with_headed(true), + &focus_test_lease(), + ) + .unwrap_err(); + + assert_eq!(error.code(), "ACTION_FAILED"); + assert_eq!(adapter.focus_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn permission_failure_during_focus_is_preserved() { + let adapter = FocusFailureAdapter { + resolve_error: None, + focus_error: Some(AdapterError::permission_denied()), + focus_calls: AtomicU32::new(0), + }; + + let err = focus_for_physical_input( + Some(&focus_ref_entry()), + &adapter, + &CommandContext::default().with_headed(true), + &focus_test_lease(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "PERM_DENIED"); +} diff --git a/crates/core/src/commands/pointer_action.rs b/crates/core/src/commands/pointer_action.rs new file mode 100644 index 0000000..2ae8b47 --- /dev/null +++ b/crates/core/src/commands/pointer_action.rs @@ -0,0 +1,390 @@ +use crate::{ + AppError, + adapter::PlatformAdapter, + commands::helpers::{load_ref_entry, resolve_handle_within_deadline}, + context::CommandContext, + ref_resolve_deadline::POLL_INTERVAL, + refs::RefEntry, +}; +use serde_json::{Value, json}; + +struct EntryPointResolve<'a> { + ref_id: &'a str, + entry: &'a RefEntry, + stability: Option>, + lease: Option<&'a crate::InteractionLease>, + verify_receives_events: bool, +} + +pub(crate) struct PointResolveAttempt<'a> { + pub args: crate::commands::point_resolve::PointResolveArgs<'a>, + pub stability: Option>, + pub allow_scroll: bool, +} + +fn resolve_point_from_entry( + request: EntryPointResolve<'_>, + deadline: crate::Deadline, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let EntryPointResolve { + ref_id, + entry, + stability, + lease, + verify_receives_events, + } = request; + let mut handle = + resolve_handle_within_deadline(adapter, entry, deadline).inspect_err(|err| { + let _ = context.trace_lazy("ref.resolve.error", || { + json!({ + "ref": ref_id, + "code": err.code.as_str(), + "message": err.message.clone(), + "details": err.details.clone() + }) + }); + })?; + context.trace_lazy("ref.resolve.ok", || json!({ "ref": ref_id }))?; + let (mut bounds, mut states) = pointer_live_observation(adapter, &handle, deadline)?; + if !pointer_is_visible(bounds, &states) { + if deadline.is_expired() { + return Err(crate::AdapterError::timeout( + "Pointer target did not become visible within the wait budget", + ) + .into()); + } + let Some(lease) = lease else { + return Err(crate::AdapterError::new( + crate::ErrorCode::ActionFailed, + "Pointer target is not visible in the live accessibility tree", + ) + .with_details(json!({ "check": "visible", "requires_scroll": true })) + .into()); + }; + adapter.scroll_into_view(&handle, lease)?; + handle = resolve_handle_within_deadline(adapter, entry, deadline)?; + (bounds, states) = pointer_live_observation(adapter, &handle, deadline)?; + if !pointer_is_visible(bounds, &states) { + return Err(crate::AdapterError::new( + crate::ErrorCode::ActionFailed, + "Pointer target is not visible in the live accessibility tree", + ) + .with_details(json!({ "check": "visible" })) + .into()); + } + } + let Some(bounds) = bounds else { + return Err(crate::AdapterError::new( + crate::ErrorCode::ActionFailed, + format!("Element {ref_id} has no live bounds"), + ) + .with_details(json!({ "check": "visible" })) + .into()); + }; + let observed_bounds_hash = bounds.bounds_hash().ok_or_else(|| { + crate::AdapterError::new( + crate::ErrorCode::ActionFailed, + "Pointer target exposes invalid live bounds", + ) + })?; + if stability.is_some_and(|expected| expected != Some(observed_bounds_hash)) { + return Err(crate::AdapterError::new( + crate::ErrorCode::ActionFailed, + "Pointer target bounds are not stable yet", + ) + .with_details(json!({ + "check": "stable", + "observed_bounds_hash": observed_bounds_hash + })) + .into()); + } + let point = crate::Point { + x: bounds.x + bounds.width / 2.0, + y: bounds.y + bounds.height / 2.0, + }; + if verify_receives_events { + crate::actionability::require_receives_events(&handle, point.clone(), adapter, deadline)?; + } + if deadline.is_expired() { + return Err(crate::AdapterError::timeout( + "Pointer target did not become actionable within the wait budget", + ) + .into()); + } + Ok(crate::commands::point_resolve::ResolvedPoint { + point, + focused: false, + source_entry: Some(entry.clone()), + bounds_hash: Some(observed_bounds_hash), + }) +} + +fn pointer_live_observation( + adapter: &dyn PlatformAdapter, + handle: &crate::adapter::NativeHandle, + deadline: crate::Deadline, +) -> Result<(Option, Vec), AppError> { + let bounds = adapter.get_element_bounds(handle, deadline)?; + let states = crate::adapter::optional_live_read(adapter.get_live_state(handle, deadline))? + .map(|state| state.states) + .unwrap_or_default(); + Ok((bounds, states)) +} + +fn pointer_is_visible(bounds: Option, states: &[String]) -> bool { + !crate::state::has_state(states, crate::state::HIDDEN) + && !crate::state::has_state(states, crate::state::OFFSCREEN) + && bounds.is_some_and(|bounds| { + bounds.validate().is_ok() && bounds.width > 0.0 && bounds.height > 0.0 + }) +} + +pub(crate) fn point_deadline(timeout_ms: Option) -> Result { + timeout_ms + .map_or_else(crate::Deadline::standard, crate::Deadline::after) + .map_err(AppError::Adapter) +} + +pub(crate) fn wait_for_point_with_deadline<'a>( + args: crate::commands::point_resolve::PointResolveArgs<'a>, + deadline: crate::Deadline, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + use crate::commands::point_resolve::resolve_point_from_ref_or_xy_with_context; + + let Some(ref_id) = args.ref_id else { + let lease = crate::InteractionLease::guarded(deadline, ())?; + return resolve_point_from_ref_or_xy_with_context(args, adapter, context, deadline, &lease); + }; + let entry = load_ref_entry(ref_id, args.snapshot_id, context)?; + let mut stability = Some(None); + let mut last_report = None; + loop { + if deadline.is_expired() { + return Err(point_actionability_timeout(last_report)); + } + match resolve_point_from_entry( + EntryPointResolve { + ref_id, + entry: &entry, + stability, + lease: None, + verify_receives_events: false, + }, + deadline, + adapter, + context, + ) { + Ok(mut point) => { + point.focused = false; + return Ok(point); + } + Err(err) => { + let remaining = deadline.remaining(); + if !is_retryable_point_error(&err) { + return Err(err); + } + last_report = Some(point_error_report(&err)); + if remaining.is_zero() { + return Err(point_actionability_timeout(last_report)); + } + if point_requires_scroll(&err) { + scroll_point_target(&entry, deadline, adapter)?; + std::thread::sleep(POLL_INTERVAL.min(deadline.remaining())); + continue; + } + let observed = point_observed_bounds_hash(&err); + if let Some(observed) = observed { + stability = Some(Some(observed)); + } + let interval = if observed.is_some() { + std::time::Duration::from_millis(16) + } else { + POLL_INTERVAL + }; + std::thread::sleep(interval.min(remaining)); + } + } + } +} + +pub(crate) fn focus_point_under_lease( + args: crate::commands::point_resolve::PointResolveArgs<'_>, + lease: &crate::InteractionLease, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + if !args.headed_requirement.requires_focus() { + return Ok(false); + } + let Some(ref_id) = args.ref_id else { + return Ok(false); + }; + let entry = load_ref_entry(ref_id, args.snapshot_id, context)?; + crate::commands::point_resolve::focus_for_physical_input(Some(&entry), adapter, context, lease) +} + +pub(crate) fn resolve_point_under_lease<'a>( + attempt: PointResolveAttempt<'a>, + deadline: crate::Deadline, + lease: &crate::InteractionLease, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + use crate::commands::point_resolve::resolve_point_from_ref_or_xy_with_context; + + let PointResolveAttempt { + args, + stability, + allow_scroll, + } = attempt; + let Some(ref_id) = args.ref_id else { + return resolve_point_from_ref_or_xy_with_context(args, adapter, context, deadline, lease); + }; + let entry = load_ref_entry(ref_id, args.snapshot_id, context)?; + resolve_point_from_entry( + EntryPointResolve { + ref_id, + entry: &entry, + stability, + lease: allow_scroll.then_some(lease), + verify_receives_events: true, + }, + deadline, + adapter, + context, + ) +} + +pub(crate) fn retry_leased_point_phase( + timeout_ms: Option, + deadline: crate::Deadline, + mut attempt: impl FnMut() -> Result, +) -> Result { + let auto_wait = timeout_ms.is_some_and(|timeout_ms| timeout_ms > 0); + loop { + match attempt() { + Ok(value) => return Ok(value), + Err(error) if auto_wait && is_retryable_point_error(&error) => { + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(point_actionability_timeout(Some(point_error_report( + &error, + )))); + } + std::thread::sleep(POLL_INTERVAL.min(remaining)); + } + Err(error) => return Err(error), + } + } +} + +fn scroll_point_target( + entry: &RefEntry, + deadline: crate::Deadline, + adapter: &dyn PlatformAdapter, +) -> Result<(), AppError> { + let lease = adapter.acquire_interaction_lease(deadline)?; + let handle = resolve_handle_within_deadline(adapter, entry, deadline)?; + adapter.scroll_into_view(&handle, &lease)?; + Ok(()) +} + +fn point_requires_scroll(err: &AppError) -> bool { + let AppError::Adapter(error) = err else { + return false; + }; + error + .details + .as_ref() + .and_then(|details| details.get("requires_scroll")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn point_actionability_timeout(last_report: Option) -> AppError { + let mut details = json!({ "kind": "actionability_timeout" }); + if let Some(last_report) = last_report + && let Some(object) = details.as_object_mut() + { + object.insert("last_report".into(), last_report); + } + crate::AdapterError::timeout("Pointer target did not become actionable within the wait budget") + .with_details(details) + .into() +} + +fn point_error_report(error: &AppError) -> Value { + json!({ + "code": error.code(), + "message": error.to_string(), + "details": match error { + AppError::Adapter(adapter_error) => adapter_error.details.clone(), + _ => None, + } + }) +} + +pub(crate) fn ensure_point_deadline( + deadline: crate::Deadline, + last_report: Option, +) -> Result<(), AppError> { + if deadline.is_expired() { + return Err(point_actionability_timeout(last_report)); + } + Ok(()) +} + +fn point_observed_bounds_hash(err: &AppError) -> Option { + let AppError::Adapter(err) = err else { + return None; + }; + err.details + .as_ref() + .and_then(|details| details.get("observed_bounds_hash")) + .and_then(Value::as_u64) +} + +fn is_retryable_point_error(err: &AppError) -> bool { + match err { + AppError::Adapter(error) if error.is_retryable_resolution_failure() => true, + AppError::Adapter(error) if error.code == crate::ErrorCode::ActionFailed => { + ["visible", "stable", "receives_events"] + .into_iter() + .any(|check| point_failed_check(err, check)) + } + _ => false, + } +} + +fn point_failed_check(err: &AppError, expected: &str) -> bool { + let AppError::Adapter(error) = err else { + return false; + }; + let Some(details) = error.details.as_ref() else { + return false; + }; + if details.get("check").and_then(Value::as_str) == Some(expected) { + return true; + } + details + .get("checks") + .and_then(Value::as_array) + .is_some_and(|checks| { + checks.iter().any(|check| { + check.get("check").and_then(Value::as_str) == Some(expected) + && check.get("status").and_then(Value::as_str) != Some("pass") + }) + }) +} + +#[cfg(test)] +#[path = "pointer_action_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "pointer_single_shot_tests.rs"] +mod single_shot_tests; diff --git a/crates/core/src/commands/pointer_action_tests.rs b/crates/core/src/commands/pointer_action_tests.rs new file mode 100644 index 0000000..84d4da4 --- /dev/null +++ b/crates/core/src/commands/pointer_action_tests.rs @@ -0,0 +1,212 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}; +use crate::refs::{RefEntry, RefMap}; +use crate::refs_store::RefStore; +use crate::refs_test_support::HomeGuard; +use crate::{AdapterError, ErrorCode, Rect, capability, hit_test::HitTestResult}; +use std::sync::atomic::{AtomicU32, Ordering}; + +#[test] +fn unstable_bounds_are_retryable() { + let err: AppError = AdapterError::new(ErrorCode::ActionFailed, "bounds changed") + .with_details(json!({ "check": "stable", "observed_bounds_hash": 42 })) + .into(); + + assert!(is_retryable_point_error(&err)); +} + +#[test] +fn occlusion_failure_is_retryable() { + let err: AppError = AdapterError::new(ErrorCode::ActionFailed, "target is occluded") + .with_details(json!({ "check": "receives_events" })) + .into(); + + assert!(is_retryable_point_error(&err)); +} + +#[test] +fn hidden_target_failure_is_retryable() { + let err: AppError = AdapterError::new(ErrorCode::ActionFailed, "target is hidden") + .with_details(json!({ "check": "visible" })) + .into(); + + assert!(is_retryable_point_error(&err)); +} + +#[test] +fn app_unresponsive_requires_explicit_retry_evidence() { + let err: AppError = AdapterError::app_unresponsive("Fixture").into(); + + assert!(!is_retryable_point_error(&err)); +} + +fn point_ref_snapshot() -> String { + let store = RefStore::new().unwrap(); + let mut refmap = RefMap::new(); + refmap.allocate(RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Target".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + }); + store.save_new_snapshot(&refmap).unwrap() +} + +fn point_args(snapshot_id: &str) -> crate::commands::point_resolve::PointResolveArgs<'_> { + crate::commands::point_resolve::PointResolveArgs { + ref_id: Some("@e1"), + xy: None, + snapshot_id: Some(snapshot_id), + missing_input_message: "target required", + headed_requirement: crate::HeadedRequirement::None, + } +} + +struct TerminalAfterRetryAdapter { + bounds_reads: AtomicU32, +} + +impl ObservationOps for TerminalAfterRetryAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + if self.bounds_reads.fetch_add(1, Ordering::SeqCst) == 0 { + return Ok(Some(Rect { + x: 1.0, + y: 2.0, + width: 3.0, + height: 4.0, + })); + } + std::thread::sleep(std::time::Duration::from_millis(120)); + Err(AdapterError::stale_ref("terminal target")) + } +} + +impl ActionOps for TerminalAfterRetryAdapter {} +impl InputOps for TerminalAfterRetryAdapter {} +impl SystemOps for TerminalAfterRetryAdapter {} + +#[test] +fn terminal_stale_ref_is_preserved_after_retry_budget_expires() { + let _guard = HomeGuard::new(); + let snapshot_id = point_ref_snapshot(); + let adapter = TerminalAfterRetryAdapter { + bounds_reads: AtomicU32::new(0), + }; + let deadline = crate::Deadline::after(100).unwrap(); + + let err = match wait_for_point_with_deadline( + point_args(&snapshot_id), + deadline, + &adapter, + &crate::CommandContext::default(), + ) { + Ok(_) => panic!("terminal stale ref must not resolve"), + Err(err) => err, + }; + + assert_eq!(err.code(), "STALE_REF"); +} + +struct UnresponsiveThenReadyAdapter { + resolve_calls: AtomicU32, +} + +impl ObservationOps for UnresponsiveThenReadyAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + if self.resolve_calls.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(AdapterError::app_unresponsive("Fixture") + .with_details(json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(Some(Rect { + x: 10.0, + y: 20.0, + width: 40.0, + height: 60.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } +} + +impl ActionOps for UnresponsiveThenReadyAdapter {} +impl InputOps for UnresponsiveThenReadyAdapter {} +impl SystemOps for UnresponsiveThenReadyAdapter {} + +#[test] +fn explicitly_retryable_app_unresponsive_recovers_within_shared_deadline() { + let _guard = HomeGuard::new(); + let snapshot_id = point_ref_snapshot(); + let adapter = UnresponsiveThenReadyAdapter { + resolve_calls: AtomicU32::new(0), + }; + let deadline = crate::Deadline::after(1_000).unwrap(); + + let resolved = wait_for_point_with_deadline( + point_args(&snapshot_id), + deadline, + &adapter, + &crate::CommandContext::default(), + ) + .unwrap(); + + assert_eq!((resolved.point.x, resolved.point.y), (30.0, 50.0)); + assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 3); +} diff --git a/crates/core/src/commands/pointer_single_shot_tests.rs b/crates/core/src/commands/pointer_single_shot_tests.rs new file mode 100644 index 0000000..df04e27 --- /dev/null +++ b/crates/core/src/commands/pointer_single_shot_tests.rs @@ -0,0 +1,233 @@ +use crate::adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}; +use crate::commands::{drag, hover}; +use crate::hit_test::HitTestResult; +use crate::refs::{RefEntry, RefMap}; +use crate::refs_store::RefStore; +use crate::refs_test_support::HomeGuard; +use crate::{ + AdapterError, CommandContext, DragParams, InteractionLease, MouseEvent, Point, ProcessId, Rect, +}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +struct SingleShotScrollAdapter { + scrolled: AtomicBool, + scrolls: AtomicU32, + resolves: AtomicU32, + leases: AtomicU32, + mouse_moves: AtomicU32, + drags: AtomicU32, + mouse_point: Mutex>, + drag_params: Mutex>, +} + +impl SingleShotScrollAdapter { + fn new() -> Self { + Self { + scrolled: AtomicBool::new(false), + scrolls: AtomicU32::new(0), + resolves: AtomicU32::new(0), + leases: AtomicU32::new(0), + mouse_moves: AtomicU32::new(0), + drags: AtomicU32::new(0), + mouse_point: Mutex::new(None), + drag_params: Mutex::new(None), + } + } +} + +impl ObservationOps for SingleShotScrollAdapter { + fn resolve_element_strict( + &self, + entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.resolves.fetch_add(1, Ordering::SeqCst); + Ok(NativeHandle::new(entry.process.pid.get())) + } + + fn get_element_bounds( + &self, + handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + let pid = *handle.downcast_ref::().unwrap(); + if pid == 1 && !self.scrolled.load(Ordering::SeqCst) { + return Ok(None); + } + Ok(Some(Rect { + x: f64::from(pid) * 100.0, + y: f64::from(pid) * 200.0, + width: 20.0, + height: 10.0, + })) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: Point, + _deadline: crate::Deadline, + ) -> Result { + Ok(HitTestResult::ReachesTarget) + } +} + +impl ActionOps for SingleShotScrollAdapter { + fn scroll_into_view( + &self, + handle: &NativeHandle, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + assert_eq!(*handle.downcast_ref::().unwrap(), 1); + self.scrolls.fetch_add(1, Ordering::SeqCst); + self.scrolled.store(true, Ordering::SeqCst); + Ok(()) + } +} + +impl InputOps for SingleShotScrollAdapter { + fn mouse_event( + &self, + event: MouseEvent, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + self.mouse_moves.fetch_add(1, Ordering::SeqCst); + *self.mouse_point.lock().unwrap() = Some(event.point); + Ok(()) + } + + fn drag(&self, params: DragParams, _lease: &InteractionLease) -> Result<(), AdapterError> { + self.drags.fetch_add(1, Ordering::SeqCst); + *self.drag_params.lock().unwrap() = Some(params); + Ok(()) + } +} + +impl SystemOps for SingleShotScrollAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result { + self.leases.fetch_add(1, Ordering::SeqCst); + InteractionLease::guarded(deadline, ()) + } + + fn resolve_window_strict( + &self, + window: &crate::WindowInfo, + _deadline: crate::Deadline, + ) -> Result { + Ok(window.clone()) + } + + fn focus_window( + &self, + _window: &crate::WindowInfo, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + Ok(()) + } +} + +fn ref_entry(pid: u32) -> RefEntry { + RefEntry { + process: crate::RefProcess { + pid: ProcessId::new(pid), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some(format!("Target {pid}")), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: Vec::new(), + available_actions: vec![crate::capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some(format!("App {pid}")), + source_window_id: Some(format!("w-{pid}")), + source_window_title: Some(format!("Window {pid}")), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + } +} + +fn snapshot(refs: u32) -> String { + let store = RefStore::new().unwrap(); + let mut refmap = RefMap::new(); + for pid in 1..=refs { + refmap.allocate(ref_entry(pid)); + } + store.save_new_snapshot(&refmap).unwrap() +} + +#[test] +fn hover_none_scrolls_once_then_dispatches_stable_point_once() { + let _guard = HomeGuard::new(); + let adapter = SingleShotScrollAdapter::new(); + let value = hover::execute( + hover::HoverArgs { + ref_id: Some("@e1".into()), + snapshot_id: Some(snapshot(1)), + xy: None, + duration_ms: None, + timeout_ms: None, + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert_eq!(adapter.scrolls.load(Ordering::SeqCst), 1); + assert_eq!(adapter.leases.load(Ordering::SeqCst), 1); + assert_eq!(adapter.resolves.load(Ordering::SeqCst), 3); + assert_eq!(adapter.mouse_moves.load(Ordering::SeqCst), 1); + assert_eq!(value["x"], 110.0); + assert_eq!(value["y"], 205.0); +} + +#[test] +fn drag_none_scrolls_once_then_dispatches_revalidated_endpoints_once() { + let _guard = HomeGuard::new(); + let adapter = SingleShotScrollAdapter::new(); + let value = drag::execute( + drag::DragArgs { + from: drag::DragEndpoint { + ref_id: Some("@e1".into()), + xy: None, + }, + to: drag::DragEndpoint { + ref_id: Some("@e2".into()), + xy: None, + }, + snapshot_id: Some(snapshot(2)), + duration_ms: None, + drop_delay_ms: None, + timeout_ms: None, + }, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert_eq!(adapter.scrolls.load(Ordering::SeqCst), 1); + assert_eq!(adapter.leases.load(Ordering::SeqCst), 1); + assert_eq!(adapter.resolves.load(Ordering::SeqCst), 5); + assert_eq!(adapter.drags.load(Ordering::SeqCst), 1); + assert_eq!(value["from"], serde_json::json!({ "x": 110.0, "y": 205.0 })); + assert_eq!(value["to"], serde_json::json!({ "x": 210.0, "y": 405.0 })); +} diff --git a/crates/core/src/commands/press.rs b/crates/core/src/commands/press.rs index 942470f..258d3f8 100644 --- a/crates/core/src/commands/press.rs +++ b/crates/core/src/commands/press.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::combo::{ensure_combo_allowed, parse_combo_normalized}, - error::AppError, + context::CommandContext, }; use serde_json::Value; @@ -13,17 +13,39 @@ pub struct PressArgs { pub force: bool, } -pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result { +pub fn execute( + args: PressArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { let combo = parse_combo_normalized(&args.combo)?; ensure_combo_allowed(&combo, &args.combo, args.force, adapter)?; + let deadline = crate::Deadline::standard()?; if let Some(app_name) = &args.app { - let result = adapter.press_key_for_app(app_name, &combo)?; + let expected = crate::commands::helpers::resolve_app(Some(app_name), adapter, deadline)?; + let lease = adapter.acquire_interaction_lease(deadline)?; + let live = crate::commands::helpers::revalidate_app_for_mutation( + adapter, + &expected, + lease.deadline(), + )?; + let process = crate::commands::helpers::process_identity(&live)?; + if context.physical_input_policy().is_headed() { + crate::headed_focus::focus_process_window(process.clone(), adapter, context, &lease)?; + } + let result = + adapter.press_key_for_app(process, &combo, context.physical_input_policy(), &lease)?; return Ok(serde_json::to_value(result)?); } + let lease = adapter.acquire_interaction_lease(deadline)?; let handle = crate::adapter::NativeHandle::null(); - let result = adapter.execute_action(&handle, ActionRequest::headed(Action::PressKey(combo)))?; + let result = adapter.execute_action( + &handle, + context.request_base(Action::PressKey(combo)), + &lease, + )?; Ok(serde_json::to_value(result)?) } diff --git a/crates/core/src/commands/press_tests.rs b/crates/core/src/commands/press_tests.rs index 8631247..13ef95d 100644 --- a/crates/core/src/commands/press_tests.rs +++ b/crates/core/src/commands/press_tests.rs @@ -1,35 +1,121 @@ use super::{PressArgs, execute}; -use crate::action::KeyCombo; +use crate::AdapterError; +use crate::KeyCombo; use crate::action_request::ActionRequest; use crate::action_result::ActionResult; -use crate::adapter::{NativeHandle, PlatformAdapter}; -use crate::error::AdapterError; +use crate::adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}; +use crate::context::CommandContext; +use std::sync::Mutex; struct BlockingAdapter; -impl PlatformAdapter for BlockingAdapter { - fn is_blocked_combo(&self, _combo: &KeyCombo) -> bool { - true - } +impl ObservationOps for BlockingAdapter {} +impl ActionOps for BlockingAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result { - Ok(ActionResult::new("PressKey")) + Ok(ActionResult::delivered_unverified("PressKey")) + } +} + +impl InputOps for BlockingAdapter {} + +impl SystemOps for BlockingAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn is_blocked_combo(&self, _combo: &KeyCombo) -> bool { + true } } struct AllowingAdapter; -impl PlatformAdapter for AllowingAdapter { +impl ObservationOps for AllowingAdapter {} + +impl ActionOps for AllowingAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result { - Ok(ActionResult::new("PressKey")) + Ok(ActionResult::delivered_unverified("PressKey")) + } +} + +impl InputOps for AllowingAdapter {} + +impl SystemOps for AllowingAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +#[derive(Default)] +struct CapturingAdapter { + global_policy: Mutex>, + app_policy: Mutex>, +} + +impl ObservationOps for CapturingAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(vec![crate::AppInfo { + name: "Editor".into(), + pid: crate::ProcessId::new(42), + bundle_id: Some("com.example.Editor".into()), + process_instance: Some("generation-1".into()), + }]) + } + + fn list_windows( + &self, + _filter: &crate::WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(vec![crate::WindowInfo { + id: "w-42".into(), + title: "Editor".into(), + app: "Editor".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("generation-1".into()), + bounds: None, + state: crate::WindowState { + visible: Some(true), + ..Default::default() + }, + }]) + } +} + +impl ActionOps for CapturingAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + *self.global_policy.lock().unwrap() = Some(request.policy); + Ok(ActionResult::delivered_unverified("PressKey")) + } +} + +impl InputOps for CapturingAdapter {} + +impl SystemOps for CapturingAdapter { + crate::adapter::guarded_interaction_lease!(); + crate::adapter::exact_window_focus!(); + + fn press_key_for_app( + &self, + _process: crate::ProcessIdentity, + _combo: &KeyCombo, + policy: crate::InteractionPolicy, + _lease: &crate::InteractionLease, + ) -> Result { + *self.app_policy.lock().unwrap() = Some(policy); + Ok(ActionResult::delivered_unverified("PressKey")) } } @@ -43,7 +129,12 @@ fn args(combo: &str, force: bool) -> PressArgs { #[test] fn adapter_blocked_combo_is_refused_when_not_forced() { - let err = execute(args("cmd+q", false), &BlockingAdapter).unwrap_err(); + let err = execute( + args("cmd+q", false), + &BlockingAdapter, + &CommandContext::default(), + ) + .unwrap_err(); assert_eq!(err.code(), "POLICY_DENIED"); assert!( err.to_string().contains("--force"), @@ -53,12 +144,68 @@ fn adapter_blocked_combo_is_refused_when_not_forced() { #[test] fn force_bypasses_the_adapter_block() { - execute(args("cmd+q", true), &BlockingAdapter) - .expect("--force must let the agent send a blocked combo"); + execute( + args("cmd+q", true), + &BlockingAdapter, + &CommandContext::default(), + ) + .expect("--force must let the agent send a blocked combo"); } #[test] fn core_blocks_nothing_by_default() { - execute(args("cmd+q", false), &AllowingAdapter) - .expect("core must not hardcode any block; the default adapter allows everything"); + execute( + args("cmd+q", false), + &AllowingAdapter, + &CommandContext::default(), + ) + .expect("core must not hardcode any block; the default adapter allows everything"); +} + +#[test] +fn global_press_uses_only_caller_authorized_policy() { + let adapter = CapturingAdapter::default(); + + execute(args("a", false), &adapter, &CommandContext::default()).unwrap(); + assert_eq!( + *adapter.global_policy.lock().unwrap(), + Some(crate::InteractionPolicy::focus_fallback()) + ); + + execute( + args("a", false), + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + assert_eq!( + *adapter.global_policy.lock().unwrap(), + Some(crate::InteractionPolicy::headed()) + ); +} + +#[test] +fn app_targeted_press_threads_focus_authorization_to_adapter() { + let adapter = CapturingAdapter::default(); + let mut request = args("a", false); + request.app = Some("Editor".into()); + + execute(request, &adapter, &CommandContext::default()).unwrap(); + assert_eq!( + *adapter.app_policy.lock().unwrap(), + Some(crate::InteractionPolicy::headless()) + ); + + let mut request = args("a", false); + request.app = Some("Editor".into()); + execute( + request, + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + assert_eq!( + *adapter.app_policy.lock().unwrap(), + Some(crate::InteractionPolicy::headed()) + ); } diff --git a/crates/core/src/commands/query.rs b/crates/core/src/commands/query.rs index af1d29d..622209a 100644 --- a/crates/core/src/commands/query.rs +++ b/crates/core/src/commands/query.rs @@ -1,22 +1,11 @@ -use crate::{error::AppError, node::AccessibilityNode, roles, search_text}; +use crate::{ + AccessibilityNode, AppError, IdentityPredicate, LocatorQuery, accessibility_node_matches, + search_text, +}; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FindQuery { - pub role: Option, - pub name: Option, - pub value: Option, - pub text: Option, -} - -impl FindQuery { - pub fn is_match_everything(&self) -> bool { - self.role.is_none() && self.name.is_none() && self.value.is_none() && self.text.is_none() - } -} - -pub fn validate_selector(raw: &str) -> Result { +pub fn validate_selector(raw: &str) -> Result { let query = parse_selector(raw); - if query.is_match_everything() { + if query.is_empty() { return Err(AppError::invalid_input_with_suggestion( "Selector must constrain at least role or text", "Use forms like \"button:Submit\", \"button\", or \":Saved!\".", @@ -25,7 +14,7 @@ pub fn validate_selector(raw: &str) -> Result { Ok(query) } -pub fn parse_selector(raw: &str) -> FindQuery { +pub fn parse_selector(raw: &str) -> LocatorQuery { let (role_part, text_part) = match raw.split_once(':') { Some((left, right)) => (Some(left.trim()), Some(right.trim())), None => (Some(raw.trim()), None), @@ -33,39 +22,26 @@ pub fn parse_selector(raw: &str) -> FindQuery { let role = role_part .filter(|part| !part.is_empty()) - .map(roles::normalize_role_query); - let text = text_part + .map(str::to_string); + let has_text = text_part .filter(|part| !part.is_empty()) .map(search_text::normalize); - FindQuery { - role, - name: None, - value: None, - text, + LocatorQuery { + identity: IdentityPredicate { + role, + ..IdentityPredicate::default() + }, + has_text, + ..LocatorQuery::default() } } -pub fn node_matches(node: &AccessibilityNode, query: &FindQuery) -> bool { - let role_match = query.role.as_deref().is_none_or(|r| node.role == r); - let name_match = query.name.as_deref().is_none_or(|n| { - node.name - .as_deref() - .is_some_and(|text| search_text::contains(text, n)) - }); - let value_match = query.value.as_deref().is_none_or(|v| { - node.value - .as_deref() - .is_some_and(|val| search_text::contains(val, v)) - }); - let text_match = query - .text - .as_deref() - .is_none_or(|t| search_text::node_contains(node, t)); - role_match && name_match && value_match && text_match +pub fn node_matches(node: &AccessibilityNode, query: &LocatorQuery) -> bool { + accessibility_node_matches(node, query) } -pub fn tree_has_match(tree: &AccessibilityNode, query: &FindQuery) -> bool { +pub fn tree_has_match(tree: &AccessibilityNode, query: &LocatorQuery) -> bool { if node_matches(tree, query) { return true; } diff --git a/crates/core/src/commands/query_tests.rs b/crates/core/src/commands/query_tests.rs index 075ded4..dd347f1 100644 --- a/crates/core/src/commands/query_tests.rs +++ b/crates/core/src/commands/query_tests.rs @@ -1,17 +1,16 @@ use super::*; -use crate::{node::AccessibilityNode, search_text}; +use crate::{AccessibilityNode, IdentityPredicate, LocatorQuery, search_text}; fn node(role: &str, name: Option<&str>, value: Option<&str>) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: role.into(), - name: name.map(String::from), - value: value.map(String::from), - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: name.map(String::from), + value: value.map(String::from), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children: vec![], } @@ -20,64 +19,62 @@ fn node(role: &str, name: Option<&str>, value: Option<&str>) -> AccessibilityNod #[test] fn parse_selector_role_and_text() { let query = parse_selector("button:Submit"); - assert_eq!(query.role.as_deref(), Some("button")); - assert_eq!(query.text.as_deref(), Some("submit")); + assert_eq!(query.identity.role.as_deref(), Some("button")); + assert_eq!(query.has_text.as_deref(), Some("submit")); } #[test] fn parse_selector_role_only() { let query = parse_selector("button"); - assert_eq!(query.role.as_deref(), Some("button")); - assert!(query.text.is_none()); + assert_eq!(query.identity.role.as_deref(), Some("button")); + assert!(query.has_text.is_none()); } #[test] fn parse_selector_text_only() { let query = parse_selector(":Saved!"); - assert!(query.role.is_none()); - assert_eq!(query.text.as_deref(), Some("saved!")); + assert!(query.identity.role.is_none()); + assert_eq!(query.has_text.as_deref(), Some("saved!")); } #[test] fn parse_selector_match_everything_variants() { for raw in ["", ":", " : "] { let query = parse_selector(raw); - assert!( - query.is_match_everything(), - "expected match-everything for {raw:?}" - ); + assert!(query.is_empty(), "expected empty query for {raw:?}"); } } #[test] fn parse_selector_empty_text_side_becomes_none() { let query = parse_selector("button:"); - assert_eq!(query.role.as_deref(), Some("button")); - assert!(query.text.is_none()); + assert_eq!(query.identity.role.as_deref(), Some("button")); + assert!(query.has_text.is_none()); } #[test] fn parse_selector_trims_whitespace() { let query = parse_selector(" button : Submit "); - assert_eq!(query.role.as_deref(), Some("button")); - assert_eq!(query.text.as_deref(), Some("submit")); + assert_eq!(query.identity.role.as_deref(), Some("button")); + assert_eq!(query.has_text.as_deref(), Some("submit")); } #[test] fn parse_selector_splits_on_first_colon_only() { let query = parse_selector("textfield:a:b"); - assert_eq!(query.role.as_deref(), Some("textfield")); - assert_eq!(query.text.as_deref(), Some("a:b")); + assert_eq!(query.identity.role.as_deref(), Some("textfield")); + assert_eq!(query.has_text.as_deref(), Some("a:b")); } #[test] fn find_query_without_text_never_calls_contains_with_empty_needle() { let root = node("button", Some("Save"), None); - let query = FindQuery { - role: Some("button".into()), - name: None, - value: None, - text: None, + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() }; assert!(tree_has_match(&root, &query)); } @@ -86,11 +83,13 @@ fn find_query_without_text_never_calls_contains_with_empty_needle() { fn tree_has_match_finds_nested_node() { let mut root = node("window", None, None); root.children.push(node("button", Some("Submit"), None)); - let query = FindQuery { - role: Some("button".into()), - name: None, - value: None, - text: Some(search_text::normalize("submit")), + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + ..IdentityPredicate::default() + }, + has_text: Some(search_text::normalize("submit")), + ..LocatorQuery::default() }; assert!(tree_has_match(&root, &query)); } diff --git a/crates/core/src/commands/ref_policy_tests.rs b/crates/core/src/commands/ref_policy_tests.rs index 9cd66c0..f4c58c7 100644 --- a/crates/core/src/commands/ref_policy_tests.rs +++ b/crates/core/src/commands/ref_policy_tests.rs @@ -1,14 +1,14 @@ +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - action::{Action, Direction}, + Action, AdapterError, Direction, action_request::ActionRequest, action_result::ActionResult, - adapter::{NativeHandle, PlatformAdapter}, + adapter::NativeHandle, commands::{ check, clear, click, collapse, double_click, expand, focus, helpers::RefArgs, right_click, scroll, scroll_to, select, set_value, toggle, triple_click, type_text, uncheck, }, context::CommandContext, - error::AdapterError, interaction_policy::InteractionPolicy, refs::{RefEntry, RefMap}, refs_store::RefStore, @@ -27,6 +27,7 @@ const POLICY_TESTED_COMMANDS: &[&str] = &[ "double_click", "expand", "focus", + "press", "right_click", "scroll", "scroll_to", @@ -54,56 +55,110 @@ impl RecordingAdapter { } } -impl PlatformAdapter for RecordingAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { +impl ObservationOps for RecordingAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { Ok(NativeHandle::null()) } + crate::adapter::complete_live_observation!( + "textfield", + "Target", + [ + crate::capability::CLICK, + crate::capability::RIGHT_CLICK, + crate::capability::SET_VALUE, + crate::capability::SET_FOCUS, + crate::capability::EXPAND, + crate::capability::COLLAPSE, + crate::capability::SELECT, + crate::capability::TOGGLE, + crate::capability::SCROLL, + crate::capability::SCROLL_TO, + crate::capability::TYPE_TEXT, + ] + ); +} + +impl ActionOps for RecordingAdapter { fn execute_action( &self, _handle: &NativeHandle, request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result { self.requests.lock().unwrap().push(request); - Ok(ActionResult::new("ok")) + Ok(ActionResult::delivered_unverified("ok")) } } +impl InputOps for RecordingAdapter {} + +impl SystemOps for RecordingAdapter { + crate::adapter::guarded_interaction_lease!(); + crate::adapter::exact_window_focus!(); +} + fn snapshot_id() -> String { let mut refmap = RefMap::new(); + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; refmap.allocate(RefEntry { - pid: 1, - role: "textfield".into(), - name: Some("Target".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![ - "Check".into(), - "Clear".into(), - "Click".into(), - "Collapse".into(), - "DoubleClick".into(), - "Expand".into(), - "RightClick".into(), - "ScrollTo".into(), - "Select".into(), - "SetFocus".into(), - "SetValue".into(), - "Toggle".into(), - "TripleClick".into(), - "TypeText".into(), - "Uncheck".into(), - ], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "textfield".into(), + name: Some("Target".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![ + "Check".into(), + "Clear".into(), + "Click".into(), + "Collapse".into(), + "DoubleClick".into(), + "Expand".into(), + "RightClick".into(), + "Scroll".into(), + "ScrollTo".into(), + "Select".into(), + "SetFocus".into(), + "SetValue".into(), + "Toggle".into(), + "TripleClick".into(), + "TypeText".into(), + "Uncheck".into(), + ], + }, + source: crate::RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-1".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() } @@ -112,6 +167,7 @@ fn ref_args(snapshot_id: &str) -> RefArgs { RefArgs { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.to_owned()), + timeout_ms: None, } } @@ -120,15 +176,19 @@ fn assert_headless(request: &ActionRequest) { } #[test] -fn default_ref_commands_are_headless() { +fn default_ref_commands_use_least_permissive_supported_policy() { let _guard = HomeGuard::new(); let snapshot_id = snapshot_id(); let adapter = RecordingAdapter::new(); let context = CommandContext::default(); click::execute(ref_args(&snapshot_id), &adapter, &context).unwrap(); - double_click::execute(ref_args(&snapshot_id), &adapter, &context).unwrap(); - triple_click::execute(ref_args(&snapshot_id), &adapter, &context).unwrap(); + let double_click_error = + double_click::execute(ref_args(&snapshot_id), &adapter, &context).unwrap_err(); + assert_eq!(double_click_error.code(), "POLICY_DENIED"); + let triple_click_error = + triple_click::execute(ref_args(&snapshot_id), &adapter, &context).unwrap_err(); + assert_eq!(triple_click_error.code(), "POLICY_DENIED"); let before_right_click = adapter.requests.lock().unwrap().len(); let _ = right_click::execute(ref_args(&snapshot_id), &adapter, &context); assert_eq!( @@ -147,6 +207,7 @@ fn default_ref_commands_are_headless() { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.clone()), value: "value".into(), + timeout_ms: None, }, &adapter, &context, @@ -157,6 +218,7 @@ fn default_ref_commands_are_headless() { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.clone()), value: "choice".into(), + timeout_ms: None, }, &adapter, &context, @@ -168,19 +230,21 @@ fn default_ref_commands_are_headless() { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.clone()), text: "text".into(), + timeout_ms: None, }, &adapter, &context, ) .unwrap(); let type_request = adapter.requests.lock().unwrap()[before_type].clone(); - assert_eq!(type_request.policy, InteractionPolicy::focus_fallback()); + assert_eq!(type_request.policy, InteractionPolicy::headless()); scroll::execute( scroll::ScrollArgs { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id), direction: Direction::Down, amount: 1, + timeout_ms: None, }, &adapter, &context, @@ -188,9 +252,6 @@ fn default_ref_commands_are_headless() { .unwrap(); for request in adapter.requests.lock().unwrap().iter() { - if matches!(request.action, Action::TypeText(_)) { - continue; - } assert_headless(request); } } @@ -209,7 +270,7 @@ fn focus_command_is_explicit_headless_policy() { } #[test] -fn headed_context_upgrades_every_ref_command_to_headed() { +fn headed_context_reaches_every_ref_action_without_policy_downgrade() { let _guard = HomeGuard::new(); let snapshot_id = snapshot_id(); let adapter = RecordingAdapter::new(); @@ -232,6 +293,7 @@ fn headed_context_upgrades_every_ref_command_to_headed() { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.clone()), value: "value".into(), + timeout_ms: None, }, &adapter, &context, @@ -242,6 +304,7 @@ fn headed_context_upgrades_every_ref_command_to_headed() { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.clone()), value: "choice".into(), + timeout_ms: None, }, &adapter, &context, @@ -252,6 +315,7 @@ fn headed_context_upgrades_every_ref_command_to_headed() { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id.clone()), text: "text".into(), + timeout_ms: None, }, &adapter, &context, @@ -263,6 +327,7 @@ fn headed_context_upgrades_every_ref_command_to_headed() { snapshot_id: Some(snapshot_id), direction: Direction::Down, amount: 1, + timeout_ms: None, }, &adapter, &context, @@ -273,7 +338,7 @@ fn headed_context_upgrades_every_ref_command_to_headed() { assert_eq!( request.policy, InteractionPolicy::headed(), - "{:?} must be headed under --headed", + "unexpected policy for {:?}", request.action ); } diff --git a/crates/core/src/commands/resize_window.rs b/crates/core/src/commands/resize_window.rs index 415c256..b53a944 100644 --- a/crates/core/src/commands/resize_window.rs +++ b/crates/core/src/commands/resize_window.rs @@ -1,23 +1,35 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_window_for_app, - error::AppError, + AppError, WindowOp, + adapter::PlatformAdapter, + commands::helpers::{resolve_window_for_app, revalidate_window_for_mutation}, }; use serde_json::{Value, json}; pub struct ResizeWindowArgs { pub app: Option, + pub window_id: Option, pub width: f64, pub height: f64, } pub fn execute(args: ResizeWindowArgs, adapter: &dyn PlatformAdapter) -> Result { - let win = resolve_window_for_app(args.app.as_deref(), adapter)?; + crate::Rect { + x: 0.0, + y: 0.0, + width: args.width, + height: args.height, + } + .validate()?; + let win = resolve_window_for_app(args.app.as_deref(), args.window_id.as_deref(), adapter)?; + let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?; + let live = revalidate_window_for_mutation(adapter, &win, &lease)?; adapter.window_op( - &win, + &live, WindowOp::Resize { width: args.width, height: args.height, }, + &lease, )?; Ok(json!({ "resized": true, "width": args.width, "height": args.height })) } diff --git a/crates/core/src/commands/restore.rs b/crates/core/src/commands/restore.rs index 091f3f1..75c117c 100644 --- a/crates/core/src/commands/restore.rs +++ b/crates/core/src/commands/restore.rs @@ -1,8 +1,7 @@ use crate::{ - action::WindowOp, + AppError, WindowOp, adapter::PlatformAdapter, commands::helpers::{AppArgs, window_op_command}, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/right_click.rs b/crates/core/src/commands/right_click.rs index fee9409..40e7235 100644 --- a/crates/core/src/commands/right_click.rs +++ b/crates/core/src/commands/right_click.rs @@ -1,14 +1,11 @@ use crate::{ + AppError, action::Action, - adapter::{PlatformAdapter, SnapshotSurface, TreeOptions}, - commands::helpers::{ - RefArgs, apply_post_action_wait, execute_ref_action_result_with_context, probe_app_name, - }, + adapter::PlatformAdapter, + commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, - snapshot, }; -use serde_json::{Value, json}; +use serde_json::Value; pub fn execute( args: RefArgs, @@ -16,70 +13,7 @@ pub fn execute( context: &CommandContext, ) -> Result { let request = context.request_base(Action::RightClick); - let (entry, result) = execute_ref_action_result_with_context( - &args.ref_id, - args.snapshot_id.as_deref(), - adapter, - request, - context, - )?; - let mut response = serde_json::to_value(&result)?; - - std::thread::sleep(std::time::Duration::from_millis(200)); - - let opts = TreeOptions { - interactive_only: true, - surface: SnapshotSurface::Menu, - ..Default::default() - }; - let probe_app = probe_app_name(adapter, &entry); - match snapshot::run_with_context(adapter, &opts, probe_app.as_deref(), None, context) { - Ok(snap) => match serde_json::to_value(&snap.tree) { - Ok(menu_json) => { - response["menu"] = menu_json; - if let Some(snapshot_id) = snap.snapshot_id { - response["menu_snapshot_id"] = json!(snapshot_id); - } - } - Err(err) => { - response["menu_probe"] = json!({ - "ok": false, - "error": { - "code": "INTERNAL", - "message": err.to_string(), - } - }) - } - }, - Err(err) => response["menu_probe"] = probe_error_json(&err), - } - - apply_post_action_wait(response, &entry, adapter, context) -} - -fn probe_error_json(err: &AppError) -> Value { - if err.code() == "ELEMENT_NOT_FOUND" { - return json!({ - "ok": false, - "error": { - "code": "ELEMENT_NOT_FOUND", - "message": "Right-click action was accepted, but no menu accessibility tree was exposed for capture.", - "suggestion": "Use 'snapshot --surface menu' only when the app exposes the context menu through accessibility." - } - }); - } - - let mut error = json!({ - "code": err.code(), - "message": err.to_string(), - }); - if let Some(suggestion) = err.suggestion() { - error["suggestion"] = json!(suggestion); - } - json!({ - "ok": false, - "error": error, - }) + execute_ref_action_with_context(args, adapter, request, context) } #[cfg(test)] diff --git a/crates/core/src/commands/right_click_tests.rs b/crates/core/src/commands/right_click_tests.rs index 701dda2..a666c05 100644 --- a/crates/core/src/commands/right_click_tests.rs +++ b/crates/core/src/commands/right_click_tests.rs @@ -1,10 +1,11 @@ use super::*; +use crate::adapter::TreeOptions; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ + AdapterError, ErrorCode, WindowInfo, action_request::ActionRequest, action_result::ActionResult, adapter::{NativeHandle, WindowFilter}, - error::{AdapterError, ErrorCode}, - node::WindowInfo, refs::{RefEntry, RefMap}, refs_store::RefStore, refs_test_support::HomeGuard, @@ -14,20 +15,20 @@ struct ProbeFailingAdapter { tree_error: Option, } -impl PlatformAdapter for ProbeFailingAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { +impl ObservationOps for ProbeFailingAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { Ok(NativeHandle::null()) } - fn execute_action( + fn list_windows( &self, - _handle: &NativeHandle, - _request: ActionRequest, - ) -> Result { - Ok(ActionResult::new("right_click")) - } - - fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError> { + filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { if filter.app.is_some() && self.tree_error.is_none() { return Err(AdapterError::new( ErrorCode::WindowNotFound, @@ -44,9 +45,13 @@ impl PlatformAdapter for ProbeFailingAdapter { id: "w1".into(), title: "Main".into(), app: "TargetApp".into(), - pid: 7, + pid: crate::ProcessId::new(7), + process_instance: Some("test-instance".into()), bounds: None, - is_focused: true, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, }]) } @@ -54,51 +59,87 @@ impl PlatformAdapter for ProbeFailingAdapter { &self, _win: &WindowInfo, _opts: &TreeOptions, - ) -> Result { + _deadline: crate::Deadline, + ) -> Result { if let Some(code) = self.tree_error.clone() { return Err(AdapterError::new(code, "menu tree unavailable")); } - Ok(crate::node::AccessibilityNode { + Ok(crate::AccessibilityNode { ref_id: None, role: "menu".into(), - name: None, - value: None, - description: None, - hint: None, - states: Vec::new(), - available_actions: Vec::new(), - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: Vec::new(), }) } + + crate::adapter::complete_live_observation!("button", "Open", [crate::capability::RIGHT_CLICK]); +} + +impl ActionOps for ProbeFailingAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("right_click")) + } +} + +impl InputOps for ProbeFailingAdapter {} + +impl SystemOps for ProbeFailingAdapter { + crate::adapter::guarded_interaction_lease!(); } fn save_refmap(source_app: Option) -> String { let mut refmap = RefMap::new(); + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; refmap.allocate(RefEntry { - pid: 7, - role: "button".into(), - name: Some("Open".into()), - value: None, - description: None, - states: Vec::new(), - bounds: None, - bounds_hash: None, - available_actions: vec!["RightClick".into()], - source_app, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(7), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Open".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: Vec::new(), + available_actions: vec!["RightClick".into()], + }, + source: crate::RefSource { + source_app, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() } #[test] -fn returns_action_success_when_menu_probe_fails() { +fn returns_action_success_without_a_synthetic_menu_probe() { let _guard = HomeGuard::new(); let snapshot_id = save_refmap(None); @@ -106,6 +147,7 @@ fn returns_action_success_when_menu_probe_fails() { RefArgs { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id), + timeout_ms: None, }, &ProbeFailingAdapter { tree_error: None }, &CommandContext::default(), @@ -113,12 +155,11 @@ fn returns_action_success_when_menu_probe_fails() { .unwrap(); assert_eq!(value["action"], "right_click"); - assert_eq!(value["menu_probe"]["ok"], false); - assert_eq!(value["menu_probe"]["error"]["code"], "WINDOW_NOT_FOUND"); + assert!(value.get("menu_probe").is_none()); } #[test] -fn element_not_found_menu_probe_uses_right_click_specific_guidance() { +fn right_click_result_does_not_depend_on_a_followup_tree_probe() { let _guard = HomeGuard::new(); let snapshot_id = save_refmap(Some("TargetApp".into())); @@ -126,6 +167,7 @@ fn element_not_found_menu_probe_uses_right_click_specific_guidance() { RefArgs { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id), + timeout_ms: None, }, &ProbeFailingAdapter { tree_error: Some(ErrorCode::ElementNotFound), @@ -135,12 +177,5 @@ fn element_not_found_menu_probe_uses_right_click_specific_guidance() { .unwrap(); assert_eq!(value["action"], "right_click"); - assert_eq!(value["menu_probe"]["ok"], false); - assert_eq!(value["menu_probe"]["error"]["code"], "ELEMENT_NOT_FOUND"); - assert!( - value["menu_probe"]["error"]["suggestion"] - .as_str() - .unwrap() - .contains("snapshot --surface menu") - ); + assert!(value.get("menu_probe").is_none()); } diff --git a/crates/core/src/commands/screenshot.rs b/crates/core/src/commands/screenshot.rs index b135c77..0435002 100644 --- a/crates/core/src/commands/screenshot.rs +++ b/crates/core/src/commands/screenshot.rs @@ -1,6 +1,6 @@ use crate::{ + AdapterError, AppError, ErrorCode, adapter::{PlatformAdapter, ScreenshotTarget, WindowFilter}, - error::AppError, }; use base64::Engine; use serde_json::{Value, json}; @@ -9,23 +9,32 @@ use std::path::PathBuf; pub struct ScreenshotArgs { pub app: Option, pub window_id: Option, + pub screen: Option, pub output_path: Option, } pub fn execute(args: ScreenshotArgs, adapter: &dyn PlatformAdapter) -> Result { - let target = resolve_target(&args, adapter)?; - let buf = adapter.screenshot(target)?; + let deadline = crate::Deadline::standard()?; + let target = resolve_target(&args, adapter, deadline)?; + let buf = adapter.screenshot(target, deadline)?; if let Some(path) = args.output_path { - std::fs::write(&path, &buf.data)?; - Ok(json!({ "path": path.to_string_lossy() })) + crate::refs::write_user_file(&path, &buf.data)?; + Ok(json!({ + "path": path.to_string_lossy(), + "format": buf.format.as_str(), + "width": buf.width, + "height": buf.height, + "scale_factor": buf.scale_factor + })) } else { let encoded = base64::engine::general_purpose::STANDARD.encode(&buf.data); Ok(json!({ "data": encoded, "format": buf.format.as_str(), "width": buf.width, - "height": buf.height + "height": buf.height, + "scale_factor": buf.scale_factor })) } } @@ -33,31 +42,99 @@ pub fn execute(args: ScreenshotArgs, adapter: &dyn PlatformAdapter) -> Result Result { + if args.screen.is_some() && (args.app.is_some() || args.window_id.is_some()) { + return Err(AppError::invalid_input_with_suggestion( + "--screen cannot be combined with --app or --window-id", + "Choose exactly one display target or one app/window target.", + )); + } + if let Some(screen) = args.screen { + let displays = adapter.list_displays(deadline).map_err(AppError::from)?; + if screen >= displays.len() { + return Err(AppError::Adapter( + AdapterError::new( + ErrorCode::InvalidArgs, + format!( + "Display index {screen} out of range; {} display(s) available", + displays.len() + ), + ) + .with_details(json!({ + "display_count": displays.len(), + "display_ids": displays.iter().map(|display| display.id.clone()).collect::>() + })), + )); + } + return Ok(ScreenshotTarget::Display { + index: screen, + expected: displays[screen].clone(), + }); + } + if let Some(window_id) = &args.window_id { + let expected_app = args + .app + .as_deref() + .map(|name| crate::commands::helpers::resolve_app(Some(name), adapter, deadline)) + .transpose()?; let filter = WindowFilter { focused_only: false, app: args.app.clone(), }; - let windows = adapter.list_windows(&filter)?; - let win = windows + let mut candidates = adapter + .list_windows(&filter, deadline)? .into_iter() - .find(|w| &w.id == window_id) - .ok_or_else(|| AppError::invalid_input(format!("Window '{window_id}' not found")))?; - return Ok(ScreenshotTarget::Window(win.pid)); + .filter(|window| &window.id == window_id) + .collect::>(); + if candidates + .iter() + .any(|window| window.process_instance.as_deref().is_none_or(str::is_empty)) + { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "Matching window has incomplete process identity", + ) + .into()); + } + if let Some(app) = expected_app.as_ref() { + let instance = crate::commands::helpers::process_identity(app)?.instance; + candidates.retain(|window| { + window.pid == app.pid + && window.process_instance.as_deref() == Some(instance.as_str()) + }); + } + let win = select_unique_window(candidates, window_id)?; + return Ok(ScreenshotTarget::ExactWindow(win)); } if let Some(app_name) = &args.app { - let filter = WindowFilter { - focused_only: false, - app: Some(app_name.clone()), - }; - let windows = adapter.list_windows(&filter)?; - let win = windows.into_iter().next().ok_or_else(|| { - AppError::invalid_input(format!("No windows found for app '{app_name}'")) - })?; - return Ok(ScreenshotTarget::Window(win.pid)); + let app = crate::commands::helpers::resolve_app(Some(app_name), adapter, deadline)?; + let win = crate::window_lookup::find_window_for_process( + crate::commands::helpers::process_identity(&app)?, + adapter, + deadline, + )?; + return Ok(ScreenshotTarget::ExactWindow(win)); } Ok(ScreenshotTarget::FullScreen) } + +fn select_unique_window( + mut candidates: Vec, + window_id: &str, +) -> Result { + match candidates.len() { + 0 => Err(AppError::invalid_input(format!( + "Window '{window_id}' not found" + ))), + 1 => Ok(candidates.swap_remove(0)), + _ => Err(AdapterError::ambiguous_target(format!( + "Multiple windows matched id '{window_id}'" + )) + .with_details(json!({ "candidate_count": candidates.len() })) + .into()), + } +} diff --git a/crates/core/src/commands/screenshot_tests.rs b/crates/core/src/commands/screenshot_tests.rs new file mode 100644 index 0000000..403ce4b --- /dev/null +++ b/crates/core/src/commands/screenshot_tests.rs @@ -0,0 +1,256 @@ +use crate::{ + AdapterError, ImageBuffer, ImageFormat, Rect, WindowInfo, + adapter::{ActionOps, InputOps, ObservationOps, ScreenshotTarget, SystemOps, WindowFilter}, + commands::screenshot::{self, ScreenshotArgs}, + display_info::DisplayInfo, +}; +use std::{path::PathBuf, sync::Mutex}; + +struct ScreenshotAdapter { + displays: Vec, + windows: Vec, + target: Mutex>, +} + +impl ScreenshotAdapter { + fn new() -> Self { + let mut focused = window("w-42", 700); + focused.state.is_focused = true; + Self { + displays: vec![display("main", true, 2.0), display("secondary", false, 1.0)], + windows: vec![window("w-41", 700), focused], + target: Mutex::new(None), + } + } + + fn take_target(&self) -> Option { + self.target.lock().expect("target lock").take() + } +} + +impl ObservationOps for ScreenshotAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(vec![crate::AppInfo { + name: "Example".into(), + pid: crate::ProcessId::new(700), + bundle_id: Some("com.example.app".into()), + process_instance: Some("instance-700".into()), + }]) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { + Ok(self.windows.clone()) + } +} + +impl ActionOps for ScreenshotAdapter {} +impl InputOps for ScreenshotAdapter {} + +impl SystemOps for ScreenshotAdapter { + fn screenshot( + &self, + target: ScreenshotTarget, + _deadline: crate::Deadline, + ) -> Result { + *self.target.lock().expect("target lock") = Some(target); + Ok(ImageBuffer { + data: vec![1, 2, 3], + format: ImageFormat::Png, + width: 640, + height: 480, + scale_factor: 2.0, + }) + } + + fn list_displays(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(self.displays.clone()) + } +} + +fn display(id: &str, is_primary: bool, scale: f64) -> DisplayInfo { + DisplayInfo { + id: id.into(), + bounds: Rect { + x: 0.0, + y: 0.0, + width: 1440.0, + height: 900.0, + }, + is_primary, + scale, + } +} + +fn window(id: &str, pid: u32) -> WindowInfo { + WindowInfo { + id: id.into(), + title: format!("Window {id}"), + app: "Example".into(), + pid: crate::ProcessId::new(pid), + process_instance: Some(format!("instance-{pid}")), + bounds: None, + state: crate::WindowState { + is_focused: false, + ..Default::default() + }, + } +} + +fn args() -> ScreenshotArgs { + ScreenshotArgs { + app: None, + window_id: None, + screen: None, + output_path: None, + } +} + +#[test] +fn screen_target_pins_selected_display_identity() { + let adapter = ScreenshotAdapter::new(); + let mut request = args(); + request.screen = Some(1); + + screenshot::execute(request, &adapter).expect("screenshot"); + + match adapter.take_target() { + Some(ScreenshotTarget::Display { index, expected }) => { + assert_eq!(index, 1); + assert_eq!(expected.id, "secondary"); + assert_eq!(expected.scale, 1.0); + } + _ => panic!("expected identity-pinned display target"), + } +} + +#[test] +fn screen_rejects_app_and_window_target_conflicts() { + for (app, window_id) in [ + (Some("Example".into()), None), + (None, Some("w-42".into())), + (Some("Example".into()), Some("w-42".into())), + ] { + let adapter = ScreenshotAdapter::new(); + let mut request = args(); + request.screen = Some(0); + request.app = app; + request.window_id = window_id; + + let error = screenshot::execute(request, &adapter).expect_err("targets conflict"); + assert_eq!(error.code(), "INVALID_ARGS"); + assert!(adapter.take_target().is_none()); + } +} + +#[test] +fn window_id_target_pins_exact_window_not_only_its_pid() { + let adapter = ScreenshotAdapter::new(); + let mut request = args(); + request.window_id = Some("w-42".into()); + + screenshot::execute(request, &adapter).expect("screenshot"); + + match adapter.take_target() { + Some(ScreenshotTarget::ExactWindow(window)) => { + assert_eq!(window.id, "w-42"); + assert_eq!(window.pid, 700); + } + _ => panic!("expected exact window target"), + } +} + +#[test] +fn app_target_resolves_one_exact_focused_window() { + let adapter = ScreenshotAdapter::new(); + let mut request = args(); + request.app = Some("Example".into()); + + screenshot::execute(request, &adapter).expect("screenshot"); + + match adapter.take_target() { + Some(ScreenshotTarget::ExactWindow(window)) => { + assert_eq!(window.id, "w-42"); + assert_eq!(window.process_instance.as_deref(), Some("instance-700")); + } + _ => panic!("expected exact window target"), + } +} + +#[test] +fn missing_window_returns_an_error_without_capturing() { + let adapter = ScreenshotAdapter::new(); + let mut request = args(); + request.window_id = Some("w-404".into()); + + let error = screenshot::execute(request, &adapter).expect_err("missing window"); + + assert_eq!(error.code(), "INVALID_ARGS"); + assert!(adapter.take_target().is_none()); +} + +#[test] +fn output_path_response_keeps_image_metadata() { + let adapter = ScreenshotAdapter::new(); + let path = output_path(); + let mut request = args(); + request.output_path = Some(path.clone()); + + let response = screenshot::execute(request, &adapter).expect("screenshot"); + + assert_eq!(std::fs::read(&path).expect("saved screenshot"), [1, 2, 3]); + assert_eq!(response["path"], path.to_string_lossy().as_ref()); + assert_eq!(response["format"], "png"); + assert_eq!(response["width"], 640); + assert_eq!(response["height"], 480); + assert_eq!(response["scale_factor"], 2.0); + std::fs::remove_file(path).expect("remove screenshot"); +} + +#[cfg(unix)] +#[test] +fn output_path_rejects_symlinks_and_creates_private_files() { + use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; + + let parent = std::env::temp_dir().join(format!( + "agent-desktop-core-screenshot-private-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::create_dir_all(&parent).expect("create private output directory"); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)) + .expect("secure output directory"); + let victim = parent.join("victim.png"); + let link = parent.join("screenshot.png"); + std::fs::write(&victim, b"unchanged").expect("write victim"); + symlink(&victim, &link).expect("create output symlink"); + + let mut request = args(); + request.output_path = Some(link); + screenshot::execute(request, &ScreenshotAdapter::new()).expect_err("reject symlink"); + assert_eq!(std::fs::read(&victim).expect("read victim"), b"unchanged"); + + let output = parent.join("private.png"); + let mut request = args(); + request.output_path = Some(output.clone()); + screenshot::execute(request, &ScreenshotAdapter::new()).expect("write private screenshot"); + assert_eq!( + std::fs::metadata(output).expect("output metadata").mode() & 0o777, + 0o600 + ); + std::fs::remove_file(parent.join("screenshot.png")).expect("remove symlink"); + std::fs::remove_file(parent.join("victim.png")).expect("remove victim"); + std::fs::remove_file(parent.join("private.png")).expect("remove output"); + std::fs::remove_dir(parent).expect("remove output directory"); +} + +fn output_path() -> PathBuf { + std::env::temp_dir().join(format!( + "agent-desktop-core-screenshot-{}-{:?}.png", + std::process::id(), + std::thread::current().id() + )) +} diff --git a/crates/core/src/commands/scroll.rs b/crates/core/src/commands/scroll.rs index 33b6c5b..79bb9fc 100644 --- a/crates/core/src/commands/scroll.rs +++ b/crates/core/src/commands/scroll.rs @@ -1,9 +1,8 @@ use crate::{ - action::{Action, Direction}, + Action, AppError, Direction, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; @@ -12,6 +11,7 @@ pub struct ScrollArgs { pub snapshot_id: Option, pub direction: Direction, pub amount: u32, + pub timeout_ms: Option, } pub fn execute( @@ -24,9 +24,14 @@ pub fn execute( RefArgs { ref_id: args.ref_id, snapshot_id: args.snapshot_id, + timeout_ms: args.timeout_ms, }, adapter, request, context, ) } + +#[cfg(test)] +#[path = "scroll_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/scroll_tests.rs b/crates/core/src/commands/scroll_tests.rs new file mode 100644 index 0000000..616073c --- /dev/null +++ b/crates/core/src/commands/scroll_tests.rs @@ -0,0 +1,102 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::commands::helpers::test_support::save_one_ref_snapshot; +use crate::{ + AdapterError, Direction, action_request::ActionRequest, action_result::ActionResult, + adapter::NativeHandle, commands::stale_retry_test_support::StaleRetryCounter, refs::RefEntry, + refs_test_support::HomeGuard, +}; + +struct StaleThenOkAdapter { + retry: StaleRetryCounter, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + retry: StaleRetryCounter::new(fail_until), + } + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.retry.attempt() + } + + crate::adapter::complete_live_observation!("scrollarea", "Target", [crate::capability::SCROLL]); +} + +impl ActionOps for StaleThenOkAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("scroll")) + } +} + +impl InputOps for StaleThenOkAdapter {} +impl SystemOps for StaleThenOkAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +fn snapshot_id() -> String { + save_one_ref_snapshot("scrollarea", "Scroll") +} + +/// Regression for the F2 fix: before it, `scroll::execute` always built its +/// `RefArgs` with `timeout_ms: None`, so a caller-supplied `--timeout-ms` +/// budget was silently dropped and the command never retried a transient +/// `STALE_REF`. +#[test] +fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(2); + + let value = execute( + ScrollArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + direction: Direction::Down, + amount: 3, + timeout_ms: Some(5_000), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["action"], "scroll"); + assert!(adapter.retry.calls() >= 3); +} + +#[test] +fn timeout_none_makes_exactly_one_resolve_attempt() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(1); + + let err = execute( + ScrollArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + direction: Direction::Down, + amount: 3, + timeout_ms: None, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "STALE_REF"); + assert_eq!(adapter.retry.calls(), 1); +} diff --git a/crates/core/src/commands/scroll_to.rs b/crates/core/src/commands/scroll_to.rs index efe717b..0668a80 100644 --- a/crates/core/src/commands/scroll_to.rs +++ b/crates/core/src/commands/scroll_to.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/select.rs b/crates/core/src/commands/select.rs index 75f22d8..e30a28b 100644 --- a/crates/core/src/commands/select.rs +++ b/crates/core/src/commands/select.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; @@ -11,6 +11,7 @@ pub struct SelectArgs { pub ref_id: String, pub snapshot_id: Option, pub value: String, + pub timeout_ms: Option, } pub fn execute( @@ -23,9 +24,14 @@ pub fn execute( RefArgs { ref_id: args.ref_id, snapshot_id: args.snapshot_id, + timeout_ms: args.timeout_ms, }, adapter, request, context, ) } + +#[cfg(test)] +#[path = "select_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/select_tests.rs b/crates/core/src/commands/select_tests.rs new file mode 100644 index 0000000..fc81ff5 --- /dev/null +++ b/crates/core/src/commands/select_tests.rs @@ -0,0 +1,100 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::commands::helpers::test_support::save_one_ref_snapshot; +use crate::{ + AdapterError, action_request::ActionRequest, action_result::ActionResult, + adapter::NativeHandle, commands::stale_retry_test_support::StaleRetryCounter, refs::RefEntry, + refs_test_support::HomeGuard, +}; + +struct StaleThenOkAdapter { + retry: StaleRetryCounter, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + retry: StaleRetryCounter::new(fail_until), + } + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.retry.attempt() + } + + crate::adapter::complete_live_observation!("combobox", "Target", [crate::capability::SELECT]); +} + +impl ActionOps for StaleThenOkAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("select")) + } +} + +impl InputOps for StaleThenOkAdapter {} +impl SystemOps for StaleThenOkAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +fn snapshot_id() -> String { + save_one_ref_snapshot("combobox", "Select") +} + +/// Regression for the F2 fix: before it, `select::execute` always built its +/// `RefArgs` with `timeout_ms: None`, so a caller-supplied `--timeout-ms` +/// budget was silently dropped and the command never retried a transient +/// `STALE_REF`. +#[test] +fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(2); + + let value = execute( + SelectArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + value: "choice".into(), + timeout_ms: Some(5_000), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["action"], "select"); + assert!(adapter.retry.calls() >= 3); +} + +#[test] +fn timeout_none_makes_exactly_one_resolve_attempt() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(1); + + let err = execute( + SelectArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + value: "choice".into(), + timeout_ms: None, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "STALE_REF"); + assert_eq!(adapter.retry.calls(), 1); +} diff --git a/crates/core/src/commands/session.rs b/crates/core/src/commands/session.rs index f139a14..a09b319 100644 --- a/crates/core/src/commands/session.rs +++ b/crates/core/src/commands/session.rs @@ -1,4 +1,4 @@ -use crate::error::AppError; +use crate::AppError; use crate::session::{ ArtifactsMode, GcOptions, SessionTraceMode, StartSessionOptions, end_session, gc, list_sessions, start_session, @@ -12,10 +12,9 @@ pub enum SessionAction { name: Option, no_trace: bool, screenshots: bool, - force: bool, }, End { - id: Option, + id: String, }, List, Gc { @@ -30,7 +29,6 @@ pub fn execute(action: SessionAction) -> Result { name, no_trace, screenshots, - force, } => { let manifest = start_session(StartSessionOptions { name, @@ -44,7 +42,6 @@ pub fn execute(action: SessionAction) -> Result { } else { ArtifactsMode::Events }, - force, })?; Ok(json!({ "session_id": manifest.id, @@ -55,7 +52,7 @@ pub fn execute(action: SessionAction) -> Result { })) } SessionAction::End { id } => { - let manifest = end_session(id.as_deref())?; + let manifest = end_session(&id)?; Ok(json!({ "session_id": manifest.id, "ended_at": manifest.ended_at, diff --git a/crates/core/src/commands/set_value.rs b/crates/core/src/commands/set_value.rs index 4ab94a3..69a9029 100644 --- a/crates/core/src/commands/set_value.rs +++ b/crates/core/src/commands/set_value.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; @@ -11,6 +11,7 @@ pub struct SetValueArgs { pub ref_id: String, pub snapshot_id: Option, pub value: String, + pub timeout_ms: Option, } pub fn execute( @@ -23,9 +24,14 @@ pub fn execute( RefArgs { ref_id: args.ref_id, snapshot_id: args.snapshot_id, + timeout_ms: args.timeout_ms, }, adapter, request, context, ) } + +#[cfg(test)] +#[path = "set_value_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/set_value_tests.rs b/crates/core/src/commands/set_value_tests.rs new file mode 100644 index 0000000..46075c3 --- /dev/null +++ b/crates/core/src/commands/set_value_tests.rs @@ -0,0 +1,104 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::commands::helpers::test_support::save_one_ref_snapshot; +use crate::{ + AdapterError, action_request::ActionRequest, action_result::ActionResult, + adapter::NativeHandle, commands::stale_retry_test_support::StaleRetryCounter, refs::RefEntry, + refs_test_support::HomeGuard, +}; + +struct StaleThenOkAdapter { + retry: StaleRetryCounter, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + retry: StaleRetryCounter::new(fail_until), + } + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.retry.attempt() + } + + crate::adapter::complete_live_observation!( + "textfield", + "Target", + [crate::capability::SET_VALUE] + ); +} + +impl ActionOps for StaleThenOkAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("set_value")) + } +} + +impl InputOps for StaleThenOkAdapter {} +impl SystemOps for StaleThenOkAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +fn snapshot_id() -> String { + save_one_ref_snapshot("textfield", "SetValue") +} + +/// Regression for the F2 fix: before it, `set_value::execute` always built its +/// `RefArgs` with `timeout_ms: None`, so a caller-supplied `--timeout-ms` +/// budget was silently dropped and the command never retried a transient +/// `STALE_REF`. +#[test] +fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(2); + + let value = execute( + SetValueArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + value: "hi".into(), + timeout_ms: Some(5_000), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["action"], "set_value"); + assert!(adapter.retry.calls() >= 3); +} + +#[test] +fn timeout_none_makes_exactly_one_resolve_attempt() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(1); + + let err = execute( + SetValueArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + value: "hi".into(), + timeout_ms: None, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "STALE_REF"); + assert_eq!(adapter.retry.calls(), 1); +} diff --git a/crates/core/src/commands/skills.rs b/crates/core/src/commands/skills.rs index 5e636b7..49d85cf 100644 --- a/crates/core/src/commands/skills.rs +++ b/crates/core/src/commands/skills.rs @@ -1,4 +1,4 @@ -use crate::error::AppError; +use crate::AppError; use serde_json::{Value, json}; const SKILL_DESKTOP_MAIN: &str = include_str!("../../../../skills/agent-desktop/SKILL.md"); @@ -38,37 +38,32 @@ struct Skill { refs: fn() -> &'static [SkillRef], } -/// Returns the four platform-agnostic desktop refs, plus the macOS ref on macOS targets. -/// Built once via OnceLock so the base entries are never duplicated. +const SKILL_DESKTOP_REFS: &[SkillRef] = &[ + SkillRef { + rel_path: "references/commands-observation.md", + body: SKILL_DESKTOP_REF_OBSERVATION, + }, + SkillRef { + rel_path: "references/commands-interaction.md", + body: SKILL_DESKTOP_REF_INTERACTION, + }, + SkillRef { + rel_path: "references/commands-system.md", + body: SKILL_DESKTOP_REF_SYSTEM, + }, + SkillRef { + rel_path: "references/workflows.md", + body: SKILL_DESKTOP_REF_WORKFLOWS, + }, + #[cfg(target_os = "macos")] + SkillRef { + rel_path: "references/macos.md", + body: SKILL_DESKTOP_REF_MACOS, + }, +]; + fn skill_desktop_refs() -> &'static [SkillRef] { - use std::sync::OnceLock; - static REFS: OnceLock> = OnceLock::new(); - REFS.get_or_init(|| { - let mut v = vec![ - SkillRef { - rel_path: "references/commands-observation.md", - body: SKILL_DESKTOP_REF_OBSERVATION, - }, - SkillRef { - rel_path: "references/commands-interaction.md", - body: SKILL_DESKTOP_REF_INTERACTION, - }, - SkillRef { - rel_path: "references/commands-system.md", - body: SKILL_DESKTOP_REF_SYSTEM, - }, - SkillRef { - rel_path: "references/workflows.md", - body: SKILL_DESKTOP_REF_WORKFLOWS, - }, - ]; - #[cfg(target_os = "macos")] - v.push(SkillRef { - rel_path: "references/macos.md", - body: SKILL_DESKTOP_REF_MACOS, - }); - v - }) + SKILL_DESKTOP_REFS } const SKILL_FFI_REFS: &[SkillRef] = &[ @@ -98,7 +93,7 @@ const SKILLS: &[Skill] = &[ Skill { canonical: "agent-desktop", aliases: &["desktop", "agent-desktop"], - summary: "Primary guide. Snapshot/ref loop, JSON envelope, 56 commands including session lifecycle, observation, interaction, keyboard/mouse, app lifecycle, notifications, clipboard, wait.", + summary: "Primary guide. Snapshot/ref loop, JSON envelope, 58 commands including session lifecycle, observation, interaction, keyboard/mouse, app lifecycle, notifications, clipboard, wait.", main: SKILL_DESKTOP_MAIN, refs: skill_desktop_refs, }, diff --git a/crates/core/src/commands/skills_tests.rs b/crates/core/src/commands/skills_tests.rs index 86efef9..0892da5 100644 --- a/crates/core/src/commands/skills_tests.rs +++ b/crates/core/src/commands/skills_tests.rs @@ -31,6 +31,27 @@ fn get_full_inlines_references() { let content = v["content"].as_str().expect("string"); assert!(content.contains("--- references/workflows.md ---")); assert!(content.contains("--- references/macos.md ---")); + assert!(content.contains("@s8f3k2p9:e1")); + assert!(content.contains("session start` does not activate later processes")); + assert!(content.contains("session-owned ref still requires the same `--session`")); + assert!(content.contains("bounded isolated helper")); + assert!(!content.contains("~/.agent-desktop/current_session")); + assert!(!content.contains("resolves cross-session")); + assert!(!content.contains("snapshot IDs do not require also passing `--session`")); +} + +#[test] +fn ffi_skill_defines_ref_and_concurrency_contracts() { + let v = get(GetArgs { + name: "ffi".into(), + full: true, + reference: None, + }) + .expect("get full ffi"); + let content = v["content"].as_str().expect("string"); + assert!(content.contains("## Ref token validation")); + assert!(content.contains("read + mutation")); + assert!(content.contains("## Off-main-thread migration")); } #[test] diff --git a/crates/core/src/commands/snapshot.rs b/crates/core/src/commands/snapshot.rs index d6b021e..69e9e3d 100644 --- a/crates/core/src/commands/snapshot.rs +++ b/crates/core/src/commands/snapshot.rs @@ -1,8 +1,8 @@ use crate::{ + AppError, adapter::{PlatformAdapter, SnapshotSurface}, commands::{wait_selector, wait_selector::WaitSelectorInput}, context::CommandContext, - error::AppError, refs::validate_ref_id, snapshot, snapshot_ref, }; @@ -53,9 +53,7 @@ pub fn execute( args.compact ); - let opts = tree_options(&args); - - if let Some(root) = args.root_ref { + if let Some(root) = args.root_ref.as_deref() { if context.wait_selector().is_some() { return Err(AppError::invalid_input_with_suggestion( "--root cannot be combined with --wait-for or --wait-for-gone", @@ -67,7 +65,14 @@ pub fn execute( "--root cannot be combined with --surface", )); } - validate_ref_id(&root)?; + validate_ref_id(root)?; + } + + validate_surface_support(args.surface, adapter)?; + + let opts = tree_options(&args); + + if let Some(root) = args.root_ref { return format_result(snapshot_ref::run_from_ref_with_context( adapter, &opts, @@ -103,6 +108,33 @@ pub fn execute( format_result(result) } +fn validate_surface_support( + requested: SnapshotSurface, + adapter: &dyn PlatformAdapter, +) -> Result<(), AppError> { + let supported = adapter.supported_surfaces(); + if supported.contains(&requested) { + return Ok(()); + } + let supported = supported + .into_iter() + .map(SnapshotSurface::as_str) + .collect::>(); + Err(crate::AdapterError::new( + crate::ErrorCode::PlatformNotSupported, + format!( + "Snapshot surface '{}' is not supported on this platform", + requested.as_str() + ), + ) + .with_details(json!({ + "requested_surface": requested.as_str(), + "supported_surfaces": supported + })) + .with_suggestion("Choose one of the supported snapshot surfaces") + .into()) +} + fn format_result(result: snapshot::SnapshotResult) -> Result { format_snapshot_fields(&result, None, None) } diff --git a/crates/core/src/commands/snapshot_tests.rs b/crates/core/src/commands/snapshot_tests.rs index 7c9df0d..280288d 100644 --- a/crates/core/src/commands/snapshot_tests.rs +++ b/crates/core/src/commands/snapshot_tests.rs @@ -1,24 +1,101 @@ use super::*; -use crate::adapter::{PlatformAdapter, WindowFilter}; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps, WindowFilter}; use crate::context::{CommandContext, WaitSelector}; -use crate::error::{AdapterError, ErrorCode}; -use crate::node::{AccessibilityNode, WindowInfo}; use crate::refs_test_support::HomeGuard; +use crate::{AccessibilityNode, WindowInfo}; +use crate::{AdapterError, ErrorCode}; struct NoopAdapter; -impl PlatformAdapter for NoopAdapter {} +impl ObservationOps for NoopAdapter {} + +impl ActionOps for NoopAdapter {} + +impl InputOps for NoopAdapter {} + +impl SystemOps for NoopAdapter { + fn supported_surfaces(&self) -> Vec { + vec![SnapshotSurface::Window] + } +} + +struct DefaultSurfaceAdapter; + +impl ObservationOps for DefaultSurfaceAdapter {} +impl ActionOps for DefaultSurfaceAdapter {} +impl InputOps for DefaultSurfaceAdapter {} +impl SystemOps for DefaultSurfaceAdapter {} struct WaitSnapshotAdapter; -impl PlatformAdapter for WaitSnapshotAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result, AdapterError> { +impl ObservationOps for WaitSnapshotAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result { + crate::adapter::observed_tree( + &root, + AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some("Doc".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![ + AccessibilityNode { + ref_id: None, + role: "button".into(), + identity: crate::NodeIdentity { + name: Some("Submit".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }, + AccessibilityNode { + ref_id: None, + role: "button".into(), + identity: crate::NodeIdentity { + name: Some("zero-bounds-button".into()), + ..Default::default() + }, + presentation: crate::NodePresentation { + bounds: Some(crate::Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }), + ..Default::default() + }, + children_count: None, + children: vec![], + }, + ], + }, + ) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { Ok(vec![WindowInfo { id: "w-1".into(), title: "Doc".into(), app: "FixtureApp".into(), - pid: 1, + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), bounds: None, - is_focused: true, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, }]) } @@ -26,28 +103,25 @@ impl PlatformAdapter for WaitSnapshotAdapter { &self, _win: &WindowInfo, _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, ) -> Result { Ok(AccessibilityNode { ref_id: None, role: "window".into(), - name: Some("Doc".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some("Doc".into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children: vec![AccessibilityNode { ref_id: None, role: "button".into(), - name: Some("Submit".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some("Submit".into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children: vec![], }], @@ -55,6 +129,16 @@ impl PlatformAdapter for WaitSnapshotAdapter { } } +impl ActionOps for WaitSnapshotAdapter {} + +impl InputOps for WaitSnapshotAdapter {} + +impl SystemOps for WaitSnapshotAdapter { + fn supported_surfaces(&self) -> Vec { + vec![SnapshotSurface::Window] + } +} + fn base_args() -> SnapshotArgs { SnapshotArgs { app: None, @@ -111,6 +195,28 @@ fn test_tree_options_suppresses_skeleton_for_drill_down() { ); } +#[test] +fn default_snapshot_retains_zero_sized_node_with_ref() { + let args = base_args(); + let result = crate::snapshot::build( + &WaitSnapshotAdapter, + &tree_options(&args), + Some("FixtureApp"), + None, + crate::Deadline::standard().unwrap(), + ) + .unwrap(); + let zero = result + .tree + .children + .iter() + .find(|node| node.identity.name.as_deref() == Some("zero-bounds-button")) + .unwrap(); + + assert!(zero.ref_id.is_some()); + assert_eq!(result.refmap.len(), 2); +} + #[test] fn test_root_with_menu_surface_rejected() { let args = args_with_surface(SnapshotSurface::Menu); @@ -138,14 +244,42 @@ fn test_root_with_window_surface_does_not_short_circuit_validation() { "NoopAdapter cannot satisfy run_from_ref so this must error" ); if let AppError::Adapter(adapter_err) = result.unwrap_err() { - assert_ne!( - adapter_err.code, - ErrorCode::InvalidArgs, - "Window surface must NOT trigger the --surface validation guard" - ); + assert_eq!(adapter_err.code, ErrorCode::InvalidArgs); + assert!(adapter_err.message.contains("explicit snapshot_id")); + assert!(!adapter_err.message.contains("--surface")); } } +#[test] +fn unsupported_surface_fails_before_adapter_traversal() { + let args = SnapshotArgs { + surface: SnapshotSurface::Sheet, + ..base_args() + }; + + let err = execute(args, &DefaultSurfaceAdapter, &CommandContext::default()).unwrap_err(); + let AppError::Adapter(adapter_error) = err else { + panic!("expected adapter error") + }; + + assert_eq!(adapter_error.code, ErrorCode::PlatformNotSupported); + assert_eq!( + adapter_error + .details + .as_ref() + .and_then(|details| details.get("requested_surface")), + Some(&serde_json::json!("sheet")) + ); + assert!( + adapter_error + .details + .as_ref() + .and_then(|details| details.get("supported_surfaces")) + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) + ); +} + #[test] fn test_invalid_root_ref_format_returns_invalid_args() { let args = SnapshotArgs { @@ -167,19 +301,16 @@ fn test_invalid_root_ref_format_returns_invalid_args() { } #[test] -fn test_valid_root_ref_format_does_not_trigger_invalid_args() { +fn test_bare_root_ref_requires_an_explicit_snapshot_id() { let args = SnapshotArgs { root_ref: Some("@e42".into()), ..base_args() }; let err = execute(args, &NoopAdapter, &CommandContext::default()) - .expect_err("NoopAdapter cannot resolve ref"); + .expect_err("bare refs cannot resolve without a snapshot namespace"); if let AppError::Adapter(adapter_err) = err { - assert_ne!( - adapter_err.code, - ErrorCode::InvalidArgs, - "well-formed ref must not trigger INVALID_ARGS" - ); + assert_eq!(adapter_err.code, ErrorCode::InvalidArgs); + assert!(adapter_err.message.contains("explicit snapshot_id")); } } diff --git a/crates/core/src/commands/stale_retry_test_support.rs b/crates/core/src/commands/stale_retry_test_support.rs new file mode 100644 index 0000000..3a9e8fa --- /dev/null +++ b/crates/core/src/commands/stale_retry_test_support.rs @@ -0,0 +1,29 @@ +use crate::{AdapterError, ErrorCode, adapter::NativeHandle}; +use std::sync::atomic::{AtomicU32, Ordering}; + +pub(crate) struct StaleRetryCounter { + calls: AtomicU32, + fail_until: u32, +} + +impl StaleRetryCounter { + pub(crate) fn new(fail_until: u32) -> Self { + Self { + calls: AtomicU32::new(0), + fail_until, + } + } + + pub(crate) fn attempt(&self) -> Result { + let n = self.calls.fetch_add(1, Ordering::SeqCst) + 1; + if n <= self.fail_until { + return Err(AdapterError::new(ErrorCode::StaleRef, "not yet resolvable") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + pub(crate) fn calls(&self) -> u32 { + self.calls.load(Ordering::SeqCst) + } +} diff --git a/crates/core/src/commands/status.rs b/crates/core/src/commands/status.rs index ddb75df..b8d5f99 100644 --- a/crates/core/src/commands/status.rs +++ b/crates/core/src/commands/status.rs @@ -1,11 +1,9 @@ use crate::{ - PermissionReport, + AppError, PermissionReport, adapter::PlatformAdapter, commands::permissions::{self, PermissionsArgs}, context::CommandContext, - error::AppError, refs_store::RefStore, - session::read_current_session_pointer, }; use serde_json::{Value, json}; @@ -17,16 +15,13 @@ pub fn execute_with_report_with_context( let permissions = permissions::execute_with_report(PermissionsArgs { request: false }, adapter, report)?; - let store = RefStore::for_session(context.session_id()).ok(); - let ref_count = store - .as_ref() - .and_then(|s| s.load_latest().ok()) - .map(|m| m.len()); - let snapshot_id = store.and_then(|s| s.latest_snapshot_id()); - let session_id = context - .session_id() - .map(str::to_string) - .or_else(|| read_current_session_pointer().ok().flatten()); + let store = RefStore::for_session(context.session_id())?; + let snapshot_id = store.latest_snapshot_id()?; + let ref_count = match snapshot_id.as_deref() { + Some(snapshot_id) => Some(store.load_snapshot(snapshot_id)?.len()), + None => None, + }; + let session_id = context.session_id().map(str::to_string); let tracing = context.trace_enabled(); let artifacts = session_id .as_deref() @@ -41,6 +36,11 @@ pub fn execute_with_report_with_context( "ref_count": ref_count, "session_id": session_id, "tracing": tracing, + "supported_surfaces": adapter + .supported_surfaces() + .into_iter() + .map(|surface| surface.as_str()) + .collect::>(), }); if let Some(artifacts) = artifacts { body["artifacts"] = json!(artifacts); diff --git a/crates/core/src/commands/status_tests.rs b/crates/core/src/commands/status_tests.rs index d248cb1..1673e5e 100644 --- a/crates/core/src/commands/status_tests.rs +++ b/crates/core/src/commands/status_tests.rs @@ -1,11 +1,21 @@ use super::*; use crate::PermissionState; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; struct DeniedAdapter; -impl PlatformAdapter for DeniedAdapter { - fn permission_report(&self) -> PermissionReport { - PermissionReport { +impl ObservationOps for DeniedAdapter {} + +impl ActionOps for DeniedAdapter {} + +impl InputOps for DeniedAdapter {} + +impl SystemOps for DeniedAdapter { + fn permission_report( + &self, + _deadline: crate::Deadline, + ) -> Result { + Ok(PermissionReport { accessibility: PermissionState::Denied { suggestion: "should not be used".into(), }, @@ -13,12 +23,13 @@ impl PlatformAdapter for DeniedAdapter { suggestion: "should not be used".into(), }, automation: PermissionState::Unknown, - } + }) } } #[test] fn status_uses_precomputed_permission_report() { + let _guard = crate::refs_test_support::HomeGuard::new(); let report = PermissionReport { accessibility: PermissionState::Granted, screen_recording: PermissionState::Granted, @@ -41,7 +52,6 @@ fn status_reports_tracing_false_when_writer_failed() { let session = crate::session::start_session(crate::session::StartSessionOptions { name: None, trace: crate::session::SessionTraceMode::On, - force: true, ..Default::default() }) .unwrap(); diff --git a/crates/core/src/commands/toggle.rs b/crates/core/src/commands/toggle.rs index 86e39ce..d9e7282 100644 --- a/crates/core/src/commands/toggle.rs +++ b/crates/core/src/commands/toggle.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/trace.rs b/crates/core/src/commands/trace.rs index 7d67a03..3df00a3 100644 --- a/crates/core/src/commands/trace.rs +++ b/crates/core/src/commands/trace.rs @@ -1,6 +1,6 @@ use crate::{ + AppError, context::CommandContext, - error::AppError, refs_store::RefStore, session::resolve_active_session, trace_read::{ExportOptions, ReadOptions, export_html, read_merged}, diff --git a/crates/core/src/commands/triple_click.rs b/crates/core/src/commands/triple_click.rs index d082880..b315887 100644 --- a/crates/core/src/commands/triple_click.rs +++ b/crates/core/src/commands/triple_click.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/type_text.rs b/crates/core/src/commands/type_text.rs index 51a9484..0ff474a 100644 --- a/crates/core/src/commands/type_text.rs +++ b/crates/core/src/commands/type_text.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; @@ -13,6 +13,7 @@ pub struct TypeArgs { pub ref_id: String, pub snapshot_id: Option, pub text: String, + pub timeout_ms: Option, } pub fn execute( @@ -31,9 +32,14 @@ pub fn execute( RefArgs { ref_id: args.ref_id, snapshot_id: args.snapshot_id, + timeout_ms: args.timeout_ms, }, adapter, request, context, ) } + +#[cfg(test)] +#[path = "type_text_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/type_text_tests.rs b/crates/core/src/commands/type_text_tests.rs new file mode 100644 index 0000000..2dd8751 --- /dev/null +++ b/crates/core/src/commands/type_text_tests.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::commands::helpers::test_support::save_one_ref_snapshot; +use crate::{ + AdapterError, action_request::ActionRequest, action_result::ActionResult, + adapter::NativeHandle, commands::stale_retry_test_support::StaleRetryCounter, refs::RefEntry, + refs_test_support::HomeGuard, +}; + +struct StaleThenOkAdapter { + retry: StaleRetryCounter, +} + +impl StaleThenOkAdapter { + fn new(fail_until: u32) -> Self { + Self { + retry: StaleRetryCounter::new(fail_until), + } + } +} + +impl ObservationOps for StaleThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result { + self.retry.attempt() + } + + crate::adapter::complete_live_observation!( + "textfield", + "Target", + [crate::capability::TYPE_TEXT] + ); +} + +impl ActionOps for StaleThenOkAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result { + Ok(ActionResult::delivered_unverified("type_text")) + } +} + +impl InputOps for StaleThenOkAdapter {} +impl SystemOps for StaleThenOkAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +fn snapshot_id() -> String { + save_one_ref_snapshot("textfield", "SetValue") +} + +/// Regression for the F2 fix: before it, `type_text::execute` always built its +/// `RefArgs` with `timeout_ms: None`, so a caller-supplied `--timeout-ms` +/// budget was silently dropped and the command never retried a transient +/// `STALE_REF`. This proves the CLI-supplied budget is actually wired through +/// by observing a real retry-then-succeed round trip via `execute()`. +#[test] +fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(2); + + let value = execute( + TypeArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + text: "hi".into(), + timeout_ms: Some(5_000), + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["action"], "type_text"); + assert!(adapter.retry.calls() >= 3); +} + +#[test] +fn timeout_none_makes_exactly_one_resolve_attempt() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = StaleThenOkAdapter::new(1); + + let err = execute( + TypeArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + text: "hi".into(), + timeout_ms: None, + }, + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "STALE_REF"); + assert_eq!(adapter.retry.calls(), 1); +} diff --git a/crates/core/src/commands/uncheck.rs b/crates/core/src/commands/uncheck.rs index ab1d24f..dc68d48 100644 --- a/crates/core/src/commands/uncheck.rs +++ b/crates/core/src/commands/uncheck.rs @@ -1,9 +1,9 @@ use crate::{ + AppError, action::Action, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, - error::AppError, }; use serde_json::Value; diff --git a/crates/core/src/commands/version.rs b/crates/core/src/commands/version.rs index 0f711af..180a3a4 100644 --- a/crates/core/src/commands/version.rs +++ b/crates/core/src/commands/version.rs @@ -1,4 +1,4 @@ -use crate::error::AppError; +use crate::AppError; use serde_json::{Value, json}; pub fn execute() -> Result { diff --git a/crates/core/src/commands/wait.rs b/crates/core/src/commands/wait.rs index 678547a..0cbd45b 100644 --- a/crates/core/src/commands/wait.rs +++ b/crates/core/src/commands/wait.rs @@ -1,14 +1,14 @@ use crate::{ + AppError, ErrorCode, NotificationFilter, NotificationInfo, adapter::{PlatformAdapter, WindowFilter}, commands::{ - helpers::resolve_app_pid, + helpers::resolve_app, wait_element::{ElementWaitInput, wait_for_element}, wait_mode::WaitMode, + wait_surface::SurfaceWait, wait_text_match, wait_timeout, }, context::CommandContext, - error::{AppError, ErrorCode}, - notification::{NotificationFilter, NotificationInfo}, refs_store::RefStore, search_text, snapshot::{self, emit_snapshot_saved}, @@ -28,15 +28,15 @@ pub struct WaitArgs { pub app: Option, } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct WaitModeArgs { pub ms: Option, pub element: Option, pub window: Option, pub text: Option, - pub menu: bool, - pub menu_closed: bool, - pub notification: bool, + pub surface: Option, + pub event: Option, + pub window_id: Option, } #[derive(Clone)] @@ -56,12 +56,16 @@ pub fn execute( let timeout_ms = args.timeout_ms; match WaitMode::from_args(args)? { WaitMode::Sleep(ms) => { - std::thread::sleep(Duration::from_millis(ms)); + let deadline = crate::Deadline::after(ms)?; + std::thread::sleep(deadline.remaining()); + if deadline.was_capped() { + return Err(deadline.timeout_error().into()); + } Ok(json!({ "waited_ms": ms })) } WaitMode::Menu { app, open } => wait_for_menu(app, open, timeout_ms, adapter), WaitMode::Notification { app, text } => { - wait_for_notification(app, text, timeout_ms, adapter) + wait_for_notification(app, text, timeout_ms, adapter, context) } WaitMode::Element { ref_id, @@ -88,6 +92,22 @@ pub fn execute( adapter, context, ), + WaitMode::Event { + event, + app, + window_id, + window_title, + } => crate::commands::wait_event::wait_for_event( + crate::commands::wait_event_input::EventWaitInput { + event, + app, + window_id, + window_title, + timeout_ms, + }, + adapter, + context.event_baseline().cloned(), + ), } } @@ -97,10 +117,15 @@ fn wait_for_menu( timeout_ms: u64, adapter: &dyn PlatformAdapter, ) -> Result { - let pid = resolve_app_pid(app.as_deref(), adapter)?; let start = Instant::now(); + let deadline = crate::Deadline::at(start, timeout_ms)?; + let target = resolve_app(app.as_deref(), adapter, deadline)?; adapter - .wait_for_menu(pid, open, timeout_ms) + .wait_for_menu( + crate::commands::helpers::process_identity(&target)?, + open, + deadline, + ) .map_err(AppError::Adapter)?; let elapsed = start.elapsed().as_millis(); Ok(json!({ "found": true, "elapsed_ms": elapsed })) @@ -112,7 +137,7 @@ fn wait_for_window( adapter: &dyn PlatformAdapter, ) -> Result { let start = Instant::now(); - let timeout = Duration::from_millis(timeout_ms); + let deadline = crate::Deadline::at(start, timeout_ms)?; let filter = WindowFilter { focused_only: false, app: None, @@ -120,7 +145,10 @@ fn wait_for_window( let mut last_error = None; loop { - match adapter.list_windows(&filter) { + if deadline.is_expired() { + return wait_timeout::window(&title, timeout_ms, last_error); + } + match adapter.list_windows(&filter, deadline) { Ok(windows) => { if let Some(win) = windows.into_iter().find(|w| w.title.contains(&title)) { let elapsed = start.elapsed().as_millis(); @@ -136,7 +164,7 @@ fn wait_for_window( Err(err) => return Err(AppError::Adapter(err)), } - let remaining = timeout.saturating_sub(start.elapsed()); + let remaining = deadline.remaining(); if remaining.is_zero() { return wait_timeout::window(&title, timeout_ms, last_error); } @@ -164,15 +192,18 @@ fn wait_for_text( timeout_ms, } = input; let start = Instant::now(); - let timeout = Duration::from_millis(timeout_ms); + let deadline = crate::Deadline::at(start, timeout_ms)?; let opts = crate::adapter::TreeOptions::default(); let normalized_text = search_text::normalize(&text); let mut interval = Duration::from_millis(200); let mut last_error = None; loop { - match snapshot::build(adapter, &opts, app.as_deref(), None) { - Ok(result) => { + if deadline.is_expired() { + return wait_timeout::text(&text, timeout_ms, expected_count, last_error); + } + match snapshot::build(adapter, &opts, app.as_deref(), None, deadline) { + Ok(mut result) => { let matches = wait_text_match::find(&result.tree, &normalized_text, expected_count); let matched = expected_count .map(|expected| matches.len() == expected) @@ -186,13 +217,16 @@ fn wait_for_text( &snapshot_id, &result.refmap, )?; + result.bind_snapshot_id(snapshot_id.clone()); emit_snapshot_saved(context, &result)?; let elapsed = start.elapsed().as_millis(); let found = matches.first(); let mut body = json!({ "found": true, "text": text, - "ref": found.and_then(|found| found.ref_id.clone()), + "ref": found + .and_then(|found| found.ref_id.as_deref()) + .map(|ref_id| crate::ref_token::qualify_ref_id(&snapshot_id, ref_id)), "role": found.map(|found| found.role.clone()), "snapshot_id": snapshot_id, "elapsed_ms": elapsed @@ -212,7 +246,7 @@ fn wait_for_text( Err(err) => return Err(err), } - let remaining = timeout.saturating_sub(start.elapsed()); + let remaining = deadline.remaining(); if remaining.is_zero() { return wait_timeout::text(&text, timeout_ms, expected_count, last_error); } @@ -235,6 +269,7 @@ fn wait_for_notification( text: Option, timeout_ms: u64, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { let filter = NotificationFilter { app: app.clone(), @@ -242,12 +277,17 @@ fn wait_for_notification( ..Default::default() }; let start = Instant::now(); - let timeout = Duration::from_millis(timeout_ms); + let deadline = crate::Deadline::at(start, timeout_ms)?; let mut baseline: Option> = None; let mut last_error = None; loop { - match adapter.list_notifications(&filter) { + if deadline.is_expired() { + return wait_timeout::notification(app.as_ref(), text.as_ref(), timeout_ms, last_error); + } + match super::notification_policy::list_with_foreground_lease( + &filter, deadline, adapter, context, + ) { Ok(current) => match &baseline { None => { baseline = Some(notification_counts(¤t)); @@ -264,16 +304,16 @@ fn wait_for_notification( } } }, - Err(err) if is_retryable_wait_poll_error(&err.code) => { + Err(AppError::Adapter(err)) if is_retryable_wait_poll_error(&err.code) => { last_error = Some(json!({ "code": err.code.as_str(), "message": err.message })); } - Err(err) => return Err(AppError::Adapter(err)), + Err(err) => return Err(err), } - let remaining = timeout.saturating_sub(start.elapsed()); + let remaining = deadline.remaining(); if remaining.is_zero() { return wait_timeout::notification(app.as_ref(), text.as_ref(), timeout_ms, last_error); } @@ -336,6 +376,10 @@ mod test_support; #[path = "wait_tests.rs"] mod tests; +#[cfg(test)] +#[path = "wait_scenario_tests.rs"] +mod scenario_tests; + #[cfg(test)] #[path = "wait_element_tests.rs"] mod element_tests; diff --git a/crates/core/src/commands/wait_element.rs b/crates/core/src/commands/wait_element.rs index e13d627..c4d1d63 100644 --- a/crates/core/src/commands/wait_element.rs +++ b/crates/core/src/commands/wait_element.rs @@ -1,17 +1,14 @@ use crate::{ + AppError, adapter::PlatformAdapter, - commands::{wait_latest_ref_cache::LatestRefCache, wait_predicate, wait_timeout}, + commands::{wait_predicate, wait_timeout}, context::CommandContext, - error::{AppError, ErrorCode}, + ref_resolve_deadline::{POLL_INTERVAL, resolve_within_deadline}, refs_store::RefStore, + resolve_attempt_outcome::ResolveAttemptOutcome, }; use serde_json::{Value, json}; -use std::time::{Duration, Instant}; - -/// Per-attempt cap on ref resolution inside a wait loop, so a slow resolve -/// cannot consume the whole wait budget on the first poll; the predicate is -/// re-checked every attempt across the full timeout. -const WAIT_RESOLVE_ATTEMPT: Duration = Duration::from_millis(750); +use std::time::Instant; pub(crate) struct ElementWaitInput { pub(crate) ref_id: String, @@ -32,87 +29,80 @@ pub(crate) fn wait_for_element( timeout_ms, } = input; let start = Instant::now(); - let timeout = Duration::from_millis(timeout_ms); + let deadline = crate::Deadline::at(start, timeout_ms)?; + let (resolved_snapshot_id, local_ref) = + crate::ref_token::resolve_ref_target(&ref_id, snapshot_id.as_deref())?; let store = RefStore::for_session(context.session_id())?; - let fixed_refmap = match snapshot_id.as_deref() { - Some(id) => Some(store.load_snapshot(id)?), - None => None, - }; - let mut latest_cache = if fixed_refmap.is_none() { - Some(LatestRefCache::new(&store)?) - } else { - None - }; - - if fixed_refmap - .as_ref() - .is_some_and(|refmap| refmap.get(&ref_id).is_none()) - { - return Err(AppError::invalid_input_with_suggestion( + let refmap = store.load_snapshot(&resolved_snapshot_id)?; + let entry = refmap.get(&local_ref).cloned().ok_or_else(|| { + AppError::invalid_input_with_suggestion( format!("Ref {ref_id} is not present in the requested snapshot"), - "Use a ref returned by that snapshot_id, or omit --snapshot to wait against the latest refmap.", - )); - } + "Use a snapshot-qualified ref returned by that snapshot, or pair a legacy @eN ref with its snapshot_id.", + ) + })?; let mut last_observed = json!(null); + let mut expected_bounds_hash = None; loop { - let entry = fixed_refmap - .as_ref() - .and_then(|r| r.get(&ref_id).cloned()) - .or_else(|| latest_cache.as_ref().and_then(|c| c.entry(&ref_id))); - if let Some(entry) = entry { - let remaining = timeout.saturating_sub(start.elapsed()); - if remaining.is_zero() { + match resolve_within_deadline(adapter, &entry, deadline) { + ResolveAttemptOutcome::DeadlinePassed => { return wait_timeout::element(ref_id, predicate, timeout_ms, last_observed); } - let attempt = remaining.min(WAIT_RESOLVE_ATTEMPT); - match adapter.resolve_element_strict_with_timeout(&entry, attempt) { - Ok(handle) => { - let observed = wait_predicate::observe(&entry, &handle, &predicate, adapter); - let _ = adapter.release_handle(&handle); - last_observed = observed.map_err(AppError::Adapter)?; - if wait_predicate::satisfied(&predicate, &last_observed) { - let elapsed = start.elapsed().as_millis(); - return Ok(json!({ - "found": true, - "ref": ref_id, - "predicate": predicate.name(), - "observed": last_observed, - "elapsed_ms": elapsed - })); - } - } - Err(err) if is_retryable_wait_resolution_error(&err.code) => { - last_observed = json!({ - "error": err.code.as_str(), - "message": err.message - }); - if fixed_refmap.is_none() { - if let Some(cache) = latest_cache.as_mut() { - cache.refresh_if_due()?; + ResolveAttemptOutcome::Resolved(handle) => { + match wait_predicate::observe( + &entry, + &handle, + &predicate, + adapter, + deadline, + crate::actionability::StabilityExpectation::strict_hash(expected_bounds_hash), + ) { + Ok(observed) => { + last_observed = observed; + if let Some(observed) = last_observed + .get("observed_bounds_hash") + .and_then(Value::as_u64) + { + expected_bounds_hash = Some(observed); + } + if wait_predicate::satisfied(&predicate, &last_observed) { + let elapsed = start.elapsed().as_millis(); + return Ok(json!({ + "found": true, + "ref": ref_id, + "predicate": predicate.name(), + "observed": last_observed, + "elapsed_ms": elapsed + })); } } + Err(err) if is_retryable_wait_error(&err) => { + last_observed = json!({ + "error": err.code.as_str(), + "message": err.message, + "details": err.details + }); + } + Err(err) => return Err(AppError::Adapter(err)), } - Err(err) => return Err(AppError::Adapter(err)), } - } else if let Some(cache) = latest_cache.as_mut() { - cache.refresh_if_due()?; + ResolveAttemptOutcome::Failed(err) if is_retryable_wait_error(&err) => { + last_observed = json!({ + "error": err.code.as_str(), + "message": err.message + }); + } + ResolveAttemptOutcome::Failed(err) => return Err(AppError::Adapter(err)), } - let remaining = timeout.saturating_sub(start.elapsed()); + let remaining = deadline.remaining(); if remaining.is_zero() { return wait_timeout::element(ref_id, predicate, timeout_ms, last_observed); } - std::thread::sleep(remaining.min(Duration::from_millis(100))); + std::thread::sleep(remaining.min(POLL_INTERVAL)); } } -fn is_retryable_wait_resolution_error(code: &ErrorCode) -> bool { - matches!( - code, - ErrorCode::StaleRef - | ErrorCode::ElementNotFound - | ErrorCode::AmbiguousTarget - | ErrorCode::Timeout - ) +fn is_retryable_wait_error(error: &crate::AdapterError) -> bool { + error.is_explicitly_retryable() } diff --git a/crates/core/src/commands/wait_element_tests.rs b/crates/core/src/commands/wait_element_tests.rs index bb3a4f6..c42dbe5 100644 --- a/crates/core/src/commands/wait_element_tests.rs +++ b/crates/core/src/commands/wait_element_tests.rs @@ -2,43 +2,62 @@ use super::test_support::{ PredicateAdapter, save_ref_in_session, snapshot_with_one_ref, wait_for_element_test, }; use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - adapter::{NativeHandle, PlatformAdapter}, - commands::wait_predicate, - element_state::ElementState, - error::AdapterError, - refs::RefEntry, - refs_test_support::HomeGuard, + AdapterError, adapter::NativeHandle, commands::wait_predicate, element_state::ElementState, + refs::RefEntry, refs_test_support::HomeGuard, +}; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, }; -use std::sync::atomic::{AtomicU32, Ordering}; struct NoopAdapter; -impl PlatformAdapter for NoopAdapter {} +impl ObservationOps for NoopAdapter {} + +impl ActionOps for NoopAdapter {} + +impl InputOps for NoopAdapter {} + +impl SystemOps for NoopAdapter {} struct LiveErrorPredicateAdapter { - releases: AtomicU32, + drops: Arc, } -impl PlatformAdapter for LiveErrorPredicateAdapter { - fn resolve_element_strict_with_timeout( +struct DropProbe(Arc); + +impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +impl ObservationOps for LiveErrorPredicateAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: std::time::Duration, + _deadline: crate::Deadline, ) -> Result { - Ok(NativeHandle::null()) + Ok(NativeHandle::new(DropProbe(Arc::clone(&self.drops)))) } - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result, AdapterError> { Err(AdapterError::permission_denied()) } - - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - self.releases.fetch_add(1, Ordering::SeqCst); - Ok(()) - } } +impl ActionOps for LiveErrorPredicateAdapter {} + +impl InputOps for LiveErrorPredicateAdapter {} + +impl SystemOps for LiveErrorPredicateAdapter {} + #[test] fn snapshot_pinned_missing_ref_is_invalid_args() { let _guard = HomeGuard::new(); @@ -59,7 +78,7 @@ fn snapshot_pinned_missing_ref_is_invalid_args() { } #[test] -fn element_wait_explicit_session_snapshot_without_session_context() { +fn element_wait_explicit_session_snapshot_with_matching_session_context() { let _guard = HomeGuard::new(); let snapshot_id = save_ref_in_session("agent-a", Vec::new()); let adapter = PredicateAdapter { @@ -67,6 +86,9 @@ fn element_wait_explicit_session_snapshot_without_session_context() { role: "button".into(), states: vec![], value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), }), value: None, bounds: None, @@ -78,7 +100,7 @@ fn element_wait_explicit_session_snapshot_without_session_context() { wait_predicate::ElementPredicate::Exists, 50, &adapter, - &crate::context::CommandContext::default(), + &crate::context::CommandContext::new(Some("agent-a".into()), None, false).unwrap(), ) .unwrap(); @@ -87,11 +109,11 @@ fn element_wait_explicit_session_snapshot_without_session_context() { } #[test] -fn element_wait_propagates_live_read_errors_after_releasing_handle() { +fn element_wait_propagates_live_read_errors_after_dropping_handle() { let _guard = HomeGuard::new(); let snapshot_id = snapshot_with_one_ref(); let adapter = LiveErrorPredicateAdapter { - releases: AtomicU32::new(0), + drops: Arc::new(AtomicU32::new(0)), }; let err = wait_for_element_test( @@ -105,7 +127,7 @@ fn element_wait_propagates_live_read_errors_after_releasing_handle() { .unwrap_err(); assert_eq!(err.code(), "PERM_DENIED"); - assert_eq!(adapter.releases.load(Ordering::SeqCst), 1); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); } #[test] @@ -126,7 +148,7 @@ fn zero_timeout_returns_timeout_before_any_resolution_attempt() { let AppError::Adapter(adapter_err) = err else { panic!("expected adapter error"); }; - assert_eq!(adapter_err.code, crate::error::ErrorCode::Timeout); + assert_eq!(adapter_err.code, crate::ErrorCode::Timeout); let details = adapter_err.details.expect("timeout should carry details"); assert!(details["last_observed"].is_null()); } diff --git a/crates/core/src/commands/wait_event.rs b/crates/core/src/commands/wait_event.rs new file mode 100644 index 0000000..159c772 --- /dev/null +++ b/crates/core/src/commands/wait_event.rs @@ -0,0 +1,278 @@ +use crate::{ + AdapterError, AppError, ErrorCode, EventKind, SignalBaseline, SignalFilter, UiEvent, + adapter::PlatformAdapter, commands::wait_event_input::EventWaitInput, diff_signals, +}; +use serde_json::{Value, json}; +use std::time::{Duration, Instant}; + +/// Baseline-diff event wait: captures a `SignalBaseline` at wait start, polls +/// for a fresh one, and matches the first `diff_signals` event whose kind +/// matches the requested `--event` token. `window_id`/`window_title` are +/// optional narrowing filters, never requirements — the whole point of R16 +/// is that a caller who does not yet know a new window's id or title can +/// still wait for it to appear. +pub(crate) fn wait_for_event( + input: EventWaitInput, + adapter: &dyn PlatformAdapter, + seeded_baseline: Option>, +) -> Result { + let requested = parse_event_kind(&input.event)?; + let start = Instant::now(); + let deadline = crate::Deadline::at(start, input.timeout_ms)?; + let filter = signal_filter( + &input, + &requested, + seeded_baseline.as_ref(), + adapter, + deadline, + )?; + let (mut baseline, mut last_error) = match seeded_baseline { + Some(Ok(baseline)) => { + validate_signal_scope(&filter, &baseline)?; + (Some(baseline), None) + } + Some(Err(error)) if is_retryable(&error.code) => (None, Some(error_evidence(&error))), + Some(Err(error)) => return Err(AppError::Adapter(error)), + None => (None, None), + }; + + loop { + if deadline.is_expired() { + return timeout_err( + &input.event, + input.app.as_ref(), + input.timeout_ms, + baseline.as_ref(), + last_error, + ); + } + + let observation = adapter.capture_signal_baseline(&filter, deadline); + if deadline.is_expired() { + return timeout_err( + &input.event, + input.app.as_ref(), + input.timeout_ms, + baseline.as_ref(), + last_error, + ); + } + + match observation { + Ok(current) => { + validate_signal_scope(&filter, ¤t)?; + match &baseline { + None => baseline = Some(current), + Some(base) => { + let events = diff_signals(base, ¤t); + if let Some(found) = find_match( + &events, + &requested, + input.window_id.as_deref(), + input.window_title.as_deref(), + ) { + let elapsed = start.elapsed().as_millis(); + return Ok(json!({ + "found": true, + "event": serde_json::to_value(found)?, + "elapsed_ms": elapsed, + })); + } + } + } + } + Err(err) if is_retryable(&err.code) => { + last_error = Some(error_evidence(&err)); + } + Err(err) => return Err(AppError::Adapter(err)), + } + + let remaining = deadline.remaining(); + if remaining.is_zero() { + return timeout_err( + &input.event, + input.app.as_ref(), + input.timeout_ms, + baseline.as_ref(), + last_error, + ); + } + std::thread::sleep(remaining.min(Duration::from_millis(200))); + } +} + +fn signal_filter( + input: &EventWaitInput, + requested: &EventKind, + seeded_baseline: Option<&Result>, + adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, +) -> Result { + let Some(app) = input.app.as_deref() else { + return Ok(SignalFilter::default()); + }; + if matches!(requested, EventKind::AppLaunched) { + return Ok(SignalFilter { + app: input.app.clone(), + process: None, + }); + } + let successful_seed = seeded_baseline.and_then(|baseline| baseline.as_ref().ok()); + let seeded_process = successful_seed + .map(|baseline| process_from_baseline(baseline, app)) + .transpose()? + .flatten(); + let process = match seeded_process { + Some(process) => Some(process), + None if successful_seed.is_some() && matches!(requested, EventKind::AppTerminated) => None, + None => Some(crate::commands::helpers::process_identity( + &crate::commands::helpers::resolve_app(Some(app), adapter, deadline)?, + )?), + }; + Ok(SignalFilter { + app: input.app.clone(), + process, + }) +} + +fn process_from_baseline( + baseline: &SignalBaseline, + app: &str, +) -> Result, AppError> { + let mut matches = baseline + .apps + .iter() + .filter(|candidate| candidate.name.eq_ignore_ascii_case(app)); + let Some(first) = matches.next() else { + return Ok(None); + }; + if matches.next().is_some() { + return Err(AdapterError::ambiguous_target(format!( + "Multiple application instances matched '{app}' in the event baseline" + )) + .into()); + } + crate::commands::helpers::process_identity(first).map(Some) +} + +fn validate_signal_scope(filter: &SignalFilter, baseline: &SignalBaseline) -> Result<(), AppError> { + let Some(expected) = filter.process.as_ref() else { + return Ok(()); + }; + let observed = baseline + .apps + .iter() + .map(|app| (app.pid, app.process_instance.as_deref())) + .chain( + baseline + .windows + .iter() + .map(|window| (window.pid, window.process_instance.as_deref())), + ) + .chain( + baseline + .surfaces + .iter() + .map(|surface| (surface.pid, Some(surface.process_instance.as_str()))), + ); + if let Some((pid, instance)) = observed.into_iter().find(|(pid, instance)| { + *pid != expected.pid || *instance != Some(expected.instance.as_str()) + }) { + return Err(AdapterError::new( + ErrorCode::StaleRef, + "Target process identity changed during event observation", + ) + .with_details(json!({ + "kind": "process_changed", + "expected_pid": expected.pid, + "observed_pid": pid, + "observed_instance_matches": instance == Some(expected.instance.as_str()), + "retryable": false, + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) + .into()); + } + Ok(()) +} + +fn find_match<'a>( + events: &'a [UiEvent], + requested: &EventKind, + window_id: Option<&str>, + window_title: Option<&str>, +) -> Option<&'a UiEvent> { + events.iter().find(|event| { + if !requested.same_variant(&event.kind) { + return false; + } + if let Some(id) = window_id { + if event.window_id.as_deref() != Some(id) { + return false; + } + } + if let Some(title) = window_title { + if !title.is_empty() && event.title.as_deref() != Some(title) { + return false; + } + } + true + }) +} + +fn is_retryable(code: &ErrorCode) -> bool { + matches!( + code, + ErrorCode::Timeout | ErrorCode::ElementNotFound | ErrorCode::AppUnresponsive + ) +} + +fn error_evidence(error: &AdapterError) -> Value { + json!({ + "code": error.code.as_str(), + "message": error.message, + "details": error.details, + }) +} + +pub(crate) fn parse_event_kind(event: &str) -> Result { + EventKind::parse(event).ok_or_else(|| { + AppError::invalid_input_with_suggestion( + format!("Unknown --event value: {event}"), + format!("Use one of: {}", EventKind::all_tokens().join(", ")), + ) + }) +} + +fn timeout_err( + event: &str, + app: Option<&String>, + timeout_ms: u64, + baseline: Option<&SignalBaseline>, + last_error: Option, +) -> Result { + let mut details = json!({ + "kind": "wait_timeout", + "predicate": "event", + "event": event, + "app": app, + "timeout_ms": timeout_ms, + "baseline_counts": baseline.map(|base| json!({ + "windows": base.windows.len(), + "apps": base.apps.len(), + "surfaces": base.surfaces.len(), + })), + }); + if let Some(err) = last_error { + details["last_error"] = err; + } + Err(AppError::Adapter( + AdapterError::timeout(format!( + "Event '{event}' did not occur within {timeout_ms}ms" + )) + .with_details(details), + )) +} + +#[cfg(test)] +#[path = "wait_event_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/wait_event_deadline_tests.rs b/crates/core/src/commands/wait_event_deadline_tests.rs new file mode 100644 index 0000000..c79c855 --- /dev/null +++ b/crates/core/src/commands/wait_event_deadline_tests.rs @@ -0,0 +1,49 @@ +use super::*; + +#[test] +fn late_matching_observation_is_discarded() { + let current = baseline_with_windows(vec![window("w-late", "Too late")]); + let adapter = SequenceAdapter::new(vec![current]).with_delay(Duration::from_millis(30)); + let mut request = input("window-opened", None); + request.timeout_ms = 10; + + let error = wait_for_event(request, &adapter, Some(Ok(empty_baseline()))) + .expect_err("an observation returned after the deadline must not match"); + + assert_eq!(error.code(), "TIMEOUT"); + assert_eq!(*adapter.calls.lock().unwrap(), 1); + let remaining = adapter.remaining_at_call.lock().unwrap()[0]; + assert!(remaining > Duration::ZERO); + assert!(remaining <= Duration::from_millis(10)); +} + +#[test] +fn zero_timeout_does_not_start_an_observation() { + let adapter = SequenceAdapter::new(vec![empty_baseline()]); + let mut request = input("window-opened", None); + request.timeout_ms = 0; + + let error = wait_for_event(request, &adapter, None) + .expect_err("a zero timeout must expire before baseline capture"); + + assert_eq!(error.code(), "TIMEOUT"); + assert_eq!(*adapter.calls.lock().unwrap(), 0); +} + +#[test] +fn oversized_timeout_cannot_overflow_the_deadline() { + let start = Instant::now(); + + let error = crate::Deadline::at(start, u64::MAX).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +#[test] +fn parse_event_kind_accepts_every_documented_token() { + for token in EventKind::all_tokens() { + assert!( + parse_event_kind(token).is_ok(), + "token '{token}' from EventKind::all_tokens() must parse" + ); + } +} diff --git a/crates/core/src/commands/wait_event_input.rs b/crates/core/src/commands/wait_event_input.rs new file mode 100644 index 0000000..0d540b8 --- /dev/null +++ b/crates/core/src/commands/wait_event_input.rs @@ -0,0 +1,7 @@ +pub(crate) struct EventWaitInput { + pub event: String, + pub app: Option, + pub window_id: Option, + pub window_title: Option, + pub timeout_ms: u64, +} diff --git a/crates/core/src/commands/wait_event_lifecycle_tests.rs b/crates/core/src/commands/wait_event_lifecycle_tests.rs new file mode 100644 index 0000000..48aab5f --- /dev/null +++ b/crates/core/src/commands/wait_event_lifecycle_tests.rs @@ -0,0 +1,76 @@ +use super::*; + +#[test] +fn named_app_launch_wait_observes_absent_to_present_without_resolution() { + let adapter = SequenceAdapter::new(vec![ + empty_baseline(), + baseline_with_apps(vec![app("TextEdit", "launched-generation")]), + ]) + .with_apps(Vec::new()); + + let result = wait_for_event(input("app-launched", Some("TextEdit")), &adapter, None).unwrap(); + + assert_eq!(result["event"]["kind"], "app_launched"); + assert_eq!(*adapter.app_calls.lock().unwrap(), 0); +} + +#[test] +fn seeded_app_launch_wait_remains_name_scoped_across_generations() { + let existing = app("TextEdit", "existing-generation"); + let launched = app("TextEdit", "launched-generation"); + let adapter = SequenceAdapter::new(vec![baseline_with_apps(vec![existing.clone(), launched])]); + let seeded = baseline_with_apps(vec![existing]); + + let result = wait_for_event( + input("app-launched", Some("TextEdit")), + &adapter, + Some(Ok(seeded)), + ) + .unwrap(); + + assert_eq!(result["event"]["kind"], "app_launched"); + assert_eq!(*adapter.app_calls.lock().unwrap(), 0); +} + +#[test] +fn seeded_app_termination_uses_pre_action_identity_after_process_exit() { + let adapter = SequenceAdapter::new(vec![empty_baseline()]).with_apps(Vec::new()); + let seeded = baseline_with_apps(vec![app("TextEdit", "terminated-generation")]); + + let result = wait_for_event( + input("app-terminated", Some("TextEdit")), + &adapter, + Some(Ok(seeded)), + ) + .unwrap(); + + assert_eq!(result["event"]["kind"], "app_terminated"); + assert_eq!(*adapter.app_calls.lock().unwrap(), 0); +} + +#[test] +fn seeded_process_generation_change_is_rejected_during_polling() { + let adapter = SequenceAdapter::new(vec![baseline_with_apps(vec![app( + "TextEdit", + "new-generation", + )])]); + let seeded = baseline_with_apps(vec![app("TextEdit", "old-generation")]); + + let error = wait_for_event( + input("app-terminated", Some("TextEdit")), + &adapter, + Some(Ok(seeded)), + ) + .expect_err("a reused process identity must terminate the seeded wait"); + + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!(error.code, ErrorCode::StaleRef); + assert_eq!( + error.disposition.delivery(), + crate::DeliveryDisposition::NotDelivered + ); + assert_eq!(error.details.unwrap()["kind"], "process_changed"); + assert_eq!(*adapter.calls.lock().unwrap(), 1); +} diff --git a/crates/core/src/commands/wait_event_tests.rs b/crates/core/src/commands/wait_event_tests.rs new file mode 100644 index 0000000..f42a746 --- /dev/null +++ b/crates/core/src/commands/wait_event_tests.rs @@ -0,0 +1,349 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{AppInfo, WindowInfo}; + +struct SequenceAdapter { + snapshots: Vec, + apps: Vec, + app_calls: std::sync::Mutex, + calls: std::sync::Mutex, + delay: std::time::Duration, + remaining_at_call: std::sync::Mutex>, +} + +impl SequenceAdapter { + fn new(snapshots: Vec) -> Self { + Self { + snapshots, + apps: vec![app("TextEdit", "test-instance")], + app_calls: std::sync::Mutex::new(0), + calls: std::sync::Mutex::new(0), + delay: std::time::Duration::ZERO, + remaining_at_call: std::sync::Mutex::new(Vec::new()), + } + } + + fn with_delay(mut self, delay: std::time::Duration) -> Self { + self.delay = delay; + self + } + + fn with_apps(mut self, apps: Vec) -> Self { + self.apps = apps; + self + } +} + +impl ObservationOps for SequenceAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + *self.app_calls.lock().unwrap() += 1; + Ok(self.apps.clone()) + } +} +impl ActionOps for SequenceAdapter {} +impl InputOps for SequenceAdapter {} + +impl SystemOps for SequenceAdapter { + fn capture_signal_baseline( + &self, + _filter: &SignalFilter, + deadline: crate::Deadline, + ) -> Result { + self.remaining_at_call + .lock() + .unwrap() + .push(deadline.remaining()); + std::thread::sleep(self.delay); + let mut calls = self.calls.lock().unwrap(); + let idx = (*calls).min(self.snapshots.len().saturating_sub(1)); + *calls += 1; + Ok(self.snapshots[idx].clone()) + } +} + +struct FailingInventoryAdapter { + calls: std::sync::Mutex, +} + +impl ObservationOps for FailingInventoryAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result, AdapterError> { + Ok(vec![app("TextEdit", "test-instance")]) + } +} +impl ActionOps for FailingInventoryAdapter {} +impl InputOps for FailingInventoryAdapter {} + +impl SystemOps for FailingInventoryAdapter { + fn capture_signal_baseline( + &self, + _filter: &SignalFilter, + _deadline: crate::Deadline, + ) -> Result { + *self.calls.lock().unwrap() += 1; + Err( + AdapterError::app_unresponsive("macOS inventory").with_details(json!({ + "kind": "inventory_sources", + })), + ) + } +} + +fn empty_baseline() -> SignalBaseline { + SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + } +} + +fn app(name: &str, instance: &str) -> AppInfo { + AppInfo { + name: name.into(), + pid: crate::ProcessId::new(42), + bundle_id: Some("com.example.editor".into()), + process_instance: Some(instance.into()), + } +} + +fn baseline_with_windows(windows: Vec) -> SignalBaseline { + SignalBaseline { + windows, + apps: Vec::new(), + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + } +} + +fn baseline_with_apps(apps: Vec) -> SignalBaseline { + SignalBaseline { + windows: Vec::new(), + apps, + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + } +} + +fn window(id: &str, title: &str) -> WindowInfo { + WindowInfo { + id: id.into(), + title: title.into(), + app: "TextEdit".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + } +} + +fn input(event: &str, app: Option<&str>) -> EventWaitInput { + EventWaitInput { + event: event.into(), + app: app.map(str::to_string), + window_id: None, + window_title: None, + timeout_ms: 5_000, + } +} + +#[test] +fn window_opened_matches_without_caller_supplied_id_or_title() { + let adapter = SequenceAdapter::new(vec![ + empty_baseline(), + empty_baseline(), + baseline_with_windows(vec![window("w-99", "Untitled")]), + ]); + + let result = wait_for_event(input("window-opened", Some("TextEdit")), &adapter, None).unwrap(); + + assert_eq!(result["found"], true); + assert_eq!(result["event"]["window_id"], "w-99"); + assert_eq!(result["event"]["title"], "Untitled"); +} + +#[test] +fn window_id_narrows_match_to_the_named_window() { + let adapter = SequenceAdapter::new(vec![ + empty_baseline(), + baseline_with_windows(vec![window("w-1", "Untitled"), window("w-2", "Untitled")]), + ]); + + let mut request = input("window-opened", None); + request.window_id = Some("w-2".into()); + let result = wait_for_event(request, &adapter, None).unwrap(); + + assert_eq!(result["event"]["window_id"], "w-2"); +} + +#[test] +fn timeout_carries_wait_timeout_kind_in_details() { + let adapter = SequenceAdapter::new(vec![empty_baseline()]); + + let mut request = input("window-opened", None); + request.timeout_ms = 50; + let err = wait_for_event(request, &adapter, None).unwrap_err(); + + let AppError::Adapter(adapter_err) = err else { + panic!("expected AppError::Adapter"); + }; + assert_eq!(adapter_err.code, ErrorCode::Timeout); + let details = adapter_err.details.expect("timeout must carry details"); + assert_eq!(details["kind"], "wait_timeout"); + assert_eq!(details["predicate"], "event"); + assert_eq!(details["event"], "window-opened"); +} + +#[test] +fn unknown_event_value_returns_invalid_args() { + let adapter = SequenceAdapter::new(vec![empty_baseline()]); + + let mut request = input("bogus-event", None); + request.timeout_ms = 10; + let err = wait_for_event(request, &adapter, None).unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); +} + +#[test] +fn seeded_pre_action_baseline_observes_synchronous_event() { + let current = baseline_with_windows(vec![window("w-99", "Opened during action")]); + let adapter = SequenceAdapter::new(vec![current]); + + let result = wait_for_event( + input("window-opened", Some("TextEdit")), + &adapter, + Some(Ok(empty_baseline())), + ) + .unwrap(); + + assert_eq!(result["event"]["window_id"], "w-99"); + assert_eq!(*adapter.calls.lock().unwrap(), 1); +} + +#[test] +fn process_generation_change_during_poll_is_terminal() { + let current = baseline_with_windows(vec![WindowInfo { + process_instance: Some("new-generation".into()), + ..window("w-new", "Reused pid") + }]); + let adapter = SequenceAdapter::new(vec![current]); + + let error = wait_for_event( + input("window-opened", Some("TextEdit")), + &adapter, + Some(Ok(empty_baseline())), + ) + .expect_err("a reused pid must terminate the old-generation wait"); + + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!(error.code, ErrorCode::StaleRef); + assert_eq!( + error.disposition.delivery(), + crate::DeliveryDisposition::NotDelivered + ); + assert_eq!(error.details.unwrap()["retryable"], false); + assert_eq!(*adapter.calls.lock().unwrap(), 1); +} + +#[test] +fn seeded_baseline_failure_is_not_disguised_as_timeout() { + let adapter = SequenceAdapter::new(vec![empty_baseline()]); + let seed_error = AdapterError::new(ErrorCode::ActionFailed, "baseline failed"); + + let err = wait_for_event( + input("window-opened", None), + &adapter, + Some(Err(seed_error)), + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_FAILED"); + assert_eq!(*adapter.calls.lock().unwrap(), 0); +} + +#[test] +fn retryable_seed_inventory_failure_is_preserved_in_timeout_evidence() { + let adapter = SequenceAdapter::new(vec![empty_baseline()]); + let seed_error = AdapterError::app_unresponsive("macOS inventory").with_details(json!({ + "kind": "inventory_sources", + })); + let mut request = input("window-opened", None); + request.timeout_ms = 20; + + let err = wait_for_event(request, &adapter, Some(Err(seed_error))).unwrap_err(); + + let AppError::Adapter(adapter_err) = err else { + panic!("expected AppError::Adapter"); + }; + let details = adapter_err.details.expect("timeout must carry evidence"); + assert_eq!(details["last_error"]["code"], "APP_UNRESPONSIVE"); + assert_eq!( + details["last_error"]["details"]["kind"], + "inventory_sources" + ); + assert!(*adapter.calls.lock().unwrap() > 0); +} + +#[test] +fn failed_inventory_poll_never_diffs_against_an_empty_baseline() { + let adapter = FailingInventoryAdapter { + calls: std::sync::Mutex::new(0), + }; + let mut request = input("window-closed", None); + request.timeout_ms = 20; + + let err = wait_for_event( + request, + &adapter, + Some(Ok(baseline_with_windows(vec![window( + "w-still-open", + "Still open", + )]))), + ) + .unwrap_err(); + + let AppError::Adapter(adapter_err) = err else { + panic!("expected AppError::Adapter"); + }; + assert_eq!(adapter_err.code, ErrorCode::Timeout); + let details = adapter_err.details.unwrap(); + assert_eq!(details["last_error"]["code"], "APP_UNRESPONSIVE"); + assert_eq!(details["baseline_counts"]["windows"], 1); + assert!(*adapter.calls.lock().unwrap() > 0); +} + +#[test] +fn failed_inventory_poll_never_reports_app_termination() { + let adapter = FailingInventoryAdapter { + calls: std::sync::Mutex::new(0), + }; + let mut request = input("app-terminated", Some("TextEdit")); + request.timeout_ms = 20; + let baseline = baseline_with_apps(vec![AppInfo { + name: "TextEdit".into(), + pid: crate::ProcessId::new(42), + bundle_id: Some("com.apple.TextEdit".into()), + process_instance: Some("test-instance".into()), + }]); + + let err = wait_for_event(request, &adapter, Some(Ok(baseline))).unwrap_err(); + + let AppError::Adapter(adapter_err) = err else { + panic!("expected AppError::Adapter"); + }; + assert_eq!(adapter_err.code, ErrorCode::Timeout); + let details = adapter_err.details.unwrap(); + assert_eq!(details["last_error"]["code"], "APP_UNRESPONSIVE"); + assert_eq!(details["baseline_counts"]["apps"], 1); +} + +#[path = "wait_event_deadline_tests.rs"] +mod deadline_tests; + +#[path = "wait_event_lifecycle_tests.rs"] +mod lifecycle_tests; diff --git a/crates/core/src/commands/wait_latest_ref_cache.rs b/crates/core/src/commands/wait_latest_ref_cache.rs deleted file mode 100644 index 3411b46..0000000 --- a/crates/core/src/commands/wait_latest_ref_cache.rs +++ /dev/null @@ -1,69 +0,0 @@ -use crate::{error::AppError, refs::RefMap, refs_store::RefStore}; -use std::time::{Duration, Instant}; - -pub(crate) struct LatestRefCache<'a> { - store: &'a RefStore, - snapshot_id: Option, - refmap: RefMap, - last_refresh: Instant, -} - -impl<'a> LatestRefCache<'a> { - pub(crate) fn new(store: &'a RefStore) -> Result { - let mut snapshot_id = store.latest_snapshot_id(); - let refmap = if let Some(id) = snapshot_id.as_deref() { - store.load_snapshot(id)? - } else { - let refmap = store.load_latest()?; - snapshot_id = store.latest_snapshot_id(); - refmap - }; - Ok(Self { - store, - snapshot_id, - refmap, - last_refresh: Instant::now() - Duration::from_millis(500), - }) - } - - pub(crate) fn entry(&self, ref_id: &str) -> Option { - self.refmap.get(ref_id).cloned() - } - - pub(crate) fn refresh_if_due(&mut self) -> Result<(), AppError> { - if self.last_refresh.elapsed() < Duration::from_millis(500) { - return Ok(()); - } - self.last_refresh = Instant::now(); - if let Some(snapshot_id) = self.snapshot_id.as_deref() { - match self.store.load_snapshot(snapshot_id) { - Ok(refmap) => { - self.refmap = refmap; - Ok(()) - } - Err(err) => { - tracing::warn!( - "latest snapshot {snapshot_id} unreadable during wait refresh: {err}" - ); - Ok(()) - } - } - } else { - match self.store.load_latest() { - Ok(refmap) => { - self.refmap = refmap; - self.snapshot_id = self.store.latest_snapshot_id(); - Ok(()) - } - Err(err) => { - tracing::warn!("latest refmap unreadable during wait refresh: {err}"); - Ok(()) - } - } - } - } -} - -#[cfg(test)] -#[path = "wait_latest_ref_cache_tests.rs"] -mod tests; diff --git a/crates/core/src/commands/wait_latest_ref_cache_tests.rs b/crates/core/src/commands/wait_latest_ref_cache_tests.rs deleted file mode 100644 index 84aa479..0000000 --- a/crates/core/src/commands/wait_latest_ref_cache_tests.rs +++ /dev/null @@ -1,135 +0,0 @@ -use super::*; -use crate::{ - adapter::SnapshotSurface, - capability, - refs::{RefEntry, RefMap}, - refs_test_support::HomeGuard, -}; - -fn save_ref(pid: i32, name: Option<&str>) -> String { - RefStore::new() - .unwrap() - .save_new_snapshot(&refmap_with_ref(pid, name)) - .unwrap() -} - -fn refmap_with_ref(pid: i32, name: Option<&str>) -> RefMap { - let mut refmap = RefMap::new(); - refmap.allocate(RefEntry { - pid, - role: "button".into(), - name: name.map(String::from), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), - }); - refmap -} - -#[test] -fn latest_ref_cache_pins_starting_snapshot_when_latest_advances() { - let _guard = HomeGuard::new(); - let first_id = save_ref(1, Some("First")); - let store = RefStore::new().unwrap(); - - let mut cache = LatestRefCache::new(&store).unwrap(); - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - - let second_id = save_ref(99, Some("Second")); - assert_ne!(first_id, second_id); - - cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2); - cache.refresh_if_due().unwrap(); - - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - assert_eq!( - store.latest_snapshot_id().as_deref(), - Some(second_id.as_str()) - ); - assert_eq!(cache.entry("@e1").unwrap().pid, 1); -} - -#[test] -fn latest_ref_cache_debounces_consecutive_refreshes() { - let _guard = HomeGuard::new(); - let _first_id = save_ref(1, Some("First")); - let store = RefStore::new().unwrap(); - - let mut cache = LatestRefCache::new(&store).unwrap(); - let pinned_snapshot_id = cache.snapshot_id.clone(); - - let _ = save_ref(2, None); - - let debounced_refresh = std::time::Instant::now(); - cache.last_refresh = debounced_refresh; - cache.refresh_if_due().unwrap(); - - assert_eq!(cache.snapshot_id, pinned_snapshot_id); - assert_eq!(cache.last_refresh, debounced_refresh); -} - -#[test] -fn latest_ref_cache_keeps_last_good_map_when_pinned_snapshot_disappears() { - let _guard = HomeGuard::new(); - let first_id = save_ref(1, Some("First")); - let store = RefStore::new().unwrap(); - - let mut cache = LatestRefCache::new(&store).unwrap(); - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - - let home = crate::refs::home_dir().unwrap(); - let snapshot_dir = home - .join(".agent-desktop") - .join("snapshots") - .join(&first_id); - std::fs::remove_dir_all(snapshot_dir).unwrap(); - - cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2); - cache.refresh_if_due().unwrap(); - - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - assert_eq!(cache.entry("@e1").unwrap().pid, 1); -} - -#[test] -fn latest_ref_cache_retries_unreadable_pinned_snapshot_after_refresh_error() { - let _guard = HomeGuard::new(); - let first_id = save_ref(1, Some("First")); - let store = RefStore::new().unwrap(); - - let mut cache = LatestRefCache::new(&store).unwrap(); - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - - let refmap_path = crate::refs::home_dir() - .unwrap() - .join(".agent-desktop") - .join("snapshots") - .join(&first_id) - .join("refmap.json"); - std::fs::write(&refmap_path, b"{not-json").unwrap(); - - cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2); - cache.refresh_if_due().unwrap(); - - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - assert_eq!(cache.entry("@e1").unwrap().pid, 1); - - store - .save_snapshot(&first_id, &refmap_with_ref(3, Some("Recovered"))) - .unwrap(); - cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2); - cache.refresh_if_due().unwrap(); - - assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); - assert_eq!(cache.entry("@e1").unwrap().pid, 3); -} diff --git a/crates/core/src/commands/wait_mode.rs b/crates/core/src/commands/wait_mode.rs index a80b30b..8c7cd48 100644 --- a/crates/core/src/commands/wait_mode.rs +++ b/crates/core/src/commands/wait_mode.rs @@ -1,6 +1,6 @@ use crate::{ - commands::{wait::WaitArgs, wait_predicate}, - error::AppError, + AppError, + commands::{wait::WaitArgs, wait_predicate, wait_surface::SurfaceWait}, refs::validate_ref_id, }; @@ -25,6 +25,12 @@ pub(crate) enum WaitMode { app: Option, text: Option, }, + Event { + event: String, + app: Option, + window_id: Option, + window_title: Option, + }, } impl WaitMode { @@ -33,16 +39,33 @@ impl WaitMode { if let Some(ms) = args.mode.ms { return Ok(Self::Sleep(ms)); } - if args.mode.menu || args.mode.menu_closed { - return Ok(Self::Menu { - app: args.app, - open: args.mode.menu, - }); + match args.mode.surface { + Some(SurfaceWait::Menu) => { + return Ok(Self::Menu { + app: args.app, + open: true, + }); + } + Some(SurfaceWait::MenuClosed) => { + return Ok(Self::Menu { + app: args.app, + open: false, + }); + } + Some(SurfaceWait::Notification) => { + return Ok(Self::Notification { + app: args.app, + text: args.mode.text, + }); + } + None => {} } - if args.mode.notification { - return Ok(Self::Notification { + if let Some(event) = args.mode.event { + return Ok(Self::Event { + event, app: args.app, - text: args.mode.text, + window_id: args.mode.window_id, + window_title: args.mode.window, }); } if let Some(ref_id) = args.mode.element { @@ -72,6 +95,10 @@ impl WaitMode { } } +/// `--window` is dual-purposed: alone it selects `WaitMode::Window`, but +/// alongside `--event` it narrows the event wait to a specific window title +/// instead of choosing a second mode — so it is excluded from the +/// exactly-one-mode count whenever `--event` is also present. pub(crate) fn validate_wait_mode(args: &WaitArgs) -> Result<(), AppError> { if args.predicate.predicate.is_some() && args.mode.element.is_none() { return Err(AppError::invalid_input_with_suggestion( @@ -85,20 +112,21 @@ pub(crate) fn validate_wait_mode(args: &WaitArgs) -> Result<(), AppError> { "Use --element --predicate value --value .", )); } - if args.predicate.count.is_some() && (args.mode.text.is_none() || args.mode.notification) { + let waits_for_notification = matches!(args.mode.surface, Some(SurfaceWait::Notification)); + if args.predicate.count.is_some() && (args.mode.text.is_none() || waits_for_notification) { return Err(AppError::invalid_input_with_suggestion( "--count is only valid for --text waits", "Use --text --count without --notification, or remove --count.", )); } + validate_event_filters(args)?; let selected = [ args.mode.ms.is_some(), args.mode.element.is_some(), - args.mode.window.is_some(), - args.mode.text.is_some() && !args.mode.notification, - args.mode.menu, - args.mode.menu_closed, - args.mode.notification, + args.mode.window.is_some() && args.mode.event.is_none(), + args.mode.text.is_some() && !waits_for_notification, + args.mode.surface.is_some(), + args.mode.event.is_some(), ] .into_iter() .filter(|selected| *selected) @@ -109,14 +137,53 @@ pub(crate) fn validate_wait_mode(args: &WaitArgs) -> Result<(), AppError> { if selected == 0 { return Err(missing_wait_mode()); } - Err(AppError::invalid_input_with_suggestion( + Err(ambiguous_wait_mode()) +} + +pub(crate) fn ambiguous_wait_mode() -> AppError { + AppError::invalid_input_with_suggestion( "wait accepts exactly one mode", - "Use one of: ms, --element, --window, --text, --menu, --menu-closed, or --notification.", - )) + "Use one of: ms, --element, --window, --text, --menu, --menu-closed, --notification, or --event.", + ) +} + +fn validate_event_filters(args: &WaitArgs) -> Result<(), AppError> { + let Some(token) = args.mode.event.as_deref() else { + return Ok(()); + }; + let event = crate::commands::wait_event::parse_event_kind(token)?; + let has_window_filter = args.mode.window.is_some() || args.mode.window_id.is_some(); + let supports_window_filter = matches!( + &event, + crate::EventKind::WindowOpened + | crate::EventKind::WindowClosed + | crate::EventKind::FocusChangedWindow + ); + if has_window_filter && !supports_window_filter { + return Err(AppError::invalid_input_with_suggestion( + format!("--event {token} does not carry window identity"), + "Remove --window and --window-id, or choose a window lifecycle event.", + )); + } + let is_surface = matches!( + &event, + crate::EventKind::SurfaceAppeared { .. } | crate::EventKind::SurfaceDismissed { .. } + ); + if is_surface && args.app.is_none() { + return Err(AppError::invalid_input_with_suggestion( + format!("--event {token} requires --app"), + "Add --app so the adapter can inspect that application's surfaces.", + )); + } + Ok(()) } fn missing_wait_mode() -> AppError { AppError::invalid_input( - "Provide a duration (ms), --menu, --notification, --element , --window , or --text <text>", + "Provide a duration (ms), --menu, --notification, --event, --element <ref>, --window <title>, or --text <text>", ) } + +#[cfg(test)] +#[path = "wait_mode_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/wait_mode_tests.rs b/crates/core/src/commands/wait_mode_tests.rs new file mode 100644 index 0000000..f516589 --- /dev/null +++ b/crates/core/src/commands/wait_mode_tests.rs @@ -0,0 +1,156 @@ +use super::*; +use crate::commands::wait::{WaitModeArgs, WaitPredicateArgs}; + +fn args(mode: WaitModeArgs) -> WaitArgs { + WaitArgs { + mode, + predicate: WaitPredicateArgs { + snapshot_id: None, + predicate: None, + value: None, + action: None, + count: None, + }, + timeout_ms: 1_000, + app: None, + } +} + +fn mode() -> WaitModeArgs { + WaitModeArgs { + ms: None, + element: None, + window: None, + text: None, + surface: None, + event: None, + window_id: None, + } +} + +#[test] +fn event_with_window_title_narrowing_is_a_single_mode() { + let result = validate_wait_mode(&args(WaitModeArgs { + event: Some("window-opened".into()), + window: Some("Untitled".into()), + ..mode() + })); + assert!( + result.is_ok(), + "--window alongside --event must narrow the event wait, not count as a second mode: {result:?}" + ); +} + +#[test] +fn event_alone_is_still_a_single_mode() { + let result = validate_wait_mode(&args(WaitModeArgs { + event: Some("window-opened".into()), + ..mode() + })); + assert!(result.is_ok()); +} + +#[test] +fn window_and_element_together_remain_ambiguous() { + let result = validate_wait_mode(&args(WaitModeArgs { + window: Some("Untitled".into()), + element: Some("@e1".into()), + ..mode() + })); + assert!(result.is_err()); +} + +#[test] +fn from_args_threads_window_title_into_event_mode() { + let parsed = WaitMode::from_args(args(WaitModeArgs { + event: Some("window-opened".into()), + window: Some("Untitled".into()), + ..mode() + })) + .unwrap(); + match parsed { + WaitMode::Event { + event, + window_title, + .. + } => { + assert_eq!(event, "window-opened"); + assert_eq!(window_title.as_deref(), Some("Untitled")); + } + _ => panic!("expected WaitMode::Event, got a different mode"), + } +} + +#[test] +fn from_args_maps_surface_variants_to_menu_open_state() { + let open = WaitMode::from_args(args(WaitModeArgs { + surface: Some(SurfaceWait::Menu), + ..mode() + })) + .unwrap(); + assert!(matches!(open, WaitMode::Menu { open: true, .. })); + + let closed = WaitMode::from_args(args(WaitModeArgs { + surface: Some(SurfaceWait::MenuClosed), + ..mode() + })) + .unwrap(); + assert!(matches!(closed, WaitMode::Menu { open: false, .. })); +} + +#[test] +fn from_args_threads_text_filter_into_notification_mode() { + let parsed = WaitMode::from_args(args(WaitModeArgs { + surface: Some(SurfaceWait::Notification), + text: Some("done".into()), + ..mode() + })) + .unwrap(); + match parsed { + WaitMode::Notification { text, .. } => assert_eq!(text.as_deref(), Some("done")), + _ => panic!("expected WaitMode::Notification, got a different mode"), + } +} + +#[test] +fn surface_and_element_together_remain_ambiguous() { + let result = validate_wait_mode(&args(WaitModeArgs { + surface: Some(SurfaceWait::Menu), + element: Some("@e1".into()), + ..mode() + })); + + assert_eq!(result.unwrap_err().code(), "INVALID_ARGS"); +} + +#[test] +fn app_event_rejects_window_filter_immediately() { + let result = validate_wait_mode(&args(WaitModeArgs { + event: Some("app-launched".into()), + window_id: Some("w-1".into()), + ..mode() + })); + + assert_eq!(result.unwrap_err().code(), "INVALID_ARGS"); +} + +#[test] +fn surface_event_requires_app_scope() { + let result = validate_wait_mode(&args(WaitModeArgs { + event: Some("surface-appeared".into()), + ..mode() + })); + + assert_eq!(result.unwrap_err().code(), "INVALID_ARGS"); +} + +#[test] +fn surface_event_accepts_app_scope() { + let mut request = args(WaitModeArgs { + event: Some("surface-dismissed".into()), + ..mode() + }); + request.app = Some("TextEdit".into()); + + assert!(validate_wait_mode(&request).is_ok()); +} diff --git a/crates/core/src/commands/wait_predicate.rs b/crates/core/src/commands/wait_predicate.rs index 952160b..9567b33 100644 --- a/crates/core/src/commands/wait_predicate.rs +++ b/crates/core/src/commands/wait_predicate.rs @@ -1,9 +1,8 @@ use crate::{ + AdapterError, AppError, ErrorCode, action::Action, action_request::ActionRequest, - actionability::{bounds_are_visible, states_are_enabled}, adapter::{NativeHandle, PlatformAdapter, optional_live_read}, - error::{AdapterError, AppError, ErrorCode}, refs::RefEntry, }; use serde_json::{Value, json}; @@ -77,9 +76,7 @@ impl ElementPredicate { fn parse_actionability_action(action: Option<&str>) -> Result<ActionRequest, AppError> { match action.unwrap_or("click") { "click" => Ok(ActionRequest::headless(Action::Click)), - "type" => Ok(ActionRequest::focus_fallback(Action::TypeText( - String::new(), - ))), + "type" => Ok(ActionRequest::headless(Action::TypeText(String::new()))), "set-value" => Ok(ActionRequest::headless(Action::SetValue(String::new()))), "clear" => Ok(ActionRequest::headless(Action::Clear)), other => Err(AppError::invalid_input_with_suggestion( @@ -104,13 +101,17 @@ pub(crate) fn observe( handle: &NativeHandle, predicate: &ElementPredicate, adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, + stability: crate::actionability::StabilityExpectation, ) -> Result<Value, AdapterError> { match predicate { ElementPredicate::Exists => Ok(json!({ "exists": true })), - ElementPredicate::Enabled => enabled(entry, handle, adapter), - ElementPredicate::Visible => visible(entry, handle, adapter), - ElementPredicate::Actionable(request) => actionable(entry, handle, request, adapter), - ElementPredicate::Value(expected) => value(entry, handle, expected, adapter), + ElementPredicate::Enabled => enabled(entry, handle, adapter, deadline), + ElementPredicate::Visible => visible(entry, handle, adapter, deadline), + ElementPredicate::Actionable(request) => { + actionable(entry, handle, request, adapter, deadline, stability) + } + ElementPredicate::Value(expected) => value(entry, handle, expected, adapter, deadline), } } @@ -135,23 +136,33 @@ fn reject_unused_value(value: Option<String>) -> Result<(), AppError> { } fn enabled( - entry: &RefEntry, + _entry: &RefEntry, handle: &NativeHandle, adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, ) -> Result<Value, AdapterError> { - let enabled = optional_live_read(adapter.get_live_state(handle))? - .map(|state| states_are_enabled(&state.states)) - .unwrap_or_else(|| states_are_enabled(&entry.states)); - Ok(json!({ "enabled": enabled })) + let enabled = optional_live_read(adapter.get_live_state(handle, deadline))? + .and_then(|state| state.enabled); + Ok(json!({ "enabled": enabled, "applicable": enabled.is_some() })) } fn visible( - entry: &RefEntry, + _entry: &RefEntry, handle: &NativeHandle, adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, ) -> Result<Value, AdapterError> { - let bounds = optional_live_read(adapter.get_element_bounds(handle))?.or(entry.bounds); - Ok(json!({ "visible": bounds_are_visible(bounds) })) + let live = adapter.get_live_element(handle, deadline)?; + let evidence = crate::state::VisibilityEvidence { + bounds: live.bounds, + states: live.state.states, + bounds_from_live: live.bounds.is_some(), + states_from_live: true, + }; + Ok(json!({ + "visible": evidence.result(), + "applicable": evidence.applicable(), + })) } fn actionable( @@ -159,8 +170,19 @@ fn actionable( handle: &NativeHandle, request: &ActionRequest, adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, + stability: crate::actionability::StabilityExpectation, ) -> Result<Value, AdapterError> { - match crate::actionability::check_live(entry, handle, adapter, request) { + match crate::actionability::check_live_with_stability( + &crate::actionability::LiveCheckTarget { + entry, + handle, + adapter, + deadline, + }, + request, + stability, + ) { Ok(report) => Ok(json!(report)), Err(err) if err.code == ErrorCode::ActionFailed => match err.details { Some(report) => Ok(report), @@ -171,12 +193,13 @@ fn actionable( } fn value( - entry: &RefEntry, + _entry: &RefEntry, handle: &NativeHandle, expected: &str, adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, ) -> Result<Value, AdapterError> { - let observed = optional_live_read(adapter.get_live_value(handle))?.or(entry.value.clone()); + let observed = optional_live_read(adapter.get_live_value(handle, deadline))?; let matched = observed.as_deref() == Some(expected); Ok(json!({ "matched": matched, diff --git a/crates/core/src/commands/wait_predicate_tests.rs b/crates/core/src/commands/wait_predicate_tests.rs index 6bd8333..4d58fed 100644 --- a/crates/core/src/commands/wait_predicate_tests.rs +++ b/crates/core/src/commands/wait_predicate_tests.rs @@ -2,39 +2,118 @@ use super::test_support::{ PredicateAdapter, snapshot_with_disabled_ref, snapshot_with_one_ref, wait_for_element_test, }; use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - adapter::{NativeHandle, PlatformAdapter}, - commands::wait_predicate, - element_state::ElementState, - error::AdapterError, - refs::RefEntry, - refs_test_support::HomeGuard, + AdapterError, adapter::NativeHandle, commands::wait_predicate, element_state::ElementState, + refs::RefEntry, refs_test_support::HomeGuard, }; use std::sync::Mutex; -struct FlippingPredicateAdapter { - states: Mutex<Vec<Vec<String>>>, +fn live_bounds() -> Option<crate::Rect> { + Some(crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }) } -impl PlatformAdapter for FlippingPredicateAdapter { - fn resolve_element_strict_with_timeout( +struct FlippingPredicateAdapter { + states: Mutex<Vec<Vec<String>>>, + remaining_errors: Mutex<usize>, +} + +impl ObservationOps for FlippingPredicateAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: std::time::Duration, + _deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } - fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<crate::LiveElement, AdapterError> { + let mut remaining_errors = self.remaining_errors.lock().unwrap(); + if *remaining_errors > 0 { + *remaining_errors -= 1; + return Err(AdapterError::app_unresponsive("Fixture") + .with_details(serde_json::json!({ "retryable": true }))); + } + drop(remaining_errors); + let states = self.states.lock().unwrap().pop().unwrap_or_default(); + Ok(crate::LiveElement { + identity: crate::adapter::live_identity("Run"), + state: ElementState { + role: "button".into(), + states, + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, + bounds: live_bounds(), + available_actions: vec![crate::capability::CLICK.into()], + }) + } + + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<ElementState>, AdapterError> { let states = self.states.lock().unwrap().pop().unwrap_or_default(); Ok(Some(ElementState { role: "button".into(), states, value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), })) } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<crate::Rect>, AdapterError> { + Ok(Some(crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + })) + } + + fn get_live_actions( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<Vec<String>>, AdapterError> { + Ok(Some(vec![crate::capability::CLICK.into()])) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<crate::hit_test::HitTestResult, AdapterError> { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } } +impl ActionOps for FlippingPredicateAdapter {} + +impl InputOps for FlippingPredicateAdapter {} + +impl SystemOps for FlippingPredicateAdapter {} + #[test] fn element_wait_enabled_predicate_uses_live_state() { let _guard = HomeGuard::new(); @@ -44,9 +123,12 @@ fn element_wait_enabled_predicate_uses_live_state() { role: "button".into(), states: vec![], value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), }), value: None, - bounds: None, + bounds: live_bounds(), }; let value = wait_for_element_test( @@ -70,7 +152,7 @@ fn element_wait_value_predicate_matches_live_value_without_leaking_it() { let adapter = PredicateAdapter { state: None, value: Some("ready".into()), - bounds: None, + bounds: live_bounds(), }; let value = wait_for_element_test( @@ -98,9 +180,12 @@ fn element_wait_timeout_reports_last_actionability_observation() { role: "button".into(), states: vec!["disabled".into()], value: None, + enabled: Some(false), + hidden: Some(false), + offscreen: Some(false), }), value: None, - bounds: None, + bounds: live_bounds(), }; let err = wait_for_element_test( @@ -135,9 +220,12 @@ fn element_wait_actionable_uses_live_state() { role: "button".into(), states: vec![], value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), }), value: None, - bounds: None, + bounds: live_bounds(), }; let value = wait_for_element_test( @@ -146,7 +234,7 @@ fn element_wait_actionable_uses_live_state() { wait_predicate::ElementPredicate::Actionable( crate::action_request::ActionRequest::headless(crate::action::Action::Click), ), - 50, + 500, &adapter, &crate::context::CommandContext::default(), ) @@ -162,6 +250,7 @@ fn element_wait_actionable_retries_until_live_state_converges() { let snapshot_id = snapshot_with_disabled_ref(); let adapter = FlippingPredicateAdapter { states: Mutex::new(vec![vec![], vec!["disabled".into()]]), + remaining_errors: Mutex::new(0), }; let value = wait_for_element_test( @@ -180,6 +269,31 @@ fn element_wait_actionable_retries_until_live_state_converges() { assert_eq!(value["observed"]["actionable"], true); } +#[test] +fn element_wait_actionable_retries_transient_observation_errors() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_with_disabled_ref(); + let adapter = FlippingPredicateAdapter { + states: Mutex::new(vec![vec![]]), + remaining_errors: Mutex::new(1), + }; + + let value = wait_for_element_test( + "@e1".into(), + Some(snapshot_id), + wait_predicate::ElementPredicate::Actionable( + crate::action_request::ActionRequest::headless(crate::action::Action::Click), + ), + 500, + &adapter, + &crate::context::CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["observed"]["actionable"], true); + assert_eq!(*adapter.remaining_errors.lock().unwrap(), 0); +} + #[test] fn element_wait_actionable_type_fails_on_uneditable_role() { let _guard = HomeGuard::new(); @@ -189,16 +303,19 @@ fn element_wait_actionable_type_fails_on_uneditable_role() { role: "button".into(), states: vec![], value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), }), value: None, - bounds: None, + bounds: live_bounds(), }; let err = wait_for_element_test( "@e1".into(), Some(snapshot_id), wait_predicate::ElementPredicate::Actionable( - crate::action_request::ActionRequest::focus_fallback(crate::action::Action::TypeText( + crate::action_request::ActionRequest::headless(crate::action::Action::TypeText( String::new(), )), ), @@ -208,11 +325,18 @@ fn element_wait_actionable_type_fails_on_uneditable_role() { ) .unwrap_err(); - assert_eq!(err.code(), "TIMEOUT"); + assert_eq!(err.code(), "ACTION_NOT_SUPPORTED"); match err { AppError::Adapter(adapter_error) => { let details = adapter_error.details.unwrap(); - assert_eq!(details["last_observed"]["actionable"], false); + assert_eq!(details["actionable"], false); + let editable = details["checks"] + .as_array() + .expect("a terminal actionability failure carries the check report") + .iter() + .find(|check| check["check"] == "editable") + .expect("the editable check is reported"); + assert_eq!(editable["status"], "fail"); } _ => panic!("expected adapter error"), } @@ -256,7 +380,7 @@ fn actionable_parse_mirrors_each_real_command_policy() { ); assert_eq!( request_for(Some("type")).policy, - InteractionPolicy::focus_fallback() + InteractionPolicy::headless() ); assert_eq!( request_for(Some("set-value")).policy, diff --git a/crates/core/src/commands/wait_resolution_tests.rs b/crates/core/src/commands/wait_resolution_tests.rs index cec45d3..30e2286 100644 --- a/crates/core/src/commands/wait_resolution_tests.rs +++ b/crates/core/src/commands/wait_resolution_tests.rs @@ -1,101 +1,155 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - adapter::{NativeHandle, PlatformAdapter}, + AdapterError, ErrorCode, + adapter::NativeHandle, capability, commands::wait_predicate, - error::{AdapterError, ErrorCode}, refs::{RefEntry, RefMap}, refs_store::RefStore, refs_test_support::HomeGuard, }; use std::sync::Mutex; -use std::time::Duration; use super::test_support::wait_for_element_test; struct AmbiguousResolveAdapter; -impl PlatformAdapter for AmbiguousResolveAdapter { - fn resolve_element_strict_with_timeout( +impl ObservationOps for AmbiguousResolveAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: Duration, + _deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { - Err(AdapterError::ambiguous_target("2 candidates matched")) + Err(AdapterError::ambiguous_target("2 candidates matched") + .with_details(serde_json::json!({ "retryable": true }))) } } +impl ActionOps for AmbiguousResolveAdapter {} + +impl InputOps for AmbiguousResolveAdapter {} + +impl SystemOps for AmbiguousResolveAdapter {} + struct TransientResolveAdapter { errors: Mutex<Vec<ErrorCode>>, } -impl PlatformAdapter for TransientResolveAdapter { - fn resolve_element_strict_with_timeout( +impl ObservationOps for TransientResolveAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: Duration, + _deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { if let Some(code) = self.errors.lock().unwrap().pop() { - return Err(AdapterError::new(code, "transient resolution failure")); + return Err(AdapterError::new(code, "transient resolution failure") + .with_details(serde_json::json!({ "retryable": true }))); } Ok(NativeHandle::null()) } } +impl ActionOps for TransientResolveAdapter {} + +impl InputOps for TransientResolveAdapter {} + +impl SystemOps for TransientResolveAdapter {} + struct PermissionResolveAdapter; -impl PlatformAdapter for PermissionResolveAdapter { - fn resolve_element_strict_with_timeout( +impl ObservationOps for PermissionResolveAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: Duration, + _deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { Err(AdapterError::permission_denied()) } } +impl ActionOps for PermissionResolveAdapter {} + +impl InputOps for PermissionResolveAdapter {} + +impl SystemOps for PermissionResolveAdapter {} + struct StrictOnlyResolveAdapter; -impl PlatformAdapter for StrictOnlyResolveAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { +impl ObservationOps for StrictOnlyResolveAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } } +impl ActionOps for StrictOnlyResolveAdapter {} + +impl InputOps for StrictOnlyResolveAdapter {} + +impl SystemOps for StrictOnlyResolveAdapter {} + struct TimeoutCaptureAdapter { captured_ms: Mutex<Vec<u128>>, } -impl PlatformAdapter for TimeoutCaptureAdapter { - fn resolve_element_strict_with_timeout( +impl ObservationOps for TimeoutCaptureAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - timeout: Duration, + deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { - self.captured_ms.lock().unwrap().push(timeout.as_millis()); + self.captured_ms + .lock() + .unwrap() + .push(deadline.remaining().as_millis()); Ok(NativeHandle::null()) } } +impl ActionOps for TimeoutCaptureAdapter {} + +impl InputOps for TimeoutCaptureAdapter {} + +impl SystemOps for TimeoutCaptureAdapter {} + fn snapshot_with_one_ref() -> String { let mut refmap = RefMap::new(); refmap.allocate(RefEntry { - pid: 1, - role: "button".into(), - name: Some("Run".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() } @@ -168,7 +222,7 @@ fn element_wait_passes_remaining_budget_to_resolver() { } #[test] -fn element_wait_delegates_to_strict_only_resolution() { +fn element_wait_uses_the_strict_deadline_aware_resolver() { let _guard = HomeGuard::new(); let snapshot_id = snapshot_with_one_ref(); @@ -182,7 +236,6 @@ fn element_wait_delegates_to_strict_only_resolution() { ) .unwrap(); - assert_eq!(value["found"], true); assert_eq!(value["observed"]["exists"], true); } diff --git a/crates/core/src/commands/wait_scenario_tests.rs b/crates/core/src/commands/wait_scenario_tests.rs new file mode 100644 index 0000000..066c0b4 --- /dev/null +++ b/crates/core/src/commands/wait_scenario_tests.rs @@ -0,0 +1,159 @@ +use super::test_support::wait_args; +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use crate::{AdapterError, WindowInfo, adapter::WindowFilter}; + +struct TextlessTreeAdapter; + +impl ObservationOps for TextlessTreeAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result<crate::live_locator::ObservedTree, AdapterError> { + crate::adapter::observed_tree( + &root, + crate::AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some("Doc".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }, + ) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { + Ok(vec![WindowInfo { + id: "w-1".into(), + title: "Doc".into(), + app: "TestApp".into(), + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }]) + } + + fn get_tree( + &self, + _win: &WindowInfo, + _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, + ) -> Result<crate::AccessibilityNode, AdapterError> { + Ok(crate::AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: crate::NodeIdentity { + name: Some("Doc".into()), + ..Default::default() + }, + presentation: Default::default(), + children_count: None, + children: vec![], + }) + } +} + +impl ActionOps for TextlessTreeAdapter {} + +impl InputOps for TextlessTreeAdapter {} + +impl SystemOps for TextlessTreeAdapter {} + +#[test] +fn text_wait_with_count_zero_detects_absence() { + let _guard = crate::refs_test_support::HomeGuard::new(); + + let value = execute( + WaitArgs { + mode: WaitModeArgs { + text: Some("Gone".into()), + ..wait_args().mode + }, + predicate: WaitPredicateArgs { + count: Some(0), + ..wait_args().predicate + }, + timeout_ms: 1_000, + app: Some("TestApp".into()), + }, + &TextlessTreeAdapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["found"], true); + assert_eq!(value["count"], 0); +} + +struct MenuWaitAdapter { + open_seen: std::sync::Mutex<Option<bool>>, +} + +impl ObservationOps for MenuWaitAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result<Vec<crate::AppInfo>, AdapterError> { + Ok(vec![crate::AppInfo { + name: "MenuApp".into(), + pid: crate::ProcessId::new(42), + bundle_id: None, + process_instance: Some("test-instance".into()), + }]) + } +} + +impl ActionOps for MenuWaitAdapter {} + +impl InputOps for MenuWaitAdapter {} + +impl SystemOps for MenuWaitAdapter { + fn wait_for_menu( + &self, + process: crate::ProcessIdentity, + open: bool, + _deadline: crate::Deadline, + ) -> Result<(), AdapterError> { + assert_eq!(process.pid, 42); + assert_eq!(process.instance, "test-instance"); + *self.open_seen.lock().unwrap() = Some(open); + Ok(()) + } +} + +#[test] +fn menu_closed_wait_requests_closed_state_and_reports_found() { + let adapter = MenuWaitAdapter { + open_seen: std::sync::Mutex::new(None), + }; + let value = execute( + WaitArgs { + mode: WaitModeArgs { + surface: Some(SurfaceWait::MenuClosed), + ..wait_args().mode + }, + app: Some("MenuApp".into()), + ..wait_args() + }, + &adapter, + &CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["found"], true); + assert_eq!( + *adapter.open_seen.lock().unwrap(), + Some(false), + "--menu-closed must wait for the menu to be closed (open=false)" + ); +} diff --git a/crates/core/src/commands/wait_selector.rs b/crates/core/src/commands/wait_selector.rs index 405396e..11dec5d 100644 --- a/crates/core/src/commands/wait_selector.rs +++ b/crates/core/src/commands/wait_selector.rs @@ -1,8 +1,12 @@ use crate::{ + AppError, ErrorCode, adapter::{PlatformAdapter, TreeOptions}, commands::{query, snapshot as snapshot_cmd, wait_timeout}, context::CommandContext, - error::{AppError, ErrorCode}, + live_locator::{ + LocatorMaterialization, LocatorResolveRequest, LocatorSelection, ObservationRoot, + resolve_query, + }, refs_store::RefStore, snapshot::{self, emit_snapshot_saved}, trace_artifacts, @@ -10,6 +14,9 @@ use crate::{ use serde_json::{Value, json}; use std::time::{Duration, Instant}; +const SELECTOR_POLL_INTERVAL: Duration = Duration::from_millis(75); +const DIAGNOSTIC_SNAPSHOT_BUDGET: Duration = Duration::from_millis(600); + pub struct WaitSelectorInput { pub query_raw: String, pub gone: bool, @@ -27,40 +34,50 @@ pub fn execute( let query = query::validate_selector(&input.query_raw)?; let start = Instant::now(); - let timeout = Duration::from_millis(input.timeout_ms); - let mut interval = Duration::from_millis(200); + let deadline = crate::Deadline::at(start, input.timeout_ms)?; let mut last_error = None; let mut last_built = None; loop { - match snapshot::build( - adapter, - &input.opts, - input.app.as_deref(), - input.window_id.as_deref(), - ) { - Ok(mut result) => { - let present = query::tree_has_match(&result.tree, &query); - let matched = if input.gone { !present } else { present }; - if matched { - let store = RefStore::for_session(context.session_id())?; - let snapshot_id = store.save_new_snapshot(&result.refmap)?; - trace_artifacts::copy_refmap_if_full( + match observe_selector(adapter, &input, &query, deadline) { + Ok(Some(true)) if input.gone => {} + Ok(Some(true)) => match build_materialization(adapter, &input, &query, deadline) { + Ok((true, result)) => { + return materialized_response( + result, context, - &store, - &snapshot_id, - &result.refmap, - )?; - result.snapshot_id = Some(snapshot_id.clone()); - emit_snapshot_saved(context, &result)?; - let elapsed = start.elapsed().as_millis(); - return snapshot_cmd::format_snapshot_fields( - &result, - Some(elapsed), - Some(&input.query_raw), + &input, + start.elapsed().as_millis(), ); } - last_built = Some(result); + Ok((false, result)) => last_built = Some(result), + Err(err) if err.code() == ErrorCode::Timeout.as_str() && deadline.is_expired() => { + return timeout_response( + adapter, + &input, + context, + last_built, + Some(poll_error_json(&err)), + ); + } + Err(err) + if err.code() == ErrorCode::Timeout.as_str() + || is_transient_poll_error(&err) + || is_target_gone_error(&err) => + { + last_error = Some(poll_error_json(&err)); + } + Err(err) => return Err(err), + }, + Ok(Some(false)) if input.gone => { + return Ok(target_absent_response( + &input.query_raw, + start.elapsed().as_millis(), + )); + } + Ok(Some(false)) => {} + Ok(None) => { + last_error = Some(json!({ "kind": "locator_incomplete" })); } Err(err) if is_target_gone_error(&err) => { if input.gone { @@ -71,40 +88,70 @@ pub fn execute( } last_error = Some(poll_error_json(&err)); } + Err(err) if err.code() == ErrorCode::Timeout.as_str() && deadline.is_expired() => { + return timeout_response(adapter, &input, context, last_built, last_error); + } + Err(err) if err.code() == ErrorCode::Timeout.as_str() => { + last_error = Some(poll_error_json(&err)); + } Err(err) if is_transient_poll_error(&err) => { last_error = Some(poll_error_json(&err)); } Err(err) => return Err(err), } - let remaining = timeout.saturating_sub(start.elapsed()); + let mut remaining = deadline.remaining(); + if !remaining.is_zero() && remaining <= SELECTOR_POLL_INTERVAL && last_built.is_none() { + match snapshot::build( + adapter, + &input.opts, + input.app.as_deref(), + input.window_id.as_deref(), + deadline, + ) { + Ok(result) => last_built = Some(result), + Err(error) => last_error = Some(poll_error_json(&error)), + } + remaining = deadline.remaining(); + } if remaining.is_zero() { - let last_snapshot_id = persist_last_built(context, last_built.as_ref())?; - return wait_timeout::selector( - &input.query_raw, - input.gone, - input.timeout_ms, - last_error, - last_snapshot_id, - ); + return timeout_response(adapter, &input, context, last_built, last_error); } - std::thread::sleep(remaining.min(interval)); - interval = (interval * 2).min(Duration::from_millis(1000)); + std::thread::sleep(remaining.min(SELECTOR_POLL_INTERVAL)); } } -fn target_absent_response(query_raw: &str, elapsed_ms: u128) -> Value { - json!({ - "matched_selector": query_raw, - "gone": true, - "target_absent": true, - "elapsed_ms": elapsed_ms, - }) -} - -fn poll_error_json(err: &AppError) -> Value { - json!({ "code": err.code(), "message": err.to_string() }) +fn timeout_response( + adapter: &dyn PlatformAdapter, + input: &WaitSelectorInput, + context: &CommandContext, + mut last_built: Option<snapshot::SnapshotResult>, + mut last_error: Option<Value>, +) -> Result<Value, AppError> { + if last_built.is_none() { + let diagnostic_deadline = + crate::Deadline::after(DIAGNOSTIC_SNAPSHOT_BUDGET.as_millis() as u64)?; + match snapshot::build( + adapter, + &input.opts, + input.app.as_deref(), + input.window_id.as_deref(), + diagnostic_deadline, + ) { + Ok(result) => last_built = Some(result), + Err(error) if last_error.is_none() => last_error = Some(poll_error_json(&error)), + Err(_) => {} + } + } + let snapshot_id = persist_last_built(context, last_built.as_ref())?; + wait_timeout::selector( + &input.query_raw, + input.gone, + input.timeout_ms, + last_error, + snapshot_id, + ) } fn persist_last_built( @@ -120,6 +167,79 @@ fn persist_last_built( Ok(Some(snapshot_id)) } +fn observe_selector( + adapter: &dyn PlatformAdapter, + input: &WaitSelectorInput, + query: &crate::LocatorQuery, + deadline: crate::Deadline, +) -> Result<Option<bool>, AppError> { + let window = snapshot::resolve_window( + adapter, + input.app.as_deref(), + input.window_id.as_deref(), + deadline, + )?; + let resolution = resolve_query( + adapter, + query, + ObservationRoot::Window(&window), + &LocatorResolveRequest { + selection: LocatorSelection::First, + deadline, + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + }, + )?; + if !resolution.meta.selection_complete { + return Ok(None); + } + Ok(Some(resolution.meta.total_matches > 0)) +} + +fn build_materialization( + adapter: &dyn PlatformAdapter, + input: &WaitSelectorInput, + query: &crate::LocatorQuery, + deadline: crate::Deadline, +) -> Result<(bool, snapshot::SnapshotResult), AppError> { + let result = snapshot::build( + adapter, + &input.opts, + input.app.as_deref(), + input.window_id.as_deref(), + deadline, + )?; + let matched = query::tree_has_match(&result.tree, query); + Ok((matched, result)) +} + +fn materialized_response( + mut result: snapshot::SnapshotResult, + context: &CommandContext, + input: &WaitSelectorInput, + elapsed_ms: u128, +) -> Result<Value, AppError> { + let store = RefStore::for_session(context.session_id())?; + let snapshot_id = store.save_new_snapshot(&result.refmap)?; + trace_artifacts::copy_refmap_if_full(context, &store, &snapshot_id, &result.refmap)?; + result.bind_snapshot_id(snapshot_id); + emit_snapshot_saved(context, &result)?; + snapshot_cmd::format_snapshot_fields(&result, Some(elapsed_ms), Some(&input.query_raw)) +} + +fn target_absent_response(query_raw: &str, elapsed_ms: u128) -> Value { + json!({ + "matched_selector": query_raw, + "gone": true, + "target_absent": true, + "elapsed_ms": elapsed_ms, + }) +} + +fn poll_error_json(err: &AppError) -> Value { + json!({ "code": err.code(), "message": err.to_string() }) +} + fn is_target_gone_error(err: &AppError) -> bool { matches!( err, @@ -129,11 +249,7 @@ fn is_target_gone_error(err: &AppError) -> bool { } fn is_transient_poll_error(err: &AppError) -> bool { - matches!( - err, - AppError::Adapter(e) - if matches!(e.code, ErrorCode::Timeout | ErrorCode::ElementNotFound) - ) + matches!(err, AppError::Adapter(error) if error.is_explicitly_retryable()) } #[cfg(test)] diff --git a/crates/core/src/commands/wait_selector_retry_tests.rs b/crates/core/src/commands/wait_selector_retry_tests.rs new file mode 100644 index 0000000..2fc2d67 --- /dev/null +++ b/crates/core/src/commands/wait_selector_retry_tests.rs @@ -0,0 +1,229 @@ +use super::*; + +struct MaterializeTimeoutAdapter { + observations: AtomicUsize, +} + +impl ObservationOps for MaterializeTimeoutAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result<crate::live_locator::ObservedTree, AdapterError> { + if self.observations.fetch_add(1, Ordering::SeqCst) > 0 { + return Err(AdapterError::timeout("final snapshot timed out")); + } + crate::adapter::observed_tree(&root, window_node(vec![button_node("saved")])) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { + StaticTreeAdapter { + tree: window_node(Vec::new()), + } + .list_windows(_filter, _deadline) + } + + fn get_tree( + &self, + _win: &WindowInfo, + _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, + ) -> Result<AccessibilityNode, AdapterError> { + Err(AdapterError::timeout("final snapshot timed out")) + } +} + +impl ActionOps for MaterializeTimeoutAdapter {} +impl InputOps for MaterializeTimeoutAdapter {} +impl SystemOps for MaterializeTimeoutAdapter {} + +struct MaterializeMissAdapter { + observations: AtomicUsize, +} + +impl ObservationOps for MaterializeMissAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result<crate::live_locator::ObservedTree, AdapterError> { + let tree = match self.observations.fetch_add(1, Ordering::SeqCst) { + 0 => window_node(vec![button_node("saved")]), + 1 => window_node(vec![button_node("newest")]), + _ => return Err(AdapterError::timeout("poll timed out")), + }; + crate::adapter::observed_tree(&root, tree) + } + + fn list_windows( + &self, + filter: &WindowFilter, + deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { + StaticTreeAdapter { + tree: window_node(Vec::new()), + } + .list_windows(filter, deadline) + } +} + +impl ActionOps for MaterializeMissAdapter {} +impl InputOps for MaterializeMissAdapter {} +impl SystemOps for MaterializeMissAdapter {} + +#[test] +fn materialization_timeout_uses_the_wait_timeout_envelope() { + let _guard = HomeGuard::new(); + let error = execute( + WaitSelectorInput { + timeout_ms: 30, + ..base_input("button:saved", false) + }, + &MaterializeTimeoutAdapter { + observations: AtomicUsize::new(0), + }, + &CommandContext::default(), + ) + .unwrap_err(); + + let AppError::Adapter(error) = error else { + panic!("expected adapter timeout"); + }; + assert_eq!(error.details.unwrap()["kind"], "wait_timeout"); +} + +#[test] +fn appearance_window_not_found_swallowed_until_timeout() { + let _guard = HomeGuard::new(); + let err = execute( + WaitSelectorInput { + timeout_ms: 50, + ..base_input("button:saved", false) + }, + &CodeErrorAdapter { + code: ErrorCode::WindowNotFound, + }, + &CommandContext::default(), + ) + .unwrap_err(); + assert_eq!(err.code(), "TIMEOUT"); +} + +#[test] +fn appearance_element_not_found_without_retry_evidence_fails_fast() { + let _guard = HomeGuard::new(); + let err = execute( + WaitSelectorInput { + timeout_ms: 50, + ..base_input("button:saved", false) + }, + &CodeErrorAdapter { + code: ErrorCode::ElementNotFound, + }, + &CommandContext::default(), + ) + .unwrap_err(); + assert_eq!(err.code(), "ELEMENT_NOT_FOUND"); +} + +#[test] +fn incomplete_locator_timeout_is_polled_until_wait_deadline() { + let _guard = HomeGuard::new(); + let started = std::time::Instant::now(); + let err = execute( + WaitSelectorInput { + timeout_ms: 50, + ..base_input("button:saved", false) + }, + &CodeErrorAdapter { + code: ErrorCode::Timeout, + }, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(err.code(), "TIMEOUT"); + assert!(started.elapsed() >= std::time::Duration::from_millis(40)); + let AppError::Adapter(error) = err else { + panic!("expected adapter error"); + }; + assert_eq!(error.details.unwrap()["kind"], "wait_timeout"); +} + +#[test] +fn persisted_snapshot_is_loadable() { + let _guard = HomeGuard::new(); + let adapter = StaticTreeAdapter { + tree: window_node(vec![button_node("saved")]), + }; + let value = execute( + base_input("button:saved", false), + &adapter, + &CommandContext::default(), + ) + .unwrap(); + let snapshot_id = value["snapshot_id"].as_str().unwrap(); + let store = RefStore::new().unwrap(); + let refmap = store.load(Some(snapshot_id)).unwrap(); + assert!(!refmap.is_empty()); +} + +#[test] +fn zero_timeout_persists_a_diagnostic_snapshot() { + let _guard = HomeGuard::new(); + let error = execute( + WaitSelectorInput { + timeout_ms: 0, + ..base_input(":missing", false) + }, + &StaticTreeAdapter { + tree: window_node(vec![button_node("other")]), + }, + &CommandContext::default(), + ) + .expect_err("missing selector must time out"); + + let AppError::Adapter(error) = error else { + panic!("expected adapter timeout"); + }; + let snapshot_id = error.details.expect("timeout details")["snapshot_id"] + .as_str() + .expect("diagnostic snapshot id") + .to_owned(); + assert!(RefStore::new().unwrap().load(Some(&snapshot_id)).is_ok()); +} + +#[test] +fn materialization_miss_retains_its_fresh_snapshot_for_timeout() { + let _guard = HomeGuard::new(); + let error = execute( + WaitSelectorInput { + timeout_ms: 30, + ..base_input("button:saved", false) + }, + &MaterializeMissAdapter { + observations: AtomicUsize::new(0), + }, + &CommandContext::default(), + ) + .expect_err("materialization miss must resume waiting"); + + let AppError::Adapter(error) = error else { + panic!("expected adapter timeout"); + }; + let snapshot_id = error.details.expect("timeout details")["snapshot_id"] + .as_str() + .expect("diagnostic snapshot id") + .to_owned(); + let refmap = RefStore::new().unwrap().load(Some(&snapshot_id)).unwrap(); + assert_eq!( + refmap + .get("@e1") + .and_then(|entry| entry.identity.name.as_deref()), + Some("newest") + ); +} diff --git a/crates/core/src/commands/wait_selector_tests.rs b/crates/core/src/commands/wait_selector_tests.rs index 78ac5e0..5aa3a3b 100644 --- a/crates/core/src/commands/wait_selector_tests.rs +++ b/crates/core/src/commands/wait_selector_tests.rs @@ -1,11 +1,8 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - adapter::{PlatformAdapter, WindowFilter}, - context::CommandContext, - error::{AdapterError, ErrorCode}, - node::{AccessibilityNode, WindowInfo}, - refs_store::RefStore, - refs_test_support::HomeGuard, + AccessibilityNode, AdapterError, ErrorCode, WindowInfo, adapter::WindowFilter, + context::CommandContext, refs_store::RefStore, refs_test_support::HomeGuard, }; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -13,13 +10,11 @@ fn window_node(children: Vec<AccessibilityNode>) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: "window".into(), - name: Some("Doc".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some("Doc".into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children, } @@ -29,13 +24,11 @@ fn button_node(label: &str) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: "button".into(), - name: Some(label.into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some(label.into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children: vec![], } @@ -45,15 +38,31 @@ struct StaticTreeAdapter { tree: AccessibilityNode, } -impl PlatformAdapter for StaticTreeAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { +impl ObservationOps for StaticTreeAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result<crate::live_locator::ObservedTree, AdapterError> { + crate::adapter::observed_tree(&root, self.tree.clone()) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { Ok(vec![WindowInfo { id: "w-1".into(), title: "Doc".into(), app: "TestApp".into(), - pid: 1, + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), bounds: None, - is_focused: true, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, }]) } @@ -61,26 +70,55 @@ impl PlatformAdapter for StaticTreeAdapter { &self, _win: &WindowInfo, _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, ) -> Result<AccessibilityNode, AdapterError> { Ok(self.tree.clone()) } } +impl ActionOps for StaticTreeAdapter {} + +impl InputOps for StaticTreeAdapter {} + +impl SystemOps for StaticTreeAdapter {} + struct FlippingTreeAdapter { calls: AtomicUsize, before: AccessibilityNode, after: AccessibilityNode, } -impl PlatformAdapter for FlippingTreeAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { +impl ObservationOps for FlippingTreeAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result<crate::live_locator::ObservedTree, AdapterError> { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + let tree = if call == 0 { + self.before.clone() + } else { + self.after.clone() + }; + crate::adapter::observed_tree(&root, tree) + } + + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { Ok(vec![WindowInfo { id: "w-1".into(), title: "Doc".into(), app: "TestApp".into(), - pid: 1, + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), bounds: None, - is_focused: true, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, }]) } @@ -88,6 +126,7 @@ impl PlatformAdapter for FlippingTreeAdapter { &self, _win: &WindowInfo, _opts: &crate::adapter::TreeOptions, + _deadline: crate::Deadline, ) -> Result<AccessibilityNode, AdapterError> { let call = self.calls.fetch_add(1, Ordering::SeqCst); if call == 0 { @@ -98,24 +137,50 @@ impl PlatformAdapter for FlippingTreeAdapter { } } +impl ActionOps for FlippingTreeAdapter {} + +impl InputOps for FlippingTreeAdapter {} + +impl SystemOps for FlippingTreeAdapter {} + struct ErrorThenTreeAdapter; -impl PlatformAdapter for ErrorThenTreeAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { +impl ObservationOps for ErrorThenTreeAdapter { + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { Err(AdapterError::new(ErrorCode::AppNotFound, "app missing")) } } +impl ActionOps for ErrorThenTreeAdapter {} + +impl InputOps for ErrorThenTreeAdapter {} + +impl SystemOps for ErrorThenTreeAdapter {} + struct CodeErrorAdapter { code: ErrorCode, } -impl PlatformAdapter for CodeErrorAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { +impl ObservationOps for CodeErrorAdapter { + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { Err(AdapterError::new(self.code.clone(), "poll error")) } } +impl ActionOps for CodeErrorAdapter {} + +impl InputOps for CodeErrorAdapter {} + +impl SystemOps for CodeErrorAdapter {} + fn base_input(query_raw: &str, gone: bool) -> WaitSelectorInput { WaitSelectorInput { query_raw: query_raw.into(), @@ -218,7 +283,7 @@ fn gone_true_when_never_present_returns_immediately() { } #[test] -fn timeout_includes_last_snapshot_id() { +fn timeout_persists_the_last_diagnostic_snapshot() { let _guard = HomeGuard::new(); let adapter = StaticTreeAdapter { tree: window_node(vec![button_node("other")]), @@ -239,7 +304,10 @@ fn timeout_includes_last_snapshot_id() { }; assert_eq!(details["kind"], "wait_timeout"); assert_eq!(details["predicate"], "selector"); - assert!(details["snapshot_id"].as_str().is_some()); + let snapshot_id = details["snapshot_id"] + .as_str() + .expect("diagnostic snapshot id"); + assert!(RefStore::new().unwrap().load(Some(snapshot_id)).is_ok()); assert!( details.get("last_error").is_none(), "last_error must be omitted when no poll error occurred, got {details}" @@ -300,54 +368,5 @@ fn gone_true_with_window_not_found_returns_immediately() { assert_eq!(value["target_absent"], true); } -#[test] -fn appearance_window_not_found_swallowed_until_timeout() { - let _guard = HomeGuard::new(); - let err = execute( - WaitSelectorInput { - timeout_ms: 50, - ..base_input("button:saved", false) - }, - &CodeErrorAdapter { - code: ErrorCode::WindowNotFound, - }, - &CommandContext::default(), - ) - .unwrap_err(); - assert_eq!(err.code(), "TIMEOUT"); -} - -#[test] -fn appearance_element_not_found_swallowed_until_timeout() { - let _guard = HomeGuard::new(); - let err = execute( - WaitSelectorInput { - timeout_ms: 50, - ..base_input("button:saved", false) - }, - &CodeErrorAdapter { - code: ErrorCode::ElementNotFound, - }, - &CommandContext::default(), - ) - .unwrap_err(); - assert_eq!(err.code(), "TIMEOUT"); -} - -#[test] -fn persisted_snapshot_is_loadable() { - let _guard = HomeGuard::new(); - let adapter = StaticTreeAdapter { - tree: window_node(vec![button_node("saved")]), - }; - let value = execute( - base_input("button:saved", false), - &adapter, - &CommandContext::default(), - ) - .unwrap(); - let snapshot_id = value["snapshot_id"].as_str().unwrap(); - let store = RefStore::new().unwrap(); - let refmap = store.load(Some(snapshot_id)).unwrap(); - assert!(!refmap.is_empty()); -} +#[path = "wait_selector_retry_tests.rs"] +mod retry_tests; diff --git a/crates/core/src/commands/wait_surface.rs b/crates/core/src/commands/wait_surface.rs new file mode 100644 index 0000000..3d5032e --- /dev/null +++ b/crates/core/src/commands/wait_surface.rs @@ -0,0 +1,31 @@ +use crate::AppError; + +/// Which surface-lifecycle condition a `wait` targets: `--menu`, +/// `--menu-closed`, or `--notification`. One variant per flag makes the three +/// modes structurally mutually exclusive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SurfaceWait { + Menu, + MenuClosed, + Notification, +} + +impl SurfaceWait { + pub fn from_flags( + menu: bool, + menu_closed: bool, + notification: bool, + ) -> Result<Option<Self>, AppError> { + match (menu, menu_closed, notification) { + (false, false, false) => Ok(None), + (true, false, false) => Ok(Some(Self::Menu)), + (false, true, false) => Ok(Some(Self::MenuClosed)), + (false, false, true) => Ok(Some(Self::Notification)), + _ => Err(crate::commands::wait_mode::ambiguous_wait_mode()), + } + } +} + +#[cfg(test)] +#[path = "wait_surface_tests.rs"] +mod tests; diff --git a/crates/core/src/commands/wait_surface_tests.rs b/crates/core/src/commands/wait_surface_tests.rs new file mode 100644 index 0000000..20426e5 --- /dev/null +++ b/crates/core/src/commands/wait_surface_tests.rs @@ -0,0 +1,38 @@ +use super::SurfaceWait; + +#[test] +fn no_flags_selects_no_surface_wait() { + assert_eq!(SurfaceWait::from_flags(false, false, false).unwrap(), None); +} + +#[test] +fn each_flag_maps_to_its_variant() { + assert_eq!( + SurfaceWait::from_flags(true, false, false).unwrap(), + Some(SurfaceWait::Menu) + ); + assert_eq!( + SurfaceWait::from_flags(false, true, false).unwrap(), + Some(SurfaceWait::MenuClosed) + ); + assert_eq!( + SurfaceWait::from_flags(false, false, true).unwrap(), + Some(SurfaceWait::Notification) + ); +} + +#[test] +fn conflicting_flags_report_exactly_one_mode_error() { + let err = SurfaceWait::from_flags(true, false, true).unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert_eq!(err.to_string(), "wait accepts exactly one mode"); + assert!(err.suggestion().is_some()); +} + +#[test] +fn menu_and_menu_closed_together_are_rejected() { + let err = SurfaceWait::from_flags(true, true, false).unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); +} diff --git a/crates/core/src/commands/wait_test_support.rs b/crates/core/src/commands/wait_test_support.rs index 43b7767..be3886f 100644 --- a/crates/core/src/commands/wait_test_support.rs +++ b/crates/core/src/commands/wait_test_support.rs @@ -1,16 +1,43 @@ +use crate::adapter::{ActionOps, InputOps, ObservationOps, PlatformAdapter, SystemOps}; +use crate::commands::wait::{WaitArgs, WaitModeArgs, WaitPredicateArgs}; use crate::commands::wait_element::{ElementWaitInput, wait_for_element}; use crate::commands::wait_predicate; use crate::{ - adapter::{NativeHandle, PlatformAdapter}, + AdapterError, AppError, Rect, + adapter::NativeHandle, context::CommandContext, element_state::ElementState, - error::{AdapterError, AppError}, - node::Rect, refs::{RefEntry, RefMap}, refs_store::RefStore, }; use serde_json::Value; +/// Baseline `WaitArgs` with every mode/predicate field cleared, shared by the +/// notification- and text/menu-scenario test groups so both can build a +/// specific mode via `..wait_args()` struct-update syntax. +pub(super) fn wait_args() -> WaitArgs { + WaitArgs { + mode: WaitModeArgs { + ms: None, + element: None, + window: None, + text: None, + surface: None, + event: None, + window_id: None, + }, + predicate: WaitPredicateArgs { + snapshot_id: None, + predicate: None, + value: None, + action: None, + count: None, + }, + timeout_ms: 1, + app: None, + } +} + pub(super) fn wait_for_element_test( ref_id: String, snapshot_id: Option<String>, @@ -37,28 +64,84 @@ pub(super) struct PredicateAdapter { pub(super) bounds: Option<Rect>, } -impl PlatformAdapter for PredicateAdapter { - fn resolve_element_strict_with_timeout( +impl ObservationOps for PredicateAdapter { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<crate::LiveElement, AdapterError> { + Ok(crate::LiveElement { + identity: crate::adapter::live_identity("Run"), + state: self.state.clone().unwrap_or_else(|| ElementState { + role: "button".into(), + states: Vec::new(), + value: self.value.clone(), + enabled: None, + hidden: None, + offscreen: None, + }), + states_complete: true, + bounds: self.bounds, + available_actions: vec![crate::capability::CLICK.into()], + }) + } + + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: std::time::Duration, + _deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } - fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> { + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<ElementState>, AdapterError> { Ok(self.state.clone()) } - fn get_live_value(&self, _handle: &NativeHandle) -> Result<Option<String>, AdapterError> { + fn get_live_value( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<String>, AdapterError> { Ok(self.value.clone()) } - fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> { + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<Rect>, AdapterError> { Ok(self.bounds) } + + fn get_live_actions( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<Vec<String>>, AdapterError> { + Ok(Some(vec![crate::capability::CLICK.into()])) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<crate::hit_test::HitTestResult, AdapterError> { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } } +impl ActionOps for PredicateAdapter {} + +impl InputOps for PredicateAdapter {} + +impl SystemOps for PredicateAdapter {} + pub(super) fn snapshot_with_one_ref() -> String { save_ref(Vec::new()) } @@ -78,22 +161,37 @@ pub(super) fn save_ref_in_session(session_id: &str, states: Vec<String>) -> Stri fn save_ref_in_store(store: RefStore, states: Vec<String>) -> String { let mut refmap = RefMap::new(); refmap.allocate(RefEntry { - pid: 1, - role: "button".into(), - name: Some("Run".into()), - value: None, - description: None, - states, - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states, + available_actions: vec!["Click".into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); store.save_new_snapshot(&refmap).unwrap() } diff --git a/crates/core/src/commands/wait_tests.rs b/crates/core/src/commands/wait_tests.rs index f4d57d8..e2357e0 100644 --- a/crates/core/src/commands/wait_tests.rs +++ b/crates/core/src/commands/wait_tests.rs @@ -1,21 +1,36 @@ +use super::test_support::wait_args; use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ - adapter::{PlatformAdapter, WindowFilter}, - error::{AdapterError, ErrorCode}, - node::WindowInfo, - notification::{NotificationFilter, NotificationInfo}, + AdapterError, ErrorCode, NotificationFilter, NotificationInfo, WindowInfo, + adapter::WindowFilter, }; struct NoopAdapter; -impl PlatformAdapter for NoopAdapter {} +impl ObservationOps for NoopAdapter {} + +impl ActionOps for NoopAdapter {} + +impl InputOps for NoopAdapter {} + +impl SystemOps for NoopAdapter {} struct NotificationErrorAdapter; -impl PlatformAdapter for NotificationErrorAdapter { +impl ObservationOps for NotificationErrorAdapter {} + +impl ActionOps for NotificationErrorAdapter {} + +impl InputOps for NotificationErrorAdapter {} + +impl SystemOps for NotificationErrorAdapter { fn list_notifications( &self, _filter: &NotificationFilter, + _policy: crate::InteractionPolicy, + _deadline: crate::Deadline, + _lease: Option<&crate::InteractionLease>, ) -> Result<Vec<NotificationInfo>, AdapterError> { Err(AdapterError::new( ErrorCode::PlatformNotSupported, @@ -38,10 +53,19 @@ impl FlakyNotificationAdapter { } } -impl PlatformAdapter for FlakyNotificationAdapter { +impl ObservationOps for FlakyNotificationAdapter {} + +impl ActionOps for FlakyNotificationAdapter {} + +impl InputOps for FlakyNotificationAdapter {} + +impl SystemOps for FlakyNotificationAdapter { fn list_notifications( &self, _filter: &NotificationFilter, + _policy: crate::InteractionPolicy, + _deadline: crate::Deadline, + _lease: Option<&crate::InteractionLease>, ) -> Result<Vec<NotificationInfo>, AdapterError> { self.responses .lock() @@ -51,6 +75,75 @@ impl PlatformAdapter for FlakyNotificationAdapter { } } +struct LeaseState { + active: std::sync::atomic::AtomicBool, + acquisitions: std::sync::atomic::AtomicUsize, + calls: std::sync::atomic::AtomicUsize, +} + +struct LeaseRelease(std::sync::Arc<LeaseState>); + +impl Drop for LeaseRelease { + fn drop(&mut self) { + self.0 + .active + .store(false, std::sync::atomic::Ordering::SeqCst); + } +} + +struct LeaseTrackingNotificationAdapter { + state: std::sync::Arc<LeaseState>, +} + +impl ObservationOps for LeaseTrackingNotificationAdapter {} +impl ActionOps for LeaseTrackingNotificationAdapter {} +impl InputOps for LeaseTrackingNotificationAdapter {} + +impl SystemOps for LeaseTrackingNotificationAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result<crate::InteractionLease, AdapterError> { + assert!( + self.state + .active + .compare_exchange( + false, + true, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) + .is_ok(), + "a wait poll retained the previous interaction lease" + ); + self.state + .acquisitions + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, LeaseRelease(self.state.clone())) + } + + fn list_notifications( + &self, + _filter: &NotificationFilter, + policy: crate::InteractionPolicy, + _deadline: crate::Deadline, + lease: Option<&crate::InteractionLease>, + ) -> Result<Vec<NotificationInfo>, AdapterError> { + assert!(policy.is_headed()); + assert!(lease.is_some()); + assert!(self.state.active.load(std::sync::atomic::Ordering::SeqCst)); + let call = self + .state + .calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if call == 0 { + Ok(vec![notification(0, "old")]) + } else { + Ok(vec![notification(0, "old"), notification(1, "fresh")]) + } + } +} + fn notification(index: usize, title: &str) -> NotificationInfo { NotificationInfo { index, @@ -64,7 +157,7 @@ fn notification(index: usize, title: &str) -> NotificationInfo { fn notification_wait_args(timeout_ms: u64) -> WaitArgs { WaitArgs { mode: WaitModeArgs { - notification: true, + surface: Some(SurfaceWait::Notification), ..wait_args().mode }, timeout_ms, @@ -74,41 +167,28 @@ fn notification_wait_args(timeout_ms: u64) -> WaitArgs { struct WindowErrorAdapter; -impl PlatformAdapter for WindowErrorAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { +impl ObservationOps for WindowErrorAdapter { + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { Err(AdapterError::permission_denied()) } } -fn wait_args() -> WaitArgs { - WaitArgs { - mode: WaitModeArgs { - ms: None, - element: None, - window: None, - text: None, - menu: false, - menu_closed: false, - notification: false, - }, - predicate: WaitPredicateArgs { - snapshot_id: None, - predicate: None, - value: None, - action: None, - count: None, - }, - timeout_ms: 1, - app: None, - } -} +impl ActionOps for WindowErrorAdapter {} + +impl InputOps for WindowErrorAdapter {} + +impl SystemOps for WindowErrorAdapter {} #[test] fn notification_wait_propagates_adapter_error() { let err = execute( WaitArgs { mode: WaitModeArgs { - notification: true, + surface: Some(SurfaceWait::Notification), ..wait_args().mode }, ..wait_args() @@ -140,6 +220,32 @@ fn notification_wait_retries_transient_baseline_errors() { assert_eq!(value["notification"]["title"], "fresh"); } +#[test] +fn headed_notification_wait_leases_each_poll_without_holding_across_sleep() { + let state = std::sync::Arc::new(LeaseState { + active: std::sync::atomic::AtomicBool::new(false), + acquisitions: std::sync::atomic::AtomicUsize::new(0), + calls: std::sync::atomic::AtomicUsize::new(0), + }); + let adapter = LeaseTrackingNotificationAdapter { + state: state.clone(), + }; + + let value = execute( + notification_wait_args(5_000), + &adapter, + &CommandContext::default().with_headed(true), + ) + .unwrap(); + + assert_eq!(value["notification"]["title"], "fresh"); + assert_eq!( + state.acquisitions.load(std::sync::atomic::Ordering::SeqCst), + 2 + ); + assert!(!state.active.load(std::sync::atomic::Ordering::SeqCst)); +} + #[test] fn notification_wait_fingerprint_ignores_reindexed_existing_notification() { let baseline = notification_counts(&[notification(0, "old")]); @@ -177,6 +283,21 @@ fn notification_wait_times_out_with_last_error_after_transient_failures() { assert_eq!(details["last_error"]["code"], "TIMEOUT"); } +#[test] +fn expired_notification_wait_does_not_start_an_adapter_read() { + let adapter = FlakyNotificationAdapter::with_responses(vec![Ok(Vec::new())]); + + let error = execute( + notification_wait_args(0), + &adapter, + &CommandContext::default(), + ) + .unwrap_err(); + + assert_eq!(error.code(), "TIMEOUT"); + assert_eq!(adapter.responses.lock().unwrap().len(), 1); +} + #[test] fn rejects_multiple_wait_modes() { let err = execute( @@ -251,7 +372,7 @@ fn notification_wait_allows_text_filter() { let result = validate_wait_mode(&WaitArgs { mode: WaitModeArgs { text: Some("done".into()), - notification: true, + surface: Some(SurfaceWait::Notification), ..wait_args().mode }, ..wait_args() @@ -273,110 +394,3 @@ fn predicate_requires_element_mode() { assert_eq!(err.code(), "INVALID_ARGS"); } - -struct TextlessTreeAdapter; - -impl PlatformAdapter for TextlessTreeAdapter { - fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { - Ok(vec![WindowInfo { - id: "w-1".into(), - title: "Doc".into(), - app: "TestApp".into(), - pid: 1, - bounds: None, - is_focused: true, - }]) - } - - fn get_tree( - &self, - _win: &WindowInfo, - _opts: &crate::adapter::TreeOptions, - ) -> Result<crate::node::AccessibilityNode, AdapterError> { - Ok(crate::node::AccessibilityNode { - ref_id: None, - role: "window".into(), - name: Some("Doc".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, - children_count: None, - children: vec![], - }) - } -} - -#[test] -fn text_wait_with_count_zero_detects_absence() { - let _guard = crate::refs_test_support::HomeGuard::new(); - - let value = execute( - WaitArgs { - mode: WaitModeArgs { - text: Some("Gone".into()), - ..wait_args().mode - }, - predicate: WaitPredicateArgs { - count: Some(0), - ..wait_args().predicate - }, - timeout_ms: 1_000, - app: Some("TestApp".into()), - }, - &TextlessTreeAdapter, - &CommandContext::default(), - ) - .unwrap(); - - assert_eq!(value["found"], true); - assert_eq!(value["count"], 0); -} - -struct MenuWaitAdapter { - open_seen: std::sync::Mutex<Option<bool>>, -} - -impl PlatformAdapter for MenuWaitAdapter { - fn list_apps(&self) -> Result<Vec<crate::node::AppInfo>, AdapterError> { - Ok(vec![crate::node::AppInfo { - name: "MenuApp".into(), - pid: 42, - bundle_id: None, - }]) - } - - fn wait_for_menu(&self, _pid: i32, open: bool, _timeout_ms: u64) -> Result<(), AdapterError> { - *self.open_seen.lock().unwrap() = Some(open); - Ok(()) - } -} - -#[test] -fn menu_closed_wait_requests_closed_state_and_reports_found() { - let adapter = MenuWaitAdapter { - open_seen: std::sync::Mutex::new(None), - }; - let value = execute( - WaitArgs { - mode: WaitModeArgs { - menu_closed: true, - ..wait_args().mode - }, - app: Some("MenuApp".into()), - ..wait_args() - }, - &adapter, - &CommandContext::default(), - ) - .unwrap(); - - assert_eq!(value["found"], true); - assert_eq!( - *adapter.open_seen.lock().unwrap(), - Some(false), - "--menu-closed must wait for the menu to be closed (open=false)" - ); -} diff --git a/crates/core/src/commands/wait_text_match.rs b/crates/core/src/commands/wait_text_match.rs index 6659bbf..bbfbde2 100644 --- a/crates/core/src/commands/wait_text_match.rs +++ b/crates/core/src/commands/wait_text_match.rs @@ -1,4 +1,4 @@ -use crate::{node::AccessibilityNode, search_text}; +use crate::{AccessibilityNode, search_text}; pub(crate) struct TextMatch { pub ref_id: Option<String>, @@ -48,13 +48,11 @@ mod tests { AccessibilityNode { ref_id: None, role: "group".into(), - name: Some(name.into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some(name.into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children, } diff --git a/crates/core/src/commands/wait_timeout.rs b/crates/core/src/commands/wait_timeout.rs index 211fcdc..d1ba8c3 100644 --- a/crates/core/src/commands/wait_timeout.rs +++ b/crates/core/src/commands/wait_timeout.rs @@ -1,7 +1,4 @@ -use crate::{ - commands::wait_predicate, - error::{AdapterError, AppError}, -}; +use crate::{AdapterError, AppError, commands::wait_predicate}; use serde_json::{Value, json}; /// Builds the wait-loop TIMEOUT error. Every payload carries diff --git a/crates/core/src/commands/window_target.rs b/crates/core/src/commands/window_target.rs new file mode 100644 index 0000000..1318fd0 --- /dev/null +++ b/crates/core/src/commands/window_target.rs @@ -0,0 +1,94 @@ +use crate::{ + AppError, WindowInfo, WindowOp, + adapter::{PlatformAdapter, WindowFilter}, + window_lookup, +}; +use serde_json::{Value, json}; + +pub struct AppArgs { + pub app: Option<String>, + pub window_id: Option<String>, +} + +pub(crate) fn window_op_command( + args: AppArgs, + adapter: &dyn PlatformAdapter, + op: WindowOp, + response_key: &'static str, +) -> Result<Value, AppError> { + let deadline = crate::Deadline::standard()?; + let win = resolve_window( + args.app.as_deref(), + args.window_id.as_deref(), + adapter, + deadline, + )?; + let lease = adapter.acquire_interaction_lease(deadline)?; + let live = revalidate_window_for_mutation(adapter, &win, &lease)?; + adapter.window_op(&live, op, &lease)?; + Ok(json!({ response_key: true })) +} + +pub(crate) fn resolve_window_for_app( + app: Option<&str>, + window_id: Option<&str>, + adapter: &dyn PlatformAdapter, +) -> Result<WindowInfo, AppError> { + resolve_window(app, window_id, adapter, crate::Deadline::standard()?) +} + +fn resolve_window( + app: Option<&str>, + window_id: Option<&str>, + adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, +) -> Result<WindowInfo, AppError> { + if let Some(window_id) = window_id { + let candidates = adapter + .list_windows( + &WindowFilter { + focused_only: false, + app: app.map(str::to_string), + }, + deadline, + )? + .into_iter() + .filter(|window| window.id == window_id) + .filter(|window| app.is_none_or(|app| window.app.eq_ignore_ascii_case(app))) + .collect(); + return window_lookup::select_window( + candidates, + crate::AdapterError::new( + crate::ErrorCode::WindowNotFound, + format!("Window '{window_id}' was not found"), + ) + .with_suggestion("Run 'list-windows' to refresh window IDs, then retry."), + "Multiple windows matched the requested window ID", + ); + } + let app = super::helpers::resolve_app(app, adapter, deadline)?; + window_lookup::find_window_for_process( + super::helpers::process_identity(&app)?, + adapter, + deadline, + ) +} + +pub(crate) fn revalidate_window_for_mutation( + adapter: &dyn PlatformAdapter, + expected: &WindowInfo, + lease: &crate::InteractionLease, +) -> Result<WindowInfo, AppError> { + let live = adapter.resolve_window_strict(expected, lease.deadline())?; + if live.id != expected.id + || live.pid != expected.pid + || live.process_instance != expected.process_instance + { + return Err(crate::AdapterError::new( + crate::ErrorCode::StaleRef, + "Window identity changed before mutation", + ) + .into()); + } + Ok(live) +} diff --git a/crates/core/src/containment_predicate.rs b/crates/core/src/containment_predicate.rs new file mode 100644 index 0000000..c80cc64 --- /dev/null +++ b/crates/core/src/containment_predicate.rs @@ -0,0 +1,10 @@ +use crate::LocatorQuery; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ContainmentPredicate { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub has: Option<Box<LocatorQuery>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub has_not: Option<Box<LocatorQuery>>, +} diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs index c6f5bd0..01f0456 100644 --- a/crates/core/src/context.rs +++ b/crates/core/src/context.rs @@ -1,5 +1,5 @@ use crate::{ - action::Action, action_request::ActionRequest, error::AppError, + AdapterError, AppError, SignalBaseline, action::Action, action_request::ActionRequest, interaction_policy::InteractionPolicy, session, trace::TraceConfig, }; use serde_json::{Value, json}; @@ -7,13 +7,19 @@ use std::cell::Cell; use std::path::PathBuf; use std::time::Instant; +mod session_scope; + +use session_scope::SessionScope; + #[derive(Debug, Clone, Default)] pub struct CommandContext { - session_id: Option<String>, + session: Option<SessionScope>, + inherited_deadline: Option<crate::Deadline>, trace: TraceConfig, artifacts_full: bool, - headed: bool, + interaction_policy: InteractionPolicy, wait_selector: Option<WaitSelector>, + event_baseline: Option<Result<SignalBaseline, AdapterError>>, } #[derive(Debug, Clone)] @@ -34,23 +40,28 @@ pub struct WaitSelector { pub struct CommandScope<'a> { context: &'a CommandContext, command: &'static str, + success_disposition: crate::DeliverySemantics, started: Instant, finished: Cell<bool>, + _deadline_scope: crate::deadline::DeadlineScope, } impl CommandScope<'_> { - pub fn complete(self, result: &Result<Value, AppError>) { + pub fn complete(self, result: &Result<Value, AppError>) -> Result<(), AppError> { self.finished.set(true); - match result { - Ok(_) => self.emit_end(true, None, None), - Err(err) => { - let message = err.to_string(); - self.emit_end(false, Some(err.code()), Some(message.as_str())); - } - } + let emitted = match result { + Ok(_) => self.emit_end(true, None), + Err(err) => self.emit_end(false, Some(err.code())), + }; + emitted.map_err(|error| { + trace_error_with_disposition( + error, + result_disposition(result, self.success_disposition), + ) + }) } - fn emit_end(&self, ok: bool, code: Option<&str>, message: Option<&str>) { + fn emit_end(&self, ok: bool, code: Option<&str>) -> Result<(), AppError> { let mut fields = json!({ "command": self.command, "ok": ok, @@ -59,10 +70,7 @@ impl CommandScope<'_> { if let Some(code) = code { fields["code"] = json!(code); } - if let Some(message) = message { - fields["message"] = json!(message); - } - let _ = self.context.trace("command.end", fields); + self.context.trace("command.end", fields) } } @@ -71,11 +79,7 @@ impl Drop for CommandScope<'_> { if self.finished.get() { return; } - self.emit_end( - false, - Some("INTERNAL"), - Some("command scope dropped without completion"), - ); + let _ = self.emit_end(false, Some("INTERNAL")); } } @@ -90,17 +94,29 @@ impl CommandContext { } let (segment_dir, artifacts_full) = session_trace_state(session_id.as_deref(), trace_path.is_some())?; + let session = acquire_session_scope(session_id, None)?; Ok(Self { - session_id, + session, + inherited_deadline: None, trace: TraceConfig::build(trace_path, segment_dir, trace_strict)?, artifacts_full, - headed: false, + interaction_policy: InteractionPolicy::headless(), wait_selector: None, + event_baseline: None, }) } pub fn with_headed(mut self, headed: bool) -> Self { - self.headed = headed; + self.interaction_policy = if headed { + InteractionPolicy::headed() + } else { + InteractionPolicy::headless() + }; + self + } + + pub fn with_interaction_policy(mut self, policy: InteractionPolicy) -> Self { + self.interaction_policy = policy; self } @@ -113,20 +129,64 @@ impl CommandContext { self.wait_selector.as_ref() } - pub fn command_scope(&self, command: &'static str) -> CommandScope<'_> { - let _ = self.trace("command.start", json!({ "command": command })); - CommandScope { + pub fn with_event_baseline( + mut self, + baseline: Option<Result<SignalBaseline, AdapterError>>, + ) -> Self { + self.event_baseline = baseline; + self + } + + pub fn with_inherited_deadline(mut self, deadline: crate::Deadline) -> Self { + self.inherited_deadline = Some(deadline); + self + } + + pub fn event_baseline(&self) -> Option<&Result<SignalBaseline, AdapterError>> { + self.event_baseline.as_ref() + } + + pub fn command_scope(&self, command: &'static str) -> Result<CommandScope<'_>, AppError> { + self.command_scope_with_disposition(command, crate::DeliverySemantics::not_delivered()) + } + + pub fn mutating_command_scope( + &self, + command: &'static str, + ) -> Result<CommandScope<'_>, AppError> { + self.command_scope_with_disposition( + command, + crate::DeliverySemantics::delivered_unverified(), + ) + } + + fn command_scope_with_disposition( + &self, + command: &'static str, + success_disposition: crate::DeliverySemantics, + ) -> Result<CommandScope<'_>, AppError> { + let deadline_scope = crate::deadline::enter_scope(self.inherited_deadline); + self.trace("command.start", json!({ "command": command })) + .map_err(|error| { + trace_error_with_disposition(error, crate::DeliverySemantics::not_delivered()) + })?; + Ok(CommandScope { context: self, command, + success_disposition, started: Instant::now(), finished: Cell::new(false), - } + _deadline_scope: deadline_scope, + }) } pub fn request(&self, action: Action, base: InteractionPolicy) -> ActionRequest { ActionRequest { action, policy: self.policy_with_base(base), + timeout_ms: None, + verified_point: None, + expected_process: None, } } @@ -140,19 +200,16 @@ impl CommandContext { } fn policy_with_base(&self, base: InteractionPolicy) -> InteractionPolicy { - if self.headed { - InteractionPolicy::headed() - } else { - base - } + base.join(self.interaction_policy) } pub fn for_batch_item(&self, session_id: Option<String>) -> Result<Self, AppError> { - let session_id = session_id.or_else(|| self.session_id.clone()); + let session_id = session_id.or_else(|| self.session_id().map(str::to_owned)); if let Some(id) = session_id.as_deref() { validate_session_id(id)?; } - let reuses_parent_trace = session_id == self.session_id + let reuses_parent_session = session_id.as_deref() == self.session_id(); + let reuses_parent_trace = reuses_parent_session || (self.trace.pending_file_path().is_some() && self.trace.has_sink()); let (trace, artifacts_full) = if reuses_parent_trace { (self.trace.clone(), self.artifacts_full) @@ -163,26 +220,32 @@ impl CommandContext { artifacts_full, ) }; + let session = if reuses_parent_session { + self.session.clone() + } else { + acquire_session_scope(session_id, self.inherited_deadline)? + }; Ok(Self { - session_id, + session, + inherited_deadline: self.inherited_deadline, trace, artifacts_full, - headed: self.headed, + interaction_policy: self.interaction_policy, wait_selector: None, + event_baseline: None, }) } pub fn trace(&self, event: &str, fields: Value) -> Result<(), AppError> { - self.trace.emit(event, self.session_id.as_deref(), fields) + self.trace.emit(event, self.session_id(), fields) } pub fn trace_lazy(&self, event: &str, fields: impl FnOnce() -> Value) -> Result<(), AppError> { - self.trace - .emit_lazy(event, self.session_id.as_deref(), fields) + self.trace.emit_lazy(event, self.session_id(), fields) } pub fn session_id(&self) -> Option<&str> { - self.session_id.as_deref() + self.session.as_ref().map(|session| session.id.as_str()) } pub fn trace_enabled(&self) -> bool { @@ -194,6 +257,49 @@ impl CommandContext { } } +fn acquire_session_scope( + session_id: Option<String>, + deadline: Option<crate::Deadline>, +) -> Result<Option<SessionScope>, AppError> { + let Some(session_id) = session_id else { + return Ok(None); + }; + let lease = match deadline { + Some(deadline) => session::acquire_liveness_lease_with_deadline(&session_id, deadline), + None => session::acquire_liveness_lease(&session_id), + }?; + Ok(Some(SessionScope { + id: session_id, + lease, + })) +} + +fn result_disposition( + result: &Result<Value, AppError>, + success_fallback: crate::DeliverySemantics, +) -> crate::DeliverySemantics { + match result { + Ok(value) => value + .get("disposition") + .cloned() + .and_then(|value| serde_json::from_value(value).ok()) + .unwrap_or(success_fallback), + Err(AppError::Adapter(error)) => error.disposition, + Err(_) => crate::DeliverySemantics::unknown(), + } +} + +pub(crate) fn trace_error_with_disposition( + error: AppError, + disposition: crate::DeliverySemantics, +) -> AppError { + let adapter_error = match error { + AppError::Adapter(error) => error, + other => crate::AdapterError::internal(other.to_string()), + }; + AppError::Adapter(adapter_error.with_disposition(disposition)) +} + fn session_trace_state( session_id: Option<&str>, explicit_trace: bool, diff --git a/crates/core/src/context/session_scope.rs b/crates/core/src/context/session_scope.rs new file mode 100644 index 0000000..ade632a --- /dev/null +++ b/crates/core/src/context/session_scope.rs @@ -0,0 +1,14 @@ +#[derive(Debug)] +pub(super) struct SessionScope { + pub(super) id: String, + pub(super) lease: Option<crate::session::SessionLivenessLease>, +} + +impl Clone for SessionScope { + fn clone(&self) -> Self { + Self { + id: self.id.clone(), + lease: self.lease.clone(), + } + } +} diff --git a/crates/core/src/context_scope_tests.rs b/crates/core/src/context_scope_tests.rs index b8e1b61..c24ef22 100644 --- a/crates/core/src/context_scope_tests.rs +++ b/crates/core/src/context_scope_tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::error::AppError; +use crate::AppError; use crate::session::{SessionTraceMode, StartSessionOptions, start_session}; use serde_json::json; @@ -13,8 +13,8 @@ fn command_scope_emits_start_and_success_end() { .as_nanos() )); let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); - let scope = context.command_scope("snapshot"); - scope.complete(&Ok(json!({ "ok": true }))); + let scope = context.command_scope("snapshot").unwrap(); + scope.complete(&Ok(json!({ "ok": true }))).unwrap(); let body = std::fs::read_to_string(&path).unwrap(); assert!(body.contains(r#""event":"command.start""#)); @@ -39,9 +39,9 @@ fn command_scope_emits_error_end_with_code_and_message() { .as_nanos() )); let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); - let scope = context.command_scope("wait"); + let scope = context.command_scope("wait").unwrap(); let err = AppError::invalid_input("bad args"); - scope.complete(&Err(err)); + scope.complete(&Err(err)).unwrap(); let body = std::fs::read_to_string(&path).unwrap(); assert!(body.contains(r#""ok":false"#)); @@ -60,7 +60,7 @@ fn command_scope_drop_emits_internal_end_once() { )); let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); { - let _scope = context.command_scope("click"); + let _scope = context.command_scope("click").unwrap(); } let body = std::fs::read_to_string(&path).unwrap(); @@ -72,8 +72,8 @@ fn command_scope_drop_emits_internal_end_once() { #[test] fn command_scope_is_noop_without_trace_sink() { let context = CommandContext::default(); - let scope = context.command_scope("status"); - scope.complete(&Ok(json!({}))); + let scope = context.command_scope("status").unwrap(); + scope.complete(&Ok(json!({}))).unwrap(); } #[test] @@ -141,11 +141,106 @@ fn wait_text_timeout_message_omits_raw_text_from_trace_segment() { let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); let marker = "zzq93f_super_secret_marker_do_not_leak"; let err = crate::commands::wait_timeout::text(marker, 50, None, None).unwrap_err(); - let scope = context.command_scope("wait"); - scope.complete(&Err(err)); + let scope = context.command_scope("wait").unwrap(); + scope.complete(&Err(err)).unwrap(); let body = std::fs::read_to_string(&path).unwrap(); assert!(body.contains(r#""event":"command.end""#)); assert!(!body.contains(marker)); let _ = std::fs::remove_file(path); } + +#[test] +fn strict_command_start_failure_is_returned() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-scope-start-failure-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(crate::trace::MAX_TRACE_FILE_BYTES) + .unwrap(); + + let error = match context.command_scope("snapshot") { + Ok(_) => panic!("strict command start unexpectedly succeeded"), + Err(error) => error, + }; + + assert_eq!(error.code(), "INVALID_ARGS"); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn strict_read_only_command_end_failure_is_safe_to_retry() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-scope-end-failure-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); + let scope = context.command_scope("snapshot").unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(crate::trace::MAX_TRACE_FILE_BYTES) + .unwrap(); + + let error = scope.complete(&Ok(json!({}))).unwrap_err(); + + assert_eq!(error.code(), "INVALID_ARGS"); + let AppError::Adapter(error) = error else { + panic!("trace failure must be an adapter error"); + }; + assert_eq!(error.disposition, crate::DeliverySemantics::not_delivered()); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn strict_mutating_command_end_failure_is_unsafe_to_retry() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-scope-mutating-end-failure-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); + let scope = context.mutating_command_scope("click").unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(crate::trace::MAX_TRACE_FILE_BYTES) + .unwrap(); + + let error = scope.complete(&Ok(json!({}))).unwrap_err(); + + let AppError::Adapter(error) = error else { + panic!("trace failure must be an adapter error"); + }; + assert_eq!( + error.disposition, + crate::DeliverySemantics::delivered_unverified() + ); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn command_end_omits_secret_and_huge_error_messages() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-scope-secret-error-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); + let scope = context.command_scope("click").unwrap(); + let secret = format!("secret-marker-{}", "x".repeat(2 * 1024 * 1024)); + + scope + .complete(&Err(AppError::invalid_input(secret.clone()))) + .unwrap(); + + let body = std::fs::read_to_string(&path).unwrap(); + assert!(!body.contains("secret-marker")); + assert!(body.len() < 4096); + std::fs::remove_file(path).unwrap(); +} diff --git a/crates/core/src/context_tests.rs b/crates/core/src/context_tests.rs index 1ecf6f3..e9d46a3 100644 --- a/crates/core/src/context_tests.rs +++ b/crates/core/src/context_tests.rs @@ -236,7 +236,6 @@ fn trace_on_session_writes_segment_without_explicit_trace_flag() { let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); @@ -258,7 +257,6 @@ fn no_trace_session_still_namespaces_snapshots() { let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: false, ..Default::default() }) .unwrap(); @@ -274,7 +272,6 @@ fn explicit_trace_overrides_session_sink() { let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); @@ -305,14 +302,12 @@ fn batch_item_session_override_uses_its_own_segment_dir() { let parent_session = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); let child_session = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: true, ..Default::default() }) .unwrap(); @@ -345,14 +340,12 @@ fn strict_parent_allows_no_trace_batch_override() { let traced = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: true, ..Default::default() }) .unwrap(); let untraced = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: true, ..Default::default() }) .unwrap(); diff --git a/crates/core/src/deadline.rs b/crates/core/src/deadline.rs new file mode 100644 index 0000000..0d8aa81 --- /dev/null +++ b/crates/core/src/deadline.rs @@ -0,0 +1,185 @@ +use std::cell::Cell; +use std::time::{Duration, Instant}; + +use serde_json::json; + +use crate::{AdapterError, ErrorCode, wait_timeout_duration}; + +pub const DEFAULT_OPERATION_TIMEOUT_MS: u64 = 5_000; + +thread_local! { + static INHERITED_DEADLINE: Cell<Option<Deadline>> = const { Cell::new(None) }; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Deadline { + started_at: Instant, + expires_at: Instant, + timeout_ms: u64, + capped: bool, +} + +impl Deadline { + pub fn after(timeout_ms: u64) -> Result<Self, AdapterError> { + let deadline = Self::detached_after(timeout_ms)?; + Ok(INHERITED_DEADLINE.with(|inherited| { + inherited + .get() + .map_or(deadline, |parent| deadline.min_expiry(parent)) + })) + } + + /// Creates a bounded deadline that is intentionally independent of the + /// current command scope. Use only for recovery work that must run after + /// an operation deadline expires. + pub fn detached_after(timeout_ms: u64) -> Result<Self, AdapterError> { + let started_at = Instant::now(); + let duration = wait_timeout_duration(timeout_ms)?; + let expires_at = started_at.checked_add(duration).ok_or_else(|| { + AdapterError::new( + ErrorCode::InvalidArgs, + "Timeout cannot be represented safely", + ) + .with_details(json!({ "timeout_ms": timeout_ms })) + })?; + Ok(Self { + started_at, + expires_at, + timeout_ms, + capped: false, + }) + } + + pub fn standard() -> Result<Self, AdapterError> { + Self::after(DEFAULT_OPERATION_TIMEOUT_MS) + } + + pub fn from_duration(duration: Duration) -> Result<Self, AdapterError> { + let whole_ms = duration.as_millis(); + let rounded_ms = if duration.subsec_nanos().is_multiple_of(1_000_000) { + whole_ms + } else { + whole_ms.saturating_add(1) + }; + let timeout_ms = u64::try_from(rounded_ms) + .map_err(|_| AdapterError::new(ErrorCode::InvalidArgs, "Timeout is too large"))?; + Self::after(timeout_ms) + } + + pub(crate) fn at(started_at: Instant, timeout_ms: u64) -> Result<Self, AdapterError> { + let duration = wait_timeout_duration(timeout_ms)?; + let expires_at = started_at.checked_add(duration).ok_or_else(|| { + AdapterError::new( + ErrorCode::InvalidArgs, + "Timeout cannot be represented safely", + ) + .with_details(json!({ "timeout_ms": timeout_ms })) + })?; + let deadline = Self { + started_at, + expires_at, + timeout_ms, + capped: false, + }; + Ok(INHERITED_DEADLINE.with(|inherited| { + inherited + .get() + .map_or(deadline, |parent| deadline.min_expiry(parent)) + })) + } + + pub fn remaining(self) -> Duration { + self.expires_at.saturating_duration_since(Instant::now()) + } + + pub fn remaining_slice(self, maximum: Duration) -> Result<Duration, AdapterError> { + let remaining = self.remaining(); + if remaining.is_zero() { + Err(self.timeout_error()) + } else { + Ok(remaining.min(maximum)) + } + } + + pub fn capped(self, maximum: Duration) -> Self { + let expires_at = Instant::now() + .checked_add(maximum) + .map_or(self.expires_at, |slice| self.expires_at.min(slice)); + Self { + capped: self.capped || expires_at < self.expires_at, + expires_at, + ..self + } + } + + pub fn is_expired(self) -> bool { + self.remaining().is_zero() + } + + pub fn remaining_ms(self) -> u64 { + let remaining = self.remaining(); + let whole_ms = remaining.as_millis(); + let rounded_ms = if remaining.subsec_nanos().is_multiple_of(1_000_000) { + whole_ms + } else { + whole_ms.saturating_add(1) + }; + u64::try_from(rounded_ms).unwrap_or(u64::MAX) + } + + pub fn elapsed(self) -> Duration { + self.started_at.elapsed() + } + + pub fn timeout_ms(self) -> u64 { + self.timeout_ms + } + + pub(crate) fn was_capped(self) -> bool { + self.capped + } + + pub fn timeout_error(self) -> AdapterError { + AdapterError::timeout("Operation exceeded its deadline").with_details(json!({ + "kind": "deadline", + "timeout_ms": self.timeout_ms, + "elapsed_ms": self.elapsed().as_millis(), + })) + } + + fn min_expiry(self, other: Self) -> Self { + Self { + capped: self.capped || other.expires_at < self.expires_at, + expires_at: self.expires_at.min(other.expires_at), + ..self + } + } +} + +pub(crate) struct DeadlineScope { + previous: Option<Deadline>, + _not_send: std::marker::PhantomData<std::rc::Rc<()>>, +} + +impl Drop for DeadlineScope { + fn drop(&mut self) { + INHERITED_DEADLINE.set(self.previous); + } +} + +pub(crate) fn enter_scope(deadline: Option<Deadline>) -> DeadlineScope { + let previous = INHERITED_DEADLINE.replace(None); + let effective = match (previous, deadline) { + (Some(parent), Some(child)) => Some(child.min_expiry(parent)), + (parent, child) => parent.or(child), + }; + INHERITED_DEADLINE.set(effective); + DeadlineScope { + previous, + _not_send: std::marker::PhantomData, + } +} + +#[cfg(test)] +#[path = "deadline_tests.rs"] +mod tests; diff --git a/crates/core/src/deadline_tests.rs b/crates/core/src/deadline_tests.rs new file mode 100644 index 0000000..b629b51 --- /dev/null +++ b/crates/core/src/deadline_tests.rs @@ -0,0 +1,48 @@ +use std::time::Duration; + +use crate::{Deadline, ErrorCode}; + +#[test] +fn oversized_deadline_is_rejected() { + assert_eq!( + Deadline::after(u64::MAX).unwrap_err().code, + ErrorCode::InvalidArgs + ); +} + +#[test] +fn slices_never_extend_the_parent_budget() { + let deadline = Deadline::after(100).unwrap(); + assert!( + deadline.remaining_slice(Duration::from_secs(1)).unwrap() <= Duration::from_millis(100) + ); +} + +#[test] +fn inherited_deadline_caps_longer_local_budget() { + let inherited = Deadline::after(25).unwrap(); + let _scope = super::enter_scope(Some(inherited)); + let local = Deadline::after(5_000).unwrap(); + + assert!(local.remaining() <= Duration::from_millis(25)); +} + +#[test] +fn shorter_local_budget_wins_inside_inherited_scope() { + let inherited = Deadline::after(5_000).unwrap(); + let _scope = super::enter_scope(Some(inherited)); + let local = Deadline::after(20).unwrap(); + + assert!(local.remaining() <= Duration::from_millis(20)); +} + +#[test] +fn detached_recovery_budget_survives_an_expired_inherited_scope() { + let inherited = Deadline::after(0).unwrap(); + let _scope = super::enter_scope(Some(inherited)); + + let cleanup = Deadline::detached_after(100).unwrap(); + + assert!(!cleanup.is_expired()); + assert!(cleanup.remaining() > Duration::from_millis(50)); +} diff --git a/crates/core/src/delivery_disposition.rs b/crates/core/src/delivery_disposition.rs new file mode 100644 index 0000000..68360c0 --- /dev/null +++ b/crates/core/src/delivery_disposition.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DeliveryDisposition { + #[default] + Unknown, + NotDelivered, + DeliveryUncertain, + DeliveredUnverified, + DeliveredVerified, +} diff --git a/crates/core/src/delivery_semantics.rs b/crates/core/src/delivery_semantics.rs new file mode 100644 index 0000000..613ab06 --- /dev/null +++ b/crates/core/src/delivery_semantics.rs @@ -0,0 +1,104 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeStruct}; + +use crate::{DeliveryDisposition, RetryDisposition}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum DeliverySemantics { + #[default] + Unknown, + NotDelivered, + DeliveryUncertain, + DeliveredUnverified, + DeliveredVerified, +} + +impl DeliverySemantics { + pub const fn unknown() -> Self { + Self::Unknown + } + + pub const fn not_delivered() -> Self { + Self::NotDelivered + } + + pub const fn uncertain() -> Self { + Self::DeliveryUncertain + } + + pub const fn delivered_unverified() -> Self { + Self::DeliveredUnverified + } + + pub const fn delivered_verified() -> Self { + Self::DeliveredVerified + } + + pub const fn delivery(self) -> DeliveryDisposition { + match self { + Self::Unknown => DeliveryDisposition::Unknown, + Self::NotDelivered => DeliveryDisposition::NotDelivered, + Self::DeliveryUncertain => DeliveryDisposition::DeliveryUncertain, + Self::DeliveredUnverified => DeliveryDisposition::DeliveredUnverified, + Self::DeliveredVerified => DeliveryDisposition::DeliveredVerified, + } + } + + pub const fn retry(self) -> RetryDisposition { + match self { + Self::NotDelivered => RetryDisposition::Safe, + Self::DeliveryUncertain | Self::DeliveredUnverified | Self::DeliveredVerified => { + RetryDisposition::Unsafe + } + Self::Unknown => RetryDisposition::Unknown, + } + } +} + +impl Serialize for DeliverySemantics { + fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + let mut state = serializer.serialize_struct("DeliverySemantics", 2)?; + state.serialize_field("delivery", &self.delivery())?; + state.serialize_field("retry", &self.retry())?; + state.end() + } +} + +impl<'de> Deserialize<'de> for DeliverySemantics { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + let value = serde_json::Value::deserialize(deserializer)?; + let delivery = value + .get("delivery") + .cloned() + .ok_or_else(|| serde::de::Error::missing_field("delivery"))?; + let retry = value + .get("retry") + .cloned() + .ok_or_else(|| serde::de::Error::missing_field("retry"))?; + let delivery = + DeliveryDisposition::deserialize(delivery).map_err(serde::de::Error::custom)?; + let retry = RetryDisposition::deserialize(retry).map_err(serde::de::Error::custom)?; + let semantics = match delivery { + DeliveryDisposition::Unknown => Self::Unknown, + DeliveryDisposition::NotDelivered => Self::NotDelivered, + DeliveryDisposition::DeliveryUncertain => Self::DeliveryUncertain, + DeliveryDisposition::DeliveredUnverified => Self::DeliveredUnverified, + DeliveryDisposition::DeliveredVerified => Self::DeliveredVerified, + }; + if semantics.retry() != retry { + return Err(serde::de::Error::custom( + "delivery and retry dispositions are inconsistent", + )); + } + Ok(semantics) + } +} + +#[cfg(test)] +#[path = "delivery_semantics_tests.rs"] +mod tests; diff --git a/crates/core/src/delivery_semantics_tests.rs b/crates/core/src/delivery_semantics_tests.rs new file mode 100644 index 0000000..1193435 --- /dev/null +++ b/crates/core/src/delivery_semantics_tests.rs @@ -0,0 +1,29 @@ +use super::*; + +#[test] +fn canonical_json_roundtrips() { + for semantics in [ + DeliverySemantics::unknown(), + DeliverySemantics::not_delivered(), + DeliverySemantics::uncertain(), + DeliverySemantics::delivered_unverified(), + DeliverySemantics::delivered_verified(), + ] { + let value = serde_json::to_value(semantics).unwrap(); + assert_eq!( + serde_json::from_value::<DeliverySemantics>(value).unwrap(), + semantics + ); + } +} + +#[test] +fn impossible_retry_safe_delivery_is_rejected() { + let error = serde_json::from_value::<DeliverySemantics>(serde_json::json!({ + "delivery": "delivered_verified", + "retry": "safe", + })) + .unwrap_err(); + + assert!(error.to_string().contains("inconsistent")); +} diff --git a/crates/core/src/direction.rs b/crates/core/src/direction.rs new file mode 100644 index 0000000..a8ee815 --- /dev/null +++ b/crates/core/src/direction.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Direction { + Up, + Down, + Left, + Right, +} diff --git a/crates/core/src/dismiss_all_notifications_request.rs b/crates/core/src/dismiss_all_notifications_request.rs new file mode 100644 index 0000000..8589254 --- /dev/null +++ b/crates/core/src/dismiss_all_notifications_request.rs @@ -0,0 +1,6 @@ +use crate::InteractionPolicy; + +pub struct DismissAllNotificationsRequest<'a> { + pub app_filter: Option<&'a str>, + pub policy: InteractionPolicy, +} diff --git a/crates/core/src/dismiss_notification_request.rs b/crates/core/src/dismiss_notification_request.rs new file mode 100644 index 0000000..1589cba --- /dev/null +++ b/crates/core/src/dismiss_notification_request.rs @@ -0,0 +1,8 @@ +use crate::{InteractionPolicy, NotificationIdentity}; + +pub struct DismissNotificationRequest<'a> { + pub index: usize, + pub app_filter: Option<&'a str>, + pub identity: &'a NotificationIdentity, + pub policy: InteractionPolicy, +} diff --git a/crates/core/src/display_info.rs b/crates/core/src/display_info.rs new file mode 100644 index 0000000..d5e3edf --- /dev/null +++ b/crates/core/src/display_info.rs @@ -0,0 +1,11 @@ +use serde::{Deserialize, Serialize}; + +use crate::Rect; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct DisplayInfo { + pub id: String, + pub bounds: Rect, + pub is_primary: bool, + pub scale: f64, +} diff --git a/crates/core/src/drag_params.rs b/crates/core/src/drag_params.rs new file mode 100644 index 0000000..5f62827 --- /dev/null +++ b/crates/core/src/drag_params.rs @@ -0,0 +1,46 @@ +use serde::{Deserialize, Serialize}; + +use crate::Point; + +pub const MAX_DRAG_DURATION_MS: u64 = 60_000; +pub const MAX_DRAG_DROP_DELAY_MS: u64 = 10_000; +pub const MAX_DRAG_STEPS: u64 = 4_096; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DragParams { + pub from: Point, + pub to: Point, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option<u64>, + #[serde(skip_serializing_if = "Option::is_none")] + pub drop_delay_ms: Option<u64>, +} + +impl DragParams { + pub fn validate(&self, deadline: crate::Deadline) -> Result<(), crate::AdapterError> { + self.from.validate()?; + self.to.validate()?; + let duration = self.duration_ms.unwrap_or(0); + let drop_delay = self.drop_delay_ms.unwrap_or(0); + if duration > MAX_DRAG_DURATION_MS || drop_delay > MAX_DRAG_DROP_DELAY_MS { + return Err(crate::AdapterError::new( + crate::ErrorCode::InvalidArgs, + "Drag timing exceeds the supported maximum", + )); + } + let total = duration.checked_add(drop_delay).ok_or_else(|| { + crate::AdapterError::new(crate::ErrorCode::InvalidArgs, "Drag timing overflows") + })?; + let steps = duration.div_ceil(16).max(1); + if steps > MAX_DRAG_STEPS { + return Err(crate::AdapterError::new( + crate::ErrorCode::InvalidArgs, + "Drag would exceed the maximum synthesized step count", + )); + } + if deadline.remaining() < std::time::Duration::from_millis(total) { + return Err(deadline.timeout_error()); + } + Ok(()) + } +} diff --git a/crates/core/src/element_identifier.rs b/crates/core/src/element_identifier.rs new file mode 100644 index 0000000..e578dae --- /dev/null +++ b/crates/core/src/element_identifier.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +use crate::IdentifierKind; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ElementIdentifier { + pub kind: IdentifierKind, + pub value: String, +} diff --git a/crates/core/src/element_state.rs b/crates/core/src/element_state.rs index 64f57b5..6c9be48 100644 --- a/crates/core/src/element_state.rs +++ b/crates/core/src/element_state.rs @@ -7,4 +7,10 @@ pub struct ElementState { pub states: Vec<String>, #[serde(skip_serializing_if = "Option::is_none")] pub value: Option<String>, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub enabled: Option<bool>, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub hidden: Option<bool>, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub offscreen: Option<bool>, } diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs deleted file mode 100644 index b3fd504..0000000 --- a/crates/core/src/error.rs +++ /dev/null @@ -1,342 +0,0 @@ -use serde::Serialize; -use serde_json::Value; -use thiserror::Error; - -use crate::interaction_policy::InteractionPolicy; - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum ErrorCode { - PermDenied, - ElementNotFound, - AppNotFound, - ActionFailed, - ActionNotSupported, - StaleRef, - AmbiguousTarget, - WindowNotFound, - PlatformNotSupported, - Timeout, - InvalidArgs, - NotificationNotFound, - SnapshotNotFound, - PolicyDenied, - Internal, -} - -impl ErrorCode { - pub fn as_str(&self) -> &'static str { - match self { - ErrorCode::PermDenied => "PERM_DENIED", - ErrorCode::ElementNotFound => "ELEMENT_NOT_FOUND", - ErrorCode::AppNotFound => "APP_NOT_FOUND", - ErrorCode::ActionFailed => "ACTION_FAILED", - ErrorCode::ActionNotSupported => "ACTION_NOT_SUPPORTED", - ErrorCode::StaleRef => "STALE_REF", - ErrorCode::AmbiguousTarget => "AMBIGUOUS_TARGET", - ErrorCode::WindowNotFound => "WINDOW_NOT_FOUND", - ErrorCode::PlatformNotSupported => "PLATFORM_NOT_SUPPORTED", - ErrorCode::Timeout => "TIMEOUT", - ErrorCode::InvalidArgs => "INVALID_ARGS", - ErrorCode::NotificationNotFound => "NOTIFICATION_NOT_FOUND", - ErrorCode::SnapshotNotFound => "SNAPSHOT_NOT_FOUND", - ErrorCode::PolicyDenied => "POLICY_DENIED", - ErrorCode::Internal => "INTERNAL", - } - } -} - -#[derive(Debug, Error, Clone)] -#[error("{message}")] -pub struct AdapterError { - pub code: ErrorCode, - pub message: String, - #[source] - pub source: Option<Box<SourceError>>, - pub suggestion: Option<String>, - pub platform_detail: Option<String>, - pub details: Option<Value>, -} - -#[derive(Debug, Error, Clone)] -#[error("{0}")] -pub struct SourceError(pub String); - -impl AdapterError { - pub fn new(code: ErrorCode, message: impl Into<String>) -> Self { - Self { - code, - message: message.into(), - source: None, - suggestion: None, - platform_detail: None, - details: None, - } - } - - pub fn with_suggestion(mut self, s: impl Into<String>) -> Self { - self.suggestion = Some(s.into()); - self - } - - pub fn with_platform_detail(mut self, d: impl Into<String>) -> Self { - self.platform_detail = Some(d.into()); - self - } - - pub fn with_details(mut self, details: Value) -> Self { - self.details = Some(details); - self - } - - pub fn stale_ref(ref_id: &str) -> Self { - Self::new( - ErrorCode::StaleRef, - format!("{ref_id} not found in current RefMap"), - ) - .with_suggestion( - "Re-run a snapshot to obtain fresh refs, then retry with the new ref \ - (CLI: snapshot [--skeleton]; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)", - ) - } - - pub fn ambiguous_target(message: impl Into<String>) -> Self { - Self::new(ErrorCode::AmbiguousTarget, message).with_suggestion( - "Re-run a snapshot to refresh refs, then retry with a more specific ref", - ) - } - - pub fn not_supported(method: &str) -> Self { - Self::new( - ErrorCode::PlatformNotSupported, - format!("{method} is not supported on this platform"), - ) - .with_suggestion("This platform adapter ships in Phase 2") - } - - pub fn element_not_found(ref_id: &str) -> Self { - Self::new( - ErrorCode::ElementNotFound, - format!("Element {ref_id} could not be resolved"), - ) - .with_suggestion("Re-run a snapshot to obtain fresh refs, then retry with the new ref") - } - - pub fn timeout(msg: impl Into<String>) -> Self { - Self::new(ErrorCode::Timeout, msg) - .with_suggestion("The target application may be busy or unresponsive") - } - - pub fn notification_not_found(index: usize) -> Self { - Self::new( - ErrorCode::NotificationNotFound, - format!("Notification at index {index} not found"), - ) - .with_suggestion( - "Notification may have been dismissed or expired. \ - Re-run a notification list to see current notifications \ - (CLI: list-notifications; FFI: ad_list_notifications)", - ) - } - - pub fn internal(msg: impl Into<String>) -> Self { - Self::new(ErrorCode::Internal, msg) - } - - pub fn permission_denied() -> Self { - Self::new( - ErrorCode::PermDenied, - "Accessibility permission not granted", - ) - .with_suggestion( - "Open System Settings > Privacy & Security > Accessibility and add the app that launches agent-desktop", - ) - } - - pub fn snapshot_not_found(snapshot_id: &str) -> Self { - Self::new( - ErrorCode::SnapshotNotFound, - format!("Snapshot '{snapshot_id}' not found"), - ) - .with_suggestion( - "Re-run a snapshot and retry with the returned snapshot_id \ - (CLI: snapshot, then pass --snapshot <id>; FFI: ad_snapshot then supply snapshot_id to ad_execute_by_ref)", - ) - } - - pub fn policy_denied(message: impl Into<String>) -> Self { - Self::new(ErrorCode::PolicyDenied, message).with_suggestion( - "Use an explicit mouse/focus command if physical interaction is intended", - ) - } - - pub fn policy_denied_for_policy(message: impl Into<String>, policy: InteractionPolicy) -> Self { - Self::new(ErrorCode::PolicyDenied, message) - .with_suggestion(policy_denied_suggestion(policy)) - } -} - -fn policy_denied_suggestion(policy: InteractionPolicy) -> &'static str { - if policy.allow_focus_steal && !policy.allow_cursor_move { - "Enable cursor movement in the interaction policy to permit cursor-driven actions \ - (CLI: --headed; FFI: set allow_cursor_move in the policy), \ - or use an explicit mouse command if physical input is intended" - } else if !policy.allow_focus_steal && !policy.allow_cursor_move { - "Headless mode allows only accessibility-backed actions; \ - enable physical interaction in the policy (CLI: --headed) only if cursor/focus movement is intended, \ - otherwise refresh the snapshot or target an element with the needed semantic action" - } else { - "Use an explicit mouse/focus command if physical interaction is intended" - } -} - -#[derive(Debug, Error)] -pub enum AppError { - #[error(transparent)] - Adapter(#[from] AdapterError), - - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - - #[error("JSON error: {0}")] - Json(#[from] serde_json::Error), - - #[error("{0}")] - Internal(String), -} - -impl AppError { - pub fn code(&self) -> &str { - match self { - AppError::Adapter(e) => e.code.as_str(), - AppError::Io(_) | AppError::Json(_) | AppError::Internal(_) => "INTERNAL", - } - } - - pub fn suggestion(&self) -> Option<&str> { - match self { - AppError::Adapter(e) => e.suggestion.as_deref(), - _ => None, - } - } - - pub fn stale_ref(ref_id: &str) -> Self { - AppError::Adapter(AdapterError::stale_ref(ref_id)) - } - - pub fn invalid_input(msg: impl Into<String>) -> Self { - AppError::Adapter(AdapterError::new(ErrorCode::InvalidArgs, msg)) - } - - pub fn invalid_input_with_suggestion( - msg: impl Into<String>, - suggestion: impl Into<String>, - ) -> Self { - AppError::Adapter( - AdapterError::new(ErrorCode::InvalidArgs, msg).with_suggestion(suggestion), - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn notification_not_found_error_has_correct_code() { - let err = AdapterError::notification_not_found(5); - assert_eq!(err.code, ErrorCode::NotificationNotFound); - assert!(err.message.contains("5")); - assert!(err.suggestion.is_some()); - } - - #[test] - fn stale_ref_suggestion_is_transport_neutral() { - let err = AdapterError::stale_ref("@e7"); - assert_eq!(err.code, ErrorCode::StaleRef); - assert!(err.message.contains("@e7")); - let suggestion = err - .suggestion - .as_deref() - .expect("stale_ref should carry a suggestion"); - assert!( - suggestion.contains("snapshot"), - "stale-ref suggestion should mention running a snapshot, got: {suggestion}" - ); - assert!( - suggestion.contains("FFI"), - "stale-ref suggestion should include FFI transport guidance, got: {suggestion}" - ); - } - - #[test] - fn ambiguous_target_error_has_machine_readable_code() { - let err = AdapterError::ambiguous_target("2 candidates matched"); - - assert_eq!(err.code, ErrorCode::AmbiguousTarget); - assert_eq!(err.code.as_str(), "AMBIGUOUS_TARGET"); - assert!(err.suggestion.is_some()); - } - - #[test] - fn policy_denied_suggestion_is_mode_aware() { - let headless = - AdapterError::policy_denied_for_policy("blocked", InteractionPolicy::headless()); - assert!(headless.suggestion.unwrap().contains("--headed")); - - let focus_fallback = - AdapterError::policy_denied_for_policy("blocked", InteractionPolicy::focus_fallback()); - assert!(focus_fallback.suggestion.unwrap().contains("--headed")); - } - - #[test] - fn all_error_codes_as_str_and_serde_are_consistent() { - let cases = [ - (ErrorCode::PermDenied, "PERM_DENIED"), - (ErrorCode::ElementNotFound, "ELEMENT_NOT_FOUND"), - (ErrorCode::AppNotFound, "APP_NOT_FOUND"), - (ErrorCode::ActionFailed, "ACTION_FAILED"), - (ErrorCode::ActionNotSupported, "ACTION_NOT_SUPPORTED"), - (ErrorCode::StaleRef, "STALE_REF"), - (ErrorCode::AmbiguousTarget, "AMBIGUOUS_TARGET"), - (ErrorCode::WindowNotFound, "WINDOW_NOT_FOUND"), - (ErrorCode::PlatformNotSupported, "PLATFORM_NOT_SUPPORTED"), - (ErrorCode::Timeout, "TIMEOUT"), - (ErrorCode::InvalidArgs, "INVALID_ARGS"), - (ErrorCode::NotificationNotFound, "NOTIFICATION_NOT_FOUND"), - (ErrorCode::SnapshotNotFound, "SNAPSHOT_NOT_FOUND"), - (ErrorCode::PolicyDenied, "POLICY_DENIED"), - (ErrorCode::Internal, "INTERNAL"), - ]; - for (code, expected) in cases { - assert_eq!(code.as_str(), expected, "as_str() mismatch for {expected}"); - let serialized = serde_json::to_string(&code).expect("serializable"); - assert_eq!( - serialized, - format!("\"{expected}\""), - "serde output mismatch for {expected}" - ); - } - } - - #[test] - fn non_adapter_app_errors_yield_internal_code_and_no_suggestion() { - let io_err = AppError::Io(std::io::Error::new(std::io::ErrorKind::NotFound, "missing")); - let json_err = - AppError::Json(serde_json::from_str::<serde_json::Value>("not json").unwrap_err()); - let internal_err = AppError::Internal("unexpected state".into()); - - for err in [&io_err, &json_err, &internal_err] { - assert_eq!( - err.code(), - "INTERNAL", - "expected INTERNAL code, got {}", - err.code() - ); - assert!( - err.suggestion().is_none(), - "non-adapter AppError must not carry a suggestion" - ); - } - } -} diff --git a/crates/core/src/error_code.rs b/crates/core/src/error_code.rs new file mode 100644 index 0000000..7f46315 --- /dev/null +++ b/crates/core/src/error_code.rs @@ -0,0 +1,45 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ErrorCode { + PermDenied, + ElementNotFound, + AppNotFound, + ActionFailed, + ActionNotSupported, + StaleRef, + AmbiguousTarget, + WindowNotFound, + PlatformNotSupported, + Timeout, + InvalidArgs, + NotificationNotFound, + SnapshotNotFound, + PolicyDenied, + AppUnresponsive, + Internal, +} + +impl ErrorCode { + pub fn as_str(&self) -> &'static str { + match self { + Self::PermDenied => "PERM_DENIED", + Self::ElementNotFound => "ELEMENT_NOT_FOUND", + Self::AppNotFound => "APP_NOT_FOUND", + Self::ActionFailed => "ACTION_FAILED", + Self::ActionNotSupported => "ACTION_NOT_SUPPORTED", + Self::StaleRef => "STALE_REF", + Self::AmbiguousTarget => "AMBIGUOUS_TARGET", + Self::WindowNotFound => "WINDOW_NOT_FOUND", + Self::PlatformNotSupported => "PLATFORM_NOT_SUPPORTED", + Self::Timeout => "TIMEOUT", + Self::InvalidArgs => "INVALID_ARGS", + Self::NotificationNotFound => "NOTIFICATION_NOT_FOUND", + Self::SnapshotNotFound => "SNAPSHOT_NOT_FOUND", + Self::PolicyDenied => "POLICY_DENIED", + Self::AppUnresponsive => "APP_UNRESPONSIVE", + Self::Internal => "INTERNAL", + } + } +} diff --git a/crates/core/src/error_tests.rs b/crates/core/src/error_tests.rs new file mode 100644 index 0000000..dfd61ca --- /dev/null +++ b/crates/core/src/error_tests.rs @@ -0,0 +1,81 @@ +use crate::{AdapterError, AppError, ErrorCode, InteractionPolicy}; + +#[test] +fn notification_not_found_error_has_correct_code() { + let error = AdapterError::notification_not_found(5); + assert_eq!(error.code, ErrorCode::NotificationNotFound); + assert!(error.message.contains('5')); + assert!(error.suggestion.is_some()); +} + +#[test] +fn stale_ref_suggestion_is_transport_neutral() { + let error = AdapterError::stale_ref("@e7"); + assert_eq!(error.code, ErrorCode::StaleRef); + assert!(error.message.contains("@e7")); + let suggestion = error + .suggestion + .as_deref() + .expect("stale_ref should carry a suggestion"); + assert!(suggestion.contains("snapshot")); + assert!(suggestion.contains("FFI")); +} + +#[test] +fn ambiguous_target_error_has_machine_readable_code() { + let error = AdapterError::ambiguous_target("2 candidates matched"); + assert_eq!(error.code, ErrorCode::AmbiguousTarget); + assert_eq!(error.code.as_str(), "AMBIGUOUS_TARGET"); + assert!(error.suggestion.is_some()); +} + +#[test] +fn policy_denied_suggestion_is_mode_aware() { + let headless = AdapterError::policy_denied_for_policy("blocked", InteractionPolicy::headless()); + assert!(headless.suggestion.unwrap().contains("--headed")); + let focus = + AdapterError::policy_denied_for_policy("blocked", InteractionPolicy::focus_fallback()); + assert!(focus.suggestion.unwrap().contains("--headed")); +} + +#[test] +fn all_error_codes_as_str_and_serde_are_consistent() { + let cases = [ + (ErrorCode::PermDenied, "PERM_DENIED"), + (ErrorCode::ElementNotFound, "ELEMENT_NOT_FOUND"), + (ErrorCode::AppNotFound, "APP_NOT_FOUND"), + (ErrorCode::ActionFailed, "ACTION_FAILED"), + (ErrorCode::ActionNotSupported, "ACTION_NOT_SUPPORTED"), + (ErrorCode::StaleRef, "STALE_REF"), + (ErrorCode::AmbiguousTarget, "AMBIGUOUS_TARGET"), + (ErrorCode::WindowNotFound, "WINDOW_NOT_FOUND"), + (ErrorCode::PlatformNotSupported, "PLATFORM_NOT_SUPPORTED"), + (ErrorCode::Timeout, "TIMEOUT"), + (ErrorCode::InvalidArgs, "INVALID_ARGS"), + (ErrorCode::NotificationNotFound, "NOTIFICATION_NOT_FOUND"), + (ErrorCode::SnapshotNotFound, "SNAPSHOT_NOT_FOUND"), + (ErrorCode::PolicyDenied, "POLICY_DENIED"), + (ErrorCode::AppUnresponsive, "APP_UNRESPONSIVE"), + (ErrorCode::Internal, "INTERNAL"), + ]; + for (code, expected) in cases { + assert_eq!(code.as_str(), expected); + assert_eq!( + serde_json::to_string(&code).unwrap(), + format!("\"{expected}\"") + ); + } +} + +#[test] +fn non_adapter_app_errors_yield_internal_code_and_no_suggestion() { + let io_error = AppError::Io(std::io::Error::new(std::io::ErrorKind::NotFound, "missing")); + let json_error = + AppError::Json(serde_json::from_str::<serde_json::Value>("not json").unwrap_err()); + let internal_error = AppError::Internal("unexpected state".into()); + + for error in [&io_error, &json_error, &internal_error] { + assert_eq!(error.code(), "INTERNAL"); + assert!(error.suggestion().is_none()); + } +} diff --git a/crates/core/src/event_kind.rs b/crates/core/src/event_kind.rs new file mode 100644 index 0000000..f2439f0 --- /dev/null +++ b/crates/core/src/event_kind.rs @@ -0,0 +1,61 @@ +use crate::snapshot_surface::SnapshotSurface; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum EventKind { + WindowOpened, + WindowClosed, + AppLaunched, + AppTerminated, + FocusChangedWindow, + SurfaceAppeared { surface: SnapshotSurface }, + SurfaceDismissed { surface: SnapshotSurface }, +} + +impl EventKind { + pub fn same_variant(&self, other: &EventKind) -> bool { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + pub fn cli_token(&self) -> &'static str { + match self { + EventKind::WindowOpened => "window-opened", + EventKind::WindowClosed => "window-closed", + EventKind::AppLaunched => "app-launched", + EventKind::AppTerminated => "app-terminated", + EventKind::FocusChangedWindow => "focus-changed", + EventKind::SurfaceAppeared { .. } => "surface-appeared", + EventKind::SurfaceDismissed { .. } => "surface-dismissed", + } + } + + pub fn all_tokens() -> &'static [&'static str] { + &[ + "window-opened", + "window-closed", + "app-launched", + "app-terminated", + "focus-changed", + "surface-appeared", + "surface-dismissed", + ] + } + + pub fn parse(token: &str) -> Option<EventKind> { + match token { + "window-opened" => Some(EventKind::WindowOpened), + "window-closed" => Some(EventKind::WindowClosed), + "app-launched" => Some(EventKind::AppLaunched), + "app-terminated" => Some(EventKind::AppTerminated), + "focus-changed" => Some(EventKind::FocusChangedWindow), + "surface-appeared" => Some(EventKind::SurfaceAppeared { + surface: SnapshotSurface::Window, + }), + "surface-dismissed" => Some(EventKind::SurfaceDismissed { + surface: SnapshotSurface::Window, + }), + _ => None, + } + } +} diff --git a/crates/core/src/file_lock.rs b/crates/core/src/file_lock.rs new file mode 100644 index 0000000..4d1361a --- /dev/null +++ b/crates/core/src/file_lock.rs @@ -0,0 +1,155 @@ +use std::fs::File; +use std::path::Path; +use std::time::Duration; + +use crate::{AdapterError, Deadline, ErrorCode}; + +pub(crate) struct FileLock { + file: File, + #[cfg(unix)] + contention_count: u64, +} + +impl FileLock { + pub(crate) fn acquire( + path: &Path, + deadline: Deadline, + purpose: &str, + ) -> Result<Self, AdapterError> { + let file = crate::private_file::open_private_lock(path, true).map_err(io_error)?; + lock_file(file, deadline, purpose, path) + } + + pub(crate) fn is_held(path: &Path) -> bool { + let Ok(file) = crate::private_file::open_private_lock(path, false) else { + return false; + }; + match file.try_lock() { + Ok(()) => { + let _ = file.unlock(); + false + } + Err(std::fs::TryLockError::WouldBlock) => true, + Err(std::fs::TryLockError::Error(_)) => true, + } + } + + #[cfg(unix)] + pub(crate) fn contention_count(&self) -> u64 { + self.contention_count + } + + #[cfg(unix)] + pub(crate) fn duplicate_inheritable(&self) -> Result<std::os::fd::OwnedFd, AdapterError> { + use std::os::fd::{AsRawFd, FromRawFd}; + + let duplicated = unsafe { libc::fcntl(self.file.as_raw_fd(), libc::F_DUPFD, 3) }; + if duplicated < 0 { + return Err(io_error(std::io::Error::last_os_error())); + } + Ok(unsafe { std::os::fd::OwnedFd::from_raw_fd(duplicated) }) + } + + #[cfg(unix)] + pub(crate) fn adopt_inherited( + raw_fd: std::os::fd::RawFd, + canonical_path: &Path, + deadline: Deadline, + purpose: &str, + ) -> Result<Self, AdapterError> { + use std::os::fd::{FromRawFd, OwnedFd}; + use std::os::unix::fs::MetadataExt; + + let duplicated = unsafe { libc::fcntl(raw_fd, libc::F_DUPFD_CLOEXEC, 3) }; + if duplicated < 0 { + return Err(io_error(std::io::Error::last_os_error())); + } + let original_flags = unsafe { libc::fcntl(raw_fd, libc::F_GETFD) }; + if original_flags < 0 + || unsafe { libc::fcntl(raw_fd, libc::F_SETFD, original_flags | libc::FD_CLOEXEC) } != 0 + { + unsafe { libc::close(duplicated) }; + return Err(io_error(std::io::Error::last_os_error())); + } + let inherited = File::from(unsafe { OwnedFd::from_raw_fd(duplicated) }); + let inherited_meta = + crate::private_file::validate_private_regular(&inherited).map_err(io_error)?; + let canonical = + crate::private_file::open_private_lock(canonical_path, false).map_err(io_error)?; + let canonical_meta = canonical.metadata().map_err(io_error)?; + if inherited_meta.dev() != canonical_meta.dev() + || inherited_meta.ino() != canonical_meta.ino() + { + return Err(AdapterError::new( + ErrorCode::PolicyDenied, + "Inherited interaction lease does not identify the canonical lock file", + )); + } + lock_file(inherited, deadline, purpose, canonical_path) + } +} + +impl Drop for FileLock { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} + +fn lock_file( + file: File, + deadline: Deadline, + purpose: &str, + path: &Path, +) -> Result<FileLock, AdapterError> { + let mut contention_count = 0_u64; + loop { + if deadline.is_expired() { + return Err(lock_timeout(deadline, purpose, path, contention_count)); + } + match file.try_lock() { + Ok(()) => { + if deadline.is_expired() { + let _ = file.unlock(); + return Err(lock_timeout(deadline, purpose, path, contention_count)); + } + return Ok(FileLock { + file, + #[cfg(unix)] + contention_count, + }); + } + Err(std::fs::TryLockError::WouldBlock) => { + contention_count = contention_count.saturating_add(1); + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(lock_timeout(deadline, purpose, path, contention_count)); + } + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } + Err(std::fs::TryLockError::Error(error)) => { + return Err(AdapterError::new( + ErrorCode::Internal, + format!("Failed to acquire {purpose}: {error}"), + )); + } + } + } +} + +fn lock_timeout( + deadline: Deadline, + purpose: &str, + path: &Path, + contention_count: u64, +) -> AdapterError { + deadline.timeout_error().with_details(serde_json::json!({ + "kind": "lock_timeout", + "purpose": purpose, + "path": path.display().to_string(), + "contention_count": contention_count, + })) +} + +fn io_error(error: std::io::Error) -> AdapterError { + AdapterError::new(ErrorCode::Internal, error.to_string()) +} diff --git a/crates/core/src/headed_focus.rs b/crates/core/src/headed_focus.rs new file mode 100644 index 0000000..e932b41 --- /dev/null +++ b/crates/core/src/headed_focus.rs @@ -0,0 +1,72 @@ +use serde_json::json; + +use crate::{AdapterError, AppError, ErrorCode, PlatformAdapter, ProcessIdentity, WindowInfo}; + +pub(crate) fn focus_entry_window( + entry: &crate::RefEntry, + adapter: &dyn PlatformAdapter, + context: &crate::CommandContext, + lease: &crate::InteractionLease, +) -> Result<WindowInfo, AppError> { + let process_instance = required_text( + entry.process.process_instance.as_deref(), + "Headed input requires target process-instance identity", + )?; + let window_id = required_text( + entry.source.source_window_id.as_deref(), + "Headed input requires an exact source window id", + )?; + let expected = WindowInfo { + id: window_id.to_string(), + title: entry.source.source_window_title.clone().unwrap_or_default(), + app: entry.source.source_app.clone().unwrap_or_default(), + pid: entry.process.pid, + process_instance: Some(process_instance.to_string()), + bounds: None, + state: crate::WindowState::default(), + }; + focus_exact_window(&expected, adapter, context, lease) +} + +pub(crate) fn focus_process_window( + process: ProcessIdentity, + adapter: &dyn PlatformAdapter, + context: &crate::CommandContext, + lease: &crate::InteractionLease, +) -> Result<WindowInfo, AppError> { + let expected = + crate::window_lookup::find_window_for_process(process, adapter, lease.deadline())?; + focus_exact_window(&expected, adapter, context, lease) +} + +fn focus_exact_window( + expected: &WindowInfo, + adapter: &dyn PlatformAdapter, + context: &crate::CommandContext, + lease: &crate::InteractionLease, +) -> Result<WindowInfo, AppError> { + let live = adapter.resolve_window_strict(expected, lease.deadline())?; + if live.id != expected.id + || live.pid != expected.pid + || live.process_instance != expected.process_instance + { + return Err(AdapterError::stale_ref( + "Headed target window belongs to a different process instance", + ) + .into()); + } + adapter.focus_window(&live, lease)?; + context.trace_lazy( + "input.focus_window", + || json!({ "pid": live.pid, "window_id": live.id, "ok": true }), + )?; + Ok(live) +} + +fn required_text<'a>(value: Option<&'a str>, message: &str) -> Result<&'a str, AppError> { + value.filter(|value| !value.is_empty()).ok_or_else(|| { + AdapterError::new(ErrorCode::ActionNotSupported, message) + .with_details(json!({ "physical_delivery_started": false })) + .into() + }) +} diff --git a/crates/core/src/headed_requirement.rs b/crates/core/src/headed_requirement.rs new file mode 100644 index 0000000..da1e42c --- /dev/null +++ b/crates/core/src/headed_requirement.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HeadedRequirement { + None, + FocusedWindow, + FocusedWindowAndCursor, +} + +impl HeadedRequirement { + pub fn requires_focus(self) -> bool { + !matches!(self, Self::None) + } + + pub fn requires_cursor(self) -> bool { + matches!(self, Self::FocusedWindowAndCursor) + } +} diff --git a/crates/core/src/hints.rs b/crates/core/src/hints.rs index 7fd256d..1154f9c 100644 --- a/crates/core/src/hints.rs +++ b/crates/core/src/hints.rs @@ -1,10 +1,10 @@ -use crate::node::AccessibilityNode; +use crate::AccessibilityNode; pub fn add_structural_hints(node: &mut AccessibilityNode) { if node.role == "splitter" && node.children.len() > 1 { let total = node.children.len(); for (i, child) in node.children.iter_mut().enumerate() { - child.hint = Some(format!("column {} of {}", i + 1, total)); + child.presentation.hint = Some(format!("column {} of {}", i + 1, total)); } } diff --git a/crates/core/src/hit_test.rs b/crates/core/src/hit_test.rs new file mode 100644 index 0000000..3ee9a74 --- /dev/null +++ b/crates/core/src/hit_test.rs @@ -0,0 +1,27 @@ +use crate::Rect; +use serde::{Deserialize, Serialize}; + +/// Classifies whether a hit-tested point reaches the intended target. A hit +/// on the target itself or one of its descendants reaches it; a hit outside +/// the target's ancestor chain names a real occluder (modal, overlay, +/// sibling); a hit on the target's own ancestor is `Unknown` rather than a +/// false occlusion, since composited or custom-drawn views often expose no +/// distinct child node to hit-test. Probe failures are `Unknown` for the +/// same reason: unavailable evidence is never a false failure. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum HitTestResult { + ReachesTarget, + InterceptedBy { + #[serde(skip_serializing_if = "Option::is_none")] + role: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + bounds: Option<Rect>, + }, + Unknown, +} + +#[cfg(test)] +#[path = "hit_test_tests.rs"] +mod tests; diff --git a/crates/core/src/hit_test_tests.rs b/crates/core/src/hit_test_tests.rs new file mode 100644 index 0000000..6c7fe7c --- /dev/null +++ b/crates/core/src/hit_test_tests.rs @@ -0,0 +1,391 @@ +use super::*; +use crate::{ + AdapterError, ErrorCode, + action::Action, + action_request::ActionRequest, + actionability::{ActionabilityCheck, ActionabilityStatus, check_live}, + adapter::{ActionOps, InputOps, LiveElement, NativeHandle, ObservationOps, SystemOps}, + capability, + element_state::ElementState, + refs::RefEntry, +}; +use smallvec::SmallVec; +use std::collections::VecDeque; +use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +struct HitTestAdapter { + outcome: Result<HitTestResult, AdapterError>, + actions: Vec<String>, + calls: AtomicUsize, +} + +impl ObservationOps for HitTestAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) + } + + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<LiveElement, AdapterError> { + Ok(live_element(self.actions.clone())) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<HitTestResult, AdapterError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.outcome.clone() + } +} + +impl ActionOps for HitTestAdapter {} +impl InputOps for HitTestAdapter {} +impl SystemOps for HitTestAdapter {} + +struct SequencedHitAdapter { + outcomes: Mutex<VecDeque<HitTestResult>>, + actions: Vec<String>, + calls: AtomicUsize, +} + +impl ObservationOps for SequencedHitAdapter { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<LiveElement, AdapterError> { + Ok(live_element(self.actions.clone())) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<HitTestResult, AdapterError> { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(self + .outcomes + .lock() + .unwrap() + .pop_front() + .unwrap_or(HitTestResult::Unknown)) + } +} + +impl ActionOps for SequencedHitAdapter {} +impl InputOps for SequencedHitAdapter {} +impl SystemOps for SequencedHitAdapter {} + +fn live_element(actions: Vec<String>) -> LiveElement { + LiveElement { + identity: crate::adapter::live_identity("Save"), + state: ElementState { + role: "button".into(), + states: vec![], + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, + bounds: Some(crate::Rect { + x: 10.0, + y: 10.0, + width: 40.0, + height: 20.0, + }), + available_actions: actions, + } +} + +fn clickable_entry() -> RefEntry { + let bounds = crate::Rect { + x: 10.0, + y: 10.0, + width: 40.0, + height: 20.0, + }; + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Save".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec!["Click".into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: SmallVec::new(), + }, + } +} + +fn run_receives_events_check(outcome: Result<HitTestResult, AdapterError>) -> ActionabilityCheck { + let adapter = HitTestAdapter { + outcome, + actions: Vec::new(), + calls: AtomicUsize::new(0), + }; + let entry = clickable_entry(); + let request = ActionRequest::headed(Action::Click); + let report = check_live(&entry, &NativeHandle::null(), &adapter, &request) + .expect("a confirmed target hit should pass actionability"); + report + .checks + .into_iter() + .find(|check| check.check == "receives_events") + .expect("Click requires a receives_events check") +} + +fn run_receives_events_error(outcome: Result<HitTestResult, AdapterError>) -> AdapterError { + let adapter = HitTestAdapter { + outcome, + actions: Vec::new(), + calls: AtomicUsize::new(0), + }; + check_live( + &clickable_entry(), + &NativeHandle::null(), + &adapter, + &ActionRequest::headed(Action::Click), + ) + .expect_err("incomplete hit-test evidence must fail closed") +} + +#[test] +fn headless_semantic_click_skips_inconclusive_screen_hit_testing() { + let adapter = HitTestAdapter { + outcome: Ok(HitTestResult::Unknown), + actions: vec![capability::CLICK.into()], + calls: AtomicUsize::new(0), + }; + + let report = check_live( + &clickable_entry(), + &NativeHandle::null(), + &adapter, + &ActionRequest::headless(Action::Click), + ) + .expect("a direct semantic click must not depend on screen hit testing"); + + assert!(report.actionable); + assert_eq!(adapter.calls.load(Ordering::SeqCst), 0); + assert!( + report + .checks + .iter() + .all(|check| check.check != "receives_events") + ); +} + +#[test] +fn semantic_click_skips_occluded_screen_hit_testing() { + let adapter = HitTestAdapter { + outcome: Ok(HitTestResult::InterceptedBy { + role: Some("window".into()), + name: None, + bounds: None, + }), + actions: vec![capability::CLICK.into()], + calls: AtomicUsize::new(0), + }; + + let report = check_live( + &clickable_entry(), + &NativeHandle::null(), + &adapter, + &ActionRequest::headless(Action::Click), + ) + .expect("occlusion is irrelevant to a direct semantic click"); + + assert!(report.actionable); + assert_eq!(adapter.calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn reaches_target_result_passes_receives_events_check() { + let check = run_receives_events_check(Ok(HitTestResult::ReachesTarget)); + assert_eq!(check.status, ActionabilityStatus::Pass); +} + +#[test] +fn unknown_hit_test_result_is_non_blocking_evidence() { + let check = run_receives_events_check(Ok(HitTestResult::Unknown)); + assert_eq!(check.status, ActionabilityStatus::Unknown); +} + +#[test] +fn not_supported_hit_test_is_non_blocking_evidence() { + let check = run_receives_events_check(Err(AdapterError::not_supported("hit_test"))); + assert_eq!(check.status, ActionabilityStatus::Unknown); +} + +#[test] +fn hit_test_probe_error_is_not_reclassified() { + let err = run_receives_events_error(Err(AdapterError::internal( + "AXUIElementCopyElementAtPosition failed", + ))); + assert_eq!(err.code, ErrorCode::Internal); +} + +#[test] +fn intercepted_by_result_fails_and_carries_redactable_occluder() { + let adapter = HitTestAdapter { + outcome: Ok(HitTestResult::InterceptedBy { + role: Some("AXSheet".into()), + name: Some("Save changes?".into()), + bounds: None, + }), + actions: Vec::new(), + calls: AtomicUsize::new(0), + }; + let entry = clickable_entry(); + let request = ActionRequest::headed(Action::Click); + + let err = check_live(&entry, &NativeHandle::null(), &adapter, &request) + .expect_err("a hit outside the target's ancestor chain must fail actionability"); + + assert_eq!(err.code, ErrorCode::ActionFailed); + assert!(err.message.contains("AXSheet")); + assert!(!err.message.contains("Save changes?")); + + let details = err + .details + .expect("a Fail report attaches actionability details"); + let checks = details["checks"] + .as_array() + .expect("details.checks is an array"); + let receives_events = checks + .iter() + .find(|check| check["check"] == "receives_events") + .expect("receives_events check must be present when Click requires a hit test"); + assert_eq!(receives_events["status"], "fail"); + assert_eq!(receives_events["occluder"]["name"], "Save changes?"); + assert_eq!(receives_events["occluder"]["role"], "AXSheet"); +} + +#[test] +fn mixed_unknown_and_occluded_points_block_physical_delivery() { + let intercepted = HitTestResult::InterceptedBy { + role: Some("sheet".into()), + name: None, + bounds: None, + }; + let adapter = SequencedHitAdapter { + outcomes: Mutex::new(VecDeque::from([ + HitTestResult::Unknown, + intercepted.clone(), + intercepted.clone(), + intercepted.clone(), + intercepted, + ])), + actions: Vec::new(), + calls: AtomicUsize::new(0), + }; + + let error = check_live( + &clickable_entry(), + &NativeHandle::null(), + &adapter, + &ActionRequest::headed(Action::Click), + ) + .expect_err("known occlusion must not degrade to center-point delivery"); + let details = error.details.unwrap(); + let check = details["checks"] + .as_array() + .unwrap() + .iter() + .find(|check| check["check"] == "receives_events") + .unwrap(); + + assert_eq!(check["status"], "unknown"); + assert_eq!(check["hit_test"]["attempted"], 5); + assert_eq!(check["hit_test"]["unknown"], 1); + assert_eq!(check["hit_test"]["occluded"], 4); + assert_eq!(check["occluder"]["role"], "sheet"); + assert_eq!( + check["reason"], + "hit test evidence mixed unknown and occluded outcomes" + ); +} + +#[test] +fn unknown_only_hit_test_evidence_never_claims_occlusion() { + let report = check_live( + &clickable_entry(), + &NativeHandle::null(), + &HitTestAdapter { + outcome: Ok(HitTestResult::Unknown), + actions: Vec::new(), + calls: AtomicUsize::new(0), + }, + &ActionRequest::headed(Action::Click), + ) + .unwrap(); + let details = serde_json::to_value(report).unwrap(); + let check = details["checks"] + .as_array() + .unwrap() + .iter() + .find(|check| check["check"] == "receives_events") + .unwrap(); + + assert_eq!(check["status"], "unknown"); + assert_eq!(check["reason"], "hit test result inconclusive"); + assert!(check.get("occluder").is_none()); + assert_eq!(check["hit_test"]["unknown"], 5); + assert_eq!(check["hit_test"]["occluded"], 0); +} + +#[test] +fn terminal_capability_policy_failure_skips_hit_testing() { + let adapter = SequencedHitAdapter { + outcomes: Mutex::new(VecDeque::new()), + actions: Vec::new(), + calls: AtomicUsize::new(0), + }; + + let error = check_live( + &clickable_entry(), + &NativeHandle::null(), + &adapter, + &ActionRequest::headless(Action::Click), + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::PolicyDenied); + assert_eq!(adapter.calls.load(Ordering::SeqCst), 0); +} diff --git a/crates/core/src/identifier_kind.rs b/crates/core/src/identifier_kind.rs new file mode 100644 index 0000000..f0333c7 --- /dev/null +++ b/crates/core/src/identifier_kind.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentifierKind { + AxIdentifier, + AxDomIdentifier, + AutomationId, + RuntimeId, + AtspiObjectPath, + Unknown, +} diff --git a/crates/core/src/identity_predicate.rs b/crates/core/src/identity_predicate.rs new file mode 100644 index 0000000..13e80f9 --- /dev/null +++ b/crates/core/src/identity_predicate.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct IdentityPredicate { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_id: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option<String>, +} diff --git a/crates/core/src/image_buffer.rs b/crates/core/src/image_buffer.rs new file mode 100644 index 0000000..9f236bc --- /dev/null +++ b/crates/core/src/image_buffer.rs @@ -0,0 +1,117 @@ +use crate::ImageFormat; + +#[derive(Debug)] +pub struct ImageBuffer { + pub data: Vec<u8>, + pub format: ImageFormat, + pub width: u32, + pub height: u32, + pub scale_factor: f64, +} + +pub const MAX_PNG_INPUT_BYTES: usize = 64 * 1024 * 1024; +const MAX_PNG_PIXELS: u64 = 64 * 1024 * 1024; +const PNG_SIGNATURE: &[u8; 8] = b"\x89PNG\r\n\x1a\n"; +const PNG_HEADER_BYTES: usize = 33; + +pub fn parse_png_dimensions(data: &[u8]) -> Option<(u32, u32)> { + if data.len() < PNG_HEADER_BYTES || data.len() > MAX_PNG_INPUT_BYTES { + return None; + } + if data.get(..PNG_SIGNATURE.len())? != PNG_SIGNATURE { + return None; + } + if read_u32(data, 8)? != 13 || data.get(12..16)? != b"IHDR" { + return None; + } + let width = read_u32(data, 16)?; + let height = read_u32(data, 20)?; + let depth = *data.get(24)?; + let color = *data.get(25)?; + valid_png_header(width, height, depth, color, data.get(26..29)?).then_some((width, height)) +} + +fn read_u32(data: &[u8], offset: usize) -> Option<u32> { + Some(u32::from_be_bytes( + data.get(offset..offset.checked_add(4)?)?.try_into().ok()?, + )) +} + +fn valid_png_header(width: u32, height: u32, depth: u8, color: u8, tail: &[u8]) -> bool { + let valid_depth = match color { + 0 => matches!(depth, 1 | 2 | 4 | 8 | 16), + 2 | 4 | 6 => matches!(depth, 8 | 16), + 3 => matches!(depth, 1 | 2 | 4 | 8), + _ => false, + }; + let pixels = u64::from(width).checked_mul(u64::from(height)); + width > 0 + && height > 0 + && pixels.is_some_and(|pixels| pixels <= MAX_PNG_PIXELS) + && valid_depth + && matches!(tail, [0, 0, 0 | 1]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn one_pixel_png() -> Vec<u8> { + vec![ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, + 8, 4, 0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 100, 248, 15, + 0, 1, 5, 1, 1, 39, 24, 227, 102, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ] + } + + #[test] + fn decodes_valid_png_before_returning_dimensions() { + let bytes = one_pixel_png(); + + assert_eq!(parse_png_dimensions(&bytes), Some((1, 1))); + } + + #[test] + fn accepts_adam7_interlaced_png_header() { + let mut bytes = one_pixel_png(); + bytes[28] = 1; + + assert_eq!(parse_png_dimensions(&bytes), Some((1, 1))); + } + + #[test] + fn rejects_unknown_png_interlace_method() { + let mut bytes = one_pixel_png(); + bytes[28] = 2; + + assert_eq!(parse_png_dimensions(&bytes), None); + } + + #[test] + fn rejects_undersized_buffer() { + assert_eq!(parse_png_dimensions(&[1, 2, 3]), None); + } + + #[test] + fn rejects_buffer_with_wrong_signature() { + let mut bytes = one_pixel_png(); + bytes[0] = 0x00; + + assert_eq!(parse_png_dimensions(&bytes), None); + } + + #[test] + fn metadata_parse_does_not_duplicate_platform_payload_validation() { + let mut bytes = one_pixel_png(); + bytes[45] ^= 0xff; + + assert_eq!(parse_png_dimensions(&bytes), Some((1, 1))); + } + + #[test] + fn rejects_header_only_png() { + let bytes = one_pixel_png(); + + assert_eq!(parse_png_dimensions(&bytes[..24]), None); + } +} diff --git a/crates/core/src/image_format.rs b/crates/core/src/image_format.rs new file mode 100644 index 0000000..e83da1f --- /dev/null +++ b/crates/core/src/image_format.rs @@ -0,0 +1,14 @@ +#[derive(Debug)] +pub enum ImageFormat { + Png, + Jpg, +} + +impl ImageFormat { + pub fn as_str(&self) -> &'static str { + match self { + ImageFormat::Png => "png", + ImageFormat::Jpg => "jpg", + } + } +} diff --git a/crates/core/src/interaction_lease.rs b/crates/core/src/interaction_lease.rs new file mode 100644 index 0000000..26eb658 --- /dev/null +++ b/crates/core/src/interaction_lease.rs @@ -0,0 +1,208 @@ +#[cfg(unix)] +use std::path::PathBuf; + +use crate::{AdapterError, Deadline}; +#[cfg(unix)] +use crate::{ErrorCode, file_lock::FileLock}; + +pub struct InteractionLease { + _guard: Option<Box<dyn Send + Sync>>, + #[cfg(unix)] + file_guard: Option<FileLock>, + #[cfg(unix)] + _process_guard: Option<crate::process_lease_guard::ProcessLeaseGuard>, + deadline: Deadline, + contention_count: u64, +} + +#[cfg(unix)] +pub const INTERACTION_LEASE_FD_ENV: &str = "AGENT_DESKTOP_INTERACTION_LEASE_FD"; + +impl InteractionLease { + pub fn guarded( + deadline: Deadline, + guard: impl Send + Sync + 'static, + ) -> Result<Self, AdapterError> { + Ok(Self { + _guard: Some(Box::new(guard)), + #[cfg(unix)] + file_guard: None, + #[cfg(unix)] + _process_guard: None, + deadline, + contention_count: 0, + }) + } + + pub fn deadline(&self) -> Deadline { + self.deadline + } + + pub fn contention_count(&self) -> u64 { + self.contention_count + } + + #[cfg(unix)] + pub fn duplicate_inheritable_fd(&self) -> Result<std::os::fd::OwnedFd, AdapterError> { + self.file_guard + .as_ref() + .ok_or_else(|| AdapterError::not_supported("interaction lease descriptor inheritance"))? + .duplicate_inheritable() + } + + #[cfg(unix)] + fn from_file_lock( + deadline: Deadline, + file_guard: FileLock, + process_guard: crate::process_lease_guard::ProcessLeaseGuard, + ) -> Self { + let contention_count = file_guard + .contention_count() + .saturating_add(process_guard.contention_count()); + Self { + _guard: None, + file_guard: Some(file_guard), + _process_guard: Some(process_guard), + deadline, + contention_count, + } + } + + #[cfg(test)] + pub(crate) fn guarded_with_contention( + deadline: Deadline, + guard: impl Send + Sync + 'static, + contention_count: u64, + ) -> Self { + Self { + _guard: Some(Box::new(guard)), + #[cfg(unix)] + file_guard: None, + #[cfg(unix)] + _process_guard: None, + deadline, + contention_count, + } + } +} + +#[cfg(unix)] +pub fn acquire_unix_interaction_lease( + deadline: Deadline, +) -> Result<InteractionLease, AdapterError> { + let process_guard = crate::process_lease_guard::ProcessLeaseGuard::acquire(deadline)?; + let path = interaction_lock_path_at(std::path::Path::new("/tmp"))?; + let lock = FileLock::acquire(&path, deadline, "desktop interaction lease")?; + Ok(InteractionLease::from_file_lock( + deadline, + lock, + process_guard, + )) +} + +#[cfg(unix)] +pub fn adopt_inherited_unix_interaction_lease( + raw_fd: std::os::fd::RawFd, + deadline: Deadline, +) -> Result<InteractionLease, AdapterError> { + adopt_inherited_unix_interaction_lease_at(raw_fd, deadline, std::path::Path::new("/tmp")) +} + +#[cfg(unix)] +fn interaction_lock_path_at(root: &std::path::Path) -> Result<PathBuf, AdapterError> { + let uid = unsafe { libc::geteuid() }; + let directory = root.join(format!("agent-desktop-{uid}")); + ensure_unix_runtime_directory(&directory, uid)?; + Ok(directory.join("interaction.lock")) +} + +#[cfg(all(unix, test))] +fn acquire_unix_interaction_lease_at( + deadline: Deadline, + root: &std::path::Path, +) -> Result<InteractionLease, AdapterError> { + let process_guard = crate::process_lease_guard::ProcessLeaseGuard::acquire(deadline)?; + let path = interaction_lock_path_at(root)?; + let lock = FileLock::acquire(&path, deadline, "test desktop interaction lease")?; + Ok(InteractionLease::from_file_lock( + deadline, + lock, + process_guard, + )) +} + +#[cfg(unix)] +fn adopt_inherited_unix_interaction_lease_at( + raw_fd: std::os::fd::RawFd, + deadline: Deadline, + root: &std::path::Path, +) -> Result<InteractionLease, AdapterError> { + let process_guard = crate::process_lease_guard::ProcessLeaseGuard::acquire(deadline)?; + let uid = unsafe { libc::geteuid() }; + let directory = root.join(format!("agent-desktop-{uid}")); + validate_unix_runtime_directory(&directory, uid)?; + let path = directory.join("interaction.lock"); + let lock = FileLock::adopt_inherited( + raw_fd, + &path, + deadline, + "inherited desktop interaction lease", + )?; + Ok(InteractionLease::from_file_lock( + deadline, + lock, + process_guard, + )) +} + +#[cfg(unix)] +fn ensure_unix_runtime_directory( + directory: &std::path::Path, + uid: u32, +) -> Result<(), AdapterError> { + use std::os::unix::fs::DirBuilderExt; + + match std::fs::symlink_metadata(directory) { + Ok(_) => validate_unix_runtime_directory(directory, uid), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + match std::fs::DirBuilder::new().mode(0o700).create(directory) { + Ok(()) => validate_unix_runtime_directory(directory, uid), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + validate_unix_runtime_directory(directory, uid) + } + Err(error) => Err(runtime_io_error(error)), + } + } + Err(error) => Err(runtime_io_error(error)), + } +} + +#[cfg(unix)] +fn validate_unix_runtime_directory( + directory: &std::path::Path, + uid: u32, +) -> Result<(), AdapterError> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let metadata = std::fs::symlink_metadata(directory).map_err(runtime_io_error)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() || metadata.uid() != uid { + return Err(AdapterError::new( + ErrorCode::Internal, + "Interaction runtime directory is not a private user-owned directory", + )); + } + if metadata.permissions().mode() & 0o077 != 0 { + std::fs::set_permissions(directory, std::fs::Permissions::from_mode(0o700)) + .map_err(runtime_io_error)?; + } + Ok(()) +} + +#[cfg(unix)] +fn runtime_io_error(error: std::io::Error) -> AdapterError { + AdapterError::new(ErrorCode::Internal, error.to_string()) +} + +#[cfg(test)] +#[path = "interaction_lease_tests.rs"] +mod tests; diff --git a/crates/core/src/interaction_lease_tests.rs b/crates/core/src/interaction_lease_tests.rs new file mode 100644 index 0000000..054fb6a --- /dev/null +++ b/crates/core/src/interaction_lease_tests.rs @@ -0,0 +1,283 @@ +#![cfg(unix)] + +use std::time::Duration; + +use super::*; + +fn test_root(label: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "agent-desktop-interaction-{label}-{}", + crate::refs::new_snapshot_id() + )); + std::fs::create_dir_all(&root).unwrap(); + root +} + +#[test] +fn a_second_lease_times_out_and_drop_releases_the_first() { + let root = test_root("contention"); + let first = acquire_unix_interaction_lease_at(Deadline::after(100).unwrap(), &root).unwrap(); + let deadline = Deadline::after(10).unwrap(); + let error = match acquire_unix_interaction_lease_at(deadline, &root) { + Ok(_) => panic!("contended lease unexpectedly acquired"), + Err(error) => error, + }; + assert_eq!(error.code, ErrorCode::Timeout); + assert!( + error.details.as_ref().unwrap()["contention_count"] + .as_u64() + .unwrap() + >= 1 + ); + assert!(deadline.elapsed() < Duration::from_secs(1)); + drop(first); + acquire_unix_interaction_lease_at(Deadline::after(100).unwrap(), &root).unwrap(); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn home_changes_do_not_change_the_physical_interaction_lock() { + let root = test_root("home"); + let first_home = crate::refs_test_support::HomeGuard::new(); + let first = acquire_unix_interaction_lease_at(Deadline::after(100).unwrap(), &root).unwrap(); + drop(first_home); + let _second_home = crate::refs_test_support::HomeGuard::new(); + let error = match acquire_unix_interaction_lease_at(Deadline::after(10).unwrap(), &root) { + Ok(_) => panic!("different HOME bypassed the interaction lock"), + Err(error) => error, + }; + assert_eq!(error.code, ErrorCode::Timeout); + drop(first); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn subprocess_holder() { + let Some(ready_path) = std::env::var_os("AGENT_DESKTOP_LOCK_HELPER_READY") else { + return; + }; + let root = PathBuf::from(std::env::var_os("AGENT_DESKTOP_LOCK_HELPER_ROOT").unwrap()); + let _lease = acquire_unix_interaction_lease_at(Deadline::after(5_000).unwrap(), &root).unwrap(); + std::fs::write(ready_path, b"ready").unwrap(); + std::thread::sleep(Duration::from_secs(5)); +} + +#[test] +fn subprocess_adopted_holder() { + use std::os::fd::RawFd; + + let Some(ready_path) = std::env::var_os("AGENT_DESKTOP_ADOPT_HELPER_READY") else { + return; + }; + let root = PathBuf::from(std::env::var_os("AGENT_DESKTOP_ADOPT_HELPER_ROOT").unwrap()); + let raw_fd = std::env::var("AGENT_DESKTOP_ADOPT_HELPER_FD") + .unwrap() + .parse::<RawFd>() + .unwrap(); + let _lease = + adopt_inherited_unix_interaction_lease_at(raw_fd, Deadline::after(5_000).unwrap(), &root) + .unwrap(); + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("interaction_lease::tests::subprocess_inherited_fd_is_closed") + .arg("--nocapture") + .env("AGENT_DESKTOP_CLOSED_HELPER_FD", raw_fd.to_string()) + .status() + .unwrap(); + assert!(status.success()); + std::fs::write(ready_path, b"ready").unwrap(); + std::thread::sleep(Duration::from_secs(5)); +} + +#[test] +fn subprocess_inherited_fd_is_closed() { + let Some(raw_fd) = std::env::var_os("AGENT_DESKTOP_CLOSED_HELPER_FD") else { + return; + }; + let raw_fd = raw_fd + .to_string_lossy() + .parse::<std::os::fd::RawFd>() + .unwrap(); + assert_eq!(unsafe { libc::fcntl(raw_fd, libc::F_GETFD) }, -1); +} + +#[test] +fn different_home_subprocess_contends_and_crash_releases() { + let token = crate::refs::new_snapshot_id(); + let directory = std::env::temp_dir().join(format!("agent-desktop-lock-proof-{token}")); + let interaction_root = directory.join("runtime-root"); + let other_home = directory.join("other-home"); + let ready = directory.join("ready"); + std::fs::create_dir_all(&other_home).unwrap(); + std::fs::create_dir_all(&interaction_root).unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("interaction_lease::tests::subprocess_holder") + .arg("--nocapture") + .env("HOME", &other_home) + .env("AGENT_DESKTOP_LOCK_HELPER_READY", &ready) + .env("AGENT_DESKTOP_LOCK_HELPER_ROOT", &interaction_root) + .spawn() + .unwrap(); + let ready_deadline = std::time::Instant::now() + Duration::from_secs(2); + while !ready.is_file() && std::time::Instant::now() < ready_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + ready.is_file(), + "subprocess did not acquire interaction lease" + ); + let error = + match acquire_unix_interaction_lease_at(Deadline::after(25).unwrap(), &interaction_root) { + Ok(_) => panic!("different HOME subprocess bypassed interaction lock"), + Err(error) => error, + }; + assert_eq!(error.code, ErrorCode::Timeout); + let _ = child.kill(); + reap_child(&mut child, Duration::from_secs(2)); + acquire_unix_interaction_lease_at(Deadline::after(1_000).unwrap(), &interaction_root).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn inherited_descriptor_survives_parent_drop_and_crash_releases() { + use std::os::fd::AsRawFd; + + let root = test_root("inherited"); + let ready = root.join("adopted-ready"); + let lease = acquire_unix_interaction_lease_at(Deadline::after(1_000).unwrap(), &root).unwrap(); + let inherited = lease.duplicate_inheritable_fd().unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("interaction_lease::tests::subprocess_adopted_holder") + .arg("--nocapture") + .env("AGENT_DESKTOP_ADOPT_HELPER_READY", &ready) + .env("AGENT_DESKTOP_ADOPT_HELPER_ROOT", &root) + .env( + "AGENT_DESKTOP_ADOPT_HELPER_FD", + inherited.as_raw_fd().to_string(), + ) + .spawn() + .unwrap(); + drop(inherited); + drop(lease); + let ready_deadline = std::time::Instant::now() + Duration::from_secs(2); + while !ready.is_file() && std::time::Instant::now() < ready_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + ready.is_file(), + "subprocess did not adopt interaction lease" + ); + let error = acquire_unix_interaction_lease_at(Deadline::after(25).unwrap(), &root) + .err() + .expect("adopted descriptor must retain the lease"); + assert_eq!(error.code, ErrorCode::Timeout); + child.kill().unwrap(); + reap_child(&mut child, Duration::from_secs(2)); + acquire_unix_interaction_lease_at(Deadline::after(1_000).unwrap(), &root).unwrap(); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn inherited_descriptor_must_match_canonical_lock_identity() { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + + let root = test_root("inherited-mismatch"); + let lease = acquire_unix_interaction_lease_at(Deadline::after(1_000).unwrap(), &root).unwrap(); + let unrelated_path = root.join("unrelated.lock"); + let unrelated = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .open(&unrelated_path) + .unwrap(); + drop(lease); + let error = adopt_inherited_unix_interaction_lease_at( + unrelated.as_raw_fd(), + Deadline::after(100).unwrap(), + &root, + ) + .err() + .expect("an unrelated descriptor must be rejected"); + assert_eq!(error.code, ErrorCode::PolicyDenied); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn inherited_descriptor_serializes_threads_in_one_host_process() { + use std::os::fd::AsRawFd; + + let root = test_root("inherited-threads"); + let original = + acquire_unix_interaction_lease_at(Deadline::after(1_000).unwrap(), &root).unwrap(); + let inherited = original.duplicate_inheritable_fd().unwrap(); + drop(original); + let first = adopt_inherited_unix_interaction_lease_at( + inherited.as_raw_fd(), + Deadline::after(1_000).unwrap(), + &root, + ) + .unwrap(); + let raw_fd = inherited.as_raw_fd(); + let thread_root = root.clone(); + let contender = std::thread::spawn(move || { + adopt_inherited_unix_interaction_lease_at( + raw_fd, + Deadline::after(25).unwrap(), + &thread_root, + ) + .err() + .expect("concurrent adoption must not share a critical section") + }); + + let error = contender.join().unwrap(); + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!( + error.details.as_ref().unwrap()["kind"], + "interaction_process_lock_timeout" + ); + drop(first); + adopt_inherited_unix_interaction_lease_at( + inherited.as_raw_fd(), + Deadline::after(1_000).unwrap(), + &root, + ) + .unwrap(); + std::fs::remove_dir_all(root).unwrap(); +} + +fn reap_child(child: &mut std::process::Child, timeout: Duration) { + let deadline = std::time::Instant::now() + timeout; + loop { + if child.try_wait().unwrap().is_some() { + return; + } + assert!( + std::time::Instant::now() < deadline, + "lock-holder subprocess did not exit after kill" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[cfg(unix)] +#[test] +fn runtime_directory_is_private_and_rejects_symlinks() { + use std::os::unix::fs::PermissionsExt; + + let token = crate::refs::new_snapshot_id(); + let root = std::env::temp_dir().join(format!("agent-desktop-runtime-proof-{token}")); + let runtime = root.join("runtime"); + std::fs::create_dir_all(&root).unwrap(); + let uid = unsafe { libc::geteuid() }; + ensure_unix_runtime_directory(&runtime, uid).unwrap(); + let mode = std::fs::metadata(&runtime).unwrap().permissions().mode(); + assert_eq!(mode & 0o077, 0); + std::fs::remove_dir(&runtime).unwrap(); + std::os::unix::fs::symlink(&root, &runtime).unwrap(); + assert!(ensure_unix_runtime_directory(&runtime, uid).is_err()); + std::fs::remove_dir_all(&root).unwrap(); +} diff --git a/crates/core/src/interaction_policy.rs b/crates/core/src/interaction_policy.rs index 6c33fe8..140ca57 100644 --- a/crates/core/src/interaction_policy.rs +++ b/crates/core/src/interaction_policy.rs @@ -28,10 +28,10 @@ impl InteractionPolicy { } } - /// Returns the least-restrictive policy that satisfies both `self` and - /// `other`. Used by FFI callers that supply an explicit caller policy: the - /// result is always at least as permissive as the action's CLI base, and - /// at least as permissive as what the caller requested. + pub fn is_headed(self) -> bool { + self.allow_focus_steal && self.allow_cursor_move + } + pub fn join(self, other: InteractionPolicy) -> InteractionPolicy { InteractionPolicy { allow_focus_steal: self.allow_focus_steal || other.allow_focus_steal, diff --git a/crates/core/src/key_combo.rs b/crates/core/src/key_combo.rs new file mode 100644 index 0000000..4f99023 --- /dev/null +++ b/crates/core/src/key_combo.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +use crate::Modifier; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KeyCombo { + pub key: String, + pub modifiers: Vec<Modifier>, +} diff --git a/crates/core/src/launch_options.rs b/crates/core/src/launch_options.rs new file mode 100644 index 0000000..110d113 --- /dev/null +++ b/crates/core/src/launch_options.rs @@ -0,0 +1,32 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; + +/// Options accepted by `launch_app`. `attach_if_running` defaults to `true`, +/// which preserves the behavior of attaching to an already-running instance +/// instead of failing. Set it to `false` to +/// require a fresh launch: the adapter then fails with a structured error +/// naming the running pid instead of attaching. +#[derive(Debug, Clone)] +pub struct LaunchOptions { + pub args: Vec<String>, + pub env: BTreeMap<String, String>, + pub cwd: Option<PathBuf>, + pub timeout_ms: u64, + pub attach_if_running: bool, +} + +impl Default for LaunchOptions { + fn default() -> Self { + Self { + args: Vec::new(), + env: BTreeMap::new(), + cwd: None, + timeout_ms: 5_000, + attach_if_running: true, + } + } +} + +#[cfg(test)] +#[path = "launch_options_tests.rs"] +mod tests; diff --git a/crates/core/src/launch_options_tests.rs b/crates/core/src/launch_options_tests.rs new file mode 100644 index 0000000..831e873 --- /dev/null +++ b/crates/core/src/launch_options_tests.rs @@ -0,0 +1,26 @@ +use super::*; + +#[test] +fn default_preserves_attach_if_running_semantics() { + let options = LaunchOptions::default(); + + assert!( + options.attach_if_running, + "default LaunchOptions must attach to an already-running instance, matching \ + launch_app's historical behavior; a derived Default would silently flip this to \ + false and turn every unmodified caller into a --no-attach launch" + ); + assert!(options.args.is_empty()); + assert!(options.env.is_empty()); + assert!(options.cwd.is_none()); +} + +#[test] +fn explicit_no_attach_overrides_the_default() { + let options = LaunchOptions { + attach_if_running: false, + ..Default::default() + }; + + assert!(!options.attach_if_running); +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 1cf76e5..d572aec 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,63 +1,261 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + +mod accname; pub mod action; pub mod action_request; pub mod action_result; pub mod action_step; pub mod action_step_outcome; pub(crate) mod actionability; -pub mod adapter; +mod adapter; +mod adapter_error; +mod adapter_session; +mod app_error; +mod app_info; +mod app_lookup; pub mod capability; +mod clipboard_content; +mod clipboard_format; pub mod commands; +mod containment_predicate; pub mod context; +mod deadline; +mod delivery_disposition; +mod delivery_semantics; +mod direction; +mod dismiss_all_notifications_request; +mod dismiss_notification_request; +pub mod display_info; +mod drag_params; +mod element_identifier; pub mod element_state; -pub mod error; +mod error_code; +mod event_kind; +mod file_lock; +mod headed_focus; +mod headed_requirement; pub mod hints; +pub mod hit_test; +mod identifier_kind; +mod identity_predicate; +mod image_buffer; +mod image_format; +mod interaction_lease; pub mod interaction_policy; -pub mod node; -pub mod notification; +mod key_combo; +pub mod launch_options; +pub mod live_element; +mod live_identity; +mod live_locator; +pub(crate) mod locator; +mod modifier; +mod mouse_button; +mod mouse_click_count; +mod mouse_event; +mod mouse_event_kind; +mod name_evidence; +pub mod native_handle; +mod node; +mod node_identity; +mod node_match_context; +mod node_presentation; +mod notification_action_request; +mod notification_filter; +mod notification_identity; +mod notification_info; pub mod output; pub mod permission_report; pub mod permission_state; +mod point; +mod private_file; +mod private_file_parent; +mod process_id; +mod process_identity; +#[cfg(unix)] +mod process_lease_guard; +pub mod process_state; +mod recovery_hint; +mod rect; pub mod ref_action; +mod ref_action_context; +mod ref_action_poll; +mod ref_action_poll_state; +mod ref_action_single; +mod ref_action_wait; +mod ref_action_wait_context; +mod ref_action_wait_evidence; +mod ref_action_wait_support; pub mod ref_alloc; +mod ref_alloc_config; +mod ref_alloc_options; +mod ref_alloc_scope; +mod ref_alloc_source; +mod ref_allocation; +mod ref_capabilities; +mod ref_entry; +mod ref_entry_identity; +mod ref_geometry; pub mod ref_identity; +mod ref_identity_match; +mod ref_process; +mod ref_resolve_deadline; +mod ref_scope; +mod ref_source; +mod ref_token; pub mod refs; mod refs_lock; pub mod refs_store; #[cfg(test)] mod refs_test_support; -pub(crate) mod resolved_element; +mod refs_validate; +mod renderer_accessibility; +mod resolve_attempt_outcome; +mod retry_disposition; +mod retryability; +mod role; pub mod roles; +pub mod screenshot_target; pub(crate) mod search_text; pub mod session; +mod session_affinity; +mod signal_baseline; +mod signal_completeness; +mod signal_filter; +pub(crate) mod signals; pub mod snapshot; pub mod snapshot_ref; +pub mod snapshot_surface; +pub mod state; +mod state_predicate; +pub mod step_mechanism; +mod surface_info; +mod surface_signal; pub(crate) mod trace; +mod trace_artifact_budget; pub(crate) mod trace_artifacts; pub mod trace_read; pub mod trace_sanitize; +mod trace_state; +pub mod tree_options; +mod ui_event; +mod wait_budget; +pub mod window_filter; +mod window_focus; +mod window_info; mod window_lookup; +mod window_op; +mod window_state; -pub use action::{ - Action, Direction, DragParams, KeyCombo, Modifier, MouseButton, MouseEvent, MouseEventKind, - Point, WindowOp, -}; +pub use accname::{compute_description, compute_name}; +pub use action::Action; pub use action_request::ActionRequest; pub use action_result::ActionResult; pub use action_step::ActionStep; pub use action_step_outcome::ActionStepOutcome; -pub use adapter::{ - ImageBuffer, ImageFormat, NativeHandle, PlatformAdapter, ScreenshotTarget, TreeOptions, - WindowFilter, -}; +pub use adapter::PlatformAdapter; +pub use adapter::actions::ActionOps; +pub use adapter::input::InputOps; +pub use adapter::observation::ObservationOps; +pub use adapter::system::SystemOps; +pub use adapter_error::AdapterError; +pub use adapter_session::AdapterSession; +pub use app_error::AppError; +pub use app_info::AppInfo; +pub use clipboard_content::ClipboardContent; +pub use clipboard_format::ClipboardFormat; +pub use containment_predicate::ContainmentPredicate; pub use context::{CommandContext, WaitSelector}; +pub use deadline::{DEFAULT_OPERATION_TIMEOUT_MS, Deadline}; +pub use delivery_disposition::DeliveryDisposition; +pub use delivery_semantics::DeliverySemantics; +pub use direction::Direction; +pub use dismiss_all_notifications_request::DismissAllNotificationsRequest; +pub use dismiss_notification_request::DismissNotificationRequest; +pub use display_info::DisplayInfo; +pub use drag_params::{DragParams, MAX_DRAG_DROP_DELAY_MS, MAX_DRAG_DURATION_MS, MAX_DRAG_STEPS}; +pub use element_identifier::ElementIdentifier; pub use element_state::ElementState; -pub use error::{AdapterError, AppError, ErrorCode}; +pub use error_code::ErrorCode; +pub use event_kind::EventKind; +pub use headed_requirement::HeadedRequirement; +pub use hit_test::HitTestResult; +pub use identifier_kind::IdentifierKind; +pub use identity_predicate::IdentityPredicate; +pub use image_buffer::{ImageBuffer, MAX_PNG_INPUT_BYTES, parse_png_dimensions}; +pub use image_format::ImageFormat; +pub use interaction_lease::InteractionLease; +#[cfg(unix)] +pub use interaction_lease::{ + INTERACTION_LEASE_FD_ENV, acquire_unix_interaction_lease, + adopt_inherited_unix_interaction_lease, +}; pub use interaction_policy::InteractionPolicy; -pub use node::{AccessibilityNode, AppInfo, Rect, WindowInfo}; -pub use notification::{NotificationFilter, NotificationInfo}; +pub use key_combo::KeyCombo; +pub use live_element::LiveElement; +pub use live_identity::LiveIdentity; +pub use live_locator::{ + EvidenceRequirements, IdentifierEvidence, LocatorCardinality, LocatorEvidence, LocatorField, + LocatorIdentifierStats, LocatorLimitStats, LocatorMaterialization, LocatorReadCounts, + LocatorReadHealth, LocatorReadStats, LocatorRefEvidence, LocatorResolution, + LocatorResolveRequest, LocatorSelection, LocatorSemanticReadStats, LocatorStats, + LocatorTraversalStats, ObservationBudget, ObservationRequest, ObservationRoot, + ObservationSource, ObservedSubtree, ObservedTree, classify_query_result, evaluate_locator_tree, + find_first_entry, require_unique, resolve_query, +}; +pub use locator::{ + LocatorQuery, accessibility_node_matches, node_context, node_matches, role_matches, +}; +pub use modifier::Modifier; +pub use mouse_button::MouseButton; +pub use mouse_click_count::{MAX_MOUSE_CLICK_COUNT, validate_mouse_click_count}; +pub use mouse_event::MouseEvent; +pub use mouse_event_kind::MouseEventKind; +pub use name_evidence::NameEvidence; +pub use native_handle::NativeHandle; +pub use node::AccessibilityNode; +pub use node_identity::NodeIdentity; +pub use node_match_context::NodeMatchContext; +pub use node_presentation::NodePresentation; +pub use notification_action_request::NotificationActionRequest; +pub use notification_filter::NotificationFilter; +pub use notification_identity::NotificationIdentity; +pub use notification_info::NotificationInfo; pub use output::{ErrorPayload, Response}; pub use permission_report::PermissionReport; pub use permission_state::PermissionState; -pub use refs::{RefEntry, RefMap}; +pub use point::Point; +pub use process_id::ProcessId; +pub use process_identity::ProcessIdentity; +pub use recovery_hint::RecoveryHint; +pub use rect::Rect; +pub use ref_capabilities::RefCapabilities; +pub use ref_entry::RefEntry; +pub use ref_entry_identity::RefEntryIdentity; +pub use ref_geometry::RefGeometry; +pub use ref_identity_match::IdentityMatch; +pub use ref_process::RefProcess; +pub use ref_scope::RefScope; +pub use ref_source::RefSource; +pub use refs::RefMap; pub use refs_store::RefStore; +pub use retry_disposition::RetryDisposition; +pub use role::Role; +pub use screenshot_target::ScreenshotTarget; +pub use session_affinity::SessionAffinity; +pub use signal_baseline::SignalBaseline; +pub use signal_completeness::SignalCompleteness; +pub use signal_filter::SignalFilter; +pub use signals::diff_signals; +pub use snapshot_surface::SnapshotSurface; +pub use state_predicate::StatePredicate; +pub use step_mechanism::StepMechanism; +pub use surface_info::SurfaceInfo; +pub use surface_signal::SurfaceSignal; pub use trace_sanitize::sanitize_trace_value; +pub use tree_options::TreeOptions; +pub use ui_event::UiEvent; +pub use wait_budget::{MAX_WAIT_TIMEOUT_MS, wait_timeout_duration}; +pub use window_filter::WindowFilter; +pub use window_info::WindowInfo; +pub use window_op::WindowOp; +pub use window_state::WindowState; diff --git a/crates/core/src/live_element.rs b/crates/core/src/live_element.rs new file mode 100644 index 0000000..7c29424 --- /dev/null +++ b/crates/core/src/live_element.rs @@ -0,0 +1,10 @@ +use crate::{LiveIdentity, Rect, element_state::ElementState}; + +#[derive(Debug, Clone)] +pub struct LiveElement { + pub identity: LiveIdentity, + pub state: ElementState, + pub states_complete: bool, + pub bounds: Option<Rect>, + pub available_actions: Vec<String>, +} diff --git a/crates/core/src/live_identity.rs b/crates/core/src/live_identity.rs new file mode 100644 index 0000000..b495b56 --- /dev/null +++ b/crates/core/src/live_identity.rs @@ -0,0 +1,8 @@ +use crate::{IdentifierEvidence, LocatorField}; + +#[derive(Debug, Clone, PartialEq)] +pub struct LiveIdentity { + pub name: LocatorField<String>, + pub description: LocatorField<String>, + pub identifiers: IdentifierEvidence, +} diff --git a/crates/core/src/live_locator/cardinality.rs b/crates/core/src/live_locator/cardinality.rs new file mode 100644 index 0000000..4f5b9b2 --- /dev/null +++ b/crates/core/src/live_locator/cardinality.rs @@ -0,0 +1,72 @@ +use super::{LocatorCardinality, LocatorMatch, LocatorResolution}; +use crate::{AdapterError, AppError, ErrorCode}; +use serde_json::json; + +pub fn classify_query_result(resolution: &LocatorResolution) -> LocatorCardinality { + let observed = resolution.meta.total_matches; + if observed >= 2 { + return LocatorCardinality::Many { + observed, + exact: resolution.meta.complete, + }; + } + if !resolution.meta.complete { + return LocatorCardinality::Incomplete { observed }; + } + if observed == 0 { + LocatorCardinality::Zero + } else { + LocatorCardinality::One + } +} + +pub fn require_unique(resolution: LocatorResolution) -> Result<LocatorMatch, AppError> { + match classify_query_result(&resolution) { + LocatorCardinality::Zero => Err(AppError::Adapter( + AdapterError::new( + ErrorCode::ElementNotFound, + "Locator query matched no elements", + ) + .with_suggestion("Use a broader locator or inspect roles_present"), + )), + LocatorCardinality::One => resolution.matches.into_iter().next().ok_or_else(|| { + AppError::Adapter(AdapterError::internal( + "unique locator resolution did not retain its match", + )) + }), + LocatorCardinality::Many { observed, exact } => { + let candidates = resolution + .matches + .iter() + .take(10) + .map(|candidate| { + json!({ + "document_order": candidate.document_order, + "name": candidate.data.name, + "ref_id": candidate.data.ref_id, + "role": candidate.data.role, + }) + }) + .collect::<Vec<_>>(); + Err(AppError::Adapter( + AdapterError::ambiguous_target(format!( + "Locator query matched at least {observed} elements" + )) + .with_details(json!({ + "candidate_count": observed, + "candidate_count_exact": exact, + "candidates": candidates, + "query_stats": resolution.stats, + })), + )) + } + LocatorCardinality::Incomplete { observed } => Err(AppError::Adapter( + AdapterError::timeout("Locator traversal could not prove a unique result") + .with_details(json!({ + "kind": "locator_incomplete", + "observed_matches": observed, + "query_stats": resolution.stats, + })), + )), + } +} diff --git a/crates/core/src/live_locator/compiled_clause.rs b/crates/core/src/live_locator/compiled_clause.rs new file mode 100644 index 0000000..d16bced --- /dev/null +++ b/crates/core/src/live_locator/compiled_clause.rs @@ -0,0 +1,30 @@ +use crate::locator::LocatorQuery; + +pub(crate) struct CompiledClause<'a> { + pub query: &'a LocatorQuery, + pub has: Option<usize>, + pub has_not: Option<usize>, +} + +pub(crate) fn compile_clauses<'a>( + query: &'a LocatorQuery, + clauses: &mut Vec<CompiledClause<'a>>, +) -> usize { + let has = query + .containment + .has + .as_deref() + .map(|nested| compile_clauses(nested, clauses)); + let has_not = query + .containment + .has_not + .as_deref() + .map(|nested| compile_clauses(nested, clauses)); + let index = clauses.len(); + clauses.push(CompiledClause { + query, + has, + has_not, + }); + index +} diff --git a/crates/core/src/live_locator/evaluate.rs b/crates/core/src/live_locator/evaluate.rs new file mode 100644 index 0000000..0d8b857 --- /dev/null +++ b/crates/core/src/live_locator/evaluate.rs @@ -0,0 +1,254 @@ +use super::{ + LocatorField, LocatorMatch, LocatorMaterialization, LocatorResolution, LocatorResolutionMeta, + LocatorResolveRequest, ObservedTree, + compiled_clause::{CompiledClause, compile_clauses}, + evaluation_buffers::EvaluationBuffers, + match_verdict::MatchVerdict, + materialize::{materialize_refmap, ref_entry}, + predicate::{normalize_query, self_text_verdict, self_verdict}, + select::{match_data, selected_indices}, + selection_completeness::first_is_authoritative, + tree_order::validated_postorder, + validate::{validate_query, validate_request}, +}; +use crate::{AdapterError, locator::LocatorQuery}; +use std::collections::BTreeSet; + +pub fn evaluate_locator_tree( + mut tree: ObservedTree, + query: &LocatorQuery, + request: &LocatorResolveRequest, +) -> Result<LocatorResolution, AdapterError> { + validate_query(query)?; + validate_request(request)?; + let normalized = normalize_query(query); + let mut clauses = Vec::new(); + let root_clause = compile_clauses(&normalized, &mut clauses); + let (postorder, parents) = validated_postorder(&tree)?; + let cells = tree + .nodes + .len() + .checked_mul(clauses.len()) + .ok_or_else(|| AdapterError::internal("locator evaluation matrix is too large"))?; + let mut matches = vec![MatchVerdict::NoMatch; cells]; + let mut subtree_matches = vec![MatchVerdict::NoMatch; cells]; + let mut subtree_text = vec![MatchVerdict::NoMatch; cells]; + let mut stats = std::mem::take(&mut tree.stats); + stats.evaluation.query_clause_count = clauses.len() as u32; + stats.evaluation.text_clause_count = clauses + .iter() + .filter(|clause| clause.query.has_text.is_some()) + .count() as u32; + + { + let mut buffers = EvaluationBuffers { + matches: matches.as_mut_slice(), + subtree_matches: subtree_matches.as_mut_slice(), + subtree_text: subtree_text.as_mut_slice(), + stats: &mut stats, + }; + for node_index in postorder { + evaluate_node(&tree, node_index, &clauses, &mut buffers); + } + } + + let mut matched_indices = Vec::new(); + let mut unknown = false; + for index in 0..tree.nodes.len() { + match matches[cell(index, root_clause, clauses.len())] { + MatchVerdict::Match => matched_indices.push(index), + MatchVerdict::Unknown => unknown = true, + MatchVerdict::NoMatch => {} + } + } + matched_indices.sort_by_key(|index| tree.nodes[*index].document_order); + stats.evaluation.matched_nodes = matched_indices.len() as u64; + let (selected, truncated) = selected_indices(&matched_indices, request.selection); + let mut complete = tree.structurally_complete && !unknown; + let refmap = match request.materialization { + LocatorMaterialization::None => None, + LocatorMaterialization::SelectedMatches => { + let (refmap, materialization_complete) = + materialize_refmap(&mut tree, &normalized, Some(&selected))?; + complete &= materialization_complete; + Some(refmap) + } + LocatorMaterialization::FullRefMap => { + let (refmap, materialization_complete) = + materialize_refmap(&mut tree, &normalized, None)?; + complete &= materialization_complete; + Some(refmap) + } + }; + let mut selected_matches = Vec::with_capacity(selected.len()); + for index in selected { + let data = match_data(&tree, index, &parents) + .ok_or_else(|| AdapterError::internal("selected locator node is missing"))?; + let node = tree + .nodes + .get(index) + .ok_or_else(|| AdapterError::internal("selected locator node is out of bounds"))?; + selected_matches.push(LocatorMatch { + data, + document_order: node.document_order, + entry: ref_entry(node, &tree.source, &normalized), + }); + } + let roles_present = if matched_indices.is_empty() { + tree.nodes + .iter() + .filter_map(|node| match &node.evidence.role { + LocatorField::Known(role) => Some(role.clone()), + LocatorField::Absent | LocatorField::Unknown => None, + }) + .collect::<BTreeSet<_>>() + .into_iter() + .collect() + } else { + Vec::new() + }; + let total_matches = u32::try_from(matched_indices.len()) + .map_err(|_| AdapterError::internal("locator match count exceeds u32"))?; + let selection_complete = complete || { + matches!(request.selection, super::LocatorSelection::First) + && selected_matches.first().is_some_and(|selected| { + let root_verdicts = (0..tree.nodes.len()) + .map(|index| matches[cell(index, root_clause, clauses.len())]) + .collect::<Vec<_>>(); + tree.nodes + .iter() + .position(|node| node.document_order == selected.document_order) + .is_some_and(|index| { + first_is_authoritative(&tree, index, &parents, &root_verdicts) + }) + }) + }; + Ok(LocatorResolution { + matches: selected_matches, + refmap, + stats, + meta: LocatorResolutionMeta { + total_matches, + complete, + selection_complete, + truncated, + roles_present, + }, + }) +} + +fn evaluate_node( + tree: &ObservedTree, + node_index: usize, + clauses: &[CompiledClause<'_>], + buffers: &mut EvaluationBuffers<'_>, +) { + let Some(node) = tree.nodes.get(node_index) else { + return; + }; + for (clause_index, clause) in clauses.iter().enumerate() { + let offset = cell(node_index, clause_index, clauses.len()); + let own = self_verdict(clause.query, &node.evidence, &mut buffers.stats.identifiers); + if own != MatchVerdict::NoMatch { + buffers.stats.evaluation.self_filter_candidates += 1; + } + let text = if clause.query.has_text.is_some() { + buffers.stats.evaluation.memo_cells_evaluated += 1; + aggregate_subtree( + tree, + node_index, + (clause_index, clauses.len()), + self_text_verdict( + clause.query.has_text.as_deref(), + &node.evidence, + clause.query.exact, + ), + buffers.subtree_text, + ) + } else { + MatchVerdict::Match + }; + buffers.subtree_text[offset] = text; + let has = clause + .has + .map(|nested| { + aggregate_descendants( + tree, + node_index, + nested, + clauses.len(), + buffers.subtree_matches, + ) + }) + .unwrap_or(MatchVerdict::Match); + let has_not = clause + .has_not + .map(|nested| { + aggregate_descendants( + tree, + node_index, + nested, + clauses.len(), + buffers.subtree_matches, + ) + .negate() + }) + .unwrap_or(MatchVerdict::Match); + let verdict = own.and(text).and(has).and(has_not); + buffers.matches[offset] = verdict; + buffers.subtree_matches[offset] = aggregate_subtree( + tree, + node_index, + (clause_index, clauses.len()), + verdict, + buffers.subtree_matches, + ); + buffers.stats.evaluation.memo_cells_evaluated += 2; + } +} + +fn aggregate_descendants( + tree: &ObservedTree, + node_index: usize, + clause_index: usize, + clause_count: usize, + matrix: &[MatchVerdict], +) -> MatchVerdict { + let Some(node) = tree.nodes.get(node_index) else { + return MatchVerdict::Unknown; + }; + let mut verdict = MatchVerdict::NoMatch; + for child in &node.children { + verdict = verdict.or(matrix[cell(*child as usize, clause_index, clause_count)]); + } + if verdict == MatchVerdict::NoMatch && !node.completeness.subtree_complete { + MatchVerdict::Unknown + } else { + verdict + } +} + +fn aggregate_subtree( + tree: &ObservedTree, + node_index: usize, + clause: (usize, usize), + own: MatchVerdict, + matrix: &[MatchVerdict], +) -> MatchVerdict { + let Some(node) = tree.nodes.get(node_index) else { + return MatchVerdict::Unknown; + }; + let mut verdict = own; + for child in &node.children { + verdict = verdict.or(matrix[cell(*child as usize, clause.0, clause.1)]); + } + if verdict == MatchVerdict::NoMatch && !node.completeness.subtree_complete { + MatchVerdict::Unknown + } else { + verdict + } +} + +fn cell(node: usize, clause: usize, clause_count: usize) -> usize { + node * clause_count + clause +} diff --git a/crates/core/src/live_locator/evaluation_buffers.rs b/crates/core/src/live_locator/evaluation_buffers.rs new file mode 100644 index 0000000..504f3a3 --- /dev/null +++ b/crates/core/src/live_locator/evaluation_buffers.rs @@ -0,0 +1,8 @@ +use super::{LocatorStats, match_verdict::MatchVerdict}; + +pub(crate) struct EvaluationBuffers<'a> { + pub matches: &'a mut [MatchVerdict], + pub subtree_matches: &'a mut [MatchVerdict], + pub subtree_text: &'a mut [MatchVerdict], + pub stats: &'a mut LocatorStats, +} diff --git a/crates/core/src/live_locator/evaluator_identifier_tests.rs b/crates/core/src/live_locator/evaluator_identifier_tests.rs new file mode 100644 index 0000000..69ca579 --- /dev/null +++ b/crates/core/src/live_locator/evaluator_identifier_tests.rs @@ -0,0 +1,102 @@ +use super::{ + IdentifierEvidence, LocatorMaterialization, LocatorResolveRequest, LocatorSelection, + evaluate_locator_tree, +}; +use crate::{ + locator::{IdentityPredicate, LocatorQuery}, + search_text, +}; + +use super::test_support::{evidence, node, tree}; + +fn request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + } +} + +fn native_id_query(role: Option<&str>) -> LocatorQuery { + LocatorQuery { + identity: IdentityPredicate { + role: role.map(search_text::normalize), + native_id: Some("checkout".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + } +} + +#[test] +fn native_id_matches_either_identifier_source() { + let mut candidate = evidence("button", Some("Checkout")); + candidate.identifiers = + IdentifierEvidence::new(["internal-id".into(), "checkout".into()], Some(0), true); + let resolution = evaluate_locator_tree( + tree(vec![node(0, candidate, vec![], &[])], vec![0], true), + &native_id_query(None), + &request(), + ) + .unwrap(); + assert_eq!(resolution.meta.total_matches, 1); + assert_eq!(resolution.stats.identifiers.fallback_matches, 1); + assert_eq!(resolution.stats.identifiers.preferred_matches, 0); +} + +#[test] +fn duplicate_identifier_value_counts_as_a_preferred_match() { + let mut candidate = evidence("button", Some("Checkout")); + candidate.identifiers = IdentifierEvidence::typed( + [ + crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: "checkout".into(), + }, + crate::ElementIdentifier { + kind: crate::IdentifierKind::AxDomIdentifier, + value: "checkout".into(), + }, + ], + Some(1), + true, + ); + let resolution = evaluate_locator_tree( + tree(vec![node(0, candidate, vec![], &[])], vec![0], true), + &native_id_query(None), + &request(), + ) + .unwrap(); + assert_eq!(resolution.stats.identifiers.preferred_matches, 1); + assert_eq!(resolution.stats.identifiers.fallback_matches, 0); +} + +#[test] +fn identifier_stats_count_only_complete_query_matches() { + let mut candidate = evidence("button", Some("Cancel")); + candidate.identifiers = IdentifierEvidence::new(["checkout".into()], Some(0), true); + let resolution = evaluate_locator_tree( + tree(vec![node(0, candidate, vec![], &[])], vec![0], true), + &native_id_query(Some("link")), + &request(), + ) + .unwrap(); + assert_eq!(resolution.meta.total_matches, 0); + assert_eq!(resolution.stats.identifiers.preferred_matches, 0); + assert_eq!(resolution.stats.identifiers.fallback_matches, 0); +} + +#[test] +fn unknown_identifier_source_prevents_false_not_found() { + let mut candidate = evidence("button", Some("Checkout")); + candidate.identifiers = IdentifierEvidence::unknown(); + let resolution = evaluate_locator_tree( + tree(vec![node(0, candidate, vec![], &[])], vec![0], true), + &native_id_query(None), + &request(), + ) + .unwrap(); + assert!(!resolution.meta.complete); + assert_eq!(resolution.meta.total_matches, 0); +} diff --git a/crates/core/src/live_locator/evaluator_tests.rs b/crates/core/src/live_locator/evaluator_tests.rs new file mode 100644 index 0000000..e760f77 --- /dev/null +++ b/crates/core/src/live_locator/evaluator_tests.rs @@ -0,0 +1,305 @@ +use super::{ + LocatorCardinality, LocatorField, LocatorMaterialization, LocatorResolveRequest, + LocatorSelection, classify_query_result, evaluate_locator_tree, +}; +use crate::{ + locator::{ContainmentPredicate, IdentityPredicate, LocatorQuery}, + search_text, +}; + +use super::test_support::{evidence, node, tree}; + +fn request(selection: LocatorSelection) -> LocatorResolveRequest { + LocatorResolveRequest { + selection, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + } +} + +fn named_query(name: &str, exact: bool) -> LocatorQuery { + LocatorQuery { + identity: IdentityPredicate { + name: Some(search_text::normalize(name)), + ..IdentityPredicate::default() + }, + exact, + ..LocatorQuery::default() + } +} + +fn names_tree() -> super::ObservedTree { + tree( + vec![ + node(2, evidence("button", Some("SAVE")), vec![], &[]), + node(0, evidence("button", Some("Save draft")), vec![], &[]), + node(1, evidence("button", Some("save")), vec![], &[]), + ], + vec![0, 1, 2], + true, + ) +} + +#[test] +fn exact_and_substring_matching_use_normalized_document_order() { + let substring = evaluate_locator_tree( + names_tree(), + &named_query("SAVE", false), + &request(LocatorSelection::All { limit: None }), + ) + .unwrap(); + assert_eq!(substring.meta.total_matches, 3); + assert_eq!(substring.matches[0].data.name, "Save draft"); + assert_eq!(substring.matches[1].data.name, "save"); + assert_eq!(substring.matches[2].data.name, "SAVE"); + + let exact = evaluate_locator_tree( + names_tree(), + &named_query("SAVE", true), + &request(LocatorSelection::All { limit: None }), + ) + .unwrap(); + assert_eq!(exact.meta.total_matches, 2); + assert_eq!(exact.matches[0].document_order, 1); + assert_eq!(exact.matches[1].document_order, 2); +} + +fn deep_tree(structurally_complete: bool) -> super::ObservedTree { + let mut nodes = vec![node( + 30, + evidence("button", Some("Deep Needle")), + vec![], + &vec![0; 30], + )]; + let mut child = 0; + for depth in (0..30).rev() { + let name = (depth == 0).then_some("root"); + let index = nodes.len() as u32; + let mut parent = node( + depth, + evidence("group", name), + vec![child], + &vec![0; depth as usize], + ); + if depth == 0 { + parent.completeness.subtree_complete = structurally_complete; + } + nodes.push(parent); + child = index; + } + tree(nodes, vec![child], structurally_complete) +} + +#[test] +fn deep_has_text_and_has_walk_all_descendants() { + let has_text = LocatorQuery { + identity: IdentityPredicate { + name: Some("root".into()), + ..IdentityPredicate::default() + }, + has_text: Some("needle".into()), + ..LocatorQuery::default() + }; + let text_resolution = evaluate_locator_tree( + deep_tree(true), + &has_text, + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert_eq!( + classify_query_result(&text_resolution), + LocatorCardinality::One + ); + assert_eq!(text_resolution.stats.evaluation.memo_cells_evaluated, 93); + + let has_button = LocatorQuery { + identity: IdentityPredicate { + name: Some("root".into()), + ..IdentityPredicate::default() + }, + containment: ContainmentPredicate { + has: Some(Box::new(LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + })), + has_not: None, + }, + ..LocatorQuery::default() + }; + let has_resolution = evaluate_locator_tree( + deep_tree(true), + &has_button, + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert_eq!(has_resolution.meta.total_matches, 1); +} + +#[test] +fn incomplete_deep_has_not_is_unknown() { + let query = LocatorQuery { + identity: IdentityPredicate { + name: Some("root".into()), + ..IdentityPredicate::default() + }, + containment: ContainmentPredicate { + has: None, + has_not: Some(Box::new(named_query("missing", false))), + }, + ..LocatorQuery::default() + }; + let resolution = + evaluate_locator_tree(deep_tree(false), &query, &request(LocatorSelection::Strict)) + .unwrap(); + assert_eq!(resolution.meta.total_matches, 0); + assert!(!resolution.meta.complete); + assert_eq!( + classify_query_result(&resolution), + LocatorCardinality::Incomplete { observed: 0 } + ); +} + +#[test] +fn known_role_mismatch_dominates_unknown_name() { + let mut unknown = evidence("button", None); + unknown.name = LocatorField::Unknown; + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("link".into()), + name: Some("save".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + let resolution = evaluate_locator_tree( + tree(vec![node(0, unknown, vec![], &[])], vec![0], true), + &query, + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert!(resolution.meta.complete); + assert_eq!(resolution.meta.total_matches, 0); +} + +#[test] +fn candidate_role_with_unknown_name_remains_incomplete() { + let mut unknown = evidence("button", None); + unknown.name = LocatorField::Unknown; + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + name: Some("save".into()), + ..IdentityPredicate::default() + }, + exact: true, + ..LocatorQuery::default() + }; + + let resolution = evaluate_locator_tree( + tree(vec![node(0, unknown, vec![], &[])], vec![0], true), + &query, + &request(LocatorSelection::Strict), + ) + .unwrap(); + + assert!(!resolution.meta.complete); + assert_eq!( + classify_query_result(&resolution), + LocatorCardinality::Incomplete { observed: 0 } + ); +} + +#[test] +fn ordinal_selection_uses_document_order_not_arena_order() { + for (selection, expected) in [ + (LocatorSelection::First, 0), + (LocatorSelection::Nth(1), 1), + (LocatorSelection::Last, 2), + ] { + let resolution = + evaluate_locator_tree(names_tree(), &LocatorQuery::default(), &request(selection)) + .unwrap(); + assert_eq!(resolution.matches[0].document_order, expected); + } +} + +#[test] +fn strict_classification_preserves_zero_one_many_and_incomplete() { + let zero = evaluate_locator_tree( + tree(Vec::new(), Vec::new(), true), + &LocatorQuery::default(), + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert_eq!(classify_query_result(&zero), LocatorCardinality::Zero); + + let one = evaluate_locator_tree( + tree( + vec![node(0, evidence("button", Some("one")), vec![], &[])], + vec![0], + true, + ), + &LocatorQuery::default(), + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert_eq!(classify_query_result(&one), LocatorCardinality::One); + + let many = evaluate_locator_tree( + names_tree(), + &LocatorQuery::default(), + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert_eq!( + classify_query_result(&many), + LocatorCardinality::Many { + observed: 3, + exact: true + } + ); + + let mut incomplete_tree = names_tree(); + incomplete_tree.structurally_complete = false; + let incomplete_many = evaluate_locator_tree( + incomplete_tree, + &LocatorQuery::default(), + &request(LocatorSelection::Strict), + ) + .unwrap(); + assert_eq!( + classify_query_result(&incomplete_many), + LocatorCardinality::Many { + observed: 3, + exact: false + } + ); +} + +#[test] +fn dense_evaluator_has_a_linear_memo_cell_bound_at_twenty_thousand_nodes() { + let nodes = (0..20_000) + .map(|order| node(order, evidence("button", Some("Match")), vec![], &[])) + .collect(); + let roots = (0..20_000).collect(); + let query = LocatorQuery { + containment: ContainmentPredicate { + has: None, + has_not: Some(Box::new(LocatorQuery::default())), + }, + ..LocatorQuery::default() + }; + let resolution = evaluate_locator_tree( + tree(nodes, roots, true), + &query, + &request(LocatorSelection::Count), + ) + .unwrap(); + assert_eq!(resolution.meta.total_matches, 20_000); + assert_eq!(resolution.stats.evaluation.query_clause_count, 2); + assert_eq!(resolution.stats.evaluation.memo_cells_evaluated, 80_000); +} diff --git a/crates/core/src/live_locator/evidence_plan.rs b/crates/core/src/live_locator/evidence_plan.rs new file mode 100644 index 0000000..8a8cb7c --- /dev/null +++ b/crates/core/src/live_locator/evidence_plan.rs @@ -0,0 +1,87 @@ +use crate::{AdapterError, ErrorCode}; + +use super::EvidenceRequirements; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct EvidencePlan { + root: EvidenceRequirements, + descendants: EvidenceRequirements, +} + +impl EvidencePlan { + pub(super) fn uniform(requirements: EvidenceRequirements) -> Self { + Self { + root: requirements, + descendants: requirements, + } + } + + pub(super) fn rooted(root: EvidenceRequirements, descendants: EvidenceRequirements) -> Self { + Self { root, descendants } + } + + pub(super) fn validate(self) -> Result<(), AdapterError> { + if !self.descendants.role || !self.root.role { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "role evidence is required for every observation", + )); + } + if !self.root.covers(self.descendants) { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "root evidence must cover descendant evidence", + )); + } + Ok(()) + } + + pub(super) fn for_raw_depth(self, raw_depth: u8) -> EvidenceRequirements { + if raw_depth == 0 { + self.root + } else { + self.descendants + } + } + + pub(super) fn descendants(self) -> EvidenceRequirements { + self.descendants + } + + pub(super) fn hydrates_root_name_from_children(self) -> bool { + self.root != self.descendants && (self.root.name || self.root.description) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn root_evidence_must_cover_descendant_evidence() { + let error = EvidencePlan::rooted( + EvidenceRequirements { + role: true, + ..EvidenceRequirements::default() + }, + EvidenceRequirements::snapshot(), + ) + .validate() + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + } + + #[test] + fn root_name_hydration_is_explicit_in_the_plan() { + let descendants = EvidenceRequirements { + role: true, + ..EvidenceRequirements::default() + }; + let rooted = EvidencePlan::rooted(EvidenceRequirements::snapshot(), descendants); + + assert!(rooted.validate().is_ok()); + assert!(rooted.hydrates_root_name_from_children()); + assert!(!EvidencePlan::uniform(descendants).hydrates_root_name_from_children()); + } +} diff --git a/crates/core/src/live_locator/evidence_requirements.rs b/crates/core/src/live_locator/evidence_requirements.rs new file mode 100644 index 0000000..cb860b4 --- /dev/null +++ b/crates/core/src/live_locator/evidence_requirements.rs @@ -0,0 +1,116 @@ +use crate::locator::LocatorQuery; + +use super::{LocatorMaterialization, LocatorResolveRequest, RefEvidenceRequirements}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct EvidenceRequirements { + pub role: bool, + pub name: bool, + pub description: bool, + pub value: bool, + pub identifiers: bool, + pub states: bool, + pub ref_evidence: RefEvidenceRequirements, +} + +impl EvidenceRequirements { + pub fn snapshot() -> Self { + Self { + role: true, + name: true, + description: true, + value: true, + identifiers: true, + states: true, + ref_evidence: RefEvidenceRequirements { + bounds: true, + actions: true, + }, + } + } + + pub(crate) fn locator(query: &LocatorQuery, request: &LocatorResolveRequest) -> Self { + if request.materialization == LocatorMaterialization::FullRefMap { + return Self::snapshot(); + } + let mut requirements = Self::query(query); + if request.materialization == LocatorMaterialization::SelectedMatches { + requirements.identifiers = true; + requirements.ref_evidence.bounds = true; + } + requirements + } + + pub(crate) fn query(query: &LocatorQuery) -> Self { + let needs_subtree_text = query.has_text.is_some(); + let mut requirements = Self { + role: true, + name: query.identity.name.is_some() || needs_subtree_text, + description: query.identity.description.is_some() || needs_subtree_text, + value: query.identity.value.is_some() || needs_subtree_text, + identifiers: query.identity.native_id.is_some(), + states: !query.states.is_empty(), + ref_evidence: RefEvidenceRequirements::default(), + }; + for nested in [ + query.containment.has.as_deref(), + query.containment.has_not.as_deref(), + ] + .into_iter() + .flatten() + { + requirements = requirements.union(Self::query(nested)); + } + requirements + } + + fn union(self, other: Self) -> Self { + Self { + role: self.role || other.role, + name: self.name || other.name, + description: self.description || other.description, + value: self.value || other.value, + identifiers: self.identifiers || other.identifiers, + states: self.states || other.states, + ref_evidence: self.ref_evidence.union(other.ref_evidence), + } + } + + pub(super) fn covers(self, other: Self) -> bool { + self.union(other) == self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::live_locator::LocatorSelection; + + #[test] + fn selected_matches_use_typed_identifiers_and_bounds_without_global_semantic_reads() { + let query = LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + ..Default::default() + }, + ..Default::default() + }; + let request = LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::after(500).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::SelectedMatches, + }; + + let requirements = EvidenceRequirements::locator(&query, &request); + + assert!(requirements.role); + assert!(!requirements.name); + assert!(!requirements.description); + assert!(!requirements.value); + assert!(requirements.identifiers); + assert!(requirements.ref_evidence.bounds); + assert!(!requirements.states); + assert!(!requirements.ref_evidence.actions); + } +} diff --git a/crates/core/src/live_locator/hydrate.rs b/crates/core/src/live_locator/hydrate.rs new file mode 100644 index 0000000..fe05976 --- /dev/null +++ b/crates/core/src/live_locator/hydrate.rs @@ -0,0 +1,300 @@ +use super::{ + LocatorField, LocatorResolution, ObservationRequest, ObservationRoot, ObservedNode, + ObservedTree, +}; +use crate::{ + AdapterError, AppError, ErrorCode, IdentityMatch, adapter::PlatformAdapter, + locator::LocatorQuery, refs::RefEntry, refs::RefMap, +}; +use serde_json::json; + +pub(super) fn retryable_error(error: &AdapterError) -> bool { + if !error.permits_retry_by_default() { + return false; + } + matches!( + error.code, + ErrorCode::StaleRef + | ErrorCode::AmbiguousTarget + | ErrorCode::ElementNotFound + | ErrorCode::Timeout + | ErrorCode::AppUnresponsive + ) +} + +pub(super) fn selected_matches( + adapter: &dyn PlatformAdapter, + query: &LocatorQuery, + request: &super::LocatorResolveRequest, + resolution: &mut LocatorResolution, +) -> Result<(), AppError> { + let deadline = request.deadline; + let mut refmap = RefMap::new(); + for matched in &mut resolution.matches { + ensure_remaining(deadline)?; + let preliminary = matched.entry.clone(); + let has_identity = crate::ref_identity::has_meaningful_identity(&preliminary); + let has_bounds = preliminary.geometry.bounds_hash.is_some(); + tracing::debug!( + role = preliminary.identity.role, + path_len = preliminary.scope.path.len(), + has_identity, + has_bounds, + "hydrating selected locator anchor" + ); + if !has_identity && !has_bounds { + return Err(anchor_missing(&preliminary).into()); + } + let handle = adapter.resolve_locator_anchor(&preliminary, deadline)?; + ensure_remaining(deadline)?; + let hydration_root = ObservationRoot::Element { + handle: &handle, + entry: &preliminary, + root_ref: None, + }; + let mut hydrated = crate::renderer_accessibility::observe_tree( + adapter, + hydration_root, + &ObservationRequest::selected_hydration(query, request, hydration_root, deadline) + .validate()?, + )?; + hydrated.stats.reads.counts.observation_attempts = + hydrated.stats.reads.counts.observation_attempts.max(1); + resolution.stats.merge_observation(&hydrated.stats); + if hydrated.roots.len() != 1 { + return Err(evidence_incomplete(&hydrated, None, query).into()); + } + let root_index = hydrated.roots[0] as usize; + let (entry, name, value, root_order) = { + let node = hydrated + .nodes + .get(root_index) + .ok_or_else(|| AdapterError::internal("hydrated locator root is missing"))?; + if !evidence_complete(query, node) { + return Err(evidence_incomplete(&hydrated, Some(node), query).into()); + } + if node.evidence.role.known().map(String::as_str) + != Some(preliminary.identity.role.as_str()) + || !anchor_matches(&preliminary, node) + { + return Err(changed_during_hydration().into()); + } + let mut entry = super::materialize::ref_entry(node, &hydrated.source, query); + preserve_verified_identity(&preliminary, node, &mut entry); + let name = display_name(node, &entry.identity.role); + let value = node.evidence.value.meaningful_string(); + (entry, name, value, node.document_order) + }; + if !crate::ref_identity::has_meaningful_identity(&entry) + && entry.geometry.bounds_hash.is_none() + { + return Err(anchor_missing(&entry).into()); + } + let validation = super::evaluate_locator_tree( + hydrated, + query, + &super::LocatorResolveRequest { + selection: super::LocatorSelection::First, + materialization: super::LocatorMaterialization::None, + ..*request + }, + )?; + resolution.stats.merge_evaluation(&validation.stats); + let root_is_authoritative = validation.meta.selection_complete + && validation + .matches + .first() + .is_some_and(|candidate| candidate.document_order == root_order); + if !root_is_authoritative { + if validation.meta.complete { + return Err(changed_during_hydration().into()); + } + return Err(query_incomplete(&validation).into()); + } + matched.data.role = entry.identity.role.clone(); + matched.data.name = name; + matched.data.value = value; + matched.data.states = entry.capabilities.states.clone(); + matched.data.interactive = crate::ref_alloc::is_ref_able_role_actions( + &entry.identity.role, + &entry.capabilities.available_actions, + ); + if matched.data.interactive { + matched.data.ref_id = Some(refmap.try_allocate(entry.clone())?); + } + matched.entry = entry; + } + resolution.refmap = Some(refmap); + Ok(()) +} + +fn ensure_remaining(deadline: crate::Deadline) -> Result<(), AppError> { + if deadline.remaining().is_zero() { + return Err( + AdapterError::timeout("Selected locator hydration exceeded its deadline").into(), + ); + } + Ok(()) +} + +fn evidence_complete(query: &LocatorQuery, node: &ObservedNode) -> bool { + !node.evidence.role.is_unknown() + && (query.identity.name.is_none() || !node.evidence.name.is_unknown()) + && (query.identity.description.is_none() || !node.evidence.description.is_unknown()) + && (query.identity.value.is_none() || !node.evidence.value.is_unknown()) + && (query.identity.native_id.is_none() || node.evidence.identifiers.is_complete()) + && !node.evidence.states.is_unknown() + && !node.evidence.ref_evidence.bounds.is_unknown() + && super::materialize::addressability(&node.evidence).1 +} + +fn anchor_matches(preliminary: &RefEntry, node: &ObservedNode) -> bool { + let actual_bounds_hash = node + .evidence + .ref_evidence + .bounds + .known() + .and_then(crate::Rect::bounds_hash); + if crate::ref_identity::has_meaningful_identity(preliminary) { + return match crate::ref_identity::identity_match( + preliminary, + &node.evidence.name, + &node.evidence.value, + &node.evidence.description, + &node.evidence.identifiers, + ) { + IdentityMatch::Match => true, + IdentityMatch::NoMatch => false, + IdentityMatch::Unknown => preliminary + .geometry + .bounds_hash + .is_none_or(|expected| actual_bounds_hash == Some(expected)), + }; + } + preliminary.geometry.bounds_hash.is_some() + && preliminary.geometry.bounds_hash == actual_bounds_hash +} + +fn preserve_verified_identity(preliminary: &RefEntry, node: &ObservedNode, entry: &mut RefEntry) { + if node.evidence.name.is_unknown() { + entry.identity.name.clone_from(&preliminary.identity.name); + } + if node.evidence.description.is_unknown() { + entry + .identity + .description + .clone_from(&preliminary.identity.description); + } + if node.evidence.value.is_unknown() + && !crate::roles::is_mutable_value_role(&entry.identity.role) + { + entry.identity.value.clone_from(&preliminary.identity.value); + } + if !node.evidence.identifiers.is_complete() { + entry + .identity + .native_id + .clone_from(&preliminary.identity.native_id); + } +} + +fn display_name(node: &ObservedNode, role: &str) -> String { + match &node.evidence.name { + LocatorField::Unknown => "(name unavailable)".into(), + LocatorField::Known(name) if !name.is_empty() => name.clone(), + LocatorField::Known(_) | LocatorField::Absent => node + .evidence + .value + .meaningful_string() + .or_else(|| node.evidence.description.meaningful_string()) + .unwrap_or_else(|| format!("(unnamed {role})")), + } +} + +fn anchor_missing(entry: &RefEntry) -> AdapterError { + AdapterError::new( + ErrorCode::StaleRef, + "Selected locator lacks a verifiable identity or geometry anchor", + ) + .with_suggestion("Use a locator that resolves to an element with stable identity or bounds") + .with_details(json!({ + "kind": "locator_selected_anchor_missing", + "retryable": false, + "role": entry.identity.role, + "path_len": entry.scope.path.len(), + "has_identity": crate::ref_identity::has_meaningful_identity(entry), + "has_bounds": entry.geometry.bounds_hash.is_some(), + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +fn changed_during_hydration() -> AdapterError { + AdapterError::new( + ErrorCode::StaleRef, + "Selected locator changed during hydration", + ) + .with_suggestion("Retry the locator against the current accessibility tree") + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +fn query_incomplete(validation: &LocatorResolution) -> AdapterError { + let deterministic = has_deterministic_limit(&validation.stats); + AdapterError::timeout("Selected locator query could not be revalidated") + .with_suggestion("Retry against a stable subtree or use a narrower locator") + .with_details(json!({ + "kind": if deterministic { + "locator_selected_query_budget_limit" + } else { + "locator_selected_query_incomplete" + }, + "retryable": !deterministic, + "selection_complete": validation.meta.selection_complete, + "observed_matches": validation.meta.total_matches, + "query_stats": &validation.stats, + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +fn has_deterministic_limit(stats: &super::LocatorStats) -> bool { + let limits = &stats.traversal.limits; + limits.node_hits > 0 + || limits.edge_hits > 0 + || limits.child_hits > 0 + || limits.text_hits > 0 + || limits.depth_hits > 0 +} + +fn evidence_incomplete( + tree: &ObservedTree, + node: Option<&ObservedNode>, + query: &LocatorQuery, +) -> AdapterError { + AdapterError::timeout("Selected locator evidence was incomplete") + .with_details(json!({ + "kind": "locator_selected_evidence_incomplete", + "retryable": true, + "structurally_complete": tree.structurally_complete, + "root_count": tree.roots.len(), + "required": { + "name": query.identity.name.is_some(), + "description": query.identity.description.is_some(), + "value": query.identity.value.is_some(), + "identifiers": query.identity.native_id.is_some(), + "states": true, + "bounds": true, + "actions": true, + }, + "unknown": node.map(|node| json!({ + "role": node.evidence.role.is_unknown(), + "name": node.evidence.name.is_unknown(), + "description": node.evidence.description.is_unknown(), + "value": node.evidence.value.is_unknown(), + "identifiers": !node.evidence.identifiers.is_complete(), + "states": node.evidence.states.is_unknown(), + "bounds": node.evidence.ref_evidence.bounds.is_unknown(), + "actions": node.evidence.ref_evidence.available_actions.is_unknown(), + })), + "query_stats": &tree.stats, + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} diff --git a/crates/core/src/live_locator/identifier_evidence.rs b/crates/core/src/live_locator/identifier_evidence.rs new file mode 100644 index 0000000..41970c1 --- /dev/null +++ b/crates/core/src/live_locator/identifier_evidence.rs @@ -0,0 +1,113 @@ +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IdentifierEvidence { + identifiers: Vec<crate::ElementIdentifier>, + preferred: Option<usize>, + complete: bool, +} + +impl IdentifierEvidence { + pub fn new( + values: impl IntoIterator<Item = String>, + preferred: Option<usize>, + complete: bool, + ) -> Self { + Self::typed( + values.into_iter().map(|value| crate::ElementIdentifier { + kind: crate::IdentifierKind::Unknown, + value, + }), + preferred, + complete, + ) + } + + pub fn typed( + identifiers: impl IntoIterator<Item = crate::ElementIdentifier>, + preferred: Option<usize>, + complete: bool, + ) -> Self { + let original = identifiers.into_iter().collect::<Vec<_>>(); + let preferred_value = preferred + .and_then(|index| original.get(index)) + .filter(|identifier| !identifier.value.trim().is_empty()) + .cloned(); + let mut normalized = Vec::new(); + for identifier in original { + if !identifier.value.trim().is_empty() && !normalized.contains(&identifier) { + normalized.push(identifier); + } + } + let preferred = preferred_value + .as_ref() + .and_then(|value| normalized.iter().position(|candidate| candidate == value)); + Self { + identifiers: normalized, + preferred, + complete, + } + } + + pub fn absent() -> Self { + Self::new([], None, true) + } + + pub fn unknown() -> Self { + Self::new([], None, false) + } + + pub fn identifiers(&self) -> &[crate::ElementIdentifier] { + &self.identifiers + } + + #[cfg(test)] + pub fn values(&self) -> Vec<&str> { + self.identifiers + .iter() + .map(|identifier| identifier.value.as_str()) + .collect() + } + + pub fn preferred_value(&self) -> Option<&str> { + self.preferred + .and_then(|index| self.identifiers.get(index)) + .map(|identifier| identifier.value.as_str()) + } + + pub fn preferred_identifier(&self) -> Option<&crate::ElementIdentifier> { + self.preferred.and_then(|index| self.identifiers.get(index)) + } + + pub fn preferred_index(&self) -> Option<usize> { + self.preferred + } + + pub fn is_complete(&self) -> bool { + self.complete + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preferred_index_is_resolved_before_empty_values_are_removed() { + let evidence = IdentifierEvidence::new([String::new(), "dom-id".into()], Some(1), true); + + assert_eq!(evidence.values(), ["dom-id"]); + assert_eq!(evidence.preferred_index(), Some(0)); + assert_eq!(evidence.preferred_value(), Some("dom-id")); + } + + #[test] + fn preferred_duplicate_maps_to_the_deduplicated_value() { + let evidence = IdentifierEvidence::new( + ["shared".into(), "other".into(), "shared".into()], + Some(2), + true, + ); + + assert_eq!(evidence.values(), ["shared", "other"]); + assert_eq!(evidence.preferred_index(), Some(0)); + } +} diff --git a/crates/core/src/live_locator/locator_activation_stats.rs b/crates/core/src/live_locator/locator_activation_stats.rs new file mode 100644 index 0000000..7e68e93 --- /dev/null +++ b/crates/core/src/live_locator/locator_activation_stats.rs @@ -0,0 +1,8 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorActivationStats { + pub attempted: bool, + pub succeeded: bool, + pub ready: bool, +} diff --git a/crates/core/src/live_locator/locator_cardinality.rs b/crates/core/src/live_locator/locator_cardinality.rs new file mode 100644 index 0000000..db27707 --- /dev/null +++ b/crates/core/src/live_locator/locator_cardinality.rs @@ -0,0 +1,7 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LocatorCardinality { + Zero, + One, + Many { observed: u32, exact: bool }, + Incomplete { observed: u32 }, +} diff --git a/crates/core/src/live_locator/locator_evaluation_stats.rs b/crates/core/src/live_locator/locator_evaluation_stats.rs new file mode 100644 index 0000000..e4c450c --- /dev/null +++ b/crates/core/src/live_locator/locator_evaluation_stats.rs @@ -0,0 +1,10 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorEvaluationStats { + pub query_clause_count: u32, + pub text_clause_count: u32, + pub memo_cells_evaluated: u64, + pub self_filter_candidates: u64, + pub matched_nodes: u64, +} diff --git a/crates/core/src/live_locator/locator_evidence.rs b/crates/core/src/live_locator/locator_evidence.rs new file mode 100644 index 0000000..d8594ec --- /dev/null +++ b/crates/core/src/live_locator/locator_evidence.rs @@ -0,0 +1,43 @@ +use super::{EvidenceRequirements, IdentifierEvidence, LocatorField, LocatorRefEvidence}; + +#[derive(Debug, Clone, PartialEq)] +pub struct LocatorEvidence { + pub role: LocatorField<String>, + pub name: LocatorField<String>, + pub description: LocatorField<String>, + pub value: LocatorField<String>, + pub identifiers: IdentifierEvidence, + pub states: LocatorField<Vec<String>>, + pub ref_evidence: LocatorRefEvidence, +} + +impl LocatorEvidence { + pub fn satisfies(&self, requirements: EvidenceRequirements) -> bool { + (!requirements.role || !self.role.is_unknown()) + && (!requirements.name || !self.name.is_unknown()) + && (!requirements.description || !self.description.is_unknown()) + && (!requirements.value || !self.value.is_unknown()) + && (!requirements.identifiers || self.identifiers.is_complete()) + && (!requirements.states || !self.states.is_unknown()) + && (!requirements.ref_evidence.bounds || !self.ref_evidence.bounds.is_unknown()) + && (!requirements.ref_evidence.actions + || !self.ref_evidence.available_actions.is_unknown()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snapshot_requirements_reject_unknown_identity_and_ref_evidence() { + let mut evidence = crate::live_locator::test_support::evidence("button", Some("Save")); + assert!(evidence.satisfies(EvidenceRequirements::snapshot())); + + evidence.name = LocatorField::Unknown; + assert!(!evidence.satisfies(EvidenceRequirements::snapshot())); + evidence.name = LocatorField::Known("Save".into()); + evidence.ref_evidence.bounds = LocatorField::Unknown; + assert!(!evidence.satisfies(EvidenceRequirements::snapshot())); + } +} diff --git a/crates/core/src/live_locator/locator_field.rs b/crates/core/src/live_locator/locator_field.rs new file mode 100644 index 0000000..09a1b60 --- /dev/null +++ b/crates/core/src/live_locator/locator_field.rs @@ -0,0 +1,25 @@ +#[derive(Debug, Clone, PartialEq)] +pub enum LocatorField<T> { + Known(T), + Absent, + Unknown, +} + +impl<T> LocatorField<T> { + pub fn known(&self) -> Option<&T> { + match self { + Self::Known(value) => Some(value), + Self::Absent | Self::Unknown => None, + } + } + + pub fn is_unknown(&self) -> bool { + matches!(self, Self::Unknown) + } +} + +impl LocatorField<String> { + pub(crate) fn meaningful_string(&self) -> Option<String> { + self.known().filter(|value| !value.is_empty()).cloned() + } +} diff --git a/crates/core/src/live_locator/locator_identifier_stats.rs b/crates/core/src/live_locator/locator_identifier_stats.rs new file mode 100644 index 0000000..e73d45a --- /dev/null +++ b/crates/core/src/live_locator/locator_identifier_stats.rs @@ -0,0 +1,10 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorIdentifierStats { + pub values_observed: u64, + pub nodes_with_identifiers: u64, + pub nodes_with_multiple_identifiers: u64, + pub preferred_matches: u64, + pub fallback_matches: u64, +} diff --git a/crates/core/src/live_locator/locator_limit_stats.rs b/crates/core/src/live_locator/locator_limit_stats.rs new file mode 100644 index 0000000..db325a3 --- /dev/null +++ b/crates/core/src/live_locator/locator_limit_stats.rs @@ -0,0 +1,24 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorLimitStats { + pub node_hits: u64, + pub edge_hits: u64, + pub child_hits: u64, + pub child_label_hits: u64, + pub text_hits: u64, + pub depth_hits: u64, + pub child_count_changes: u64, +} + +impl LocatorLimitStats { + pub(crate) fn merge(&mut self, other: &Self) { + self.node_hits += other.node_hits; + self.edge_hits += other.edge_hits; + self.child_hits += other.child_hits; + self.child_label_hits += other.child_label_hits; + self.text_hits += other.text_hits; + self.depth_hits += other.depth_hits; + self.child_count_changes += other.child_count_changes; + } +} diff --git a/crates/core/src/live_locator/locator_match.rs b/crates/core/src/live_locator/locator_match.rs new file mode 100644 index 0000000..9dec6b0 --- /dev/null +++ b/crates/core/src/live_locator/locator_match.rs @@ -0,0 +1,14 @@ +use super::LocatorMatchData; +use crate::refs::RefEntry; + +pub struct LocatorMatch { + pub data: LocatorMatchData, + pub document_order: u32, + pub(crate) entry: RefEntry, +} + +impl LocatorMatch { + pub fn into_entry(self) -> RefEntry { + self.entry + } +} diff --git a/crates/core/src/live_locator/locator_match_data.rs b/crates/core/src/live_locator/locator_match_data.rs new file mode 100644 index 0000000..31f1724 --- /dev/null +++ b/crates/core/src/live_locator/locator_match_data.rs @@ -0,0 +1,12 @@ +use serde::Serialize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct LocatorMatchData { + pub ref_id: Option<String>, + pub role: String, + pub name: String, + pub value: Option<String>, + pub states: Vec<String>, + pub interactive: bool, + pub path: Vec<String>, +} diff --git a/crates/core/src/live_locator/locator_materialization.rs b/crates/core/src/live_locator/locator_materialization.rs new file mode 100644 index 0000000..6d696fb --- /dev/null +++ b/crates/core/src/live_locator/locator_materialization.rs @@ -0,0 +1,7 @@ +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum LocatorMaterialization { + #[default] + None, + SelectedMatches, + FullRefMap, +} diff --git a/crates/core/src/live_locator/locator_read_counts.rs b/crates/core/src/live_locator/locator_read_counts.rs new file mode 100644 index 0000000..3f2b5f1 --- /dev/null +++ b/crates/core/src/live_locator/locator_read_counts.rs @@ -0,0 +1,11 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorReadCounts { + pub observation_attempts: u64, + pub attribute_batches: u64, + pub attributes_requested: u64, + pub child_reads: u64, + pub action_reads: u64, + pub fallback_reads: u64, +} diff --git a/crates/core/src/live_locator/locator_read_health.rs b/crates/core/src/live_locator/locator_read_health.rs new file mode 100644 index 0000000..b98ce84 --- /dev/null +++ b/crates/core/src/live_locator/locator_read_health.rs @@ -0,0 +1,8 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorReadHealth { + pub cannot_complete: u64, + pub native_read_failures: u64, + pub deadline_exhausted: u64, +} diff --git a/crates/core/src/live_locator/locator_read_stats.rs b/crates/core/src/live_locator/locator_read_stats.rs new file mode 100644 index 0000000..77f6f4e --- /dev/null +++ b/crates/core/src/live_locator/locator_read_stats.rs @@ -0,0 +1,11 @@ +use serde::Serialize; + +use super::{LocatorReadCounts, LocatorReadHealth}; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorReadStats { + #[serde(flatten)] + pub counts: LocatorReadCounts, + #[serde(flatten)] + pub health: LocatorReadHealth, +} diff --git a/crates/core/src/live_locator/locator_ref_evidence.rs b/crates/core/src/live_locator/locator_ref_evidence.rs new file mode 100644 index 0000000..2e6958c --- /dev/null +++ b/crates/core/src/live_locator/locator_ref_evidence.rs @@ -0,0 +1,8 @@ +use super::LocatorField; +use crate::Rect; + +#[derive(Debug, Clone, PartialEq)] +pub struct LocatorRefEvidence { + pub bounds: LocatorField<Rect>, + pub available_actions: LocatorField<Vec<String>>, +} diff --git a/crates/core/src/live_locator/locator_resolution.rs b/crates/core/src/live_locator/locator_resolution.rs new file mode 100644 index 0000000..cb6a62b --- /dev/null +++ b/crates/core/src/live_locator/locator_resolution.rs @@ -0,0 +1,9 @@ +use super::{LocatorMatch, LocatorResolutionMeta, LocatorStats}; +use crate::refs::RefMap; + +pub struct LocatorResolution { + pub matches: Vec<LocatorMatch>, + pub refmap: Option<RefMap>, + pub stats: LocatorStats, + pub meta: LocatorResolutionMeta, +} diff --git a/crates/core/src/live_locator/locator_resolution_meta.rs b/crates/core/src/live_locator/locator_resolution_meta.rs new file mode 100644 index 0000000..d3d5992 --- /dev/null +++ b/crates/core/src/live_locator/locator_resolution_meta.rs @@ -0,0 +1,10 @@ +use serde::Serialize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct LocatorResolutionMeta { + pub total_matches: u32, + pub complete: bool, + pub selection_complete: bool, + pub truncated: bool, + pub roles_present: Vec<String>, +} diff --git a/crates/core/src/live_locator/locator_resolve_request.rs b/crates/core/src/live_locator/locator_resolve_request.rs new file mode 100644 index 0000000..80851fe --- /dev/null +++ b/crates/core/src/live_locator/locator_resolve_request.rs @@ -0,0 +1,10 @@ +use super::{LocatorMaterialization, LocatorSelection}; +use crate::Deadline; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LocatorResolveRequest { + pub selection: LocatorSelection, + pub deadline: Deadline, + pub max_raw_depth: u8, + pub materialization: LocatorMaterialization, +} diff --git a/crates/core/src/live_locator/locator_selection.rs b/crates/core/src/live_locator/locator_selection.rs new file mode 100644 index 0000000..321e66c --- /dev/null +++ b/crates/core/src/live_locator/locator_selection.rs @@ -0,0 +1,9 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LocatorSelection { + Strict, + All { limit: Option<u32> }, + Count, + First, + Last, + Nth(u32), +} diff --git a/crates/core/src/live_locator/locator_semantic_read_stats.rs b/crates/core/src/live_locator/locator_semantic_read_stats.rs new file mode 100644 index 0000000..97d7886 --- /dev/null +++ b/crates/core/src/live_locator/locator_semantic_read_stats.rs @@ -0,0 +1,8 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorSemanticReadStats { + pub child_label_reads: u64, + pub promotion_reads: u64, + pub settable_reads: u64, +} diff --git a/crates/core/src/live_locator/locator_stats.rs b/crates/core/src/live_locator/locator_stats.rs new file mode 100644 index 0000000..1dc2229 --- /dev/null +++ b/crates/core/src/live_locator/locator_stats.rs @@ -0,0 +1,77 @@ +use super::{ + LocatorActivationStats, LocatorEvaluationStats, LocatorIdentifierStats, LocatorReadStats, + LocatorSemanticReadStats, LocatorTraversalStats, +}; +use serde::Serialize; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorStats { + pub activation: LocatorActivationStats, + pub traversal: LocatorTraversalStats, + pub reads: LocatorReadStats, + pub identifiers: LocatorIdentifierStats, + pub semantic_reads: LocatorSemanticReadStats, + pub evaluation: LocatorEvaluationStats, + pub elapsed_us: u64, +} + +impl LocatorStats { + pub(crate) fn merge_observation(&mut self, other: &Self) { + self.activation.attempted |= other.activation.attempted; + self.activation.succeeded |= other.activation.succeeded; + self.activation.ready |= other.activation.ready; + self.traversal.nodes_visited += other.traversal.nodes_visited; + self.traversal.peak_handles_owned = self + .traversal + .peak_handles_owned + .max(other.traversal.peak_handles_owned); + self.traversal.max_raw_depth = self + .traversal + .max_raw_depth + .max(other.traversal.max_raw_depth); + self.traversal.max_logical_depth = self + .traversal + .max_logical_depth + .max(other.traversal.max_logical_depth); + self.traversal.web_wrapper_nodes += other.traversal.web_wrapper_nodes; + self.traversal.cycles_skipped += other.traversal.cycles_skipped; + self.traversal.limits.merge(&other.traversal.limits); + self.reads.counts.observation_attempts += other.reads.counts.observation_attempts; + self.reads.counts.attribute_batches += other.reads.counts.attribute_batches; + self.reads.counts.attributes_requested += other.reads.counts.attributes_requested; + self.reads.counts.child_reads += other.reads.counts.child_reads; + self.reads.counts.action_reads += other.reads.counts.action_reads; + self.reads.counts.fallback_reads += other.reads.counts.fallback_reads; + self.reads.health.cannot_complete += other.reads.health.cannot_complete; + self.reads.health.native_read_failures += other.reads.health.native_read_failures; + self.reads.health.deadline_exhausted += other.reads.health.deadline_exhausted; + self.identifiers.values_observed += other.identifiers.values_observed; + self.identifiers.nodes_with_identifiers += other.identifiers.nodes_with_identifiers; + self.identifiers.nodes_with_multiple_identifiers += + other.identifiers.nodes_with_multiple_identifiers; + self.semantic_reads.child_label_reads += other.semantic_reads.child_label_reads; + self.semantic_reads.promotion_reads += other.semantic_reads.promotion_reads; + self.semantic_reads.settable_reads += other.semantic_reads.settable_reads; + } + + pub(crate) fn merge_attempt(&mut self, other: &Self) { + self.merge_observation(other); + self.merge_evaluation(other); + } + + pub(crate) fn merge_evaluation(&mut self, other: &Self) { + self.identifiers.preferred_matches += other.identifiers.preferred_matches; + self.identifiers.fallback_matches += other.identifiers.fallback_matches; + self.evaluation.query_clause_count = self + .evaluation + .query_clause_count + .max(other.evaluation.query_clause_count); + self.evaluation.text_clause_count = self + .evaluation + .text_clause_count + .max(other.evaluation.text_clause_count); + self.evaluation.memo_cells_evaluated += other.evaluation.memo_cells_evaluated; + self.evaluation.self_filter_candidates += other.evaluation.self_filter_candidates; + self.evaluation.matched_nodes += other.evaluation.matched_nodes; + } +} diff --git a/crates/core/src/live_locator/locator_stats_tests.rs b/crates/core/src/live_locator/locator_stats_tests.rs new file mode 100644 index 0000000..6659e0c --- /dev/null +++ b/crates/core/src/live_locator/locator_stats_tests.rs @@ -0,0 +1,58 @@ +use super::LocatorStats; + +#[test] +fn observation_attempts_are_serialized_and_merged() { + let mut aggregate = LocatorStats::default(); + aggregate.reads.counts.observation_attempts = 1; + let mut next = LocatorStats::default(); + next.reads.counts.observation_attempts = 2; + + aggregate.merge_observation(&next); + + assert_eq!(aggregate.reads.counts.observation_attempts, 3); + let json = serde_json::to_value(aggregate).unwrap(); + assert_eq!(json["reads"]["observation_attempts"], 3); +} + +#[test] +fn child_label_limit_hits_are_serialized_and_merged_separately() { + let mut aggregate = LocatorStats::default(); + aggregate.traversal.limits.child_hits = 2; + let mut next = LocatorStats::default(); + next.traversal.limits.child_label_hits = 3; + + aggregate.merge_observation(&next); + + assert_eq!(aggregate.traversal.limits.child_hits, 2); + assert_eq!(aggregate.traversal.limits.child_label_hits, 3); + let json = serde_json::to_value(aggregate).unwrap(); + assert_eq!(json["traversal"]["limits"]["child_hits"], 2); + assert_eq!(json["traversal"]["limits"]["child_label_hits"], 3); +} + +#[test] +fn unclassified_native_read_failures_are_serialized_and_merged() { + let mut aggregate = LocatorStats::default(); + let mut next = LocatorStats::default(); + next.reads.health.native_read_failures = 2; + + aggregate.merge_observation(&next); + + assert_eq!(aggregate.reads.health.native_read_failures, 2); + let json = serde_json::to_value(aggregate).unwrap(); + assert_eq!(json["reads"]["native_read_failures"], 2); +} + +#[test] +fn read_stat_groups_keep_the_flat_json_contract() { + let mut stats = LocatorStats::default(); + stats.reads.counts.attribute_batches = 3; + stats.reads.health.deadline_exhausted = 2; + + let json = serde_json::to_value(stats).unwrap(); + + assert_eq!(json["reads"]["attribute_batches"], 3); + assert_eq!(json["reads"]["deadline_exhausted"], 2); + assert!(json["reads"].get("counts").is_none()); + assert!(json["reads"].get("health").is_none()); +} diff --git a/crates/core/src/live_locator/locator_traversal_stats.rs b/crates/core/src/live_locator/locator_traversal_stats.rs new file mode 100644 index 0000000..4a5a81c --- /dev/null +++ b/crates/core/src/live_locator/locator_traversal_stats.rs @@ -0,0 +1,14 @@ +use serde::Serialize; + +use super::LocatorLimitStats; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct LocatorTraversalStats { + pub nodes_visited: u64, + pub peak_handles_owned: u64, + pub max_raw_depth: u8, + pub max_logical_depth: u8, + pub web_wrapper_nodes: u64, + pub cycles_skipped: u64, + pub limits: LocatorLimitStats, +} diff --git a/crates/core/src/live_locator/match_verdict.rs b/crates/core/src/live_locator/match_verdict.rs new file mode 100644 index 0000000..ef076d2 --- /dev/null +++ b/crates/core/src/live_locator/match_verdict.rs @@ -0,0 +1,32 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MatchVerdict { + Match, + NoMatch, + Unknown, +} + +impl MatchVerdict { + pub(crate) fn and(self, other: Self) -> Self { + match (self, other) { + (Self::NoMatch, _) | (_, Self::NoMatch) => Self::NoMatch, + (Self::Match, Self::Match) => Self::Match, + _ => Self::Unknown, + } + } + + pub(crate) fn or(self, other: Self) -> Self { + match (self, other) { + (Self::Match, _) | (_, Self::Match) => Self::Match, + (Self::NoMatch, Self::NoMatch) => Self::NoMatch, + _ => Self::Unknown, + } + } + + pub(crate) fn negate(self) -> Self { + match self { + Self::Match => Self::NoMatch, + Self::NoMatch => Self::Match, + Self::Unknown => Self::Unknown, + } + } +} diff --git a/crates/core/src/live_locator/materialize.rs b/crates/core/src/live_locator/materialize.rs new file mode 100644 index 0000000..b2cdfd2 --- /dev/null +++ b/crates/core/src/live_locator/materialize.rs @@ -0,0 +1,153 @@ +use super::{LocatorEvidence, LocatorField, ObservationSource, ObservedTree}; +use crate::{ + adapter::SnapshotSurface, + locator::LocatorQuery, + ref_alloc::is_ref_able_role_actions, + refs::{RefEntry, RefMap, RefPath}, +}; + +pub(crate) fn materialize_refmap( + tree: &mut ObservedTree, + query: &LocatorQuery, + selected: Option<&[usize]>, +) -> Result<(RefMap, bool), crate::AdapterError> { + let source = &tree.source; + let nodes = &mut tree.nodes; + let mut refmap = RefMap::new(); + let mut complete = true; + let mut indices = selected + .map(|indices| indices.to_vec()) + .unwrap_or_else(|| (0..nodes.len()).collect()); + indices.sort_by_key(|index| nodes[*index].document_order); + for index in indices { + let node = &nodes[index]; + let (addressable, known) = addressability(&node.evidence); + complete &= known; + if !addressable { + continue; + } + let entry = ref_entry(node, source, query); + let ref_id = refmap.try_allocate(entry).map_err(|error| { + crate::AdapterError::new(crate::ErrorCode::InvalidArgs, error.to_string()) + })?; + nodes[index].ref_id = Some(ref_id); + } + Ok((refmap, complete)) +} + +pub(crate) fn addressability(evidence: &LocatorEvidence) -> (bool, bool) { + let Some(role) = evidence.role.known() else { + return (false, false); + }; + match &evidence.ref_evidence.available_actions { + LocatorField::Known(actions) => (is_ref_able_role_actions(role, actions), true), + LocatorField::Absent => (is_ref_able_role_actions(role, &[]), true), + LocatorField::Unknown => (is_ref_able_role_actions(role, &[]), false), + } +} + +pub(crate) fn ref_entry( + node: &super::ObservedNode, + source: &ObservationSource, + query: &LocatorQuery, +) -> RefEntry { + let role = node + .evidence + .role + .known() + .cloned() + .unwrap_or_else(|| "unknown".into()); + let bounds = node.evidence.ref_evidence.bounds.known().copied(); + let identifier = selected_identifier(&node.evidence, query.identity.native_id.as_deref()); + let mut entry = RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(0), + process_instance: None, + }, + identity: crate::RefEntryIdentity { + role: role.clone(), + name: node.evidence.name.meaningful_string(), + value: node.evidence.value.meaningful_string(), + description: node.evidence.description.meaningful_string(), + native_id: identifier, + }, + geometry: crate::RefGeometry { + bounds, + bounds_hash: bounds.as_ref().and_then(|value| value.bounds_hash()), + }, + capabilities: crate::RefCapabilities { + states: node.evidence.states.known().cloned().unwrap_or_default(), + available_actions: available_actions(&node.evidence), + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: node.path.clone(), + }, + }; + apply_source(&mut entry, source, &node.path); + entry +} + +fn apply_source(entry: &mut RefEntry, source: &ObservationSource, node_path: &RefPath) { + match source { + ObservationSource::Window(window) => { + entry.process.pid = window.pid; + entry.source.source_app = Some(window.app.clone()); + entry.source.source_window_id = Some(window.id.clone()); + entry.source.source_window_title = Some(window.title.clone()); + entry.source.source_window_bounds_hash = + window.bounds.as_ref().and_then(crate::Rect::bounds_hash); + entry.process.process_instance = window.process_instance.clone(); + } + ObservationSource::Element { + entry: source_entry, + root_ref, + } => { + entry.process = source_entry.process.clone(); + entry.source = source_entry.source.clone(); + entry.scope.root_ref = root_ref + .clone() + .or_else(|| source_entry.scope.root_ref.clone()); + entry.scope.path_is_absolute = + entry.scope.root_ref.is_some() || source_entry.scope.path_is_absolute; + let mut path = source_entry.scope.path.clone(); + path.extend(node_path.iter().copied()); + entry.scope.path = path; + } + } +} + +fn selected_identifier( + evidence: &LocatorEvidence, + expected: Option<&str>, +) -> Option<crate::ElementIdentifier> { + if let Some(expected) = expected { + if let Some(preferred) = evidence.identifiers.preferred_identifier() + && preferred.value == expected + { + return Some(preferred.clone()); + } + return evidence + .identifiers + .identifiers() + .iter() + .find(|identifier| identifier.value == expected) + .cloned(); + } + evidence.identifiers.preferred_identifier().cloned() +} + +fn available_actions(evidence: &LocatorEvidence) -> Vec<String> { + match &evidence.ref_evidence.available_actions { + LocatorField::Known(actions) => actions.clone(), + LocatorField::Absent | LocatorField::Unknown => Vec::new(), + } +} diff --git a/crates/core/src/live_locator/materialize_tests.rs b/crates/core/src/live_locator/materialize_tests.rs new file mode 100644 index 0000000..4be7535 --- /dev/null +++ b/crates/core/src/live_locator/materialize_tests.rs @@ -0,0 +1,267 @@ +use super::{ + IdentifierEvidence, LocatorField, LocatorMaterialization, LocatorResolveRequest, + LocatorSelection, ObservationSource, evaluate_locator_tree, +}; +use crate::{ + adapter::SnapshotSurface, + capability, + locator::IdentityPredicate, + locator::LocatorQuery, + refs::{RefEntry, RefPath}, +}; + +use super::test_support::{evidence, node, tree}; + +fn request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::FullRefMap, + } +} + +#[test] +fn selected_materialization_persists_only_returned_matches() { + let buttons = (0..512) + .map(|index| { + node( + index + 1, + evidence("button", Some(&format!("Button {index}"))), + Vec::new(), + &[index as usize], + ) + }) + .collect::<Vec<_>>(); + let children = (0..512).map(|index| index + 1).collect::<Vec<_>>(); + let mut nodes = vec![node(0, evidence("window", Some("Fixture")), children, &[])]; + nodes.extend(buttons); + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + let resolution = evaluate_locator_tree( + tree(nodes, vec![0], true), + &query, + &LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::SelectedMatches, + }, + ) + .unwrap(); + + assert_eq!(resolution.refmap.as_ref().unwrap().len(), 1); + assert_eq!(resolution.matches[0].data.ref_id.as_deref(), Some("@e1")); + assert_eq!(resolution.matches[0].data.name, "Button 0"); +} + +#[test] +fn full_refmap_uses_same_arena_evidence_and_document_order() { + let mut button = evidence("button", Some("Checkout")); + button.identifiers = IdentifierEvidence::typed( + ["internal-id", "checkout"].map(|value| crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: value.into(), + }), + Some(0), + true, + ); + let mut disclosure = evidence("disclosure", Some("Details")); + disclosure.ref_evidence.available_actions = + LocatorField::Known(vec![capability::EXPAND.into()]); + let mut root = evidence("window", Some("Fixture")); + root.ref_evidence.available_actions = LocatorField::Known(vec![capability::SET_FOCUS.into()]); + let query = LocatorQuery { + identity: IdentityPredicate { + native_id: Some("checkout".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + + let resolution = evaluate_locator_tree( + tree( + vec![ + node(1, button, vec![], &[0]), + node(2, disclosure, vec![], &[1]), + node(0, root, vec![0, 1], &[]), + ], + vec![2], + true, + ), + &query, + &request(), + ) + .unwrap(); + + let refmap = resolution.refmap.as_ref().unwrap(); + assert_eq!(refmap.len(), 2); + let checkout = refmap.get("@e1").unwrap(); + assert_eq!( + checkout + .identity + .native_id + .as_ref() + .map(|identifier| identifier.value.as_str()), + Some("checkout") + ); + assert_eq!(checkout.scope.path.as_slice(), &[0]); + assert!(checkout.capabilities.available_actions.is_empty()); + assert_eq!(resolution.matches[0].data.ref_id.as_deref(), Some("@e1")); + assert_eq!(resolution.matches[0].data.path, ["window:Fixture"]); +} + +#[test] +fn selected_identifier_prefers_typed_preferred_duplicate() { + let mut button = evidence("button", Some("Checkout")); + button.identifiers = IdentifierEvidence::typed( + [ + crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: "checkout".into(), + }, + crate::ElementIdentifier { + kind: crate::IdentifierKind::AxDomIdentifier, + value: "checkout".into(), + }, + ], + Some(1), + true, + ); + let query = LocatorQuery { + identity: IdentityPredicate { + native_id: Some("checkout".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + + let resolution = evaluate_locator_tree( + tree(vec![node(0, button, vec![], &[])], vec![0], true), + &query, + &request(), + ) + .unwrap(); + + assert_eq!( + resolution + .refmap + .unwrap() + .get("@e1") + .unwrap() + .identity + .native_id, + Some(crate::ElementIdentifier { + kind: crate::IdentifierKind::AxDomIdentifier, + value: "checkout".into(), + }) + ); +} + +#[test] +fn unknown_noninteractive_actions_make_full_refmap_incomplete() { + let mut group = evidence("group", Some("Container")); + group.ref_evidence.available_actions = LocatorField::Unknown; + let resolution = evaluate_locator_tree( + tree(vec![node(0, group, vec![], &[])], vec![0], true), + &LocatorQuery::default(), + &request(), + ) + .unwrap(); + assert!(!resolution.meta.complete); + assert!(resolution.refmap.unwrap().is_empty()); +} + +#[test] +fn window_source_materialization_preserves_geometry_generation_evidence() { + let bounds = crate::Rect { + x: 10.0, + y: 20.0, + width: 800.0, + height: 600.0, + }; + let mut observed = tree( + vec![node(0, evidence("button", Some("Save")), vec![], &[])], + vec![0], + true, + ); + let ObservationSource::Window(window) = &mut observed.source else { + panic!("fixture must use a window source"); + }; + window.bounds = Some(bounds); + + let resolution = evaluate_locator_tree(observed, &LocatorQuery::default(), &request()).unwrap(); + let saved = resolution.refmap.unwrap().get("@e1").unwrap().clone(); + + assert_eq!(saved.source.source_window_bounds_hash, bounds.bounds_hash()); +} + +#[test] +fn scoped_refmap_extends_source_path_without_index_correlation() { + let root = evidence("group", Some("Scope")); + let button = evidence("button", Some("Save")); + let mut scoped = tree( + vec![node(1, button, vec![], &[0]), node(0, root, vec![0], &[])], + vec![1], + true, + ); + scoped.source = ObservationSource::Element { + entry: Box::new(source_entry()), + root_ref: Some("@e7".into()), + }; + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + let resolution = evaluate_locator_tree(scoped, &query, &request()).unwrap(); + let refmap = resolution.refmap.unwrap(); + let entry = refmap.get("@e1").unwrap(); + assert_eq!(entry.scope.path.as_slice(), &[3, 0]); + assert_eq!(entry.scope.root_ref.as_deref(), Some("@e7")); + assert!(entry.scope.path_is_absolute); + assert_eq!(entry.source.source_surface, SnapshotSurface::Menu); +} + +fn source_entry() -> RefEntry { + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "group".into(), + name: Some("Scope".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: Vec::new(), + available_actions: Vec::new(), + }, + source: crate::RefSource { + source_app: Some("FixtureApp".into()), + source_window_id: Some("w-1".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Menu, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: RefPath::from_slice(&[3]), + }, + } +} diff --git a/crates/core/src/live_locator/mod.rs b/crates/core/src/live_locator/mod.rs new file mode 100644 index 0000000..79d7592 --- /dev/null +++ b/crates/core/src/live_locator/mod.rs @@ -0,0 +1,118 @@ +mod cardinality; +mod compiled_clause; +mod evaluate; +mod evaluation_buffers; +mod evidence_plan; +mod evidence_requirements; +mod hydrate; +mod identifier_evidence; +mod locator_activation_stats; +mod locator_cardinality; +mod locator_evaluation_stats; +mod locator_evidence; +mod locator_field; +mod locator_identifier_stats; +mod locator_limit_stats; +mod locator_match; +mod locator_match_data; +mod locator_materialization; +mod locator_read_counts; +mod locator_read_health; +mod locator_read_stats; +mod locator_ref_evidence; +mod locator_resolution; +mod locator_resolution_meta; +mod locator_resolve_request; +mod locator_selection; +mod locator_semantic_read_stats; +mod locator_stats; +#[cfg(test)] +mod locator_stats_tests; +mod locator_traversal_stats; +mod match_verdict; +mod materialize; +mod observation_budget; +mod observation_completeness; +mod observation_request; +mod observation_root; +mod observation_source; +mod observed_node; +mod observed_subtree; +mod observed_tree; +mod predicate; +mod ref_evidence_requirements; +mod resolve; +mod select; +mod selection_completeness; +mod tree_order; +mod validate; + +pub use cardinality::{classify_query_result, require_unique}; +pub use evaluate::evaluate_locator_tree; +pub use evidence_requirements::EvidenceRequirements; +pub use identifier_evidence::IdentifierEvidence; +pub use locator_activation_stats::LocatorActivationStats; +pub use locator_cardinality::LocatorCardinality; +pub use locator_evaluation_stats::LocatorEvaluationStats; +pub use locator_evidence::LocatorEvidence; +pub use locator_field::LocatorField; +pub use locator_identifier_stats::LocatorIdentifierStats; +pub use locator_limit_stats::LocatorLimitStats; +pub use locator_match::LocatorMatch; +pub use locator_match_data::LocatorMatchData; +pub use locator_materialization::LocatorMaterialization; +pub use locator_read_counts::LocatorReadCounts; +pub use locator_read_health::LocatorReadHealth; +pub use locator_read_stats::LocatorReadStats; +pub use locator_ref_evidence::LocatorRefEvidence; +pub use locator_resolution::LocatorResolution; +pub use locator_resolution_meta::LocatorResolutionMeta; +pub use locator_resolve_request::LocatorResolveRequest; +pub use locator_selection::LocatorSelection; +pub use locator_semantic_read_stats::LocatorSemanticReadStats; +pub use locator_stats::LocatorStats; +pub use locator_traversal_stats::LocatorTraversalStats; +pub use observation_budget::ObservationBudget; +pub(crate) use observation_completeness::ObservationCompleteness; +pub use observation_request::ObservationRequest; +pub use observation_root::ObservationRoot; +pub use observation_source::ObservationSource; +pub use observed_node::ObservedNode; +pub use observed_subtree::ObservedSubtree; +pub use observed_tree::ObservedTree; +pub use ref_evidence_requirements::RefEvidenceRequirements; +pub use resolve::{find_first_entry, resolve_query}; +pub use validate::{validate_query, validate_request}; + +#[cfg(test)] +mod evaluator_identifier_tests; +#[cfg(test)] +mod evaluator_tests; +#[cfg(test)] +mod materialize_tests; +#[cfg(test)] +mod observed_tree_tests; +#[cfg(test)] +mod ownership_tests; +#[cfg(test)] +mod resolve_query_hydration_tests; +#[cfg(test)] +mod resolve_query_tests; +#[cfg(test)] +mod resolve_tests; +#[cfg(test)] +mod scrollarea_name_tests; +#[cfg(test)] +mod selected_hydration_churn_tests; +#[cfg(test)] +mod selected_hydration_contract_tests; +#[cfg(test)] +mod selected_hydration_subtree_tests; +#[cfg(test)] +mod selected_hydration_tests; +#[cfg(test)] +mod selection_completeness_tests; +#[cfg(test)] +mod test_support; +#[cfg(test)] +mod validation_tests; diff --git a/crates/core/src/live_locator/observation_budget.rs b/crates/core/src/live_locator/observation_budget.rs new file mode 100644 index 0000000..a470cc8 --- /dev/null +++ b/crates/core/src/live_locator/observation_budget.rs @@ -0,0 +1,71 @@ +use crate::{AdapterError, ErrorCode}; + +pub const MAX_OBSERVATION_NODES: usize = 50_000; +pub const MAX_OBSERVATION_EDGES: usize = 100_000; +pub const MAX_CHILDREN_PER_NODE: usize = 4_096; +pub const MAX_OBSERVATION_FIELD_BYTES: usize = 64 * 1024; +pub const MAX_OBSERVATION_TEXT_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObservationBudget { + pub max_nodes: usize, + pub max_edges: usize, + pub max_children_per_node: usize, + pub max_field_bytes: usize, + pub max_text_bytes: usize, +} + +impl ObservationBudget { + pub fn validate(self) -> Result<Self, AdapterError> { + validate_limit("max_nodes", self.max_nodes, MAX_OBSERVATION_NODES)?; + validate_limit("max_edges", self.max_edges, MAX_OBSERVATION_EDGES)?; + validate_limit( + "max_children_per_node", + self.max_children_per_node, + MAX_CHILDREN_PER_NODE, + )?; + validate_limit( + "max_field_bytes", + self.max_field_bytes, + MAX_OBSERVATION_FIELD_BYTES, + )?; + validate_limit( + "max_text_bytes", + self.max_text_bytes, + MAX_OBSERVATION_TEXT_BYTES, + )?; + if self.max_field_bytes > self.max_text_bytes { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "max_field_bytes cannot exceed max_text_bytes", + )); + } + Ok(self) + } +} + +impl Default for ObservationBudget { + fn default() -> Self { + Self { + max_nodes: MAX_OBSERVATION_NODES, + max_edges: MAX_OBSERVATION_EDGES, + max_children_per_node: MAX_CHILDREN_PER_NODE, + max_field_bytes: MAX_OBSERVATION_FIELD_BYTES, + max_text_bytes: MAX_OBSERVATION_TEXT_BYTES, + } + } +} + +fn validate_limit(name: &str, value: usize, maximum: usize) -> Result<(), AdapterError> { + if (1..=maximum).contains(&value) { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("{name} must be between 1 and {maximum}, got {value}"), + )) +} + +#[cfg(test)] +#[path = "observation_budget_tests.rs"] +mod tests; diff --git a/crates/core/src/live_locator/observation_budget_tests.rs b/crates/core/src/live_locator/observation_budget_tests.rs new file mode 100644 index 0000000..a512829 --- /dev/null +++ b/crates/core/src/live_locator/observation_budget_tests.rs @@ -0,0 +1,51 @@ +use super::*; + +#[test] +fn defaults_are_valid_hard_limits() { + let budget = ObservationBudget::default().validate().unwrap(); + assert_eq!(budget.max_nodes, MAX_OBSERVATION_NODES); + assert_eq!(budget.max_edges, MAX_OBSERVATION_EDGES); + assert_eq!(budget.max_children_per_node, MAX_CHILDREN_PER_NODE); + assert_eq!(budget.max_field_bytes, MAX_OBSERVATION_FIELD_BYTES); + assert_eq!(budget.max_text_bytes, MAX_OBSERVATION_TEXT_BYTES); +} + +#[test] +fn zero_and_oversized_limits_are_rejected() { + for budget in [ + ObservationBudget { + max_nodes: 0, + ..ObservationBudget::default() + }, + ObservationBudget { + max_edges: MAX_OBSERVATION_EDGES + 1, + ..ObservationBudget::default() + }, + ObservationBudget { + max_children_per_node: 0, + ..ObservationBudget::default() + }, + ObservationBudget { + max_field_bytes: MAX_OBSERVATION_FIELD_BYTES + 1, + ..ObservationBudget::default() + }, + ObservationBudget { + max_text_bytes: MAX_OBSERVATION_TEXT_BYTES + 1, + ..ObservationBudget::default() + }, + ] { + assert_eq!(budget.validate().unwrap_err().code, ErrorCode::InvalidArgs); + } +} + +#[test] +fn field_budget_must_fit_total_text_budget() { + let error = ObservationBudget { + max_field_bytes: 2, + max_text_bytes: 1, + ..ObservationBudget::default() + } + .validate() + .unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidArgs); +} diff --git a/crates/core/src/live_locator/observation_completeness.rs b/crates/core/src/live_locator/observation_completeness.rs new file mode 100644 index 0000000..a272894 --- /dev/null +++ b/crates/core/src/live_locator/observation_completeness.rs @@ -0,0 +1,14 @@ +#[derive(Debug, Clone, Copy)] +pub(crate) struct ObservationCompleteness { + pub(crate) subtree_complete: bool, + pub(crate) predecessors_complete: bool, +} + +impl ObservationCompleteness { + pub(crate) fn new(subtree_complete: bool) -> Self { + Self { + subtree_complete, + predecessors_complete: true, + } + } +} diff --git a/crates/core/src/live_locator/observation_request.rs b/crates/core/src/live_locator/observation_request.rs new file mode 100644 index 0000000..98010f2 --- /dev/null +++ b/crates/core/src/live_locator/observation_request.rs @@ -0,0 +1,132 @@ +use crate::{ + AdapterError, Deadline, ErrorCode, adapter::TreeOptions, locator::LocatorQuery, + snapshot_surface::SnapshotSurface, +}; + +use super::{ + EvidenceRequirements, LocatorResolveRequest, ObservationBudget, evidence_plan::EvidencePlan, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObservationRequest { + pub deadline: Deadline, + pub max_raw_depth: u8, + pub max_logical_depth: u8, + pub surface: SnapshotSurface, + pub skeleton: bool, + evidence_plan: EvidencePlan, + pub budget: ObservationBudget, +} + +impl ObservationRequest { + pub fn validate(self) -> Result<Self, AdapterError> { + self.budget.validate()?; + if !(1..=50).contains(&self.max_raw_depth) { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "max_raw_depth must be between 1 and 50", + )); + } + if self.max_logical_depth > self.max_raw_depth { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "max_logical_depth cannot exceed max_raw_depth", + )); + } + if self.skeleton && self.max_logical_depth > 3 { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "skeleton observations support a maximum logical depth of 3", + )); + } + self.evidence_plan.validate()?; + Ok(self) + } + + pub fn snapshot(options: &TreeOptions, deadline: Deadline) -> Self { + Self { + deadline, + max_raw_depth: 50, + max_logical_depth: if options.skeleton { + options.max_depth.min(3) + } else { + options.max_depth + }, + surface: options.surface, + skeleton: options.skeleton, + evidence_plan: EvidencePlan::uniform(EvidenceRequirements::snapshot()), + budget: ObservationBudget::default(), + } + } + + pub fn locator( + query: &LocatorQuery, + request: &LocatorResolveRequest, + deadline: Deadline, + ) -> Self { + Self { + deadline, + max_raw_depth: request.max_raw_depth, + max_logical_depth: request.max_raw_depth, + surface: SnapshotSurface::Window, + skeleton: false, + evidence_plan: EvidencePlan::uniform(EvidenceRequirements::locator(query, request)), + budget: ObservationBudget::default(), + } + } + + pub(crate) fn locator_for_root( + query: &LocatorQuery, + request: &LocatorResolveRequest, + root: super::ObservationRoot<'_>, + deadline: Deadline, + ) -> Self { + Self { + surface: root.surface(), + ..Self::locator(query, request, deadline) + } + } + + pub(crate) fn selected_hydration( + query: &LocatorQuery, + request: &LocatorResolveRequest, + root: super::ObservationRoot<'_>, + deadline: Deadline, + ) -> Self { + let subtree_query = query.has_text.is_some() + || query.containment.has.is_some() + || query.containment.has_not.is_some(); + Self { + deadline, + max_raw_depth: if subtree_query { + request.max_raw_depth + } else { + 1 + }, + max_logical_depth: if subtree_query { + request.max_raw_depth + } else { + 0 + }, + surface: root.surface(), + skeleton: false, + evidence_plan: EvidencePlan::rooted( + EvidenceRequirements::snapshot(), + EvidenceRequirements::query(query), + ), + budget: ObservationBudget::default(), + } + } + + pub fn evidence_for_raw_depth(self, raw_depth: u8) -> EvidenceRequirements { + self.evidence_plan.for_raw_depth(raw_depth) + } + + pub fn descendant_evidence(self) -> EvidenceRequirements { + self.evidence_plan.descendants() + } + + pub fn hydrates_root_name_from_children(self) -> bool { + self.evidence_plan.hydrates_root_name_from_children() + } +} diff --git a/crates/core/src/live_locator/observation_root.rs b/crates/core/src/live_locator/observation_root.rs new file mode 100644 index 0000000..0093c2d --- /dev/null +++ b/crates/core/src/live_locator/observation_root.rs @@ -0,0 +1,20 @@ +use crate::{WindowInfo, native_handle::NativeHandle, refs::RefEntry}; + +#[derive(Clone, Copy)] +pub enum ObservationRoot<'a> { + Window(&'a WindowInfo), + Element { + handle: &'a NativeHandle, + entry: &'a RefEntry, + root_ref: Option<&'a str>, + }, +} + +impl ObservationRoot<'_> { + pub fn surface(self) -> crate::SnapshotSurface { + match self { + Self::Window(_) => crate::SnapshotSurface::Window, + Self::Element { entry, .. } => entry.source.source_surface, + } + } +} diff --git a/crates/core/src/live_locator/observation_source.rs b/crates/core/src/live_locator/observation_source.rs new file mode 100644 index 0000000..3c68bc3 --- /dev/null +++ b/crates/core/src/live_locator/observation_source.rs @@ -0,0 +1,26 @@ +use crate::{WindowInfo, refs::RefEntry}; + +use super::ObservationRoot; + +#[derive(Debug, Clone)] +pub enum ObservationSource { + Window(WindowInfo), + Element { + entry: Box<RefEntry>, + root_ref: Option<String>, + }, +} + +impl ObservationSource { + pub fn from_root(root: &ObservationRoot<'_>) -> Self { + match root { + ObservationRoot::Window(window) => Self::Window((*window).clone()), + ObservationRoot::Element { + entry, root_ref, .. + } => Self::Element { + entry: Box::new((*entry).clone()), + root_ref: root_ref.map(str::to_string), + }, + } + } +} diff --git a/crates/core/src/live_locator/observed_node.rs b/crates/core/src/live_locator/observed_node.rs new file mode 100644 index 0000000..fceb65d --- /dev/null +++ b/crates/core/src/live_locator/observed_node.rs @@ -0,0 +1,14 @@ +use crate::refs::RefPath; + +use super::{LocatorEvidence, ObservationCompleteness}; + +#[derive(Debug, Clone)] +pub struct ObservedNode { + pub(crate) evidence: LocatorEvidence, + pub(crate) path: RefPath, + pub(crate) children: Vec<u32>, + pub(crate) document_order: u32, + pub(crate) completeness: ObservationCompleteness, + pub(crate) children_count: Option<u32>, + pub(crate) ref_id: Option<String>, +} diff --git a/crates/core/src/live_locator/observed_subtree.rs b/crates/core/src/live_locator/observed_subtree.rs new file mode 100644 index 0000000..c2e1442 --- /dev/null +++ b/crates/core/src/live_locator/observed_subtree.rs @@ -0,0 +1,41 @@ +use super::{LocatorEvidence, ObservationCompleteness}; + +pub struct ObservedSubtree { + pub(crate) evidence: LocatorEvidence, + pub(crate) children: Vec<Self>, + pub(crate) completeness: ObservationCompleteness, + pub(crate) children_count: Option<u32>, + pub(crate) source_child_index: Option<usize>, +} + +impl ObservedSubtree { + pub fn new( + evidence: LocatorEvidence, + children: Vec<Self>, + subtree_complete: bool, + children_count: Option<u32>, + ) -> Self { + Self { + evidence, + children, + completeness: ObservationCompleteness::new(subtree_complete), + children_count, + source_child_index: None, + } + } + + /// Preserves this node's index in its platform-native parent child list. + pub fn with_source_child_index(mut self, source_child_index: usize) -> Self { + self.source_child_index = Some(source_child_index); + self + } + + pub fn with_predecessors_complete(mut self, predecessors_complete: bool) -> Self { + self.completeness.predecessors_complete = predecessors_complete; + self + } + + pub fn is_complete(&self) -> bool { + self.completeness.subtree_complete + } +} diff --git a/crates/core/src/live_locator/observed_tree.rs b/crates/core/src/live_locator/observed_tree.rs new file mode 100644 index 0000000..8fba464 --- /dev/null +++ b/crates/core/src/live_locator/observed_tree.rs @@ -0,0 +1,158 @@ +use serde_json::json; + +use crate::{AccessibilityNode, AdapterError, ErrorCode, refs::RefPath}; + +use super::{LocatorStats, ObservationSource, ObservedNode, ObservedSubtree}; + +#[derive(Debug, Clone)] +pub struct ObservedTree { + pub(crate) nodes: Vec<ObservedNode>, + pub(crate) roots: Vec<u32>, + pub(crate) source: ObservationSource, + pub(crate) stats: LocatorStats, + pub(crate) structurally_complete: bool, +} + +impl ObservedTree { + pub fn from_roots( + roots: Vec<ObservedSubtree>, + source: ObservationSource, + stats: LocatorStats, + structurally_complete: bool, + ) -> Result<Self, AdapterError> { + if roots.is_empty() { + return Err(AdapterError::internal("observation contains no roots")); + } + let mut tree = Self { + nodes: Vec::new(), + roots: Vec::new(), + source, + stats, + structurally_complete, + }; + for root in roots { + let index = tree.append(root, RefPath::new())?; + tree.roots.push(index); + } + tree.structurally_complete &= tree + .roots + .iter() + .all(|root| tree.nodes[*root as usize].completeness.subtree_complete); + Ok(tree) + } + + pub fn retained_handle_count(&self) -> usize { + 0 + } + + pub fn node_count(&self) -> usize { + self.nodes.len() + } + + pub fn is_complete(&self) -> bool { + self.structurally_complete + } + + pub fn into_accessibility_tree(self) -> Result<AccessibilityNode, AdapterError> { + if !self.structurally_complete { + return Err(AdapterError::new( + ErrorCode::Timeout, + "Accessibility observation was incomplete", + ) + .with_suggestion("Retry with a larger timeout or a narrower tree depth") + .with_details(json!({ + "kind": "observation_incomplete", + "nodes_observed": self.nodes.len(), + "query_stats": self.stats, + }))); + } + if self.roots.len() != 1 { + return Err(AdapterError::internal( + "accessibility projection requires exactly one root", + )); + } + self.project(self.roots[0] as usize) + } + + fn append(&mut self, subtree: ObservedSubtree, path: RefPath) -> Result<u32, AdapterError> { + let index = u32::try_from(self.nodes.len()) + .map_err(|_| AdapterError::internal("observed tree exceeds u32"))?; + let document_order = index; + let ObservedSubtree { + evidence, + children, + completeness, + children_count, + source_child_index: _, + } = subtree; + self.nodes.push(ObservedNode { + evidence, + path: path.clone(), + children: Vec::new(), + document_order, + completeness, + children_count, + ref_id: None, + }); + let mut child_indices = Vec::with_capacity(children.len()); + let mut complete = completeness.subtree_complete; + for (child_order, child) in children.into_iter().enumerate() { + let mut child_path = path.clone(); + child_path.push(child.source_child_index.unwrap_or(child_order)); + let child_index = self.append(child, child_path)?; + complete &= self.nodes[child_index as usize] + .completeness + .subtree_complete; + child_indices.push(child_index); + } + let node = self + .nodes + .get_mut(index as usize) + .ok_or_else(|| AdapterError::internal("observed node disappeared during build"))?; + node.children = child_indices; + node.completeness.subtree_complete = complete; + Ok(index) + } + + fn project(&self, index: usize) -> Result<AccessibilityNode, AdapterError> { + let node = self + .nodes + .get(index) + .ok_or_else(|| AdapterError::internal("observed child index is out of bounds"))?; + let role = node + .evidence + .role + .known() + .cloned() + .unwrap_or_else(|| "unknown".into()); + let children = node + .children + .iter() + .map(|child| self.project(*child as usize)) + .collect::<Result<Vec<_>, _>>()?; + Ok(AccessibilityNode { + ref_id: node.ref_id.clone(), + role, + identity: crate::NodeIdentity { + name: node.evidence.name.meaningful_string(), + value: node.evidence.value.meaningful_string(), + description: node.evidence.description.meaningful_string(), + native_id: node.evidence.identifiers.preferred_identifier().cloned(), + }, + presentation: crate::NodePresentation { + hint: None, + states: node.evidence.states.known().cloned().unwrap_or_default(), + available_actions: node + .evidence + .ref_evidence + .available_actions + .known() + .cloned() + .unwrap_or_default(), + bounds: node.evidence.ref_evidence.bounds.known().copied(), + }, + children_count: node.children_count, + children, + }) + } +} diff --git a/crates/core/src/live_locator/observed_tree_tests.rs b/crates/core/src/live_locator/observed_tree_tests.rs new file mode 100644 index 0000000..edbd108 --- /dev/null +++ b/crates/core/src/live_locator/observed_tree_tests.rs @@ -0,0 +1,154 @@ +use super::{ + IdentifierEvidence, LocatorMaterialization, LocatorResolveRequest, LocatorSelection, + ObservationSource, ObservedSubtree, ObservedTree, evaluate_locator_tree, +}; +use crate::{ + WindowInfo, + locator::{IdentityPredicate, LocatorQuery}, +}; + +use super::test_support::evidence; + +fn source() -> ObservationSource { + ObservationSource::Window(WindowInfo { + id: "w-1".into(), + title: "Fixture".into(), + app: "FixtureApp".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }) +} + +fn subtree(role: &str, name: &str, children: Vec<ObservedSubtree>) -> ObservedSubtree { + ObservedSubtree::new(evidence(role, Some(name)), children, true, None) +} + +#[test] +fn core_builder_owns_preorder_paths_and_child_indices() { + let tree = ObservedTree::from_roots( + vec![subtree( + "window", + "Fixture", + vec![ + subtree("button", "Save", Vec::new()), + subtree("link", "Help", Vec::new()), + ], + )], + source(), + Default::default(), + true, + ) + .unwrap(); + + assert_eq!(tree.roots, [0]); + assert_eq!(tree.nodes[0].children, [1, 2]); + assert_eq!(tree.nodes[0].document_order, 0); + assert_eq!(tree.nodes[1].document_order, 1); + assert_eq!(tree.nodes[2].document_order, 2); + assert!(tree.nodes[0].path.is_empty()); + assert_eq!(tree.nodes[1].path.as_slice(), &[0]); + assert_eq!(tree.nodes[2].path.as_slice(), &[1]); + assert_eq!(tree.retained_handle_count(), 0); +} + +#[test] +fn source_child_indices_survive_omitted_siblings() { + let target = subtree("scrollarea", "Nested", Vec::new()).with_source_child_index(0); + let retained = subtree("group", "Wrapper", vec![target]) + .with_source_child_index(2) + .with_predecessors_complete(false); + let tree = ObservedTree::from_roots( + vec![subtree("window", "Fixture", vec![retained])], + source(), + Default::default(), + true, + ) + .unwrap(); + + assert_eq!(tree.nodes[1].path.as_slice(), &[2]); + assert_eq!(tree.nodes[2].path.as_slice(), &[2, 0]); + assert!(!tree.nodes[1].completeness.predecessors_complete); + assert!(tree.nodes[2].completeness.predecessors_complete); +} + +#[test] +fn incomplete_descendant_propagates_to_the_observation_root() { + let incomplete = + ObservedSubtree::new(evidence("button", Some("Save")), Vec::new(), false, None); + let tree = ObservedTree::from_roots( + vec![subtree("window", "Fixture", vec![incomplete])], + source(), + Default::default(), + true, + ) + .unwrap(); + + assert!(!tree.is_complete()); + assert!(tree.into_accessibility_tree().is_err()); +} + +#[test] +fn snapshot_projection_and_find_share_the_same_observation() { + let tree = ObservedTree::from_roots( + vec![subtree( + "window", + "Fixture", + vec![subtree("button", "Save", Vec::new())], + )], + source(), + Default::default(), + true, + ) + .unwrap(); + let snapshot = tree.clone().into_accessibility_tree().unwrap(); + let query = LocatorQuery { + identity: IdentityPredicate { + name: Some("save".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + let resolution = evaluate_locator_tree( + tree, + &query, + &LocatorResolveRequest { + selection: LocatorSelection::Count, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(1)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + }, + ) + .unwrap(); + + assert_eq!(resolution.meta.total_matches, 1); + assert_eq!( + snapshot + .children + .iter() + .filter(|node| crate::accessibility_node_matches(node, &query)) + .count(), + 1 + ); +} + +#[test] +fn identifier_evidence_preserves_preference_without_platform_names() { + let identifiers = IdentifierEvidence::new( + [ + "automation-id".into(), + "runtime-id".into(), + "automation-id".into(), + ], + Some(0), + true, + ); + + assert_eq!(identifiers.values(), ["automation-id", "runtime-id"]); + assert_eq!(identifiers.preferred_value(), Some("automation-id")); + assert!(identifiers.is_complete()); +} diff --git a/crates/core/src/live_locator/ownership_tests.rs b/crates/core/src/live_locator/ownership_tests.rs new file mode 100644 index 0000000..8a00a08 --- /dev/null +++ b/crates/core/src/live_locator/ownership_tests.rs @@ -0,0 +1,62 @@ +use super::{ + LocatorMaterialization, LocatorResolveRequest, LocatorSelection, evaluate_locator_tree, +}; +use crate::{locator::LocatorQuery, refs::RefPath}; + +use super::test_support::{evidence, node, tree}; + +fn request(selection: LocatorSelection) -> LocatorResolveRequest { + LocatorResolveRequest { + selection, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + } +} + +#[test] +fn six_thousand_four_hundred_one_nodes_retain_zero_native_handles() { + let nodes = (0..6_401) + .map(|order| node(order, evidence("button", Some("match")), vec![], &[])) + .collect::<Vec<_>>(); + let roots = (0..6_401).collect::<Vec<_>>(); + let observed = tree(nodes, roots, true); + + assert_eq!(observed.node_count(), 6_401); + assert_eq!(observed.retained_handle_count(), 0); +} + +#[test] +fn count_returns_no_target_ownership() { + let resolution = evaluate_locator_tree( + tree( + vec![node(0, evidence("button", Some("match")), vec![], &[])], + vec![0], + true, + ), + &LocatorQuery::default(), + &request(LocatorSelection::Count), + ) + .unwrap(); + + assert!(resolution.matches.is_empty()); +} + +#[test] +fn invalid_tree_fails_before_evaluation() { + let mut invalid = tree( + vec![node(0, evidence("button", Some("match")), vec![], &[])], + vec![0], + true, + ); + invalid.nodes[0].path = RefPath::from_slice(&[1]); + let error = evaluate_locator_tree( + invalid, + &LocatorQuery::default(), + &request(LocatorSelection::Strict), + ) + .err() + .unwrap(); + + assert_eq!(error.code, crate::ErrorCode::Internal); +} diff --git a/crates/core/src/live_locator/predicate.rs b/crates/core/src/live_locator/predicate.rs new file mode 100644 index 0000000..e2fa763 --- /dev/null +++ b/crates/core/src/live_locator/predicate.rs @@ -0,0 +1,196 @@ +use super::match_verdict::MatchVerdict; +use super::{LocatorEvidence, LocatorField, LocatorIdentifierStats}; +use crate::{ + locator::{ContainmentPredicate, IdentityPredicate, LocatorQuery}, + roles, search_text, state, +}; + +pub(crate) fn normalize_query(query: &LocatorQuery) -> LocatorQuery { + LocatorQuery { + identity: IdentityPredicate { + role: query + .identity + .role + .as_deref() + .map(roles::normalize_role_query), + name: query.identity.name.as_deref().map(search_text::normalize), + description: query + .identity + .description + .as_deref() + .map(search_text::normalize), + native_id: query.identity.native_id.clone(), + value: query.identity.value.as_deref().map(search_text::normalize), + }, + has_text: query.has_text.as_deref().map(search_text::normalize), + exact: query.exact, + states: query.states.clone(), + containment: ContainmentPredicate { + has: query + .containment + .has + .as_deref() + .map(normalize_query) + .map(Box::new), + has_not: query + .containment + .has_not + .as_deref() + .map(normalize_query) + .map(Box::new), + }, + } +} + +pub(crate) fn self_verdict( + query: &LocatorQuery, + evidence: &LocatorEvidence, + identifier_stats: &mut LocatorIdentifierStats, +) -> MatchVerdict { + let (identifier, identifier_match) = + identifier_verdict(query.identity.native_id.as_deref(), evidence); + let verdict = role_verdict(query.identity.role.as_deref(), &evidence.role) + .and(text_verdict( + query.identity.name.as_deref(), + &evidence.name, + query.exact, + )) + .and(text_verdict( + query.identity.description.as_deref(), + &evidence.description, + query.exact, + )) + .and(identifier) + .and(text_verdict( + query.identity.value.as_deref(), + &evidence.value, + query.exact, + )) + .and(states_verdict(query, evidence)); + if verdict == MatchVerdict::Match { + match identifier_match { + Some(IdentifierMatch::Preferred) => identifier_stats.preferred_matches += 1, + Some(IdentifierMatch::Fallback) => identifier_stats.fallback_matches += 1, + None => {} + } + } + verdict +} + +pub(crate) fn self_text_verdict( + expected: Option<&str>, + evidence: &LocatorEvidence, + exact: bool, +) -> MatchVerdict { + let Some(expected) = expected else { + return MatchVerdict::Match; + }; + let mut verdict = MatchVerdict::NoMatch; + for field in [&evidence.name, &evidence.description, &evidence.value] { + verdict = verdict.or(match field { + LocatorField::Known(actual) => bool_verdict(if exact { + search_text::normalize(actual) == expected + } else { + search_text::contains(actual, expected) + }), + LocatorField::Absent => MatchVerdict::NoMatch, + LocatorField::Unknown => MatchVerdict::Unknown, + }); + } + verdict +} + +fn role_verdict(expected: Option<&str>, actual: &LocatorField<String>) -> MatchVerdict { + let Some(expected) = expected else { + return MatchVerdict::Match; + }; + field_equality(actual, expected) +} + +fn text_verdict( + expected: Option<&str>, + actual: &LocatorField<String>, + exact: bool, +) -> MatchVerdict { + let Some(expected) = expected else { + return MatchVerdict::Match; + }; + match actual { + LocatorField::Known(actual) => { + let matched = if exact { + search_text::normalize(actual) == expected + } else { + search_text::contains(actual, expected) + }; + bool_verdict(matched) + } + LocatorField::Absent => MatchVerdict::NoMatch, + LocatorField::Unknown => MatchVerdict::Unknown, + } +} + +fn identifier_verdict( + expected: Option<&str>, + evidence: &LocatorEvidence, +) -> (MatchVerdict, Option<IdentifierMatch>) { + let Some(expected) = expected else { + return (MatchVerdict::Match, None); + }; + if evidence.identifiers.preferred_value() == Some(expected) { + return (MatchVerdict::Match, Some(IdentifierMatch::Preferred)); + } + if evidence + .identifiers + .identifiers() + .iter() + .any(|actual| actual.value == expected) + { + return (MatchVerdict::Match, Some(IdentifierMatch::Fallback)); + } + let verdict = if !evidence.identifiers.is_complete() { + MatchVerdict::Unknown + } else { + MatchVerdict::NoMatch + }; + (verdict, None) +} + +#[derive(Clone, Copy)] +enum IdentifierMatch { + Preferred, + Fallback, +} + +fn states_verdict(query: &LocatorQuery, evidence: &LocatorEvidence) -> MatchVerdict { + if query.states.is_empty() { + return MatchVerdict::Match; + } + match &evidence.states { + LocatorField::Known(states) => bool_verdict(query.states.iter().all(|predicate| { + state::has_state(states, &predicate.token) == predicate.expected.unwrap_or(true) + })), + LocatorField::Absent => bool_verdict( + query + .states + .iter() + .all(|predicate| predicate.expected == Some(false)), + ), + LocatorField::Unknown => MatchVerdict::Unknown, + } +} + +fn field_equality(actual: &LocatorField<String>, expected: &str) -> MatchVerdict { + match actual { + LocatorField::Known(actual) => bool_verdict(actual == expected), + LocatorField::Absent => MatchVerdict::NoMatch, + LocatorField::Unknown => MatchVerdict::Unknown, + } +} + +fn bool_verdict(matched: bool) -> MatchVerdict { + if matched { + MatchVerdict::Match + } else { + MatchVerdict::NoMatch + } +} diff --git a/crates/core/src/live_locator/ref_evidence_requirements.rs b/crates/core/src/live_locator/ref_evidence_requirements.rs new file mode 100644 index 0000000..7792144 --- /dev/null +++ b/crates/core/src/live_locator/ref_evidence_requirements.rs @@ -0,0 +1,14 @@ +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RefEvidenceRequirements { + pub bounds: bool, + pub actions: bool, +} + +impl RefEvidenceRequirements { + pub(crate) fn union(self, other: Self) -> Self { + Self { + bounds: self.bounds || other.bounds, + actions: self.actions || other.actions, + } + } +} diff --git a/crates/core/src/live_locator/resolve.rs b/crates/core/src/live_locator/resolve.rs new file mode 100644 index 0000000..814a0b3 --- /dev/null +++ b/crates/core/src/live_locator/resolve.rs @@ -0,0 +1,245 @@ +use super::{ + LocatorResolution, LocatorResolveRequest, LocatorStats, ObservationRequest, ObservationRoot, + evaluate_locator_tree, validate_query, validate_request, +}; +use crate::{ + AdapterError, AppError, ErrorCode, WindowInfo, adapter::PlatformAdapter, locator::LocatorQuery, + refs::RefEntry, +}; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub fn resolve_query( + adapter: &dyn PlatformAdapter, + query: &LocatorQuery, + root: ObservationRoot<'_>, + request: &LocatorResolveRequest, +) -> Result<LocatorResolution, AppError> { + validate_query(query)?; + validate_request(request)?; + let started = Instant::now(); + let deadline = request.deadline; + let mut aggregate = LocatorStats::default(); + let mut last_incomplete = None; + let mut selection_retry_used = false; + loop { + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(transient_incomplete_timeout(deadline, &aggregate, last_incomplete).into()); + } + let attempt_request = LocatorResolveRequest { ..*request }; + match resolve_query_attempt(adapter, query, root, &attempt_request, &mut aggregate) { + Ok(mut resolution) => { + resolution.stats.reads.counts.observation_attempts = + resolution.stats.reads.counts.observation_attempts.max(1); + if !resolution.meta.selection_complete && has_deterministic_limit(&resolution.stats) + { + return Err(deterministic_limit_error(&resolution).into()); + } + if !resolution.meta.selection_complete + && has_transient_incompleteness(&resolution.stats) + { + last_incomplete = Some(incomplete_evidence(&resolution)); + aggregate.merge_attempt(&resolution.stats); + if deadline.is_expired() { + return Err(transient_incomplete_timeout( + deadline, + &aggregate, + last_incomplete, + ) + .into()); + } + let remaining = deadline.remaining(); + std::thread::sleep(remaining.min(Duration::from_millis(10))); + continue; + } + if !resolution.meta.selection_complete && requires_authoritative_result(request) { + return Err(unknown_incomplete_error(&resolution).into()); + } + aggregate.merge_attempt(&resolution.stats); + aggregate.elapsed_us = + u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); + resolution.stats = aggregate; + return Ok(resolution); + } + Err(AppError::Adapter(error)) + if request.materialization == super::LocatorMaterialization::SelectedMatches + && super::hydrate::retryable_error(&error) + && !selection_retry_used + && !deadline.is_expired() => + { + selection_retry_used = true; + last_incomplete = Some(json!({ + "phase": "hydration", + "code": error.code.as_str(), + "details": error.details, + })); + let remaining = deadline.remaining(); + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } + Err(error) => return Err(error), + } + } +} + +fn requires_authoritative_result(request: &LocatorResolveRequest) -> bool { + request.materialization == super::LocatorMaterialization::SelectedMatches + || !matches!( + request.selection, + super::LocatorSelection::All { .. } | super::LocatorSelection::Count + ) +} + +fn unknown_incomplete_error(resolution: &LocatorResolution) -> AdapterError { + AdapterError::timeout("Locator traversal returned incomplete evidence") + .with_suggestion("Retry with a narrower locator or a larger observation budget") + .with_details(json!({ + "kind": "locator_incomplete", + "retryable": false, + "observed_matches": resolution.meta.total_matches, + "query_stats": resolution.stats, + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +fn has_deterministic_limit(stats: &LocatorStats) -> bool { + let limits = &stats.traversal.limits; + limits.node_hits > 0 + || limits.edge_hits > 0 + || limits.child_hits > 0 + || limits.text_hits > 0 + || limits.depth_hits > 0 +} + +fn has_transient_incompleteness(stats: &LocatorStats) -> bool { + stats.traversal.limits.child_count_changes > 0 + || stats.reads.health.cannot_complete > 0 + || stats.reads.health.deadline_exhausted > 0 +} + +fn deterministic_limit_error(resolution: &LocatorResolution) -> AdapterError { + AdapterError::new( + ErrorCode::Timeout, + "Locator traversal reached a deterministic observation budget", + ) + .with_suggestion( + "Use a narrower locator, a shallower scope, or increase the matching observation budget", + ) + .with_details(json!({ + "kind": "locator_budget_limit", + "retryable": false, + "observed_matches": resolution.meta.total_matches, + "limits": resolution.stats.traversal.limits, + "query_stats": resolution.stats, + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +fn incomplete_evidence(resolution: &LocatorResolution) -> serde_json::Value { + json!({ + "observed_matches": resolution.meta.total_matches, + "child_count_changes": resolution.stats.traversal.limits.child_count_changes, + "cannot_complete": resolution.stats.reads.health.cannot_complete, + "deadline_exhausted": resolution.stats.reads.health.deadline_exhausted, + }) +} + +fn transient_incomplete_timeout( + deadline: crate::Deadline, + stats: &LocatorStats, + last_incomplete: Option<serde_json::Value>, +) -> AdapterError { + AdapterError::timeout("Locator observation did not stabilize before its deadline") + .with_suggestion( + "Retry with a larger timeout or narrow the locator to a more stable subtree", + ) + .with_details(json!({ + "kind": "locator_transient_incomplete", + "retryable": true, + "timeout_ms": deadline.timeout_ms(), + "observation_attempts": stats.reads.counts.observation_attempts, + "last_incomplete": last_incomplete, + "query_stats": stats, + })) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +fn resolve_query_attempt( + adapter: &dyn PlatformAdapter, + query: &LocatorQuery, + root: ObservationRoot<'_>, + request: &LocatorResolveRequest, + aggregate: &mut LocatorStats, +) -> Result<LocatorResolution, AppError> { + let observation_request = + ObservationRequest::locator_for_root(query, request, root, request.deadline).validate()?; + let tree = crate::renderer_accessibility::observe_tree(adapter, root, &observation_request)?; + let mut tree = tree; + tree.stats.reads.counts.observation_attempts = + tree.stats.reads.counts.observation_attempts.max(1); + let evaluation_request = LocatorResolveRequest { + materialization: match request.materialization { + super::LocatorMaterialization::SelectedMatches => super::LocatorMaterialization::None, + other => other, + }, + ..*request + }; + let mut resolution = evaluate_locator_tree(tree, query, &evaluation_request)?; + if request.materialization == super::LocatorMaterialization::SelectedMatches { + if let Err(error) = + super::hydrate::selected_matches(adapter, query, request, &mut resolution) + { + aggregate.merge_attempt(&resolution.stats); + return Err(error); + } + } + Ok(resolution) +} + +pub fn find_first_entry( + adapter: &dyn PlatformAdapter, + window: &WindowInfo, + query: &LocatorQuery, + timeout: Duration, +) -> Result<RefEntry, AdapterError> { + let resolution = resolve_query( + adapter, + query, + ObservationRoot::Window(window), + &LocatorResolveRequest { + selection: super::LocatorSelection::First, + deadline: crate::Deadline::from_duration(timeout)?, + max_raw_depth: 50, + materialization: super::LocatorMaterialization::SelectedMatches, + }, + ) + .map_err(app_error_to_adapter)?; + if !resolution.meta.selection_complete { + return Err( + AdapterError::timeout("Locator traversal was incomplete").with_details(json!({ + "kind": "locator_incomplete", + "observed_matches": resolution.meta.total_matches, + "query_stats": resolution.stats, + })), + ); + } + resolution + .matches + .into_iter() + .next() + .map(|matched| matched.entry) + .ok_or_else(|| { + AdapterError::new( + ErrorCode::ElementNotFound, + "Locator query matched no elements", + ) + .with_suggestion("Use a broader locator or inspect the accessibility tree") + }) +} + +fn app_error_to_adapter(error: AppError) -> AdapterError { + match error { + AppError::Adapter(error) => error, + other => AdapterError::internal(other.to_string()), + } +} diff --git a/crates/core/src/live_locator/resolve_query_hydration_tests.rs b/crates/core/src/live_locator/resolve_query_hydration_tests.rs new file mode 100644 index 0000000..e7ed426 --- /dev/null +++ b/crates/core/src/live_locator/resolve_query_hydration_tests.rs @@ -0,0 +1,152 @@ +use super::test_support::window; +use super::{ + LocatorMaterialization, LocatorResolveRequest, LocatorSelection, ObservationRequest, + ObservationRoot, ObservedTree, resolve_query, +}; +use crate::{ + AdapterError, ErrorCode, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, + locator::LocatorQuery, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct HydrationRetryAdapter { + window_observations: AtomicUsize, + hydration_observations: AtomicUsize, +} + +impl ObservationOps for HydrationRetryAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + match root { + ObservationRoot::Window(_) => { + self.window_observations.fetch_add(1, Ordering::SeqCst); + Ok(button_tree()) + } + ObservationRoot::Element { .. } => { + let attempt = self.hydration_observations.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + return Err(AdapterError::stale_ref("hydration retry") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(button_tree()) + } + } + } + + fn resolve_locator_anchor( + &self, + _entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<crate::NativeHandle, AdapterError> { + Ok(crate::NativeHandle::null()) + } +} + +impl ActionOps for HydrationRetryAdapter {} +impl InputOps for HydrationRetryAdapter {} +impl SystemOps for HydrationRetryAdapter {} + +fn button_tree() -> ObservedTree { + super::test_support::tree( + vec![super::test_support::node( + 0, + super::test_support::evidence("button", Some("Save")), + Vec::new(), + &[], + )], + vec![0], + true, + ) +} + +#[test] +fn hydration_retry_preserves_failed_attempt_statistics() { + let adapter = HydrationRetryAdapter { + window_observations: AtomicUsize::new(0), + hydration_observations: AtomicUsize::new(0), + }; + let request = LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::after(1_000).unwrap(), + max_raw_depth: 10, + materialization: LocatorMaterialization::SelectedMatches, + }; + + let resolution = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request, + ) + .unwrap(); + + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 2); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 2); + assert_eq!(resolution.stats.reads.counts.observation_attempts, 3); +} + +struct IncompleteHydrationAdapter { + hydration_observations: AtomicUsize, +} + +impl ObservationOps for IncompleteHydrationAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + if matches!(root, ObservationRoot::Window(_)) { + return Ok(button_tree()); + } + self.hydration_observations.fetch_add(1, Ordering::SeqCst); + let mut evidence = super::test_support::evidence("button", Some("Save")); + evidence.ref_evidence.bounds = super::LocatorField::Unknown; + Ok(super::test_support::tree( + vec![super::test_support::node(0, evidence, Vec::new(), &[])], + vec![0], + true, + )) + } + + fn resolve_locator_anchor( + &self, + _entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<crate::NativeHandle, AdapterError> { + Ok(crate::NativeHandle::null()) + } +} + +impl ActionOps for IncompleteHydrationAdapter {} +impl InputOps for IncompleteHydrationAdapter {} +impl SystemOps for IncompleteHydrationAdapter {} + +#[test] +fn selected_hydration_rejects_incomplete_snapshot_evidence() { + let adapter = IncompleteHydrationAdapter { + hydration_observations: AtomicUsize::new(0), + }; + let request = LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::after(35).unwrap(), + max_raw_depth: 10, + materialization: LocatorMaterialization::SelectedMatches, + }; + + let error = match resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request, + ) { + Ok(_) => panic!("incomplete hydration unexpectedly succeeded"), + Err(error) => error, + }; + + assert_eq!(error.code(), ErrorCode::Timeout.as_str()); + assert!(adapter.hydration_observations.load(Ordering::SeqCst) >= 1); +} diff --git a/crates/core/src/live_locator/resolve_query_tests.rs b/crates/core/src/live_locator/resolve_query_tests.rs new file mode 100644 index 0000000..d776b62 --- /dev/null +++ b/crates/core/src/live_locator/resolve_query_tests.rs @@ -0,0 +1,253 @@ +use super::test_support::window; +use super::{ + LocatorMaterialization, LocatorResolveRequest, LocatorSelection, LocatorStats, + ObservationRequest, ObservationRoot, ObservedTree, resolve_query, +}; +use crate::{ + AdapterError, AppError, ErrorCode, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, + locator::{LocatorQuery, StatePredicate}, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct CountingAdapter { + builds: AtomicUsize, + observe: Option<fn(usize) -> ObservedTree>, +} + +impl ObservationOps for CountingAdapter { + fn observe_tree( + &self, + _root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + let attempt = self.builds.fetch_add(1, Ordering::SeqCst); + self.observe + .map(|observe| observe(attempt)) + .ok_or_else(|| AdapterError::not_supported("fixture locator tree")) + } +} + +impl ActionOps for CountingAdapter {} +impl InputOps for CountingAdapter {} +impl SystemOps for CountingAdapter {} + +fn request() -> LocatorResolveRequest { + request_with_timeout(std::time::Duration::from_secs(5)) +} + +fn request_with_timeout(timeout: std::time::Duration) -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::from_duration(timeout).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + } +} + +fn observed_tree(complete: bool, stats: LocatorStats) -> ObservedTree { + let root = super::test_support::node( + 0, + super::test_support::evidence("window", Some("Fixture")), + Vec::new(), + &[], + ); + let mut tree = super::test_support::tree(vec![root], vec![0], complete); + tree.stats = stats; + tree +} + +fn churn_then_stable(attempt: usize) -> ObservedTree { + let mut stats = LocatorStats::default(); + if attempt == 0 { + stats.traversal.limits.child_count_changes = 1; + observed_tree(false, stats) + } else { + observed_tree(true, stats) + } +} + +fn always_cannot_complete(_: usize) -> ObservedTree { + let mut stats = LocatorStats::default(); + stats.reads.health.cannot_complete = 1; + observed_tree(false, stats) +} + +fn node_limit(_: usize) -> ObservedTree { + let mut stats = LocatorStats::default(); + stats.traversal.limits.node_hits = 1; + observed_tree(false, stats) +} + +fn child_label_cap(_: usize) -> ObservedTree { + let mut stats = LocatorStats::default(); + stats.traversal.limits.child_label_hits = 1; + observed_tree(true, stats) +} + +#[test] +fn resolve_query_validates_before_calling_the_adapter() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: None, + }; + let query = LocatorQuery { + states: vec![StatePredicate { + token: "imaginary".into(), + expected: None, + }], + ..LocatorQuery::default() + }; + let error = resolve_query( + &adapter, + &query, + ObservationRoot::Window(&window()), + &request(), + ) + .err() + .unwrap(); + assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str()); + assert_eq!(adapter.builds.load(Ordering::SeqCst), 0); +} + +#[test] +fn resolve_query_delegates_valid_queries_once() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: None, + }; + let error = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request(), + ) + .err() + .unwrap(); + assert_eq!(error.code(), ErrorCode::PlatformNotSupported.as_str()); + assert_eq!(adapter.builds.load(Ordering::SeqCst), 1); +} + +#[test] +fn child_count_churn_retries_until_the_observation_is_stable() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: Some(churn_then_stable), + }; + + let resolution = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request(), + ) + .unwrap(); + + assert!(resolution.meta.complete); + assert_eq!(adapter.builds.load(Ordering::SeqCst), 2); + assert_eq!(resolution.stats.reads.counts.observation_attempts, 2); + assert_eq!(resolution.stats.traversal.limits.child_count_changes, 1); +} + +#[test] +fn persistent_cannot_complete_exhausts_one_deadline_with_last_evidence() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: Some(always_cannot_complete), + }; + + let error = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request_with_timeout(std::time::Duration::from_millis(35)), + ) + .err() + .expect("persistent incompleteness must time out"); + + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!(error.code, ErrorCode::Timeout); + let details = error.details.unwrap(); + assert_eq!(details["kind"], "locator_transient_incomplete"); + let observation_attempts = details["observation_attempts"].as_u64().unwrap(); + assert!(observation_attempts >= 1); + assert_eq!( + observation_attempts, + adapter.builds.load(Ordering::SeqCst) as u64 + ); + assert_eq!(details["last_incomplete"]["cannot_complete"], 1); + assert_eq!( + details["query_stats"]["reads"]["cannot_complete"], + observation_attempts + ); +} + +#[test] +fn child_label_cap_alone_is_not_a_global_budget_failure() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: Some(child_label_cap), + }; + + let resolution = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request(), + ) + .expect("a per-node child-label cap must not fail a complete resolution"); + + assert_eq!(resolution.stats.traversal.limits.child_label_hits, 1); + assert_eq!(adapter.builds.load(Ordering::SeqCst), 1); +} + +#[test] +fn deterministic_observation_limit_fails_without_retrying() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: Some(node_limit), + }; + + let error = resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request(), + ) + .err() + .expect("deterministic limits must reject the resolution"); + + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(error.details.unwrap()["kind"], "locator_budget_limit"); + assert_eq!(adapter.builds.load(Ordering::SeqCst), 1); +} + +#[test] +fn strict_unknown_incomplete_result_fails_closed() { + let adapter = CountingAdapter { + builds: AtomicUsize::new(0), + observe: Some(|_| observed_tree(false, LocatorStats::default())), + }; + + let error = match resolve_query( + &adapter, + &LocatorQuery::default(), + ObservationRoot::Window(&window()), + &request(), + ) { + Ok(_) => panic!("strict incomplete result unexpectedly succeeded"), + Err(error) => error, + }; + + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(error.details.unwrap()["kind"], "locator_incomplete"); + assert_eq!(adapter.builds.load(Ordering::SeqCst), 1); +} diff --git a/crates/core/src/live_locator/resolve_tests.rs b/crates/core/src/live_locator/resolve_tests.rs new file mode 100644 index 0000000..f1bbd59 --- /dev/null +++ b/crates/core/src/live_locator/resolve_tests.rs @@ -0,0 +1,89 @@ +use super::{ + LocatorMaterialization, LocatorResolveRequest, LocatorSelection, evaluate_locator_tree, + require_unique, +}; +use crate::{ErrorCode, locator::LocatorQuery}; + +use super::test_support::{evidence, node, tree}; + +fn request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + } +} + +#[test] +fn require_unique_returns_the_only_owned_match() { + let resolution = evaluate_locator_tree( + tree( + vec![node(0, evidence("button", Some("Save")), vec![], &[])], + vec![0], + true, + ), + &LocatorQuery::default(), + &request(), + ) + .unwrap(); + let matched = require_unique(resolution).unwrap(); + assert_eq!(matched.document_order, 0); +} + +#[test] +fn require_unique_maps_complete_zero_to_not_found() { + let resolution = evaluate_locator_tree( + tree(Vec::new(), Vec::new(), true), + &LocatorQuery::default(), + &request(), + ) + .unwrap(); + let error = require_unique(resolution).err().unwrap(); + assert_eq!(error.code(), ErrorCode::ElementNotFound.as_str()); +} + +#[test] +fn require_unique_caps_ambiguous_candidate_summaries() { + let nodes = (0..11) + .map(|order| node(order, evidence("button", Some("Duplicate")), vec![], &[])) + .collect(); + let roots = (0..11).collect(); + let resolution = evaluate_locator_tree( + tree(nodes, roots, true), + &LocatorQuery::default(), + &request(), + ) + .unwrap(); + let error = require_unique(resolution).err().unwrap(); + assert_eq!(error.code(), ErrorCode::AmbiguousTarget.as_str()); + let details = match error { + crate::AppError::Adapter(error) => error.details.unwrap(), + other => panic!("expected adapter error, got {other}"), + }; + assert_eq!(details["candidate_count"], 11); + assert_eq!(details["candidate_count_exact"], true); + assert_eq!(details["candidates"].as_array().unwrap().len(), 10); +} + +#[test] +fn require_unique_maps_incomplete_zero_or_one_to_timeout() { + let resolution = evaluate_locator_tree( + tree( + vec![node(0, evidence("button", Some("Save")), vec![], &[])], + vec![0], + false, + ), + &LocatorQuery::default(), + &request(), + ) + .unwrap(); + let error = require_unique(resolution).err().unwrap(); + assert_eq!(error.code(), ErrorCode::Timeout.as_str()); + let details = match error { + crate::AppError::Adapter(error) => error.details.unwrap(), + other => panic!("expected adapter error, got {other}"), + }; + assert_eq!(details["kind"], "locator_incomplete"); + assert_eq!(details["observed_matches"], 1); +} diff --git a/crates/core/src/live_locator/scrollarea_name_tests.rs b/crates/core/src/live_locator/scrollarea_name_tests.rs new file mode 100644 index 0000000..e8f2d26 --- /dev/null +++ b/crates/core/src/live_locator/scrollarea_name_tests.rs @@ -0,0 +1,40 @@ +use super::{ + LocatorField, LocatorMaterialization, LocatorResolveRequest, LocatorSelection, + evaluate_locator_tree, +}; +use crate::{IdentityPredicate, LocatorQuery}; + +#[test] +fn exact_named_scroll_area_is_unique_among_unnamed_scroll_containers() { + let named = super::test_support::node( + 0, + super::test_support::evidence("scrollarea", Some("scroll-area")), + Vec::new(), + &[], + ); + let mut unnamed_evidence = super::test_support::evidence("scrollarea", None); + unnamed_evidence.name = LocatorField::Absent; + let unnamed = super::test_support::node(1, unnamed_evidence, Vec::new(), &[]); + let tree = super::test_support::tree(vec![named, unnamed], vec![0, 1], true); + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("scrollarea".into()), + name: Some("scroll-area".into()), + ..Default::default() + }, + exact: true, + ..Default::default() + }; + let request = LocatorResolveRequest { + selection: LocatorSelection::Strict, + deadline: crate::Deadline::after(500).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + }; + + let resolution = evaluate_locator_tree(tree, &query, &request).unwrap(); + + assert!(resolution.meta.complete); + assert!(resolution.meta.selection_complete); + assert_eq!(resolution.meta.total_matches, 1); +} diff --git a/crates/core/src/live_locator/select.rs b/crates/core/src/live_locator/select.rs new file mode 100644 index 0000000..cea7040 --- /dev/null +++ b/crates/core/src/live_locator/select.rs @@ -0,0 +1,88 @@ +use super::{LocatorMatchData, LocatorSelection, ObservedTree}; + +pub(crate) fn selected_indices( + matches: &[usize], + selection: LocatorSelection, +) -> (Vec<usize>, bool) { + match selection { + LocatorSelection::Strict => take(matches, 10), + LocatorSelection::All { + limit: Some(0) | None, + } => (matches.to_vec(), false), + LocatorSelection::All { limit: Some(limit) } => take(matches, limit as usize), + LocatorSelection::Count => (Vec::new(), false), + LocatorSelection::First => take(matches, 1), + LocatorSelection::Last => ( + matches.last().copied().into_iter().collect(), + matches.len() > 1, + ), + LocatorSelection::Nth(index) => ( + matches.get(index as usize).copied().into_iter().collect(), + matches.len() > 1, + ), + } +} + +pub(crate) fn match_data( + tree: &ObservedTree, + index: usize, + parents: &[Option<usize>], +) -> Option<LocatorMatchData> { + let node = tree.nodes.get(index)?; + let role = node + .evidence + .role + .known() + .cloned() + .unwrap_or_else(|| "unknown".into()); + let name = node + .evidence + .name + .meaningful_string() + .or_else(|| node.evidence.value.meaningful_string()) + .or_else(|| node.evidence.description.meaningful_string()) + .unwrap_or_else(|| format!("(unnamed {role})")); + let mut ancestor_indices = Vec::new(); + let mut parent = parents.get(index).copied().flatten(); + while let Some(parent_index) = parent { + ancestor_indices.push(parent_index); + parent = parents.get(parent_index).copied().flatten(); + } + ancestor_indices.reverse(); + let path = ancestor_indices + .into_iter() + .filter_map(|ancestor| tree.nodes.get(ancestor)) + .map(node_label) + .collect(); + Some(LocatorMatchData { + ref_id: node.ref_id.clone(), + role, + name, + value: node.evidence.value.meaningful_string(), + states: node.evidence.states.known().cloned().unwrap_or_default(), + interactive: super::materialize::addressability(&node.evidence).0, + path, + }) +} + +fn take(matches: &[usize], limit: usize) -> (Vec<usize>, bool) { + ( + matches.iter().take(limit).copied().collect(), + matches.len() > limit, + ) +} + +fn node_label(node: &super::ObservedNode) -> String { + let role = node + .evidence + .role + .known() + .map(String::as_str) + .unwrap_or("unknown"); + node.evidence + .name + .meaningful_string() + .or_else(|| node.evidence.value.meaningful_string()) + .map(|label| format!("{role}:{label}")) + .unwrap_or_else(|| role.to_string()) +} diff --git a/crates/core/src/live_locator/selected_hydration_churn_tests.rs b/crates/core/src/live_locator/selected_hydration_churn_tests.rs new file mode 100644 index 0000000..7e0e977 --- /dev/null +++ b/crates/core/src/live_locator/selected_hydration_churn_tests.rs @@ -0,0 +1,282 @@ +use super::test_support::window; +use super::{ + IdentifierEvidence, LocatorField, LocatorMaterialization, LocatorResolveRequest, + LocatorSelection, ObservationRequest, ObservationRoot, ObservationSource, ObservedTree, + resolve_query, +}; +use crate::{ + AdapterError, ElementIdentifier, IdentifierKind, LocatorQuery, NativeHandle, Rect, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct SingleAnchorAdapter { + initial_id: Option<&'static str>, + hydrated_id: Option<&'static str>, + window_observations: AtomicUsize, + strict_resolutions: AtomicUsize, + hydration_observations: AtomicUsize, + hydration_topology_complete: bool, + hydrated_bounds: Rect, + path_churn: bool, +} + +impl ObservationOps for SingleAnchorAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + match root { + ObservationRoot::Window(_) => { + self.window_observations.fetch_add(1, Ordering::SeqCst); + let evidence = request.descendant_evidence(); + assert!(evidence.identifiers); + assert!(evidence.ref_evidence.bounds); + assert!(!evidence.ref_evidence.actions); + Ok(single_anchor_tree(self.initial_id)) + } + ObservationRoot::Element { .. } => { + self.hydration_observations.fetch_add(1, Ordering::SeqCst); + Ok(single_hydration_tree( + root, + self.hydrated_id, + self.hydration_topology_complete, + self.hydrated_bounds, + )) + } + } + } + + fn resolve_locator_anchor( + &self, + entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + self.strict_resolutions.fetch_add(1, Ordering::SeqCst); + assert_eq!(entry.scope.path.as_slice(), &[0]); + assert_eq!(entry.geometry.bounds, Some(button_bounds())); + if self.path_churn { + return Err(AdapterError::new( + crate::ErrorCode::StaleRef, + "selected locator path changed", + )); + } + Ok(NativeHandle::null()) + } +} + +impl ActionOps for SingleAnchorAdapter {} +impl InputOps for SingleAnchorAdapter {} +impl SystemOps for SingleAnchorAdapter {} + +#[test] +fn unnamed_selected_control_uses_original_geometry_as_its_hydration_anchor() { + let adapter = adapter(None, None); + + let resolution = resolve_query( + &adapter, + &role_query("button"), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .expect("geometry must anchor an unnamed selected control"); + + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 1); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 1); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 1); + let entry = resolution.refmap.unwrap().get("@e1").unwrap().clone(); + assert_eq!(entry.identity.name, None); + assert_eq!(entry.geometry.bounds, Some(button_bounds())); +} + +#[test] +fn same_role_path_replacement_is_rejected_after_one_bounded_reobservation() { + let adapter = adapter(Some("original-control"), Some("replacement-control")); + + let error = resolve_query( + &adapter, + &role_query("button"), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .err() + .expect("a same-role replacement at the observed path must be rejected"); + + assert_eq!(error.code(), "STALE_REF"); + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 2); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 2); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 2); +} + +#[test] +fn geometry_churn_is_rejected_after_one_bounded_reobservation() { + let adapter = SingleAnchorAdapter { + hydrated_bounds: moved_bounds(), + ..adapter(None, None) + }; + + let error = resolve_query( + &adapter, + &role_query("button"), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .err() + .expect("geometry churn must invalidate a geometry-only anchor"); + + assert_eq!(error.code(), "STALE_REF"); + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 2); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 2); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 2); +} + +#[test] +fn exact_path_churn_never_falls_through_to_hydration() { + let adapter = SingleAnchorAdapter { + path_churn: true, + ..adapter(None, None) + }; + + let error = resolve_query( + &adapter, + &role_query("button"), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .err() + .expect("an exact-path miss must fail before selected hydration"); + + assert_eq!(error.code(), "STALE_REF"); + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 2); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 2); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 0); +} + +#[test] +fn complete_selected_root_evidence_does_not_require_descendant_topology() { + let adapter = SingleAnchorAdapter { + hydration_topology_complete: false, + ..adapter(None, None) + }; + + let resolution = resolve_query( + &adapter, + &role_query("button"), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .expect("selected hydration consumes only its complete root evidence"); + + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 1); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 1); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 1); + assert_eq!(resolution.refmap.unwrap().len(), 1); +} + +fn adapter( + initial_id: Option<&'static str>, + hydrated_id: Option<&'static str>, +) -> SingleAnchorAdapter { + SingleAnchorAdapter { + initial_id, + hydrated_id, + window_observations: AtomicUsize::new(0), + strict_resolutions: AtomicUsize::new(0), + hydration_observations: AtomicUsize::new(0), + hydration_topology_complete: true, + hydrated_bounds: button_bounds(), + path_churn: false, + } +} + +fn role_query(role: &str) -> LocatorQuery { + LocatorQuery { + identity: crate::IdentityPredicate { + role: Some(role.into()), + ..Default::default() + }, + ..Default::default() + } +} + +fn selected_request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::after(500).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::SelectedMatches, + } +} + +fn single_anchor_tree(identifier: Option<&str>) -> ObservedTree { + let mut anchor = super::test_support::evidence("button", identifier.map(|_| "Shared control")); + anchor.identifiers = identifier_evidence(identifier); + anchor.states = LocatorField::Unknown; + anchor.ref_evidence.bounds = LocatorField::Known(button_bounds()); + anchor.ref_evidence.available_actions = LocatorField::Unknown; + super::test_support::tree( + vec![ + super::test_support::node( + 0, + super::test_support::evidence("window", None), + vec![1], + &[], + ), + super::test_support::node(1, anchor, Vec::new(), &[0]), + ], + vec![0], + true, + ) +} + +fn single_hydration_tree( + root: ObservationRoot<'_>, + identifier: Option<&str>, + topology_complete: bool, + bounds: Rect, +) -> ObservedTree { + let mut evidence = + super::test_support::evidence("button", identifier.map(|_| "Shared control")); + evidence.identifiers = identifier_evidence(identifier); + evidence.ref_evidence.bounds = LocatorField::Known(bounds); + evidence.ref_evidence.available_actions = + LocatorField::Known(vec![crate::capability::CLICK.into()]); + let mut tree = super::test_support::tree( + vec![super::test_support::node(0, evidence, Vec::new(), &[])], + vec![0], + true, + ); + tree.source = ObservationSource::from_root(&root); + tree.structurally_complete = topology_complete; + tree +} + +fn identifier_evidence(identifier: Option<&str>) -> IdentifierEvidence { + identifier.map_or_else(IdentifierEvidence::absent, |value| { + IdentifierEvidence::typed( + [ElementIdentifier { + kind: IdentifierKind::AxIdentifier, + value: value.into(), + }], + Some(0), + true, + ) + }) +} + +fn button_bounds() -> Rect { + Rect { + x: 10.0, + y: 20.0, + width: 100.0, + height: 30.0, + } +} + +fn moved_bounds() -> Rect { + Rect { + x: 40.0, + ..button_bounds() + } +} diff --git a/crates/core/src/live_locator/selected_hydration_contract_tests.rs b/crates/core/src/live_locator/selected_hydration_contract_tests.rs new file mode 100644 index 0000000..9b5f78f --- /dev/null +++ b/crates/core/src/live_locator/selected_hydration_contract_tests.rs @@ -0,0 +1,256 @@ +use super::test_support::window; +use super::{ + IdentifierEvidence, LocatorField, LocatorMaterialization, LocatorResolveRequest, + LocatorSelection, ObservationRequest, ObservationRoot, ObservationSource, ObservedTree, + resolve_query, +}; +use crate::{ + AdapterError, AppError, LocatorQuery, NativeHandle, Rect, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone, Copy)] +enum Mode { + OptionalIdentityUnknown, + MissingAnchor, + RequiredNameUnknown, + RequiredDescriptionUnknown, +} + +struct HydrationContractAdapter { + mode: Mode, + window_observations: AtomicUsize, + strict_resolutions: AtomicUsize, + hydration_observations: AtomicUsize, +} + +impl ObservationOps for HydrationContractAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + match root { + ObservationRoot::Window(_) => { + self.window_observations.fetch_add(1, Ordering::SeqCst); + Ok(initial_tree(self.mode)) + } + ObservationRoot::Element { .. } => { + self.hydration_observations.fetch_add(1, Ordering::SeqCst); + let root_evidence = request.evidence_for_raw_depth(0); + assert!(root_evidence.states); + assert!(root_evidence.ref_evidence.bounds); + assert!(root_evidence.ref_evidence.actions); + Ok(hydrated_tree(root, self.mode)) + } + } + } + + fn resolve_locator_anchor( + &self, + entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + self.strict_resolutions.fetch_add(1, Ordering::SeqCst); + assert!( + crate::ref_identity::has_meaningful_identity(entry) + || entry.geometry.bounds_hash.is_some() + ); + Ok(NativeHandle::null()) + } +} + +impl ActionOps for HydrationContractAdapter {} +impl InputOps for HydrationContractAdapter {} +impl SystemOps for HydrationContractAdapter {} + +#[test] +fn role_only_geometry_anchor_accepts_unknown_optional_identity_once() { + let adapter = adapter(Mode::OptionalIdentityUnknown); + + let resolution = resolve_query( + &adapter, + &role_query(), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .expect("optional identity gaps must not invalidate a verified geometry anchor"); + + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 1); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 1); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 1); + assert_eq!(resolution.matches[0].data.name, "(name unavailable)"); + let entry = resolution.refmap.unwrap().get("@e1").unwrap().clone(); + assert_eq!(entry.identity.name, None); + assert_eq!(entry.geometry.bounds, Some(bounds())); +} + +#[test] +fn selected_path_without_identity_or_geometry_rejects_without_retrying() { + let adapter = adapter(Mode::MissingAnchor); + + let error = resolve_query( + &adapter, + &role_query(), + ObservationRoot::Window(&window()), + &selected_request(), + ) + .err() + .expect("path-only selected anchors must fail closed"); + let error = adapter_error(error); + + assert_eq!(error.code.as_str(), "STALE_REF"); + assert_eq!(error.details.as_ref().unwrap()["retryable"], false); + assert_eq!( + error.details.as_ref().unwrap()["kind"], + "locator_selected_anchor_missing" + ); + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 1); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 0); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 0); +} + +#[test] +fn name_query_rejects_unknown_hydrated_name() { + let query = LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + name: Some("Save".into()), + ..Default::default() + }, + ..Default::default() + }; + + assert_required_field_rejected(Mode::RequiredNameUnknown, query, "name"); +} + +#[test] +fn description_query_rejects_unknown_hydrated_description() { + let query = LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + description: Some("Primary".into()), + ..Default::default() + }, + ..Default::default() + }; + + assert_required_field_rejected(Mode::RequiredDescriptionUnknown, query, "description"); +} + +fn assert_required_field_rejected(mode: Mode, query: LocatorQuery, field: &str) { + let adapter = adapter(mode); + let error = resolve_query( + &adapter, + &query, + ObservationRoot::Window(&window()), + &selected_request(), + ) + .err() + .expect("query-requested identity evidence must remain authoritative"); + let error = adapter_error(error); + let details = error.details.as_ref().unwrap(); + + assert_eq!(error.code.as_str(), "TIMEOUT"); + assert_eq!(details["kind"], "locator_selected_evidence_incomplete"); + assert_eq!(details["required"][field], true); + assert_eq!(details["unknown"][field], true); + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 2); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 2); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 2); +} + +fn adapter(mode: Mode) -> HydrationContractAdapter { + HydrationContractAdapter { + mode, + window_observations: AtomicUsize::new(0), + strict_resolutions: AtomicUsize::new(0), + hydration_observations: AtomicUsize::new(0), + } +} + +fn initial_tree(mode: Mode) -> ObservedTree { + let mut evidence = evidence(mode, false); + evidence.ref_evidence.bounds = match mode { + Mode::MissingAnchor => LocatorField::Absent, + _ => LocatorField::Known(bounds()), + }; + super::test_support::tree( + vec![super::test_support::node(0, evidence, Vec::new(), &[])], + vec![0], + true, + ) +} + +fn hydrated_tree(root: ObservationRoot<'_>, mode: Mode) -> ObservedTree { + let mut evidence = evidence(mode, true); + evidence.states = LocatorField::Known(Vec::new()); + evidence.ref_evidence.bounds = LocatorField::Known(bounds()); + evidence.ref_evidence.available_actions = + LocatorField::Known(vec![crate::capability::CLICK.into()]); + let mut tree = super::test_support::tree( + vec![super::test_support::node(0, evidence, Vec::new(), &[])], + vec![0], + true, + ); + tree.source = ObservationSource::from_root(&root); + tree +} + +fn evidence(mode: Mode, hydrating: bool) -> super::LocatorEvidence { + let mut evidence = super::test_support::evidence("button", None); + evidence.name = match mode { + Mode::RequiredNameUnknown if !hydrating => LocatorField::Known("Save".into()), + Mode::RequiredNameUnknown | Mode::OptionalIdentityUnknown | Mode::MissingAnchor => { + LocatorField::Unknown + } + Mode::RequiredDescriptionUnknown => LocatorField::Known("Save".into()), + }; + evidence.description = match mode { + Mode::RequiredDescriptionUnknown if !hydrating => LocatorField::Known("Primary".into()), + Mode::RequiredDescriptionUnknown | Mode::OptionalIdentityUnknown | Mode::MissingAnchor => { + LocatorField::Unknown + } + Mode::RequiredNameUnknown => LocatorField::Known("Primary".into()), + }; + evidence.identifiers = IdentifierEvidence::absent(); + evidence.states = LocatorField::Unknown; + evidence.ref_evidence.available_actions = LocatorField::Unknown; + evidence +} + +fn role_query() -> LocatorQuery { + LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + ..Default::default() + }, + ..Default::default() + } +} + +fn selected_request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::after(500).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::SelectedMatches, + } +} + +fn adapter_error(error: AppError) -> AdapterError { + match error { + AppError::Adapter(error) => error, + other => panic!("expected adapter error, got {other}"), + } +} + +fn bounds() -> Rect { + Rect { + x: 10.0, + y: 20.0, + width: 100.0, + height: 30.0, + } +} diff --git a/crates/core/src/live_locator/selected_hydration_subtree_tests.rs b/crates/core/src/live_locator/selected_hydration_subtree_tests.rs new file mode 100644 index 0000000..b5a63f0 --- /dev/null +++ b/crates/core/src/live_locator/selected_hydration_subtree_tests.rs @@ -0,0 +1,317 @@ +use super::test_support::window; +use super::{ + IdentifierEvidence, LocatorField, LocatorMaterialization, LocatorResolveRequest, + LocatorSelection, ObservationRequest, ObservationRoot, ObservationSource, ObservedTree, + resolve_query, +}; +use crate::{ + AdapterError, AppError, ContainmentPredicate, LocatorQuery, NativeHandle, Rect, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone, Copy)] +enum Mode { + TextWitnessRemoved, + WitnessRemoved, + ForbiddenAdded, + NegativeIncomplete, + PositiveIncompleteSuffix, +} + +struct SubtreeHydrationAdapter { + mode: Mode, + window_observations: AtomicUsize, + anchor_resolutions: AtomicUsize, + selected_observations: AtomicUsize, +} + +impl ObservationOps for SubtreeHydrationAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + match root { + ObservationRoot::Window(_) => { + self.window_observations.fetch_add(1, Ordering::SeqCst); + let evidence = request.descendant_evidence(); + assert!(evidence.identifiers); + assert!(evidence.ref_evidence.bounds); + assert!(!evidence.states); + assert!(!evidence.ref_evidence.actions); + Ok(initial_tree(self.mode)) + } + ObservationRoot::Element { .. } => { + self.selected_observations.fetch_add(1, Ordering::SeqCst); + assert_eq!(request.max_logical_depth, 10); + assert_eq!( + request.evidence_for_raw_depth(0), + super::EvidenceRequirements::snapshot() + ); + assert!(!request.evidence_for_raw_depth(1).ref_evidence.actions); + assert!(!request.evidence_for_raw_depth(1).states); + Ok(selected_tree(root, self.mode)) + } + } + } + + fn resolve_locator_anchor( + &self, + entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + self.anchor_resolutions.fetch_add(1, Ordering::SeqCst); + assert_eq!(entry.geometry.bounds, Some(root_bounds())); + Ok(NativeHandle::null()) + } +} + +impl ActionOps for SubtreeHydrationAdapter {} +impl InputOps for SubtreeHydrationAdapter {} +impl SystemOps for SubtreeHydrationAdapter {} + +#[test] +fn has_text_witness_removed_between_phases_is_rejected() { + assert_stale(Mode::TextWitnessRemoved, query(Mode::TextWitnessRemoved)); +} + +#[test] +fn containment_witness_removed_between_phases_is_rejected() { + assert_stale(Mode::WitnessRemoved, query(Mode::WitnessRemoved)); +} + +#[test] +fn forbidden_descendant_added_between_phases_is_rejected() { + assert_stale(Mode::ForbiddenAdded, query(Mode::ForbiddenAdded)); +} + +#[test] +fn incomplete_negative_absence_proof_fails_closed() { + let adapter = adapter(Mode::NegativeIncomplete); + let error = resolve_query( + &adapter, + &query(Mode::NegativeIncomplete), + ObservationRoot::Window(&window()), + &request(), + ) + .err() + .expect("incomplete has-not evidence must never prove absence"); + let error = adapter_error(error); + + assert_eq!(error.code.as_str(), "TIMEOUT"); + assert_eq!( + error.details.as_ref().unwrap()["kind"], + "locator_selected_query_incomplete" + ); + assert_attempts(&adapter, 2); +} + +#[test] +fn positive_witness_survives_an_irrelevant_incomplete_suffix() { + let adapter = adapter(Mode::PositiveIncompleteSuffix); + let resolution = resolve_query( + &adapter, + &query(Mode::PositiveIncompleteSuffix), + ObservationRoot::Window(&window()), + &request(), + ) + .expect("a witnessed positive predicate does not need an irrelevant suffix"); + + assert_attempts(&adapter, 1); + assert!(resolution.meta.selection_complete); + assert_eq!(resolution.matches.len(), 1); +} + +fn assert_stale(mode: Mode, query: LocatorQuery) { + let adapter = adapter(mode); + let error = resolve_query( + &adapter, + &query, + ObservationRoot::Window(&window()), + &request(), + ) + .err() + .expect("descendant churn must invalidate the selected match"); + let error = adapter_error(error); + + assert_eq!(error.code.as_str(), "STALE_REF"); + assert_attempts(&adapter, 2); +} + +fn assert_attempts(adapter: &SubtreeHydrationAdapter, expected: usize) { + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), expected); + assert_eq!(adapter.anchor_resolutions.load(Ordering::SeqCst), expected); + assert_eq!( + adapter.selected_observations.load(Ordering::SeqCst), + expected + ); +} + +fn adapter(mode: Mode) -> SubtreeHydrationAdapter { + SubtreeHydrationAdapter { + mode, + window_observations: AtomicUsize::new(0), + anchor_resolutions: AtomicUsize::new(0), + selected_observations: AtomicUsize::new(0), + } +} + +fn initial_tree(mode: Mode) -> ObservedTree { + let child = match mode { + Mode::TextWitnessRemoved => text_evidence("needle"), + Mode::WitnessRemoved | Mode::PositiveIncompleteSuffix => role_evidence("button"), + Mode::ForbiddenAdded | Mode::NegativeIncomplete => role_evidence("link"), + }; + super::test_support::tree( + vec![ + super::test_support::node(0, phase_one_root(mode), vec![1], &[]), + super::test_support::node(1, child, Vec::new(), &[0]), + ], + vec![0], + true, + ) +} + +fn selected_tree(root: ObservationRoot<'_>, mode: Mode) -> ObservedTree { + let children = match mode { + Mode::TextWitnessRemoved => vec![text_evidence("other")], + Mode::WitnessRemoved => vec![role_evidence("link")], + Mode::ForbiddenAdded => vec![role_evidence("button")], + Mode::NegativeIncomplete => vec![role_evidence("link")], + Mode::PositiveIncompleteSuffix => { + vec![role_evidence("button"), role_evidence("group")] + } + }; + let child_indices = (1..=children.len()) + .map(|index| u32::try_from(index).unwrap()) + .collect(); + let mut nodes = vec![super::test_support::node( + 0, + hydrated_root(), + child_indices, + &[], + )]; + for (index, evidence) in children.into_iter().enumerate() { + nodes.push(super::test_support::node( + u32::try_from(index + 1).unwrap(), + evidence, + Vec::new(), + &[index], + )); + } + let incomplete = matches!( + mode, + Mode::NegativeIncomplete | Mode::PositiveIncompleteSuffix + ); + if incomplete { + nodes[0].completeness.subtree_complete = false; + let last = nodes.len() - 1; + nodes[last].completeness.subtree_complete = false; + } + let mut tree = super::test_support::tree(nodes, vec![0], !incomplete); + tree.source = ObservationSource::from_root(&root); + tree +} + +fn phase_one_root(mode: Mode) -> super::LocatorEvidence { + let mut evidence = role_evidence("group"); + if matches!(mode, Mode::TextWitnessRemoved) { + evidence.name = LocatorField::Absent; + evidence.description = LocatorField::Absent; + evidence.value = LocatorField::Absent; + } + evidence.identifiers = IdentifierEvidence::absent(); + evidence.ref_evidence.bounds = LocatorField::Known(root_bounds()); + evidence +} + +fn hydrated_root() -> super::LocatorEvidence { + let mut evidence = super::test_support::evidence("group", None); + evidence.ref_evidence.bounds = LocatorField::Known(root_bounds()); + evidence +} + +fn role_evidence(role: &str) -> super::LocatorEvidence { + let mut evidence = super::test_support::evidence(role, None); + evidence.name = LocatorField::Unknown; + evidence.description = LocatorField::Unknown; + evidence.value = LocatorField::Unknown; + evidence.identifiers = IdentifierEvidence::unknown(); + evidence.states = LocatorField::Unknown; + evidence.ref_evidence.bounds = LocatorField::Unknown; + evidence.ref_evidence.available_actions = LocatorField::Unknown; + evidence +} + +fn text_evidence(text: &str) -> super::LocatorEvidence { + let mut evidence = role_evidence("statictext"); + evidence.name = LocatorField::Known(text.into()); + evidence.description = LocatorField::Absent; + evidence.value = LocatorField::Absent; + evidence +} + +fn query(mode: Mode) -> LocatorQuery { + let mut query = LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("group".into()), + ..Default::default() + }, + ..Default::default() + }; + match mode { + Mode::TextWitnessRemoved => query.has_text = Some("needle".into()), + Mode::WitnessRemoved | Mode::PositiveIncompleteSuffix => { + query.containment = containment(true) + } + Mode::ForbiddenAdded | Mode::NegativeIncomplete => query.containment = containment(false), + } + query +} + +fn containment(positive: bool) -> ContainmentPredicate { + let nested = Box::new(LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + ..Default::default() + }, + ..Default::default() + }); + if positive { + ContainmentPredicate { + has: Some(nested), + has_not: None, + } + } else { + ContainmentPredicate { + has: None, + has_not: Some(nested), + } + } +} + +fn request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::after(500).unwrap(), + max_raw_depth: 10, + materialization: LocatorMaterialization::SelectedMatches, + } +} + +fn adapter_error(error: AppError) -> AdapterError { + match error { + AppError::Adapter(error) => error, + other => panic!("expected adapter error, got {other}"), + } +} + +fn root_bounds() -> Rect { + Rect { + x: 10.0, + y: 20.0, + width: 300.0, + height: 200.0, + } +} diff --git a/crates/core/src/live_locator/selected_hydration_tests.rs b/crates/core/src/live_locator/selected_hydration_tests.rs new file mode 100644 index 0000000..7a9edb5 --- /dev/null +++ b/crates/core/src/live_locator/selected_hydration_tests.rs @@ -0,0 +1,258 @@ +use super::test_support::window; +use super::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorMaterialization, LocatorRefEvidence, + LocatorResolveRequest, LocatorSelection, LocatorStats, ObservationRequest, ObservationRoot, + ObservationSource, ObservedTree, resolve_query, +}; +use crate::{ + AdapterError, ElementIdentifier, IdentifierKind, LocatorQuery, NativeHandle, Rect, + adapter::{ActionOps, InputOps, ObservationOps, SystemOps}, +}; +use std::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; + +const WRAPPER_COUNT: usize = 255; +const INITIAL_NODE_COUNT: u64 = WRAPPER_COUNT as u64 + 2; +const INITIAL_CHILD_READS: u64 = INITIAL_NODE_COUNT * 4; +const INITIAL_ATTRIBUTES_REQUESTED: u64 = INITIAL_NODE_COUNT * 6; + +struct RoleOnlyHydrationAdapter { + window_observations: AtomicUsize, + strict_resolutions: AtomicUsize, + hydration_observations: AtomicUsize, +} + +impl ObservationOps for RoleOnlyHydrationAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + match root { + ObservationRoot::Window(_) => { + self.window_observations.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(10)); + let evidence = request.descendant_evidence(); + assert!(!evidence.name); + assert!(!evidence.description); + assert!(!evidence.value); + assert!(evidence.identifiers); + assert!(evidence.ref_evidence.bounds); + assert!(!evidence.states); + assert!(!evidence.ref_evidence.actions); + Ok(anchored_wrapper_tree()) + } + ObservationRoot::Element { .. } => { + self.hydration_observations.fetch_add(1, Ordering::SeqCst); + assert_eq!(request.max_logical_depth, 0); + assert_eq!( + request.evidence_for_raw_depth(0), + super::EvidenceRequirements::snapshot() + ); + assert!(!request.evidence_for_raw_depth(1).name); + assert!(!request.evidence_for_raw_depth(1).ref_evidence.actions); + Ok(hydrated_button_tree(root)) + } + } + } + + fn resolve_locator_anchor( + &self, + entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + self.strict_resolutions.fetch_add(1, Ordering::SeqCst); + assert_eq!(entry.identity.name, None); + assert_eq!( + entry + .identity + .native_id + .as_ref() + .map(|id| id.value.as_str()), + Some("primary-action") + ); + assert_eq!(entry.geometry.bounds, Some(button_bounds())); + assert_eq!(entry.scope.path.len(), WRAPPER_COUNT + 1); + Ok(NativeHandle::null()) + } +} + +impl ActionOps for RoleOnlyHydrationAdapter {} +impl InputOps for RoleOnlyHydrationAdapter {} +impl SystemOps for RoleOnlyHydrationAdapter {} + +#[test] +fn role_only_selected_hydration_anchors_before_hydrating_without_rewalking_large_tree() { + let adapter = RoleOnlyHydrationAdapter { + window_observations: AtomicUsize::new(0), + strict_resolutions: AtomicUsize::new(0), + hydration_observations: AtomicUsize::new(0), + }; + let query = LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + ..Default::default() + }, + ..Default::default() + }; + let request = LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::after(5_000).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::SelectedMatches, + }; + + let resolution = resolve_query( + &adapter, + &query, + ObservationRoot::Window(&window()), + &request, + ) + .expect("role-only selected hydration should use the observed path"); + + assert_eq!(adapter.window_observations.load(Ordering::SeqCst), 1); + assert_eq!(adapter.strict_resolutions.load(Ordering::SeqCst), 1); + assert_eq!(adapter.hydration_observations.load(Ordering::SeqCst), 1); + assert_eq!( + resolution.stats.traversal.nodes_visited, + INITIAL_NODE_COUNT + 1 + ); + assert_eq!( + resolution.stats.reads.counts.child_reads, + INITIAL_CHILD_READS + ); + assert_eq!( + resolution.stats.reads.counts.attributes_requested, + INITIAL_ATTRIBUTES_REQUESTED + 23 + ); + assert_eq!(resolution.stats.reads.counts.action_reads, 1); + assert_eq!(resolution.stats.semantic_reads.child_label_reads, 0); + assert_eq!(resolution.stats.reads.counts.observation_attempts, 2); + + let matched = &resolution.matches[0]; + assert_eq!(matched.data.name, "Primary action"); + assert_eq!(matched.data.ref_id.as_deref(), Some("@e1")); + let entry = resolution.refmap.as_ref().unwrap().get("@e1").unwrap(); + assert_eq!(entry.identity.name.as_deref(), Some("Primary action")); + assert_eq!( + entry + .identity + .native_id + .as_ref() + .map(|id| id.value.as_str()), + Some("primary-action") + ); + assert_eq!(entry.geometry.bounds, Some(button_bounds())); + assert_eq!( + entry.capabilities.available_actions, + [crate::capability::CLICK] + ); + assert_eq!(entry.scope.path.len(), WRAPPER_COUNT + 1); +} + +fn anchored_wrapper_tree() -> ObservedTree { + let last = WRAPPER_COUNT + 1; + let mut nodes = Vec::with_capacity(last + 1); + for index in 0..=last { + let role = if index == 0 { + "window" + } else if index == last { + "button" + } else { + "group" + }; + let children = (index < last) + .then_some(vec![u32::try_from(index + 1).unwrap()]) + .unwrap_or_default(); + let path = vec![0; index]; + let evidence = if index == last { + selected_anchor_evidence() + } else { + role_only_evidence(role) + }; + nodes.push(super::test_support::node( + u32::try_from(index).unwrap(), + evidence, + children, + &path, + )); + } + let mut tree = super::test_support::tree(nodes, vec![0], true); + tree.stats = initial_stats(); + tree +} + +fn selected_anchor_evidence() -> LocatorEvidence { + let mut evidence = role_only_evidence("button"); + evidence.identifiers = IdentifierEvidence::typed( + [ElementIdentifier { + kind: IdentifierKind::AxIdentifier, + value: "primary-action".into(), + }], + Some(0), + true, + ); + evidence.ref_evidence.bounds = LocatorField::Known(button_bounds()); + evidence +} + +fn role_only_evidence(role: &str) -> LocatorEvidence { + LocatorEvidence { + role: LocatorField::Known(role.into()), + name: LocatorField::Unknown, + description: LocatorField::Unknown, + value: LocatorField::Unknown, + identifiers: IdentifierEvidence::unknown(), + states: LocatorField::Unknown, + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Unknown, + available_actions: LocatorField::Unknown, + }, + } +} + +fn hydrated_button_tree(root: ObservationRoot<'_>) -> ObservedTree { + let mut evidence = super::test_support::evidence("button", Some("Primary action")); + evidence.identifiers = IdentifierEvidence::typed( + [ElementIdentifier { + kind: IdentifierKind::AxIdentifier, + value: "primary-action".into(), + }], + Some(0), + true, + ); + evidence.ref_evidence.bounds = LocatorField::Known(button_bounds()); + evidence.ref_evidence.available_actions = + LocatorField::Known(vec![crate::capability::CLICK.into()]); + let mut tree = super::test_support::tree( + vec![super::test_support::node(0, evidence, Vec::new(), &[])], + vec![0], + true, + ); + tree.source = ObservationSource::from_root(&root); + tree.stats.traversal.nodes_visited = 1; + tree.stats.reads.counts.attribute_batches = 1; + tree.stats.reads.counts.attributes_requested = 23; + tree.stats.reads.counts.action_reads = 1; + tree +} + +fn initial_stats() -> LocatorStats { + let mut stats = LocatorStats::default(); + stats.traversal.nodes_visited = INITIAL_NODE_COUNT; + stats.reads.counts.attribute_batches = INITIAL_NODE_COUNT; + stats.reads.counts.attributes_requested = INITIAL_ATTRIBUTES_REQUESTED; + stats.reads.counts.child_reads = INITIAL_CHILD_READS; + stats +} + +fn button_bounds() -> Rect { + Rect { + x: 10.0, + y: 20.0, + width: 100.0, + height: 30.0, + } +} diff --git a/crates/core/src/live_locator/selection_completeness.rs b/crates/core/src/live_locator/selection_completeness.rs new file mode 100644 index 0000000..c6cd0dc --- /dev/null +++ b/crates/core/src/live_locator/selection_completeness.rs @@ -0,0 +1,85 @@ +use super::{ObservedTree, match_verdict::MatchVerdict}; + +pub(crate) fn first_is_authoritative( + tree: &ObservedTree, + target: usize, + parents: &[Option<usize>], + verdicts: &[MatchVerdict], +) -> bool { + let mut chain = Vec::new(); + let mut current = Some(target); + while let Some(index) = current { + chain.push(index); + current = parents.get(index).copied().flatten(); + } + chain.reverse(); + let Some(root) = chain.first().copied() else { + return false; + }; + if !tree + .nodes + .get(root) + .is_some_and(|node| node.completeness.predecessors_complete) + { + return false; + } + let Some(root_order) = tree + .roots + .iter() + .position(|candidate| *candidate as usize == root) + else { + return false; + }; + if tree.roots[..root_order] + .iter() + .any(|index| !subtree_is_authoritative(tree, *index as usize, verdicts)) + { + return false; + } + for pair in chain.windows(2) { + let ancestor = pair[0]; + let selected_child = pair[1]; + if verdicts.get(ancestor) == Some(&MatchVerdict::Unknown) { + return false; + } + let Some(node) = tree.nodes.get(ancestor) else { + return false; + }; + if !tree + .nodes + .get(selected_child) + .is_some_and(|child| child.completeness.predecessors_complete) + { + return false; + } + let Some(child_order) = node + .children + .iter() + .position(|child| *child as usize == selected_child) + else { + return false; + }; + if node.children[..child_order] + .iter() + .any(|child| !subtree_is_authoritative(tree, *child as usize, verdicts)) + { + return false; + } + } + verdicts.get(target) == Some(&MatchVerdict::Match) +} + +fn subtree_is_authoritative(tree: &ObservedTree, root: usize, verdicts: &[MatchVerdict]) -> bool { + let Some(node) = tree.nodes.get(root) else { + return false; + }; + if !node.completeness.subtree_complete + || !node.completeness.predecessors_complete + || verdicts.get(root) == Some(&MatchVerdict::Unknown) + { + return false; + } + node.children + .iter() + .all(|child| subtree_is_authoritative(tree, *child as usize, verdicts)) +} diff --git a/crates/core/src/live_locator/selection_completeness_tests.rs b/crates/core/src/live_locator/selection_completeness_tests.rs new file mode 100644 index 0000000..6be039c --- /dev/null +++ b/crates/core/src/live_locator/selection_completeness_tests.rs @@ -0,0 +1,127 @@ +use super::{ + LocatorMaterialization, LocatorResolveRequest, LocatorSelection, evaluate_locator_tree, +}; +use crate::LocatorQuery; + +#[test] +fn first_match_is_authoritative_when_only_its_descendants_or_later_nodes_are_incomplete() { + let mut tree = fixture_tree(); + tree.nodes[1].completeness.subtree_complete = false; + tree.nodes[2].completeness.subtree_complete = false; + tree.nodes[0].completeness.subtree_complete = false; + tree.structurally_complete = false; + + let resolution = evaluate_locator_tree(tree, &button_query(), &first_request()).unwrap(); + + assert!(!resolution.meta.complete); + assert!(resolution.meta.selection_complete); + assert_eq!(resolution.matches[0].document_order, 1); +} + +#[test] +fn first_match_is_not_authoritative_when_an_earlier_subtree_is_incomplete() { + let mut tree = fixture_tree(); + tree.nodes[1].evidence = super::test_support::evidence("group", Some("Earlier")); + tree.nodes[1].completeness.subtree_complete = false; + tree.nodes[2].evidence = super::test_support::evidence("button", Some("Selected")); + tree.nodes[0].completeness.subtree_complete = false; + tree.structurally_complete = false; + + let resolution = evaluate_locator_tree(tree, &button_query(), &first_request()).unwrap(); + + assert!(!resolution.meta.complete); + assert!(!resolution.meta.selection_complete); + assert_eq!(resolution.matches[0].document_order, 2); +} + +#[test] +fn first_match_is_authoritative_when_native_predecessors_are_complete() { + let mut tree = single_retained_child_tree(0); + tree.nodes[0].completeness.subtree_complete = false; + tree.structurally_complete = false; + + let resolution = evaluate_locator_tree(tree, &button_query(), &first_request()).unwrap(); + + assert!(!resolution.meta.complete); + assert!(resolution.meta.selection_complete); +} + +#[test] +fn first_match_is_not_authoritative_after_unobserved_native_predecessors() { + let mut tree = single_retained_child_tree(2); + tree.nodes[0].completeness.subtree_complete = false; + tree.nodes[1].completeness.predecessors_complete = false; + tree.structurally_complete = false; + + let resolution = evaluate_locator_tree(tree, &button_query(), &first_request()).unwrap(); + + assert!(!resolution.meta.complete); + assert!(!resolution.meta.selection_complete); +} + +fn single_retained_child_tree(native_index: usize) -> super::ObservedTree { + super::test_support::tree( + vec![ + super::test_support::node( + 0, + super::test_support::evidence("window", Some("Fixture")), + vec![1], + &[], + ), + super::test_support::node( + 1, + super::test_support::evidence("button", Some("Selected")), + Vec::new(), + &[native_index], + ), + ], + vec![0], + true, + ) +} + +fn fixture_tree() -> super::ObservedTree { + super::test_support::tree( + vec![ + super::test_support::node( + 0, + super::test_support::evidence("window", Some("Fixture")), + vec![1, 2], + &[], + ), + super::test_support::node( + 1, + super::test_support::evidence("button", Some("Selected")), + Vec::new(), + &[0], + ), + super::test_support::node( + 2, + super::test_support::evidence("group", Some("Later")), + Vec::new(), + &[1], + ), + ], + vec![0], + true, + ) +} + +fn button_query() -> LocatorQuery { + LocatorQuery { + identity: crate::IdentityPredicate { + role: Some("button".into()), + ..Default::default() + }, + ..Default::default() + } +} + +fn first_request() -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::First, + deadline: crate::Deadline::after(500).unwrap(), + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + } +} diff --git a/crates/core/src/live_locator/test_support.rs b/crates/core/src/live_locator/test_support.rs new file mode 100644 index 0000000..bc90ea5 --- /dev/null +++ b/crates/core/src/live_locator/test_support.rs @@ -0,0 +1,68 @@ +use super::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, LocatorStats, + ObservationSource, ObservedNode, ObservedTree, +}; +use crate::{WindowInfo, refs::RefPath}; + +pub(super) fn window() -> WindowInfo { + WindowInfo { + id: "w-1".into(), + title: "Fixture".into(), + app: "FixtureApp".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + } +} + +pub(crate) fn evidence(role: &str, name: Option<&str>) -> LocatorEvidence { + LocatorEvidence { + role: LocatorField::Known(role.to_string()), + name: name + .map(|value| LocatorField::Known(value.to_string())) + .unwrap_or(LocatorField::Absent), + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::absent(), + states: LocatorField::Known(Vec::new()), + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Known(Vec::new()), + }, + } +} + +pub(crate) fn node( + document_order: u32, + evidence: LocatorEvidence, + children: Vec<u32>, + path: &[usize], +) -> ObservedNode { + ObservedNode { + evidence, + path: RefPath::from_slice(path), + children, + document_order, + completeness: super::ObservationCompleteness::new(true), + children_count: None, + ref_id: None, + } +} + +pub(crate) fn tree( + nodes: Vec<ObservedNode>, + roots: Vec<u32>, + structurally_complete: bool, +) -> ObservedTree { + ObservedTree { + nodes, + roots, + source: ObservationSource::Window(window()), + stats: LocatorStats::default(), + structurally_complete, + } +} diff --git a/crates/core/src/live_locator/tree_order.rs b/crates/core/src/live_locator/tree_order.rs new file mode 100644 index 0000000..fe9f595 --- /dev/null +++ b/crates/core/src/live_locator/tree_order.rs @@ -0,0 +1,87 @@ +use super::ObservedTree; +use crate::AdapterError; +use std::collections::HashSet; + +pub(crate) fn validated_postorder( + tree: &ObservedTree, +) -> Result<(Vec<usize>, Vec<Option<usize>>), AdapterError> { + let mut state = ( + vec![0_u8; tree.nodes.len()], + vec![None; tree.nodes.len()], + Vec::with_capacity(tree.nodes.len()), + ); + for root in &tree.roots { + visit(*root as usize, None, tree, &mut state)?; + } + if state.2.len() != tree.nodes.len() { + return Err(AdapterError::internal( + "locator tree contains unreachable nodes", + )); + } + let mut document_orders = HashSet::new(); + for node in &tree.nodes { + if !document_orders.insert(node.document_order) { + return Err(AdapterError::internal( + "locator tree has duplicate document_order values", + )); + } + } + Ok((state.2, state.1)) +} + +fn visit( + index: usize, + parent: Option<usize>, + tree: &ObservedTree, + state: &mut (Vec<u8>, Vec<Option<usize>>, Vec<usize>), +) -> Result<(), AdapterError> { + let mark = state + .0 + .get(index) + .copied() + .ok_or_else(|| AdapterError::internal("locator tree child index is out of bounds"))?; + if mark == 1 { + return Err(AdapterError::internal("locator tree contains a cycle")); + } + if mark == 2 { + return Err(AdapterError::internal( + "locator tree node has multiple parents", + )); + } + let node = tree + .nodes + .get(index) + .ok_or_else(|| AdapterError::internal("locator tree node index is out of bounds"))?; + let path_matches_edge = match parent { + Some(parent) => tree.nodes.get(parent).is_some_and(|parent| { + node.path.len() == parent.path.len() + 1 + && node.path.starts_with(parent.path.as_slice()) + }), + None => node.path.is_empty(), + }; + if !path_matches_edge { + return Err(AdapterError::internal( + "locator tree node path does not match its edges", + )); + } + state.0[index] = 1; + state.1[index] = parent; + let mut previous_native_index = None; + for child in node.children.iter().copied() { + let native_index = tree + .nodes + .get(child as usize) + .and_then(|child| child.path.last().copied()) + .ok_or_else(|| AdapterError::internal("locator tree child path is empty"))?; + if previous_native_index.is_some_and(|previous| previous >= native_index) { + return Err(AdapterError::internal( + "locator tree children are not in native document order", + )); + } + previous_native_index = Some(native_index); + visit(child as usize, Some(index), tree, state)?; + } + state.0[index] = 2; + state.2.push(index); + Ok(()) +} diff --git a/crates/core/src/live_locator/validate.rs b/crates/core/src/live_locator/validate.rs new file mode 100644 index 0000000..fe53ab4 --- /dev/null +++ b/crates/core/src/live_locator/validate.rs @@ -0,0 +1,81 @@ +use super::LocatorResolveRequest; +use crate::{AdapterError, ErrorCode, locator::LocatorQuery}; + +const MAX_QUERY_CLAUSES: usize = 64; +const MAX_RAW_DEPTH: u8 = 50; +const MAX_QUERY_FIELD_BYTES: usize = 4_096; +const MAX_QUERY_TOTAL_BYTES: usize = 65_536; + +pub fn validate_query(query: &LocatorQuery) -> Result<(), AdapterError> { + query.validate_states()?; + let mut clauses = 0; + let mut total_bytes = 0; + validate_clause(query, &mut clauses, &mut total_bytes) +} + +pub fn validate_request(request: &LocatorResolveRequest) -> Result<(), AdapterError> { + if (1..=MAX_RAW_DEPTH).contains(&request.max_raw_depth) { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!( + "max_raw_depth must be between 1 and {MAX_RAW_DEPTH}, got {}", + request.max_raw_depth + ), + )) +} + +fn validate_clause( + query: &LocatorQuery, + clauses: &mut usize, + total_bytes: &mut usize, +) -> Result<(), AdapterError> { + *clauses += 1; + if *clauses > MAX_QUERY_CLAUSES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("LocatorQuery may contain at most {MAX_QUERY_CLAUSES} recursive clauses"), + )); + } + if let Some(role) = query.identity.role.as_deref() + && !crate::roles::is_valid_role_query(role) + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Unknown role query '{role}'"), + )); + } + for value in [ + query.identity.role.as_deref(), + query.identity.name.as_deref(), + query.identity.description.as_deref(), + query.identity.native_id.as_deref(), + query.identity.value.as_deref(), + query.has_text.as_deref(), + ] + .into_iter() + .flatten() + { + if value.len() > MAX_QUERY_FIELD_BYTES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Locator query field exceeds {MAX_QUERY_FIELD_BYTES} bytes"), + )); + } + *total_bytes = total_bytes.saturating_add(value.len()); + if *total_bytes > MAX_QUERY_TOTAL_BYTES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Locator query exceeds {MAX_QUERY_TOTAL_BYTES} aggregate bytes"), + )); + } + } + if let Some(has) = &query.containment.has { + validate_clause(has, clauses, total_bytes)?; + } + if let Some(has_not) = &query.containment.has_not { + validate_clause(has_not, clauses, total_bytes)?; + } + Ok(()) +} diff --git a/crates/core/src/live_locator/validation_tests.rs b/crates/core/src/live_locator/validation_tests.rs new file mode 100644 index 0000000..7ef333b --- /dev/null +++ b/crates/core/src/live_locator/validation_tests.rs @@ -0,0 +1,170 @@ +use super::{ + EvidenceRequirements, LocatorMaterialization, LocatorResolveRequest, LocatorSelection, + ObservationRequest, ObservationRoot, validate_query, validate_request, +}; +use crate::{ + ErrorCode, WindowInfo, + adapter::ObservationOps, + locator::{ContainmentPredicate, LocatorQuery, StatePredicate}, +}; + +struct UnsupportedAdapter; + +impl ObservationOps for UnsupportedAdapter {} + +fn request(max_raw_depth: u8) -> LocatorResolveRequest { + LocatorResolveRequest { + selection: LocatorSelection::All { limit: None }, + deadline: crate::Deadline::from_duration(std::time::Duration::from_secs(5)).unwrap(), + max_raw_depth, + materialization: LocatorMaterialization::None, + } +} + +#[test] +fn match_all_query_remains_valid() { + validate_query(&LocatorQuery::default()).unwrap(); +} + +#[test] +fn recursive_state_validation_runs_before_native_work() { + let query = LocatorQuery { + containment: ContainmentPredicate { + has: Some(Box::new(LocatorQuery { + states: vec![StatePredicate { + token: "imaginary".into(), + expected: None, + }], + ..LocatorQuery::default() + })), + has_not: None, + }, + ..LocatorQuery::default() + }; + let error = validate_query(&query).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +#[test] +fn recursive_query_clause_limit_is_bounded() { + let mut query = LocatorQuery::default(); + for _ in 0..64 { + query = LocatorQuery { + containment: ContainmentPredicate { + has: Some(Box::new(query)), + has_not: None, + }, + ..LocatorQuery::default() + }; + } + let error = validate_query(&query).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +#[test] +fn raw_depth_must_fit_native_safety_cap() { + for invalid in [0, 51] { + let error = validate_request(&request(invalid)).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidArgs); + } + validate_request(&request(50)).unwrap(); +} + +#[test] +fn tree_builder_default_is_not_supported() { + let adapter = UnsupportedAdapter; + let window = WindowInfo { + id: "w-1".into(), + title: "Fixture".into(), + app: "FixtureApp".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }; + let error = match adapter.observe_tree( + ObservationRoot::Window(&window), + &ObservationRequest::locator( + &LocatorQuery::default(), + &request(50), + crate::Deadline::standard().unwrap(), + ), + ) { + Ok(_) => panic!("default implementation should fail"), + Err(error) => error, + }; + assert_eq!(error.code, ErrorCode::PlatformNotSupported); +} + +#[test] +fn element_root_derives_surface_with_relative_depth_budgets() { + let entry = crate::RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(42), + process_instance: Some("instance-1".into()), + }, + identity: crate::RefEntryIdentity { + role: "group".into(), + name: Some("Menu root".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: Vec::new(), + available_actions: Vec::new(), + }, + source: crate::RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-1".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::SnapshotSurface::Menu, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: true, + path: smallvec::smallvec![3, 4, 5], + }, + }; + let handle = crate::NativeHandle::null(); + let root = ObservationRoot::Element { + handle: &handle, + entry: &entry, + root_ref: Some("@e1"), + }; + let locator = ObservationRequest::locator_for_root( + &LocatorQuery::default(), + &request(12), + root, + crate::Deadline::standard().unwrap(), + ); + let hydration = ObservationRequest::selected_hydration( + &LocatorQuery::default(), + &request(12), + root, + crate::Deadline::standard().unwrap(), + ); + + assert_eq!(locator.surface, crate::SnapshotSurface::Menu); + assert_eq!(locator.max_raw_depth, 12); + assert_eq!(locator.max_logical_depth, 12); + assert_eq!(hydration.surface, crate::SnapshotSurface::Menu); + assert_eq!(hydration.max_raw_depth, 1); + assert_eq!(hydration.max_logical_depth, 0); + assert_eq!( + hydration.evidence_for_raw_depth(0), + EvidenceRequirements::snapshot() + ); + assert_eq!( + hydration.evidence_for_raw_depth(1), + EvidenceRequirements::query(&LocatorQuery::default()) + ); +} diff --git a/crates/core/src/locator.rs b/crates/core/src/locator.rs new file mode 100644 index 0000000..1a587a5 --- /dev/null +++ b/crates/core/src/locator.rs @@ -0,0 +1,199 @@ +use crate::{ + AccessibilityNode, AdapterError, ErrorCode, roles, search_text, + state::{self, STATE_VOCABULARY}, +}; +pub(crate) use crate::{ContainmentPredicate, IdentityPredicate, NodeMatchContext, StatePredicate}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct LocatorQuery { + #[serde(flatten)] + pub identity: IdentityPredicate, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub has_text: Option<String>, + #[serde(default)] + pub exact: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub states: Vec<StatePredicate>, + #[serde(flatten)] + pub containment: ContainmentPredicate, +} + +impl LocatorQuery { + pub fn is_empty(&self) -> bool { + self.identity.role.is_none() + && self.identity.name.is_none() + && self.identity.description.is_none() + && self.identity.native_id.is_none() + && self.identity.value.is_none() + && self.has_text.is_none() + && self.states.is_empty() + && self.containment.has.is_none() + && self.containment.has_not.is_none() + } + + pub fn validate_states(&self) -> Result<(), AdapterError> { + for predicate in &self.states { + if !STATE_VOCABULARY.contains(&predicate.token.as_str()) { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Unknown state token '{}'", predicate.token), + ) + .with_suggestion(format!("Use one of: {}", STATE_VOCABULARY.join(", ")))); + } + } + if let Some(has) = &self.containment.has { + has.validate_states()?; + } + if let Some(has_not) = &self.containment.has_not { + has_not.validate_states()?; + } + Ok(()) + } +} + +pub fn node_context(node: &AccessibilityNode) -> NodeMatchContext<'_> { + NodeMatchContext { + role: &node.role, + name: node.identity.name.as_deref(), + description: node.identity.description.as_deref(), + native_id: node + .identity + .native_id + .as_ref() + .map(|identifier| identifier.value.as_str()), + value: node.identity.value.as_deref(), + states: &node.presentation.states, + children: &node.children, + } +} + +pub fn node_matches(query: &LocatorQuery, ctx: NodeMatchContext<'_>) -> bool { + if !role_matches(query, ctx.role) { + return false; + } + if !text_field_matches(query.identity.name.as_deref(), ctx.name, query.exact) { + return false; + } + if !text_field_matches( + query.identity.description.as_deref(), + ctx.description, + query.exact, + ) { + return false; + } + if !native_id_matches(query.identity.native_id.as_deref(), ctx.native_id) { + return false; + } + if !text_field_matches(query.identity.value.as_deref(), ctx.value, query.exact) { + return false; + } + if !has_text_matches(query.has_text.as_deref(), &ctx, query.exact) { + return false; + } + if !state_predicates_match(&query.states, ctx.states) { + return false; + } + if let Some(has) = &query.containment.has { + if !subtree_contains(has, ctx.children) { + return false; + } + } + if let Some(has_not) = &query.containment.has_not { + if subtree_contains(has_not, ctx.children) { + return false; + } + } + true +} + +pub fn accessibility_node_matches(node: &AccessibilityNode, query: &LocatorQuery) -> bool { + node_matches(query, node_context(node)) +} + +/// Cheap standalone role predicate, factored out of [`node_matches`] so +/// callers building a full match context (states, children, name/value +/// text) can short-circuit before paying for that work when the role alone +/// already rules a candidate out. +pub fn role_matches(query: &LocatorQuery, role: &str) -> bool { + query + .identity + .role + .as_deref() + .is_none_or(|expected| roles::normalize_role_query(expected) == role) +} + +fn text_field_matches(expected: Option<&str>, actual: Option<&str>, exact: bool) -> bool { + let Some(expected) = expected else { + return true; + }; + let Some(actual) = actual else { + return false; + }; + if exact { + search_text::normalize(actual) == expected + } else { + search_text::contains(actual, expected) + } +} + +fn native_id_matches(expected: Option<&str>, actual: Option<&str>) -> bool { + let Some(expected) = expected else { + return true; + }; + actual == Some(expected) +} + +fn has_text_matches(expected: Option<&str>, ctx: &NodeMatchContext<'_>, exact: bool) -> bool { + let Some(expected) = expected else { + return true; + }; + [ctx.name, ctx.description, ctx.value] + .into_iter() + .flatten() + .any(|text| text_matches(text, expected, exact)) + || ctx + .children + .iter() + .any(|child| subtree_text_matches(child, expected, exact)) +} + +fn subtree_text_matches(node: &AccessibilityNode, expected: &str, exact: bool) -> bool { + [ + node.identity.name.as_deref(), + node.identity.description.as_deref(), + node.identity.value.as_deref(), + ] + .into_iter() + .flatten() + .any(|text| text_matches(text, expected, exact)) + || node + .children + .iter() + .any(|child| subtree_text_matches(child, expected, exact)) +} + +fn text_matches(actual: &str, expected: &str, exact: bool) -> bool { + if exact { + search_text::normalize(actual) == expected + } else { + search_text::contains(actual, expected) + } +} + +fn state_predicates_match(predicates: &[StatePredicate], states: &[String]) -> bool { + predicates.iter().all(|predicate| { + let present = state::has_state(states, &predicate.token); + predicate.expected.unwrap_or(true) == present + }) +} + +fn subtree_contains(query: &LocatorQuery, children: &[AccessibilityNode]) -> bool { + children.iter().any(|child| { + accessibility_node_matches(child, query) || subtree_contains(query, &child.children) + }) +} + +#[cfg(test)] +#[path = "locator_tests.rs"] +mod tests; diff --git a/crates/core/src/locator_tests.rs b/crates/core/src/locator_tests.rs new file mode 100644 index 0000000..3446c9b --- /dev/null +++ b/crates/core/src/locator_tests.rs @@ -0,0 +1,275 @@ +use super::*; +use crate::{AccessibilityNode, search_text, state}; + +fn node( + role: &str, + name: Option<&str>, + value: Option<&str>, + states: Vec<&str>, +) -> AccessibilityNode { + AccessibilityNode { + ref_id: None, + role: role.into(), + identity: crate::NodeIdentity { + name: name.map(String::from), + value: value.map(String::from), + ..Default::default() + }, + presentation: crate::NodePresentation { + states: states.into_iter().map(String::from).collect(), + ..Default::default() + }, + children_count: None, + children: vec![], + } +} + +#[test] +fn role_and_name_substring_match() { + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + name: Some(search_text::normalize("save")), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + let target = node("button", Some("Save Changes"), None, vec![]); + assert!(accessibility_node_matches(&target, &query)); +} + +#[test] +fn exact_name_requires_equality() { + let query = LocatorQuery { + identity: IdentityPredicate { + name: Some(search_text::normalize("save")), + ..IdentityPredicate::default() + }, + exact: true, + ..LocatorQuery::default() + }; + assert!(!accessibility_node_matches( + &node("button", Some("Save Changes"), None, vec![]), + &query + )); + assert!(accessibility_node_matches( + &node("button", Some("Save"), None, vec![]), + &query + )); +} + +#[test] +fn state_predicate_filters_checked() { + let query = LocatorQuery { + states: vec![StatePredicate { + token: state::CHECKED.into(), + expected: Some(true), + }], + ..LocatorQuery::default() + }; + assert!(accessibility_node_matches( + &node("checkbox", None, None, vec![state::CHECKED]), + &query + )); + assert!(!accessibility_node_matches( + &node("checkbox", None, None, vec![]), + &query + )); +} + +#[test] +fn native_id_exact_match() { + let mut target = node("button", Some("X"), None, vec![]); + target.identity.native_id = Some(crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: "submit-btn".into(), + }); + let query = LocatorQuery { + identity: IdentityPredicate { + native_id: Some("submit-btn".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + }; + assert!(accessibility_node_matches(&target, &query)); +} + +#[test] +fn has_subquery_matches_descendant() { + let mut root = node("group", None, None, vec![]); + root.children + .push(node("statictext", Some("Hello"), None, vec![])); + let query = LocatorQuery { + containment: ContainmentPredicate { + has: Some(Box::new(LocatorQuery { + has_text: Some(search_text::normalize("hello")), + ..LocatorQuery::default() + })), + ..ContainmentPredicate::default() + }, + ..LocatorQuery::default() + }; + assert!(accessibility_node_matches(&root, &query)); +} + +#[test] +fn has_subquery_matches_deep_descendant() { + let mut root = node("group", None, None, vec![]); + let mut wrapper = node("group", None, None, vec![]); + wrapper + .children + .push(node("statictext", Some("Hello"), None, vec![])); + root.children.push(wrapper); + let query = LocatorQuery { + containment: ContainmentPredicate { + has: Some(Box::new(LocatorQuery { + has_text: Some(search_text::normalize("hello")), + ..LocatorQuery::default() + })), + ..ContainmentPredicate::default() + }, + ..LocatorQuery::default() + }; + assert!(accessibility_node_matches(&root, &query)); +} + +#[test] +fn has_text_matches_deep_descendant_text() { + let mut root = node("group", Some("Settings"), None, vec![]); + let mut wrapper = node("group", None, None, vec![]); + wrapper.children.push(node( + "statictext", + Some("Workspace notifications"), + None, + vec![], + )); + root.children.push(wrapper); + let query = LocatorQuery { + has_text: Some(search_text::normalize("notifications")), + ..LocatorQuery::default() + }; + assert!(accessibility_node_matches(&root, &query)); +} + +#[test] +fn invalid_state_token_rejected() { + let query = LocatorQuery { + states: vec![StatePredicate { + token: "bogus".into(), + expected: None, + }], + ..LocatorQuery::default() + }; + assert!(query.validate_states().is_err()); +} + +#[test] +fn has_not_excludes_subtree_match() { + let mut root = node("group", None, None, vec![]); + root.children + .push(node("button", Some("Delete"), None, vec![])); + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("group".into()), + ..IdentityPredicate::default() + }, + containment: ContainmentPredicate { + has_not: Some(Box::new(LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + name: Some(search_text::normalize("delete")), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + })), + ..ContainmentPredicate::default() + }, + ..LocatorQuery::default() + }; + assert!(!accessibility_node_matches(&root, &query)); +} + +#[test] +fn has_not_excludes_deep_subtree_match() { + let mut root = node("group", None, None, vec![]); + let mut wrapper = node("group", None, None, vec![]); + wrapper + .children + .push(node("button", Some("Delete"), None, vec![])); + root.children.push(wrapper); + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("group".into()), + ..IdentityPredicate::default() + }, + containment: ContainmentPredicate { + has_not: Some(Box::new(LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + name: Some(search_text::normalize("delete")), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + })), + ..ContainmentPredicate::default() + }, + ..LocatorQuery::default() + }; + assert!(!accessibility_node_matches(&root, &query)); +} + +/// The `IdentityPredicate`/`ContainmentPredicate` split (god-object fix) must +/// stay wire-compatible: JSON consumers still see a single flat object, not +/// `{"identity": {...}}`. Reverting the `#[serde(flatten)]` attributes (e.g. +/// accidentally serializing the grouped sub-structs as nested objects) would +/// fail this assertion. +#[test] +fn identity_and_containment_fields_serialize_flat_not_nested() { + let query = LocatorQuery { + identity: IdentityPredicate { + role: Some("button".into()), + ..IdentityPredicate::default() + }, + containment: ContainmentPredicate { + has: Some(Box::new(LocatorQuery { + identity: IdentityPredicate { + role: Some("statictext".into()), + ..IdentityPredicate::default() + }, + ..LocatorQuery::default() + })), + ..ContainmentPredicate::default() + }, + ..LocatorQuery::default() + }; + + let json = serde_json::to_value(&query).unwrap(); + + assert_eq!(json["role"], "button"); + assert!(json.get("identity").is_none()); + assert!(json.get("containment").is_none()); + assert_eq!(json["has"]["role"], "statictext"); +} + +/// Round-trips a flat JSON payload (the shape any pre-existing caller sends) +/// back through the new nested Rust representation. +#[test] +fn flat_json_deserializes_into_nested_identity_and_containment() { + let query: LocatorQuery = serde_json::from_value(serde_json::json!({ + "role": "button", + "native_id": "submit-btn", + "has_not": { "role": "textfield" } + })) + .unwrap(); + + assert_eq!(query.identity.role.as_deref(), Some("button")); + assert_eq!(query.identity.native_id.as_deref(), Some("submit-btn")); + assert_eq!( + query + .containment + .has_not + .as_ref() + .and_then(|q| q.identity.role.as_deref()), + Some("textfield") + ); +} diff --git a/crates/core/src/modifier.rs b/crates/core/src/modifier.rs new file mode 100644 index 0000000..cf8ef0c --- /dev/null +++ b/crates/core/src/modifier.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum Modifier { + #[serde(alias = "Cmd")] + Meta, + Ctrl, + Alt, + Shift, +} diff --git a/crates/core/src/mouse_button.rs b/crates/core/src/mouse_button.rs new file mode 100644 index 0000000..86e5980 --- /dev/null +++ b/crates/core/src/mouse_button.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MouseButton { + Left, + Right, + Middle, +} diff --git a/crates/core/src/mouse_click_count.rs b/crates/core/src/mouse_click_count.rs new file mode 100644 index 0000000..c8b270b --- /dev/null +++ b/crates/core/src/mouse_click_count.rs @@ -0,0 +1,38 @@ +use crate::{AdapterError, ErrorCode}; + +pub const MAX_MOUSE_CLICK_COUNT: u32 = 100; + +pub fn validate_mouse_click_count(count: u32) -> Result<(), AdapterError> { + if (1..=MAX_MOUSE_CLICK_COUNT).contains(&count) { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Mouse click count must be between 1 and {MAX_MOUSE_CLICK_COUNT}"), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_bounded_positive_counts() { + assert!(validate_mouse_click_count(1).is_ok()); + assert!(validate_mouse_click_count(MAX_MOUSE_CLICK_COUNT).is_ok()); + } + + #[test] + fn rejects_zero_and_unbounded_counts() { + assert_eq!( + validate_mouse_click_count(0).unwrap_err().code, + ErrorCode::InvalidArgs + ); + assert_eq!( + validate_mouse_click_count(MAX_MOUSE_CLICK_COUNT + 1) + .unwrap_err() + .code, + ErrorCode::InvalidArgs + ); + } +} diff --git a/crates/core/src/mouse_event.rs b/crates/core/src/mouse_event.rs new file mode 100644 index 0000000..0810cda --- /dev/null +++ b/crates/core/src/mouse_event.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +use crate::{Modifier, MouseButton, MouseEventKind, Point}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MouseEvent { + pub kind: MouseEventKind, + pub point: Point, + pub button: MouseButton, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub modifiers: Vec<Modifier>, +} diff --git a/crates/core/src/mouse_event_kind.rs b/crates/core/src/mouse_event_kind.rs new file mode 100644 index 0000000..8f9e665 --- /dev/null +++ b/crates/core/src/mouse_event_kind.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MouseEventKind { + Move, + Down, + Up, + Click { count: u32 }, + Wheel { delta_x: f64, delta_y: f64 }, +} diff --git a/crates/core/src/name_evidence.rs b/crates/core/src/name_evidence.rs new file mode 100644 index 0000000..7b170ab --- /dev/null +++ b/crates/core/src/name_evidence.rs @@ -0,0 +1,10 @@ +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NameEvidence { + pub explicit_label: Option<String>, + pub labelled_by_text: Option<String>, + pub native_title: Option<String>, + pub static_value: Option<String>, + pub child_label: Option<String>, + pub placeholder: Option<String>, + pub description: Option<String>, +} diff --git a/crates/core/src/native_handle.rs b/crates/core/src/native_handle.rs new file mode 100644 index 0000000..3adf55a --- /dev/null +++ b/crates/core/src/native_handle.rs @@ -0,0 +1,96 @@ +use std::{any::Any, marker::PhantomData, rc::Rc}; + +/// An owned platform accessibility payload. +/// +/// Platform adapters may store a native RAII wrapper or a worker-thread token. +/// The payload is destroyed exactly once when this value is dropped. The handle +/// is intentionally neither `Send` nor `Sync` so thread-affine platform objects +/// cannot cross threads through the platform-neutral API. +/// +/// ```compile_fail +/// use agent_desktop_core::NativeHandle; +/// fn require_send<T: Send>() {} +/// require_send::<NativeHandle>(); +/// ``` +/// +/// ```compile_fail +/// use agent_desktop_core::NativeHandle; +/// fn require_sync<T: Sync>() {} +/// require_sync::<NativeHandle>(); +/// ``` +pub struct NativeHandle { + payload: Option<Box<dyn Any>>, + _thread_affinity: PhantomData<Rc<()>>, +} + +impl NativeHandle { + /// Owns a platform-specific payload until this handle is dropped. + pub fn new<T: Any>(payload: T) -> Self { + Self { + payload: Some(Box::new(payload)), + _thread_affinity: PhantomData, + } + } + + /// Borrows the payload when it has the requested platform-specific type. + pub fn downcast_ref<T: Any>(&self) -> Option<&T> { + self.payload.as_deref()?.downcast_ref::<T>() + } + + /// Returns an empty handle for adapters and tests without a native payload. + pub fn null() -> Self { + Self { + payload: None, + _thread_affinity: PhantomData, + } + } + + /// Reports whether this handle has no platform payload. + pub fn is_null(&self) -> bool { + self.payload.is_none() + } +} + +#[cfg(test)] +mod tests { + use super::NativeHandle; + use std::{cell::Cell, rc::Rc}; + + struct DropProbe(Rc<Cell<u32>>); + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + #[test] + fn owns_and_downcasts_typed_payload() { + let handle = NativeHandle::new(String::from("platform-token")); + + assert_eq!( + handle.downcast_ref::<String>().map(String::as_str), + Some("platform-token") + ); + assert!(handle.downcast_ref::<u64>().is_none()); + assert!(!handle.is_null()); + } + + #[test] + fn drops_payload_exactly_once() { + let drops = Rc::new(Cell::new(0)); + { + let _handle = NativeHandle::new(DropProbe(Rc::clone(&drops))); + assert_eq!(drops.get(), 0); + } + assert_eq!(drops.get(), 1); + } + + #[test] + fn null_has_no_typed_payload() { + let handle = NativeHandle::null(); + + assert!(handle.is_null()); + assert!(handle.downcast_ref::<String>().is_none()); + } +} diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 950fd0b..ea88124 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -1,4 +1,8 @@ -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; + +#[cfg(test)] +use crate::{AppInfo, Rect}; +use crate::{NodeIdentity, NodePresentation}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AccessibilityNode { @@ -7,26 +11,11 @@ pub struct AccessibilityNode { pub role: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub name: Option<String>, + #[serde(flatten)] + pub identity: NodeIdentity, - #[serde(skip_serializing_if = "Option::is_none")] - pub value: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub hint: Option<String>, - - #[serde(skip_serializing_if = "Vec::is_empty", default)] - pub states: Vec<String>, - - #[serde(skip_serializing_if = "Vec::is_empty", default)] - pub available_actions: Vec<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub bounds: Option<Rect>, + #[serde(flatten)] + pub presentation: NodePresentation, #[serde(skip_serializing_if = "Option::is_none", default)] pub children_count: Option<u32>, @@ -35,69 +24,6 @@ pub struct AccessibilityNode { pub children: Vec<AccessibilityNode>, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub struct Rect { - #[serde(default, deserialize_with = "f64_or_zero")] - pub x: f64, - #[serde(default, deserialize_with = "f64_or_zero")] - pub y: f64, - #[serde(default, deserialize_with = "f64_or_zero")] - pub width: f64, - #[serde(default, deserialize_with = "f64_or_zero")] - pub height: f64, -} - -fn f64_or_zero<'de, D: Deserializer<'de>>(deserializer: D) -> Result<f64, D::Error> { - Option::<f64>::deserialize(deserializer).map(|opt| opt.unwrap_or(0.0)) -} - -impl Rect { - pub fn bounds_hash(&self) -> u64 { - use rustc_hash::FxHasher; - use std::hash::{Hash, Hasher}; - let mut h = FxHasher::default(); - let x = (self.x * 100.0) as i64; - let y = (self.y * 100.0) as i64; - let w = (self.width * 100.0) as i64; - let hh = (self.height * 100.0) as i64; - x.hash(&mut h); - y.hash(&mut h); - w.hash(&mut h); - hh.hash(&mut h); - h.finish() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WindowInfo { - pub id: String, - pub title: String, - #[serde(rename = "app_name")] - pub app: String, - pub pid: i32, - #[serde(skip_serializing_if = "Option::is_none")] - pub bounds: Option<Rect>, - pub is_focused: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AppInfo { - pub name: String, - pub pid: i32, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub bundle_id: Option<String>, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SurfaceInfo { - #[serde(rename = "type")] - pub kind: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub title: Option<String>, - #[serde(skip_serializing_if = "Option::is_none")] - pub item_count: Option<usize>, -} - #[cfg(test)] #[path = "node_tests.rs"] mod tests; diff --git a/crates/core/src/node_identity.rs b/crates/core/src/node_identity.rs new file mode 100644 index 0000000..88877a9 --- /dev/null +++ b/crates/core/src/node_identity.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct NodeIdentity { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_id: Option<crate::ElementIdentifier>, +} diff --git a/crates/core/src/node_match_context.rs b/crates/core/src/node_match_context.rs new file mode 100644 index 0000000..ba4cfef --- /dev/null +++ b/crates/core/src/node_match_context.rs @@ -0,0 +1,11 @@ +use crate::AccessibilityNode; + +pub struct NodeMatchContext<'a> { + pub role: &'a str, + pub name: Option<&'a str>, + pub description: Option<&'a str>, + pub native_id: Option<&'a str>, + pub value: Option<&'a str>, + pub states: &'a [String], + pub children: &'a [AccessibilityNode], +} diff --git a/crates/core/src/node_presentation.rs b/crates/core/src/node_presentation.rs new file mode 100644 index 0000000..15f9f13 --- /dev/null +++ b/crates/core/src/node_presentation.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; + +use crate::Rect; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct NodePresentation { + #[serde(skip_serializing_if = "Option::is_none")] + pub hint: Option<String>, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub states: Vec<String>, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub available_actions: Vec<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub bounds: Option<Rect>, +} diff --git a/crates/core/src/node_tests.rs b/crates/core/src/node_tests.rs index 3b6f47b..d3b8a86 100644 --- a/crates/core/src/node_tests.rs +++ b/crates/core/src/node_tests.rs @@ -22,13 +22,11 @@ fn test_children_count_omitted_when_none() { let node = AccessibilityNode { ref_id: None, role: "group".into(), - name: Some("Sidebar".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some("Sidebar".into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children: vec![], }; @@ -41,13 +39,11 @@ fn test_children_count_present_when_set() { let node = AccessibilityNode { ref_id: None, role: "group".into(), - name: Some("Sidebar".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some("Sidebar".into()), + ..Default::default() + }, + presentation: Default::default(), children_count: Some(47), children: vec![], }; @@ -83,8 +79,9 @@ fn test_rect_normal_roundtrip() { fn app_info_bundle_id_none_omitted_from_json() { let info = AppInfo { name: "Finder".into(), - pid: 42, + pid: crate::ProcessId::new(42), bundle_id: None, + process_instance: Some("test-instance".into()), }; let json = serde_json::to_string(&info).unwrap(); assert!( @@ -98,8 +95,9 @@ fn app_info_bundle_id_none_omitted_from_json() { fn app_info_bundle_id_some_present_in_json() { let info = AppInfo { name: "Safari".into(), - pid: 7, + pid: crate::ProcessId::new(7), bundle_id: Some("com.apple.Safari".into()), + process_instance: Some("test-instance".into()), }; let json = serde_json::to_string(&info).unwrap(); assert!( @@ -114,8 +112,9 @@ fn app_info_bundle_id_some_present_in_json() { fn app_info_roundtrip_preserves_all_fields() { let original = AppInfo { name: "TextEdit".into(), - pid: 1234, + pid: crate::ProcessId::new(1234), bundle_id: Some("com.apple.TextEdit".into()), + process_instance: Some("test-instance".into()), }; let json = serde_json::to_string(&original).unwrap(); let back: AppInfo = serde_json::from_str(&json).unwrap(); @@ -145,13 +144,11 @@ fn accessibility_node_bounds_none_omitted_from_json() { let node = AccessibilityNode { ref_id: None, role: "button".into(), - name: Some("OK".into()), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: Some("OK".into()), + ..Default::default() + }, + presentation: Default::default(), children_count: None, children: vec![], }; @@ -169,13 +166,8 @@ fn accessibility_node_hint_none_omitted_from_json() { let node = AccessibilityNode { ref_id: None, role: "textfield".into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: vec![], }; diff --git a/crates/core/src/notification.rs b/crates/core/src/notification.rs deleted file mode 100644 index 82618cc..0000000 --- a/crates/core/src/notification.rs +++ /dev/null @@ -1,154 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NotificationInfo { - pub index: usize, - pub app_name: String, - pub title: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub body: Option<String>, - #[serde(skip_serializing_if = "Vec::is_empty", default)] - pub actions: Vec<String>, -} - -#[derive(Debug, Clone, Default)] -pub struct NotificationFilter { - pub app: Option<String>, - pub text: Option<String>, - pub limit: Option<usize>, -} - -/// Identity fingerprint used to verify that a notification-action call -/// targets the same notification a host observed via `list_notifications`. -/// -/// Notification Center reindexes entries on every listing: opening, -/// dismissing, or receiving a new notification shifts what sits at any -/// given `index`. Calling `notification_action(index, ...)` without -/// verification can therefore press the action button on a different -/// notification than the host intended (confused-deputy at the OS -/// boundary). -/// -/// The host should pass the `app_name` and/or `title` observed on the -/// notification at the moment of listing. If either field is `Some` and -/// the row currently at `index` does not match, the action is rejected -/// with `ErrorCode::NotificationNotFound`. Leaving both `None` preserves -/// the legacy index-only behavior for callers that have already done -/// their own reconciliation. -#[derive(Debug, Clone, Default)] -pub struct NotificationIdentity { - pub expected_app: Option<String>, - pub expected_title: Option<String>, -} - -impl NotificationIdentity { - pub fn is_empty(&self) -> bool { - self.expected_app.is_none() && self.expected_title.is_none() - } - - /// Returns true when `info` matches every field that is `Some` on - /// this identity. `None` fields are treated as wildcards. - pub fn matches(&self, info: &NotificationInfo) -> bool { - if let Some(ref app) = self.expected_app { - if app != &info.app_name { - return false; - } - } - if let Some(ref title) = self.expected_title { - if title != &info.title { - return false; - } - } - true - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn notification_info_serialization_omits_none_fields() { - let info = NotificationInfo { - index: 1, - app_name: "Messages".into(), - title: "New message".into(), - body: None, - actions: vec![], - }; - let json = serde_json::to_value(&info).unwrap(); - assert!(!json.as_object().unwrap().contains_key("body")); - assert!(!json.as_object().unwrap().contains_key("actions")); - } - - #[test] - fn notification_info_serialization_includes_present_fields() { - let info = NotificationInfo { - index: 2, - app_name: "Slack".into(), - title: "Channel update".into(), - body: Some("New message in #general".into()), - actions: vec!["Reply".into(), "Open".into()], - }; - let json = serde_json::to_value(&info).unwrap(); - assert_eq!(json["body"], "New message in #general"); - assert_eq!(json["actions"], serde_json::json!(["Reply", "Open"])); - } - - #[test] - fn notification_filter_default_is_unfiltered() { - let filter = NotificationFilter::default(); - assert!(filter.app.is_none()); - assert!(filter.text.is_none()); - assert!(filter.limit.is_none()); - } - - fn sample_info(app: &str, title: &str) -> NotificationInfo { - NotificationInfo { - index: 1, - app_name: app.into(), - title: title.into(), - body: None, - actions: vec![], - } - } - - #[test] - fn identity_default_is_empty_and_matches_everything() { - let id = NotificationIdentity::default(); - assert!(id.is_empty()); - assert!(id.matches(&sample_info("Messages", "Hi"))); - assert!(id.matches(&sample_info("Slack", "New"))); - } - - #[test] - fn identity_with_only_app_matches_any_title_on_that_app() { - let id = NotificationIdentity { - expected_app: Some("Messages".into()), - expected_title: None, - }; - assert!(!id.is_empty()); - assert!(id.matches(&sample_info("Messages", "anything"))); - assert!(!id.matches(&sample_info("Slack", "anything"))); - } - - #[test] - fn identity_with_only_title_matches_any_app_with_that_title() { - let id = NotificationIdentity { - expected_app: None, - expected_title: Some("Meeting starting".into()), - }; - assert!(id.matches(&sample_info("Calendar", "Meeting starting"))); - assert!(!id.matches(&sample_info("Calendar", "Reminder"))); - } - - #[test] - fn identity_with_both_requires_both_to_match() { - let id = NotificationIdentity { - expected_app: Some("Calendar".into()), - expected_title: Some("Meeting".into()), - }; - assert!(id.matches(&sample_info("Calendar", "Meeting"))); - assert!(!id.matches(&sample_info("Calendar", "Other"))); - assert!(!id.matches(&sample_info("Other", "Meeting"))); - } -} diff --git a/crates/core/src/notification_action_request.rs b/crates/core/src/notification_action_request.rs new file mode 100644 index 0000000..c0a1873 --- /dev/null +++ b/crates/core/src/notification_action_request.rs @@ -0,0 +1,8 @@ +use crate::{InteractionPolicy, NotificationIdentity}; + +pub struct NotificationActionRequest<'a> { + pub index: usize, + pub identity: &'a NotificationIdentity, + pub action_name: &'a str, + pub policy: InteractionPolicy, +} diff --git a/crates/core/src/notification_filter.rs b/crates/core/src/notification_filter.rs new file mode 100644 index 0000000..82f8b1e --- /dev/null +++ b/crates/core/src/notification_filter.rs @@ -0,0 +1,10 @@ +#[derive(Debug, Clone, Default)] +pub struct NotificationFilter { + pub app: Option<String>, + pub text: Option<String>, + pub limit: Option<usize>, +} + +#[cfg(test)] +#[path = "notification_filter_tests.rs"] +mod tests; diff --git a/crates/core/src/notification_filter_tests.rs b/crates/core/src/notification_filter_tests.rs new file mode 100644 index 0000000..770a998 --- /dev/null +++ b/crates/core/src/notification_filter_tests.rs @@ -0,0 +1,9 @@ +use crate::NotificationFilter; + +#[test] +fn default_is_unfiltered() { + let filter = NotificationFilter::default(); + assert!(filter.app.is_none()); + assert!(filter.text.is_none()); + assert!(filter.limit.is_none()); +} diff --git a/crates/core/src/notification_identity.rs b/crates/core/src/notification_identity.rs new file mode 100644 index 0000000..2da5ce0 --- /dev/null +++ b/crates/core/src/notification_identity.rs @@ -0,0 +1,28 @@ +use crate::NotificationInfo; + +#[derive(Debug, Clone, Default)] +pub struct NotificationIdentity { + pub expected_app: Option<String>, + pub expected_title: Option<String>, +} + +impl NotificationIdentity { + pub fn is_empty(&self) -> bool { + self.expected_app.as_deref().is_none_or(str::is_empty) + && self.expected_title.as_deref().is_none_or(str::is_empty) + } + + pub fn matches(&self, info: &NotificationInfo) -> bool { + self.expected_app + .as_ref() + .is_none_or(|expected| expected == &info.app_name) + && self + .expected_title + .as_ref() + .is_none_or(|expected| expected == &info.title) + } +} + +#[cfg(test)] +#[path = "notification_identity_tests.rs"] +mod tests; diff --git a/crates/core/src/notification_identity_tests.rs b/crates/core/src/notification_identity_tests.rs new file mode 100644 index 0000000..6581217 --- /dev/null +++ b/crates/core/src/notification_identity_tests.rs @@ -0,0 +1,53 @@ +use crate::{NotificationIdentity, NotificationInfo}; + +fn sample_info(app: &str, title: &str) -> NotificationInfo { + NotificationInfo { + index: 1, + app_name: app.into(), + title: title.into(), + body: None, + actions: vec![], + } +} + +#[test] +fn default_is_empty_and_matches_everything() { + let identity = NotificationIdentity::default(); + assert!(identity.is_empty()); + assert!(identity.matches(&sample_info("Messages", "Hi"))); + assert!(identity.matches(&sample_info("Slack", "New"))); +} + +#[test] +fn empty_string_fields_do_not_form_an_identity() { + let identity = NotificationIdentity { + expected_app: Some(String::new()), + expected_title: None, + }; + assert!(identity.is_empty()); +} + +#[test] +fn present_fields_must_match() { + let app_only = NotificationIdentity { + expected_app: Some("Messages".into()), + expected_title: None, + }; + assert!(app_only.matches(&sample_info("Messages", "anything"))); + assert!(!app_only.matches(&sample_info("Slack", "anything"))); + + let title_only = NotificationIdentity { + expected_app: None, + expected_title: Some("Meeting".into()), + }; + assert!(title_only.matches(&sample_info("Calendar", "Meeting"))); + assert!(!title_only.matches(&sample_info("Calendar", "Reminder"))); + + let both = NotificationIdentity { + expected_app: Some("Calendar".into()), + expected_title: Some("Meeting".into()), + }; + assert!(both.matches(&sample_info("Calendar", "Meeting"))); + assert!(!both.matches(&sample_info("Calendar", "Other"))); + assert!(!both.matches(&sample_info("Other", "Meeting"))); +} diff --git a/crates/core/src/notification_info.rs b/crates/core/src/notification_info.rs new file mode 100644 index 0000000..bf5471f --- /dev/null +++ b/crates/core/src/notification_info.rs @@ -0,0 +1,16 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NotificationInfo { + pub index: usize, + pub app_name: String, + pub title: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option<String>, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub actions: Vec<String>, +} + +#[cfg(test)] +#[path = "notification_info_tests.rs"] +mod tests; diff --git a/crates/core/src/notification_info_tests.rs b/crates/core/src/notification_info_tests.rs new file mode 100644 index 0000000..163239b --- /dev/null +++ b/crates/core/src/notification_info_tests.rs @@ -0,0 +1,29 @@ +use crate::NotificationInfo; + +#[test] +fn serialization_omits_absent_and_empty_fields() { + let info = NotificationInfo { + index: 1, + app_name: "Messages".into(), + title: "New message".into(), + body: None, + actions: vec![], + }; + let json = serde_json::to_value(&info).unwrap(); + assert!(!json.as_object().unwrap().contains_key("body")); + assert!(!json.as_object().unwrap().contains_key("actions")); +} + +#[test] +fn serialization_includes_present_fields() { + let info = NotificationInfo { + index: 2, + app_name: "Slack".into(), + title: "Channel update".into(), + body: Some("New message in #general".into()), + actions: vec!["Reply".into(), "Open".into()], + }; + let json = serde_json::to_value(&info).unwrap(); + assert_eq!(json["body"], "New message in #general"); + assert_eq!(json["actions"], serde_json::json!(["Reply", "Open"])); +} diff --git a/crates/core/src/output.rs b/crates/core/src/output.rs index 99d205a..d579160 100644 --- a/crates/core/src/output.rs +++ b/crates/core/src/output.rs @@ -1,9 +1,10 @@ use serde::Serialize; use serde_json::Value; -use crate::error::{AppError, ErrorCode}; +use crate::recovery_hint::RecoveryHint; +use crate::{AppError, DeliverySemantics, ErrorCode, RetryDisposition}; -pub const ENVELOPE_VERSION: &str = "2.0"; +pub const ENVELOPE_VERSION: &str = "2.1"; /// Structured output envelope used by the CLI and future programmatic transports. #[derive(Debug, Serialize)] @@ -24,11 +25,12 @@ pub struct ErrorPayload { #[serde(skip_serializing_if = "Option::is_none")] pub suggestion: Option<String>, #[serde(skip_serializing_if = "Option::is_none")] - pub retry_command: Option<String>, + pub recovery: Option<RecoveryHint>, #[serde(skip_serializing_if = "Option::is_none")] pub platform_detail: Option<String>, #[serde(skip_serializing_if = "Option::is_none")] pub details: Option<Value>, + pub disposition: DeliverySemantics, } impl Response { @@ -53,12 +55,29 @@ impl Response { } } -fn retry_token_for_code(code: &ErrorCode) -> Option<String> { +fn recovery_for_code(code: &ErrorCode, disposition: DeliverySemantics) -> Option<RecoveryHint> { + if disposition.retry() != RetryDisposition::Safe { + return None; + } match code { - ErrorCode::StaleRef | ErrorCode::SnapshotNotFound => { - Some("snapshot;execute_by_ref".to_owned()) - } - ErrorCode::PolicyDenied => Some("escalate_policy".to_owned()), + ErrorCode::StaleRef | ErrorCode::SnapshotNotFound => Some(RecoveryHint { + strategy: "refresh_snapshot_then_retry_original".into(), + retryable: true, + requires_fresh_snapshot: true, + retry_after_ms: None, + }), + ErrorCode::PolicyDenied => Some(RecoveryHint { + strategy: "request_explicit_policy_then_retry_original".into(), + retryable: true, + requires_fresh_snapshot: false, + retry_after_ms: None, + }), + ErrorCode::AppUnresponsive => Some(RecoveryHint { + strategy: "inspect_state_then_retry_original".into(), + retryable: true, + requires_fresh_snapshot: true, + retry_after_ms: Some(250), + }), _ => None, } } @@ -72,7 +91,8 @@ impl ErrorPayload { if let AppError::Adapter(adapter_error) = err { payload.platform_detail = adapter_error.platform_detail.clone(); payload.details = adapter_error.details.clone(); - payload.retry_command = retry_token_for_code(&adapter_error.code); + payload.disposition = adapter_error.disposition; + payload.recovery = recovery_for_code(&adapter_error.code, adapter_error.disposition); } payload } @@ -82,9 +102,10 @@ impl ErrorPayload { code: code.into(), message: message.into(), suggestion: None, - retry_command: None, + recovery: None, platform_detail: None, details: None, + disposition: DeliverySemantics::unknown(), } } @@ -93,177 +114,12 @@ impl ErrorPayload { self } - pub fn with_retry(mut self, cmd: impl Into<String>) -> Self { - self.retry_command = Some(cmd.into()); + pub fn with_recovery(mut self, recovery: RecoveryHint) -> Self { + self.recovery = Some(recovery); self } } #[cfg(test)] -mod tests { - use super::*; - use crate::error::{AdapterError, ErrorCode}; - use serde_json::json; - - #[test] - fn app_error_payload_preserves_adapter_recovery_fields() { - let err = AppError::Adapter( - AdapterError::new(ErrorCode::ActionFailed, "not actionable") - .with_suggestion("wait and retry") - .with_platform_detail("native press action failed") - .with_details(json!({ "check": "visible" })), - ); - - let payload = ErrorPayload::from_app_error(&err); - - assert_eq!(payload.code, "ACTION_FAILED"); - assert_eq!(payload.message, "not actionable"); - assert_eq!(payload.suggestion.as_deref(), Some("wait and retry")); - assert_eq!( - payload.platform_detail.as_deref(), - Some("native press action failed") - ); - assert_eq!(payload.details, Some(json!({ "check": "visible" }))); - assert_eq!( - payload.retry_command, None, - "ACTION_FAILED must not carry a retry token" - ); - } - - #[test] - fn stale_ref_payload_carries_snapshot_retry_token() { - let err = AppError::stale_ref("@e5"); - let payload = ErrorPayload::from_app_error(&err); - assert_eq!(payload.code, "STALE_REF"); - assert_eq!( - payload.retry_command.as_deref(), - Some("snapshot;execute_by_ref"), - "STALE_REF must carry the canonical retry token" - ); - } - - #[test] - fn snapshot_not_found_payload_carries_snapshot_retry_token() { - let err = AppError::Adapter(AdapterError::snapshot_not_found("snap-abc")); - let payload = ErrorPayload::from_app_error(&err); - assert_eq!(payload.code, "SNAPSHOT_NOT_FOUND"); - assert_eq!( - payload.retry_command.as_deref(), - Some("snapshot;execute_by_ref"), - "SNAPSHOT_NOT_FOUND must carry the canonical retry token" - ); - } - - #[test] - fn policy_denied_payload_carries_escalate_policy_token() { - let err = AppError::Adapter(AdapterError::policy_denied("blocked by policy")); - let payload = ErrorPayload::from_app_error(&err); - assert_eq!(payload.code, "POLICY_DENIED"); - assert_eq!( - payload.retry_command.as_deref(), - Some("escalate_policy"), - "POLICY_DENIED must carry the escalate_policy token, not a snapshot token" - ); - } - - #[test] - fn retry_command_absent_for_non_retryable_errors() { - for err in [ - AppError::Adapter(AdapterError::new(ErrorCode::InvalidArgs, "bad input")), - AppError::Adapter(AdapterError::not_supported("method_x")), - AppError::Adapter(AdapterError::new(ErrorCode::ActionFailed, "failed")), - ] { - let payload = ErrorPayload::from_app_error(&err); - assert!( - payload.retry_command.is_none(), - "non-retryable error {} must not carry a retry token", - payload.code - ); - } - } - - #[test] - fn ok_response_json_shape_has_version_ok_command_data_and_no_error_field() { - let resp = Response::ok("snapshot", json!({"app": "Finder"})); - let map: serde_json::Map<String, serde_json::Value> = - serde_json::from_value(serde_json::to_value(&resp).expect("serializable")) - .expect("map"); - - assert_eq!(map["version"].as_str(), Some("2.0"), "version must be 2.0"); - assert_eq!(map["ok"].as_bool(), Some(true), "ok must be true"); - assert_eq!( - map["command"].as_str(), - Some("snapshot"), - "command must match" - ); - assert!(map.contains_key("data"), "ok response must have data field"); - assert!( - !map.contains_key("error"), - "ok response must not serialize an error field (skip_serializing_if = is_none)" - ); - } - - #[test] - fn err_response_json_shape_has_version_ok_command_error_and_no_data_field() { - let payload = - ErrorPayload::new("STALE_REF", "ref @e1 is stale").with_suggestion("re-run snapshot"); - let resp = Response::err("click", payload); - let map: serde_json::Map<String, serde_json::Value> = - serde_json::from_value(serde_json::to_value(&resp).expect("serializable")) - .expect("map"); - - assert_eq!(map["version"].as_str(), Some("2.0"), "version must be 2.0"); - assert_eq!(map["ok"].as_bool(), Some(false), "ok must be false"); - assert_eq!(map["command"].as_str(), Some("click"), "command must match"); - assert!( - !map.contains_key("data"), - "err response must not serialize a data field (skip_serializing_if = is_none)" - ); - assert!( - map.contains_key("error"), - "err response must have error field" - ); - assert_eq!( - map["error"]["code"].as_str(), - Some("STALE_REF"), - "error code must round-trip" - ); - assert_eq!( - map["error"]["message"].as_str(), - Some("ref @e1 is stale"), - "error message must round-trip" - ); - assert_eq!( - map["error"]["suggestion"].as_str(), - Some("re-run snapshot"), - "suggestion must be present when set" - ); - } - - #[test] - fn err_response_omits_optional_error_subfields_when_absent() { - let payload = ErrorPayload::new("INTERNAL", "something broke"); - let resp = Response::err("snapshot", payload); - let map: serde_json::Map<String, serde_json::Value> = - serde_json::from_value(serde_json::to_value(&resp).expect("serializable")) - .expect("map"); - - let error = map["error"].as_object().expect("error must be an object"); - assert!( - !error.contains_key("suggestion"), - "absent suggestion must be omitted from JSON" - ); - assert!( - !error.contains_key("retry_command"), - "absent retry_command must be omitted from JSON" - ); - assert!( - !error.contains_key("platform_detail"), - "absent platform_detail must be omitted from JSON" - ); - assert!( - !error.contains_key("details"), - "absent details must be omitted from JSON" - ); - } -} +#[path = "output_tests.rs"] +mod tests; diff --git a/crates/core/src/output_tests.rs b/crates/core/src/output_tests.rs new file mode 100644 index 0000000..79f883d --- /dev/null +++ b/crates/core/src/output_tests.rs @@ -0,0 +1,194 @@ +use super::*; +use crate::{AdapterError, ErrorCode}; +use serde_json::json; + +#[test] +fn app_error_payload_preserves_adapter_recovery_fields() { + let err = AppError::Adapter( + AdapterError::new(ErrorCode::ActionFailed, "not actionable") + .with_suggestion("wait and retry") + .with_platform_detail("native press action failed") + .with_details(json!({ "check": "visible" })), + ); + + let payload = ErrorPayload::from_app_error(&err); + + assert_eq!(payload.code, "ACTION_FAILED"); + assert_eq!(payload.message, "not actionable"); + assert_eq!(payload.suggestion.as_deref(), Some("wait and retry")); + assert_eq!( + payload.platform_detail.as_deref(), + Some("native press action failed") + ); + assert_eq!(payload.details, Some(json!({ "check": "visible" }))); + assert_eq!( + payload.recovery, None, + "ACTION_FAILED must not carry a retry token" + ); +} + +#[test] +fn stale_ref_payload_carries_snapshot_recovery() { + let err = AppError::stale_ref("@e5"); + let payload = ErrorPayload::from_app_error(&err); + assert_eq!(payload.code, "STALE_REF"); + assert_eq!( + payload + .recovery + .as_ref() + .map(|recovery| recovery.strategy.as_str()), + Some("refresh_snapshot_then_retry_original") + ); + assert!( + payload + .recovery + .is_some_and(|recovery| recovery.requires_fresh_snapshot) + ); +} + +#[test] +fn snapshot_not_found_payload_carries_snapshot_recovery() { + let err = AppError::Adapter(AdapterError::snapshot_not_found("snap-abc")); + let payload = ErrorPayload::from_app_error(&err); + assert_eq!(payload.code, "SNAPSHOT_NOT_FOUND"); + assert_eq!( + payload + .recovery + .as_ref() + .map(|recovery| recovery.strategy.as_str()), + Some("refresh_snapshot_then_retry_original") + ); +} + +#[test] +fn policy_denied_payload_carries_policy_recovery() { + let err = AppError::Adapter(AdapterError::policy_denied("blocked by policy")); + let payload = ErrorPayload::from_app_error(&err); + assert_eq!(payload.code, "POLICY_DENIED"); + assert_eq!( + payload + .recovery + .as_ref() + .map(|recovery| recovery.strategy.as_str()), + Some("request_explicit_policy_then_retry_original") + ); +} + +#[test] +fn app_unresponsive_payload_carries_declarative_recovery() { + let err = AppError::Adapter(AdapterError::app_unresponsive("Finder")); + let payload = ErrorPayload::from_app_error(&err); + assert_eq!(payload.code, "APP_UNRESPONSIVE"); + assert_eq!( + payload + .recovery + .as_ref() + .map(|recovery| recovery.strategy.as_str()), + Some("inspect_state_then_retry_original") + ); + assert_eq!( + payload + .recovery + .as_ref() + .and_then(|recovery| recovery.retry_after_ms), + Some(250) + ); +} + +#[test] +fn recovery_absent_for_non_retryable_errors() { + for err in [ + AppError::Adapter(AdapterError::new(ErrorCode::InvalidArgs, "bad input")), + AppError::Adapter(AdapterError::not_supported("method_x")), + AppError::Adapter(AdapterError::new(ErrorCode::ActionFailed, "failed")), + ] { + let payload = ErrorPayload::from_app_error(&err); + assert!( + payload.recovery.is_none(), + "non-retryable error {} must not carry a retry token", + payload.code + ); + } +} + +#[test] +fn ok_response_json_shape_has_version_ok_command_data_and_no_error_field() { + let resp = Response::ok("snapshot", json!({"app": "Finder"})); + let map: serde_json::Map<String, serde_json::Value> = + serde_json::from_value(serde_json::to_value(&resp).expect("serializable")).expect("map"); + + assert_eq!(map["version"].as_str(), Some("2.1"), "version must be 2.1"); + assert_eq!(map["ok"].as_bool(), Some(true), "ok must be true"); + assert_eq!( + map["command"].as_str(), + Some("snapshot"), + "command must match" + ); + assert!(map.contains_key("data"), "ok response must have data field"); + assert!( + !map.contains_key("error"), + "ok response must not serialize an error field (skip_serializing_if = is_none)" + ); +} + +#[test] +fn err_response_json_shape_has_version_ok_command_error_and_no_data_field() { + let payload = + ErrorPayload::new("STALE_REF", "ref @e1 is stale").with_suggestion("re-run snapshot"); + let resp = Response::err("click", payload); + let map: serde_json::Map<String, serde_json::Value> = + serde_json::from_value(serde_json::to_value(&resp).expect("serializable")).expect("map"); + + assert_eq!(map["version"].as_str(), Some("2.1"), "version must be 2.1"); + assert_eq!(map["ok"].as_bool(), Some(false), "ok must be false"); + assert_eq!(map["command"].as_str(), Some("click"), "command must match"); + assert!( + !map.contains_key("data"), + "err response must not serialize a data field (skip_serializing_if = is_none)" + ); + assert!( + map.contains_key("error"), + "err response must have error field" + ); + assert_eq!( + map["error"]["code"].as_str(), + Some("STALE_REF"), + "error code must round-trip" + ); + assert_eq!( + map["error"]["message"].as_str(), + Some("ref @e1 is stale"), + "error message must round-trip" + ); + assert_eq!( + map["error"]["suggestion"].as_str(), + Some("re-run snapshot"), + "suggestion must be present when set" + ); +} + +#[test] +fn err_response_omits_optional_error_subfields_when_absent() { + let payload = ErrorPayload::new("INTERNAL", "something broke"); + let resp = Response::err("snapshot", payload); + let map: serde_json::Map<String, serde_json::Value> = + serde_json::from_value(serde_json::to_value(&resp).expect("serializable")).expect("map"); + + let error = map["error"].as_object().expect("error must be an object"); + assert!( + !error.contains_key("suggestion"), + "absent suggestion must be omitted from JSON" + ); + assert!( + !error.contains_key("recovery"), + "absent recovery must be omitted from JSON" + ); + assert!( + !error.contains_key("platform_detail"), + "absent platform_detail must be omitted from JSON" + ); + assert!( + !error.contains_key("details"), + "absent details must be omitted from JSON" + ); +} diff --git a/crates/core/src/point.rs b/crates/core/src/point.rs new file mode 100644 index 0000000..f34c38c --- /dev/null +++ b/crates/core/src/point.rs @@ -0,0 +1,24 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +impl Point { + pub fn validate(&self) -> Result<(), crate::AdapterError> { + const MAX_COORDINATE: f64 = 10_000_000.0; + if self.x.is_finite() + && self.y.is_finite() + && self.x.abs() <= MAX_COORDINATE + && self.y.abs() <= MAX_COORDINATE + { + return Ok(()); + } + Err(crate::AdapterError::new( + crate::ErrorCode::InvalidArgs, + "Point coordinates must be finite and within platform geometry bounds", + )) + } +} diff --git a/crates/core/src/private_file.rs b/crates/core/src/private_file.rs new file mode 100644 index 0000000..3b05389 --- /dev/null +++ b/crates/core/src/private_file.rs @@ -0,0 +1,354 @@ +use std::fs::File; +#[cfg(not(windows))] +use std::fs::OpenOptions; +use std::hash::{BuildHasher, RandomState}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[cfg(windows)] +#[path = "private_file_windows.rs"] +pub(crate) mod windows; + +pub(crate) fn open_private_lock(path: &Path, create: bool) -> std::io::Result<File> { + #[cfg(windows)] + { + windows::open_lock(path, create) + } + #[cfg(not(windows))] + { + let parent = path + .parent() + .ok_or_else(|| invalid_input("private file path has no parent"))?; + crate::private_file_parent::ensure_private(parent)?; + let mut options = OpenOptions::new(); + options.read(true).write(true).create(create); + configure_unix(&mut options, 0o600); + let file = options.open(path)?; + validate_private_regular(&file)?; + Ok(file) + } +} + +pub(crate) fn open_private_append(path: &Path) -> std::io::Result<File> { + #[cfg(windows)] + { + windows::open_append(path) + } + #[cfg(not(windows))] + { + let mut options = OpenOptions::new(); + options.create(true).append(true); + configure_unix(&mut options, 0o600); + let file = options.open(path)?; + validate_private_regular(&file)?; + Ok(file) + } +} + +pub(crate) fn read_private_bounded(path: &Path, max_bytes: u64) -> std::io::Result<Vec<u8>> { + let file = open_private_read(path)?; + read_bounded(file, max_bytes) +} + +pub(crate) fn read_regular_bounded(path: &Path, max_bytes: u64) -> std::io::Result<Vec<u8>> { + #[cfg(windows)] + let file = windows::open_regular_read(path)?; + #[cfg(not(windows))] + let file = { + let mut options = OpenOptions::new(); + options.read(true); + configure_unix(&mut options, 0); + options.open(path)? + }; + validate_regular(&file)?; + validate_local_filesystem(&file)?; + read_bounded(file, max_bytes) +} + +pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + write_atomic_with( + path, + bytes, + crate::private_file_parent::ensure_private, + sync_directory, + validate_private_destination, + ) +} + +pub(crate) fn write_user_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + write_atomic_with( + path, + bytes, + crate::private_file_parent::ensure_user, + sync_user_directory, + validate_user_destination, + ) +} + +fn write_atomic_with( + path: &Path, + bytes: &[u8], + ensure_parent: fn(&Path) -> std::io::Result<()>, + sync_parent: fn(&Path) -> std::io::Result<()>, + validate_destination: fn(&Path) -> std::io::Result<()>, +) -> std::io::Result<()> { + let parent = path + .parent() + .ok_or_else(|| invalid_input("private file path has no parent"))?; + ensure_parent(parent)?; + validate_destination(path)?; + let (temporary, mut file) = create_temporary(path)?; + let result = (|| { + file.write_all(bytes)?; + file.sync_all()?; + #[cfg(test)] + crash_before_rename_if_requested(path); + replace_atomic(&file, &temporary, path)?; + validate_private_regular(&open_private_read(path)?)?; + sync_parent(parent) + })(); + drop(file); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result +} + +fn validate_private_destination(path: &Path) -> std::io::Result<()> { + match open_private_read(path) { + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn validate_user_destination(path: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + Err(invalid_input("user output path is a symlink")) + } + Ok(metadata) if !metadata.is_file() => { + Err(invalid_input("user output path is not a regular file")) + } + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +pub(crate) fn validate_private_regular(file: &File) -> std::io::Result<std::fs::Metadata> { + let metadata = validate_regular(file)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(permission_denied( + "private file is not owned by the effective user", + )); + } + if metadata.mode() & 0o077 != 0 { + return Err(permission_denied( + "private file is accessible by group or other users", + )); + } + if metadata.nlink() != 1 { + return Err(permission_denied("private file must not be hard-linked")); + } + } + #[cfg(windows)] + windows::validate_private(file)?; + Ok(metadata) +} + +pub(crate) fn validate_regular(file: &File) -> std::io::Result<std::fs::Metadata> { + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(permission_denied("path is not a regular file")); + } + #[cfg(windows)] + windows::validate_regular(file)?; + Ok(metadata) +} + +fn open_private_read(path: &Path) -> std::io::Result<File> { + #[cfg(windows)] + { + windows::open_read(path) + } + #[cfg(not(windows))] + { + let mut options = OpenOptions::new(); + options.read(true); + configure_unix(&mut options, 0); + let file = options.open(path)?; + validate_private_regular(&file)?; + Ok(file) + } +} + +fn read_bounded(file: File, max_bytes: u64) -> std::io::Result<Vec<u8>> { + let metadata = file.metadata()?; + if metadata.len() > max_bytes { + return Err(invalid_input("file exceeds its read limit")); + } + let capacity = usize::try_from(metadata.len().min(max_bytes)).unwrap_or(usize::MAX); + let mut bytes = Vec::with_capacity(capacity); + file.take(max_bytes.saturating_add(1)) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > max_bytes { + return Err(invalid_input("file grew beyond its read limit")); + } + Ok(bytes) +} + +fn create_temporary(path: &Path) -> std::io::Result<(PathBuf, File)> { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| invalid_input("private file path has an invalid filename"))?; + for _ in 0..32 { + let nonce = RandomState::new().hash_one(( + std::process::id(), + TEMP_COUNTER.fetch_add(1, Ordering::Relaxed), + std::time::SystemTime::now(), + )); + let temporary = path.with_file_name(format!(".{file_name}.{nonce:016x}.tmp")); + #[cfg(windows)] + match windows::create_new(&temporary) { + Ok(file) => return Ok((temporary, file)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + #[cfg(not(windows))] + { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + configure_unix(&mut options, 0o600); + match options.open(&temporary) { + Ok(file) => { + validate_private_regular(&file)?; + return Ok((temporary, file)); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not allocate a private temporary file", + )) +} + +#[cfg(windows)] +fn sync_directory(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +#[cfg(not(windows))] +fn sync_directory(path: &Path) -> std::io::Result<()> { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + } + options.open(path)?.sync_all() +} + +#[cfg(windows)] +fn sync_user_directory(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +#[cfg(not(windows))] +fn sync_user_directory(path: &Path) -> std::io::Result<()> { + OpenOptions::new().read(true).open(path)?.sync_all() +} + +#[cfg(windows)] +fn replace_atomic(file: &File, source: &Path, destination: &Path) -> std::io::Result<()> { + windows::replace_atomic(file, source, destination) +} + +#[cfg(not(windows))] +fn replace_atomic(_file: &File, source: &Path, destination: &Path) -> std::io::Result<()> { + std::fs::rename(source, destination) +} + +#[cfg(unix)] +fn configure_unix(options: &mut OpenOptions, mode: u32) { + use std::os::unix::fs::OpenOptionsExt; + options + .mode(mode) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK); +} + +#[cfg(not(any(unix, windows)))] +fn configure_unix(_options: &mut OpenOptions, _mode: u32) {} + +#[cfg(target_os = "macos")] +fn validate_local_filesystem(file: &File) -> std::io::Result<()> { + use std::os::fd::AsRawFd; + let mut stats = std::mem::MaybeUninit::<libc::statfs>::uninit(); + if unsafe { libc::fstatfs(file.as_raw_fd(), stats.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { stats.assume_init() }.f_flags & libc::MNT_LOCAL as u32 == 0 { + return Err(permission_denied( + "network filesystems are not accepted here", + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn validate_local_filesystem(file: &File) -> std::io::Result<()> { + use std::os::fd::AsRawFd; + let mut stats = std::mem::MaybeUninit::<libc::statfs>::uninit(); + if unsafe { libc::fstatfs(file.as_raw_fd(), stats.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let fs_type = unsafe { stats.assume_init() }.f_type as u64; + if matches!(fs_type, 0x6969 | 0x517b | 0xff53_4d42 | 0x6573_5546) { + return Err(permission_denied( + "network filesystems are not accepted here", + )); + } + Ok(()) +} + +#[cfg(windows)] +fn validate_local_filesystem(file: &File) -> std::io::Result<()> { + windows::validate_local(file) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux", windows)))] +fn validate_local_filesystem(_file: &File) -> std::io::Result<()> { + Ok(()) +} + +#[cfg(test)] +fn crash_before_rename_if_requested(path: &Path) { + if std::env::var_os("AGENT_DESKTOP_TEST_CRASH_BEFORE_RENAME") + .is_some_and(|value| Path::new(&value) == path) + { + std::process::abort(); + } +} + +pub(super) fn invalid_input(message: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, message) +} + +pub(super) fn permission_denied(message: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::PermissionDenied, message) +} + +#[cfg(test)] +#[path = "private_file_tests.rs"] +mod tests; diff --git a/crates/core/src/private_file_parent.rs b/crates/core/src/private_file_parent.rs new file mode 100644 index 0000000..257cceb --- /dev/null +++ b/crates/core/src/private_file_parent.rs @@ -0,0 +1,102 @@ +use std::path::Path; + +pub(super) fn ensure_private(path: &Path) -> std::io::Result<()> { + #[cfg(windows)] + { + super::private_file::windows::ensure_private_parent(path) + } + #[cfg(not(windows))] + { + ensure_directory_path(path)?; + let metadata = std::fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(super::private_file::permission_denied( + "private file parent must be a real directory", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(super::private_file::permission_denied( + "private file parent is not owned by the effective user", + )); + } + if metadata.mode() & 0o077 != 0 { + return Err(super::private_file::permission_denied( + "private file parent is accessible by group or other users", + )); + } + } + Ok(()) + } +} + +pub(super) fn ensure_user(path: &Path) -> std::io::Result<()> { + #[cfg(windows)] + { + super::private_file::windows::ensure_user_parent(path) + } + #[cfg(not(windows))] + { + ensure_directory_path(path)?; + let metadata = std::fs::metadata(path)?; + if !metadata.is_dir() { + return Err(super::private_file::permission_denied( + "user output parent must be a directory", + )); + } + Ok(()) + } +} + +#[cfg(unix)] +fn ensure_directory_path(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::{DirBuilderExt, MetadataExt}; + + let mut current = std::path::PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::CurDir => continue, + std::path::Component::ParentDir => { + return Err(super::private_file::invalid_input( + "private file parent must not contain parent traversal", + )); + } + _ => current.push(component.as_os_str()), + } + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() && metadata.uid() != 0 => { + return Err(super::private_file::permission_denied( + "private file parent path must not contain user-controlled symlinks", + )); + } + Ok(metadata) if !metadata.file_type().is_symlink() && !metadata.is_dir() => { + return Err(super::private_file::permission_denied( + "private file parent path must contain only directories", + )); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + match std::fs::DirBuilder::new().mode(0o700).create(¤t) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + let metadata = std::fs::symlink_metadata(¤t)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(super::private_file::permission_denied( + "private file parent creation raced with a non-directory path", + )); + } + } + Err(error) => return Err(error), + } + } + Ok(()) +} + +#[cfg(not(any(unix, windows)))] +fn ensure_directory_path(path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path) +} diff --git a/crates/core/src/private_file_tests.rs b/crates/core/src/private_file_tests.rs new file mode 100644 index 0000000..5791267 --- /dev/null +++ b/crates/core/src/private_file_tests.rs @@ -0,0 +1,188 @@ +#![cfg(unix)] + +use super::*; +use std::ffi::CString; +use std::os::fd::AsRawFd; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::fs::PermissionsExt; + +fn directory(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "agent-desktop-private-{label}-{}", + crate::refs::new_snapshot_id() + )); + std::fs::create_dir_all(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + path +} + +#[test] +fn private_open_sets_nonblocking_and_close_on_exec() { + let directory = directory("flags"); + let path = directory.join("lock"); + let file = open_private_lock(&path, true).unwrap(); + + let descriptor_flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFD) }; + let status_flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFL) }; + assert_ne!(descriptor_flags & libc::FD_CLOEXEC, 0); + assert_ne!(status_flags & libc::O_NONBLOCK, 0); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn private_open_rejects_symlink_fifo_device_and_hardlink() { + let directory = directory("special"); + let target = directory.join("target"); + let target_file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&target) + .unwrap(); + drop(target_file); + let symlink = directory.join("symlink"); + std::os::unix::fs::symlink(&target, &symlink).unwrap(); + assert!(open_private_lock(&symlink, false).is_err()); + + let fifo = directory.join("fifo"); + let fifo_path = CString::new(fifo.to_string_lossy().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(fifo_path.as_ptr(), 0o600) }, 0); + let started = std::time::Instant::now(); + assert!(open_private_lock(&fifo, false).is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(1)); + + let hardlink = directory.join("hardlink"); + std::fs::hard_link(&target, &hardlink).unwrap(); + assert!(open_private_lock(&hardlink, false).is_err()); + assert!(open_private_lock(Path::new("/dev/null"), false).is_err()); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn private_read_enforces_the_bound_before_allocating() { + let directory = directory("bound"); + let path = directory.join("data"); + write_atomic(&path, b"12345").unwrap(); + + let error = read_private_bounded(&path, 4).unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn private_writes_and_locks_reject_group_accessible_parent() { + let directory = directory("hostile-parent"); + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o770)).unwrap(); + let path = directory.join("data"); + + assert_eq!( + write_atomic(&path, b"private").unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + assert_eq!( + open_private_lock(&path, true).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn private_write_rejects_an_intermediate_directory_symlink() { + let directory = directory("intermediate-symlink"); + let outside = directory.join("outside"); + let nested = outside.join("nested"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::set_permissions(&outside, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&nested, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = directory.join("redirect"); + std::os::unix::fs::symlink(&outside, &link).unwrap(); + + let error = write_atomic(&link.join("nested/artifact"), b"private").unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert!(!nested.join("artifact").exists()); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn user_write_overwrites_an_existing_group_readable_file() { + let directory = directory("user-overwrite"); + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let path = directory.join("out.png"); + std::fs::write(&path, b"old").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + write_user_atomic(&path, b"new").unwrap(); + + assert_eq!(std::fs::read(&path).unwrap(), b"new"); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn user_write_refuses_symlink_and_directory_destinations() { + let directory = directory("user-refuse"); + let target = directory.join("target"); + std::fs::write(&target, b"kept").unwrap(); + let symlink = directory.join("symlink"); + std::os::unix::fs::symlink(&target, &symlink).unwrap(); + let subdirectory = directory.join("subdirectory"); + std::fs::create_dir(&subdirectory).unwrap(); + + assert_eq!( + write_user_atomic(&symlink, b"new").unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + assert_eq!( + write_user_atomic(&subdirectory, b"new").unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + assert_eq!(std::fs::read(&target).unwrap(), b"kept"); + + let error = crate::refs::write_user_file(&symlink, b"new").unwrap_err(); + assert_eq!(error.code(), "INVALID_ARGS"); + assert!( + error + .suggestion() + .unwrap() + .contains(&symlink.display().to_string()) + ); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn private_write_still_rejects_a_group_readable_destination() { + let directory = directory("private-loose-destination"); + let path = directory.join("data"); + std::fs::write(&path, b"old").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + let error = write_atomic(&path, b"new").unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert_eq!(std::fs::read(&path).unwrap(), b"old"); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn user_write_allows_the_system_temporary_directory() { + let path = Path::new("/tmp").join(format!( + "agent-desktop-user-output-{}", + crate::refs::new_snapshot_id() + )); + + write_user_atomic(&path, b"private").unwrap(); + + assert_eq!(std::fs::read(&path).unwrap(), b"private"); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::remove_file(path).unwrap(); +} diff --git a/crates/core/src/private_file_windows.rs b/crates/core/src/private_file_windows.rs new file mode 100644 index 0000000..4f30433 --- /dev/null +++ b/crates/core/src/private_file_windows.rs @@ -0,0 +1,299 @@ +use std::fs::File; +use std::mem::size_of; +use std::os::windows::io::AsRawHandle; +use std::path::Path; +use std::ptr::null; +use windows_sys::Win32::Foundation::{ + ERROR_ALREADY_EXISTS, ERROR_INVALID_PARAMETER, GENERIC_READ, GENERIC_WRITE, GetLastError, + HANDLE, +}; +use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, CREATE_NEW, CreateDirectoryW, DELETE, FILE_APPEND_DATA, + FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_REMOTE_PROTOCOL_INFO, + FILE_SHARE_READ, FILE_SHARE_WRITE, FileRemoteProtocolInfo, GetFileInformationByHandle, + GetFileInformationByHandleEx, OPEN_ALWAYS, OPEN_EXISTING, READ_CONTROL, +}; + +#[path = "private_file_windows_guard.rs"] +mod guard; +#[path = "private_file_windows_open.rs"] +mod open; +#[path = "private_file_windows_path.rs"] +mod path; +#[path = "private_file_windows_rename.rs"] +mod rename; +#[path = "private_file_windows_security.rs"] +mod security; + +use open::FileOpen; +use security::PrivateSecurity; + +const LEAF_SHARING: u32 = FILE_SHARE_READ | FILE_SHARE_WRITE; +const GUARD_SHARING: u32 = FILE_SHARE_READ; +const TEMP_ACCESS: u32 = GENERIC_WRITE | READ_CONTROL | DELETE; + +pub(super) fn open_lock(path: &Path, create: bool) -> std::io::Result<File> { + let creation = if create { OPEN_ALWAYS } else { OPEN_EXISTING }; + open_private( + path, + GENERIC_READ | GENERIC_WRITE | READ_CONTROL, + creation, + true, + ) +} + +pub(super) fn open_append(path: &Path) -> std::io::Result<File> { + open_private(path, FILE_APPEND_DATA | READ_CONTROL, OPEN_ALWAYS, false) +} + +pub(super) fn open_read(path: &Path) -> std::io::Result<File> { + open_private(path, GENERIC_READ | READ_CONTROL, OPEN_EXISTING, false) +} + +pub(super) fn open_regular_read(path: &Path) -> std::io::Result<File> { + let path = path::normalized(path)?; + path::validate_file_name(&path)?; + with_leaf_parent(&path, false, None, || { + FileOpen::leaf(&path, GENERIC_READ, OPEN_EXISTING, null()).execute() + }) +} + +pub(super) fn create_new(path: &Path) -> std::io::Result<File> { + open_private(path, TEMP_ACCESS, CREATE_NEW, false) +} + +pub(crate) fn ensure_private_parent(path: &Path) -> std::io::Result<()> { + let path = path::normalized(path)?; + let security = PrivateSecurity::new_directory()?; + guard::with_ancestor_guards(&path, true, Some(&security), |guards| { + security::validate_private_acl(guards.leaf()?) + }) +} + +pub(crate) fn ensure_user_parent(path: &Path) -> std::io::Result<()> { + let path = path::normalized(path)?; + let security = PrivateSecurity::new_directory()?; + guard::with_ancestor_guards(&path, true, Some(&security), |_| Ok(())) +} + +pub(super) fn validate_private(file: &File) -> std::io::Result<()> { + let info = file_information(file)?; + if info.dwFileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT) != 0 { + return Err(super::permission_denied( + "private path must be a regular non-reparse file", + )); + } + if info.nNumberOfLinks != 1 { + return Err(super::permission_denied( + "private file must not be hard-linked", + )); + } + validate_local(file)?; + security::validate_private_acl(file) +} + +pub(super) fn validate_regular(file: &File) -> std::io::Result<()> { + let info = file_information(file)?; + if info.dwFileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT) != 0 { + return Err(super::permission_denied( + "path must be a regular non-reparse file", + )); + } + validate_local(file) +} + +pub(super) fn validate_local(file: &File) -> std::io::Result<()> { + let mut remote = remote_protocol_query(); + let success = unsafe { + GetFileInformationByHandleEx( + raw_handle(file), + FileRemoteProtocolInfo, + (&raw mut remote).cast(), + size_of::<FILE_REMOTE_PROTOCOL_INFO>() as u32, + ) + }; + let error = if success == 0 { + unsafe { GetLastError() } + } else { + 0 + }; + classify_locality(success != 0, error) +} + +fn classify_locality(success: bool, error: u32) -> std::io::Result<()> { + if success { + return Err(super::permission_denied( + "network filesystems are not accepted here", + )); + } + if error == ERROR_INVALID_PARAMETER { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "cannot verify that the Windows storage is local", + )) + } +} + +pub(super) fn replace_atomic( + source_file: &File, + source: &Path, + destination: &Path, +) -> std::io::Result<()> { + let source = path::normalized(source)?; + let destination = path::normalized(destination)?; + path::validate_file_name(&source)?; + path::validate_file_name(&destination)?; + let source_parent = parent(&source)?; + let destination_parent = parent(&destination)?; + if source_parent == destination_parent { + return guard::with_ancestor_guards(source_parent, false, None, |_guards| { + rename::replace(source_file, &destination) + }); + } + guard::with_ancestor_guards(source_parent, false, None, |_source_guards| { + guard::with_ancestor_guards(destination_parent, false, None, |_destination_guards| { + rename::replace(source_file, &destination) + }) + }) +} + +fn open_private( + path: &Path, + access: u32, + creation: u32, + create_parent: bool, +) -> std::io::Result<File> { + let path = path::normalized(path)?; + path::validate_file_name(&path)?; + let (directory_security, file_security) = if create_parent { + let (directory, file) = PrivateSecurity::new_pair()?; + (Some(directory), file) + } else { + (None, PrivateSecurity::new_file()?) + }; + with_leaf_parent(&path, create_parent, directory_security.as_ref(), || { + let file = FileOpen::leaf(&path, access, creation, file_security.attributes()).execute()?; + validate_private(&file)?; + Ok(file) + }) +} + +fn with_leaf_parent<T>( + path: &Path, + create_parent: bool, + security: Option<&PrivateSecurity>, + operation: impl FnOnce() -> std::io::Result<T>, +) -> std::io::Result<T> { + guard::with_ancestor_guards(parent(path)?, create_parent, security, |_guards| { + operation() + }) +} + +fn create_private_directory(path: &Path, security: &PrivateSecurity) -> std::io::Result<()> { + let path = path::wide_normalized(path)?; + if unsafe { CreateDirectoryW(path.as_ptr(), security.attributes()) } != 0 { + return Ok(()); + } + let code = unsafe { GetLastError() }; + if code == ERROR_ALREADY_EXISTS { + Ok(()) + } else { + Err(std::io::Error::from_raw_os_error(code as i32)) + } +} + +fn open_guarded_directory(path: &Path) -> std::io::Result<File> { + FileOpen::guarded_directory(path, GENERIC_READ | READ_CONTROL, OPEN_EXISTING).execute() +} + +fn validate_directory(file: &File) -> std::io::Result<()> { + let info = file_information(file)?; + if info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY == 0 + || info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + { + return Err(super::permission_denied( + "private file parent must be a real non-reparse directory", + )); + } + validate_local(file) +} + +fn file_information(file: &File) -> std::io::Result<BY_HANDLE_FILE_INFORMATION> { + let mut info = BY_HANDLE_FILE_INFORMATION::default(); + if unsafe { GetFileInformationByHandle(raw_handle(file), &raw mut info) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(info) + } +} + +fn raw_handle(file: &File) -> HANDLE { + file.as_raw_handle() +} + +fn parent(path: &Path) -> std::io::Result<&Path> { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or_else(|| invalid_input("private file path has no parent")) +} + +fn remote_protocol_query() -> FILE_REMOTE_PROTOCOL_INFO { + FILE_REMOTE_PROTOCOL_INFO { + StructureVersion: 2, + StructureSize: size_of::<FILE_REMOTE_PROTOCOL_INFO>() as u16, + ..Default::default() + } +} + +fn invalid_input(message: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidInput, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ancestor_guards_exclude_delete_sharing() { + assert_ne!( + windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OPEN_REPARSE_POINT, + 0 + ); + assert_eq!(LEAF_SHARING, FILE_SHARE_READ | FILE_SHARE_WRITE); + assert_eq!( + LEAF_SHARING & windows_sys::Win32::Storage::FileSystem::FILE_SHARE_DELETE, + 0 + ); + assert_eq!(GUARD_SHARING, FILE_SHARE_READ); + assert_eq!(GUARD_SHARING & FILE_SHARE_WRITE, 0); + assert_eq!( + GUARD_SHARING & windows_sys::Win32::Storage::FileSystem::FILE_SHARE_DELETE, + 0 + ); + assert_ne!(TEMP_ACCESS & DELETE, 0); + } + + #[test] + fn remote_protocol_query_uses_the_current_contract_shape() { + let query = remote_protocol_query(); + + assert_eq!(query.StructureVersion, 2); + assert_eq!( + usize::from(query.StructureSize), + size_of::<FILE_REMOTE_PROTOCOL_INFO>() + ); + assert_eq!(query.Protocol, 0); + assert_eq!(ERROR_INVALID_PARAMETER, 87); + assert!(classify_locality(false, ERROR_INVALID_PARAMETER).is_ok()); + assert_eq!( + classify_locality(true, 0).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + assert_eq!( + classify_locality(false, 5).unwrap_err().kind(), + std::io::ErrorKind::Unsupported + ); + } +} diff --git a/crates/core/src/private_file_windows_guard.rs b/crates/core/src/private_file_windows_guard.rs new file mode 100644 index 0000000..a1f6743 --- /dev/null +++ b/crates/core/src/private_file_windows_guard.rs @@ -0,0 +1,84 @@ +use std::fs::File; +use std::path::{Component, Path, PathBuf}; + +use super::security::PrivateSecurity; + +pub(super) struct AncestorGuards { + handles: Vec<File>, +} + +impl AncestorGuards { + fn acquire( + path: &Path, + create: bool, + security: Option<&PrivateSecurity>, + ) -> std::io::Result<Self> { + let absolute = super::path::normalized(path)?; + let mut current = PathBuf::new(); + let mut handles = Vec::new(); + for component in absolute.components() { + current.push(component); + if matches!(component, Component::Prefix(_)) { + continue; + } + let handle = match super::open_guarded_directory(¤t) { + Ok(handle) => handle, + Err(error) if create && error.kind() == std::io::ErrorKind::NotFound => { + let security = security.ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::Unsupported, + "secure Windows directory creation requires a private descriptor", + ) + })?; + super::create_private_directory(¤t, security)?; + super::open_guarded_directory(¤t)? + } + Err(error) => return Err(error), + }; + super::validate_directory(&handle)?; + handles.push(handle); + } + if handles.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "Windows private path has no guardable directory", + )); + } + Ok(Self { handles }) + } + + pub(super) fn leaf(&self) -> std::io::Result<&File> { + self.handles.last().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "Windows private path lost its ancestor guards", + ) + }) + } +} + +pub(super) fn with_ancestor_guards<T>( + path: &Path, + create: bool, + security: Option<&PrivateSecurity>, + operation: impl FnOnce(&AncestorGuards) -> std::io::Result<T>, +) -> std::io::Result<T> { + let guards = AncestorGuards::acquire(path, create, security)?; + operation(&guards) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lifetime_contract(guards: &AncestorGuards) -> std::io::Result<&File> { + guards.leaf() + } + + #[test] + fn operation_borrows_the_live_guard_chain() { + let contract: for<'a> fn(&'a AncestorGuards) -> std::io::Result<&'a File> = + lifetime_contract; + let _ = contract; + } +} diff --git a/crates/core/src/private_file_windows_open.rs b/crates/core/src/private_file_windows_open.rs new file mode 100644 index 0000000..a10de35 --- /dev/null +++ b/crates/core/src/private_file_windows_open.rs @@ -0,0 +1,77 @@ +use std::fs::File; +use std::os::windows::io::FromRawHandle; +use std::path::Path; +use std::ptr::null_mut; +use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE}; +use windows_sys::Win32::Security::SECURITY_ATTRIBUTES; +use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, +}; + +pub(super) struct FileOpen<'a> { + path: &'a Path, + access: u32, + creation: u32, + directory: bool, + sharing: u32, + security: *const SECURITY_ATTRIBUTES, +} + +impl<'a> FileOpen<'a> { + pub(super) fn leaf( + path: &'a Path, + access: u32, + creation: u32, + security: *const SECURITY_ATTRIBUTES, + ) -> Self { + Self { + path, + access, + creation, + directory: false, + sharing: super::LEAF_SHARING, + security, + } + } + + pub(super) fn guarded_directory(path: &'a Path, access: u32, creation: u32) -> Self { + Self { + path, + access, + creation, + directory: true, + sharing: super::GUARD_SHARING, + security: std::ptr::null(), + } + } + + pub(super) fn execute(self) -> std::io::Result<File> { + let path = super::path::wide_normalized(self.path)?; + let flags = FILE_FLAG_OPEN_REPARSE_POINT + | if self.directory { + FILE_FLAG_BACKUP_SEMANTICS + } else { + FILE_ATTRIBUTE_NORMAL + }; + let handle = unsafe { + CreateFileW( + path.as_ptr(), + self.access, + self.sharing, + self.security, + self.creation, + flags, + null_mut(), + ) + }; + file_from_handle(handle) + } +} + +fn file_from_handle(handle: HANDLE) -> std::io::Result<File> { + if handle == INVALID_HANDLE_VALUE { + Err(std::io::Error::last_os_error()) + } else { + Ok(unsafe { File::from_raw_handle(handle) }) + } +} diff --git a/crates/core/src/private_file_windows_path.rs b/crates/core/src/private_file_windows_path.rs new file mode 100644 index 0000000..41fb758 --- /dev/null +++ b/crates/core/src/private_file_windows_path.rs @@ -0,0 +1,247 @@ +use std::os::windows::ffi::OsStrExt; +use std::path::{Component, Path, PathBuf, Prefix}; + +const VERBATIM: &[u16] = &[b'\\' as u16, b'\\' as u16, b'?' as u16, b'\\' as u16]; +const VERBATIM_UNC: &[u16] = &[ + b'\\' as u16, + b'\\' as u16, + b'?' as u16, + b'\\' as u16, + b'U' as u16, + b'N' as u16, + b'C' as u16, + b'\\' as u16, +]; + +pub(super) fn normalized(path: &Path) -> std::io::Result<PathBuf> { + validate_path_nul(path)?; + validate_components(path)?; + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + if path.has_root() || matches!(path.components().next(), Some(Component::Prefix(_))) { + return Err(invalid_input( + "drive-relative and root-relative Windows paths are not accepted here", + )); + } + std::env::current_dir()?.join(path) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + return Err(invalid_input( + "Windows private paths must not contain parent traversal", + )); + } + _ => normalized.push(component), + } + } + if !normalized.is_absolute() { + return Err(invalid_input( + "Windows path did not normalize to an absolute path", + )); + } + validate_components(&normalized)?; + Ok(normalized) +} + +pub(super) fn wide_normalized(path: &Path) -> std::io::Result<Vec<u16>> { + validate_path_nul(path)?; + validate_components(path)?; + let absolute: Vec<u16> = path.as_os_str().encode_wide().collect(); + let mut verbatim = if absolute.starts_with(VERBATIM) { + absolute + } else if absolute.starts_with(&[b'\\' as u16, b'\\' as u16]) { + let mut path = Vec::with_capacity(VERBATIM_UNC.len() + absolute.len() - 2 + 1); + path.extend_from_slice(VERBATIM_UNC); + path.extend_from_slice(&absolute[2..]); + path + } else if is_drive_absolute(&absolute) { + let mut path = Vec::with_capacity(VERBATIM.len() + absolute.len() + 1); + path.extend_from_slice(VERBATIM); + path.extend_from_slice(&absolute); + path + } else { + return Err(invalid_input( + "Windows path did not normalize to an absolute path", + )); + }; + if verbatim.len() + 1 > 32_767 { + return Err(invalid_input("Windows verbatim path exceeds 32,767 units")); + } + verbatim.push(0); + Ok(verbatim) +} + +pub(super) fn validate_file_name(path: &Path) -> std::io::Result<()> { + let file_name = path + .file_name() + .ok_or_else(|| invalid_input("private path has no filename"))?; + validate_component(file_name) +} + +fn validate_components(path: &Path) -> std::io::Result<()> { + for component in path.components() { + match component { + Component::Prefix(prefix) => validate_prefix(prefix.kind())?, + Component::Normal(component) => validate_component(component)?, + Component::ParentDir => { + return Err(invalid_input( + "Windows private paths must not contain parent traversal", + )); + } + Component::RootDir | Component::CurDir => {} + } + } + Ok(()) +} + +fn validate_prefix(prefix: Prefix<'_>) -> std::io::Result<()> { + match prefix { + Prefix::Disk(_) + | Prefix::UNC(_, _) + | Prefix::VerbatimDisk(_) + | Prefix::VerbatimUNC(_, _) => Ok(()), + Prefix::DeviceNS(_) | Prefix::Verbatim(_) => Err(invalid_input( + "Windows device namespace paths are not accepted here", + )), + } +} + +fn validate_component(component: &std::ffi::OsStr) -> std::io::Result<()> { + let wide: Vec<u16> = component.encode_wide().collect(); + if wide.contains(&0) { + return Err(invalid_input("Windows paths must not contain NUL")); + } + if wide.contains(&(b':' as u16)) { + return Err(invalid_input( + "Windows alternate data streams are not accepted here", + )); + } + if matches!(wide.last(), Some(last) if *last == b'.' as u16 || *last == b' ' as u16) { + return Err(invalid_input( + "Windows private path components must not end in a dot or space", + )); + } + if is_dos_reserved(&wide) { + return Err(invalid_input( + "Windows DOS device basenames are not accepted here", + )); + } + Ok(()) +} + +fn is_dos_reserved(component: &[u16]) -> bool { + let base_end = component + .iter() + .position(|unit| *unit == b'.' as u16) + .unwrap_or(component.len()); + let mut base = &component[..base_end]; + while matches!(base.last(), Some(last) if *last == b' ' as u16 || *last == b'.' as u16) { + base = &base[..base.len() - 1]; + } + ascii_eq(base, b"NUL") + || ascii_eq(base, b"CON") + || ascii_eq(base, b"AUX") + || ascii_eq(base, b"PRN") + || ascii_eq(base, b"CLOCK$") + || numbered_device(base, b"COM") + || numbered_device(base, b"LPT") +} + +fn numbered_device(base: &[u16], prefix: &[u8; 3]) -> bool { + base.len() == 4 + && ascii_eq(&base[..3], prefix) + && matches!(base[3], unit if (b'1' as u16..=b'9' as u16).contains(&unit) || matches!(unit, 0x00b9 | 0x00b2 | 0x00b3)) +} + +fn ascii_eq(value: &[u16], expected: &[u8]) -> bool { + value.len() == expected.len() + && value.iter().zip(expected).all(|(actual, expected)| { + u8::try_from(*actual).is_ok_and(|actual| actual.to_ascii_uppercase() == *expected) + }) +} + +fn validate_path_nul(path: &Path) -> std::io::Result<()> { + if path.as_os_str().encode_wide().any(|unit| unit == 0) { + return Err(invalid_input("Windows paths must not contain NUL")); + } + Ok(()) +} + +fn is_drive_absolute(path: &[u16]) -> bool { + matches!(path, [drive, colon, slash, ..] if *drive != b'\\' as u16 && *colon == b':' as u16 && *slash == b'\\' as u16) +} + +fn invalid_input(message: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidInput, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_names_reject_windows_aliasing_syntax() { + assert!(validate_file_name(Path::new("state:stream")).is_err()); + assert!(validate_file_name(Path::new("state.")).is_err()); + assert!(validate_file_name(Path::new("state ")).is_err()); + assert!(validate_file_name(Path::new("state.json")).is_ok()); + } + + #[test] + fn dos_device_names_are_rejected_case_insensitively_with_extensions() { + for name in [ + "NUL", + "con.txt", + "AUX.json", + "prn", + "CLOCK$", + "COM1.log", + "com9", + "LPT1.txt", + "lpt9", + "COM¹.txt", + "LPT²", + "lpt³.json", + ] { + assert!(validate_file_name(Path::new(name)).is_err(), "{name}"); + } + for name in ["COM0", "COM10", "LPT0", "LPT10", "console", "auxiliary"] { + assert!(validate_file_name(Path::new(name)).is_ok(), "{name}"); + } + } + + #[test] + fn drive_absolute_detection_requires_a_root_separator() { + assert!(is_drive_absolute(&[ + b'C' as u16, + b':' as u16, + b'\\' as u16, + b'x' as u16 + ])); + assert!(!is_drive_absolute(&[b'C' as u16, b':' as u16, b'x' as u16])); + } + + #[test] + fn drive_and_unc_paths_receive_verbatim_prefixes() { + let drive = wide_normalized(Path::new(r"C:\state\refs.json")).unwrap(); + let unc = wide_normalized(Path::new(r"\\server\share\refs.json")).unwrap(); + + assert!(drive.starts_with(VERBATIM)); + assert!(unc.starts_with(VERBATIM_UNC)); + assert_eq!(drive.last(), Some(&0)); + assert_eq!(unc.last(), Some(&0)); + } + + #[test] + fn long_drive_path_is_encoded_without_legacy_max_path_truncation() { + let path = format!(r"C:\state\{}\refs.json", "a".repeat(300)); + let encoded = wide_normalized(Path::new(&path)).unwrap(); + + assert!(encoded.starts_with(VERBATIM)); + assert!(encoded.len() > 260); + } +} diff --git a/crates/core/src/private_file_windows_rename.rs b/crates/core/src/private_file_windows_rename.rs new file mode 100644 index 0000000..3d8769b --- /dev/null +++ b/crates/core/src/private_file_windows_rename.rs @@ -0,0 +1,114 @@ +use std::fs::File; +use std::mem::{offset_of, size_of}; +use std::os::windows::io::AsRawHandle; +use std::path::Path; +use std::ptr::null_mut; +use windows_sys::Win32::Storage::FileSystem::{ + FILE_RENAME_INFO, FileRenameInfo, SetFileInformationByHandle, +}; + +pub(super) struct RenameBuffer { + words: Vec<usize>, + byte_len: u32, +} + +impl RenameBuffer { + fn new(destination: &Path) -> std::io::Result<Self> { + let name = super::path::wide_normalized(destination)?; + if name.last() != Some(&0) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Windows rename target is not NUL-terminated", + )); + } + let name_units = name.len().checked_sub(1).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Windows rename target is not NUL-terminated", + ) + })?; + let name_bytes = name_units.checked_mul(size_of::<u16>()).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Windows rename target is too large", + ) + })?; + let stored_name_bytes = name.len().checked_mul(size_of::<u16>()).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Windows rename target is too large", + ) + })?; + let byte_len = offset_of!(FILE_RENAME_INFO, FileName) + .checked_add(stored_name_bytes) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Windows rename buffer is too large", + ) + })?; + let mut words = vec![0usize; byte_len.div_ceil(size_of::<usize>())]; + let info = words.as_mut_ptr().cast::<FILE_RENAME_INFO>(); + unsafe { + (*info).Anonymous.ReplaceIfExists = true; + (*info).RootDirectory = null_mut(); + (*info).FileNameLength = name_bytes as u32; + std::ptr::copy_nonoverlapping( + name.as_ptr(), + (&raw mut (*info).FileName).cast::<u16>(), + name.len(), + ); + } + Ok(Self { + words, + byte_len: byte_len as u32, + }) + } + + fn as_ptr(&self) -> *const std::ffi::c_void { + self.words.as_ptr().cast() + } +} + +pub(super) fn replace(source: &File, destination: &Path) -> std::io::Result<()> { + let rename = RenameBuffer::new(destination)?; + if unsafe { + SetFileInformationByHandle( + source.as_raw_handle(), + FileRenameInfo, + rename.as_ptr(), + rename.byte_len, + ) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + source.sync_all() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replacement_contract_requires_the_open_source_descriptor() { + let contract: fn(&File, &Path) -> std::io::Result<()> = replace; + let _ = contract; + } + + #[test] + fn rename_layout_declares_the_terminal_nul_for_one_unit_basename() { + let rename = RenameBuffer::new(Path::new(r"C:\x")).unwrap(); + let info = rename.as_ptr().cast::<FILE_RENAME_INFO>(); + let file_name_bytes = unsafe { (*info).FileNameLength } as usize; + let nul_index = file_name_bytes / size_of::<u16>(); + let nul = unsafe { *(&raw const (*info).FileName).cast::<u16>().add(nul_index) }; + + assert!(rename.byte_len as usize >= size_of::<FILE_RENAME_INFO>()); + assert!( + offset_of!(FILE_RENAME_INFO, FileName) + file_name_bytes + size_of::<u16>() + <= rename.byte_len as usize + ); + assert_eq!(nul, 0); + } +} diff --git a/crates/core/src/private_file_windows_security.rs b/crates/core/src/private_file_windows_security.rs new file mode 100644 index 0000000..4f1e1c7 --- /dev/null +++ b/crates/core/src/private_file_windows_security.rs @@ -0,0 +1,241 @@ +use std::ffi::c_void; +use std::fs::File; +use std::mem::{size_of, zeroed}; +use std::os::windows::io::AsRawHandle; +use std::ptr::null_mut; +use std::sync::Arc; +use windows_sys::Win32::Foundation::{CloseHandle, LocalFree}; +use windows_sys::Win32::Security::Authorization::{GetSecurityInfo, SE_FILE_OBJECT}; +use windows_sys::Win32::Security::{ + ACCESS_ALLOWED_ACE, ACL, ACL_REVISION, ACL_SIZE_INFORMATION, AclSizeInformation, + AddAccessAllowedAceEx, CONTAINER_INHERIT_ACE, DACL_SECURITY_INFORMATION, EqualSid, GetAce, + GetAclInformation, GetLengthSid, GetSecurityDescriptorControl, GetTokenInformation, + InitializeAcl, InitializeSecurityDescriptor, OBJECT_INHERIT_ACE, OWNER_SECURITY_INFORMATION, + PSID, SE_DACL_PROTECTED, SECURITY_ATTRIBUTES, SECURITY_DESCRIPTOR, + SetSecurityDescriptorControl, SetSecurityDescriptorDacl, SetSecurityDescriptorOwner, + TOKEN_QUERY, TOKEN_USER, TokenUser, +}; +use windows_sys::Win32::Storage::FileSystem::FILE_ALL_ACCESS; +use windows_sys::Win32::System::SystemServices::{ + ACCESS_ALLOWED_ACE_TYPE, SECURITY_DESCRIPTOR_REVISION, +}; +use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; + +pub(super) fn validate_private_acl(file: &File) -> std::io::Result<()> { + let (current_sid, _) = current_user_sid()?; + let mut owner = null_mut(); + let mut dacl = null_mut(); + let mut descriptor = null_mut(); + let status = unsafe { + GetSecurityInfo( + file.as_raw_handle(), + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, + &raw mut owner, + null_mut(), + &raw mut dacl, + null_mut(), + &raw mut descriptor, + ) + }; + if status != 0 { + return Err(std::io::Error::from_raw_os_error(status as i32)); + } + let current_sid_ptr = current_sid.as_ptr().cast_mut().cast(); + let result = if owner.is_null() || unsafe { EqualSid(owner, current_sid_ptr) } == 0 { + Err(permission_denied( + "private path is not owned by the current user", + )) + } else { + validate_owner_only_dacl(descriptor, dacl, current_sid_ptr) + }; + unsafe { LocalFree(descriptor) }; + result +} + +fn validate_owner_only_dacl( + descriptor: *mut c_void, + dacl: *mut ACL, + current_sid: PSID, +) -> std::io::Result<()> { + let mut control = 0; + let mut revision = 0; + if dacl.is_null() + || unsafe { GetSecurityDescriptorControl(descriptor, &raw mut control, &raw mut revision) } + == 0 + || control & SE_DACL_PROTECTED == 0 + { + return Err(permission_denied( + "private path must have a protected owner-only DACL", + )); + } + let mut acl_info = ACL_SIZE_INFORMATION::default(); + if unsafe { + GetAclInformation( + dacl, + (&raw mut acl_info).cast(), + size_of::<ACL_SIZE_INFORMATION>() as u32, + AclSizeInformation, + ) + } == 0 + || acl_info.AceCount != 1 + { + return Err(permission_denied( + "private path DACL must contain exactly one owner entry", + )); + } + let mut raw_ace = null_mut(); + if unsafe { GetAce(dacl, 0, &raw mut raw_ace) } == 0 || raw_ace.is_null() { + return Err(std::io::Error::last_os_error()); + } + let ace = unsafe { &*raw_ace.cast::<ACCESS_ALLOWED_ACE>() }; + let ace_sid = (&raw const ace.SidStart).cast_mut().cast(); + if ace.Header.AceType != ACCESS_ALLOWED_ACE_TYPE as u8 + || ace.Mask & FILE_ALL_ACCESS != FILE_ALL_ACCESS + || unsafe { EqualSid(ace_sid, current_sid) } == 0 + { + return Err(permission_denied( + "private path grants access outside the current user", + )); + } + Ok(()) +} + +pub(super) struct PrivateSecurity { + _sid: Arc<Vec<usize>>, + _acl: Vec<usize>, + _descriptor: Box<SECURITY_DESCRIPTOR>, + attributes: SECURITY_ATTRIBUTES, +} + +impl PrivateSecurity { + pub(super) fn new_directory() -> std::io::Result<Self> { + let (sid, sid_bytes) = current_user_sid()?; + Self::new_with_sid(Arc::new(sid), sid_bytes, true) + } + + pub(super) fn new_file() -> std::io::Result<Self> { + let (sid, sid_bytes) = current_user_sid()?; + Self::new_with_sid(Arc::new(sid), sid_bytes, false) + } + + pub(super) fn new_pair() -> std::io::Result<(Self, Self)> { + let (sid, sid_bytes) = current_user_sid()?; + let sid = Arc::new(sid); + let directory = Self::new_with_sid(Arc::clone(&sid), sid_bytes, true)?; + let file = Self::new_with_sid(sid, sid_bytes, false)?; + Ok((directory, file)) + } + + fn new_with_sid( + sid: Arc<Vec<usize>>, + sid_bytes: usize, + directory: bool, + ) -> std::io::Result<Self> { + let sid_ptr = sid.as_ref().as_ptr().cast_mut().cast(); + let acl_bytes = + size_of::<ACL>() + size_of::<ACCESS_ALLOWED_ACE>() - size_of::<u32>() + sid_bytes; + let mut acl = aligned_words(acl_bytes); + let acl_ptr = acl.as_mut_ptr().cast::<ACL>(); + if unsafe { InitializeAcl(acl_ptr, acl_bytes as u32, ACL_REVISION) } == 0 { + return Err(std::io::Error::last_os_error()); + } + let inheritance = if directory { + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE + } else { + 0 + }; + if unsafe { + AddAccessAllowedAceEx(acl_ptr, ACL_REVISION, inheritance, FILE_ALL_ACCESS, sid_ptr) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + let mut descriptor = Box::new(unsafe { zeroed::<SECURITY_DESCRIPTOR>() }); + let descriptor_ptr = (&raw mut *descriptor).cast(); + if unsafe { InitializeSecurityDescriptor(descriptor_ptr, SECURITY_DESCRIPTOR_REVISION) } + == 0 + || unsafe { SetSecurityDescriptorOwner(descriptor_ptr, sid_ptr, 0) } == 0 + || unsafe { SetSecurityDescriptorDacl(descriptor_ptr, 1, acl_ptr, 0) } == 0 + || unsafe { + SetSecurityDescriptorControl(descriptor_ptr, SE_DACL_PROTECTED, SE_DACL_PROTECTED) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + let attributes = SECURITY_ATTRIBUTES { + nLength: size_of::<SECURITY_ATTRIBUTES>() as u32, + lpSecurityDescriptor: descriptor_ptr, + bInheritHandle: 0, + }; + Ok(Self { + _sid: sid, + _acl: acl, + _descriptor: descriptor, + attributes, + }) + } + + pub(super) fn attributes(&self) -> *const SECURITY_ATTRIBUTES { + &raw const self.attributes + } +} + +fn current_user_sid() -> std::io::Result<(Vec<usize>, usize)> { + let mut token = null_mut(); + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &raw mut token) } == 0 { + return Err(std::io::Error::last_os_error()); + } + let result = (|| { + let mut bytes = 0; + unsafe { GetTokenInformation(token, TokenUser, null_mut(), 0, &raw mut bytes) }; + if bytes == 0 { + return Err(std::io::Error::last_os_error()); + } + let mut buffer = aligned_words(bytes as usize); + if unsafe { + GetTokenInformation( + token, + TokenUser, + buffer.as_mut_ptr().cast(), + bytes, + &raw mut bytes, + ) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + let sid = unsafe { (*buffer.as_ptr().cast::<TOKEN_USER>()).User.Sid }; + let sid_bytes = unsafe { GetLengthSid(sid) } as usize; + if sid_bytes == 0 { + return Err(std::io::Error::last_os_error()); + } + let mut owned = aligned_words(sid_bytes); + unsafe { + std::ptr::copy_nonoverlapping(sid.cast::<u8>(), owned.as_mut_ptr().cast(), sid_bytes) + }; + Ok((owned, sid_bytes)) + })(); + unsafe { CloseHandle(token) }; + result +} + +fn aligned_words(byte_len: usize) -> Vec<usize> { + vec![0; byte_len.div_ceil(size_of::<usize>())] +} + +fn permission_denied(message: &'static str) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::PermissionDenied, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn private_acl_is_protected_and_owner_only_by_construction() { + assert_ne!(SE_DACL_PROTECTED, 0); + assert_ne!(FILE_ALL_ACCESS, 0); + assert_eq!(ACL_REVISION, 2); + } +} diff --git a/crates/core/src/process_id.rs b/crates/core/src/process_id.rs new file mode 100644 index 0000000..1cc4b5a --- /dev/null +++ b/crates/core/src/process_id.rs @@ -0,0 +1,66 @@ +use serde::{Deserialize, Serialize}; + +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ProcessId(u32); + +impl ProcessId { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +impl From<u32> for ProcessId { + fn from(value: u32) -> Self { + Self::new(value) + } +} + +impl From<ProcessId> for u32 { + fn from(value: ProcessId) -> Self { + value.get() + } +} + +impl PartialEq<u32> for ProcessId { + fn eq(&self, other: &u32) -> bool { + self.get() == *other + } +} + +impl PartialEq<ProcessId> for u32 { + fn eq(&self, other: &ProcessId) -> bool { + *self == other.get() + } +} + +impl TryFrom<i32> for ProcessId { + type Error = std::num::TryFromIntError; + + fn try_from(value: i32) -> Result<Self, Self::Error> { + u32::try_from(value).map(Self::new) + } +} + +impl TryFrom<ProcessId> for i32 { + type Error = std::num::TryFromIntError; + + fn try_from(value: ProcessId) -> Result<Self, Self::Error> { + i32::try_from(value.get()) + } +} + +impl std::fmt::Display for ProcessId { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} + +#[cfg(test)] +#[path = "process_id_tests.rs"] +mod tests; diff --git a/crates/core/src/process_id_tests.rs b/crates/core/src/process_id_tests.rs new file mode 100644 index 0000000..d8258bc --- /dev/null +++ b/crates/core/src/process_id_tests.rs @@ -0,0 +1,19 @@ +use super::ProcessId; + +#[test] +fn serializes_as_a_json_number() { + let pid = ProcessId::new(u32::MAX); + + assert_eq!(serde_json::to_string(&pid).unwrap(), u32::MAX.to_string()); + assert_eq!( + serde_json::from_str::<ProcessId>("4294967295").unwrap(), + pid + ); +} + +#[test] +fn signed_conversion_rejects_invalid_platform_values() { + assert!(ProcessId::try_from(-1).is_err()); + let above_pid_t = u32::try_from(i32::MAX).unwrap() + 1; + assert!(i32::try_from(ProcessId::new(above_pid_t)).is_err()); +} diff --git a/crates/core/src/process_identity.rs b/crates/core/src/process_identity.rs new file mode 100644 index 0000000..4bd4638 --- /dev/null +++ b/crates/core/src/process_identity.rs @@ -0,0 +1,16 @@ +use crate::ProcessId; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ProcessIdentity { + pub pid: ProcessId, + pub instance: String, +} + +impl ProcessIdentity { + pub fn new(pid: impl Into<ProcessId>, instance: impl Into<String>) -> Self { + Self { + pid: pid.into(), + instance: instance.into(), + } + } +} diff --git a/crates/core/src/process_lease_guard.rs b/crates/core/src/process_lease_guard.rs new file mode 100644 index 0000000..4de9d70 --- /dev/null +++ b/crates/core/src/process_lease_guard.rs @@ -0,0 +1,51 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use crate::{AdapterError, Deadline}; + +static PROCESS_LEASE_HELD: AtomicBool = AtomicBool::new(false); + +pub(crate) struct ProcessLeaseGuard { + contention_count: u64, +} + +impl ProcessLeaseGuard { + pub(crate) fn acquire(deadline: Deadline) -> Result<Self, AdapterError> { + let mut contention_count = 0_u64; + loop { + if PROCESS_LEASE_HELD + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + if deadline.is_expired() { + PROCESS_LEASE_HELD.store(false, Ordering::Release); + return Err(timeout(deadline, contention_count)); + } + return Ok(Self { contention_count }); + } + contention_count = contention_count.saturating_add(1); + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(timeout(deadline, contention_count)); + } + std::thread::sleep(remaining.min(Duration::from_millis(1))); + } + } + + pub(crate) fn contention_count(&self) -> u64 { + self.contention_count + } +} + +impl Drop for ProcessLeaseGuard { + fn drop(&mut self) { + PROCESS_LEASE_HELD.store(false, Ordering::Release); + } +} + +fn timeout(deadline: Deadline, contention_count: u64) -> AdapterError { + deadline.timeout_error().with_details(serde_json::json!({ + "kind": "interaction_process_lock_timeout", + "contention_count": contention_count, + })) +} diff --git a/crates/core/src/process_state.rs b/crates/core/src/process_state.rs new file mode 100644 index 0000000..f9b02f2 --- /dev/null +++ b/crates/core/src/process_state.rs @@ -0,0 +1,40 @@ +use serde::{Deserialize, Serialize}; + +/// Liveness/responsiveness classification for a target process, per KTD8. +/// +/// macOS can only emit `Running`, `Exited { code: None }`, and +/// `Unresponsive` — it has no way to read the exit code of a detached +/// process (apps launched via `open -g -a` are not children of this +/// process). `Crashed` stays in the contract for adapters with real crash +/// evidence (e.g. Windows `GetExitCodeProcess`). +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum ProcessState { + Running, + Exited { + #[serde(default, skip_serializing_if = "Option::is_none")] + code: Option<i32>, + }, + Crashed { + signal_or_code: i32, + }, + Unresponsive, +} + +impl ProcessState { + /// Compact lowercase tag for best-effort `details.process_state` + /// enrichment on terminal errors — a short label, not the full + /// serialized shape (which carries `code`/`signal_or_code` payloads). + pub fn label(&self) -> &'static str { + match self { + ProcessState::Running => "running", + ProcessState::Exited { .. } => "exited", + ProcessState::Crashed { .. } => "crashed", + ProcessState::Unresponsive => "unresponsive", + } + } +} + +#[cfg(test)] +#[path = "process_state_tests.rs"] +mod tests; diff --git a/crates/core/src/process_state_tests.rs b/crates/core/src/process_state_tests.rs new file mode 100644 index 0000000..be0fc4a --- /dev/null +++ b/crates/core/src/process_state_tests.rs @@ -0,0 +1,47 @@ +use super::*; + +#[test] +fn label_reports_short_lowercase_tag_per_variant() { + assert_eq!(ProcessState::Running.label(), "running"); + assert_eq!(ProcessState::Exited { code: None }.label(), "exited"); + assert_eq!(ProcessState::Exited { code: Some(1) }.label(), "exited"); + assert_eq!( + ProcessState::Crashed { signal_or_code: 11 }.label(), + "crashed" + ); + assert_eq!(ProcessState::Unresponsive.label(), "unresponsive"); +} + +#[test] +fn exited_serializes_with_optional_code_field() { + let value = serde_json::to_value(ProcessState::Exited { code: None }).expect("serializable"); + assert_eq!(value, serde_json::json!({ "state": "exited" })); + + let value = serde_json::to_value(ProcessState::Exited { code: Some(9) }).expect("serializable"); + assert_eq!(value, serde_json::json!({ "state": "exited", "code": 9 })); +} + +#[test] +fn crashed_round_trips_through_serde() { + let value = + serde_json::to_value(ProcessState::Crashed { signal_or_code: 11 }).expect("serializable"); + assert_eq!( + value, + serde_json::json!({ "state": "crashed", "signal_or_code": 11 }) + ); + + let round_tripped: ProcessState = serde_json::from_value(value).expect("deserializable"); + assert_eq!(round_tripped, ProcessState::Crashed { signal_or_code: 11 }); +} + +#[test] +fn running_and_unresponsive_serialize_as_tag_only() { + assert_eq!( + serde_json::to_value(ProcessState::Running).expect("serializable"), + serde_json::json!({ "state": "running" }) + ); + assert_eq!( + serde_json::to_value(ProcessState::Unresponsive).expect("serializable"), + serde_json::json!({ "state": "unresponsive" }) + ); +} diff --git a/crates/core/src/recovery_hint.rs b/crates/core/src/recovery_hint.rs new file mode 100644 index 0000000..5060590 --- /dev/null +++ b/crates/core/src/recovery_hint.rs @@ -0,0 +1,10 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct RecoveryHint { + pub strategy: String, + pub retryable: bool, + pub requires_fresh_snapshot: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry_after_ms: Option<u64>, +} diff --git a/crates/core/src/rect.rs b/crates/core/src/rect.rs new file mode 100644 index 0000000..02407c9 --- /dev/null +++ b/crates/core/src/rect.rs @@ -0,0 +1,52 @@ +use serde::{Deserialize, Deserializer, Serialize}; + +const MAX_GEOMETRY_MAGNITUDE: f64 = 10_000_000.0; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] +pub struct Rect { + #[serde(default, deserialize_with = "f64_or_zero")] + pub x: f64, + #[serde(default, deserialize_with = "f64_or_zero")] + pub y: f64, + #[serde(default, deserialize_with = "f64_or_zero")] + pub width: f64, + #[serde(default, deserialize_with = "f64_or_zero")] + pub height: f64, +} + +fn f64_or_zero<'de, D: Deserializer<'de>>(deserializer: D) -> Result<f64, D::Error> { + Option::<f64>::deserialize(deserializer).map(|value| value.unwrap_or(0.0)) +} + +impl Rect { + pub fn validate(self) -> Result<Self, crate::AdapterError> { + let coordinates_valid = self.x.is_finite() + && self.y.is_finite() + && self.x.abs() <= MAX_GEOMETRY_MAGNITUDE + && self.y.abs() <= MAX_GEOMETRY_MAGNITUDE; + let size_valid = self.width.is_finite() + && self.height.is_finite() + && (0.0..=MAX_GEOMETRY_MAGNITUDE).contains(&self.width) + && (0.0..=MAX_GEOMETRY_MAGNITUDE).contains(&self.height); + if coordinates_valid && size_valid { + return Ok(self); + } + Err(crate::AdapterError::new( + crate::ErrorCode::InvalidArgs, + "Rectangle must contain finite bounded coordinates and non-negative bounded dimensions", + )) + } + + pub fn bounds_hash(&self) -> Option<u64> { + use rustc_hash::FxHasher; + use std::hash::{Hash, Hasher}; + + self.validate().ok()?; + let mut hasher = FxHasher::default(); + for value in [self.x, self.y, self.width, self.height] { + let canonical = (value * 100.0).round() as i64; + canonical.hash(&mut hasher); + } + Some(hasher.finish()) + } +} diff --git a/crates/core/src/ref_action.rs b/crates/core/src/ref_action.rs index 06cd66d..2e99b45 100644 --- a/crates/core/src/ref_action.rs +++ b/crates/core/src/ref_action.rs @@ -1,86 +1,302 @@ use crate::{ + AdapterError, AppError, action_request::ActionRequest, action_result::ActionResult, actionability, adapter::{NativeHandle, PlatformAdapter}, context::CommandContext, - error::{AdapterError, AppError}, + ref_action_context::RefActionContext, refs::RefEntry, - resolved_element::ResolvedElement, - trace_artifacts, }; use serde_json::json; +const TRACE_CAPTURE_BUDGET_MS: u64 = 1_000; +const PRE_CAPTURE_DISPATCH_RESERVE_MS: u64 = 100; + /// A strictly resolved ref-action target plus the tracing identity for it. -/// Both the CLI command path and the FFI entry path execute through -/// [`execute_resolved`], so actionability, dispatch, and trace semantics -/// live in exactly one place. pub(crate) struct ResolvedRefAction<'a> { - pub(crate) adapter: &'a dyn PlatformAdapter, - pub(crate) entry: &'a RefEntry, + target: RefActionContext<'a>, pub(crate) handle: &'a NativeHandle, - pub(crate) ref_id: &'a str, - pub(crate) context: &'a CommandContext, } +pub(crate) struct ActionabilityPreflight { + verified_point: Option<crate::Point>, + pointer_delivery: actionability::PointerDelivery, +} + +impl<'a> ResolvedRefAction<'a> { + pub(crate) fn new(target: RefActionContext<'a>, handle: &'a NativeHandle) -> Self { + Self { target, handle } + } +} + +impl<'a> std::ops::Deref for ResolvedRefAction<'a> { + type Target = RefActionContext<'a>; + + fn deref(&self) -> &Self::Target { + &self.target + } +} + +pub(crate) fn preflight_resolved( + target: &ResolvedRefAction<'_>, + request: &ActionRequest, + stability: actionability::StabilityExpectation, +) -> Result<ActionabilityPreflight, AppError> { + check_actionability_with_trace(target, request, stability) +} + +pub(crate) fn dispatch_resolved( + target: RefActionContext<'_>, + mut request: ActionRequest, + lease: &crate::InteractionLease, +) -> Result<ActionResult, AppError> { + let process_instance = target + .entry + .process + .process_instance + .as_deref() + .filter(|instance| !instance.is_empty()) + .ok_or_else(|| AdapterError::stale_ref("target process instance is unavailable"))?; + let expected_process = crate::ProcessIdentity::new(target.entry.process.pid, process_instance); + let mut handle = target + .adapter + .resolve_element_strict(target.entry, target.deadline) + .map_err(mark_pre_dispatch_resolution_failure) + .inspect_err(|error| { + crate::ref_action_wait_support::trace_resolve_error( + target.context, + target.ref_id, + error, + ) + })?; + crate::ref_action_wait_support::trace_resolve_ok(target.context, target.ref_id); + let initial_target = ResolvedRefAction::new(target, &handle); + let preflight = match stable_preflight(&initial_target, &request) { + Ok(preflight) => preflight, + Err(error) => { + let error = into_adapter_error(error); + if !crate::ref_action_wait_evidence::should_scroll_after_preflight(&request, &error) { + return Err(error.into()); + } + target + .adapter + .scroll_into_view(&handle, lease) + .inspect_err(|error| trace_scroll_error(&target, error))?; + handle = target + .adapter + .resolve_element_strict(target.entry, target.deadline) + .map_err(mark_pre_dispatch_resolution_failure)?; + let scrolled_target = ResolvedRefAction::new(target, &handle); + stable_preflight(&scrolled_target, &request)? + } + }; + if matches!( + preflight.pointer_delivery, + actionability::PointerDelivery::Semantic + ) { + request.policy = crate::InteractionPolicy::headless(); + } + request = request + .with_verified_point(preflight.verified_point) + .with_expected_process(expected_process); + let final_target = ResolvedRefAction::new(target, &handle); + final_target.context.trace_lazy( + "action.dispatch.start", + || json!({ "ref": final_target.ref_id, "action": request.action.name() }), + )?; + let action_name = request.action.name(); + let dispatch_result = final_target + .adapter + .execute_action(final_target.handle, request, lease); + let result = dispatch_result?; + final_target + .context + .trace_lazy( + "action.dispatch.ok", + || json!({ "ref": final_target.ref_id, "action": action_name, "result": &result }), + ) + .map_err(trace_error_after_delivery)?; + Ok(result) +} + +pub(crate) fn mark_pre_dispatch_resolution_failure(error: AdapterError) -> AdapterError { + match error.disposition { + crate::DeliverySemantics::Unknown | crate::DeliverySemantics::NotDelivered => { + error.with_disposition(crate::DeliverySemantics::not_delivered()) + } + crate::DeliverySemantics::DeliveryUncertain + | crate::DeliverySemantics::DeliveredUnverified + | crate::DeliverySemantics::DeliveredVerified => error, + } +} + +pub(crate) fn capture_pre_artifact( + context: &CommandContext, + adapter: &dyn PlatformAdapter, + entry: &RefEntry, + deadline: crate::Deadline, +) -> crate::trace_artifacts::ArtifactOutcome { + let Some(capture_deadline) = pre_trace_capture_deadline(deadline) else { + return crate::trace_artifacts::ArtifactOutcome::Skipped("dispatch_budget".into()); + }; + crate::trace_artifacts::capture_action_screenshot( + context, + adapter, + entry, + "pre", + capture_deadline, + ) +} + +pub(crate) fn finish_artifacts( + target: RefActionContext<'_>, + pre: &crate::trace_artifacts::ArtifactOutcome, +) { + let post = crate::trace_artifacts::capture_action_screenshot( + target.context, + target.adapter, + target.entry, + "post", + trace_capture_deadline(target.deadline), + ); + if let Err(error) = + crate::trace_artifacts::emit_action_artifacts(target.context, target.ref_id, pre, &post) + { + tracing::warn!(error = %error, ref_id = target.ref_id, "action artifact emission failed"); + } +} + +fn trace_error_after_delivery(error: AppError) -> AppError { + crate::context::trace_error_with_disposition( + error, + crate::DeliverySemantics::delivered_unverified(), + ) +} + +fn trace_capture_deadline(parent: crate::Deadline) -> crate::Deadline { + parent.capped(std::time::Duration::from_millis(TRACE_CAPTURE_BUDGET_MS)) +} + +fn pre_trace_capture_deadline(parent: crate::Deadline) -> Option<crate::Deadline> { + let remaining = parent.remaining(); + let reserve = std::time::Duration::from_millis(PRE_CAPTURE_DISPATCH_RESERVE_MS); + if remaining <= reserve { + return None; + } + let capture_budget = remaining + .saturating_sub(reserve) + .min(std::time::Duration::from_millis(TRACE_CAPTURE_BUDGET_MS)); + Some(parent.capped(capture_budget)) +} + +fn stable_preflight( + target: &ResolvedRefAction<'_>, + request: &ActionRequest, +) -> Result<ActionabilityPreflight, AppError> { + let permissive = check_actionability_with_trace( + target, + request, + actionability::StabilityExpectation::permissive(target.entry.geometry.bounds_hash), + )?; + if !actionability::requires_stability(&request.action) + || matches!( + permissive.pointer_delivery, + actionability::PointerDelivery::Semantic + ) + { + return Ok(permissive); + } + let started = std::time::Instant::now(); + let mut sampler = actionability::stability_sampler::StabilitySampler::new(); + loop { + let bounds = target + .adapter + .get_element_bounds(target.handle, target.deadline)?; + let elapsed = started.elapsed(); + if sampler.observe(bounds, elapsed) { + let stability = actionability::StabilityExpectation::strict( + sampler.bounds(), + sampler.samples(), + u64::try_from(sampler.stable_span(elapsed).as_millis()).unwrap_or(u64::MAX), + ); + return check_actionability_with_trace(target, request, stability); + } + let sleep = target + .deadline + .remaining_slice(actionability::stability_sampler::STABILITY_SAMPLE_INTERVAL) + .map_err(|error| error.with_disposition(crate::DeliverySemantics::not_delivered()))?; + std::thread::sleep(sleep); + } +} + +fn trace_scroll_error(target: &RefActionContext<'_>, error: &AdapterError) { + let _ = target.context.trace_lazy("ref.scroll_into_view.error", || { + serde_json::json!({ + "ref": target.ref_id, + "code": error.code.as_str(), + "message": error.message, + }) + }); +} + +#[cfg(test)] pub(crate) fn execute_resolved( target: ResolvedRefAction<'_>, request: ActionRequest, + lease: &crate::InteractionLease, ) -> Result<ActionResult, AppError> { - check_actionability_with_trace(&target, &request)?; - let pre = trace_artifacts::capture_action_screenshot( + let pre = capture_pre_artifact( target.context, target.adapter, - target.entry.pid, - "pre", + target.entry, + target.deadline, ); - target.context.trace_lazy( - "action.dispatch.start", - || json!({ "ref": target.ref_id, "action": request.action.name() }), - )?; - let action_name = request.action.name(); - let dispatch_result = target.adapter.execute_action(target.handle, request); - let post = trace_artifacts::capture_action_screenshot( - target.context, - target.adapter, - target.entry.pid, - "post", - ); - let _ = trace_artifacts::emit_action_artifacts(target.context, target.ref_id, &pre, &post); - let result = dispatch_result?; - let _ = target.context.trace_lazy( - "action.dispatch.ok", - || json!({ "ref": target.ref_id, "action": action_name, "result": &result }), - ); - Ok(result) + let stability = + actionability::StabilityExpectation::permissive(target.entry.geometry.bounds_hash); + preflight_resolved(&target, &request, stability)?; + let result = dispatch_resolved(target.target, request, lease); + finish_artifacts(target.target, &pre); + result } fn check_actionability_with_trace( target: &ResolvedRefAction<'_>, request: &ActionRequest, -) -> Result<(), AppError> { + stability: actionability::StabilityExpectation, +) -> Result<ActionabilityPreflight, AppError> { target.context.trace_lazy( "actionability.check.start", || json!({ "ref": target.ref_id, "action": request.action.name() }), )?; - actionability::check_live(target.entry, target.handle, target.adapter, request).inspect_err( - |err| { - let _ = target.context.trace_lazy("actionability.check.error", || { - json!({ - "ref": target.ref_id, - "action": request.action.name(), - "code": err.code.as_str(), - "message": err.message.clone(), - "details": err.details.clone() - }) - }); + let report = actionability::check_live_with_stability( + &actionability::LiveCheckTarget { + entry: target.entry, + handle: target.handle, + adapter: target.adapter, + deadline: target.deadline, }, - )?; - target.context.trace_lazy( - "actionability.check.ok", - || json!({ "ref": target.ref_id, "action": request.action.name() }), - )?; - Ok(()) + request, + stability, + ) + .inspect_err(|err| { + let _ = target.context.trace_lazy("actionability.check.error", || { + json!({ + "ref": target.ref_id, + "action": request.action.name(), + "code": err.code.as_str(), + "message": err.message.clone(), + "details": err.details.clone() + }) + }); + })?; + target.context.trace_lazy("actionability.check.ok", || { + json!({ "ref": target.ref_id, "action": request.action.name(), "report": json!(report) }) + })?; + Ok(ActionabilityPreflight { + verified_point: report.verified_point, + pointer_delivery: report.pointer_delivery, + }) } /// Builds a stable, non-sensitive trace label from a `RefEntry`. The label @@ -88,11 +304,11 @@ fn check_actionability_with_trace( /// safe to emit in the `"ref"` trace key without redaction risk. Path indices /// are deterministic within a snapshot but carry no secret information. fn ref_label_from_entry(entry: &RefEntry) -> String { - if entry.path.is_empty() { - return format!("<{}>", entry.role); + if entry.scope.path.is_empty() { + return format!("<{}>", entry.identity.role); } - let indices: Vec<String> = entry.path.iter().map(|i| i.to_string()).collect(); - format!("<{}/{}>", entry.role, indices.join("/")) + let indices: Vec<String> = entry.scope.path.iter().map(|i| i.to_string()).collect(); + format!("<{}/{}>", entry.identity.role, indices.join("/")) } /// Executes a pre-resolved ref-action entry using the provided `context` for @@ -111,19 +327,16 @@ pub fn execute_entry_with_context( context: &CommandContext, ) -> Result<ActionResult, AdapterError> { let label = ref_label_from_entry(entry); - let handle = adapter.resolve_element_strict(entry)?; - let handle = ResolvedElement::new(adapter, handle); - let result = execute_resolved( - ResolvedRefAction { + crate::ref_action_wait::execute_with_auto_wait( + crate::ref_action_wait_context::RefActionWaitContext { adapter, entry, - handle: handle.handle(), ref_id: &label, context, }, request, - ); - result.map_err(into_adapter_error) + dispatch_resolved, + ) } /// Executes a pre-resolved ref-action entry with a default (no-session, @@ -137,7 +350,7 @@ pub fn execute_entry( execute_entry_with_context(adapter, entry, request, &CommandContext::default()) } -fn into_adapter_error(err: AppError) -> AdapterError { +pub(crate) fn into_adapter_error(err: AppError) -> AdapterError { match err { AppError::Adapter(err) => err, other => AdapterError::internal(other.to_string()), diff --git a/crates/core/src/ref_action_context.rs b/crates/core/src/ref_action_context.rs new file mode 100644 index 0000000..dc2f583 --- /dev/null +++ b/crates/core/src/ref_action_context.rs @@ -0,0 +1,21 @@ +use crate::ref_action_wait_context::RefActionWaitContext; + +#[derive(Clone, Copy)] +pub(crate) struct RefActionContext<'a> { + wait: RefActionWaitContext<'a>, + pub(crate) deadline: crate::Deadline, +} + +impl<'a> RefActionContext<'a> { + pub(crate) fn new(wait: RefActionWaitContext<'a>, deadline: crate::Deadline) -> Self { + Self { wait, deadline } + } +} + +impl<'a> std::ops::Deref for RefActionContext<'a> { + type Target = RefActionWaitContext<'a>; + + fn deref(&self) -> &Self::Target { + &self.wait + } +} diff --git a/crates/core/src/ref_action_exactly_once_tests.rs b/crates/core/src/ref_action_exactly_once_tests.rs new file mode 100644 index 0000000..42bd9ea --- /dev/null +++ b/crates/core/src/ref_action_exactly_once_tests.rs @@ -0,0 +1,323 @@ +use super::*; +use crate::{ + AdapterError, ErrorCode, Rect, + action::Action, + action_request::ActionRequest, + action_result::ActionResult, + adapter::{ActionOps, InputOps, LiveElement, NativeHandle, ObservationOps, SystemOps}, + capability, + element_state::ElementState, + refs::RefEntry, + snapshot_surface::SnapshotSurface, +}; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, + time::Duration, +}; + +struct DispatchGuardAdapter { + resolve_calls: AtomicU32, + live_calls: AtomicU32, + dispatch_calls: AtomicU32, + scroll_calls: AtomicU32, + lease_held: Arc<AtomicBool>, + preflight_delay_ms: u64, + mode: &'static str, +} + +struct DispatchGuardLease(Arc<AtomicBool>); + +impl Drop for DispatchGuardLease { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +impl DispatchGuardAdapter { + fn new(mode: &'static str, preflight_delay_ms: u64) -> Self { + Self { + resolve_calls: AtomicU32::new(0), + live_calls: AtomicU32::new(0), + dispatch_calls: AtomicU32::new(0), + scroll_calls: AtomicU32::new(0), + lease_held: Arc::new(AtomicBool::new(false)), + preflight_delay_ms, + mode, + } + } +} + +impl ObservationOps for DispatchGuardAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + let call = self.resolve_calls.fetch_add(1, Ordering::SeqCst) + 1; + if self.mode == "resolve_timeout" { + return Err(AdapterError::timeout("strict resolution timed out")); + } + if self.mode == "resolve_timeout_under_lease" && self.lease_held.load(Ordering::SeqCst) { + return Err(AdapterError::timeout("strict re-resolution timed out")); + } + if self.mode == "resolve_uncertain" { + return Err(AdapterError::timeout("strict resolution was uncertain") + .with_disposition(crate::DeliverySemantics::uncertain())); + } + if self.mode == "ambiguous" && call == 1 { + return Err(AdapterError::ambiguous_target("two live candidates") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<LiveElement, AdapterError> { + std::thread::sleep(Duration::from_millis(self.preflight_delay_ms)); + let call = self.live_calls.fetch_add(1, Ordering::SeqCst) + 1; + let live_bounds = match self.mode { + "moving_once" => bounds_at(100.0), + "moving_continuously" => bounds_at(f64::from(call + 1) * 10.0), + _ => bounds(), + }; + Ok(LiveElement { + identity: crate::adapter::live_identity("Run"), + state: ElementState { + role: "button".into(), + states: Vec::new(), + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(self.mode == "offscreen_slow"), + }, + states_complete: true, + bounds: Some(live_bounds), + available_actions: vec![capability::CLICK.into()], + }) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<Rect>, AdapterError> { + let observed = self.live_calls.load(Ordering::SeqCst); + let bounds = match self.mode { + "moving_once" => bounds_at(100.0), + "moving_continuously" => bounds_at(f64::from(observed + 1) * 10.0), + _ => bounds(), + }; + Ok(Some(bounds)) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<crate::hit_test::HitTestResult, AdapterError> { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } +} + +impl ActionOps for DispatchGuardAdapter { + fn scroll_into_view( + &self, + _handle: &NativeHandle, + _lease: &crate::InteractionLease, + ) -> Result<(), AdapterError> { + self.scroll_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<ActionResult, AdapterError> { + self.dispatch_calls.fetch_add(1, Ordering::SeqCst); + if self.mode == "dispatch_error" { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "post-dispatch verification failed", + )); + } + let result = ActionResult::delivered_unverified("click"); + if self.mode == "ambiguous" { + return Ok(result.with_details(serde_json::json!({ "mechanism": "AXPress" }))); + } + Ok(result) + } +} + +impl InputOps for DispatchGuardAdapter {} +impl SystemOps for DispatchGuardAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result<crate::InteractionLease, AdapterError> { + self.lease_held.store(true, Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, DispatchGuardLease(Arc::clone(&self.lease_held))) + } +} + +fn bounds() -> Rect { + bounds_at(10.0) +} + +fn bounds_at(x: f64) -> Rect { + Rect { + x, + y: 10.0, + width: 20.0, + height: 20.0, + } +} + +fn entry() -> RefEntry { + let bounds = bounds(); + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: Vec::new(), + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + } +} + +fn execute(adapter: &DispatchGuardAdapter, timeout_ms: u64) -> Result<ActionResult, AdapterError> { + execute_with_auto_wait( + RefActionWaitCtx { + adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click).with_timeout_ms(Some(timeout_ms)), + crate::ref_action::dispatch_resolved, + ) +} + +#[test] +fn dispatch_failure_is_returned_without_repeating_the_action() { + let adapter = DispatchGuardAdapter::new("dispatch_error", 0); + let err = execute(&adapter, 500).unwrap_err(); + assert_eq!(err.code, ErrorCode::ActionFailed); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn poll_resolution_timeout_is_retry_safe_before_dispatch() { + let adapter = DispatchGuardAdapter::new("resolve_timeout", 0); + let err = execute(&adapter, 500).unwrap_err(); + + assert_eq!(err.code, ErrorCode::Timeout); + assert_eq!(err.disposition, crate::DeliverySemantics::not_delivered()); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn under_lease_resolution_timeout_is_retry_safe_before_dispatch() { + let adapter = DispatchGuardAdapter::new("resolve_timeout_under_lease", 0); + let err = execute(&adapter, 500).unwrap_err(); + + assert_eq!(err.code, ErrorCode::Timeout); + assert_eq!(err.disposition, crate::DeliverySemantics::not_delivered()); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 0); + assert!(!adapter.lease_held.load(Ordering::SeqCst)); +} + +#[test] +fn pre_dispatch_resolution_annotation_preserves_stronger_delivery_evidence() { + let adapter = DispatchGuardAdapter::new("resolve_uncertain", 0); + let err = execute(&adapter, 500).unwrap_err(); + + assert_eq!(err.disposition, crate::DeliverySemantics::uncertain()); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn expired_deadline_after_preflight_prevents_dispatch() { + let adapter = DispatchGuardAdapter::new("success", 20); + let err = execute(&adapter, 1).unwrap_err(); + assert_eq!(err.code, ErrorCode::Timeout); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn zero_timeout_is_an_explicit_single_attempt() { + let adapter = DispatchGuardAdapter::new("success", 0); + execute(&adapter, 0).unwrap(); + assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn ambiguity_diagnostic_preserves_adapter_result_details() { + let adapter = DispatchGuardAdapter::new("ambiguous", 0); + let result = execute(&adapter, 500).unwrap(); + assert_eq!(result.details.as_ref().unwrap()["mechanism"], "AXPress"); + assert_eq!( + result.details.as_ref().unwrap()["transient_ambiguity"], + true + ); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn moved_semantic_target_dispatches_without_positional_wait() { + let adapter = DispatchGuardAdapter::new("moving_once", 0); + execute(&adapter, 500).unwrap(); + assert_eq!(adapter.live_calls.load(Ordering::SeqCst), 2); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn continuously_moving_semantic_target_dispatches_exactly_once() { + let adapter = DispatchGuardAdapter::new("moving_continuously", 0); + execute(&adapter, 60).unwrap(); + assert_eq!(adapter.live_calls.load(Ordering::SeqCst), 2); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn expired_preflight_never_scrolls_after_the_deadline() { + let adapter = DispatchGuardAdapter::new("offscreen_slow", 20); + let err = execute(&adapter, 1).unwrap_err(); + assert_eq!(err.code, ErrorCode::Timeout); + assert_eq!(adapter.scroll_calls.load(Ordering::SeqCst), 0); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 0); +} diff --git a/crates/core/src/ref_action_poll.rs b/crates/core/src/ref_action_poll.rs new file mode 100644 index 0000000..3be977c --- /dev/null +++ b/crates/core/src/ref_action_poll.rs @@ -0,0 +1,187 @@ +use std::time::Duration; + +use serde_json::json; + +use crate::{ + ActionRequest, AdapterError, Deadline, ErrorCode, + ref_action::{ResolvedRefAction, into_adapter_error, preflight_resolved}, + ref_action_context::RefActionContext, + ref_action_poll_state::RefActionPollState, + ref_action_wait_context::RefActionWaitContext, + ref_action_wait_evidence::{failed_check, should_scroll_after_preflight}, + ref_action_wait_support::{trace_resolve_error, trace_resolve_ok}, + ref_resolve_deadline::{POLL_INTERVAL, resolve_within_deadline}, + resolve_attempt_outcome::ResolveAttemptOutcome, +}; + +const STABILITY_POLL_INTERVAL: Duration = Duration::from_millis(16); + +pub(crate) fn execute_poll_loop( + context: RefActionWaitContext<'_>, + request: &ActionRequest, + deadline: Deadline, +) -> Result<RefActionPollState, AdapterError> { + let mut state = RefActionPollState::default(); + loop { + state.resolve_attempts = state.resolve_attempts.saturating_add(1); + match resolve_within_deadline(context.adapter, context.entry, deadline) { + ResolveAttemptOutcome::DeadlinePassed => return Err(timeout(&state, deadline)), + ResolveAttemptOutcome::Failed(error) => { + handle_resolve_failure( + context, + &mut state, + crate::ref_action::mark_pre_dispatch_resolution_failure(error), + deadline, + )?; + } + ResolveAttemptOutcome::Resolved(handle) => { + trace_resolve_ok(context.context, context.ref_id); + ensure_before_deadline(deadline, &state)?; + let target = + ResolvedRefAction::new(RefActionContext::new(context, deadline), &handle); + state.preflight_attempts = state.preflight_attempts.saturating_add(1); + if let Err(error) = preflight_resolved(&target, request, state.stability()) { + let error = into_adapter_error(error); + if !should_scroll_after_preflight(request, &error) { + handle_actionability_failure(&mut state, error, deadline)?; + continue; + } + } + ensure_before_deadline(deadline, &state)?; + return Ok(state); + } + } + } +} + +fn handle_actionability_failure( + state: &mut RefActionPollState, + error: AdapterError, + deadline: Deadline, +) -> Result<(), AdapterError> { + if is_permanent_actionability_error(&error.code) { + return Err(error); + } + let stability_changed = failed_check(&error, "stable"); + state.record_preflight_error(&error); + ensure_before_deadline(deadline, state)?; + sleep_until_next_poll( + deadline, + if stability_changed { + STABILITY_POLL_INTERVAL + } else { + POLL_INTERVAL + }, + ); + Ok(()) +} + +fn handle_resolve_failure( + context: RefActionWaitContext<'_>, + state: &mut RefActionPollState, + error: AdapterError, + deadline: Deadline, +) -> Result<(), AdapterError> { + if !error.is_retryable_resolution_failure() { + trace_resolve_error(context.context, context.ref_id, &error); + return Err(error); + } + state.saw_ambiguity |= error.code == ErrorCode::AmbiguousTarget; + state.record_resolve_error(&error); + sleep_until_next_poll(deadline, POLL_INTERVAL); + Ok(()) +} + +fn ensure_before_deadline( + deadline: Deadline, + state: &RefActionPollState, +) -> Result<(), AdapterError> { + if deadline.is_expired() { + return Err(timeout(state, deadline)); + } + Ok(()) +} + +fn sleep_until_next_poll(deadline: Deadline, interval: Duration) { + let remaining = deadline.remaining(); + if !remaining.is_zero() { + std::thread::sleep(interval.min(remaining)); + } +} + +fn is_permanent_error(code: &ErrorCode) -> bool { + matches!( + code, + ErrorCode::PermDenied + | ErrorCode::AppNotFound + | ErrorCode::ActionNotSupported + | ErrorCode::InvalidArgs + | ErrorCode::PolicyDenied + | ErrorCode::Internal + ) +} + +fn is_permanent_actionability_error(code: &ErrorCode) -> bool { + is_permanent_error(code) || matches!(code, ErrorCode::StaleRef | ErrorCode::AmbiguousTarget) +} + +fn timeout(state: &RefActionPollState, deadline: Deadline) -> AdapterError { + let mut details = json!({ + "kind": "actionability_timeout", + "timeout_ms": deadline.timeout_ms(), + "elapsed_ms": deadline.elapsed().as_millis(), + }); + if let Some(last_report) = state.last_report.clone() + && let Some(object) = details.as_object_mut() + { + object.insert("last_report".into(), last_report); + } + if state.saw_ambiguity + && let Some(object) = details.as_object_mut() + { + object.insert("transient_ambiguity".into(), true.into()); + } + AdapterError::timeout("Target did not become actionable within the wait budget") + .with_details(details) + .with_disposition(crate::DeliverySemantics::not_delivered()) +} + +#[cfg(test)] +pub(crate) fn timeout_with_last_report(last_report: serde_json::Value) -> AdapterError { + let deadline = Deadline::after(1).expect("test deadline"); + timeout( + &RefActionPollState { + last_report: Some(last_report), + ..Default::default() + }, + deadline, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preflight_identity_failures_are_terminal() { + assert!(is_permanent_actionability_error(&ErrorCode::StaleRef)); + assert!(is_permanent_actionability_error( + &ErrorCode::AmbiguousTarget + )); + assert!(!is_permanent_error(&ErrorCode::StaleRef)); + } + + #[test] + fn timeout_preserves_transient_ambiguity_evidence() { + let state = RefActionPollState { + saw_ambiguity: true, + ..Default::default() + }; + let error = timeout(&state, Deadline::after(1).expect("deadline")); + + assert_eq!( + error.details.expect("timeout details")["transient_ambiguity"], + true + ); + } +} diff --git a/crates/core/src/ref_action_poll_state.rs b/crates/core/src/ref_action_poll_state.rs new file mode 100644 index 0000000..cdc8365 --- /dev/null +++ b/crates/core/src/ref_action_poll_state.rs @@ -0,0 +1,94 @@ +use serde_json::{Value, json}; + +use crate::{ActionResult, AdapterError, actionability::StabilityExpectation}; + +#[derive(Default)] +pub(crate) struct RefActionPollState { + pub(crate) last_report: Option<Value>, + pub(crate) saw_ambiguity: bool, + pub(crate) expected_bounds_hash: Option<u64>, + pub(crate) resolve_attempts: u64, + pub(crate) preflight_attempts: u64, +} + +impl RefActionPollState { + pub(crate) fn stability(&self) -> StabilityExpectation { + StabilityExpectation::strict_hash(self.expected_bounds_hash) + } + + pub(crate) fn record_preflight_error(&mut self, error: &AdapterError) { + if let Some(observed) = crate::ref_action_wait_evidence::observed_bounds_hash(error) { + self.expected_bounds_hash = Some(observed); + } + if let Some(report) = error.details.clone() { + self.last_report = Some(json!({ "phase": "preflight", "report": report })); + } + } + + pub(crate) fn record_resolve_error(&mut self, error: &AdapterError) { + self.last_report = Some(json!({ + "phase": "resolve", + "code": error.code.as_str(), + "message": error.message, + "details": error.details, + })); + } + + pub(crate) fn attach_transient_ambiguity(&self, result: &mut ActionResult) { + if !self.saw_ambiguity { + return; + } + let mut details = result.details.take().unwrap_or_else(|| json!({})); + if let Some(object) = details.as_object_mut() { + object.insert("transient_ambiguity".into(), json!(true)); + result.details = Some(details); + } else { + result.details = Some(json!({ + "action_details": details, + "transient_ambiguity": true, + })); + } + } + + pub(crate) fn attach_wait_metrics( + &self, + result: &mut ActionResult, + lease: &crate::InteractionLease, + lease_hold_ms: u64, + ) { + let mut details = result.details.take().unwrap_or_else(|| json!({})); + let metrics = json!({ + "read_only_resolve_attempts": self.resolve_attempts, + "read_only_preflight_attempts": self.preflight_attempts, + "lease_contention_count": lease.contention_count(), + "lease_hold_ms": lease_hold_ms, + }); + if let Some(object) = details.as_object_mut() { + object.insert("auto_wait".into(), metrics); + result.details = Some(details); + } else { + result.details = Some(json!({ + "action_details": details, + "auto_wait": metrics, + })); + } + } + + pub(crate) fn attach_error_metrics(&self, mut error: AdapterError) -> AdapterError { + let mut details = error.details.take().unwrap_or_else(|| json!({})); + let metrics = json!({ + "read_only_resolve_attempts": self.resolve_attempts, + "read_only_preflight_attempts": self.preflight_attempts, + }); + if let Some(object) = details.as_object_mut() { + object.insert("auto_wait".into(), metrics); + error.details = Some(details); + } else { + error.details = Some(json!({ + "error_details": details, + "auto_wait": metrics, + })); + } + error + } +} diff --git a/crates/core/src/ref_action_single.rs b/crates/core/src/ref_action_single.rs new file mode 100644 index 0000000..60d9782 --- /dev/null +++ b/crates/core/src/ref_action_single.rs @@ -0,0 +1,27 @@ +use crate::{ + ActionRequest, ActionResult, AdapterError, ref_action::into_adapter_error, + ref_action_context::RefActionContext, ref_action_wait_context::RefActionWaitContext, +}; + +pub(crate) fn execute_single_shot( + context: RefActionWaitContext<'_>, + request: ActionRequest, + deadline: crate::Deadline, + lease: &crate::InteractionLease, + dispatch: impl Fn( + RefActionContext<'_>, + ActionRequest, + &crate::InteractionLease, + ) -> Result<ActionResult, crate::AppError>, +) -> Result<ActionResult, AdapterError> { + if request.headed_requirement().requires_focus() { + crate::headed_focus::focus_entry_window( + context.entry, + context.adapter, + context.context, + lease, + ) + .map_err(crate::ref_action::into_adapter_error)?; + } + dispatch(RefActionContext::new(context, deadline), request, lease).map_err(into_adapter_error) +} diff --git a/crates/core/src/ref_action_tests.rs b/crates/core/src/ref_action_tests.rs index 9dcacf1..42b9459 100644 --- a/crates/core/src/ref_action_tests.rs +++ b/crates/core/src/ref_action_tests.rs @@ -1,106 +1,284 @@ use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; use crate::{ action::Action, action_result::ActionResult, adapter::{NativeHandle, SnapshotSurface}, capability, }; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU32, Ordering}, +}; -struct ReleaseFailingAdapter; +struct DropProbe(Arc<AtomicU32>); -impl PlatformAdapter for ReleaseFailingAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { - Ok(NativeHandle::null()) +impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +struct SuccessfulAdapter { + drops: Arc<AtomicU32>, + dispatched_policies: Mutex<Vec<crate::InteractionPolicy>>, +} + +impl SuccessfulAdapter { + fn new() -> Self { + Self { + drops: Arc::new(AtomicU32::new(0)), + dispatched_policies: Mutex::new(Vec::new()), + } + } +} + +impl ObservationOps for SuccessfulAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::new(DropProbe(Arc::clone(&self.drops)))) } + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for SuccessfulAdapter { fn execute_action( &self, _handle: &NativeHandle, - _request: ActionRequest, + request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<ActionResult, AdapterError> { - Ok(ActionResult::new("click")) - } - - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - Err(AdapterError::internal("release failed")) + self.dispatched_policies + .lock() + .unwrap() + .push(request.policy); + Ok(ActionResult::delivered_unverified("click")) } } -struct ErrorReleasingAdapter { - releases: AtomicU32, +impl InputOps for SuccessfulAdapter {} + +impl SystemOps for SuccessfulAdapter { + crate::adapter::guarded_interaction_lease!(); + crate::adapter::exact_window_focus!(); } -impl PlatformAdapter for ErrorReleasingAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { - Ok(NativeHandle::null()) +struct FailingAdapter { + drops: Arc<AtomicU32>, +} + +impl ObservationOps for FailingAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::new(DropProbe(Arc::clone(&self.drops)))) } + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for FailingAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<ActionResult, AdapterError> { Err(AdapterError::internal("dispatch failed")) } +} - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - self.releases.fetch_add(1, Ordering::SeqCst); - Ok(()) +impl InputOps for FailingAdapter {} + +impl SystemOps for FailingAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +struct ProcessReplacingAdapter; + +impl ObservationOps for ProcessReplacingAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) } + + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for ProcessReplacingAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<ActionResult, AdapterError> { + let expected = request + .expected_process() + .ok_or_else(|| AdapterError::internal("missing expected process identity"))?; + if expected.instance != "replacement-generation" { + return Err(AdapterError::stale_ref( + "process generation changed before physical dispatch", + )); + } + Ok(ActionResult::delivered_unverified("click")) + } +} + +impl InputOps for ProcessReplacingAdapter {} +impl SystemOps for ProcessReplacingAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +struct TraceFailureAdapter { + path: std::path::PathBuf, + fail_after_dispatch: bool, + dispatches: AtomicU32, +} + +impl ObservationOps for TraceFailureAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for TraceFailureAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<ActionResult, AdapterError> { + self.dispatches.fetch_add(1, Ordering::SeqCst); + if self.fail_after_dispatch { + std::fs::OpenOptions::new() + .write(true) + .open(&self.path) + .unwrap() + .set_len(crate::trace::MAX_TRACE_FILE_BYTES) + .unwrap(); + } + Ok(ActionResult::delivered_unverified("click")) + } +} + +impl InputOps for TraceFailureAdapter {} +impl SystemOps for TraceFailureAdapter { + crate::adapter::guarded_interaction_lease!(); } fn entry() -> RefEntry { + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; RefEntry { - pid: 1, - role: "button".into(), - name: Some("Run".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-42".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } #[test] -fn successful_action_survives_release_failure() { - let result = execute_entry( - &ReleaseFailingAdapter, - &entry(), - ActionRequest::headless(Action::Click), - ) - .unwrap(); +fn successful_action_drops_resolved_payload() { + let adapter = SuccessfulAdapter::new(); + let result = execute_entry(&adapter, &entry(), ActionRequest::headless(Action::Click)).unwrap(); assert_eq!(result.action, "click"); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); } #[test] -fn failed_action_still_releases_resolved_handle() { - let adapter = ErrorReleasingAdapter { - releases: AtomicU32::new(0), +fn headed_preflight_preserves_requested_physical_delivery() { + let adapter = SuccessfulAdapter::new(); + + execute_entry(&adapter, &entry(), ActionRequest::headed(Action::Click)).unwrap(); + + assert_eq!( + adapter.dispatched_policies.lock().unwrap().as_slice(), + &[crate::InteractionPolicy::headed()] + ); +} + +#[test] +fn failed_action_still_drops_resolved_payload() { + let adapter = FailingAdapter { + drops: Arc::new(AtomicU32::new(0)), }; let err = execute_entry(&adapter, &entry(), ActionRequest::headless(Action::Click)).unwrap_err(); - assert_eq!(err.code, crate::error::ErrorCode::Internal); - assert_eq!(adapter.releases.load(Ordering::SeqCst), 1); + assert_eq!(err.code, crate::ErrorCode::Internal); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn replacement_between_resolution_and_dispatch_fails_before_delivery() { + let error = execute_entry( + &ProcessReplacingAdapter, + &entry(), + ActionRequest::headless(Action::Click), + ) + .unwrap_err(); + + assert_eq!(error.code, crate::ErrorCode::StaleRef); + assert_eq!( + error.disposition.delivery(), + crate::DeliveryDisposition::NotDelivered + ); } #[test] fn execute_entry_with_context_succeeds_and_matches_execute_entry() { let context = CommandContext::default(); + let adapter = SuccessfulAdapter::new(); let result = execute_entry_with_context( - &ReleaseFailingAdapter, + &adapter, &entry(), ActionRequest::headless(Action::Click), &context, @@ -121,9 +299,10 @@ fn execute_entry_with_context_emits_trace_events() { )); let context = CommandContext::new(Some("test-session".into()), Some(trace_path.clone()), false).unwrap(); + let adapter = SuccessfulAdapter::new(); let _ = execute_entry_with_context( - &ReleaseFailingAdapter, + &adapter, &entry(), ActionRequest::headless(Action::Click), &context, @@ -158,9 +337,10 @@ fn trace_records_omit_session_id_when_context_has_none() { .as_nanos() )); let context = CommandContext::new(None, Some(trace_path.clone()), false).unwrap(); + let adapter = SuccessfulAdapter::new(); let _ = execute_entry_with_context( - &ReleaseFailingAdapter, + &adapter, &entry(), ActionRequest::headless(Action::Click), &context, @@ -183,6 +363,9 @@ fn ref_label_from_entry_uses_role_and_path_indices() { assert_eq!(ref_label_from_entry(&no_path), "<button>"); let mut with_path = entry(); - with_path.path = smallvec::smallvec![2, 0, 3]; + with_path.scope.path = smallvec::smallvec![2, 0, 3]; assert_eq!(ref_label_from_entry(&with_path), "<button/2/0/3>"); } + +#[path = "ref_action_trace_failure_tests.rs"] +mod trace_failure_tests; diff --git a/crates/core/src/ref_action_trace_failure_tests.rs b/crates/core/src/ref_action_trace_failure_tests.rs new file mode 100644 index 0000000..7f68913 --- /dev/null +++ b/crates/core/src/ref_action_trace_failure_tests.rs @@ -0,0 +1,62 @@ +use super::*; + +#[test] +fn strict_trace_failure_before_dispatch_is_not_delivered() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-ref-trace-start-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(crate::trace::MAX_TRACE_FILE_BYTES) + .unwrap(); + let adapter = TraceFailureAdapter { + path: path.clone(), + fail_after_dispatch: false, + dispatches: AtomicU32::new(0), + }; + + let error = execute_entry_with_context( + &adapter, + &entry(), + ActionRequest::headless(Action::Click), + &context, + ) + .unwrap_err(); + + assert_eq!(adapter.dispatches.load(Ordering::SeqCst), 0); + assert_eq!(error.disposition, crate::DeliverySemantics::not_delivered()); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn strict_trace_failure_after_dispatch_is_unsafe_to_retry() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-ref-trace-end-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let context = CommandContext::new(None, Some(path.clone()), true).unwrap(); + let adapter = TraceFailureAdapter { + path: path.clone(), + fail_after_dispatch: true, + dispatches: AtomicU32::new(0), + }; + + let error = execute_entry_with_context( + &adapter, + &entry(), + ActionRequest::headless(Action::Click), + &context, + ) + .unwrap_err(); + + assert_eq!(adapter.dispatches.load(Ordering::SeqCst), 1); + assert_eq!( + error.disposition, + crate::DeliverySemantics::delivered_unverified() + ); + std::fs::remove_file(path).unwrap(); +} diff --git a/crates/core/src/ref_action_wait.rs b/crates/core/src/ref_action_wait.rs new file mode 100644 index 0000000..4cbf0d1 --- /dev/null +++ b/crates/core/src/ref_action_wait.rs @@ -0,0 +1,138 @@ +use crate::{ + ActionRequest, ActionResult, AdapterError, AppError, Deadline, + ref_action_context::RefActionContext, ref_action_poll::execute_poll_loop, + ref_action_poll_state::RefActionPollState, ref_action_single::execute_single_shot, + ref_action_wait_context::RefActionWaitContext, + ref_action_wait_support::enrich_with_process_state, +}; + +#[cfg(test)] +use crate::{context::CommandContext, refs::RefEntry}; +#[cfg(test)] +type RefActionWaitCtx<'a> = RefActionWaitContext<'a>; + +pub(crate) fn execute_with_auto_wait( + context: RefActionWaitContext<'_>, + request: ActionRequest, + dispatch: impl Fn( + RefActionContext<'_>, + ActionRequest, + &crate::InteractionLease, + ) -> Result<ActionResult, AppError>, +) -> Result<ActionResult, AdapterError> { + let (result, lease, pre, deadline, _lease_started) = + execute_with_auto_wait_and_lease(context, request, dispatch)?; + drop(lease); + crate::ref_action::finish_artifacts(RefActionContext::new(context, deadline), &pre); + Ok(result) +} + +pub(crate) fn execute_with_auto_wait_and_lease( + context: RefActionWaitContext<'_>, + request: ActionRequest, + dispatch: impl Fn( + RefActionContext<'_>, + ActionRequest, + &crate::InteractionLease, + ) -> Result<ActionResult, AppError>, +) -> Result< + ( + ActionResult, + crate::InteractionLease, + crate::trace_artifacts::ArtifactOutcome, + Deadline, + std::time::Instant, + ), + AdapterError, +> { + let deadline = operation_deadline(&request)?; + let result = execute_with_deadline(context, request, deadline, dispatch); + result.map_err(|error| { + if deadline.is_expired() { + error + } else { + enrich_with_process_state(context.adapter, context.entry, error, deadline) + } + }) +} + +fn execute_with_deadline( + context: RefActionWaitContext<'_>, + mut request: ActionRequest, + deadline: Deadline, + dispatch: impl Fn( + RefActionContext<'_>, + ActionRequest, + &crate::InteractionLease, + ) -> Result<ActionResult, AppError>, +) -> Result< + ( + ActionResult, + crate::InteractionLease, + crate::trace_artifacts::ArtifactOutcome, + Deadline, + std::time::Instant, + ), + AdapterError, +> { + let state = if request.timeout_ms.is_some_and(|timeout_ms| timeout_ms > 0) { + execute_poll_loop(context, &request, deadline)? + } else { + RefActionPollState::default() + }; + let pre = crate::ref_action::capture_pre_artifact( + context.context, + context.adapter, + context.entry, + deadline, + ); + let lease = context + .adapter + .acquire_interaction_lease(deadline) + .map_err(|error| state.attach_error_metrics(error))?; + if deadline.is_expired() { + return Err(state.attach_error_metrics(deadline.timeout_error())); + } + if request.timeout_ms.is_some_and(|timeout_ms| timeout_ms > 0) { + request.timeout_ms = Some(deadline.remaining_ms()); + } + let lease_started = std::time::Instant::now(); + let mut result = match execute_single_shot(context, request, deadline, &lease, dispatch) { + Ok(result) => result, + Err(error) => { + drop(lease); + crate::ref_action::finish_artifacts(RefActionContext::new(context, deadline), &pre); + return Err(error); + } + }; + state.attach_transient_ambiguity(&mut result); + state.attach_wait_metrics( + &mut result, + &lease, + u64::try_from(lease_started.elapsed().as_millis()).unwrap_or(u64::MAX), + ); + Ok((result, lease, pre, deadline, lease_started)) +} + +pub(crate) fn operation_deadline(request: &ActionRequest) -> Result<Deadline, AdapterError> { + request + .timeout_ms + .filter(|timeout_ms| *timeout_ms > 0) + .map_or_else(Deadline::standard, Deadline::after) +} + +#[cfg(test)] +#[path = "ref_action_wait_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "ref_action_wait_process_state_tests.rs"] +mod process_state_tests; + +#[cfg(test)] +#[path = "ref_action_wait_app_not_found_tests.rs"] +mod app_not_found_tests; + +#[cfg(test)] +#[path = "ref_action_exactly_once_tests.rs"] +mod exactly_once_tests; diff --git a/crates/core/src/ref_action_wait_app_not_found_tests.rs b/crates/core/src/ref_action_wait_app_not_found_tests.rs new file mode 100644 index 0000000..99e9e69 --- /dev/null +++ b/crates/core/src/ref_action_wait_app_not_found_tests.rs @@ -0,0 +1,287 @@ +use super::*; +use crate::{ + AdapterError, AppInfo, ErrorCode, + action::Action, + adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}, + capability, +}; +use std::sync::atomic::{AtomicU32, Ordering}; + +/// F23 follow-up: covers the `APP_NOT_FOUND` branch of terminal process-state +/// enrichment, split out of `ref_action_wait_process_state_tests.rs` to keep +/// both files under the repo's 400 LOC hard limit. +fn entry() -> RefEntry { + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(crate::Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }), + bounds_hash: Some(1), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("Original".into()), + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::snapshot_surface::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + } +} + +fn request_with_timeout(timeout_ms: u64) -> ActionRequest { + ActionRequest::headless(Action::Click).with_timeout_ms(Some(timeout_ms)) +} + +struct TicksThenAppNotFoundAdapter { + resolve_calls: AtomicU32, + probe_calls: AtomicU32, +} + +impl ObservationOps for TicksThenAppNotFoundAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + let attempt = self.resolve_calls.fetch_add(1, Ordering::SeqCst) + 1; + if attempt < 3 { + Err(AdapterError::new(ErrorCode::StaleRef, "not yet") + .with_details(serde_json::json!({ "retryable": true }))) + } else { + Err(AdapterError::new(ErrorCode::AppNotFound, "app gone")) + } + } +} + +impl ActionOps for TicksThenAppNotFoundAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for TicksThenAppNotFoundAdapter {} + +impl SystemOps for TicksThenAppNotFoundAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + self.probe_calls.fetch_add(1, Ordering::SeqCst); + Ok(crate::process_state::ProcessState::Running) + } +} + +struct AppNotFoundExitedProcessAdapter; + +impl ObservationOps for AppNotFoundExitedProcessAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::new(ErrorCode::AppNotFound, "app gone")) + } +} + +impl ActionOps for AppNotFoundExitedProcessAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for AppNotFoundExitedProcessAdapter {} + +impl SystemOps for AppNotFoundExitedProcessAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + Ok(crate::process_state::ProcessState::Exited { code: None }) + } +} + +#[test] +fn terminal_app_not_found_against_exited_process_carries_process_state_detail() { + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &AppNotFoundExitedProcessAdapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!( + err.code, + ErrorCode::AppNotFound, + "an Exited (not Unresponsive) classification must not replace the original error code" + ); + assert_eq!( + err.details.as_ref().and_then(|d| d.get("process_state")), + Some(&serde_json::json!("exited")), + "APP_NOT_FOUND against a dead pid must carry details.process_state = \"exited\"" + ); +} + +struct AppNotFoundUnresponsiveProcessAdapter { + probe_calls: AtomicU32, +} + +impl ObservationOps for AppNotFoundUnresponsiveProcessAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result<Vec<AppInfo>, AdapterError> { + Ok(vec![AppInfo { + name: "Original".into(), + pid: crate::ProcessId::new(1), + bundle_id: None, + process_instance: Some("test-instance".into()), + }]) + } + + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::new(ErrorCode::AppNotFound, "app gone")) + } +} + +impl ActionOps for AppNotFoundUnresponsiveProcessAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for AppNotFoundUnresponsiveProcessAdapter {} + +impl SystemOps for AppNotFoundUnresponsiveProcessAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + self.probe_calls.fetch_add(1, Ordering::SeqCst); + Ok(crate::process_state::ProcessState::Unresponsive) + } +} + +#[test] +fn terminal_app_not_found_against_unresponsive_process_surfaces_app_unresponsive() { + let adapter = AppNotFoundUnresponsiveProcessAdapter { + probe_calls: AtomicU32::new(0), + }; + + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!( + err.code, + ErrorCode::AppUnresponsive, + "an Unresponsive classification must upgrade APP_NOT_FOUND to APP_UNRESPONSIVE" + ); + assert!( + err.suggestion.is_some(), + "APP_UNRESPONSIVE must carry a recovery suggestion" + ); + assert_eq!( + adapter.probe_calls.load(Ordering::SeqCst), + 1, + "the liveness probe must run exactly once when building the terminal error" + ); +} + +#[test] +fn probe_call_count_is_independent_of_auto_wait_tick_count() { + let adapter = TicksThenAppNotFoundAdapter { + resolve_calls: AtomicU32::new(0), + probe_calls: AtomicU32::new(0), + }; + + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + request_with_timeout(5_000), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::AppNotFound); + assert!( + adapter.resolve_calls.load(Ordering::SeqCst) >= 3, + "the poll loop must have ticked multiple times before the terminal error" + ); + assert_eq!( + adapter.probe_calls.load(Ordering::SeqCst), + 1, + "the liveness probe must run exactly once at the terminal boundary, \ + not once per auto-wait tick" + ); +} diff --git a/crates/core/src/ref_action_wait_context.rs b/crates/core/src/ref_action_wait_context.rs new file mode 100644 index 0000000..bace5c1 --- /dev/null +++ b/crates/core/src/ref_action_wait_context.rs @@ -0,0 +1,9 @@ +use crate::{CommandContext, PlatformAdapter, RefEntry}; + +#[derive(Clone, Copy)] +pub(crate) struct RefActionWaitContext<'a> { + pub(crate) adapter: &'a dyn PlatformAdapter, + pub(crate) entry: &'a RefEntry, + pub(crate) ref_id: &'a str, + pub(crate) context: &'a CommandContext, +} diff --git a/crates/core/src/ref_action_wait_evidence.rs b/crates/core/src/ref_action_wait_evidence.rs new file mode 100644 index 0000000..7ef23e8 --- /dev/null +++ b/crates/core/src/ref_action_wait_evidence.rs @@ -0,0 +1,27 @@ +use crate::{ActionRequest, AdapterError}; + +pub(crate) fn should_scroll_after_preflight(request: &ActionRequest, error: &AdapterError) -> bool { + request.action.requires_scroll_into_view() && failed_check(error, "visible") +} + +pub(crate) fn failed_check(error: &AdapterError, check_name: &str) -> bool { + error + .details + .as_ref() + .and_then(|details| details.get("checks")) + .and_then(serde_json::Value::as_array) + .is_some_and(|checks| { + checks.iter().any(|check| { + check.get("check").and_then(serde_json::Value::as_str) == Some(check_name) + && check.get("status").and_then(serde_json::Value::as_str) == Some("fail") + }) + }) +} + +pub(crate) fn observed_bounds_hash(error: &AdapterError) -> Option<u64> { + error + .details + .as_ref() + .and_then(|details| details.get("observed_bounds_hash")) + .and_then(serde_json::Value::as_u64) +} diff --git a/crates/core/src/ref_action_wait_lease_tests.rs b/crates/core/src/ref_action_wait_lease_tests.rs new file mode 100644 index 0000000..e5c4de3 --- /dev/null +++ b/crates/core/src/ref_action_wait_lease_tests.rs @@ -0,0 +1,197 @@ +use super::*; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU32, Ordering}, +}; + +struct DoubleCheckAdapter { + live_calls: AtomicU32, + lease_held: Arc<AtomicBool>, + prelease_calls: u32, + timeout_ms: u64, + expected_deadline: Option<crate::Deadline>, +} + +struct DoubleCheckLeaseGuard(Arc<AtomicBool>); + +impl Drop for DoubleCheckLeaseGuard { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +impl ObservationOps for DoubleCheckAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) + } + + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<crate::adapter::LiveElement, AdapterError> { + let call = self.live_calls.fetch_add(1, Ordering::SeqCst) + 1; + assert_eq!( + self.lease_held.load(Ordering::SeqCst), + call > self.prelease_calls, + "poll reads must be lease-free and final validation must be leased" + ); + let states = if call <= self.prelease_calls { + Vec::new() + } else { + vec!["disabled".to_string()] + }; + Ok(crate::adapter::LiveElement { + identity: crate::adapter::live_identity("Run"), + state: crate::element_state::ElementState { + role: "button".into(), + states, + value: None, + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, + bounds: Some(bounds()), + available_actions: vec![capability::CLICK.into()], + }) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<crate::Rect>, AdapterError> { + Ok(Some(bounds())) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<crate::hit_test::HitTestResult, AdapterError> { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } +} + +impl ActionOps for DoubleCheckAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for DoubleCheckAdapter {} + +impl SystemOps for DoubleCheckAdapter { + crate::adapter::exact_window_focus!(); + + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result<crate::InteractionLease, AdapterError> { + assert_eq!(self.live_calls.load(Ordering::SeqCst), self.prelease_calls); + assert_eq!(deadline.timeout_ms(), self.timeout_ms); + if let Some(expected) = self.expected_deadline { + assert_eq!(deadline, expected); + return Err(AdapterError::new(ErrorCode::Internal, "lease probe")); + } + self.lease_held.store(true, Ordering::SeqCst); + Ok(crate::InteractionLease::guarded_with_contention( + deadline, + DoubleCheckLeaseGuard(Arc::clone(&self.lease_held)), + 3, + )) + } +} + +#[test] +fn stability_revalidates_once_under_lease_before_dispatch() { + let adapter = adapter(2, 5_000, None); + let result = run(&adapter, ActionRequest::headed(Action::DoubleClick), 5_000); + + assert_eq!(result.action, "click"); + assert_eq!(adapter.live_calls.load(Ordering::SeqCst), 4); + let metrics = &result.details.as_ref().unwrap()["auto_wait"]; + assert_eq!(metrics["read_only_resolve_attempts"], 2); + assert_eq!(metrics["read_only_preflight_attempts"], 2); + assert_eq!(metrics["lease_contention_count"], 3); + assert!(metrics["lease_hold_ms"].as_u64().is_some()); + assert!(!adapter.lease_held.load(Ordering::SeqCst)); +} + +#[test] +fn lease_acquisition_receives_the_exact_original_deadline() { + let deadline = crate::Deadline::after(80).unwrap(); + let adapter = adapter(0, 80, Some(deadline)); + let error = super::super::execute_with_deadline( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + deadline, + crate::ref_action::dispatch_resolved, + ) + .err() + .unwrap(); + + assert_eq!(error.code, ErrorCode::Internal); + assert_eq!(error.message, "lease probe"); + assert_eq!(adapter.live_calls.load(Ordering::SeqCst), 0); + assert!(!adapter.lease_held.load(Ordering::SeqCst)); +} + +fn adapter( + prelease_calls: u32, + timeout_ms: u64, + expected_deadline: Option<crate::Deadline>, +) -> DoubleCheckAdapter { + DoubleCheckAdapter { + live_calls: AtomicU32::new(0), + lease_held: Arc::new(AtomicBool::new(false)), + prelease_calls, + timeout_ms, + expected_deadline, + } +} + +fn run( + adapter: &DoubleCheckAdapter, + request: ActionRequest, + timeout_ms: u64, +) -> crate::action_result::ActionResult { + execute_with_auto_wait( + RefActionWaitCtx { + adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + request.with_timeout_ms(Some(timeout_ms)), + crate::ref_action::dispatch_resolved, + ) + .unwrap() +} + +fn bounds() -> crate::Rect { + crate::Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + } +} diff --git a/crates/core/src/ref_action_wait_process_state_tests.rs b/crates/core/src/ref_action_wait_process_state_tests.rs new file mode 100644 index 0000000..9fa0965 --- /dev/null +++ b/crates/core/src/ref_action_wait_process_state_tests.rs @@ -0,0 +1,400 @@ +use super::*; +use crate::{ + AdapterError, AppInfo, ErrorCode, + action::Action, + adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}, + capability, +}; +use std::sync::atomic::{AtomicU32, Ordering}; + +fn entry() -> RefEntry { + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("Original".into()), + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::snapshot_surface::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + } +} + +struct UnresponsiveProcessAdapter { + probe_calls: AtomicU32, + inventory_calls: AtomicU32, +} + +impl ObservationOps for UnresponsiveProcessAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result<Vec<AppInfo>, AdapterError> { + self.inventory_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![app("Original")]) + } + + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::stale_ref("@e1")) + } +} + +fn app(name: &str) -> AppInfo { + AppInfo { + name: name.into(), + pid: crate::ProcessId::new(1), + bundle_id: None, + process_instance: Some("test-instance".into()), + } +} + +impl ActionOps for UnresponsiveProcessAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for UnresponsiveProcessAdapter {} + +impl SystemOps for UnresponsiveProcessAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + self.probe_calls.fetch_add(1, Ordering::SeqCst); + Ok(crate::process_state::ProcessState::Unresponsive) + } +} + +#[test] +fn terminal_stale_ref_against_unresponsive_process_surfaces_app_unresponsive() { + let adapter = UnresponsiveProcessAdapter { + probe_calls: AtomicU32::new(0), + inventory_calls: AtomicU32::new(0), + }; + + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::AppUnresponsive); + assert!(err.suggestion.is_some()); + assert_eq!( + err.details.as_ref().and_then(|v| v["retryable"].as_bool()), + Some(false) + ); + assert_eq!( + err.details.as_ref().and_then(|v| v["kind"].as_str()), + Some("app_unresponsive") + ); + assert!(!err.permits_retry_by_default()); + assert_eq!( + adapter.probe_calls.load(Ordering::SeqCst), + 1, + "the liveness probe must run exactly once when building the terminal error" + ); + assert_eq!(adapter.inventory_calls.load(Ordering::SeqCst), 1); +} + +struct RecycledPidAdapter { + probe_calls: AtomicU32, + inventory_calls: AtomicU32, +} + +impl ObservationOps for RecycledPidAdapter { + fn list_apps(&self, _deadline: crate::Deadline) -> Result<Vec<AppInfo>, AdapterError> { + self.inventory_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![app("Replacement")]) + } + + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::stale_ref("@e1")) + } +} + +impl ActionOps for RecycledPidAdapter {} +impl InputOps for RecycledPidAdapter {} + +impl SystemOps for RecycledPidAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + self.probe_calls.fetch_add(1, Ordering::SeqCst); + Ok(crate::process_state::ProcessState::Unresponsive) + } +} + +#[test] +fn recycled_pid_never_upgrades_stale_ref_to_app_unresponsive() { + let adapter = RecycledPidAdapter { + probe_calls: AtomicU32::new(0), + inventory_calls: AtomicU32::new(0), + }; + + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::StaleRef); + assert_eq!(adapter.probe_calls.load(Ordering::SeqCst), 1); + assert_eq!(adapter.inventory_calls.load(Ordering::SeqCst), 1); +} + +struct ExitedProcessAdapter; + +impl ObservationOps for ExitedProcessAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::stale_ref("@e1")) + } +} + +impl ActionOps for ExitedProcessAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for ExitedProcessAdapter {} + +impl SystemOps for ExitedProcessAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + Ok(crate::process_state::ProcessState::Exited { code: None }) + } +} + +#[test] +fn terminal_stale_ref_against_exited_process_carries_process_state_detail() { + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &ExitedProcessAdapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!( + err.code, + ErrorCode::StaleRef, + "an Exited (not Unresponsive) classification must not replace the original error code" + ); + assert_eq!( + err.details.as_ref().and_then(|d| d.get("process_state")), + Some(&serde_json::json!("exited")), + "STALE_REF against a dead pid must carry details.process_state = \"exited\"" + ); +} + +struct CrashedProcessAdapter; + +impl ObservationOps for CrashedProcessAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::stale_ref("@e1")) + } +} + +impl ActionOps for CrashedProcessAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for CrashedProcessAdapter {} + +impl SystemOps for CrashedProcessAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + Ok(crate::process_state::ProcessState::Crashed { signal_or_code: 11 }) + } +} + +#[test] +fn terminal_stale_ref_against_crashed_process_carries_process_state_detail() { + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &CrashedProcessAdapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + + assert_eq!( + err.code, + ErrorCode::StaleRef, + "a Crashed (not Unresponsive) classification must not replace the original error code" + ); + assert_eq!( + err.details.as_ref().and_then(|d| d.get("process_state")), + Some(&serde_json::json!("crashed")), + "STALE_REF against a crashed pid must carry details.process_state = \"crashed\"" + ); +} + +struct SuccessWithUnresponsiveProbeAdapter { + probe_calls: AtomicU32, +} + +impl ObservationOps for SuccessWithUnresponsiveProbeAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for SuccessWithUnresponsiveProbeAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for SuccessWithUnresponsiveProbeAdapter {} + +impl SystemOps for SuccessWithUnresponsiveProbeAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn process_state( + &self, + _process: crate::ProcessIdentity, + _deadline: crate::Deadline, + ) -> Result<crate::process_state::ProcessState, AdapterError> { + self.probe_calls.fetch_add(1, Ordering::SeqCst); + Ok(crate::process_state::ProcessState::Unresponsive) + } +} + +#[test] +fn enrichment_never_converts_a_successful_action_into_a_failure() { + let adapter = SuccessWithUnresponsiveProbeAdapter { + probe_calls: AtomicU32::new(0), + }; + + let result = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap(); + assert_eq!(result.action, "click"); + assert_eq!(adapter.probe_calls.load(Ordering::SeqCst), 0); +} diff --git a/crates/core/src/ref_action_wait_support.rs b/crates/core/src/ref_action_wait_support.rs new file mode 100644 index 0000000..4deeb5f --- /dev/null +++ b/crates/core/src/ref_action_wait_support.rs @@ -0,0 +1,111 @@ +use crate::{ + AdapterError, ErrorCode, adapter::PlatformAdapter, context::CommandContext, refs::RefEntry, +}; +use serde_json::json; + +pub(crate) fn enrich_with_process_state( + adapter: &dyn PlatformAdapter, + entry: &RefEntry, + err: AdapterError, + deadline: crate::Deadline, +) -> AdapterError { + if !matches!(err.code, ErrorCode::StaleRef | ErrorCode::AppNotFound) { + return err; + } + let Some(instance) = entry + .process + .process_instance + .as_deref() + .filter(|value| !value.is_empty()) + else { + return err; + }; + let Ok(state) = adapter.process_state( + crate::ProcessIdentity::new(entry.process.pid, instance), + deadline, + ) else { + return err; + }; + if state == crate::process_state::ProcessState::Unresponsive { + if !process_identity_matches(adapter, entry, deadline) { + return err; + } + let app = entry + .source + .source_app + .as_deref() + .unwrap_or("target application"); + let unresponsive = AdapterError::app_unresponsive(app); + let mut details = match err.details { + Some(serde_json::Value::Object(details)) => details, + Some(cause) => serde_json::Map::from_iter([("cause".into(), cause)]), + None => serde_json::Map::new(), + }; + details.insert("kind".into(), json!("app_unresponsive")); + details.insert("retryable".into(), json!(false)); + return unresponsive.with_details(details.into()); + } + attach_process_state_detail(err, state) +} + +fn process_identity_matches( + adapter: &dyn PlatformAdapter, + entry: &RefEntry, + deadline: crate::Deadline, +) -> bool { + let Some(expected_name) = entry + .source + .source_app + .as_deref() + .filter(|name| !name.is_empty()) + else { + return false; + }; + let Some(expected_instance) = entry + .process + .process_instance + .as_deref() + .filter(|instance| !instance.is_empty()) + else { + return false; + }; + let Ok(apps) = adapter.list_apps(deadline) else { + return false; + }; + let mut same_pid = apps.iter().filter(|app| app.pid == entry.process.pid); + let Some(app) = same_pid.next() else { + return false; + }; + same_pid.next().is_none() + && app.name == expected_name + && app.process_instance.as_deref() == Some(expected_instance) +} + +fn attach_process_state_detail( + err: AdapterError, + state: crate::process_state::ProcessState, +) -> AdapterError { + let mut details = err.details.clone().unwrap_or_else(|| json!({})); + match details.as_object_mut() { + Some(obj) => { + obj.insert("process_state".into(), json!(state.label())); + } + None => details = json!({ "process_state": state.label() }), + } + err.with_details(details) +} + +pub(crate) fn trace_resolve_error(context: &CommandContext, ref_id: &str, err: &AdapterError) { + let _ = context.trace_lazy("ref.resolve.error", || { + json!({ + "ref": ref_id, + "code": err.code.as_str(), + "message": err.message.clone(), + "details": err.details.clone() + }) + }); +} + +pub(crate) fn trace_resolve_ok(context: &CommandContext, ref_id: &str) { + let _ = context.trace_lazy("ref.resolve.ok", || json!({ "ref": ref_id })); +} diff --git a/crates/core/src/ref_action_wait_tests.rs b/crates/core/src/ref_action_wait_tests.rs new file mode 100644 index 0000000..f8b8d8b --- /dev/null +++ b/crates/core/src/ref_action_wait_tests.rs @@ -0,0 +1,235 @@ +use super::*; +use crate::{ + AdapterError, ErrorCode, + action::Action, + adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}, + capability, +}; +use std::sync::atomic::{AtomicU32, Ordering}; + +#[test] +fn oversized_timeout_budget_is_clamped_and_never_overflows() { + assert_eq!(crate::Deadline::after(100).unwrap().timeout_ms(), 100); + assert_eq!( + crate::Deadline::after(u64::MAX).unwrap_err().code, + ErrorCode::InvalidArgs + ); +} + +#[test] +fn actionability_timeout_exposes_the_last_report_contract() { + let error = crate::ref_action_poll::timeout_with_last_report(serde_json::json!({ + "phase": "preflight" + })); + + assert_eq!( + error + .details + .as_ref() + .and_then(|details| details.get("last_report")), + Some(&serde_json::json!({ "phase": "preflight" })) + ); + assert!( + error + .details + .as_ref() + .is_none_or(|details| details.get("report").is_none()) + ); +} + +struct RetryAdapter { + resolve_calls: AtomicU32, +} + +impl ObservationOps for RetryAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + self.resolve_calls.fetch_add(1, Ordering::SeqCst); + if self.resolve_calls.load(Ordering::SeqCst) < 3 { + return Err(AdapterError::new(ErrorCode::StaleRef, "not yet") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for RetryAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for RetryAdapter {} + +impl SystemOps for RetryAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +struct AmbiguousThenOkAdapter { + resolve_calls: AtomicU32, +} + +impl ObservationOps for AmbiguousThenOkAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + self.resolve_calls.fetch_add(1, Ordering::SeqCst); + if self.resolve_calls.load(Ordering::SeqCst) == 1 { + return Err(AdapterError::ambiguous_target("2 candidates") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for AmbiguousThenOkAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for AmbiguousThenOkAdapter {} + +impl SystemOps for AmbiguousThenOkAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +fn entry() -> RefEntry { + let bounds = crate::Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }; + RefEntry { + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-1".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::snapshot_surface::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, + } +} + +fn request_with_timeout(timeout_ms: u64) -> ActionRequest { + ActionRequest::headless(Action::Click).with_timeout_ms(Some(timeout_ms)) +} + +#[test] +fn none_timeout_uses_single_resolve_attempt() { + let adapter = RetryAdapter { + resolve_calls: AtomicU32::new(0), + }; + let err = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click), + crate::ref_action::dispatch_resolved, + ) + .unwrap_err(); + assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1); + assert_eq!(err.code, ErrorCode::StaleRef); +} + +#[test] +fn budget_timeout_retries_until_success() { + let adapter = RetryAdapter { + resolve_calls: AtomicU32::new(0), + }; + let result = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + request_with_timeout(5_000), + crate::ref_action::dispatch_resolved, + ) + .unwrap(); + assert_eq!(result.action, "click"); + assert!(adapter.resolve_calls.load(Ordering::SeqCst) >= 3); +} + +#[test] +fn transient_ambiguity_is_recorded_in_result_details() { + let adapter = AmbiguousThenOkAdapter { + resolve_calls: AtomicU32::new(0), + }; + let result = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + request_with_timeout(5_000), + crate::ref_action::dispatch_resolved, + ) + .unwrap(); + assert_eq!( + result + .details + .as_ref() + .and_then(|d| d.get("transient_ambiguity")), + Some(&serde_json::json!(true)) + ); +} + +#[path = "ref_action_wait_lease_tests.rs"] +mod lease_tests; + +#[path = "ref_action_wait_unresponsive_tests.rs"] +mod unresponsive_tests; diff --git a/crates/core/src/ref_action_wait_unresponsive_tests.rs b/crates/core/src/ref_action_wait_unresponsive_tests.rs new file mode 100644 index 0000000..699114e --- /dev/null +++ b/crates/core/src/ref_action_wait_unresponsive_tests.rs @@ -0,0 +1,62 @@ +use super::*; + +struct TransientUnresponsiveAdapter { + resolve_calls: AtomicU32, +} + +impl ObservationOps for TransientUnresponsiveAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + let call = self.resolve_calls.fetch_add(1, Ordering::SeqCst); + if call == 0 { + return Err(AdapterError::app_unresponsive("Original") + .with_details(serde_json::json!({ "retryable": true }))); + } + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for TransientUnresponsiveAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<crate::action_result::ActionResult, AdapterError> { + Ok(crate::action_result::ActionResult::delivered_unverified( + "click", + )) + } +} + +impl InputOps for TransientUnresponsiveAdapter {} +impl SystemOps for TransientUnresponsiveAdapter { + crate::adapter::guarded_interaction_lease!(); +} + +#[test] +fn auto_wait_retries_a_transiently_unresponsive_accessibility_service() { + let adapter = TransientUnresponsiveAdapter { + resolve_calls: AtomicU32::new(0), + }; + + let result = execute_with_auto_wait( + RefActionWaitCtx { + adapter: &adapter, + entry: &entry(), + ref_id: "@e1", + context: &CommandContext::default(), + }, + ActionRequest::headless(Action::Click).with_timeout_ms(Some(1_000)), + crate::ref_action::dispatch_resolved, + ) + .unwrap(); + + assert_eq!(result.action, "click"); + assert!(adapter.resolve_calls.load(Ordering::SeqCst) >= 2); +} diff --git a/crates/core/src/ref_alloc.rs b/crates/core/src/ref_alloc.rs index 34851c0..e978a1b 100644 --- a/crates/core/src/ref_alloc.rs +++ b/crates/core/src/ref_alloc.rs @@ -1,39 +1,56 @@ -use crate::adapter::SnapshotSurface; -use crate::node::AccessibilityNode; +use crate::AccessibilityNode; use crate::refs::{RefEntry, RefMap}; +pub(crate) use crate::ref_alloc_config::RefAllocConfig; + pub(crate) use crate::roles::INTERACTIVE_ROLES; pub(crate) fn ref_entry_from_node( node: &AccessibilityNode, - pid: i32, - source_app: Option<&str>, - source_window_id: Option<&str>, - source_window_title: Option<&str>, + source: &crate::ref_alloc_source::RefAllocSource<'_>, root_ref: Option<String>, path: &[usize], ) -> RefEntry { + let bounds = node + .presentation + .bounds + .filter(|bounds| bounds.validate().is_ok()); RefEntry { - pid, - role: node.role.clone(), - name: meaningful_string(node.name.clone()), - value: meaningful_string(node.value.clone()), - description: meaningful_string(node.description.clone()), - states: node.states.clone(), - bounds: node.bounds, - bounds_hash: node.bounds.as_ref().map(|b| b.bounds_hash()), - available_actions: if node.available_actions.is_empty() { - crate::capability::defaults_for_role(&node.role) - } else { - node.available_actions.clone() + process: crate::RefProcess { + pid: source.pid, + process_instance: source.process_instance.map(str::to_string), + }, + identity: crate::RefEntryIdentity { + role: node.role.clone(), + name: meaningful_string(node.identity.name.clone()), + value: meaningful_string(node.identity.value.clone()), + description: meaningful_string(node.identity.description.clone()), + native_id: node + .identity + .native_id + .clone() + .filter(|id| !id.value.trim().is_empty()), + }, + geometry: crate::RefGeometry { + bounds, + bounds_hash: bounds.and_then(|bounds| bounds.bounds_hash()), + }, + capabilities: crate::RefCapabilities { + states: node.presentation.states.clone(), + available_actions: node.presentation.available_actions.clone(), + }, + source: crate::RefSource { + source_app: source.app.map(str::to_string), + source_window_id: source.window_id.map(str::to_string), + source_window_title: source.window_title.map(str::to_string), + source_window_bounds_hash: source.window_bounds_hash, + source_surface: source.surface, + }, + scope: crate::RefScope { + root_ref, + path_is_absolute: false, + path: smallvec::SmallVec::from_slice(path), }, - source_app: source_app.map(str::to_string), - source_window_id: source_window_id.map(str::to_string), - source_window_title: source_window_title.map(str::to_string), - source_surface: SnapshotSurface::Window, - root_ref, - path_is_absolute: false, - path: smallvec::SmallVec::from_slice(path), } } @@ -42,25 +59,52 @@ pub(crate) fn ref_entry_from_node( /// role. Container roles like `scrollarea` (Scroll) and `disclosure` /// (Expand/Collapse) are not "interactive" by role but are genuinely /// actionable, and `scroll` / `expand` / `collapse` need a ref to target -/// them — so action-bearing elements must be ref-able. A bare `SetFocus` -/// affordance does not qualify on its own: focusability is not a primary -/// action and would ref-allocate large numbers of inert containers. +/// them — so action-bearing elements must be ref-able even when their current +/// bounds are zero-sized. Visibility remains a live actionability concern. A +/// bare `SetFocus` affordance does not qualify on its own: focusability is not +/// a primary action and would ref-allocate large numbers of inert containers. pub(crate) fn is_ref_able(node: &AccessibilityNode) -> bool { - INTERACTIVE_ROLES.contains(&node.role.as_str()) || advertises_primary_action(node) + is_ref_able_role_actions(&node.role, &node.presentation.available_actions) } -fn advertises_primary_action(node: &AccessibilityNode) -> bool { - node.available_actions +pub(crate) fn is_ref_able_role_actions(role: &str, available_actions: &[String]) -> bool { + INTERACTIVE_ROLES.contains(&role) || advertises_primary_action(available_actions) +} + +fn advertises_primary_action(available_actions: &[String]) -> bool { + available_actions .iter() .any(|action| action != crate::capability::SET_FOCUS) } pub(crate) fn is_collapsible(node: &AccessibilityNode) -> bool { - node.ref_id.is_none() - && node.name.as_deref().is_none_or(str::is_empty) - && node.value.as_deref().is_none_or(str::is_empty) - && node.description.as_deref().is_none_or(str::is_empty) - && node.states.is_empty() + crate::Role::from_token(&node.role).is_transparent_wrapper() + && node.ref_id.is_none() + && node + .identity + .name + .as_deref() + .is_none_or(|value| value.trim().is_empty()) + && node + .identity + .value + .as_deref() + .is_none_or(|value| value.trim().is_empty()) + && node + .identity + .description + .as_deref() + .is_none_or(|value| value.trim().is_empty()) + && node.identity.native_id.is_none() + && node + .presentation + .hint + .as_deref() + .is_none_or(|value| value.trim().is_empty()) + && node.presentation.states.is_empty() + && node.presentation.available_actions.is_empty() + && node.presentation.bounds.is_none() + && node.children_count.is_none() && node.children.len() == 1 } @@ -82,20 +126,17 @@ pub fn transform_tree( interactive_only: bool, compact: bool, ) -> AccessibilityNode { - if !include_bounds { - node.bounds = None; - } - node.children = node .children .into_iter() .filter_map(|child| { + let collapsible = compact && is_collapsible(&child); let child = transform_tree(child, include_bounds, interactive_only, compact); - if compact && is_collapsible(&child) { + if collapsible { return child.children.into_iter().next(); } if interactive_only - && !INTERACTIVE_ROLES.contains(&child.role.as_str()) + && !is_ref_able(&child) && child.children.is_empty() && child.children_count.is_none() { @@ -106,28 +147,19 @@ pub fn transform_tree( }) .collect(); - node -} + if !include_bounds { + node.presentation.bounds = None; + } -pub(crate) struct RefAllocConfig<'a> { - pub include_bounds: bool, - pub interactive_only: bool, - pub compact: bool, - pub pid: i32, - pub source_app: Option<&'a str>, - pub source_window_id: Option<&'a str>, - pub source_window_title: Option<&'a str>, - pub source_surface: SnapshotSurface, - pub root_ref_id: Option<&'a str>, - pub path_prefix: &'a [usize], + node } pub(crate) fn allocate_refs( node: AccessibilityNode, refmap: &mut RefMap, config: &RefAllocConfig, -) -> AccessibilityNode { - allocate_refs_at_path(node, refmap, config, &mut config.path_prefix.to_vec()) +) -> Result<AccessibilityNode, crate::AppError> { + allocate_refs_at_path(node, refmap, config, &mut config.scope.path_prefix.to_vec()) } fn allocate_refs_at_path( @@ -135,79 +167,87 @@ fn allocate_refs_at_path( refmap: &mut RefMap, config: &RefAllocConfig, path: &mut Vec<usize>, -) -> AccessibilityNode { - let is_ref_able = is_ref_able(&node); +) -> Result<AccessibilityNode, crate::AppError> { + let node_is_ref_able = is_ref_able(&node); - if is_ref_able { + if node_is_ref_able { let mut entry = ref_entry_from_node( &node, - config.pid, - config.source_app, - config.source_window_id, - config.source_window_title, - config.root_ref_id.map(str::to_string), + &config.source, + config.scope.root_ref_id.map(str::to_string), path, ); - entry.source_surface = config.source_surface; - entry.path_is_absolute = config.root_ref_id.is_some(); - if !config.include_bounds { - entry.bounds = None; + entry.scope.path_is_absolute = config.scope.root_ref_id.is_some(); + if !config.options.include_bounds { + entry.geometry.bounds = None; } - node.ref_id = Some(refmap.allocate(entry)); + node.ref_id = allocate_observed_ref(refmap, entry)?; } - let has_label = node.name.as_deref().is_some_and(|n| !n.is_empty()) - || node.description.as_deref().is_some_and(|d| !d.is_empty()); - let is_skeleton_anchor = - !is_ref_able && node.children_count.is_some() && has_label && config.root_ref_id.is_none(); + let has_label = node + .identity + .name + .as_deref() + .is_some_and(|name| !name.is_empty()) + || node + .identity + .description + .as_deref() + .is_some_and(|description| !description.is_empty()); + let is_skeleton_anchor = !node_is_ref_able + && node.children_count.is_some() + && has_label + && config.scope.root_ref_id.is_none(); if is_skeleton_anchor { - let mut entry = ref_entry_from_node( - &node, - config.pid, - config.source_app, - config.source_window_id, - config.source_window_title, - None, - path, - ); - entry.source_surface = config.source_surface; - entry.available_actions = vec![]; - if !config.include_bounds { - entry.bounds = None; + let mut entry = ref_entry_from_node(&node, &config.source, None, path); + entry.capabilities.available_actions = vec![]; + if !config.options.include_bounds { + entry.geometry.bounds = None; } - node.ref_id = Some(refmap.allocate(entry)); + node.ref_id = allocate_observed_ref(refmap, entry)?; } - if !config.include_bounds { - node.bounds = None; + if !config.options.include_bounds { + node.presentation.bounds = None; } - node.children = node - .children - .into_iter() - .enumerate() - .filter_map(|child| { - let (idx, child) = child; - path.push(idx); - let child = allocate_refs_at_path(child, refmap, config, path); - path.pop(); - if config.compact && is_collapsible(&child) { - return child.children.into_iter().next(); + let mut children = Vec::new(); + for (idx, child) in node.children.into_iter().enumerate() { + let collapsible = config.options.compact && is_collapsible(&child); + path.push(idx); + let child = allocate_refs_at_path(child, refmap, config, path)?; + path.pop(); + if collapsible { + if let Some(child) = child.children.into_iter().next() { + children.push(child); } - if config.interactive_only - && child.ref_id.is_none() - && child.children.is_empty() - && child.children_count.is_none() - { - None - } else { - Some(child) - } - }) - .collect(); + continue; + } + if config.options.interactive_only + && child.ref_id.is_none() + && !is_ref_able(&child) + && child.children.is_empty() + && child.children_count.is_none() + { + continue; + } + children.push(child); + } + node.children = children; - node + Ok(node) +} + +fn allocate_observed_ref( + refmap: &mut RefMap, + entry: RefEntry, +) -> Result<Option<String>, crate::AppError> { + match refmap.try_allocate_observed(entry)? { + crate::ref_allocation::RefAllocation::Allocated(ref_id) => Ok(Some(ref_id)), + crate::ref_allocation::RefAllocation::SkippedInvalidRole + | crate::ref_allocation::RefAllocation::SkippedInvalidEntry => Ok(None), + } } fn meaningful_string(value: Option<String>) -> Option<String> { diff --git a/crates/core/src/ref_alloc_config.rs b/crates/core/src/ref_alloc_config.rs new file mode 100644 index 0000000..43ac2d5 --- /dev/null +++ b/crates/core/src/ref_alloc_config.rs @@ -0,0 +1,11 @@ +use crate::{ + ref_alloc_options::RefAllocOptions, ref_alloc_scope::RefAllocScope, + ref_alloc_source::RefAllocSource, +}; + +#[derive(Clone, Copy)] +pub(crate) struct RefAllocConfig<'a> { + pub(crate) options: RefAllocOptions, + pub(crate) source: RefAllocSource<'a>, + pub(crate) scope: RefAllocScope<'a>, +} diff --git a/crates/core/src/ref_alloc_options.rs b/crates/core/src/ref_alloc_options.rs new file mode 100644 index 0000000..d2026cf --- /dev/null +++ b/crates/core/src/ref_alloc_options.rs @@ -0,0 +1,6 @@ +#[derive(Clone, Copy)] +pub(crate) struct RefAllocOptions { + pub(crate) include_bounds: bool, + pub(crate) interactive_only: bool, + pub(crate) compact: bool, +} diff --git a/crates/core/src/ref_alloc_ordering_tests.rs b/crates/core/src/ref_alloc_ordering_tests.rs new file mode 100644 index 0000000..3d2309b --- /dev/null +++ b/crates/core/src/ref_alloc_ordering_tests.rs @@ -0,0 +1,215 @@ +use super::*; + +/// Refs must be assigned in depth-first document order. +/// Given: window → [button("A"), group → [button("B"), button("C")]], +/// DFS visits A then B then C, so @e1=A, @e2=B, @e3=C. +/// A regression that allocates in BFS order (A, then skipping group to find B +/// last) would violate the contract the CLI documents and agents depend on. +#[test] +fn allocate_refs_assigns_refs_in_depth_first_order() { + let btn_a = node("button", Some("A")); + let btn_b = node("button", Some("B")); + let btn_c = node("button", Some("C")); + let mut group = node("group", None); + group.children = vec![btn_b, btn_c]; + let mut root = node("window", Some("w")); + root.children = vec![btn_a, group]; + + let mut refmap = RefMap::new(); + let config = RefAllocConfig { + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: false, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(1), + app: None, + window_id: None, + window_title: None, + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, + }; + let out = allocate_refs(root, &mut refmap, &config).unwrap(); + + let a_ref = out.children[0].ref_id.as_deref().unwrap(); + let b_ref = out.children[1].children[0].ref_id.as_deref().unwrap(); + let c_ref = out.children[1].children[1].ref_id.as_deref().unwrap(); + + assert_eq!(a_ref, "@e1", "first DFS interactive node must be @e1"); + assert_eq!(b_ref, "@e2", "second DFS interactive node must be @e2"); + assert_eq!(c_ref, "@e3", "third DFS interactive node must be @e3"); +} + +/// A node whose available_actions list contains SetFocus alongside a real +/// primary action must be ref-able, because advertises_primary_action +/// filters to actions that are not SetFocus. +#[test] +fn node_with_primary_action_alongside_set_focus_is_ref_able() { + let mut panel = node("group", Some("Panel")); + panel.presentation.available_actions = vec!["SetFocus".into(), "Scroll".into()]; + assert!( + is_ref_able(&panel), + "group with SetFocus+Scroll must be ref-able via the primary action path" + ); +} + +/// Each role in the hardcoded list must be ref-able by role alone (no actions +/// needed). Using a literal list rather than iterating INTERACTIVE_ROLES means +/// removing any of these from the constant will actually fail this test. +#[test] +fn representative_interactive_roles_are_ref_able_by_role_alone() { + for role in [ + "button", + "textfield", + "checkbox", + "link", + "slider", + "combobox", + "treeitem", + "cell", + "radiobutton", + "tab", + "menuitem", + "switch", + "colorwell", + "menubutton", + "incrementor", + "dockitem", + ] { + let n = node(role, None); + assert!( + is_ref_able(&n), + "'{role}' must be ref-able by role alone with no available_actions" + ); + } +} + +#[test] +fn allocate_refs_keeps_bounds_in_refmap_when_snapshot_includes_bounds() { + let mut root = node("window", Some("w")); + root.children = vec![node("button", Some("Open"))]; + let mut refmap = RefMap::new(); + let config = RefAllocConfig { + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: true, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(7), + app: Some("Finder"), + window_id: Some("w-42"), + window_title: Some("Documents"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, + }; + + let out = allocate_refs(root, &mut refmap, &config).unwrap(); + let open_ref = out.children[0].ref_id.as_deref().unwrap(); + let entry = refmap.get(open_ref).unwrap(); + + assert!(out.children[0].presentation.bounds.is_some()); + assert!(entry.geometry.bounds.is_some()); + assert!(entry.geometry.bounds_hash.is_some()); +} + +#[test] +fn actionable_unknown_role_keeps_node_without_consuming_ref_id() { + let first = node("button", Some("First")); + let mut unknown = node("unknown", Some("Custom control")); + unknown.presentation.available_actions = vec!["Click".into()]; + let last = node("button", Some("Last")); + let mut root = node("window", Some("w")); + root.children = vec![first, unknown, last]; + let mut refmap = RefMap::new(); + let mut config = allocation_config(true); + config.options.interactive_only = true; + + let out = allocate_refs(root, &mut refmap, &config).unwrap(); + + assert_eq!(out.children.len(), 3); + assert_eq!(out.children[0].ref_id.as_deref(), Some("@e1")); + assert!(out.children[1].ref_id.is_none()); + assert_eq!(out.children[1].role, "unknown"); + assert_eq!(out.children[2].ref_id.as_deref(), Some("@e2")); + assert_eq!(refmap.len(), 2); +} + +#[test] +fn out_of_range_bounds_stay_in_tree_but_not_ref_geometry() { + let mut bounded = node("button", Some("Far away")); + bounded.presentation.bounds = Some(Rect { + x: 10_000_001.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + let mut root = node("window", Some("w")); + root.children = vec![bounded]; + let mut refmap = RefMap::new(); + + let out = allocate_refs(root, &mut refmap, &allocation_config(true)).unwrap(); + + let child = &out.children[0]; + assert_eq!( + child.presentation.bounds.map(|bounds| bounds.x), + Some(10_000_001.0) + ); + let entry = refmap.get(child.ref_id.as_deref().unwrap()).unwrap(); + assert!(entry.geometry.bounds.is_none()); + assert!(entry.geometry.bounds_hash.is_none()); +} + +#[test] +fn structural_allocation_failure_still_aborts_snapshot() { + let mut refmap: RefMap = serde_json::from_value(serde_json::json!({ + "inner": {}, + "counter": u32::MAX + })) + .unwrap(); + let mut root = node("window", Some("w")); + root.children = vec![node("button", Some("First"))]; + + let result = allocate_refs(root, &mut refmap, &allocation_config(true)); + + let Err(error) = result else { + panic!("identifier exhaustion must remain terminal"); + }; + assert!(error.to_string().contains("identifier space")); +} + +fn allocation_config(include_bounds: bool) -> RefAllocConfig<'static> { + RefAllocConfig { + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(7), + app: Some("Finder"), + window_id: Some("w-42"), + window_title: Some("Documents"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, + } +} diff --git a/crates/core/src/ref_alloc_scope.rs b/crates/core/src/ref_alloc_scope.rs new file mode 100644 index 0000000..dc27e08 --- /dev/null +++ b/crates/core/src/ref_alloc_scope.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy)] +pub(crate) struct RefAllocScope<'a> { + pub(crate) root_ref_id: Option<&'a str>, + pub(crate) path_prefix: &'a [usize], +} diff --git a/crates/core/src/ref_alloc_source.rs b/crates/core/src/ref_alloc_source.rs new file mode 100644 index 0000000..597752b --- /dev/null +++ b/crates/core/src/ref_alloc_source.rs @@ -0,0 +1,12 @@ +use crate::{ProcessId, SnapshotSurface}; + +#[derive(Clone, Copy)] +pub(crate) struct RefAllocSource<'a> { + pub(crate) pid: ProcessId, + pub(crate) app: Option<&'a str>, + pub(crate) window_id: Option<&'a str>, + pub(crate) window_title: Option<&'a str>, + pub(crate) window_bounds_hash: Option<u64>, + pub(crate) process_instance: Option<&'a str>, + pub(crate) surface: SnapshotSurface, +} diff --git a/crates/core/src/ref_alloc_tests.rs b/crates/core/src/ref_alloc_tests.rs index 37c8442..937efc4 100644 --- a/crates/core/src/ref_alloc_tests.rs +++ b/crates/core/src/ref_alloc_tests.rs @@ -1,39 +1,52 @@ use super::*; -use crate::node::{AccessibilityNode, Rect}; +use crate::{AccessibilityNode, Rect}; fn node(role: &str, name: Option<&str>) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: role.into(), - name: name.map(str::to_string), - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: Some(Rect { - x: 0.0, - y: 0.0, - width: 10.0, - height: 10.0, - }), + identity: crate::NodeIdentity { + name: name.map(str::to_string), + ..Default::default() + }, + presentation: crate::NodePresentation { + bounds: Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }), + ..Default::default() + }, children_count: None, children: vec![], } } +fn source(pid: u32) -> crate::ref_alloc_source::RefAllocSource<'static> { + crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(pid), + app: None, + window_id: None, + window_title: None, + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + } +} + #[test] fn transform_tree_include_bounds_false_strips_bounds() { let n = node("group", None); let out = transform_tree(n, false, false, false); - assert!(out.bounds.is_none()); + assert!(out.presentation.bounds.is_none()); } #[test] fn transform_tree_include_bounds_true_preserves_bounds() { let n = node("group", None); let out = transform_tree(n, true, false, false); - assert!(out.bounds.is_some()); + assert!(out.presentation.bounds.is_some()); } #[test] @@ -60,6 +73,8 @@ fn transform_tree_interactive_only_keeps_named_containers_with_children() { fn transform_tree_compact_collapses_empty_single_child_chain() { let mut outer = node("group", None); let mut inner = node("group", None); + outer.presentation.bounds = None; + inner.presentation.bounds = None; inner.children = vec![node("button", Some("Go"))]; outer.children = vec![inner]; let mut root = node("window", Some("w")); @@ -69,6 +84,66 @@ fn transform_tree_compact_collapses_empty_single_child_chain() { assert_eq!(out.children[0].role, "button"); } +#[test] +fn compact_collapses_payload_free_chromium_group_wrapper() { + let mut wrapper = node("group", None); + wrapper.presentation.bounds = None; + wrapper.children = vec![node("button", Some("Send"))]; + assert!(is_collapsible(&wrapper)); +} + +#[test] +fn compact_preserves_semantic_container_roles() { + for role in [ + "webarea", + "banner", + "navigation", + "main", + "region", + "table", + "list", + ] { + let mut container = node(role, None); + container.presentation.bounds = None; + container.children = vec![node("button", Some("Child"))]; + assert!(!is_collapsible(&container), "{role} must retain its role"); + } +} + +#[test] +fn compact_preserves_group_identity_truncation_and_capabilities() { + let mut identified = transparent_group(); + identified.identity.native_id = Some(crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: "renderer-node".into(), + }); + assert!(!is_collapsible(&identified)); + + let mut truncated = transparent_group(); + truncated.children_count = Some(17); + assert!(!is_collapsible(&truncated)); + + let mut actionable = transparent_group(); + actionable.presentation.available_actions = vec![crate::capability::SET_FOCUS.into()]; + assert!(!is_collapsible(&actionable)); + + let mut bounded = transparent_group(); + bounded.presentation.bounds = Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + assert!(!is_collapsible(&bounded)); +} + +fn transparent_group() -> AccessibilityNode { + let mut group = node("group", None); + group.presentation.bounds = None; + group.children = vec![node("button", Some("Child"))]; + group +} + #[test] fn transform_tree_compact_preserves_labeled_containers() { let mut named = node("group", Some("Toolbar")); @@ -78,41 +153,44 @@ fn transform_tree_compact_preserves_labeled_containers() { let out = transform_tree(root, true, false, true); assert_eq!(out.children.len(), 1); assert_eq!(out.children[0].role, "group"); - assert_eq!(out.children[0].name.as_deref(), Some("Toolbar")); + assert_eq!(out.children[0].identity.name.as_deref(), Some("Toolbar")); } #[test] fn ref_entry_prefers_platform_actions() { let mut button = node("button", Some("Save")); - button.available_actions = vec!["SetFocus".into()]; + button.presentation.available_actions = vec!["SetFocus".into()]; - let entry = ref_entry_from_node(&button, 7, None, None, None, None, &[0]); + let entry = ref_entry_from_node(&button, &source(7), None, &[0]); - assert_eq!(entry.available_actions, vec!["SetFocus"]); + assert_eq!(entry.capabilities.available_actions, vec!["SetFocus"]); } #[test] fn ref_entry_drops_empty_identity_text() { let mut button = node("button", Some("")); - button.value = Some(String::new()); + button.identity.value = Some(String::new()); - let entry = ref_entry_from_node(&button, 7, None, None, None, None, &[0]); + let entry = ref_entry_from_node(&button, &source(7), None, &[0]); - assert!(entry.name.is_none()); - assert!(entry.value.is_none()); + assert!(entry.identity.name.is_none()); + assert!(entry.identity.value.is_none()); } #[test] fn ref_entry_preserves_meaningful_identity_text() { let mut button = node("button", Some("Save")); - button.value = Some("Primary".into()); - button.description = Some("Commits changes".into()); + button.identity.value = Some("Primary".into()); + button.identity.description = Some("Commits changes".into()); - let entry = ref_entry_from_node(&button, 7, None, None, None, None, &[0]); + let entry = ref_entry_from_node(&button, &source(7), None, &[0]); - assert_eq!(entry.name.as_deref(), Some("Save")); - assert_eq!(entry.value.as_deref(), Some("Primary")); - assert_eq!(entry.description.as_deref(), Some("Commits changes")); + assert_eq!(entry.identity.name.as_deref(), Some("Save")); + assert_eq!(entry.identity.value.as_deref(), Some("Primary")); + assert_eq!( + entry.identity.description.as_deref(), + Some("Commits changes") + ); } /// scrollarea/disclosure are not interactive roles, but they advertise real @@ -120,11 +198,11 @@ fn ref_entry_preserves_meaningful_identity_text() { #[test] fn actionable_container_roles_receive_refs() { let mut scroll = node("scrollarea", Some("Log")); - scroll.available_actions = vec!["Scroll".into()]; + scroll.presentation.available_actions = vec!["Scroll".into()]; assert!(is_ref_able(&scroll)); let mut disclosure = node("disclosure", Some("Details")); - disclosure.available_actions = vec!["Click".into()]; + disclosure.presentation.available_actions = vec!["Click".into()]; assert!(is_ref_able(&disclosure)); } @@ -133,7 +211,7 @@ fn actionable_container_roles_receive_refs() { #[test] fn focus_only_container_does_not_receive_a_ref() { let mut group = node("group", Some("Panel")); - group.available_actions = vec!["SetFocus".into()]; + group.presentation.available_actions = vec!["SetFocus".into()]; assert!(!is_ref_able(&group)); let inert = node("statictext", Some("Label")); @@ -155,25 +233,38 @@ fn allocate_refs_records_structural_paths() { let mut refmap = RefMap::new(); let config = RefAllocConfig { - include_bounds: true, - interactive_only: false, - compact: false, - pid: 7, - source_app: Some("Finder"), - source_window_id: Some("w-42"), - source_window_title: Some("Documents"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: None, - path_prefix: &[], + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: true, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(7), + app: Some("Finder"), + window_id: Some("w-42"), + window_title: Some("Documents"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, }; - let out = allocate_refs(root, &mut refmap, &config); + let out = allocate_refs(root, &mut refmap, &config).unwrap(); let save_ref = out.children[0].ref_id.as_deref().unwrap(); let open_ref = out.children[1].children[0].ref_id.as_deref().unwrap(); - assert_eq!(refmap.get(save_ref).unwrap().path.as_slice(), [0]); - assert_eq!(refmap.get(open_ref).unwrap().path.as_slice(), [1, 0]); + assert_eq!(refmap.get(save_ref).unwrap().scope.path.as_slice(), [0]); + assert_eq!(refmap.get(open_ref).unwrap().scope.path.as_slice(), [1, 0]); assert_eq!( - refmap.get(open_ref).unwrap().source_window_id.as_deref(), + refmap + .get(open_ref) + .unwrap() + .source + .source_window_id + .as_deref(), Some("w-42") ); } @@ -184,28 +275,39 @@ fn allocate_refs_keeps_bounds_hash_when_snapshot_hides_bounds() { root.children = vec![node("button", Some("Open"))]; let mut refmap = RefMap::new(); let config = RefAllocConfig { - include_bounds: false, - interactive_only: false, - compact: false, - pid: 7, - source_app: Some("Finder"), - source_window_id: Some("w-42"), - source_window_title: Some("Documents"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: None, - path_prefix: &[], + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: false, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(7), + app: Some("Finder"), + window_id: Some("w-42"), + window_title: Some("Documents"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, }; - let out = allocate_refs(root, &mut refmap, &config); + let out = allocate_refs(root, &mut refmap, &config).unwrap(); let open_ref = out.children[0].ref_id.as_deref().unwrap(); let entry = refmap.get(open_ref).unwrap(); - assert!(out.children[0].bounds.is_none()); - assert!(entry.bounds.is_none()); - assert_eq!(entry.bounds_hash, Some(entry_hash())); - assert_eq!(entry.path.as_slice(), [0]); - assert_eq!(entry.source_window_id.as_deref(), Some("w-42")); - assert_eq!(entry.source_window_title.as_deref(), Some("Documents")); + assert!(out.children[0].presentation.bounds.is_none()); + assert!(entry.geometry.bounds.is_none()); + assert_eq!(entry.geometry.bounds_hash, Some(entry_hash())); + assert_eq!(entry.scope.path.as_slice(), [0]); + assert_eq!(entry.source.source_window_id.as_deref(), Some("w-42")); + assert_eq!( + entry.source.source_window_title.as_deref(), + Some("Documents") + ); } fn entry_hash() -> u64 { @@ -216,114 +318,8 @@ fn entry_hash() -> u64 { height: 10.0, } .bounds_hash() + .unwrap() } -/// Refs must be assigned in depth-first document order. -/// Given: window → [button("A"), group → [button("B"), button("C")]], -/// DFS visits A then B then C, so @e1=A, @e2=B, @e3=C. -/// A regression that allocates in BFS order (A, then skipping group to find B -/// last) would violate the contract the CLI documents and agents depend on. -#[test] -fn allocate_refs_assigns_refs_in_depth_first_order() { - let btn_a = node("button", Some("A")); - let btn_b = node("button", Some("B")); - let btn_c = node("button", Some("C")); - let mut group = node("group", None); - group.children = vec![btn_b, btn_c]; - let mut root = node("window", Some("w")); - root.children = vec![btn_a, group]; - - let mut refmap = RefMap::new(); - let config = RefAllocConfig { - include_bounds: false, - interactive_only: false, - compact: false, - pid: 1, - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: None, - path_prefix: &[], - }; - let out = allocate_refs(root, &mut refmap, &config); - - let a_ref = out.children[0].ref_id.as_deref().unwrap(); - let b_ref = out.children[1].children[0].ref_id.as_deref().unwrap(); - let c_ref = out.children[1].children[1].ref_id.as_deref().unwrap(); - - assert_eq!(a_ref, "@e1", "first DFS interactive node must be @e1"); - assert_eq!(b_ref, "@e2", "second DFS interactive node must be @e2"); - assert_eq!(c_ref, "@e3", "third DFS interactive node must be @e3"); -} - -/// A node whose available_actions list contains SetFocus alongside a real -/// primary action must be ref-able, because advertises_primary_action -/// filters to actions that are not SetFocus. -#[test] -fn node_with_primary_action_alongside_set_focus_is_ref_able() { - let mut panel = node("group", Some("Panel")); - panel.available_actions = vec!["SetFocus".into(), "Scroll".into()]; - assert!( - is_ref_able(&panel), - "group with SetFocus+Scroll must be ref-able via the primary action path" - ); -} - -/// Each role in the hardcoded list must be ref-able by role alone (no actions -/// needed). Using a literal list rather than iterating INTERACTIVE_ROLES means -/// removing any of these from the constant will actually fail this test. -#[test] -fn representative_interactive_roles_are_ref_able_by_role_alone() { - for role in [ - "button", - "textfield", - "checkbox", - "link", - "slider", - "combobox", - "treeitem", - "cell", - "radiobutton", - "tab", - "menuitem", - "switch", - "colorwell", - "menubutton", - "incrementor", - "dockitem", - ] { - let n = node(role, None); - assert!( - is_ref_able(&n), - "'{role}' must be ref-able by role alone with no available_actions" - ); - } -} - -#[test] -fn allocate_refs_keeps_bounds_in_refmap_when_snapshot_includes_bounds() { - let mut root = node("window", Some("w")); - root.children = vec![node("button", Some("Open"))]; - let mut refmap = RefMap::new(); - let config = RefAllocConfig { - include_bounds: true, - interactive_only: false, - compact: false, - pid: 7, - source_app: Some("Finder"), - source_window_id: Some("w-42"), - source_window_title: Some("Documents"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: None, - path_prefix: &[], - }; - - let out = allocate_refs(root, &mut refmap, &config); - let open_ref = out.children[0].ref_id.as_deref().unwrap(); - let entry = refmap.get(open_ref).unwrap(); - - assert!(out.children[0].bounds.is_some()); - assert!(entry.bounds.is_some()); - assert!(entry.bounds_hash.is_some()); -} +#[path = "ref_alloc_ordering_tests.rs"] +mod ordering_tests; diff --git a/crates/core/src/ref_allocation.rs b/crates/core/src/ref_allocation.rs new file mode 100644 index 0000000..39fe274 --- /dev/null +++ b/crates/core/src/ref_allocation.rs @@ -0,0 +1,5 @@ +pub(crate) enum RefAllocation { + Allocated(String), + SkippedInvalidRole, + SkippedInvalidEntry, +} diff --git a/crates/core/src/ref_capabilities.rs b/crates/core/src/ref_capabilities.rs new file mode 100644 index 0000000..9ec884b --- /dev/null +++ b/crates/core/src/ref_capabilities.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefCapabilities { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub states: Vec<String>, + pub available_actions: Vec<String>, +} diff --git a/crates/core/src/ref_entry.rs b/crates/core/src/ref_entry.rs new file mode 100644 index 0000000..03928c2 --- /dev/null +++ b/crates/core/src/ref_entry.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefEntry { + #[serde(flatten)] + pub process: crate::RefProcess, + #[serde(flatten)] + pub identity: crate::RefEntryIdentity, + #[serde(flatten)] + pub geometry: crate::RefGeometry, + #[serde(flatten)] + pub capabilities: crate::RefCapabilities, + #[serde(flatten)] + pub source: crate::RefSource, + #[serde(flatten)] + pub scope: crate::RefScope, +} diff --git a/crates/core/src/ref_entry_identity.rs b/crates/core/src/ref_entry_identity.rs new file mode 100644 index 0000000..8e3c29c --- /dev/null +++ b/crates/core/src/ref_entry_identity.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefEntryIdentity { + pub role: String, + pub name: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_id: Option<crate::ElementIdentifier>, +} diff --git a/crates/core/src/ref_geometry.rs b/crates/core/src/ref_geometry.rs new file mode 100644 index 0000000..413a543 --- /dev/null +++ b/crates/core/src/ref_geometry.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefGeometry { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bounds: Option<crate::Rect>, + pub bounds_hash: Option<u64>, +} diff --git a/crates/core/src/ref_identity.rs b/crates/core/src/ref_identity.rs index 46f3b9d..320619d 100644 --- a/crates/core/src/ref_identity.rs +++ b/crates/core/src/ref_identity.rs @@ -1,35 +1,110 @@ -use crate::{adapter::SnapshotSurface, refs::RefEntry, roles::is_mutable_value_role}; +use crate::{ + IdentityMatch, + live_locator::{IdentifierEvidence, LocatorField}, + refs::RefEntry, + roles::is_mutable_value_role, +}; -/// Returns true when a saved ref has stable text identity beyond role/path/bounds. pub fn has_meaningful_identity(entry: &RefEntry) -> bool { - stable_name( - entry.role.as_str(), - entry.name.as_deref(), - entry.value.as_deref(), - ) - .is_some() - || stable_value(entry.role.as_str(), entry.value.as_deref()).is_some() - || meaningful_text(entry.description.as_deref()).is_some() + entry + .identity + .native_id + .as_ref() + .is_some_and(|identifier| meaningful_text(Some(&identifier.value)).is_some()) + || has_stable_text_identity(entry) } -/// Compares saved ref identity against live text without treating mutable -/// control values as stable identity. -pub fn identity_matches( +pub fn has_stable_text_identity(entry: &RefEntry) -> bool { + stable_name( + entry.identity.role.as_str(), + entry.identity.name.as_deref(), + entry.identity.value.as_deref(), + ) + .is_some() + || stable_value( + entry.identity.role.as_str(), + entry.identity.value.as_deref(), + ) + .is_some() + || meaningful_text(entry.identity.description.as_deref()).is_some() +} + +pub fn identity_match( + entry: &RefEntry, + actual_name: &LocatorField<String>, + actual_value: &LocatorField<String>, + actual_description: &LocatorField<String>, + actual_identifiers: &IdentifierEvidence, +) -> IdentityMatch { + if let Some(expected) = entry.identity.native_id.as_ref() { + let Some(expected_value) = meaningful_text(Some(&expected.value)) else { + return IdentityMatch::Unknown; + }; + if actual_identifiers + .identifiers() + .iter() + .any(|actual| actual.kind == expected.kind && actual.value == expected_value) + { + return IdentityMatch::Match; + } + if !actual_identifiers.is_complete() { + return IdentityMatch::Unknown; + } + return IdentityMatch::NoMatch; + } + + stable_text_match(entry, actual_name, actual_value, actual_description) +} + +#[cfg(test)] +pub(crate) fn identity_matches( entry: &RefEntry, actual_name: Option<&str>, actual_value: Option<&str>, actual_description: Option<&str>, + actual_native_id: Option<&str>, ) -> bool { - let expected_name = stable_name( - entry.role.as_str(), - entry.name.as_deref(), - entry.value.as_deref(), + let actual_name = option_field(actual_name); + let actual_value = option_field(actual_value); + let actual_description = option_field(actual_description); + let identifiers = IdentifierEvidence::typed( + actual_native_id + .into_iter() + .map(|value| crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: value.to_string(), + }), + actual_native_id.map(|_| 0), + true, ); - let expected_value = stable_value(entry.role.as_str(), entry.value.as_deref()); - let expected_description = meaningful_text(entry.description.as_deref()); - let actual_name = stable_name(entry.role.as_str(), actual_name, actual_value); - let actual_value = stable_value(entry.role.as_str(), actual_value); - let actual_description = meaningful_text(actual_description); + identity_match( + entry, + &actual_name, + &actual_value, + &actual_description, + &identifiers, + ) == IdentityMatch::Match +} + +fn stable_text_match( + entry: &RefEntry, + actual_name: &LocatorField<String>, + actual_value: &LocatorField<String>, + actual_description: &LocatorField<String>, +) -> IdentityMatch { + let expected_name = stable_name( + entry.identity.role.as_str(), + entry.identity.name.as_deref(), + entry.identity.value.as_deref(), + ); + let expected_value = stable_value( + entry.identity.role.as_str(), + entry.identity.value.as_deref(), + ); + let expected_description = meaningful_text(entry.identity.description.as_deref()); + let actual_name = stable_name_field(entry.identity.role.as_str(), actual_name, actual_value); + let actual_value = stable_value_field(entry.identity.role.as_str(), actual_value); + let actual_description = meaningful_field(actual_description); if let Some(expected) = expected_name { return match_primary_identity(expected, actual_name, actual_value); @@ -40,38 +115,97 @@ pub fn identity_matches( if let Some(expected) = expected_description { return match_primary_identity(expected, actual_description, actual_name); } - - if is_mutable_value_role(entry.role.as_str()) { - return true; + if is_mutable_value_role(entry.identity.role.as_str()) { + return IdentityMatch::Unknown; } - - actual_name.is_none() && actual_value.is_none() && actual_description.is_none() -} - -/// Allows a platform adapter to search replacement windows only when the saved -/// ref has enough non-text evidence for the shared classifier to fail closed. -/// A saved source-window title disables this fallback unless a platform first -/// finds that title uniquely; otherwise the old titled window is considered gone. -pub fn bounded_window_fallback_allowed(entry: &RefEntry) -> bool { - matches!(entry.source_surface, SnapshotSurface::Window) - && entry.source_window_id.is_some() - && entry.source_window_title.is_none() - && entry.bounds_hash.is_some() + empty_identity_match([actual_name, actual_value, actual_description]) } fn match_primary_identity( expected: &str, - actual_primary: Option<&str>, - actual_fallback: Option<&str>, -) -> bool { + actual_primary: LocatorField<&str>, + actual_fallback: LocatorField<&str>, +) -> IdentityMatch { match actual_primary { - Some(actual) => actual == expected, - None => actual_fallback == Some(expected), + LocatorField::Known(actual) => equality_match(expected, actual), + LocatorField::Unknown => IdentityMatch::Unknown, + LocatorField::Absent => match actual_fallback { + LocatorField::Known(actual) => equality_match(expected, actual), + LocatorField::Unknown => IdentityMatch::Unknown, + LocatorField::Absent => IdentityMatch::NoMatch, + }, + } +} + +fn equality_match(expected: &str, actual: &str) -> IdentityMatch { + if actual == expected { + IdentityMatch::Match + } else { + IdentityMatch::NoMatch + } +} + +fn empty_identity_match(fields: [LocatorField<&str>; 3]) -> IdentityMatch { + if fields.iter().any(LocatorField::is_unknown) { + return IdentityMatch::Unknown; + } + if fields + .iter() + .any(|field| matches!(field, LocatorField::Known(_))) + { + IdentityMatch::NoMatch + } else { + IdentityMatch::Unknown + } +} + +#[cfg(test)] +fn option_field(value: Option<&str>) -> LocatorField<String> { + value + .map(str::to_string) + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent) +} + +fn meaningful_field(field: &LocatorField<String>) -> LocatorField<&str> { + match field { + LocatorField::Known(value) => meaningful_text(Some(value.as_str())) + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent), + LocatorField::Absent => LocatorField::Absent, + LocatorField::Unknown => LocatorField::Unknown, + } +} + +fn stable_name_field<'a>( + role: &str, + name: &'a LocatorField<String>, + value: &LocatorField<String>, +) -> LocatorField<&'a str> { + let name = meaningful_field(name); + if !is_mutable_value_role(role) { + return name; + } + let LocatorField::Known(name) = name else { + return name; + }; + match meaningful_field(value) { + LocatorField::Known(value) if value_matches_name(Some(value), name) => LocatorField::Absent, + LocatorField::Known(_) | LocatorField::Absent => LocatorField::Known(name), + LocatorField::Unknown => LocatorField::Unknown, + } +} + +fn stable_value_field<'a>(role: &str, value: &'a LocatorField<String>) -> LocatorField<&'a str> { + if is_mutable_value_role(role) { + LocatorField::Absent + } else { + meaningful_field(value) } } fn meaningful_text(value: Option<&str>) -> Option<&str> { - value.filter(|text| !text.is_empty()) + value.filter(|text| !text.trim().is_empty()) } fn stable_name<'a>(role: &str, name: Option<&'a str>, value: Option<&str>) -> Option<&'a str> { diff --git a/crates/core/src/ref_identity_match.rs b/crates/core/src/ref_identity_match.rs new file mode 100644 index 0000000..39685dd --- /dev/null +++ b/crates/core/src/ref_identity_match.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdentityMatch { + Match, + NoMatch, + Unknown, +} diff --git a/crates/core/src/ref_identity_tests.rs b/crates/core/src/ref_identity_tests.rs index 4490812..02e7f1b 100644 --- a/crates/core/src/ref_identity_tests.rs +++ b/crates/core/src/ref_identity_tests.rs @@ -1,157 +1,214 @@ use super::*; +fn identifier(value: &str) -> crate::ElementIdentifier { + crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: value.into(), + } +} +use crate::{ + SnapshotSurface, + live_locator::{IdentifierEvidence, LocatorField}, +}; + fn entry() -> RefEntry { RefEntry { - pid: 1, - role: "button".into(), - name: None, - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: None, + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } #[test] -fn empty_identity_matches_missing_or_empty_ax_text() { +fn empty_identity_remains_unknown_without_positive_evidence() { let mut entry = entry(); - entry.role = "menubutton".into(); - entry.name = Some(String::new()); + entry.identity.role = "menubutton".into(); + entry.identity.name = Some(String::new()); assert!(!has_meaningful_identity(&entry)); - assert!(identity_matches(&entry, None, None, None)); - assert!(identity_matches(&entry, Some(""), None, None)); - assert!(identity_matches(&entry, None, Some(""), None)); - assert!(!identity_matches(&entry, Some("Insert Shape"), None, None)); + assert!(!identity_matches(&entry, None, None, None, None)); + assert!(!identity_matches(&entry, Some(""), None, None, None)); + assert!(!identity_matches(&entry, None, Some(""), None, None)); + assert!(!identity_matches( + &entry, + Some("Insert Shape"), + None, + None, + None + )); } #[test] fn description_identity_matches_blank_title_controls() { let mut entry = entry(); - entry.description = Some("Insert Text Box".into()); + entry.identity.description = Some("Insert Text Box".into()); assert!(has_meaningful_identity(&entry)); assert!(identity_matches( &entry, Some(""), None, - Some("Insert Text Box") + Some("Insert Text Box"), + None, )); assert!(identity_matches( &entry, Some("Insert Text Box"), None, - None + None, + None, )); - assert!(!identity_matches(&entry, Some(""), None, None)); + assert!(!identity_matches(&entry, Some(""), None, None, None)); assert!(!identity_matches( &entry, Some(""), None, - Some("Insert Shape") + Some("Insert Shape"), + None, )); } #[test] fn name_identity_cannot_be_rescued_by_matching_description() { let mut entry = entry(); - entry.name = Some("Primary".into()); - entry.description = Some("Generic".into()); + entry.identity.name = Some("Primary".into()); + entry.identity.description = Some("Generic".into()); - assert!(identity_matches(&entry, Some("Primary"), None, None)); - assert!(identity_matches(&entry, None, Some("Primary"), None)); + assert!(identity_matches(&entry, Some("Primary"), None, None, None)); + assert!(identity_matches(&entry, None, Some("Primary"), None, None)); assert!(!identity_matches( &entry, Some("Other"), None, - Some("Primary") + Some("Primary"), + None, )); - assert!(!identity_matches(&entry, Some("Generic"), None, None)); + assert!(!identity_matches(&entry, Some("Generic"), None, None, None)); } #[test] fn value_identity_cannot_be_rescued_by_matching_name_when_value_mismatches() { let mut entry = entry(); - entry.value = Some("On".into()); + entry.identity.value = Some("On".into()); - assert!(identity_matches(&entry, None, Some("On"), None)); - assert!(identity_matches(&entry, Some("On"), None, None)); - assert!(!identity_matches(&entry, Some("On"), Some("Off"), None)); -} - -#[test] -fn mutable_value_role_does_not_go_stale_when_value_changes() { - let mut entry = entry(); - entry.role = "textfield".into(); - entry.value = Some("seed".into()); - - assert!(!has_meaningful_identity(&entry)); - assert!(identity_matches(&entry, None, Some("changed"), None)); -} - -#[test] -fn unnamed_mutable_value_role_does_not_go_stale_when_content_becomes_name() { - let mut entry = entry(); - entry.role = "textfield".into(); - - assert!(!has_meaningful_identity(&entry)); - assert!(identity_matches( + assert!(identity_matches(&entry, None, Some("On"), None, None)); + assert!(identity_matches(&entry, Some("On"), None, None, None)); + assert!(!identity_matches( &entry, - Some("typed document text"), - Some("typed document text"), + Some("On"), + Some("Off"), + None, None )); } #[test] -fn mutable_value_text_promoted_to_name_is_not_stable_identity() { +fn mutable_value_change_cannot_prove_identity_by_itself() { let mut entry = entry(); - entry.role = "textfield".into(); - entry.name = Some("00:01".into()); - entry.value = Some("00:01".into()); + entry.identity.role = "textfield".into(); + entry.identity.value = Some("seed".into()); assert!(!has_meaningful_identity(&entry)); - assert!(identity_matches(&entry, Some("00:06"), Some("00:06"), None)); + assert!(!identity_matches(&entry, None, Some("changed"), None, None)); +} + +#[test] +fn unnamed_mutable_value_role_cannot_use_content_as_identity() { + let mut entry = entry(); + entry.identity.role = "textfield".into(); + + assert!(!has_meaningful_identity(&entry)); + assert!(!identity_matches( + &entry, + Some("typed document text"), + Some("typed document text"), + None, + None + )); +} + +#[test] +fn mutable_value_text_promoted_to_name_remains_unknown() { + let mut entry = entry(); + entry.identity.role = "textfield".into(); + entry.identity.name = Some("00:01".into()); + entry.identity.value = Some("00:01".into()); + + assert!(!has_meaningful_identity(&entry)); + assert!(!identity_matches( + &entry, + Some("00:06"), + Some("00:06"), + None, + None + )); } #[test] fn formatted_numeric_mutable_value_promoted_to_name_is_not_stable_identity() { let mut entry = entry(); - entry.role = "slider".into(); - entry.name = Some("50".into()); - entry.value = Some("50.0".into()); + entry.identity.role = "slider".into(); + entry.identity.name = Some("50".into()); + entry.identity.value = Some("50.0".into()); assert!(!has_meaningful_identity(&entry)); - assert!(identity_matches(&entry, Some("51"), Some("51.0"), None)); + assert!(!identity_matches( + &entry, + Some("51"), + Some("51.0"), + None, + None + )); } #[test] fn named_mutable_value_role_still_uses_name_identity() { let mut entry = entry(); - entry.role = "textfield".into(); - entry.name = Some("Search".into()); - entry.value = Some("old query".into()); + entry.identity.role = "textfield".into(); + entry.identity.name = Some("Search".into()); + entry.identity.value = Some("old query".into()); assert!(has_meaningful_identity(&entry)); assert!(identity_matches( &entry, Some("Search"), Some("new query"), + None, None )); assert!(!identity_matches( &entry, Some("Replace"), Some("new query"), + None, None )); } @@ -159,31 +216,176 @@ fn named_mutable_value_role_still_uses_name_identity() { #[test] fn mutable_role_label_different_from_value_remains_stable_identity() { let mut entry = entry(); - entry.role = "combobox".into(); - entry.name = Some("Font".into()); - entry.value = Some("Helvetica".into()); + entry.identity.role = "combobox".into(); + entry.identity.name = Some("Font".into()); + entry.identity.value = Some("Helvetica".into()); assert!(has_meaningful_identity(&entry)); - assert!(identity_matches(&entry, Some("Font"), Some("Arial"), None)); - assert!(!identity_matches(&entry, Some("Size"), Some("Arial"), None)); + assert!(identity_matches( + &entry, + Some("Font"), + Some("Arial"), + None, + None + )); + assert!(!identity_matches( + &entry, + Some("Size"), + Some("Arial"), + None, + None + )); } #[test] -fn bounded_window_fallback_requires_window_source_window_id_and_bounds() { +fn native_id_is_strongest_identity_signal() { let mut entry = entry(); - entry.source_window_id = Some("platform-window-1".into()); - entry.bounds_hash = Some(42); + entry.identity.native_id = Some(identifier("submit-btn")); + entry.identity.name = Some("Old Label".into()); - assert!(bounded_window_fallback_allowed(&entry)); - entry.source_window_title = Some("Stale Title".into()); - assert!(!bounded_window_fallback_allowed(&entry)); - entry.source_window_title = None; - entry.bounds_hash = None; - assert!(!bounded_window_fallback_allowed(&entry)); - entry.bounds_hash = Some(42); - entry.source_window_id = None; - assert!(!bounded_window_fallback_allowed(&entry)); - entry.source_window_id = Some("platform-window-1".into()); - entry.source_surface = SnapshotSurface::Menu; - assert!(!bounded_window_fallback_allowed(&entry)); + assert!(has_meaningful_identity(&entry)); + assert!(identity_matches( + &entry, + Some("Renamed"), + None, + None, + Some("submit-btn"), + )); + assert!(!identity_matches( + &entry, + Some("Renamed"), + None, + None, + Some("cancel-btn"), + )); +} + +#[test] +fn differing_native_ids_are_hard_non_match() { + let mut entry = entry(); + entry.identity.native_id = Some(identifier("field-a")); + entry.identity.name = Some("Same".into()); + + assert!(!identity_matches( + &entry, + Some("Same"), + None, + None, + Some("field-b"), + )); +} + +#[test] +fn missing_live_native_id_fails_closed_even_with_stable_name() { + let mut entry = entry(); + entry.identity.native_id = Some(identifier("compose-message")); + entry.identity.name = Some("Send".into()); + + assert!(!identity_matches(&entry, Some("Send"), None, None, None,)); + assert!(!identity_matches(&entry, Some("Cancel"), None, None, None,)); +} + +#[test] +fn saved_identifier_matches_either_live_identifier_source() { + let mut entry = entry(); + entry.identity.native_id = Some(identifier("dom-submit")); + let identifiers = + IdentifierEvidence::typed(["ax-submit", "dom-submit"].map(identifier), Some(0), true); + + assert_eq!( + identity_match( + &entry, + &LocatorField::Absent, + &LocatorField::Absent, + &LocatorField::Absent, + &identifiers, + ), + IdentityMatch::Match + ); +} + +#[test] +fn ax_only_and_dom_only_identifiers_resolve_deterministically() { + for identifier in ["ax-only", "dom-only"] { + let mut entry = entry(); + entry.identity.native_id = Some(self::identifier(identifier)); + let live = IdentifierEvidence::typed([self::identifier(identifier)], Some(0), true); + + assert_eq!( + identity_match( + &entry, + &LocatorField::Absent, + &LocatorField::Absent, + &LocatorField::Absent, + &live, + ), + IdentityMatch::Match + ); + } +} + +#[test] +fn incomplete_identifier_slots_cannot_prove_a_non_match() { + let mut entry = entry(); + entry.identity.native_id = Some(identifier("dom-submit")); + let identifiers = IdentifierEvidence::typed([identifier("ax-submit")], Some(0), false); + + assert_eq!( + identity_match( + &entry, + &LocatorField::Known("Submit".into()), + &LocatorField::Absent, + &LocatorField::Absent, + &identifiers, + ), + IdentityMatch::Unknown + ); +} + +#[test] +fn unavailable_identifier_fails_closed_before_stable_text_fallback() { + let mut entry = entry(); + entry.identity.native_id = Some(identifier("old-dom-id")); + entry.identity.name = Some("Submit".into()); + let absent = IdentifierEvidence::absent(); + + assert_eq!( + identity_match( + &entry, + &LocatorField::Known("Submit".into()), + &LocatorField::Absent, + &LocatorField::Absent, + &absent, + ), + IdentityMatch::NoMatch + ); + + entry.identity.name = None; + assert_eq!( + identity_match( + &entry, + &LocatorField::Absent, + &LocatorField::Absent, + &LocatorField::Absent, + &absent, + ), + IdentityMatch::NoMatch + ); +} + +#[test] +fn unknown_stable_text_field_does_not_collapse_to_absence() { + let mut entry = entry(); + entry.identity.name = Some("Submit".into()); + + assert_eq!( + identity_match( + &entry, + &LocatorField::Unknown, + &LocatorField::Absent, + &LocatorField::Absent, + &IdentifierEvidence::absent(), + ), + IdentityMatch::Unknown + ); } diff --git a/crates/core/src/ref_process.rs b/crates/core/src/ref_process.rs new file mode 100644 index 0000000..8ff1b41 --- /dev/null +++ b/crates/core/src/ref_process.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +use crate::ProcessId; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefProcess { + pub pid: ProcessId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub process_instance: Option<String>, +} diff --git a/crates/core/src/ref_resolve_deadline.rs b/crates/core/src/ref_resolve_deadline.rs new file mode 100644 index 0000000..9069370 --- /dev/null +++ b/crates/core/src/ref_resolve_deadline.rs @@ -0,0 +1,114 @@ +use std::time::Duration; + +use serde_json::json; + +use crate::{ + AdapterError, Deadline, ErrorCode, PlatformAdapter, RefEntry, + resolve_attempt_outcome::ResolveAttemptOutcome, +}; + +pub(crate) const POLL_INTERVAL: Duration = Duration::from_millis(100); +const RESOLVE_ATTEMPT: Duration = Duration::from_millis(750); + +pub(crate) fn resolve_within_deadline( + adapter: &dyn PlatformAdapter, + entry: &RefEntry, + deadline: Deadline, +) -> ResolveAttemptOutcome { + if deadline.remaining_slice(RESOLVE_ATTEMPT).is_err() { + return ResolveAttemptOutcome::DeadlinePassed; + } + match adapter.resolve_element_strict(entry, deadline.capped(RESOLVE_ATTEMPT)) { + Ok(handle) => ResolveAttemptOutcome::Resolved(handle), + Err(error) => classify_error(error, deadline), + } +} + +fn classify_error(error: AdapterError, deadline: Deadline) -> ResolveAttemptOutcome { + if error.code != ErrorCode::Timeout { + return ResolveAttemptOutcome::Failed(error); + } + if !error.permits_retry_by_default() + || error.disposition.retry() == crate::RetryDisposition::Unsafe + { + return ResolveAttemptOutcome::Failed(error); + } + if deadline.is_expired() { + return ResolveAttemptOutcome::DeadlinePassed; + } + ResolveAttemptOutcome::Failed(mark_retryable(error)) +} + +fn mark_retryable(mut error: AdapterError) -> AdapterError { + let mut details = error.details.take().unwrap_or_else(|| json!({})); + if let Some(object) = details.as_object_mut() { + object.insert("retryable".into(), true.into()); + } else { + details = json!({ + "error_details": details, + "retryable": true, + }); + } + error.with_details(details) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Instant; + + #[test] + fn per_attempt_timeout_is_retryable_while_outer_deadline_remains() { + let error = AdapterError::timeout("strict resolution slice expired") + .with_details(json!({ "kind": "deadline" })); + + let ResolveAttemptOutcome::Failed(error) = + classify_error(error, Deadline::after(1_000).unwrap()) + else { + panic!("per-attempt timeout must remain inside the outer poll loop"); + }; + + assert!(error.is_explicitly_retryable()); + assert_eq!(error.details.unwrap()["kind"], "deadline"); + } + + #[test] + fn explicit_non_retryable_timeout_remains_terminal() { + let error = AdapterError::timeout("deterministic limit") + .with_details(json!({ "retryable": false })); + + let ResolveAttemptOutcome::Failed(error) = + classify_error(error, Deadline::after(1_000).unwrap()) + else { + panic!("explicit non-retryable timeout must remain terminal"); + }; + + assert!(!error.permits_retry_by_default()); + } + + #[test] + fn uncertain_timeout_remains_terminal_with_stronger_delivery_evidence() { + let error = AdapterError::timeout("delivery is uncertain") + .with_disposition(crate::DeliverySemantics::uncertain()); + + let ResolveAttemptOutcome::Failed(error) = + classify_error(error, Deadline::after(1_000).unwrap()) + else { + panic!("delivery-uncertain timeout must remain terminal"); + }; + + assert_eq!(error.disposition, crate::DeliverySemantics::uncertain()); + assert!(!error.is_explicitly_retryable()); + } + + #[test] + fn outer_deadline_owns_the_terminal_timeout_shape() { + let started = Instant::now() - Duration::from_millis(10); + let deadline = Deadline::at(started, 1).unwrap(); + + assert!(matches!( + classify_error(AdapterError::timeout("attempt expired"), deadline), + ResolveAttemptOutcome::DeadlinePassed + )); + } +} diff --git a/crates/core/src/ref_scope.rs b/crates/core/src/ref_scope.rs new file mode 100644 index 0000000..da70b34 --- /dev/null +++ b/crates/core/src/ref_scope.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefScope { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_ref: Option<String>, + #[serde(default, skip_serializing_if = "is_false")] + pub path_is_absolute: bool, + #[serde(default, skip_serializing_if = "smallvec::SmallVec::is_empty")] + pub path: crate::refs::RefPath, +} + +fn is_false(value: &bool) -> bool { + !*value +} diff --git a/crates/core/src/ref_source.rs b/crates/core/src/ref_source.rs new file mode 100644 index 0000000..e3ce1a0 --- /dev/null +++ b/crates/core/src/ref_source.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RefSource { + pub source_app: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_window_id: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_window_title: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_window_bounds_hash: Option<u64>, + #[serde(default, skip_serializing_if = "crate::SnapshotSurface::is_window")] + pub source_surface: crate::SnapshotSurface, +} diff --git a/crates/core/src/ref_token.rs b/crates/core/src/ref_token.rs new file mode 100644 index 0000000..e7d8ba3 --- /dev/null +++ b/crates/core/src/ref_token.rs @@ -0,0 +1,78 @@ +use crate::{AccessibilityNode, AppError, refs::validate_snapshot_id}; + +pub(crate) fn qualify_ref_id(snapshot_id: &str, local_ref: &str) -> String { + format!( + "@{snapshot_id}:{}", + local_ref.strip_prefix('@').unwrap_or(local_ref) + ) +} + +pub(crate) fn qualify_tree_refs(tree: &mut AccessibilityNode, snapshot_id: &str) { + if let Some(local_ref) = tree.ref_id.as_deref() { + tree.ref_id = Some(qualify_ref_id(snapshot_id, local_ref)); + } + for child in &mut tree.children { + qualify_tree_refs(child, snapshot_id); + } +} + +pub(crate) fn resolve_ref_target( + ref_id: &str, + explicit_snapshot_id: Option<&str>, +) -> Result<(String, String), AppError> { + if is_local_ref(ref_id) { + let snapshot_id = explicit_snapshot_id.ok_or_else(|| { + AppError::invalid_input_with_suggestion( + "Bare refs require an explicit snapshot_id", + "Use the snapshot-qualified ref returned by snapshot, or pass --snapshot with a legacy @eN ref.", + ) + })?; + validate_snapshot_id(snapshot_id)?; + return Ok((snapshot_id.to_string(), ref_id.to_string())); + } + + let Some(without_at) = ref_id.strip_prefix('@') else { + return Err(invalid_ref(ref_id)); + }; + let Some((snapshot_id, element_number)) = without_at.rsplit_once(":e") else { + return Err(invalid_ref(ref_id)); + }; + validate_snapshot_id(snapshot_id)?; + let local_ref = format!("@e{element_number}"); + if !is_local_ref(&local_ref) { + return Err(invalid_ref(ref_id)); + } + if explicit_snapshot_id.is_some_and(|explicit| explicit != snapshot_id) { + return Err(AppError::invalid_input_with_suggestion( + "Ref snapshot does not match the explicit snapshot_id", + "Use the snapshot_id embedded in the ref, or pass a ref from the requested snapshot.", + )); + } + Ok((snapshot_id.to_string(), local_ref)) +} + +pub(crate) fn validate_ref_token(ref_id: &str) -> Result<(), AppError> { + if is_local_ref(ref_id) { + return Ok(()); + } + resolve_ref_target(ref_id, None).map(|_| ()) +} + +fn is_local_ref(ref_id: &str) -> bool { + ref_id.strip_prefix("@e").is_some_and(|digits| { + !digits.is_empty() + && digits.len() <= 10 + && digits.chars().all(|character| character.is_ascii_digit()) + && digits.parse::<u32>().is_ok_and(|number| number > 0) + }) +} + +fn invalid_ref(ref_id: &str) -> AppError { + AppError::invalid_input(format!( + "Invalid ref_id '{ref_id}': expected @<snapshot_id>:e<N> or legacy @e<N> with an explicit snapshot" + )) +} + +#[cfg(test)] +#[path = "ref_token_tests.rs"] +mod tests; diff --git a/crates/core/src/ref_token_tests.rs b/crates/core/src/ref_token_tests.rs new file mode 100644 index 0000000..e48cafe --- /dev/null +++ b/crates/core/src/ref_token_tests.rs @@ -0,0 +1,39 @@ +use crate::ref_token::{qualify_ref_id, resolve_ref_target, validate_ref_token}; + +#[test] +fn qualified_ref_selects_its_own_snapshot() { + assert_eq!( + resolve_ref_target("@sabc:e7", None).unwrap(), + ("sabc".into(), "@e7".into()) + ); + assert_eq!(qualify_ref_id("sabc", "@e7"), "@sabc:e7"); +} + +#[test] +fn bare_ref_requires_explicit_snapshot() { + assert_eq!( + resolve_ref_target("@e1", None).unwrap_err().code(), + "INVALID_ARGS" + ); + assert_eq!( + resolve_ref_target("@e1", Some("sabc")).unwrap(), + ("sabc".into(), "@e1".into()) + ); +} + +#[test] +fn explicit_snapshot_must_match_qualified_ref() { + assert_eq!( + resolve_ref_target("@sone:e1", Some("stwo")) + .unwrap_err() + .code(), + "INVALID_ARGS" + ); +} + +#[test] +fn syntax_validation_accepts_both_forms() { + assert!(validate_ref_token("@e1").is_ok()); + assert!(validate_ref_token("@sabc:e1").is_ok()); + assert!(validate_ref_token("@latest:e0").is_err()); +} diff --git a/crates/core/src/refs.rs b/crates/core/src/refs.rs index ab67c98..29e7e10 100644 --- a/crates/core/src/refs.rs +++ b/crates/core/src/refs.rs @@ -1,5 +1,5 @@ -use crate::adapter::SnapshotSurface; -use crate::error::AppError; +use crate::AppError; +pub(crate) use crate::RefEntry; use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use std::cell::RefCell; @@ -19,52 +19,8 @@ thread_local! { static HOME_OVERRIDE: RefCell<Option<PathBuf>> = const { RefCell::new(None) }; } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RefEntry { - pub pid: i32, - pub role: String, - pub name: Option<String>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option<String>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option<String>, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub states: Vec<String>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub bounds: Option<crate::node::Rect>, - pub bounds_hash: Option<u64>, - pub available_actions: Vec<String>, - pub source_app: Option<String>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_window_id: Option<String>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_window_title: Option<String>, - #[serde(default, skip_serializing_if = "SnapshotSurface::is_window")] - pub source_surface: SnapshotSurface, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub root_ref: Option<String>, - #[serde(default, skip_serializing_if = "is_false")] - pub path_is_absolute: bool, - #[serde(default, skip_serializing_if = "SmallVec::is_empty")] - pub path: RefPath, -} - -fn is_false(value: &bool) -> bool { - !*value -} - pub fn validate_ref_id(ref_id: &str) -> Result<(), AppError> { - let valid = ref_id.starts_with("@e") - && ref_id.len() >= 3 - && ref_id.len() <= 12 - && ref_id[2..].chars().all(|c| c.is_ascii_digit()) - && ref_id[2..].parse::<u32>().is_ok_and(|n| n > 0); - if valid { - return Ok(()); - } - Err(AppError::invalid_input(format!( - "Invalid ref_id '{ref_id}': must match @e{{N}} where N is a positive integer" - ))) + crate::ref_token::validate_ref_token(ref_id) } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -81,11 +37,39 @@ impl RefMap { } } - pub fn allocate(&mut self, entry: RefEntry) -> String { - self.counter += 1; + pub fn try_allocate(&mut self, entry: RefEntry) -> Result<String, AppError> { + crate::refs_validate::validate_ref_entry(&entry)?; + self.allocate_validated(entry) + } + + pub(crate) fn try_allocate_observed( + &mut self, + entry: RefEntry, + ) -> Result<crate::ref_allocation::RefAllocation, AppError> { + if !crate::Role::is_canonical(&entry.identity.role) || entry.identity.role == "unknown" { + return Ok(crate::ref_allocation::RefAllocation::SkippedInvalidRole); + } + if crate::refs_validate::validate_ref_entry(&entry).is_err() { + return Ok(crate::ref_allocation::RefAllocation::SkippedInvalidEntry); + } + self.allocate_validated(entry) + .map(crate::ref_allocation::RefAllocation::Allocated) + } + + fn allocate_validated(&mut self, entry: RefEntry) -> Result<String, AppError> { + self.counter = self + .counter + .checked_add(1) + .ok_or_else(|| AppError::invalid_input("RefMap exhausted its identifier space"))?; let ref_id = format!("@e{}", self.counter); self.inner.insert(ref_id.clone(), entry); - ref_id + Ok(ref_id) + } + + #[cfg(test)] + pub fn allocate(&mut self, entry: RefEntry) -> String { + self.try_allocate(entry) + .expect("test RefEntry must be valid") } pub fn get(&self, ref_id: &str) -> Option<&RefEntry> { @@ -102,10 +86,35 @@ impl RefMap { pub fn remove_by_root_ref(&mut self, root: &str) { self.inner - .retain(|_, entry| entry.root_ref.as_deref() != Some(root)); + .retain(|_, entry| entry.scope.root_ref.as_deref() != Some(root)); + } + + pub fn validate(&self) -> Result<(), AppError> { + let mut max_id = 0_u32; + for (ref_id, entry) in &self.inner { + let numeric = ref_id + .strip_prefix("@e") + .and_then(|value| value.parse::<u32>().ok()) + .filter(|value| *value > 0) + .ok_or_else(|| AppError::invalid_input("RefMap contains an invalid ref key"))?; + if format!("@e{numeric}") != *ref_id { + return Err(AppError::invalid_input( + "RefMap contains a non-canonical ref key", + )); + } + max_id = max_id.max(numeric); + crate::refs_validate::validate_ref_entry(entry)?; + } + if self.counter < max_id || self.counter == u32::MAX { + return Err(AppError::invalid_input( + "RefMap counter is inconsistent with its allocated refs", + )); + } + Ok(()) } pub(crate) fn serialize_with_size_check(&self) -> Result<String, AppError> { + self.validate()?; let json = serde_json::to_string(self)?; if json.len() as u64 > MAX_REFMAP_BYTES { return Err(AppError::Internal( @@ -124,16 +133,9 @@ impl RefMap { pub fn load() -> Result<Self, AppError> { let path = refmap_path()?; - - let metadata = std::fs::metadata(&path)?; - if metadata.len() > MAX_REFMAP_BYTES { - return Err(AppError::Internal( - "RefMap file exceeds 1MB size limit".into(), - )); - } - - let json = std::fs::read_to_string(&path)?; - let map: Self = serde_json::from_str(&json)?; + let json = crate::private_file::read_private_bounded(&path, MAX_REFMAP_BYTES)?; + let map: Self = serde_json::from_slice(&json)?; + map.validate()?; Ok(map) } } @@ -200,51 +202,23 @@ pub fn validate_snapshot_id(snapshot_id: &str) -> Result<(), AppError> { } pub(crate) fn write_private_file(path: &Path, bytes: &[u8]) -> Result<(), AppError> { - let dir = path - .parent() - .ok_or_else(|| AppError::Internal("invalid ref store path".into()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt; - std::fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(dir)?; - } - #[cfg(not(unix))] - std::fs::create_dir_all(dir)?; - - static TMP_COUNTER: AtomicU64 = AtomicU64::new(0); - let unique = TMP_COUNTER.fetch_add(1, Ordering::Relaxed); - let tmp = path.with_extension(format!("{}.{unique}.tmp", std::process::id())); - let written = write_tmp_then_rename(&tmp, path, bytes); - if written.is_err() { - let _ = std::fs::remove_file(&tmp); - } - written + crate::private_file::write_atomic(path, bytes).map_err(AppError::from) } -fn write_tmp_then_rename(tmp: &Path, path: &Path, bytes: &[u8]) -> Result<(), AppError> { - #[cfg(unix)] - { - use std::io::Write; - use std::os::unix::fs::OpenOptionsExt; - let mut file = std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .custom_flags(libc::O_NOFOLLOW) - .open(tmp)?; - file.write_all(bytes)?; - file.flush()?; - } - #[cfg(not(unix))] - std::fs::write(tmp, bytes)?; - - std::fs::rename(tmp, path)?; - Ok(()) +pub(crate) fn write_user_file(path: &Path, bytes: &[u8]) -> Result<(), AppError> { + crate::private_file::write_user_atomic(path, bytes).map_err(|error| { + if error.kind() == std::io::ErrorKind::InvalidData { + AppError::invalid_input_with_suggestion( + format!("Cannot write output file: {error}"), + format!( + "Pass an output path that is not '{}' or remove the conflicting entry there", + path.display() + ), + ) + } else { + AppError::from(error) + } + }) } pub(crate) fn is_symlink(path: &Path) -> bool { diff --git a/crates/core/src/refs_lock.rs b/crates/core/src/refs_lock.rs index 6be516f..f5a90f9 100644 --- a/crates/core/src/refs_lock.rs +++ b/crates/core/src/refs_lock.rs @@ -1,300 +1,66 @@ -use crate::error::AppError; -use std::io::{ErrorKind, Write}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::path::Path; -const LOCK_TIMEOUT: Duration = Duration::from_secs(2); -const MALFORMED_LOCK_WINDOW: Duration = Duration::from_secs(30); +use crate::{AppError, Deadline, file_lock::FileLock}; + +const LOCK_TIMEOUT_MS: u64 = 2_000; pub(crate) struct RefStoreLock { - path: PathBuf, - token: String, + _lock: FileLock, } impl RefStoreLock { pub(crate) fn acquire(path: &Path) -> Result<Self, AppError> { - if let Some(dir) = path.parent() { - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt; - std::fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(dir)?; - } - #[cfg(not(unix))] - std::fs::create_dir_all(dir)?; - } - let start = Instant::now(); - loop { - let token = lock_token(); - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - { - Ok(mut file) => { - if let Err(err) = - writeln!(file, "{} {} {}", std::process::id(), now_secs(), token) - { - let _ = std::fs::remove_file(path); - return Err(err.into()); - } - return Ok(Self { - path: path.to_path_buf(), - token, - }); - } - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { - if try_remove_stale_lock(path) { - continue; - } - if start.elapsed() > LOCK_TIMEOUT { - return Err(AppError::Internal(format!( - "Timed out waiting for ref store lock at {}", - path.display() - ))); - } - std::thread::sleep(Duration::from_millis(10)); - } - Err(err) => return Err(err.into()), - } - } - } -} - -fn try_remove_stale_lock(path: &Path) -> bool { - let snapshot = match LockSnapshot::read(path) { - Ok(Some(snapshot)) => snapshot, - Ok(None) => return true, - Err(()) => return false, - }; - if !snapshot.is_stale() { - return false; - } - let current = match LockSnapshot::read(path) { - Ok(Some(current)) => current, - Ok(None) => return true, - Err(()) => return false, - }; - if current.contents != snapshot.contents || current.modified != snapshot.modified { - return false; - } - match std::fs::remove_file(path) { - Ok(()) => true, - Err(err) if err.kind() == ErrorKind::NotFound => true, - Err(_) => false, - } -} - -struct LockSnapshot { - contents: String, - modified: Option<SystemTime>, -} - -impl LockSnapshot { - fn read(path: &Path) -> Result<Option<Self>, ()> { - let contents = match std::fs::read_to_string(path) { - Ok(contents) => contents, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), - Err(_) => return Err(()), - }; - let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok(); - Ok(Some(Self { contents, modified })) + let deadline = Deadline::after(LOCK_TIMEOUT_MS)?; + Self::acquire_with_deadline(path, deadline) } - fn is_stale(&self) -> bool { - if self.contents.trim().is_empty() { - return self.modified_age_exceeds(MALFORMED_LOCK_WINDOW); - } - match self.pid() { - Some(pid) => match process_is_alive(pid) { - Some(true) => self.live_pid_lock_is_stale(pid), - Some(false) => true, - None => self.modified_age_exceeds(MALFORMED_LOCK_WINDOW), - }, - None => self.modified_age_exceeds(MALFORMED_LOCK_WINDOW), - } + pub(crate) fn acquire_with_deadline(path: &Path, deadline: Deadline) -> Result<Self, AppError> { + Ok(Self { + _lock: FileLock::acquire(path, deadline, "ref store lock")?, + }) } - - fn live_pid_lock_is_stale(&self, pid: u32) -> bool { - if !self.token_shape_matches_pid(pid) { - return self.modified_age_exceeds(LOCK_TIMEOUT); - } - self.modified_age_exceeds(MALFORMED_LOCK_WINDOW) - } - - fn pid(&self) -> Option<u32> { - self.contents - .split_whitespace() - .next() - .and_then(|s| s.parse::<u32>().ok()) - } - - fn token(&self) -> Option<&str> { - self.contents.split_whitespace().nth(2) - } - - fn owned_by(&self, token: &str) -> bool { - self.pid() == Some(std::process::id()) && self.token() == Some(token) - } - - fn token_shape_matches_pid(&self, pid: u32) -> bool { - self.token() - .and_then(|token| token.split_once('-')) - .and_then(|(prefix, _)| prefix.parse::<u32>().ok()) - == Some(pid) - } - - fn modified_age_exceeds(&self, duration: Duration) -> bool { - self.modified - .and_then(|modified| SystemTime::now().duration_since(modified).ok()) - .is_some_and(|age| age > duration) - } -} - -fn now_secs() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) -} - -fn lock_token() -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - format!("{}-{nanos}", std::process::id()) -} - -#[cfg(unix)] -fn process_is_alive(pid: u32) -> Option<bool> { - let result = unsafe { libc::kill(pid as i32, 0) }; - if result == 0 { - return Some(true); - } - match std::io::Error::last_os_error().raw_os_error() { - Some(errno) if errno == libc::ESRCH => Some(false), - Some(errno) if errno == libc::EPERM => None, - _ => None, - } -} - -#[cfg(not(unix))] -fn process_is_alive(_pid: u32) -> Option<bool> { - None } pub(crate) fn lock_holder_is_live(lock_path: &Path) -> bool { - match LockSnapshot::read(lock_path) { - Ok(Some(snapshot)) => !snapshot.is_stale(), - Ok(None) | Err(()) => false, - } -} - -impl Drop for RefStoreLock { - fn drop(&mut self) { - let should_remove = LockSnapshot::read(&self.path) - .ok() - .flatten() - .is_some_and(|snapshot| snapshot.owned_by(&self.token)); - if should_remove { - let _ = std::fs::remove_file(&self.path); - } - } + FileLock::is_held(lock_path) } #[cfg(test)] mod tests { - use super::*; - use std::fs; + use std::time::Duration; - fn lock_path(name: &str) -> PathBuf { + use super::*; + + fn lock_path(name: &str) -> std::path::PathBuf { std::env::temp_dir().join(format!( "agent-desktop-{name}-{}-{}.lock", std::process::id(), - lock_token() + crate::refs::new_snapshot_id() )) } #[test] - fn acquire_removes_lock_on_drop() { - let path = lock_path("drop"); + fn lock_is_released_without_deleting_the_lock_file() { + let path = lock_path("release"); { let _lock = RefStoreLock::acquire(&path).unwrap(); - assert!(path.exists()); + assert!(lock_holder_is_live(&path)); } - assert!(!path.exists()); - } - - #[test] - fn stale_dead_pid_lock_is_replaced() { - let path = lock_path("stale-pid"); - fs::write(&path, "999999 1 stale").unwrap(); + assert!(path.is_file()); + assert!(!lock_holder_is_live(&path)); let _lock = RefStoreLock::acquire(&path).unwrap(); - let contents = fs::read_to_string(&path).unwrap(); - assert!(contents.starts_with(&std::process::id().to_string())); } #[test] - fn old_same_process_lock_with_foreign_token_shape_is_stale() { - let snapshot = LockSnapshot { - contents: format!("{} 1 old-token", std::process::id()), - modified: Some(SystemTime::now() - Duration::from_secs(60)), + fn contention_obeys_the_callers_deadline() { + let path = lock_path("deadline"); + let _held = RefStoreLock::acquire(&path).unwrap(); + let deadline = Deadline::after(10).unwrap(); + let error = match RefStoreLock::acquire_with_deadline(&path, deadline) { + Ok(_) => panic!("contended lock unexpectedly acquired"), + Err(error) => error, }; - - assert!(snapshot.is_stale()); - } - - #[test] - fn recent_live_pid_lock_is_not_stale() { - let snapshot = LockSnapshot { - contents: format!( - "{} {} {}-token", - std::process::id(), - now_secs(), - std::process::id() - ), - modified: Some(SystemTime::now()), - }; - - assert!(!snapshot.is_stale()); - } - - #[test] - fn old_live_pid_lock_with_foreign_token_shape_is_stale() { - let snapshot = LockSnapshot { - contents: "2000 1 1000-old".into(), - modified: Some(SystemTime::now() - Duration::from_secs(3)), - }; - - assert!(snapshot.live_pid_lock_is_stale(2000)); - } - - #[test] - fn recent_live_pid_lock_with_matching_token_shape_is_not_stale() { - let snapshot = LockSnapshot { - contents: "2000 1 2000-token".into(), - modified: Some(SystemTime::now() - Duration::from_secs(3)), - }; - - assert!(!snapshot.live_pid_lock_is_stale(2000)); - } - - #[test] - fn drop_does_not_remove_replaced_lock() { - let path = lock_path("replaced"); - let lock = RefStoreLock::acquire(&path).unwrap(); - fs::write( - &path, - format!("{} {} replacement-token", std::process::id(), now_secs()), - ) - .unwrap(); - - drop(lock); - - assert!(path.exists()); - fs::remove_file(&path).unwrap(); + assert_eq!(error.code(), "TIMEOUT"); + assert!(deadline.elapsed() < Duration::from_secs(1)); } } diff --git a/crates/core/src/refs_serde_tests.rs b/crates/core/src/refs_serde_tests.rs index e7b57a8..e535062 100644 --- a/crates/core/src/refs_serde_tests.rs +++ b/crates/core/src/refs_serde_tests.rs @@ -3,22 +3,37 @@ use crate::adapter::SnapshotSurface; fn minimal_entry(role: &str) -> RefEntry { RefEntry { - pid: 1, - role: role.into(), - name: None, - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: role.into(), + name: None, + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: Some(0xA11C_E551), + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } @@ -85,20 +100,16 @@ fn ref_entry_source_surface_omitted_for_window_present_for_non_window() { "Window surface must be omitted as the default, json={window_json}" ); - let alert_entry = RefEntry { - source_surface: SnapshotSurface::Alert, - ..minimal_entry("button") - }; + let mut alert_entry = minimal_entry("button"); + alert_entry.source.source_surface = SnapshotSurface::Alert; let alert_json = serde_json::to_string(&alert_entry).unwrap(); assert!( alert_json.contains("\"source_surface\":\"alert\""), "Alert surface must serialize to 'alert', json={alert_json}" ); - let menu_entry = RefEntry { - source_surface: SnapshotSurface::Menu, - ..minimal_entry("button") - }; + let mut menu_entry = minimal_entry("button"); + menu_entry.source.source_surface = SnapshotSurface::Menu; let menu_json = serde_json::to_string(&menu_entry).unwrap(); assert!( menu_json.contains("\"source_surface\":\"menu\""), @@ -119,6 +130,24 @@ fn snapshot_surface_serializes_to_snake_case_and_roundtrips() { (SnapshotSurface::Sheet, "\"sheet\""), (SnapshotSurface::Popover, "\"popover\""), (SnapshotSurface::Alert, "\"alert\""), + (SnapshotSurface::Desktop, "\"desktop\""), + (SnapshotSurface::Taskbar, "\"taskbar\""), + (SnapshotSurface::SystemTray, "\"system_tray\""), + (SnapshotSurface::QuickSettings, "\"quick_settings\""), + ( + SnapshotSurface::NotificationCenter, + "\"notification_center\"", + ), + (SnapshotSurface::Toolbar, "\"toolbar\""), + (SnapshotSurface::Dock, "\"dock\""), + (SnapshotSurface::Spotlight, "\"spotlight\""), + (SnapshotSurface::MenuBarExtras, "\"menu_bar_extras\""), + ( + SnapshotSurface::SystemTrayOverflow, + "\"system_tray_overflow\"", + ), + (SnapshotSurface::StartMenu, "\"start_menu\""), + (SnapshotSurface::ActionCenter, "\"action_center\""), ]; for (variant, expected_json) in cases { let serialized = serde_json::to_string(&variant).unwrap(); @@ -136,39 +165,43 @@ fn snapshot_surface_serializes_to_snake_case_and_roundtrips() { #[test] fn ref_entry_full_roundtrip_preserves_all_fields() { let original = RefEntry { - pid: 99, - role: "textfield".into(), - name: Some("Email".into()), - value: Some("user@example.com".into()), - description: Some("Enter email".into()), - states: vec!["focused".into()], - bounds: None, - bounds_hash: Some(0xDEAD_BEEF), - available_actions: vec!["SetValue".into(), "Click".into()], - source_app: Some("Mail".into()), - source_window_id: Some("w-7".into()), - source_window_title: Some("Compose".into()), - source_surface: SnapshotSurface::Sheet, - root_ref: Some("@e5".into()), - path_is_absolute: true, - path: smallvec::SmallVec::from_slice(&[2, 0, 1]), + process: crate::RefProcess { + pid: crate::ProcessId::new(99), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "textfield".into(), + name: Some("Email".into()), + value: Some("user@example.com".into()), + description: Some("Enter email".into()), + native_id: Some(crate::ElementIdentifier { + kind: crate::IdentifierKind::AxIdentifier, + value: "email-field".into(), + }), + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: Some(0xDEAD_BEEF), + }, + capabilities: crate::RefCapabilities { + states: vec!["focused".into()], + available_actions: vec!["SetValue".into(), "Click".into()], + }, + source: crate::RefSource { + source_app: Some("Mail".into()), + source_window_id: Some("w-7".into()), + source_window_title: Some("Compose".into()), + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Sheet, + }, + scope: crate::RefScope { + root_ref: Some("@e5".into()), + path_is_absolute: true, + path: smallvec::SmallVec::from_slice(&[2, 0, 1]), + }, }; let json = serde_json::to_string(&original).unwrap(); let back: RefEntry = serde_json::from_str(&json).unwrap(); - assert_eq!(back.pid, original.pid); - assert_eq!(back.role, original.role); - assert_eq!(back.name, original.name); - assert_eq!(back.value, original.value); - assert_eq!(back.description, original.description); - assert_eq!(back.states, original.states); - assert_eq!(back.bounds_hash, original.bounds_hash); - assert_eq!(back.available_actions, original.available_actions); - assert_eq!(back.source_app, original.source_app); - assert_eq!(back.source_window_id, original.source_window_id); - assert_eq!(back.source_window_title, original.source_window_title); - assert_eq!(back.source_surface, original.source_surface); - assert_eq!(back.root_ref, original.root_ref); - assert_eq!(back.path_is_absolute, original.path_is_absolute); - assert_eq!(back.path.as_slice(), original.path.as_slice()); + assert_eq!(back, original); } diff --git a/crates/core/src/refs_store.rs b/crates/core/src/refs_store.rs index 3cc0c57..bec98f2 100644 --- a/crates/core/src/refs_store.rs +++ b/crates/core/src/refs_store.rs @@ -1,15 +1,15 @@ use crate::{ + AdapterError, AppError, context::validate_session_id, - error::{AdapterError, AppError}, refs::{RefMap, home_dir, new_snapshot_id, validate_snapshot_id, write_private_file}, refs_lock::RefStoreLock, }; -use std::io::{ErrorKind, Read}; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; const LATEST_SNAPSHOT_FILE: &str = "latest_snapshot_id"; const MAX_SAVED_SNAPSHOTS: usize = 512; -const STALE_TMP_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(60); +pub(crate) const STALE_TMP_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(60); #[derive(Debug, Clone)] pub struct RefStore { @@ -60,55 +60,23 @@ impl RefStore { self.with_write_lock(|| self.save_snapshot_unlocked(snapshot_id, refmap)) } - /// Re-saves a snapshot to whichever store currently owns it. Ownership is - /// re-verified inside the owning store's write lock, so a snapshot pruned - /// or moved between discovery and locking is never written to a stale - /// location; discovery retries once before deterministically recreating - /// the snapshot in this store. - pub fn save_existing_snapshot( + pub(crate) fn update_existing_snapshot<T>( &self, snapshot_id: &str, - refmap: &RefMap, - ) -> Result<(), AppError> { + expected_ref_id: &str, + expected_entry: &crate::RefEntry, + update: impl FnOnce(&mut RefMap) -> Result<T, AppError>, + ) -> Result<(T, RefMap), AppError> { validate_snapshot_id(snapshot_id)?; - for _ in 0..2 { - let Some(owner) = self.snapshot_owner(snapshot_id)? else { - break; - }; - let saved = owner.with_write_lock(|| { - if owner.snapshot_path(snapshot_id).is_file() { - owner - .save_snapshot_unlocked(snapshot_id, refmap) - .map(|_| true) - } else { - Ok(false) - } - })?; - if saved { - return Ok(()); + self.with_write_lock(|| { + let mut current = self.load_snapshot_from_base(&self.base_dir, snapshot_id)?; + if current.get(expected_ref_id) != Some(expected_entry) { + return Err(AppError::Adapter(AdapterError::stale_ref(expected_ref_id))); } - } - let fallback = self.without_legacy_migration(); - fallback.with_write_lock(|| fallback.save_snapshot_unlocked(snapshot_id, refmap)) - } - - fn snapshot_owner(&self, snapshot_id: &str) -> Result<Option<Self>, AppError> { - if self.snapshot_path(snapshot_id).is_file() { - return Ok(Some(self.without_legacy_migration())); - } - Ok(self - .discover_snapshot_base(snapshot_id)? - .map(|base_dir| Self { - base_dir, - allow_legacy_migration: false, - })) - } - - fn without_legacy_migration(&self) -> Self { - Self { - base_dir: self.base_dir.clone(), - allow_legacy_migration: false, - } + let value = update(&mut current)?; + self.save_snapshot_unlocked(snapshot_id, ¤t)?; + Ok((value, current)) + }) } pub fn load(&self, snapshot_id: Option<&str>) -> Result<RefMap, AppError> { @@ -132,15 +100,7 @@ impl RefStore { pub fn load_snapshot(&self, snapshot_id: &str) -> Result<RefMap, AppError> { validate_snapshot_id(snapshot_id)?; - if let Some(refmap) = self.read_snapshot_if_present(&self.base_dir, snapshot_id)? { - return Ok(refmap); - } - match self.discover_snapshot_base(snapshot_id)? { - Some(base_dir) => self.load_snapshot_from_base(&base_dir, snapshot_id), - None => Err(AppError::Adapter(AdapterError::snapshot_not_found( - snapshot_id, - ))), - } + self.load_snapshot_from_base(&self.base_dir, snapshot_id) } fn load_snapshot_from_base( @@ -159,33 +119,23 @@ impl RefStore { snapshot_id: &str, ) -> Result<Option<RefMap>, AppError> { let path = Self::snapshot_path_for_base(base_dir, snapshot_id); - let mut file = match crate::refs::open_nofollow(&path) { - Ok(file) => file, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), - Err(err) => return Err(err.into()), - }; - let metadata = file.metadata()?; - if metadata.len() > crate::refs::MAX_REFMAP_BYTES { - return Err(AppError::Internal( - "RefMap file exceeds 1MB size limit".into(), - )); - } - let mut json = String::with_capacity(metadata.len() as usize); - file.read_to_string(&mut json)?; - if json.len() as u64 > crate::refs::MAX_REFMAP_BYTES { - return Err(AppError::Internal( - "RefMap file exceeds 1MB size limit".into(), - )); - } - Ok(Some(serde_json::from_str(&json)?)) + let json = + match crate::private_file::read_private_bounded(&path, crate::refs::MAX_REFMAP_BYTES) { + Ok(json) => json, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err.into()), + }; + let refmap: RefMap = serde_json::from_slice(&json)?; + refmap.validate()?; + Ok(Some(refmap)) } pub fn set_latest(&self, snapshot_id: &str) -> Result<(), AppError> { self.with_write_lock(|| self.set_latest_unlocked(snapshot_id)) } - pub fn latest_snapshot_id(&self) -> Option<String> { - self.read_latest_snapshot_id().ok().flatten() + pub fn latest_snapshot_id(&self) -> Result<Option<String>, AppError> { + self.read_latest_snapshot_id() } fn save_snapshot_unlocked(&self, snapshot_id: &str, refmap: &RefMap) -> Result<(), AppError> { @@ -205,14 +155,20 @@ impl RefStore { } fn read_latest_snapshot_id(&self) -> Result<Option<String>, AppError> { - let mut file = match crate::refs::open_nofollow(&self.latest_path()) { - Ok(file) => file, + let bytes = match crate::private_file::read_private_bounded(&self.latest_path(), 65) { + Ok(bytes) => bytes, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), Err(err) => return Err(err.into()), }; - let mut id = String::new(); - file.read_to_string(&mut id)?; - Ok(Some(id.trim().to_string()).filter(|id| !id.is_empty())) + let id = std::str::from_utf8(&bytes) + .map_err(|_| AppError::invalid_input("latest_snapshot_id must be UTF-8"))? + .trim() + .to_string(); + if id.is_empty() { + return Ok(None); + } + validate_snapshot_id(&id)?; + Ok(Some(id)) } fn snapshot_path(&self, snapshot_id: &str) -> PathBuf { @@ -242,48 +198,6 @@ impl RefStore { self.base_dir.join("refstore.lock") } - fn discover_snapshot_base(&self, snapshot_id: &str) -> Result<Option<PathBuf>, AppError> { - let home = - home_dir().ok_or_else(|| AppError::Internal("HOME directory not found".into()))?; - let agent_dir = home.join(".agent-desktop"); - let mut matches = Vec::new(); - if agent_dir != self.base_dir - && Self::snapshot_path_for_base(&agent_dir, snapshot_id).is_file() - { - matches.push(agent_dir.clone()); - } - let sessions_dir = agent_dir.join("sessions"); - let Ok(entries) = std::fs::read_dir(sessions_dir) else { - return Ok(matches.into_iter().next()); - }; - for entry in entries.flatten() { - let path = entry.path(); - if path == self.base_dir { - continue; - } - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if validate_session_id(name).is_err() { - continue; - } - let Ok(file_type) = entry.file_type() else { - continue; - }; - if file_type.is_dir() && Self::snapshot_path_for_base(&path, snapshot_id).is_file() { - matches.push(path); - } - } - if matches.len() > 1 { - return Err(AppError::invalid_input_with_suggestion( - format!("Snapshot '{snapshot_id}' exists in more than one session"), - "Pass the matching --session for this rare snapshot id collision.", - )); - } - Ok(matches.into_iter().next()) - } - fn with_write_lock<T>(&self, f: impl FnOnce() -> Result<T, AppError>) -> Result<T, AppError> { let _lock = RefStoreLock::acquire(&self.lock_path())?; f() @@ -313,11 +227,12 @@ impl RefStore { } /// Pruning logic is a sibling `#[path]` module rather than a separate crate -/// module so it can access `base_dir`/`snapshots_dir` directly. Exposing them -/// as `pub(crate)` would widen the visibility surface to every module in the -/// crate; the path declaration keeps them private to this module tree. +/// module so it can access `base_dir`/`snapshots_dir` directly, without +/// widening those fields' visibility beyond this module tree. The module +/// itself is `pub(crate)` so its standalone, non-`RefStore` age-based-prune +/// helper is reachable from other commands that need the same TTL sweep. #[path = "refs_store_prune.rs"] -mod prune; +pub(crate) mod prune; #[cfg(test)] #[path = "refs_store_tests.rs"] @@ -326,3 +241,7 @@ mod tests; #[cfg(test)] #[path = "refs_store_trace_tests.rs"] mod trace_tests; + +#[cfg(test)] +#[path = "refs_store_transaction_tests.rs"] +mod transaction_tests; diff --git a/crates/core/src/refs_store_prune.rs b/crates/core/src/refs_store_prune.rs index 263e5b6..0154568 100644 --- a/crates/core/src/refs_store_prune.rs +++ b/crates/core/src/refs_store_prune.rs @@ -1,5 +1,5 @@ use super::{MAX_SAVED_SNAPSHOTS, RefStore, STALE_TMP_MAX_AGE}; -use crate::error::AppError; +use crate::AppError; use crate::refs::validate_snapshot_id; impl RefStore { @@ -7,40 +7,15 @@ impl RefStore { /// the temp write and the atomic rename. Runs under the store write lock; /// the age threshold keeps any in-flight write from another process safe. pub(crate) fn remove_tmp_files_older_than(&self, max_age: std::time::Duration) { - self.remove_tmp_files_in_dir(&self.base_dir, max_age); + remove_stale_files_in_dir(&self.base_dir, max_age, is_orphaned_tmp_file); let snapshots_dir = self.snapshots_dir(); - self.remove_tmp_files_in_dir(&snapshots_dir, max_age); + remove_stale_files_in_dir(&snapshots_dir, max_age, is_orphaned_tmp_file); let Ok(entries) = std::fs::read_dir(snapshots_dir) else { return; }; for entry in entries.flatten() { if entry.file_type().is_ok_and(|kind| kind.is_dir()) { - self.remove_tmp_files_in_dir(&entry.path(), max_age); - } - } - } - - fn remove_tmp_files_in_dir(&self, dir: &std::path::Path, max_age: std::time::Duration) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().is_none_or(|ext| ext != "tmp") { - continue; - } - let is_plain_file = entry.file_type().is_ok_and(|kind| kind.is_file()); - if !is_plain_file { - continue; - } - let stale = entry - .metadata() - .ok() - .and_then(|metadata| metadata.modified().ok()) - .and_then(|modified| modified.elapsed().ok()) - .is_some_and(|age| age >= max_age); - if stale { - let _ = std::fs::remove_file(&path); + remove_stale_files_in_dir(&entry.path(), max_age, is_orphaned_tmp_file); } } } @@ -86,3 +61,41 @@ impl RefStore { Ok(()) } } + +pub(crate) fn is_orphaned_tmp_file(path: &std::path::Path) -> bool { + path.extension().is_some_and(|ext| ext == "tmp") +} + +/// Removes plain files directly under `dir` that satisfy `matches` and whose +/// mtime is at least `max_age` old. Never descends into subdirectories and +/// never removes a directory itself. Shared by refstore `*.tmp` cleanup and +/// the sessionless clipboard image sweep so both age-based prunes stay on one +/// implementation. +pub(crate) fn remove_stale_files_in_dir( + dir: &std::path::Path, + max_age: std::time::Duration, + matches: impl Fn(&std::path::Path) -> bool, +) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if !matches(&path) { + continue; + } + let is_plain_file = entry.file_type().is_ok_and(|kind| kind.is_file()); + if !is_plain_file { + continue; + } + let stale = entry + .metadata() + .ok() + .and_then(|metadata| metadata.modified().ok()) + .and_then(|modified| modified.elapsed().ok()) + .is_some_and(|age| age >= max_age); + if stale { + let _ = std::fs::remove_file(&path); + } + } +} diff --git a/crates/core/src/refs_store_pruning_tests.rs b/crates/core/src/refs_store_pruning_tests.rs new file mode 100644 index 0000000..e04c0a8 --- /dev/null +++ b/crates/core/src/refs_store_pruning_tests.rs @@ -0,0 +1,88 @@ +use super::*; + +#[test] +fn save_new_snapshot_prunes_old_snapshots_without_removing_latest() { + let _guard = HomeGuard::new(); + let store = RefStore::new().unwrap(); + let first_id = store.save_new_snapshot(&map_with("First")).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(5)); + let mut latest_id = first_id.clone(); + + for i in 0..=MAX_SAVED_SNAPSHOTS { + latest_id = store + .save_new_snapshot(&map_with(&format!("Snapshot {i}"))) + .unwrap(); + } + + let count = std::fs::read_dir(store.snapshots_dir()) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .count(); + + assert!(count <= MAX_SAVED_SNAPSHOTS); + assert!(store.snapshot_path(&latest_id).is_file()); + assert!(!store.snapshot_path(&first_id).exists()); + assert_eq!( + store.latest_snapshot_id().unwrap().as_deref(), + Some(latest_id.as_str()) + ); +} + +#[test] +fn update_existing_refuses_to_recreate_a_missing_snapshot() { + let _guard = HomeGuard::new(); + let store = RefStore::new().unwrap(); + let snapshot_id = store.save_new_snapshot(&map_with("Original")).unwrap(); + std::fs::remove_dir_all(store.snapshots_dir().join(&snapshot_id)).unwrap(); + + let err = store + .update_existing_snapshot(&snapshot_id, "@e1", &entry("Original"), |_| Ok(())) + .unwrap_err(); + + assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND"); + assert!(!store.snapshot_path(&snapshot_id).exists()); +} + +#[test] +fn duplicate_snapshot_id_across_sessions_remains_isolated() { + let _guard = HomeGuard::new(); + let store_a = RefStore::for_session(Some("agent-a")).unwrap(); + let store_b = RefStore::for_session(Some("agent-b")).unwrap(); + let snapshot_id = store_a.save_new_snapshot(&map_with("A")).unwrap(); + store_b.save_snapshot(&snapshot_id, &map_with("B")).unwrap(); + + let err = RefStore::new() + .unwrap() + .load_snapshot(&snapshot_id) + .unwrap_err(); + + assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND"); + assert_eq!( + ref_name(&store_a.load_snapshot(&snapshot_id).unwrap()), + Some("A") + ); + assert_eq!( + ref_name(&store_b.load_snapshot(&snapshot_id).unwrap()), + Some("B") + ); +} + +#[test] +fn prune_never_removes_trace_segments() { + let _guard = HomeGuard::new(); + let store = RefStore::for_session(Some("trace-retention")).unwrap(); + let trace_dir = store.trace_dir(); + crate::trace::ensure_trace_dir(&trace_dir).unwrap(); + let segment = trace_dir.join("1234-5678.jsonl"); + std::fs::write(&segment, b"{}\n").unwrap(); + for index in 0..=MAX_SAVED_SNAPSHOTS { + let snapshot_id = format!("snap-{index:04}"); + store + .save_snapshot(&snapshot_id, &map_with(&snapshot_id)) + .unwrap(); + store.set_latest(&snapshot_id).unwrap(); + } + assert!(segment.is_file()); + assert!(trace_dir.is_dir()); +} diff --git a/crates/core/src/refs_store_tests.rs b/crates/core/src/refs_store_tests.rs index 0eb6ef1..1ad0435 100644 --- a/crates/core/src/refs_store_tests.rs +++ b/crates/core/src/refs_store_tests.rs @@ -6,23 +6,44 @@ use crate::{ }; fn entry(name: &str) -> RefEntry { + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; RefEntry { - pid: 7, - role: "button".into(), - name: Some(name.into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: Some(42), - available_actions: vec![crate::capability::CLICK.into()], - source_app: Some("TestApp".into()), - source_window_id: None, - source_window_title: Some("Test Window".into()), - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(7), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some(name.into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![crate::capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("TestApp".into()), + source_window_id: None, + source_window_title: Some("Test Window".into()), + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } @@ -33,7 +54,8 @@ fn map_with(name: &str) -> RefMap { } fn ref_name(map: &RefMap) -> Option<&str> { - map.get("@e1").and_then(|entry| entry.name.as_deref()) + map.get("@e1") + .and_then(|entry| entry.identity.name.as_deref()) } #[test] @@ -45,7 +67,7 @@ fn snapshot_roundtrip_updates_latest_pointer() { let snapshot_id = store.save_new_snapshot(&map).unwrap(); assert_eq!( - store.latest_snapshot_id().as_deref(), + store.latest_snapshot_id().unwrap().as_deref(), Some(snapshot_id.as_str()) ); assert_eq!(store.load(Some(&snapshot_id)).unwrap().len(), 1); @@ -75,7 +97,7 @@ fn concurrent_writers_preserve_all_snapshots() { for id in &ids { assert_eq!(store.load_snapshot(id).unwrap().len(), 1); } - let latest = store.latest_snapshot_id().unwrap(); + let latest = store.latest_snapshot_id().unwrap().unwrap(); assert!(ids.iter().any(|id| id == &latest)); } @@ -102,13 +124,13 @@ fn sessions_are_isolated_from_default_store() { ); assert!(session_b.load(None).is_err()); assert_ne!( - default_store.latest_snapshot_id(), - session_a.latest_snapshot_id() + default_store.latest_snapshot_id().unwrap(), + session_a.latest_snapshot_id().unwrap() ); } #[test] -fn explicit_snapshot_id_loads_across_session_namespaces() { +fn explicit_snapshot_id_remains_in_its_session_namespace() { let _guard = HomeGuard::new(); let default_store = RefStore::new().unwrap(); let session_a = RefStore::for_session(Some("agent-a")).unwrap(); @@ -116,12 +138,10 @@ fn explicit_snapshot_id_loads_across_session_namespaces() { let snapshot_id = session_a.save_new_snapshot(&map_with("Session A")).unwrap(); + assert!(default_store.load(Some(&snapshot_id)).is_err()); + assert!(session_b.load(Some(&snapshot_id)).is_err()); assert_eq!( - ref_name(&default_store.load(Some(&snapshot_id)).unwrap()), - Some("Session A") - ); - assert_eq!( - ref_name(&session_b.load(Some(&snapshot_id)).unwrap()), + ref_name(&session_a.load(Some(&snapshot_id)).unwrap()), Some("Session A") ); assert!(default_store.load(None).is_err()); @@ -129,29 +149,30 @@ fn explicit_snapshot_id_loads_across_session_namespaces() { } #[test] -fn save_existing_snapshot_updates_discovered_owner_without_promoting_latest() { +fn update_existing_snapshot_cannot_cross_session_namespaces() { let _guard = HomeGuard::new(); let default_store = RefStore::new().unwrap(); let session_a = RefStore::for_session(Some("agent-a")).unwrap(); let snapshot_id = session_a.save_new_snapshot(&map_with("Session A")).unwrap(); - default_store - .save_existing_snapshot(&snapshot_id, &map_with("Updated")) - .unwrap(); + let err = default_store + .update_existing_snapshot(&snapshot_id, "@e1", &entry("Session A"), |_| Ok(())) + .unwrap_err(); + assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND"); assert_eq!( ref_name(&session_a.load(Some(&snapshot_id)).unwrap()), - Some("Updated") + Some("Session A") ); - assert!(default_store.latest_snapshot_id().is_none()); + assert!(default_store.latest_snapshot_id().unwrap().is_none()); assert_eq!( - session_a.latest_snapshot_id().as_deref(), + session_a.latest_snapshot_id().unwrap().as_deref(), Some(snapshot_id.as_str()) ); } #[test] -fn duplicate_explicit_snapshot_id_requires_session() { +fn duplicate_explicit_snapshot_ids_remain_session_scoped() { let _guard = HomeGuard::new(); let default_store = RefStore::new().unwrap(); let session_a = RefStore::for_session(Some("agent-a")).unwrap(); @@ -166,16 +187,19 @@ fn duplicate_explicit_snapshot_id_requires_session() { let err = default_store.load(Some("sdup1")).unwrap_err(); - assert_eq!(err.code(), "INVALID_ARGS"); - assert!(err.suggestion().unwrap().contains("--session")); + assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND"); assert_eq!( ref_name(&session_a.load(Some("sdup1")).unwrap()), Some("Session A") ); + assert_eq!( + ref_name(&session_b.load(Some("sdup1")).unwrap()), + Some("Session B") + ); } #[test] -fn discover_skips_invalid_session_names_when_detecting_collisions() { +fn default_store_does_not_discover_session_snapshots() { let _guard = HomeGuard::new(); let default_store = RefStore::new().unwrap(); let session_a = RefStore::for_session(Some("agent-a")).unwrap(); @@ -192,8 +216,10 @@ fn discover_skips_invalid_session_names_when_detecting_collisions() { ) .unwrap(); + let err = default_store.load(Some("sdup2")).unwrap_err(); + assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND"); assert_eq!( - ref_name(&default_store.load(Some("sdup2")).unwrap()), + ref_name(&session_a.load(Some("sdup2")).unwrap()), Some("Session A") ); } @@ -236,23 +262,26 @@ fn read_latest_rejects_symlinked_pointer() { let err = store.load_latest().unwrap_err(); assert_eq!(err.code(), "INTERNAL"); - assert!(store.latest_snapshot_id().is_none()); + assert!(store.latest_snapshot_id().is_err()); } #[test] -fn save_existing_snapshot_does_not_promote_latest_pointer() { +fn update_existing_snapshot_does_not_promote_latest_pointer() { let _guard = HomeGuard::new(); let store = RefStore::new().unwrap(); - let mut first = map_with("First"); - let first_id = store.save_new_snapshot(&first).unwrap(); + let first_id = store.save_new_snapshot(&map_with("First")).unwrap(); let second_id = store.save_new_snapshot(&map_with("Second")).unwrap(); - first.allocate(entry("First Child")); - store.save_existing_snapshot(&first_id, &first).unwrap(); + store + .update_existing_snapshot(&first_id, "@e1", &entry("First"), |map| { + map.allocate(entry("First Child")); + Ok(()) + }) + .unwrap(); assert_eq!( - store.latest_snapshot_id().as_deref(), + store.latest_snapshot_id().unwrap().as_deref(), Some(second_id.as_str()) ); assert_eq!(store.load(Some(&first_id)).unwrap().len(), 2); @@ -267,7 +296,7 @@ fn default_store_migrates_legacy_latest_refmap() { let loaded = store.load_latest().unwrap(); assert_eq!(loaded.len(), 1); - assert!(store.latest_snapshot_id().is_some()); + assert!(store.latest_snapshot_id().unwrap().is_some()); } #[test] @@ -279,7 +308,7 @@ fn session_store_does_not_migrate_global_legacy_refmap() { let err = store.load(None).unwrap_err(); assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND"); - assert!(store.latest_snapshot_id().is_none()); + assert!(store.latest_snapshot_id().unwrap().is_none()); } #[test] @@ -307,82 +336,5 @@ fn stale_tmp_files_are_swept_and_fresh_ones_kept() { assert!(base_tmp.exists()); } -#[test] -fn save_new_snapshot_prunes_old_snapshots_without_removing_latest() { - let _guard = HomeGuard::new(); - let store = RefStore::new().unwrap(); - let first_id = store.save_new_snapshot(&map_with("First")).unwrap(); - std::thread::sleep(std::time::Duration::from_millis(5)); - let mut latest_id = first_id.clone(); - - for i in 0..=MAX_SAVED_SNAPSHOTS { - latest_id = store - .save_new_snapshot(&map_with(&format!("Snapshot {i}"))) - .unwrap(); - } - - let count = std::fs::read_dir(store.snapshots_dir()) - .unwrap() - .filter_map(Result::ok) - .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) - .count(); - - assert!(count <= MAX_SAVED_SNAPSHOTS); - assert!(store.snapshot_path(&latest_id).is_file()); - assert!(!store.snapshot_path(&first_id).exists()); - assert_eq!( - store.latest_snapshot_id().as_deref(), - Some(latest_id.as_str()) - ); -} - -#[test] -fn save_existing_recreates_snapshot_pruned_from_every_store() { - let _guard = HomeGuard::new(); - let store = RefStore::new().unwrap(); - let snapshot_id = store.save_new_snapshot(&map_with("Original")).unwrap(); - std::fs::remove_dir_all(store.snapshots_dir().join(&snapshot_id)).unwrap(); - - store - .save_existing_snapshot(&snapshot_id, &map_with("Recreated")) - .unwrap(); - - let reloaded = store.load_snapshot(&snapshot_id).unwrap(); - assert_eq!(ref_name(&reloaded), Some("Recreated")); -} - -#[test] -fn duplicate_snapshot_id_across_sessions_is_rejected_on_load() { - let _guard = HomeGuard::new(); - let store_a = RefStore::for_session(Some("agent-a")).unwrap(); - let store_b = RefStore::for_session(Some("agent-b")).unwrap(); - let snapshot_id = store_a.save_new_snapshot(&map_with("A")).unwrap(); - store_b.save_snapshot(&snapshot_id, &map_with("B")).unwrap(); - - let err = RefStore::new() - .unwrap() - .load_snapshot(&snapshot_id) - .unwrap_err(); - - assert_eq!(err.code(), "INVALID_ARGS"); - assert!(err.to_string().contains("more than one session")); -} - -#[test] -fn prune_never_removes_trace_segments() { - let _guard = HomeGuard::new(); - let store = RefStore::for_session(Some("trace-retention")).unwrap(); - let trace_dir = store.trace_dir(); - std::fs::create_dir_all(&trace_dir).unwrap(); - let segment = trace_dir.join("1234-5678.jsonl"); - std::fs::write(&segment, b"{}\n").unwrap(); - for index in 0..=MAX_SAVED_SNAPSHOTS { - let snapshot_id = format!("snap-{index:04}"); - store - .save_snapshot(&snapshot_id, &map_with(&snapshot_id)) - .unwrap(); - store.set_latest(&snapshot_id).unwrap(); - } - assert!(segment.is_file()); - assert!(trace_dir.is_dir()); -} +#[path = "refs_store_pruning_tests.rs"] +mod pruning_tests; diff --git a/crates/core/src/refs_store_transaction_tests.rs b/crates/core/src/refs_store_transaction_tests.rs new file mode 100644 index 0000000..dfb33d7 --- /dev/null +++ b/crates/core/src/refs_store_transaction_tests.rs @@ -0,0 +1,180 @@ +use super::*; +use crate::{RefCapabilities, RefEntryIdentity, RefGeometry, RefProcess, RefScope, RefSource}; + +fn entry(name: &str, root_ref: Option<&str>) -> crate::RefEntry { + crate::RefEntry { + process: RefProcess { + pid: crate::ProcessId::new(7), + process_instance: Some("instance-1".into()), + }, + identity: RefEntryIdentity { + role: "button".into(), + name: Some(name.into()), + value: None, + description: None, + native_id: None, + }, + geometry: RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: RefCapabilities { + states: Vec::new(), + available_actions: vec![crate::capability::CLICK.into()], + }, + source: RefSource { + source_app: Some("Fixture".into()), + source_window_id: Some("w-1".into()), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::SnapshotSurface::Window, + }, + scope: RefScope { + root_ref: root_ref.map(str::to_string), + path_is_absolute: root_ref.is_some(), + path: smallvec::SmallVec::new(), + }, + } +} + +#[test] +fn subprocess_transaction_writer() { + let Some(snapshot_id) = std::env::var_os("AGENT_DESKTOP_TX_SNAPSHOT") else { + return; + }; + let root_ref = std::env::var("AGENT_DESKTOP_TX_ROOT_REF").unwrap(); + let label = std::env::var("AGENT_DESKTOP_TX_LABEL").unwrap(); + let ready = PathBuf::from(std::env::var_os("AGENT_DESKTOP_TX_READY").unwrap()); + let go = PathBuf::from(std::env::var_os("AGENT_DESKTOP_TX_GO").unwrap()); + let store = RefStore::new().unwrap(); + let snapshot_id = snapshot_id.to_string_lossy(); + let expected = store + .load_snapshot(&snapshot_id) + .unwrap() + .get(&root_ref) + .unwrap() + .clone(); + std::fs::write(&ready, b"ready").unwrap(); + while !go.is_file() { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + store + .update_existing_snapshot(&snapshot_id, &root_ref, &expected, |current| { + current.try_allocate(entry(&label, Some(&root_ref)))?; + Ok(()) + }) + .unwrap(); +} + +#[test] +fn two_process_transactions_preserve_disjoint_drill_updates() { + let home = std::env::temp_dir().join(format!( + "agent-desktop-refstore-two-process-{}", + crate::refs::new_snapshot_id() + )); + std::fs::create_dir_all(&home).unwrap(); + let previous = crate::refs::set_home_override(Some(home.clone())); + let store = RefStore::new().unwrap(); + let mut map = RefMap::new(); + map.try_allocate(entry("Root A", None)).unwrap(); + map.try_allocate(entry("Root B", None)).unwrap(); + let snapshot_id = store.save_new_snapshot(&map).unwrap(); + let go = home.join("go"); + let mut children = Vec::new(); + for (index, (root_ref, label)) in [("@e1", "Child A"), ("@e2", "Child B")] + .into_iter() + .enumerate() + { + let ready = home.join(format!("ready-{index}")); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("refs_store::transaction_tests::subprocess_transaction_writer") + .arg("--nocapture") + .env("HOME", &home) + .env("AGENT_DESKTOP_TX_SNAPSHOT", &snapshot_id) + .env("AGENT_DESKTOP_TX_ROOT_REF", root_ref) + .env("AGENT_DESKTOP_TX_LABEL", label) + .env("AGENT_DESKTOP_TX_READY", &ready) + .env("AGENT_DESKTOP_TX_GO", &go) + .spawn() + .unwrap(); + children.push((child, ready)); + } + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while children.iter().any(|(_, ready)| !ready.is_file()) { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + std::fs::write(&go, b"go").unwrap(); + for (child, _) in &mut children { + assert!(child.wait().unwrap().success()); + } + let updated = store.load_snapshot(&snapshot_id).unwrap(); + assert_eq!(updated.len(), 4); + assert!(updated.get("@e1").is_some()); + assert!(updated.get("@e2").is_some()); + let names = ["@e3", "@e4"] + .into_iter() + .map(|id| updated.get(id).unwrap().identity.name.as_deref().unwrap()) + .collect::<std::collections::HashSet<_>>(); + assert_eq!( + names, + std::collections::HashSet::from(["Child A", "Child B"]) + ); + crate::refs::set_home_override(previous); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn subprocess_crash_before_snapshot_rename() { + let Some(snapshot_id) = std::env::var_os("AGENT_DESKTOP_CRASH_SNAPSHOT") else { + return; + }; + let store = RefStore::new().unwrap(); + let snapshot_id = snapshot_id.to_string_lossy(); + let expected = store + .load_snapshot(&snapshot_id) + .unwrap() + .get("@e1") + .unwrap() + .clone(); + let _ = store.update_existing_snapshot(&snapshot_id, "@e1", &expected, |current| { + current.try_allocate(entry("Never committed", Some("@e1")))?; + Ok(()) + }); + panic!("crash failpoint did not abort"); +} + +#[test] +fn crash_before_rename_preserves_the_previous_snapshot() { + let home = std::env::temp_dir().join(format!( + "agent-desktop-refstore-crash-{}", + crate::refs::new_snapshot_id() + )); + std::fs::create_dir_all(&home).unwrap(); + let previous = crate::refs::set_home_override(Some(home.clone())); + let store = RefStore::new().unwrap(); + let mut map = RefMap::new(); + map.try_allocate(entry("Original", None)).unwrap(); + let snapshot_id = store.save_new_snapshot(&map).unwrap(); + let snapshot_path = store.snapshot_path(&snapshot_id); + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("refs_store::transaction_tests::subprocess_crash_before_snapshot_rename") + .arg("--nocapture") + .env("HOME", &home) + .env("AGENT_DESKTOP_CRASH_SNAPSHOT", &snapshot_id) + .env("AGENT_DESKTOP_TEST_CRASH_BEFORE_RENAME", &snapshot_path) + .status() + .unwrap(); + + assert!(!status.success()); + let persisted = store.load_snapshot(&snapshot_id).unwrap(); + assert_eq!(persisted.len(), 1); + assert_eq!( + persisted.get("@e1").unwrap().identity.name.as_deref(), + Some("Original") + ); + crate::refs::set_home_override(previous); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/crates/core/src/refs_test_support.rs b/crates/core/src/refs_test_support.rs index 9fab538..d3f5f94 100644 --- a/crates/core/src/refs_test_support.rs +++ b/crates/core/src/refs_test_support.rs @@ -16,6 +16,10 @@ impl HomeGuard { let prev = crate::refs::set_home_override(Some(dir.path().to_path_buf())); Self { _dir: dir, prev } } + + pub(crate) fn path(&self) -> &Path { + self._dir.path() + } } impl Drop for HomeGuard { diff --git a/crates/core/src/refs_tests.rs b/crates/core/src/refs_tests.rs index 9fa5cc9..835a027 100644 --- a/crates/core/src/refs_tests.rs +++ b/crates/core/src/refs_tests.rs @@ -3,22 +3,37 @@ use crate::refs_test_support::HomeGuard; fn entry(role: &str, name: Option<&str>) -> RefEntry { RefEntry { - pid: 1, - role: role.into(), - name: name.map(String::from), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: role.into(), + name: name.map(String::from), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec!["Click".into()], + }, + source: crate::RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } @@ -35,27 +50,48 @@ fn test_allocate_sequential() { #[test] fn test_get_existing() { let mut map = RefMap::new(); + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; let ref_id = map.allocate(RefEntry { - pid: 42, - role: "textfield".into(), - name: None, - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: Some(12345), - available_actions: vec![], - source_app: Some("Finder".into()), - source_window_id: None, - source_window_title: Some("Documents".into()), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "textfield".into(), + name: None, + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![], + }, + source: crate::RefSource { + source_app: Some("Finder".into()), + source_window_id: None, + source_window_title: Some("Documents".into()), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); let retrieved = map.get(&ref_id).unwrap(); - assert_eq!(retrieved.pid, 42); - assert_eq!(retrieved.role, "textfield"); + assert_eq!(retrieved.process.pid, 42); + assert_eq!(retrieved.identity.role, "textfield"); } #[test] @@ -89,10 +125,8 @@ fn test_remove_by_root_ref() { map.allocate(base.clone()); - let drilled = RefEntry { - root_ref: Some("@e1".into()), - ..base - }; + let mut drilled = base; + drilled.scope.root_ref = Some("@e1".into()); map.allocate(drilled.clone()); map.allocate(drilled); assert_eq!(map.len(), 3); @@ -113,10 +147,8 @@ fn test_counter_continues_after_skeleton_into_drill_down() { .unwrap(); assert_eq!(last_skeleton, "@e10"); - let drilled = RefEntry { - root_ref: Some("@e3".into()), - ..skeleton_entry - }; + let mut drilled = skeleton_entry; + drilled.scope.root_ref = Some("@e3".into()); let first_drilled = map.allocate(drilled.clone()); let second_drilled = map.allocate(drilled); @@ -138,14 +170,12 @@ fn test_counter_continues_after_skeleton_into_drill_down() { #[test] fn test_root_ref_serde_roundtrip() { - let entry = RefEntry { - root_ref: Some("@e5".into()), - ..entry("button", None) - }; + let mut entry = entry("button", None); + entry.scope.root_ref = Some("@e5".into()); let json = serde_json::to_string(&entry).unwrap(); assert!(json.contains("root_ref")); let back: RefEntry = serde_json::from_str(&json).unwrap(); - assert_eq!(back.root_ref.as_deref(), Some("@e5")); + assert_eq!(back.scope.root_ref.as_deref(), Some("@e5")); } #[test] @@ -181,31 +211,52 @@ fn test_serialize_with_size_check_accepts_normal() { fn test_save_load_roundtrip_with_home_override() { let _guard = HomeGuard::new(); let mut map = RefMap::new(); + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; map.allocate(RefEntry { - pid: 7, - role: "button".into(), - name: Some("Send".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: Some(42), - available_actions: vec!["Click".into()], - source_app: Some("TestApp".into()), - source_window_id: None, - source_window_title: Some("Test Window".into()), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(7), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Send".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec!["Click".into()], + }, + source: crate::RefSource { + source_app: Some("TestApp".into()), + source_window_id: None, + source_window_title: Some("Test Window".into()), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, }); map.save().expect("save should succeed under HomeGuard"); let loaded = RefMap::load().expect("load should succeed"); assert_eq!(loaded.len(), 1); let entry = loaded.get("@e1").unwrap(); - assert_eq!(entry.pid, 7); - assert_eq!(entry.name.as_deref(), Some("Send")); + assert_eq!(entry.process.pid, 7); + assert_eq!(entry.identity.name.as_deref(), Some("Send")); } #[test] @@ -261,7 +312,7 @@ fn test_save_oversize_preserves_previous_file() { let reloaded = RefMap::load().expect("previous file must still load"); assert_eq!(reloaded.len(), 1); let entry = reloaded.get("@e1").unwrap(); - assert_eq!(entry.name.as_deref(), Some("Original")); + assert_eq!(entry.identity.name.as_deref(), Some("Original")); } #[cfg(unix)] @@ -275,6 +326,8 @@ fn test_write_private_file_ignores_stale_predictable_tmp_symlink() { .as_nanos() )); std::fs::create_dir_all(&dir).unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); let path = dir.join("refmap.json"); let target = dir.join("target.json"); let stale = path.with_extension("tmp"); diff --git a/crates/core/src/refs_validate.rs b/crates/core/src/refs_validate.rs new file mode 100644 index 0000000..8ad82b7 --- /dev/null +++ b/crates/core/src/refs_validate.rs @@ -0,0 +1,72 @@ +use crate::{AppError, RefEntry}; + +pub(crate) fn validate_ref_entry(entry: &RefEntry) -> Result<(), AppError> { + const MAX_FIELD_BYTES: usize = 65_536; + const MAX_PATH_DEPTH: usize = 256; + if entry.process.pid.get() == 0 + || entry + .process + .process_instance + .as_deref() + .is_none_or(str::is_empty) + { + return Err(AppError::invalid_input( + "RefEntry requires a positive pid and process instance", + )); + } + if !crate::Role::is_canonical(&entry.identity.role) || entry.identity.role == "unknown" { + return Err(AppError::invalid_input("RefEntry contains an invalid role")); + } + for field in [ + entry.identity.name.as_deref(), + entry.identity.value.as_deref(), + entry.identity.description.as_deref(), + entry.source.source_app.as_deref(), + entry.source.source_window_id.as_deref(), + entry.source.source_window_title.as_deref(), + entry.process.process_instance.as_deref(), + ] + .into_iter() + .flatten() + { + if field.len() > MAX_FIELD_BYTES { + return Err(AppError::invalid_input( + "RefEntry string evidence exceeds the field limit", + )); + } + } + if let Some(identifier) = &entry.identity.native_id + && (identifier.value.trim().is_empty() + || identifier.value.len() > MAX_FIELD_BYTES + || identifier.kind == crate::IdentifierKind::Unknown) + { + return Err(AppError::invalid_input( + "RefEntry contains invalid typed identifier evidence", + )); + } + if entry.scope.path.len() > MAX_PATH_DEPTH { + return Err(AppError::invalid_input("RefEntry path is too deep")); + } + if let Some(bounds) = entry.geometry.bounds { + bounds.validate()?; + if entry.geometry.bounds_hash != bounds.bounds_hash() { + return Err(AppError::invalid_input( + "RefEntry bounds hash does not match its geometry", + )); + } + } + if entry.capabilities.states.len() > 256 + || entry.capabilities.available_actions.len() > 256 + || entry + .capabilities + .states + .iter() + .chain(entry.capabilities.available_actions.iter()) + .any(|value| value.len() > 256) + { + return Err(AppError::invalid_input( + "RefEntry state or action evidence exceeds its limit", + )); + } + Ok(()) +} diff --git a/crates/core/src/renderer_accessibility.rs b/crates/core/src/renderer_accessibility.rs new file mode 100644 index 0000000..26da924 --- /dev/null +++ b/crates/core/src/renderer_accessibility.rs @@ -0,0 +1,52 @@ +use crate::{ + AppError, PlatformAdapter, + live_locator::{ObservationRequest, ObservationRoot, ObservedTree}, +}; + +pub(crate) fn observe_tree( + adapter: &dyn PlatformAdapter, + root: ObservationRoot<'_>, + request: &ObservationRequest, +) -> Result<ObservedTree, AppError> { + let mut activated = false; + loop { + match adapter.observe_tree(root, request) { + Ok(tree) => return Ok(tree), + Err(error) if error.requires_renderer_accessibility_activation() => { + if !activated { + let acquired = adapter.acquire_interaction_lease(request.deadline)?; + adapter.activate_renderer_accessibility(root_process(root)?, &acquired)?; + drop(acquired); + activated = true; + } + let remaining = request.deadline.remaining(); + if remaining.is_zero() { + return Err(request.deadline.timeout_error().into()); + } + std::thread::sleep(remaining.min(std::time::Duration::from_millis(25))); + } + Err(error) => return Err(AppError::Adapter(error)), + } + } +} + +fn root_process(root: ObservationRoot<'_>) -> Result<crate::ProcessIdentity, AppError> { + match root { + ObservationRoot::Window(window) => window + .process_instance + .as_deref() + .map(|instance| crate::ProcessIdentity::new(window.pid, instance)), + ObservationRoot::Element { entry, .. } => entry + .process + .process_instance + .as_deref() + .map(|instance| crate::ProcessIdentity::new(entry.process.pid, instance)), + } + .ok_or_else(|| { + crate::AdapterError::stale_ref("renderer process instance is unavailable").into() + }) +} + +#[cfg(test)] +#[path = "renderer_accessibility_tests.rs"] +mod tests; diff --git a/crates/core/src/renderer_accessibility_tests.rs b/crates/core/src/renderer_accessibility_tests.rs new file mode 100644 index 0000000..f80be94 --- /dev/null +++ b/crates/core/src/renderer_accessibility_tests.rs @@ -0,0 +1,102 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU32, Ordering}, +}; + +struct LeaseFlag(Arc<AtomicBool>); + +impl Drop for LeaseFlag { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +struct RendererAdapter { + lease_held: Arc<AtomicBool>, + observations: AtomicU32, + activations: AtomicU32, +} + +impl ObservationOps for RendererAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result<ObservedTree, crate::AdapterError> { + let attempt = self.observations.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + return Err( + crate::AdapterError::renderer_accessibility_activation_required( + "activation required", + ), + ); + } + assert!(!self.lease_held.load(Ordering::SeqCst)); + crate::adapter::observed_tree( + &root, + crate::AccessibilityNode { + ref_id: None, + role: "window".into(), + identity: Default::default(), + presentation: Default::default(), + children_count: None, + children: Vec::new(), + }, + ) + } +} + +impl ActionOps for RendererAdapter {} +impl InputOps for RendererAdapter {} + +impl SystemOps for RendererAdapter { + fn acquire_interaction_lease( + &self, + deadline: crate::Deadline, + ) -> Result<crate::InteractionLease, crate::AdapterError> { + self.lease_held.store(true, Ordering::SeqCst); + crate::InteractionLease::guarded(deadline, LeaseFlag(Arc::clone(&self.lease_held))) + } + + fn activate_renderer_accessibility( + &self, + _process: crate::ProcessIdentity, + _lease: &crate::InteractionLease, + ) -> Result<(), crate::AdapterError> { + assert!(self.lease_held.load(Ordering::SeqCst)); + self.activations.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +#[test] +fn activation_lease_is_dropped_before_observation_retry() { + let held = Arc::new(AtomicBool::new(false)); + let adapter = RendererAdapter { + lease_held: Arc::clone(&held), + observations: AtomicU32::new(0), + activations: AtomicU32::new(0), + }; + let window = crate::WindowInfo { + id: "w-1".into(), + title: "Fixture".into(), + app: "Fixture".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("instance-1".into()), + bounds: None, + state: Default::default(), + }; + let deadline = crate::Deadline::after(500).unwrap(); + let request = ObservationRequest::snapshot(&crate::TreeOptions::default(), deadline) + .validate() + .unwrap(); + + let tree = observe_tree(&adapter, ObservationRoot::Window(&window), &request).unwrap(); + + assert_eq!(tree.node_count(), 1); + assert_eq!(adapter.observations.load(Ordering::SeqCst), 2); + assert_eq!(adapter.activations.load(Ordering::SeqCst), 1); + assert!(!held.load(Ordering::SeqCst)); +} diff --git a/crates/core/src/resolve_attempt_outcome.rs b/crates/core/src/resolve_attempt_outcome.rs new file mode 100644 index 0000000..64a5039 --- /dev/null +++ b/crates/core/src/resolve_attempt_outcome.rs @@ -0,0 +1,7 @@ +use crate::{AdapterError, adapter::NativeHandle}; + +pub(crate) enum ResolveAttemptOutcome { + Resolved(NativeHandle), + Failed(AdapterError), + DeadlinePassed, +} diff --git a/crates/core/src/resolved_element.rs b/crates/core/src/resolved_element.rs deleted file mode 100644 index a689be9..0000000 --- a/crates/core/src/resolved_element.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::adapter::{NativeHandle, PlatformAdapter}; - -pub(crate) struct ResolvedElement<'a> { - adapter: &'a dyn PlatformAdapter, - handle: NativeHandle, -} - -impl<'a> ResolvedElement<'a> { - pub(crate) fn new(adapter: &'a dyn PlatformAdapter, handle: NativeHandle) -> Self { - Self { adapter, handle } - } - - pub(crate) fn handle(&self) -> &NativeHandle { - &self.handle - } -} - -impl Drop for ResolvedElement<'_> { - fn drop(&mut self) { - let _ = self.adapter.release_handle(&self.handle); - } -} diff --git a/crates/core/src/retry_disposition.rs b/crates/core/src/retry_disposition.rs new file mode 100644 index 0000000..c4075d5 --- /dev/null +++ b/crates/core/src/retry_disposition.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RetryDisposition { + #[default] + Unknown, + Safe, + Unsafe, +} diff --git a/crates/core/src/retryability.rs b/crates/core/src/retryability.rs new file mode 100644 index 0000000..7b8fe10 --- /dev/null +++ b/crates/core/src/retryability.rs @@ -0,0 +1,20 @@ +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum Retryability { + #[default] + Unspecified, + Retryable, + NonRetryable, +} + +impl Retryability { + pub(crate) fn from_details(details: &serde_json::Value) -> Self { + match details + .get("retryable") + .and_then(serde_json::Value::as_bool) + { + Some(true) => Self::Retryable, + Some(false) => Self::NonRetryable, + None => Self::Unspecified, + } + } +} diff --git a/crates/core/src/role.rs b/crates/core/src/role.rs new file mode 100644 index 0000000..9db94ef --- /dev/null +++ b/crates/core/src/role.rs @@ -0,0 +1,298 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Role { + Alert, + AlertDialog, + Application, + Article, + Banner, + Browser, + Button, + Cell, + Checkbox, + ColorWell, + Column, + ComboBox, + Complementary, + ContentInfo, + DateField, + Definition, + Dialog, + Disclosure, + DockItem, + Document, + Drawer, + Form, + Grid, + Group, + Handle, + Heading, + HelpTag, + Image, + Incrementor, + LayoutItem, + LevelIndicator, + Link, + List, + ListBox, + Log, + Main, + Marquee, + Matte, + Math, + Menu, + MenuButton, + MenuItem, + Navigation, + Note, + Option, + Outline, + Paragraph, + Popover, + ProgressBar, + RadioButton, + RadioGroup, + Region, + RelevanceIndicator, + Row, + Ruler, + RulerMarker, + ScrollArea, + ScrollBar, + Search, + Separator, + Sheet, + Slider, + Splitter, + StaticText, + Status, + Switch, + Tab, + TabList, + TabPanel, + Table, + TextField, + Term, + TimeField, + Timer, + Toolbar, + Tooltip, + TreeItem, + WebArea, + Window, + Unknown, +} + +impl Role { + pub fn from_token(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "alert" => Self::Alert, + "alertdialog" => Self::AlertDialog, + "application" => Self::Application, + "article" => Self::Article, + "banner" => Self::Banner, + "browser" => Self::Browser, + "button" => Self::Button, + "cell" => Self::Cell, + "checkbox" => Self::Checkbox, + "colorwell" => Self::ColorWell, + "column" => Self::Column, + "combobox" => Self::ComboBox, + "complementary" => Self::Complementary, + "contentinfo" => Self::ContentInfo, + "datefield" => Self::DateField, + "definition" => Self::Definition, + "dialog" => Self::Dialog, + "disclosure" => Self::Disclosure, + "dockitem" => Self::DockItem, + "document" => Self::Document, + "drawer" => Self::Drawer, + "form" => Self::Form, + "grid" => Self::Grid, + "group" => Self::Group, + "handle" => Self::Handle, + "heading" => Self::Heading, + "helptag" => Self::HelpTag, + "image" => Self::Image, + "incrementor" => Self::Incrementor, + "layoutitem" => Self::LayoutItem, + "levelindicator" => Self::LevelIndicator, + "link" => Self::Link, + "list" => Self::List, + "listbox" => Self::ListBox, + "log" => Self::Log, + "main" => Self::Main, + "marquee" => Self::Marquee, + "matte" => Self::Matte, + "math" => Self::Math, + "menu" => Self::Menu, + "menubutton" => Self::MenuButton, + "menuitem" => Self::MenuItem, + "navigation" => Self::Navigation, + "note" => Self::Note, + "option" => Self::Option, + "outline" => Self::Outline, + "paragraph" => Self::Paragraph, + "popover" => Self::Popover, + "progressbar" => Self::ProgressBar, + "radiobutton" => Self::RadioButton, + "radiogroup" => Self::RadioGroup, + "region" => Self::Region, + "relevanceindicator" => Self::RelevanceIndicator, + "row" => Self::Row, + "ruler" => Self::Ruler, + "rulermarker" => Self::RulerMarker, + "scrollarea" => Self::ScrollArea, + "scrollbar" => Self::ScrollBar, + "search" => Self::Search, + "separator" => Self::Separator, + "sheet" => Self::Sheet, + "slider" => Self::Slider, + "splitter" => Self::Splitter, + "statictext" => Self::StaticText, + "status" => Self::Status, + "switch" => Self::Switch, + "tab" => Self::Tab, + "tablist" => Self::TabList, + "tabpanel" => Self::TabPanel, + "table" => Self::Table, + "textfield" => Self::TextField, + "term" => Self::Term, + "timefield" => Self::TimeField, + "timer" => Self::Timer, + "toolbar" => Self::Toolbar, + "tooltip" => Self::Tooltip, + "treeitem" => Self::TreeItem, + "webarea" => Self::WebArea, + "window" => Self::Window, + _ => Self::Unknown, + } + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::Alert => "alert", + Self::AlertDialog => "alertdialog", + Self::Application => "application", + Self::Article => "article", + Self::Banner => "banner", + Self::Browser => "browser", + Self::Button => "button", + Self::Cell => "cell", + Self::Checkbox => "checkbox", + Self::ColorWell => "colorwell", + Self::Column => "column", + Self::ComboBox => "combobox", + Self::Complementary => "complementary", + Self::ContentInfo => "contentinfo", + Self::DateField => "datefield", + Self::Definition => "definition", + Self::Dialog => "dialog", + Self::Disclosure => "disclosure", + Self::DockItem => "dockitem", + Self::Document => "document", + Self::Drawer => "drawer", + Self::Form => "form", + Self::Grid => "grid", + Self::Group => "group", + Self::Handle => "handle", + Self::Heading => "heading", + Self::HelpTag => "helptag", + Self::Image => "image", + Self::Incrementor => "incrementor", + Self::LayoutItem => "layoutitem", + Self::LevelIndicator => "levelindicator", + Self::Link => "link", + Self::List => "list", + Self::ListBox => "listbox", + Self::Log => "log", + Self::Main => "main", + Self::Marquee => "marquee", + Self::Matte => "matte", + Self::Math => "math", + Self::Menu => "menu", + Self::MenuButton => "menubutton", + Self::MenuItem => "menuitem", + Self::Navigation => "navigation", + Self::Note => "note", + Self::Option => "option", + Self::Outline => "outline", + Self::Paragraph => "paragraph", + Self::Popover => "popover", + Self::ProgressBar => "progressbar", + Self::RadioButton => "radiobutton", + Self::RadioGroup => "radiogroup", + Self::Region => "region", + Self::RelevanceIndicator => "relevanceindicator", + Self::Row => "row", + Self::Ruler => "ruler", + Self::RulerMarker => "rulermarker", + Self::ScrollArea => "scrollarea", + Self::ScrollBar => "scrollbar", + Self::Search => "search", + Self::Separator => "separator", + Self::Sheet => "sheet", + Self::Slider => "slider", + Self::Splitter => "splitter", + Self::StaticText => "statictext", + Self::Status => "status", + Self::Switch => "switch", + Self::Tab => "tab", + Self::TabList => "tablist", + Self::TabPanel => "tabpanel", + Self::Table => "table", + Self::TextField => "textfield", + Self::Term => "term", + Self::TimeField => "timefield", + Self::Timer => "timer", + Self::Toolbar => "toolbar", + Self::Tooltip => "tooltip", + Self::TreeItem => "treeitem", + Self::WebArea => "webarea", + Self::Window => "window", + Self::Unknown => "unknown", + } + } + + pub const fn is_interactive(self) -> bool { + matches!( + self, + Self::Button + | Self::Cell + | Self::Checkbox + | Self::ColorWell + | Self::ComboBox + | Self::DockItem + | Self::Incrementor + | Self::Link + | Self::ListBox + | Self::MenuButton + | Self::MenuItem + | Self::Option + | Self::RadioButton + | Self::Slider + | Self::Switch + | Self::Tab + | Self::TextField + | Self::TreeItem + ) + } + + pub const fn is_transparent_wrapper(self) -> bool { + matches!(self, Self::Group) + } + + pub fn is_canonical(value: &str) -> bool { + let normalized = value.trim().to_ascii_lowercase(); + normalized == "unknown" || Self::from_token(&normalized) != Self::Unknown + } +} + +impl From<&str> for Role { + fn from(value: &str) -> Self { + Self::from_token(value) + } +} + +#[cfg(test)] +#[path = "role_tests.rs"] +mod tests; diff --git a/crates/core/src/role_tests.rs b/crates/core/src/role_tests.rs new file mode 100644 index 0000000..51609c2 --- /dev/null +++ b/crates/core/src/role_tests.rs @@ -0,0 +1,68 @@ +use super::*; + +#[test] +fn structural_roles_are_canonical_but_not_interactive() { + for token in ["row", "scrollbar", "tablist"] { + let role = Role::from_token(token); + assert_eq!(role.as_str(), token); + assert!(Role::is_canonical(token)); + assert!(!role.is_interactive()); + } +} + +#[test] +fn unknown_tokens_fail_closed_to_unknown() { + assert_eq!(Role::from_token("buttn"), Role::Unknown); + assert_eq!(Role::from_token(""), Role::Unknown); + assert_eq!(Role::Unknown.as_str(), "unknown"); +} + +#[test] +fn canonical_tokens_round_trip() { + for token in [ + "alert", + "alertdialog", + "application", + "article", + "banner", + "button", + "checkbox", + "complementary", + "contentinfo", + "definition", + "dialog", + "form", + "group", + "listbox", + "log", + "main", + "marquee", + "math", + "menuitem", + "navigation", + "note", + "option", + "radiogroup", + "region", + "search", + "statictext", + "tabpanel", + "textfield", + "term", + "timer", + "webarea", + "window", + "unknown", + ] { + assert_eq!(Role::from_token(token).as_str(), token); + assert!(Role::is_canonical(token)); + } +} + +#[test] +fn interactive_classification_is_owned_by_role() { + assert!(Role::Button.is_interactive()); + assert!(Role::TextField.is_interactive()); + assert!(!Role::Group.is_interactive()); + assert!(!Role::StaticText.is_interactive()); +} diff --git a/crates/core/src/roles.rs b/crates/core/src/roles.rs index f209ceb..2748a06 100644 --- a/crates/core/src/roles.rs +++ b/crates/core/src/roles.rs @@ -12,8 +12,10 @@ pub const INTERACTIVE_ROLES: &[&str] = &[ "dockitem", "incrementor", "link", + "listbox", "menubutton", "menuitem", + "option", "radiobutton", "slider", "switch", @@ -25,43 +27,89 @@ pub const INTERACTIVE_ROLES: &[&str] = &[ /// Normalizes a caller-supplied role filter for comparison against tree /// roles: trims, lowercases, and folds a few high-frequency web-automation /// synonyms onto their canonical names so an agent's reflexive `textarea` -/// matches the `textfield` the adapters emit. This is an ergonomic shim, -/// not a vocabulary: it never rejects. Whether a role exists is answered -/// by the live tree (see `find`'s `roles_present`), so a new adapter role -/// works the instant the adapter emits it — nothing here to keep in sync. +/// matches the `textfield` the adapters emit. Callers must validate with +/// [`is_valid_role_query`] before normalizing. pub fn normalize_role_query(role: &str) -> String { let normalized = role.trim().to_ascii_lowercase(); - match normalized.as_str() { - "textarea" | "textbox" | "searchfield" => "textfield".to_string(), - _ => normalized, + let canonical = + role_query_alias(&normalized).unwrap_or_else(|| crate::Role::from_token(&normalized)); + canonical.as_str().to_string() +} + +pub fn is_valid_role_query(role: &str) -> bool { + let normalized = role.trim().to_ascii_lowercase(); + role_query_alias(&normalized).is_some() + || (normalized != "unknown" && crate::Role::is_canonical(&normalized)) +} + +fn role_query_alias(role: &str) -> Option<crate::Role> { + match role { + "gridcell" => Some(crate::Role::Cell), + "img" => Some(crate::Role::Image), + "radio" => Some(crate::Role::RadioButton), + "searchbox" | "searchfield" | "textarea" | "textbox" => Some(crate::Role::TextField), + "spinbutton" => Some(crate::Role::Incrementor), + "togglebutton" => Some(crate::Role::Button), + "tree" => Some(crate::Role::Outline), + _ => None, } } /// Returns true when `role` is in [`INTERACTIVE_ROLES`]. pub fn is_interactive_role(role: &str) -> bool { - INTERACTIVE_ROLES.contains(&role) + crate::Role::from_token(role).is_interactive() +} + +pub fn is_canonical_role(role: &str) -> bool { + crate::Role::is_canonical(role) } /// Returns true for roles whose checked/unchecked state can be queried and set. pub fn is_toggleable_role(role: &str) -> bool { - matches!(role, "checkbox" | "switch" | "radiobutton") + matches!( + crate::Role::from_token(role), + crate::Role::Checkbox | crate::Role::Switch | crate::Role::RadioButton + ) } /// Returns true for roles that carry an expanded/collapsed surface state. pub fn is_expandable_role(role: &str) -> bool { - matches!(role, "combobox" | "menubutton" | "treeitem" | "disclosure") + matches!( + crate::Role::from_token(role), + crate::Role::ComboBox + | crate::Role::MenuButton + | crate::Role::TreeItem + | crate::Role::Disclosure + ) } /// Returns true for roles whose `value` changes during normal interaction and /// must not be treated as stable ref identity. pub fn is_mutable_value_role(role: &str) -> bool { - matches!(role, "combobox" | "incrementor" | "slider" | "textfield") + matches!( + crate::Role::from_token(role), + crate::Role::ComboBox + | crate::Role::Checkbox + | crate::Role::Incrementor + | crate::Role::ListBox + | crate::Role::RadioButton + | crate::Role::Slider + | crate::Role::Switch + | crate::Role::TextField + ) } #[cfg(test)] mod tests { use super::*; + #[test] + fn is_interactive_role_matches_interactive_roles_list() { + for role in INTERACTIVE_ROLES { + assert!(is_interactive_role(role), "{role} should be interactive"); + } + } + #[test] fn interactive_roles_are_sorted_and_unique() { let mut sorted = INTERACTIVE_ROLES.to_vec(); @@ -71,10 +119,22 @@ mod tests { } #[test] - fn normalize_role_query_folds_text_input_synonyms() { - assert_eq!(normalize_role_query("textarea"), "textfield"); - assert_eq!(normalize_role_query("textbox"), "textfield"); - assert_eq!(normalize_role_query("searchfield"), "textfield"); + fn normalize_role_query_folds_playwright_aliases() { + for (alias, canonical) in [ + ("gridcell", "cell"), + ("img", "image"), + ("radio", "radiobutton"), + ("searchbox", "textfield"), + ("searchfield", "textfield"), + ("spinbutton", "incrementor"), + ("textarea", "textfield"), + ("textbox", "textfield"), + ("togglebutton", "button"), + ("tree", "outline"), + ] { + assert_eq!(normalize_role_query(alias), canonical, "{alias}"); + assert!(is_valid_role_query(alias), "{alias}"); + } } #[test] @@ -84,9 +144,9 @@ mod tests { } #[test] - fn normalize_role_query_passes_unknown_roles_through_untouched() { - assert_eq!(normalize_role_query("navbar"), "navbar"); - assert_eq!(normalize_role_query("buttn"), "buttn"); + fn normalize_role_query_fails_unknown_roles_closed() { + assert_eq!(normalize_role_query("navbar"), "unknown"); + assert_eq!(normalize_role_query("buttn"), "unknown"); } #[test] @@ -129,6 +189,7 @@ mod tests { fn every_toggleable_role_is_interactive() { for role in ["checkbox", "switch", "radiobutton"] { assert!(is_toggleable_role(role)); + assert!(is_mutable_value_role(role)); assert!( INTERACTIVE_ROLES.contains(&role), "toggleable role {role} missing from INTERACTIVE_ROLES" @@ -145,7 +206,16 @@ mod tests { #[test] fn mutable_value_roles_are_interactive() { - for role in ["combobox", "incrementor", "slider", "textfield"] { + for role in [ + "checkbox", + "combobox", + "incrementor", + "listbox", + "radiobutton", + "slider", + "switch", + "textfield", + ] { assert!(is_mutable_value_role(role)); assert!(is_interactive_role(role)); } diff --git a/crates/core/src/screenshot_target.rs b/crates/core/src/screenshot_target.rs new file mode 100644 index 0000000..f08b5fb --- /dev/null +++ b/crates/core/src/screenshot_target.rs @@ -0,0 +1,8 @@ +use crate::{WindowInfo, display_info::DisplayInfo}; + +pub enum ScreenshotTarget { + Screen(usize), + Display { index: usize, expected: DisplayInfo }, + ExactWindow(WindowInfo), + FullScreen, +} diff --git a/crates/core/src/search_text.rs b/crates/core/src/search_text.rs index 12bb56c..ba676a6 100644 --- a/crates/core/src/search_text.rs +++ b/crates/core/src/search_text.rs @@ -1,32 +1,32 @@ pub(crate) fn normalize(value: &str) -> String { - if value.is_ascii() { - value.to_ascii_lowercase() - } else { - value.to_lowercase() + let mut normalized = String::with_capacity(value.len()); + let mut pending_space = false; + for character in value.chars() { + if character.is_whitespace() { + pending_space = !normalized.is_empty(); + continue; + } + if pending_space { + normalized.push(' '); + pending_space = false; + } + normalized.extend(character.to_lowercase()); } + normalized } pub(crate) fn contains(haystack: &str, normalized_needle: &str) -> bool { if normalized_needle.is_empty() { return true; } - if haystack.is_ascii() && normalized_needle.is_ascii() { - return haystack - .as_bytes() - .windows(normalized_needle.len()) - .any(|chunk| chunk.eq_ignore_ascii_case(normalized_needle.as_bytes())); - } normalize(haystack).contains(normalized_needle) } -pub(crate) fn node_contains( - node: &crate::node::AccessibilityNode, - normalized_needle: &str, -) -> bool { +pub(crate) fn node_contains(node: &crate::AccessibilityNode, normalized_needle: &str) -> bool { [ - node.name.as_deref(), - node.value.as_deref(), - node.description.as_deref(), + node.identity.name.as_deref(), + node.identity.value.as_deref(), + node.identity.description.as_deref(), ] .into_iter() .flatten() diff --git a/crates/core/src/search_text_tests.rs b/crates/core/src/search_text_tests.rs index 10ab4ee..a2cfd36 100644 --- a/crates/core/src/search_text_tests.rs +++ b/crates/core/src/search_text_tests.rs @@ -1,17 +1,17 @@ use super::*; -use crate::node::AccessibilityNode; +use crate::AccessibilityNode; fn node(name: Option<&str>, value: Option<&str>, description: Option<&str>) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: "textfield".into(), - name: name.map(String::from), - value: value.map(String::from), - description: description.map(String::from), - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: crate::NodeIdentity { + name: name.map(String::from), + value: value.map(String::from), + description: description.map(String::from), + native_id: None, + }, + presentation: Default::default(), children_count: None, children: vec![], } diff --git a/crates/core/src/session/gc.rs b/crates/core/src/session/gc.rs index ad861dc..50c54fa 100644 --- a/crates/core/src/session/gc.rs +++ b/crates/core/src/session/gc.rs @@ -1,10 +1,7 @@ -use super::{ - SessionManifest, TRACE_LIVENESS_WINDOW, list_sessions, now_millis, - read_current_session_pointer, read_manifest, -}; +use super::{SessionManifest, TRACE_LIVENESS_WINDOW, list_sessions, now_millis, read_manifest}; use crate::{ + AppError, context::validate_session_id, - error::AppError, refs_lock::{RefStoreLock, lock_holder_is_live}, refs_store::RefStore, }; @@ -21,19 +18,15 @@ pub struct GcReport { pub removed: Vec<String>, } -pub fn pointer_references_live_session() -> Result<bool, AppError> { - let Some(id) = read_current_session_pointer()? else { - return Ok(false); - }; - is_live(&id) -} - pub fn is_live(session_id: &str) -> Result<bool, AppError> { validate_session_id(session_id)?; let store = RefStore::for_session(Some(session_id))?; if lock_holder_is_live(&store.base_dir().join("refstore.lock")) { return Ok(true); } + if super::liveness::any_held(store.base_dir()) { + return Ok(true); + } let manifest = read_manifest(session_id)?; Ok(has_recent_activity(&store, manifest.as_ref())) } @@ -51,12 +44,8 @@ fn has_recent_activity(store: &RefStore, manifest: Option<&SessionManifest>) -> } pub fn gc(options: GcOptions) -> Result<GcReport, AppError> { - let pointer = read_current_session_pointer()?; let mut removed = Vec::new(); for manifest in list_sessions()? { - if pointer.as_deref() == Some(manifest.id.as_str()) { - continue; - } if is_live(&manifest.id)? { continue; } @@ -78,7 +67,7 @@ pub fn gc(options: GcOptions) -> Result<GcReport, AppError> { Ok(lock) => lock, Err(_) => continue, }; - if has_recent_activity(&store, Some(&manifest)) { + if super::liveness::any_held(&dir) || has_recent_activity(&store, Some(&manifest)) { drop(lock); continue; } diff --git a/crates/core/src/session/liveness.rs b/crates/core/src/session/liveness.rs new file mode 100644 index 0000000..23fb627 --- /dev/null +++ b/crates/core/src/session/liveness.rs @@ -0,0 +1,77 @@ +use crate::{AppError, Deadline, file_lock::FileLock, refs_lock::RefStoreLock}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +static LEASE_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone)] +pub struct SessionLivenessLease { + inner: Arc<LeaseInner>, +} + +impl std::fmt::Debug for SessionLivenessLease { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SessionLivenessLease") + .field("path", &self.inner.path) + .finish() + } +} + +struct LeaseInner { + lock: Option<FileLock>, + path: PathBuf, +} + +impl Drop for LeaseInner { + fn drop(&mut self) { + drop(self.lock.take()); + let _ = std::fs::remove_file(&self.path); + } +} + +pub fn acquire( + session_id: &str, + deadline: Deadline, +) -> Result<Option<SessionLivenessLease>, AppError> { + let store = crate::refs_store::RefStore::for_session(Some(session_id))?; + let base = store.base_dir(); + if !base.is_dir() { + return Ok(None); + } + let _store_lock = RefStoreLock::acquire_with_deadline(&base.join("refstore.lock"), deadline)?; + if super::read_manifest(session_id)?.is_none() { + return Ok(None); + } + let directory = base.join("liveness"); + crate::private_file_parent::ensure_private(&directory)?; + let path = directory.join(lease_filename()); + let lock = FileLock::acquire(&path, deadline, "session liveness lease")?; + Ok(Some(SessionLivenessLease { + inner: Arc::new(LeaseInner { + lock: Some(lock), + path, + }), + })) +} + +pub(super) fn any_held(base: &Path) -> bool { + let directory = base.join("liveness"); + let Ok(entries) = std::fs::read_dir(directory) else { + return false; + }; + entries.flatten().any(|entry| { + entry.file_type().is_ok_and(|file_type| file_type.is_file()) + && FileLock::is_held(&entry.path()) + }) +} + +fn lease_filename() -> String { + let sequence = LEASE_COUNTER.fetch_add(1, Ordering::Relaxed); + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + format!("{}-{timestamp}-{sequence}.lock", std::process::id()) +} diff --git a/crates/core/src/session/mod.rs b/crates/core/src/session/mod.rs index 34e8752..76b577b 100644 --- a/crates/core/src/session/mod.rs +++ b/crates/core/src/session/mod.rs @@ -1,23 +1,25 @@ mod gc; +mod liveness; mod manifest; -pub use gc::{GcOptions, GcReport, gc, is_live, pointer_references_live_session}; +pub use gc::{GcOptions, GcReport, gc, is_live}; +pub use liveness::SessionLivenessLease; pub use manifest::{ArtifactsMode, SessionManifest, SessionTraceMode}; use crate::{ + AppError, context::validate_session_id, - error::AppError, refs::{home_dir, write_private_file}, refs_store::RefStore, }; use serde_json; -use std::io::{ErrorKind, Read}; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -const CURRENT_SESSION_FILE: &str = "current_session"; const SESSION_MANIFEST_FILE: &str = "session.json"; +const MAX_SESSION_MANIFEST_BYTES: u64 = 64 * 1024; pub(super) const TRACE_LIVENESS_WINDOW: Duration = Duration::from_secs(300); static SESSION_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -25,7 +27,6 @@ pub struct StartSessionOptions { pub name: Option<String>, pub trace: SessionTraceMode, pub artifacts: ArtifactsMode, - pub force: bool, } impl Default for StartSessionOptions { @@ -34,7 +35,6 @@ impl Default for StartSessionOptions { name: None, trace: SessionTraceMode::On, artifacts: ArtifactsMode::Events, - force: false, } } } @@ -53,10 +53,6 @@ pub fn trace_dir(session_id: &str) -> Result<PathBuf, AppError> { Ok(RefStore::for_session(Some(session_id))?.trace_dir()) } -pub fn current_session_path() -> Result<PathBuf, AppError> { - Ok(agent_desktop_dir()?.join(CURRENT_SESSION_FILE)) -} - pub fn resolve_active_session( explicit: Option<&str>, env: Option<&str>, @@ -75,57 +71,17 @@ pub fn resolve_active_session( validate_session_id(id)?; return Ok(Some(id.to_string())); } - read_current_session_pointer() -} - -pub fn read_current_session_pointer() -> Result<Option<String>, AppError> { - let path = current_session_path()?; - let mut file = match crate::refs::open_nofollow(&path) { - Ok(file) => file, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), - Err(err) if crate::refs::is_symlink(&path) => { - tracing::warn!( - "ignoring symlinked session pointer {}: {err}", - path.display() - ); - return Ok(None); - } - Err(err) => return Err(err.into()), - }; - let mut id = String::new(); - file.read_to_string(&mut id)?; - let id = id.trim().to_string(); - if id.is_empty() || validate_session_id(&id).is_err() { - return Ok(None); - } - Ok(Some(id)) -} - -pub fn write_current_session_pointer(session_id: &str) -> Result<(), AppError> { - validate_session_id(session_id)?; - write_private_file(¤t_session_path()?, session_id.as_bytes()) -} - -pub fn clear_current_session_pointer() -> Result<(), AppError> { - match std::fs::remove_file(current_session_path()?) { - Ok(()) => Ok(()), - Err(err) if err.kind() == ErrorKind::NotFound => Ok(()), - Err(err) => Err(err.into()), - } + Ok(None) } pub fn read_manifest(session_id: &str) -> Result<Option<SessionManifest>, AppError> { let path = manifest_path(session_id)?; - let mut file = match crate::refs::open_nofollow(&path) { - Ok(file) => file, + let json = match crate::private_file::read_private_bounded(&path, MAX_SESSION_MANIFEST_BYTES) { + Ok(json) => json, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), Err(err) => return Ok(ignore_unreadable_manifest(&path, &err)), }; - let mut json = String::new(); - if let Err(err) = file.read_to_string(&mut json) { - return Ok(ignore_unreadable_manifest(&path, &err)); - } - match serde_json::from_str(&json) { + match serde_json::from_slice(&json) { Ok(manifest) => Ok(Some(manifest)), Err(err) => Ok(ignore_unreadable_manifest(&path, &err)), } @@ -217,12 +173,6 @@ pub fn start_session(options: StartSessionOptions) -> Result<SessionManifest, Ap "Remove --no-trace or omit --screenshots.", )); } - if !options.force && pointer_references_live_session()? { - return Err(AppError::invalid_input_with_suggestion( - "Refusing to clobber the current session pointer while it references a live session", - "Run `session end` first, set AGENT_DESKTOP_SESSION for concurrent work, or pass --force.", - )); - } let id = new_session_id(); let name = options .name @@ -239,23 +189,13 @@ pub fn start_session(options: StartSessionOptions) -> Result<SessionManifest, Ap artifacts: options.artifacts, }; write_manifest(&manifest)?; - write_current_session_pointer(&id)?; Ok(manifest) } -pub fn end_session(session_id: Option<&str>) -> Result<SessionManifest, AppError> { - let id = match session_id { - Some(id) => { - validate_session_id(id)?; - id.to_string() - } - None => read_current_session_pointer()?.ok_or_else(|| { - AppError::invalid_input_with_suggestion( - "No active session to end", - "Pass a session id or run `session start` first.", - ) - })?, - }; +pub fn end_session(session_id: &str) -> Result<SessionManifest, AppError> { + validate_session_id(session_id)?; + let _lease = acquire_liveness_lease(session_id)?; + let id = session_id.to_string(); let mut manifest = read_manifest(&id)?.ok_or_else(|| { AppError::invalid_input_with_suggestion( format!("Session '{id}' has no manifest"), @@ -266,12 +206,21 @@ pub fn end_session(session_id: Option<&str>) -> Result<SessionManifest, AppError manifest.ended_at = Some(now_millis()); write_manifest(&manifest)?; } - if read_current_session_pointer()?.as_deref() == Some(id.as_str()) { - clear_current_session_pointer()?; - } Ok(manifest) } +pub fn acquire_liveness_lease(session_id: &str) -> Result<Option<SessionLivenessLease>, AppError> { + acquire_liveness_lease_with_deadline(session_id, crate::Deadline::standard()?) +} + +pub(crate) fn acquire_liveness_lease_with_deadline( + session_id: &str, + deadline: crate::Deadline, +) -> Result<Option<SessionLivenessLease>, AppError> { + validate_session_id(session_id)?; + liveness::acquire(session_id, deadline) +} + fn create_session_tree(dir: &Path) -> Result<(), AppError> { #[cfg(unix)] { diff --git a/crates/core/src/session/session_gc_tests.rs b/crates/core/src/session/session_gc_tests.rs index e783361..ebbb9a3 100644 --- a/crates/core/src/session/session_gc_tests.rs +++ b/crates/core/src/session/session_gc_tests.rs @@ -1,27 +1,112 @@ use super::*; use crate::refs_test_support::HomeGuard; use crate::session::SessionTraceMode; +#[cfg(unix)] use std::fs; use std::time::Duration; +#[test] +fn multiple_liveness_owners_do_not_serialize_each_other() { + let _guard = HomeGuard::new(); + let mut manifest = start_session(StartSessionOptions { + name: None, + trace: SessionTraceMode::Off, + ..Default::default() + }) + .unwrap(); + manifest.created_at = 0; + write_manifest(&manifest).unwrap(); + + let first = acquire_liveness_lease(&manifest.id).unwrap().unwrap(); + let second = acquire_liveness_lease(&manifest.id).unwrap().unwrap(); + drop(first); + + assert!(is_live(&manifest.id).unwrap()); + assert!(session_dir(&manifest.id).unwrap().is_dir()); + + drop(second); + let report = gc(GcOptions { + ended_only: false, + older_than: Some(Duration::ZERO), + }) + .unwrap(); + assert!(report.removed.contains(&manifest.id)); +} + +#[test] +fn subprocess_session_lease_holder() { + let Some(ready) = std::env::var_os("AGENT_DESKTOP_SESSION_LEASE_READY") else { + return; + }; + let session_id = std::env::var("AGENT_DESKTOP_SESSION_LEASE_ID").unwrap(); + let _lease = acquire_liveness_lease(&session_id).unwrap().unwrap(); + std::fs::write(ready, b"ready").unwrap(); + std::thread::sleep(Duration::from_secs(5)); +} + +#[test] +fn gc_retains_idle_cross_process_owner_and_reaps_after_crash() { + let guard = HomeGuard::new(); + let mut manifest = start_session(StartSessionOptions { + name: None, + trace: SessionTraceMode::Off, + ..Default::default() + }) + .unwrap(); + manifest.created_at = 0; + write_manifest(&manifest).unwrap(); + let ready = guard.path().join("lease-ready"); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("session::gc_tests::subprocess_session_lease_holder") + .arg("--nocapture") + .env("HOME", guard.path()) + .env("AGENT_DESKTOP_SESSION_LEASE_READY", &ready) + .env("AGENT_DESKTOP_SESSION_LEASE_ID", &manifest.id) + .spawn() + .unwrap(); + let ready_deadline = std::time::Instant::now() + Duration::from_secs(2); + while !ready.is_file() && std::time::Instant::now() < ready_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + ready.is_file(), + "subprocess did not acquire its session lease" + ); + + let retained = gc(GcOptions { + ended_only: false, + older_than: Some(Duration::ZERO), + }) + .unwrap(); + assert!(!retained.removed.contains(&manifest.id)); + + child.kill().unwrap(); + child.wait().unwrap(); + let removed = gc(GcOptions { + ended_only: false, + older_than: Some(Duration::ZERO), + }) + .unwrap(); + assert!(removed.removed.contains(&manifest.id)); +} + #[test] fn gc_removes_ended_sessions_but_not_pointer_or_live() { let _guard = HomeGuard::new(); let live = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); let ended = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: true, ..Default::default() }) .unwrap(); - end_session(Some(&ended.id)).unwrap(); + end_session(&ended.id).unwrap(); let report = gc(GcOptions { ended_only: false, older_than: None, @@ -51,12 +136,10 @@ fn gc_respects_older_than_threshold() { let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: false, ..Default::default() }) .unwrap(); - end_session(Some(&manifest.id)).unwrap(); - clear_current_session_pointer().unwrap(); + end_session(&manifest.id).unwrap(); let report = gc(GcOptions { ended_only: false, older_than: Some(Duration::from_secs(3600)), @@ -71,11 +154,9 @@ fn gc_leaves_recently_created_unended_session() { let started = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: true, ..Default::default() }) .unwrap(); - clear_current_session_pointer().unwrap(); let report = gc(GcOptions { ended_only: false, older_than: Some(Duration::from_secs(0)), @@ -97,7 +178,6 @@ fn unreadable_manifest_is_skipped_not_fatal_for_list_and_gc() { let good = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: true, ..Default::default() }) .unwrap(); diff --git a/crates/core/src/session/session_tests.rs b/crates/core/src/session/session_tests.rs index 2064d91..c9af413 100644 --- a/crates/core/src/session/session_tests.rs +++ b/crates/core/src/session/session_tests.rs @@ -1,13 +1,11 @@ use super::*; -use crate::refs_lock::RefStoreLock; use crate::refs_test_support::HomeGuard; use crate::session::{ArtifactsMode, SessionTraceMode}; use std::fs; #[test] -fn resolve_prefers_explicit_over_env_and_pointer() { +fn resolve_prefers_explicit_over_env() { let _guard = HomeGuard::new(); - write_current_session_pointer("pointer").unwrap(); unsafe { std::env::set_var("AGENT_DESKTOP_SESSION", "env-session") }; let resolved = resolve_active_session(Some("explicit"), None).unwrap(); assert_eq!(resolved.as_deref(), Some("explicit")); @@ -15,9 +13,8 @@ fn resolve_prefers_explicit_over_env_and_pointer() { } #[test] -fn resolve_prefers_env_over_pointer() { +fn resolve_uses_env_without_explicit_session() { let _guard = HomeGuard::new(); - write_current_session_pointer("pointer").unwrap(); unsafe { std::env::set_var("AGENT_DESKTOP_SESSION", "env-session") }; let resolved = resolve_active_session(None, Some("env-session")).unwrap(); assert_eq!(resolved.as_deref(), Some("env-session")); @@ -25,11 +22,10 @@ fn resolve_prefers_env_over_pointer() { } #[test] -fn resolve_falls_back_to_pointer() { +fn resolve_does_not_infer_an_active_session() { let _guard = HomeGuard::new(); - write_current_session_pointer("pointer").unwrap(); let resolved = resolve_active_session(None, None).unwrap(); - assert_eq!(resolved.as_deref(), Some("pointer")); + assert!(resolved.is_none()); } #[test] @@ -62,62 +58,47 @@ fn validate_session_name_rejects_control_chars() { } #[test] -fn start_creates_tree_manifest_and_pointer() { +fn start_creates_tree_and_manifest() { let _guard = HomeGuard::new(); let manifest = start_session(StartSessionOptions { name: Some("demo".into()), trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); assert!(session_dir(&manifest.id).unwrap().join("trace").is_dir()); - assert_eq!( - read_current_session_pointer().unwrap().as_deref(), - Some(manifest.id.as_str()) - ); + assert_eq!(read_manifest(&manifest.id).unwrap().unwrap(), manifest); } #[test] -fn start_refuses_live_pointer_without_force() { +fn start_allows_concurrent_explicit_sessions() { let _guard = HomeGuard::new(); let first = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); - let _lock = RefStoreLock::acquire( - &crate::refs_store::RefStore::for_session(Some(&first.id)) - .unwrap() - .base_dir() - .join("refstore.lock"), - ) - .unwrap(); - let err = start_session(StartSessionOptions { + let second = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) - .unwrap_err(); - assert_eq!(err.code(), "INVALID_ARGS"); + .unwrap(); + assert_ne!(first.id, second.id); } #[test] -fn end_seals_manifest_and_clears_pointer() { +fn end_seals_the_explicit_manifest() { let _guard = HomeGuard::new(); - let _manifest = start_session(StartSessionOptions { + let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); - let ended = end_session(None).unwrap(); + let ended = end_session(&manifest.id).unwrap(); assert!(ended.ended_at.is_some()); - assert!(read_current_session_pointer().unwrap().is_none()); } #[test] @@ -126,7 +107,6 @@ fn list_reports_manifest_fields_only() { let manifest = start_session(StartSessionOptions { name: Some("listed".into()), trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); @@ -141,7 +121,6 @@ fn trace_enabled_requires_manifest_on() { let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: false, ..Default::default() }) .unwrap(); @@ -161,7 +140,6 @@ fn corrupt_manifest_is_ignored_not_fatal() { let good = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: true, ..Default::default() }) .unwrap(); @@ -176,34 +154,21 @@ fn corrupt_manifest_is_ignored_not_fatal() { } #[test] -fn start_with_force_overrides_live_pointer() { +fn multiple_starts_remain_independent() { let _guard = HomeGuard::new(); let first = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); - let _lock = RefStoreLock::acquire( - &crate::refs_store::RefStore::for_session(Some(&first.id)) - .unwrap() - .base_dir() - .join("refstore.lock"), - ) - .unwrap(); let second = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: true, ..Default::default() }) .unwrap(); assert_ne!(first.id, second.id); - assert_eq!( - read_current_session_pointer().unwrap().as_deref(), - Some(second.id.as_str()) - ); } #[test] @@ -212,12 +177,11 @@ fn trace_enabled_false_once_session_ended() { let manifest = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::On, - force: false, ..Default::default() }) .unwrap(); assert!(trace_enabled_for_session(&manifest.id).unwrap()); - end_session(Some(&manifest.id)).unwrap(); + end_session(&manifest.id).unwrap(); assert!(!trace_enabled_for_session(&manifest.id).unwrap()); } @@ -254,6 +218,11 @@ fn legacy_manifest_without_artifacts_defaults_to_events() { r#"{"id":"legacy","created_at":1,"trace":"on"}"#, ) .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(dir.join("session.json"), fs::Permissions::from_mode(0o600)).unwrap(); + } let manifest = read_manifest("legacy").unwrap().expect("manifest"); assert_eq!(manifest.artifacts, ArtifactsMode::Events); } @@ -279,7 +248,7 @@ fn ended_session_reports_artifacts_full_false() { }) .unwrap(); assert!(manifest.artifacts_full()); - end_session(Some(&manifest.id)).unwrap(); + end_session(&manifest.id).unwrap(); let ended = read_manifest(&manifest.id).unwrap().expect("manifest"); assert!(!ended.artifacts_full()); } @@ -291,7 +260,6 @@ fn symlinked_manifest_is_ignored_not_fatal() { let good = start_session(StartSessionOptions { name: None, trace: SessionTraceMode::Off, - force: true, ..Default::default() }) .unwrap(); @@ -309,12 +277,16 @@ fn symlinked_manifest_is_ignored_not_fatal() { #[cfg(unix)] #[test] -fn symlinked_session_pointer_degrades_to_none() { +fn legacy_pointer_symlink_does_not_activate_a_session() { let _guard = HomeGuard::new(); let target = agent_desktop_dir().unwrap().join("pointer-target"); fs::create_dir_all(target.parent().unwrap()).unwrap(); fs::write(&target, b"whatever").unwrap(); - std::os::unix::fs::symlink(&target, current_session_path().unwrap()).unwrap(); + std::os::unix::fs::symlink( + &target, + agent_desktop_dir().unwrap().join("current_session"), + ) + .unwrap(); - assert!(read_current_session_pointer().unwrap().is_none()); + assert!(resolve_active_session(None, None).unwrap().is_none()); } diff --git a/crates/core/src/session_affinity.rs b/crates/core/src/session_affinity.rs new file mode 100644 index 0000000..ac7a856 --- /dev/null +++ b/crates/core/src/session_affinity.rs @@ -0,0 +1,5 @@ +/// Associates adapter-native connection state with a caller-managed session. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionAffinity { + pub session_id: Option<String>, +} diff --git a/crates/core/src/signal_baseline.rs b/crates/core/src/signal_baseline.rs new file mode 100644 index 0000000..a5c4769 --- /dev/null +++ b/crates/core/src/signal_baseline.rs @@ -0,0 +1,10 @@ +use crate::{AppInfo, SignalCompleteness, SurfaceSignal, WindowInfo}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SignalBaseline { + pub windows: Vec<WindowInfo>, + pub apps: Vec<AppInfo>, + pub surfaces: Vec<SurfaceSignal>, + pub completeness: SignalCompleteness, +} diff --git a/crates/core/src/signal_completeness.rs b/crates/core/src/signal_completeness.rs new file mode 100644 index 0000000..0031e3e --- /dev/null +++ b/crates/core/src/signal_completeness.rs @@ -0,0 +1,18 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct SignalCompleteness { + pub windows: bool, + pub apps: bool, + pub surfaces: bool, +} + +impl SignalCompleteness { + pub const fn complete() -> Self { + Self { + windows: true, + apps: true, + surfaces: true, + } + } +} diff --git a/crates/core/src/signal_filter.rs b/crates/core/src/signal_filter.rs new file mode 100644 index 0000000..209d250 --- /dev/null +++ b/crates/core/src/signal_filter.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Clone, Default)] +pub struct SignalFilter { + pub app: Option<String>, + pub process: Option<crate::ProcessIdentity>, +} diff --git a/crates/core/src/signals.rs b/crates/core/src/signals.rs new file mode 100644 index 0000000..7148c8e --- /dev/null +++ b/crates/core/src/signals.rs @@ -0,0 +1,193 @@ +#[cfg(test)] +use crate::snapshot_surface::SnapshotSurface; +use crate::{AppInfo, EventKind, ProcessId, SignalBaseline, SurfaceSignal, UiEvent, WindowInfo}; +use std::collections::HashSet; + +/// Pure baseline-diff over two independently captured [`SignalBaseline`] +/// snapshots. Never touches the adapter — every code path here is exercised +/// with hand-built fixtures, which is what makes it a real regression guard +/// for F8/F9-shaped bugs (wrong-id matches, masked closes) instead of one +/// that can only be verified by hand against a live desktop. +pub fn diff_signals(baseline: &SignalBaseline, current: &SignalBaseline) -> Vec<UiEvent> { + let mut events = Vec::new(); + if baseline.completeness.windows && current.completeness.windows { + diff_windows(baseline, current, &mut events); + diff_focus(baseline, current, &mut events); + } + if baseline.completeness.apps && current.completeness.apps { + diff_apps(baseline, current, &mut events); + } + if baseline.completeness.surfaces && current.completeness.surfaces { + diff_surfaces(baseline, current, &mut events); + } + events.sort_by(compare_events); + events +} + +fn compare_events(left: &UiEvent, right: &UiEvent) -> std::cmp::Ordering { + event_rank(&left.kind) + .cmp(&event_rank(&right.kind)) + .then_with(|| left.app.cmp(&right.app)) + .then_with(|| left.pid.cmp(&right.pid)) + .then_with(|| left.window_id.cmp(&right.window_id)) + .then_with(|| left.title.cmp(&right.title)) + .then_with(|| left.kind.cli_token().cmp(right.kind.cli_token())) +} + +fn event_rank(kind: &EventKind) -> u8 { + match kind { + EventKind::WindowClosed => 0, + EventKind::AppTerminated => 1, + EventKind::SurfaceDismissed { .. } => 2, + EventKind::WindowOpened => 3, + EventKind::AppLaunched => 4, + EventKind::SurfaceAppeared { .. } => 5, + EventKind::FocusChangedWindow => 6, + } +} + +type WindowIdentity<'a> = (ProcessId, &'a str, &'a str); + +fn window_identity(window: &WindowInfo) -> Option<WindowIdentity<'_>> { + window + .process_instance + .as_deref() + .map(|instance| (window.pid, instance, window.id.as_str())) +} + +fn window_event(kind: EventKind, win: &WindowInfo) -> UiEvent { + UiEvent { + kind, + window_id: Some(win.id.clone()), + title: Some(win.title.clone()), + app: Some(win.app.clone()), + pid: Some(win.pid), + } +} + +fn diff_windows(baseline: &SignalBaseline, current: &SignalBaseline, events: &mut Vec<UiEvent>) { + let baseline_ids: HashSet<WindowIdentity<'_>> = baseline + .windows + .iter() + .filter_map(window_identity) + .collect(); + for win in ¤t.windows { + if window_identity(win).is_some_and(|identity| !baseline_ids.contains(&identity)) { + events.push(window_event(EventKind::WindowOpened, win)); + } + } + let current_ids: HashSet<WindowIdentity<'_>> = + current.windows.iter().filter_map(window_identity).collect(); + for win in &baseline.windows { + if window_identity(win).is_some_and(|identity| !current_ids.contains(&identity)) { + events.push(window_event(EventKind::WindowClosed, win)); + } + } +} + +fn diff_focus(baseline: &SignalBaseline, current: &SignalBaseline, events: &mut Vec<UiEvent>) { + let baseline_focused_id = baseline + .windows + .iter() + .find(|w| w.state.is_focused) + .and_then(window_identity); + let current_focused = current + .windows + .iter() + .find(|window| window.state.is_focused); + match current_focused { + Some(window) + if window_identity(window) + .is_some_and(|identity| Some(identity) != baseline_focused_id) => + { + events.push(window_event(EventKind::FocusChangedWindow, window)); + } + None if baseline_focused_id.is_some() => events.push(UiEvent { + kind: EventKind::FocusChangedWindow, + window_id: None, + title: None, + app: None, + pid: None, + }), + Some(_) | None => {} + } +} + +fn app_event(kind: EventKind, app: &AppInfo) -> UiEvent { + UiEvent { + kind, + window_id: None, + title: None, + app: Some(app.name.clone()), + pid: Some(app.pid), + } +} + +fn diff_apps(baseline: &SignalBaseline, current: &SignalBaseline, events: &mut Vec<UiEvent>) { + let baseline_pids: HashSet<_> = baseline.apps.iter().filter_map(app_identity).collect(); + for app in ¤t.apps { + if app_identity(app).is_some_and(|identity| !baseline_pids.contains(&identity)) { + events.push(app_event(EventKind::AppLaunched, app)); + } + } + let current_pids: HashSet<_> = current.apps.iter().filter_map(app_identity).collect(); + for app in &baseline.apps { + if app_identity(app).is_some_and(|identity| !current_pids.contains(&identity)) { + events.push(app_event(EventKind::AppTerminated, app)); + } + } +} + +fn app_identity(app: &AppInfo) -> Option<(ProcessId, &str)> { + app.process_instance + .as_deref() + .map(|instance| (app.pid, instance)) +} + +type SurfaceKey<'a> = (ProcessId, &'a str, &'a str, &'static str); + +fn surface_identity(surface: &SurfaceSignal) -> SurfaceKey<'_> { + ( + surface.pid, + surface.process_instance.as_str(), + surface.id.as_str(), + surface.kind.as_str(), + ) +} + +fn surface_event(surface: &SurfaceSignal, appeared: bool) -> UiEvent { + UiEvent { + kind: if appeared { + EventKind::SurfaceAppeared { + surface: surface.kind, + } + } else { + EventKind::SurfaceDismissed { + surface: surface.kind, + } + }, + window_id: None, + title: surface.title.clone(), + app: Some(surface.app.clone()), + pid: Some(surface.pid), + } +} + +fn diff_surfaces(baseline: &SignalBaseline, current: &SignalBaseline, events: &mut Vec<UiEvent>) { + let baseline_ids: HashSet<_> = baseline.surfaces.iter().map(surface_identity).collect(); + for surface in ¤t.surfaces { + if !baseline_ids.contains(&surface_identity(surface)) { + events.push(surface_event(surface, true)); + } + } + let current_ids: HashSet<_> = current.surfaces.iter().map(surface_identity).collect(); + for surface in &baseline.surfaces { + if !current_ids.contains(&surface_identity(surface)) { + events.push(surface_event(surface, false)); + } + } +} + +#[cfg(test)] +#[path = "signals_tests.rs"] +mod tests; diff --git a/crates/core/src/signals_surface_tests.rs b/crates/core/src/signals_surface_tests.rs new file mode 100644 index 0000000..03d5e0b --- /dev/null +++ b/crates/core/src/signals_surface_tests.rs @@ -0,0 +1,99 @@ +use super::*; + +#[test] +fn surface_title_is_preserved_and_orders_simultaneous_events() { + let baseline = SignalBaseline { + completeness: crate::SignalCompleteness::complete(), + ..Default::default() + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: vec![ + SurfaceSignal { + kind: SnapshotSurface::Sheet, + app: "TextEdit".into(), + pid: crate::ProcessId::new(200), + process_instance: "instance-200".into(), + id: "sheet-zulu".into(), + title: Some("Zulu".into()), + }, + SurfaceSignal { + kind: SnapshotSurface::Sheet, + app: "TextEdit".into(), + pid: crate::ProcessId::new(200), + process_instance: "instance-200".into(), + id: "sheet-alpha".into(), + title: Some("Alpha".into()), + }, + ], + completeness: crate::SignalCompleteness::complete(), + }; + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].title.as_deref(), Some("Alpha")); + assert_eq!(events[1].title.as_deref(), Some("Zulu")); + + let mut reversed = current.clone(); + reversed.surfaces.reverse(); + assert_eq!(diff_signals(&baseline, &reversed), events); +} + +#[test] +fn same_named_app_instances_keep_surface_events_pid_scoped() { + let baseline = SignalBaseline { + completeness: crate::SignalCompleteness::complete(), + ..Default::default() + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: vec![ + SurfaceSignal { + kind: SnapshotSurface::Alert, + app: "Electron".into(), + pid: crate::ProcessId::new(202), + process_instance: "instance-202".into(), + id: "alert-202".into(), + title: Some("Confirm".into()), + }, + SurfaceSignal { + kind: SnapshotSurface::Alert, + app: "Electron".into(), + pid: crate::ProcessId::new(101), + process_instance: "instance-101".into(), + id: "alert-101".into(), + title: Some("Confirm".into()), + }, + ], + completeness: crate::SignalCompleteness::complete(), + }; + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].pid, Some(crate::ProcessId::new(101))); + assert_eq!(events[1].pid, Some(crate::ProcessId::new(202))); +} + +#[test] +fn event_kind_same_variant_ignores_surface_payload() { + let requested = EventKind::parse("surface-appeared").unwrap(); + let observed = EventKind::SurfaceAppeared { + surface: SnapshotSurface::Alert, + }; + assert!(requested.same_variant(&observed)); + + let dismissed = EventKind::SurfaceDismissed { + surface: SnapshotSurface::Alert, + }; + assert!(!requested.same_variant(&dismissed)); +} + +#[test] +fn event_kind_parse_rejects_unknown_token() { + assert!(EventKind::parse("window_opened").is_none()); + assert!(EventKind::parse("bogus").is_none()); +} diff --git a/crates/core/src/signals_tests.rs b/crates/core/src/signals_tests.rs new file mode 100644 index 0000000..dba68ba --- /dev/null +++ b/crates/core/src/signals_tests.rs @@ -0,0 +1,336 @@ +use super::*; + +fn window(id: &str, title: &str, app: &str, pid: u32, focused: bool) -> WindowInfo { + WindowInfo { + id: id.into(), + title: title.into(), + app: app.into(), + pid: crate::ProcessId::new(pid), + process_instance: Some(format!("instance-{pid}")), + bounds: None, + state: crate::WindowState { + is_focused: focused, + ..Default::default() + }, + } +} + +fn app(name: &str, pid: u32) -> AppInfo { + app_with_instance(name, pid, &format!("instance-{pid}")) +} + +fn app_with_instance(name: &str, pid: u32, instance: &str) -> AppInfo { + AppInfo { + name: name.into(), + pid: crate::ProcessId::new(pid), + bundle_id: None, + process_instance: Some(instance.into()), + } +} + +fn baseline_with_windows(windows: Vec<WindowInfo>) -> SignalBaseline { + SignalBaseline { + windows, + apps: Vec::new(), + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + } +} + +#[test] +fn new_window_id_produces_window_opened() { + let baseline = baseline_with_windows(vec![window("w-1", "Docs", "Finder", 100, true)]); + let current = baseline_with_windows(vec![ + window("w-1", "Docs", "Finder", 100, true), + window("w-2", "Untitled", "TextEdit", 200, false), + ]); + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, EventKind::WindowOpened); + assert_eq!(events[0].window_id.as_deref(), Some("w-2")); + assert_eq!(events[0].title.as_deref(), Some("Untitled")); +} + +#[test] +fn removed_window_id_produces_window_closed() { + let baseline = baseline_with_windows(vec![ + window("w-1", "Docs", "Finder", 100, true), + window("w-2", "Untitled", "TextEdit", 200, false), + ]); + let current = baseline_with_windows(vec![window("w-1", "Docs", "Finder", 100, true)]); + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, EventKind::WindowClosed); + assert_eq!(events[0].window_id.as_deref(), Some("w-2")); +} + +#[test] +fn window_closed_fires_even_as_another_window_opens() { + let baseline = baseline_with_windows(vec![ + window("w-1", "Untitled", "TextEdit", 200, true), + window("w-9", "Finder", "Finder", 100, false), + ]); + let current = baseline_with_windows(vec![ + window("w-9", "Finder", "Finder", 100, false), + window("w-42", "Preview", "Preview", 300, true), + ]); + + let events = diff_signals(&baseline, ¤t); + + let closed: Vec<_> = events + .iter() + .filter(|e| e.kind == EventKind::WindowClosed) + .collect(); + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].window_id.as_deref(), Some("w-1")); + + let opened: Vec<_> = events + .iter() + .filter(|e| e.kind == EventKind::WindowOpened) + .collect(); + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].window_id.as_deref(), Some("w-42")); +} + +#[test] +fn focus_change_produces_focus_changed_window_event() { + let baseline = baseline_with_windows(vec![ + window("w-1", "Docs", "Finder", 100, true), + window("w-2", "Untitled", "TextEdit", 200, false), + ]); + let current = baseline_with_windows(vec![ + window("w-1", "Docs", "Finder", 100, false), + window("w-2", "Untitled", "TextEdit", 200, true), + ]); + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, EventKind::FocusChangedWindow); + assert_eq!(events[0].window_id.as_deref(), Some("w-2")); +} + +#[test] +fn focus_change_between_two_same_title_windows_reports_correct_id() { + let baseline = baseline_with_windows(vec![ + window("w-1", "Untitled", "TextEdit", 200, true), + window("w-2", "Untitled", "TextEdit", 200, false), + ]); + let current = baseline_with_windows(vec![ + window("w-1", "Untitled", "TextEdit", 200, false), + window("w-2", "Untitled", "TextEdit", 200, true), + ]); + + let events = diff_signals(&baseline, ¤t); + + let focus_events: Vec<_> = events + .iter() + .filter(|e| e.kind == EventKind::FocusChangedWindow) + .collect(); + assert_eq!(focus_events.len(), 1); + assert_eq!( + focus_events[0].window_id.as_deref(), + Some("w-2"), + "focus event must key off id, not the shared title" + ); +} + +#[test] +fn app_launch_and_terminate_are_detected_by_pid() { + let baseline = SignalBaseline { + windows: Vec::new(), + apps: vec![app("Finder", 1)], + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: vec![app("Finder", 1), app("TextEdit", 2)], + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + + let events = diff_signals(&baseline, ¤t); + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, EventKind::AppLaunched); + assert_eq!(events[0].pid, Some(crate::ProcessId::new(2))); + + let events_back = diff_signals(¤t, &baseline); + assert_eq!(events_back.len(), 1); + assert_eq!(events_back[0].kind, EventKind::AppTerminated); + assert_eq!(events_back[0].pid, Some(crate::ProcessId::new(2))); +} + +#[test] +fn sheet_appears_under_app_filter_produces_surface_appeared() { + let baseline = SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: vec![SurfaceSignal { + kind: SnapshotSurface::Sheet, + app: "TextEdit".into(), + pid: crate::ProcessId::new(200), + process_instance: "instance-200".into(), + id: "sheet-1".into(), + title: None, + }], + completeness: crate::SignalCompleteness::complete(), + }; + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 1); + match &events[0].kind { + EventKind::SurfaceAppeared { surface } => assert_eq!(*surface, SnapshotSurface::Sheet), + other => panic!("expected SurfaceAppeared, got {other:?}"), + } + assert_eq!(events[0].app.as_deref(), Some("TextEdit")); + assert_eq!(events[0].pid, Some(crate::ProcessId::new(200))); +} + +#[test] +fn surface_dismissed_fires_when_sheet_count_drops() { + let baseline = SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: vec![SurfaceSignal { + kind: SnapshotSurface::Sheet, + app: "TextEdit".into(), + pid: crate::ProcessId::new(200), + process_instance: "instance-200".into(), + id: "sheet-1".into(), + title: None, + }], + completeness: crate::SignalCompleteness::complete(), + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: Vec::new(), + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 1); + assert!(matches!(events[0].kind, EventKind::SurfaceDismissed { .. })); +} + +#[test] +fn reorder_only_baselines_produce_zero_events() { + let baseline = baseline_with_windows(vec![ + window("w-1", "Docs", "Finder", 100, true), + window("w-2", "Untitled", "TextEdit", 200, false), + ]); + let current = baseline_with_windows(vec![ + window("w-2", "Untitled", "TextEdit", 200, false), + window("w-1", "Docs", "Finder", 100, true), + ]); + + let events = diff_signals(&baseline, ¤t); + + assert!( + events.is_empty(), + "reordering the same windows must not synthesize open/close/focus events, got {events:?}" + ); +} + +#[test] +fn equal_window_ids_from_different_processes_are_distinct() { + let baseline = baseline_with_windows(vec![window("42", "Old", "Alpha", 100, false)]); + let current = baseline_with_windows(vec![window("42", "New", "Beta", 200, false)]); + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].kind, EventKind::WindowClosed); + assert_eq!(events[0].pid, Some(crate::ProcessId::new(100))); + assert_eq!(events[1].kind, EventKind::WindowOpened); + assert_eq!(events[1].pid, Some(crate::ProcessId::new(200))); +} + +#[test] +fn recycled_pid_with_changed_app_identity_emits_both_lifecycle_events() { + let baseline = SignalBaseline { + windows: Vec::new(), + apps: vec![app_with_instance("Alpha", 100, "generation-a")], + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: vec![app_with_instance("Beta", 100, "generation-b")], + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].kind, EventKind::AppTerminated); + assert_eq!(events[0].app.as_deref(), Some("Alpha")); + assert_eq!(events[1].kind, EventKind::AppLaunched); + assert_eq!(events[1].app.as_deref(), Some("Beta")); +} + +#[test] +fn app_metadata_change_does_not_synthesize_lifecycle_events() { + let baseline = SignalBaseline { + windows: Vec::new(), + apps: vec![app_with_instance("Old Name", 100, "same-generation")], + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + let current = SignalBaseline { + windows: Vec::new(), + apps: vec![app_with_instance("New Name", 100, "same-generation")], + surfaces: Vec::new(), + completeness: crate::SignalCompleteness::complete(), + }; + + assert!(diff_signals(&baseline, ¤t).is_empty()); +} + +#[test] +fn focus_cleared_produces_an_identity_free_focus_event() { + let baseline = baseline_with_windows(vec![window("w-1", "Docs", "Finder", 100, true)]); + let current = baseline_with_windows(vec![window("w-1", "Docs", "Finder", 100, false)]); + + let events = diff_signals(&baseline, ¤t); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, EventKind::FocusChangedWindow); + assert_eq!(events[0].window_id, None); + assert_eq!(events[0].app, None); + assert_eq!(events[0].pid, None); +} + +#[test] +fn reused_pid_and_window_id_with_new_process_generation_is_replacement() { + let mut old = window("w-1", "Old", "Editor", 100, false); + old.process_instance = Some("generation-a".into()); + let mut new = window("w-1", "New", "Editor", 100, false); + new.process_instance = Some("generation-b".into()); + + let events = diff_signals( + &baseline_with_windows(vec![old]), + &baseline_with_windows(vec![new]), + ); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].kind, EventKind::WindowClosed); + assert_eq!(events[1].kind, EventKind::WindowOpened); +} + +#[path = "signals_surface_tests.rs"] +mod surface_tests; diff --git a/crates/core/src/snapshot.rs b/crates/core/src/snapshot.rs index e470b42..6221d67 100644 --- a/crates/core/src/snapshot.rs +++ b/crates/core/src/snapshot.rs @@ -1,9 +1,12 @@ use crate::{ - adapter::{PlatformAdapter, SnapshotSurface, TreeOptions, WindowFilter}, + AccessibilityNode, AppError, WindowInfo, + adapter::{PlatformAdapter, TreeOptions, WindowFilter}, context::CommandContext, - error::AppError, - node::{AccessibilityNode, WindowInfo}, + live_locator::{ObservationRequest, ObservationRoot}, ref_alloc::{self, RefAllocConfig}, + ref_alloc_options::RefAllocOptions, + ref_alloc_scope::RefAllocScope, + ref_alloc_source::RefAllocSource, refs::RefMap, refs_store::RefStore, trace_artifacts, @@ -18,83 +21,51 @@ pub struct SnapshotResult { pub snapshot_id: Option<String>, } +impl SnapshotResult { + pub(crate) fn bind_snapshot_id(&mut self, snapshot_id: String) { + crate::ref_token::qualify_tree_refs(&mut self.tree, &snapshot_id); + self.snapshot_id = Some(snapshot_id); + } +} + pub fn build( adapter: &dyn PlatformAdapter, opts: &TreeOptions, app_name: Option<&str>, window_id: Option<&str>, + deadline: crate::Deadline, ) -> Result<SnapshotResult, AppError> { - let filter = WindowFilter { - focused_only: app_name.is_none() && window_id.is_none(), - app: app_name.map(str::to_string), - }; - - let windows = adapter.list_windows(&filter)?; - - let window = if let Some(wid) = window_id { - windows.into_iter().find(|w| w.id == wid).ok_or_else(|| { - AppError::Adapter( - crate::error::AdapterError::new( - crate::error::ErrorCode::WindowNotFound, - format!("No window with id {wid}"), - ) - .with_suggestion("Run 'list-windows' to see available window IDs."), - ) - })? - } else if let Some(app) = app_name { - windows - .into_iter() - .find(|w| w.app.eq_ignore_ascii_case(app) && w.is_focused) - .or_else(|| { - adapter - .list_windows(&WindowFilter { - focused_only: false, - app: Some(app.to_string()), - }) - .ok() - .and_then(|ws| ws.into_iter().next()) - }) - .ok_or_else(|| { - AppError::Adapter( - crate::error::AdapterError::new( - crate::error::ErrorCode::AppNotFound, - format!("No window found for app '{app}'"), - ) - .with_suggestion( - "Verify the app is running. Use 'list-apps' to see running applications.", - ), - ) - })? - } else { - windows.into_iter().find(|w| w.is_focused).ok_or_else(|| { - AppError::Adapter( - crate::error::AdapterError::new( - crate::error::ErrorCode::WindowNotFound, - "No focused window found", - ) - .with_suggestion( - "Use --app to specify an application, or click a window to focus it.", - ), - ) - })? - }; - - let raw_tree = adapter.get_tree(&window, &opts.with_ref_identity_bounds())?; + let window = resolve_window(adapter, app_name, window_id, deadline)?; + let observation_options = opts.with_ref_identity_bounds(); + let raw_tree = crate::renderer_accessibility::observe_tree( + adapter, + ObservationRoot::Window(&window), + &ObservationRequest::snapshot(&observation_options, deadline).validate()?, + )? + .into_accessibility_tree()?; let mut refmap = RefMap::new(); let config = RefAllocConfig { - include_bounds: opts.include_bounds, - interactive_only: opts.interactive_only, - compact: opts.compact, - pid: window.pid, - source_app: Some(window.app.as_str()), - source_window_id: Some(window.id.as_str()), - source_window_title: Some(window.title.as_str()), - source_surface: opts.surface, - root_ref_id: None, - path_prefix: &[], + options: RefAllocOptions { + include_bounds: opts.include_bounds, + interactive_only: opts.interactive_only, + compact: opts.compact, + }, + source: RefAllocSource { + pid: window.pid, + app: Some(window.app.as_str()), + window_id: Some(window.id.as_str()), + window_title: Some(window.title.as_str()), + window_bounds_hash: window.bounds.as_ref().and_then(crate::Rect::bounds_hash), + process_instance: window.process_instance.as_deref(), + surface: opts.surface, + }, + scope: RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, }; - let mut tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); + let mut tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config)?; crate::hints::add_structural_hints(&mut tree); @@ -106,6 +77,51 @@ pub fn build( }) } +pub(crate) fn resolve_window( + adapter: &dyn PlatformAdapter, + app_name: Option<&str>, + window_id: Option<&str>, + deadline: crate::Deadline, +) -> Result<WindowInfo, AppError> { + let filter = WindowFilter { + focused_only: app_name.is_none() && window_id.is_none(), + app: app_name.map(str::to_string), + }; + + let windows = adapter.list_windows(&filter, deadline)?; + + if let Some(wid) = window_id { + windows.into_iter().find(|w| w.id == wid).ok_or_else(|| { + AppError::Adapter( + crate::AdapterError::new( + crate::ErrorCode::WindowNotFound, + format!("No window with id {wid}"), + ) + .with_suggestion("Run 'list-windows' to see available window IDs."), + ) + }) + } else if let Some(app) = app_name { + let candidates = windows + .into_iter() + .filter(|window| window.app.eq_ignore_ascii_case(app)) + .collect::<Vec<_>>(); + crate::window_lookup::select_window( + candidates, + crate::AdapterError::new( + crate::ErrorCode::AppNotFound, + format!("No window found for app '{app}'"), + ), + "More than one window matches the target", + ) + } else { + crate::window_lookup::select_window( + windows, + crate::AdapterError::new(crate::ErrorCode::WindowNotFound, "No focused window found"), + "More than one window matches the target", + ) + } +} + #[cfg(test)] pub fn run( adapter: &dyn PlatformAdapter, @@ -129,11 +145,17 @@ pub fn run_with_context( window_id: Option<&str>, context: &CommandContext, ) -> Result<SnapshotResult, AppError> { - let mut result = build(adapter, opts, app_name, window_id)?; + let mut result = build( + adapter, + opts, + app_name, + window_id, + crate::Deadline::after(3_000)?, + )?; let store = RefStore::for_session(context.session_id())?; let snapshot_id = store.save_new_snapshot(&result.refmap)?; trace_artifacts::copy_refmap_if_full(context, &store, &snapshot_id, &result.refmap)?; - result.snapshot_id = Some(snapshot_id); + result.bind_snapshot_id(snapshot_id); emit_snapshot_saved(context, &result)?; Ok(result) } @@ -154,67 +176,6 @@ pub(crate) fn emit_snapshot_saved( }) } -pub fn append_surface_refs( - adapter: &dyn PlatformAdapter, - pid: i32, - source_app: Option<&str>, - surface: SnapshotSurface, -) -> Result<Option<AccessibilityNode>, AppError> { - append_surface_refs_with_context( - adapter, - pid, - source_app, - surface, - &CommandContext::default(), - ) -} - -pub fn append_surface_refs_with_context( - adapter: &dyn PlatformAdapter, - pid: i32, - source_app: Option<&str>, - surface: SnapshotSurface, - context: &CommandContext, -) -> Result<Option<AccessibilityNode>, AppError> { - let filter = WindowFilter { - focused_only: false, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - let Some(window) = windows.into_iter().find(|w| w.pid == pid) else { - return Ok(None); - }; - let opts = TreeOptions { - surface, - interactive_only: true, - ..Default::default() - }; - let raw_tree = adapter.get_tree(&window, &opts.with_ref_identity_bounds())?; - let store = RefStore::for_session(context.session_id())?; - let mut refmap = store.load_latest()?; - let config = RefAllocConfig { - include_bounds: false, - interactive_only: true, - compact: false, - pid, - source_app, - source_window_id: Some(window.id.as_str()), - source_window_title: Some(window.title.as_str()), - source_surface: surface, - root_ref_id: None, - path_prefix: &[], - }; - let tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); - if let Some(id) = store.latest_snapshot_id() { - store.save_existing_snapshot(&id, &refmap)?; - trace_artifacts::copy_refmap_if_full(context, &store, &id, &refmap)?; - } else { - let id = store.save_new_snapshot(&refmap)?; - trace_artifacts::copy_refmap_if_full(context, &store, &id, &refmap)?; - } - Ok(Some(tree)) -} - #[cfg(test)] #[path = "snapshot_tests.rs"] mod tests; diff --git a/crates/core/src/snapshot_ref.rs b/crates/core/src/snapshot_ref.rs index 9532519..73e448f 100644 --- a/crates/core/src/snapshot_ref.rs +++ b/crates/core/src/snapshot_ref.rs @@ -1,11 +1,13 @@ use crate::{ + AppError, adapter::{PlatformAdapter, TreeOptions}, context::CommandContext, - error::AppError, - node::WindowInfo, + live_locator::{ObservationRequest, ObservationRoot}, ref_alloc::{self, RefAllocConfig}, + ref_alloc_options::RefAllocOptions, + ref_alloc_scope::RefAllocScope, + ref_alloc_source::RefAllocSource, refs_store::RefStore, - resolved_element::ResolvedElement, snapshot::SnapshotResult, }; @@ -33,77 +35,92 @@ pub fn run_from_ref_with_context( context: &CommandContext, ) -> Result<SnapshotResult, AppError> { let store = RefStore::for_session(context.session_id())?; - let mut refmap = store.load(snapshot_id)?; - let active_snapshot_id = snapshot_id - .map(str::to_string) - .or_else(|| store.latest_snapshot_id()); + let (active_snapshot_id, local_root_ref) = + crate::ref_token::resolve_ref_target(root_ref_id, snapshot_id)?; + let refmap = store.load_snapshot(&active_snapshot_id)?; let entry = refmap - .get(root_ref_id) + .get(&local_root_ref) .ok_or_else(|| AppError::stale_ref(root_ref_id))? .clone(); - let handle = ResolvedElement::new(adapter, adapter.resolve_element_strict(&entry)?); + let deadline = crate::Deadline::after(3_000)?; + let handle = adapter.resolve_element_strict(&entry, deadline)?; - let raw_tree = adapter.get_subtree(handle.handle(), &opts.with_ref_identity_bounds())?; + let observation_options = opts.with_ref_identity_bounds(); + let raw_tree = crate::renderer_accessibility::observe_tree( + adapter, + ObservationRoot::Element { + handle: &handle, + entry: &entry, + root_ref: Some(&local_root_ref), + }, + &ObservationRequest::snapshot(&observation_options, deadline).validate()?, + )? + .into_accessibility_tree()?; - refmap.remove_by_root_ref(root_ref_id); - - let source_app = entry.source_app.as_deref(); - let source_window_id = entry.source_window_id.as_deref(); - let source_window_title = entry.source_window_title.as_deref(); - let path_prefix = entry.path.clone(); + let source_app = entry.source.source_app.as_deref(); + let source_window_id = entry.source.source_window_id.as_deref(); + let source_window_title = entry.source.source_window_title.as_deref(); + let path_prefix = entry.scope.path.clone(); let config = RefAllocConfig { - include_bounds: opts.include_bounds, - interactive_only: opts.interactive_only, - compact: opts.compact, - pid: entry.pid, - source_app, - source_window_id, - source_window_title, - source_surface: entry.source_surface, - root_ref_id: Some(root_ref_id), - path_prefix: path_prefix.as_slice(), + options: RefAllocOptions { + include_bounds: opts.include_bounds, + interactive_only: opts.interactive_only, + compact: opts.compact, + }, + source: RefAllocSource { + pid: entry.process.pid, + app: source_app, + window_id: source_window_id, + window_title: source_window_title, + window_bounds_hash: entry.source.source_window_bounds_hash, + process_instance: entry.process.process_instance.as_deref(), + surface: entry.source.source_surface, + }, + scope: RefAllocScope { + root_ref_id: Some(&local_root_ref), + path_prefix: path_prefix.as_slice(), + }, }; - let mut tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); - - crate::hints::add_structural_hints(&mut tree); - - let saved_snapshot_id = if let Some(id) = active_snapshot_id { - store.save_existing_snapshot(&id, &refmap)?; - crate::trace_artifacts::copy_refmap_if_full(context, &store, &id, &refmap)?; - Some(id) - } else { - let id = store.save_new_snapshot(&refmap)?; - crate::trace_artifacts::copy_refmap_if_full(context, &store, &id, &refmap)?; - Some(id) - }; + let (mut tree, refmap) = store.update_existing_snapshot( + &active_snapshot_id, + &local_root_ref, + &entry, + |current| { + current.remove_by_root_ref(&local_root_ref); + let mut tree = ref_alloc::allocate_refs(raw_tree, current, &config)?; + crate::hints::add_structural_hints(&mut tree); + Ok(tree) + }, + )?; + crate::trace_artifacts::copy_refmap_if_full(context, &store, &active_snapshot_id, &refmap)?; + crate::ref_token::qualify_tree_refs(&mut tree, &active_snapshot_id); context.trace_lazy("snapshot.root.saved", || { serde_json::json!({ "root_ref": root_ref_id, - "snapshot_id": saved_snapshot_id, + "snapshot_id": active_snapshot_id, "ref_count": refmap.len() }) })?; - let window = - crate::window_lookup::find_window_for_pid(entry.pid, adapter).unwrap_or(WindowInfo { - id: String::new(), - title: entry - .source_window_title - .unwrap_or_else(|| format!("subtree from {root_ref_id}")), - app: entry.source_app.unwrap_or_default(), - pid: entry.pid, - bounds: None, - is_focused: true, - }); + let instance = entry.process.process_instance.as_deref().ok_or_else(|| { + AppError::Adapter(crate::AdapterError::stale_ref( + "root ref has no process-instance identity", + )) + })?; + let window = crate::window_lookup::find_window_for_process( + crate::ProcessIdentity::new(entry.process.pid, instance), + adapter, + deadline, + )?; Ok(SnapshotResult { tree, refmap, window, - snapshot_id: saved_snapshot_id, + snapshot_id: Some(active_snapshot_id), }) } diff --git a/crates/core/src/snapshot_ref_alloc_tests.rs b/crates/core/src/snapshot_ref_alloc_tests.rs index 5bb092f..7252436 100644 --- a/crates/core/src/snapshot_ref_alloc_tests.rs +++ b/crates/core/src/snapshot_ref_alloc_tests.rs @@ -1,4 +1,4 @@ -use crate::node::AccessibilityNode; +use crate::AccessibilityNode; use crate::ref_alloc::{self, RefAllocConfig}; use crate::refs::RefMap; @@ -6,13 +6,8 @@ fn node(role: &str) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: role.into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: vec![], } @@ -20,35 +15,43 @@ fn node(role: &str) -> AccessibilityNode { fn drill_config<'a>( source_app: Option<&'a str>, - pid: i32, + pid: u32, root_ref_id: &'a str, interactive_only: bool, compact: bool, ) -> RefAllocConfig<'a> { RefAllocConfig { - include_bounds: false, - interactive_only, - compact, - pid, - source_app, - source_window_id: None, - source_window_title: Some("Drill Window"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: Some(root_ref_id), - path_prefix: &[], + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: false, + interactive_only, + compact, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(pid), + app: source_app, + window_id: None, + window_title: Some("Drill Window"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: Some(root_ref_id), + path_prefix: &[], + }, } } #[test] fn test_drill_alloc_tags_entries() { let mut btn = node("button"); - btn.name = Some("Submit".into()); + btn.identity.name = Some("Submit".into()); let mut root = node("group"); root.children = vec![btn]; let mut refmap = RefMap::new(); let config = drill_config(Some("TestApp"), 42, "@e5", false, false); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config).unwrap(); assert_eq!(refmap.len(), 1); let btn_ref = tree.children[0] @@ -56,9 +59,9 @@ fn test_drill_alloc_tags_entries() { .as_deref() .expect("button should have ref"); let entry = refmap.get(btn_ref).expect("entry should exist"); - assert_eq!(entry.root_ref.as_deref(), Some("@e5")); - assert_eq!(entry.pid, 42); - assert_eq!(entry.source_app.as_deref(), Some("TestApp")); + assert_eq!(entry.scope.root_ref.as_deref(), Some("@e5")); + assert_eq!(entry.process.pid, 42); + assert_eq!(entry.source.source_app.as_deref(), Some("TestApp")); } #[test] @@ -70,7 +73,7 @@ fn test_drill_alloc_respects_interactive_only() { let mut refmap = RefMap::new(); let config = drill_config(None, 1, "@e1", true, false); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config).unwrap(); assert_eq!(tree.children.len(), 1); assert_eq!(tree.children[0].role, "button"); @@ -79,14 +82,14 @@ fn test_drill_alloc_respects_interactive_only() { #[test] fn test_drill_alloc_preserves_truncated_child() { let mut container = node("group"); - container.name = Some("Sidebar".into()); + container.identity.name = Some("Sidebar".into()); container.children_count = Some(4); let mut root = node("window"); root.children = vec![container]; let mut refmap = RefMap::new(); let config = drill_config(None, 1, "@e1", true, false); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config).unwrap(); assert_eq!(tree.children.len(), 1); assert_eq!(tree.children[0].children_count, Some(4)); @@ -95,7 +98,7 @@ fn test_drill_alloc_preserves_truncated_child() { #[test] fn test_drill_alloc_compact() { let mut btn = node("button"); - btn.name = Some("OK".into()); + btn.identity.name = Some("OK".into()); let mut wrapper = node("group"); wrapper.children = vec![btn]; let mut root = node("window"); @@ -103,7 +106,7 @@ fn test_drill_alloc_compact() { let mut refmap = RefMap::new(); let config = drill_config(None, 1, "@e1", false, true); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config).unwrap(); assert_eq!(tree.children.len(), 1); assert_eq!(tree.children[0].role, "button"); diff --git a/crates/core/src/snapshot_ref_merge_tests.rs b/crates/core/src/snapshot_ref_merge_tests.rs new file mode 100644 index 0000000..b9473e9 --- /dev/null +++ b/crates/core/src/snapshot_ref_merge_tests.rs @@ -0,0 +1,100 @@ +use super::*; + +#[test] +fn test_run_from_ref_multiple_drill_downs_accumulate() { + let _guard = HomeGuard::new(); + let snapshot_id = save_latest(seed_skeleton_refmap()); + + let adapter_one = StubAdapter::new(named("button", "FromE1")); + let first = run_from_ref(&adapter_one, &drill_opts(), "@e1", Some(&snapshot_id)).unwrap(); + let from_e1_ref = first.tree.ref_id.clone().expect("first drill ref"); + + let adapter_two = StubAdapter::new(named("button", "FromE2")); + let second = run_from_ref(&adapter_two, &drill_opts(), "@e2", Some(&snapshot_id)).unwrap(); + let from_e2_ref = second.tree.ref_id.clone().expect("second drill ref"); + + let on_disk = load_latest(); + assert!(on_disk.get("@e1").is_some(), "skeleton @e1 preserved"); + assert!(on_disk.get("@e2").is_some(), "skeleton @e2 preserved"); + let entry_one = on_disk + .get(&local_ref(&from_e1_ref)) + .expect("@e1 drill survives"); + assert_eq!(entry_one.scope.root_ref.as_deref(), Some("@e1")); + let entry_two = on_disk + .get(&local_ref(&from_e2_ref)) + .expect("@e2 drill survives"); + assert_eq!(entry_two.scope.root_ref.as_deref(), Some("@e2")); +} + +#[test] +fn test_drilldown_refmap_matches_golden_fixture() { + let golden = include_str!("../../../tests/fixtures/drilldown-refmap.json"); + let golden_value: serde_json::Value = serde_json::from_str(golden).unwrap(); + let expected_total = golden_value["expected_total"].as_u64().unwrap() as usize; + + let _guard = HomeGuard::new(); + let mut seed = RefMap::new(); + seed.allocate(ref_entry_from_node( + &named("group", "Sidebar"), + &source("Fixture"), + None, + &[0], + )); + seed.allocate(ref_entry_from_node( + &named("group", "Toolbar"), + &source("Fixture"), + None, + &[1], + )); + let snapshot_id = save_latest(seed); + + let mut sidebar_subtree = named("outline", "Sidebar"); + sidebar_subtree.children = vec![named("treeitem", "Recents"), named("treeitem", "Documents")]; + let adapter = StubAdapter::new(sidebar_subtree); + let _ = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)).unwrap(); + + let toolbar_subtree = named("button", "Back"); + let adapter = StubAdapter::new(toolbar_subtree); + let _ = run_from_ref(&adapter, &drill_opts(), "@e2", Some(&snapshot_id)).unwrap(); + + let on_disk = load_latest(); + assert_eq!( + on_disk.len(), + expected_total, + "merged refmap should match golden fixture's expected_total" + ); + + for anchor in golden_value["skeleton_anchors"].as_array().unwrap() { + let id = anchor["ref_id"].as_str().unwrap(); + let entry = on_disk.get(id).unwrap_or_else(|| panic!("missing {id}")); + assert_eq!(entry.identity.role, anchor["role"].as_str().unwrap()); + assert_eq!(entry.identity.name.as_deref(), anchor["name"].as_str()); + assert!( + entry.scope.root_ref.is_none(), + "skeleton {id} must have null root_ref" + ); + } + + for drill in golden_value["drilled_from_e1"].as_array().unwrap() { + let id = drill["ref_id"].as_str().unwrap(); + if let Some(entry) = on_disk.get(id) { + assert_eq!(entry.scope.root_ref.as_deref(), Some("@e1")); + } + } +} + +#[test] +fn test_run_from_ref_empty_subtree() { + let _guard = HomeGuard::new(); + let snapshot_id = save_latest(seed_skeleton_refmap()); + + let adapter = StubAdapter::new(node("group")); + let result = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)).unwrap(); + + assert!(result.tree.children.is_empty()); + assert_eq!( + result.refmap.len(), + 2, + "no new refs added for empty subtree" + ); +} diff --git a/crates/core/src/snapshot_ref_tests.rs b/crates/core/src/snapshot_ref_tests.rs index f00b2f1..f5894c1 100644 --- a/crates/core/src/snapshot_ref_tests.rs +++ b/crates/core/src/snapshot_ref_tests.rs @@ -1,24 +1,30 @@ use super::*; +use crate::AccessibilityNode; +use crate::AdapterError; use crate::action_request::ActionRequest; -use crate::adapter::{NativeHandle, PlatformAdapter}; -use crate::error::AdapterError; -use crate::node::AccessibilityNode; +use crate::adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, SystemOps}; use crate::ref_alloc::ref_entry_from_node; use crate::refs_test_support::HomeGuard; use crate::{refs::RefMap, refs_store::RefStore}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; + +struct DropProbe(Arc<AtomicU32>); + +impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} fn node(role: &str) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: role.into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: vec![], } @@ -26,15 +32,27 @@ fn node(role: &str) -> AccessibilityNode { fn named(role: &str, name: &str) -> AccessibilityNode { let mut n = node(role); - n.name = Some(name.into()); + n.identity.name = Some(name.into()); n } +fn source(app: &'static str) -> crate::ref_alloc_source::RefAllocSource<'static> { + crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(42), + app: Some(app), + window_id: None, + window_title: None, + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + } +} + struct StubAdapter { subtree: AccessibilityNode, subtree_error: Option<AdapterError>, resolve_calls: AtomicU32, - release_calls: AtomicU32, + drops: Arc<AtomicU32>, } impl StubAdapter { @@ -43,7 +61,7 @@ impl StubAdapter { subtree, subtree_error: None, resolve_calls: AtomicU32::new(0), - release_calls: AtomicU32::new(0), + drops: Arc::new(AtomicU32::new(0)), } } @@ -52,45 +70,79 @@ impl StubAdapter { subtree: node("group"), subtree_error: Some(error), resolve_calls: AtomicU32::new(0), - release_calls: AtomicU32::new(0), + drops: Arc::new(AtomicU32::new(0)), } } } -impl PlatformAdapter for StubAdapter { +impl ObservationOps for StubAdapter { + fn observe_tree( + &self, + root: crate::live_locator::ObservationRoot<'_>, + _request: &crate::live_locator::ObservationRequest, + ) -> Result<crate::live_locator::ObservedTree, AdapterError> { + if let Some(error) = &self.subtree_error { + return Err(error.clone()); + } + crate::adapter::observed_tree(&root, self.subtree.clone()) + } + + fn list_windows( + &self, + _filter: &crate::adapter::WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<crate::WindowInfo>, AdapterError> { + Ok(vec![crate::WindowInfo { + id: "w-42".into(), + title: "Test".into(), + app: "TestApp".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("test-instance".into()), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + }]) + } + fn resolve_element_strict( &self, _entry: &crate::refs::RefEntry, + _deadline: crate::Deadline, ) -> Result<NativeHandle, AdapterError> { self.resolve_calls.fetch_add(1, Ordering::SeqCst); - Ok(NativeHandle::null()) + Ok(NativeHandle::new(DropProbe(Arc::clone(&self.drops)))) } fn get_subtree( &self, _handle: &NativeHandle, _opts: &TreeOptions, + _deadline: crate::Deadline, ) -> Result<AccessibilityNode, AdapterError> { if let Some(error) = &self.subtree_error { return Err(error.clone()); } Ok(self.subtree.clone()) } +} - fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - self.release_calls.fetch_add(1, Ordering::SeqCst); - Ok(()) - } - +impl ActionOps for StubAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<crate::action_result::ActionResult, AdapterError> { Err(AdapterError::not_supported("execute_action")) } } +impl InputOps for StubAdapter {} + +impl SystemOps for StubAdapter {} + fn save_latest(refmap: RefMap) -> String { RefStore::new() .unwrap() @@ -119,27 +171,17 @@ fn load_session_snapshot(session_id: &str, snapshot_id: &str) -> RefMap { .expect("session snapshot should load") } +fn local_ref(ref_id: &str) -> String { + crate::ref_token::resolve_ref_target(ref_id, None) + .expect("result refs must be snapshot-qualified") + .1 +} + fn seed_skeleton_refmap() -> RefMap { let mut map = RefMap::new(); - let anchor = ref_entry_from_node( - &named("group", "Sidebar"), - 42, - Some("TestApp"), - None, - None, - None, - &[0], - ); + let anchor = ref_entry_from_node(&named("group", "Sidebar"), &source("TestApp"), None, &[0]); let _ = map.allocate(anchor); - let other = ref_entry_from_node( - &named("button", "Toolbar"), - 42, - Some("TestApp"), - None, - None, - None, - &[1], - ); + let other = ref_entry_from_node(&named("button", "Toolbar"), &source("TestApp"), None, &[1]); let _ = map.allocate(other); map } @@ -154,7 +196,7 @@ fn drill_opts() -> TreeOptions { #[test] fn test_run_from_ref_returns_subtree_and_persists_refs() { let _guard = HomeGuard::new(); - save_latest(seed_skeleton_refmap()); + let snapshot_id = save_latest(seed_skeleton_refmap()); let mut child_btn = named("button", "Save"); child_btn.children = vec![]; @@ -162,7 +204,8 @@ fn test_run_from_ref_returns_subtree_and_persists_refs() { subtree_root.children = vec![child_btn]; let adapter = StubAdapter::new(subtree_root); - let result = run_from_ref(&adapter, &drill_opts(), "@e1", None).expect("drill should succeed"); + let result = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)) + .expect("drill should succeed"); let on_disk = load_latest(); assert_eq!(on_disk.len(), result.refmap.len()); @@ -179,65 +222,75 @@ fn test_run_from_ref_returns_subtree_and_persists_refs() { .find(|c| c.role == "button") .and_then(|c| c.ref_id.as_deref()) .expect("button child should carry a ref"); - let drill_entry = on_disk.get(drill_ref).expect("entry persisted"); - assert_eq!(drill_entry.root_ref.as_deref(), Some("@e1")); + let drill_entry = on_disk.get(&local_ref(drill_ref)).expect("entry persisted"); + assert_eq!(drill_entry.scope.root_ref.as_deref(), Some("@e1")); assert!( - drill_entry.path_is_absolute, + drill_entry.scope.path_is_absolute, "drilled refs must retain an absolute path for fast, scoped resolution" ); - assert_eq!(drill_entry.path.as_slice(), [0, 0]); + assert_eq!(drill_entry.scope.path.as_slice(), [0, 0]); assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1); - assert_eq!(adapter.release_calls.load(Ordering::SeqCst), 1); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); } #[test] -fn test_run_from_ref_explicit_session_snapshot_without_session_context() { +fn test_run_from_ref_explicit_session_snapshot_with_matching_context() { let _guard = HomeGuard::new(); let snapshot_id = save_session("agent-a", seed_skeleton_refmap()); let adapter = StubAdapter::new(named("button", "Save")); - let result = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)) - .expect("explicit snapshot should drill without repeating --session"); + let context = crate::CommandContext::new(Some("agent-a".into()), None, false).unwrap(); + let result = + run_from_ref_with_context(&adapter, &drill_opts(), "@e1", Some(&snapshot_id), &context) + .expect("session snapshot should drill within its namespace"); assert_eq!(result.snapshot_id.as_deref(), Some(snapshot_id.as_str())); let on_disk = load_session_snapshot("agent-a", &snapshot_id); assert!(on_disk.get("@e1").is_some(), "skeleton anchor preserved"); let drill_ref = result.tree.ref_id.as_deref().expect("drill ref"); - let drill_entry = on_disk.get(drill_ref).expect("drill ref persisted"); - assert_eq!(drill_entry.name.as_deref(), Some("Save")); - assert!(RefStore::new().unwrap().latest_snapshot_id().is_none()); + let drill_entry = on_disk + .get(&local_ref(drill_ref)) + .expect("drill ref persisted"); + assert_eq!(drill_entry.identity.name.as_deref(), Some("Save")); + assert!( + RefStore::new() + .unwrap() + .latest_snapshot_id() + .unwrap() + .is_none() + ); } #[test] -fn test_run_from_ref_releases_handle_when_subtree_read_fails() { +fn test_run_from_ref_drops_handle_when_subtree_read_fails() { let _guard = HomeGuard::new(); - save_latest(seed_skeleton_refmap()); + let snapshot_id = save_latest(seed_skeleton_refmap()); let adapter = StubAdapter::with_subtree_error(AdapterError::new( - crate::error::ErrorCode::ActionFailed, + crate::ErrorCode::ActionFailed, "subtree failed", )); - let result = run_from_ref(&adapter, &drill_opts(), "@e1", None); + let result = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)); assert!(result.is_err()); assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1); - assert_eq!(adapter.release_calls.load(Ordering::SeqCst), 1); + assert_eq!(adapter.drops.load(Ordering::SeqCst), 1); } #[test] fn test_run_from_ref_stale_root_returns_stale_ref() { let _guard = HomeGuard::new(); - save_latest(RefMap::new()); + let snapshot_id = save_latest(RefMap::new()); let adapter = StubAdapter::new(named("group", "Sidebar")); - let result = run_from_ref(&adapter, &drill_opts(), "@e99", None); + let result = run_from_ref(&adapter, &drill_opts(), "@e99", Some(&snapshot_id)); let err = match result { Ok(_) => panic!("stale root must error"), Err(e) => e, }; match err { AppError::Adapter(adapter_err) => { - assert_eq!(adapter_err.code, crate::error::ErrorCode::StaleRef); + assert_eq!(adapter_err.code, crate::ErrorCode::StaleRef); let suggestion = adapter_err.suggestion.as_deref().unwrap_or(""); assert!( suggestion.contains("snapshot"), @@ -251,16 +304,16 @@ fn test_run_from_ref_stale_root_returns_stale_ref() { #[test] fn test_run_from_ref_re_drill_replaces_drill_refs_only() { let _guard = HomeGuard::new(); - save_latest(seed_skeleton_refmap()); + let snapshot_id = save_latest(seed_skeleton_refmap()); let subtree = named("button", "Save"); let adapter = StubAdapter::new(subtree); - let first = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); + let first = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)).unwrap(); let first_count = first.refmap.len(); let first_button_ref = first.tree.ref_id.clone().expect("button should get a ref"); - let second = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); + let second = run_from_ref(&adapter, &drill_opts(), "@e1", Some(&snapshot_id)).unwrap(); let second_count = second.refmap.len(); let second_button_ref = second.tree.ref_id.clone().expect("button should get a ref"); @@ -274,110 +327,12 @@ fn test_run_from_ref_re_drill_replaces_drill_refs_only() { ); let on_disk = load_latest(); assert!(on_disk.get("@e1").is_some(), "skeleton anchor preserved"); - assert!(on_disk.get(&second_button_ref).is_some()); + assert!(on_disk.get(&local_ref(&second_button_ref)).is_some()); assert!( - on_disk.get(&first_button_ref).is_none(), + on_disk.get(&local_ref(&first_button_ref)).is_none(), "first drill ref must be invalidated by remove_by_root_ref" ); } -#[test] -fn test_run_from_ref_multiple_drill_downs_accumulate() { - let _guard = HomeGuard::new(); - save_latest(seed_skeleton_refmap()); - - let adapter_one = StubAdapter::new(named("button", "FromE1")); - let first = run_from_ref(&adapter_one, &drill_opts(), "@e1", None).unwrap(); - let from_e1_ref = first.tree.ref_id.clone().expect("first drill ref"); - - let adapter_two = StubAdapter::new(named("button", "FromE2")); - let second = run_from_ref(&adapter_two, &drill_opts(), "@e2", None).unwrap(); - let from_e2_ref = second.tree.ref_id.clone().expect("second drill ref"); - - let on_disk = load_latest(); - assert!(on_disk.get("@e1").is_some(), "skeleton @e1 preserved"); - assert!(on_disk.get("@e2").is_some(), "skeleton @e2 preserved"); - let entry_one = on_disk.get(&from_e1_ref).expect("@e1 drill survives"); - assert_eq!(entry_one.root_ref.as_deref(), Some("@e1")); - let entry_two = on_disk.get(&from_e2_ref).expect("@e2 drill survives"); - assert_eq!(entry_two.root_ref.as_deref(), Some("@e2")); -} - -#[test] -fn test_drilldown_refmap_matches_golden_fixture() { - let golden = include_str!("../../../tests/fixtures/drilldown-refmap.json"); - let golden_value: serde_json::Value = serde_json::from_str(golden).unwrap(); - let expected_total = golden_value["expected_total"].as_u64().unwrap() as usize; - - let _guard = HomeGuard::new(); - let mut seed = RefMap::new(); - seed.allocate(ref_entry_from_node( - &named("group", "Sidebar"), - 42, - Some("Fixture"), - None, - None, - None, - &[0], - )); - seed.allocate(ref_entry_from_node( - &named("group", "Toolbar"), - 42, - Some("Fixture"), - None, - None, - None, - &[1], - )); - save_latest(seed); - - let mut sidebar_subtree = named("outline", "Sidebar"); - sidebar_subtree.children = vec![named("treeitem", "Recents"), named("treeitem", "Documents")]; - let adapter = StubAdapter::new(sidebar_subtree); - let _ = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); - - let toolbar_subtree = named("button", "Back"); - let adapter = StubAdapter::new(toolbar_subtree); - let _ = run_from_ref(&adapter, &drill_opts(), "@e2", None).unwrap(); - - let on_disk = load_latest(); - assert_eq!( - on_disk.len(), - expected_total, - "merged refmap should match golden fixture's expected_total" - ); - - for anchor in golden_value["skeleton_anchors"].as_array().unwrap() { - let id = anchor["ref_id"].as_str().unwrap(); - let entry = on_disk.get(id).unwrap_or_else(|| panic!("missing {id}")); - assert_eq!(entry.role, anchor["role"].as_str().unwrap()); - assert_eq!(entry.name.as_deref(), anchor["name"].as_str()); - assert!( - entry.root_ref.is_none(), - "skeleton {id} must have null root_ref" - ); - } - - for drill in golden_value["drilled_from_e1"].as_array().unwrap() { - let id = drill["ref_id"].as_str().unwrap(); - if let Some(entry) = on_disk.get(id) { - assert_eq!(entry.root_ref.as_deref(), Some("@e1")); - } - } -} - -#[test] -fn test_run_from_ref_empty_subtree() { - let _guard = HomeGuard::new(); - save_latest(seed_skeleton_refmap()); - - let adapter = StubAdapter::new(node("group")); - let result = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); - - assert!(result.tree.children.is_empty()); - assert_eq!( - result.refmap.len(), - 2, - "no new refs added for empty subtree" - ); -} +#[path = "snapshot_ref_merge_tests.rs"] +mod merge_tests; diff --git a/crates/core/src/snapshot_surface.rs b/crates/core/src/snapshot_surface.rs new file mode 100644 index 0000000..4caea06 --- /dev/null +++ b/crates/core/src/snapshot_surface.rs @@ -0,0 +1,55 @@ +#[non_exhaustive] +#[derive(Debug, Clone, Copy, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SnapshotSurface { + #[default] + Window, + Focused, + Menu, + Menubar, + Sheet, + Popover, + Alert, + Desktop, + Taskbar, + SystemTray, + QuickSettings, + NotificationCenter, + Toolbar, + Dock, + Spotlight, + MenuBarExtras, + SystemTrayOverflow, + StartMenu, + ActionCenter, +} + +impl SnapshotSurface { + pub fn is_window(surface: &Self) -> bool { + matches!(surface, Self::Window) + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Window => "window", + Self::Focused => "focused", + Self::Menu => "menu", + Self::Menubar => "menubar", + Self::Sheet => "sheet", + Self::Popover => "popover", + Self::Alert => "alert", + Self::Desktop => "desktop", + Self::Taskbar => "taskbar", + Self::SystemTray => "system_tray", + Self::QuickSettings => "quick_settings", + Self::NotificationCenter => "notification_center", + Self::Toolbar => "toolbar", + Self::Dock => "dock", + Self::Spotlight => "spotlight", + Self::MenuBarExtras => "menu_bar_extras", + Self::SystemTrayOverflow => "system_tray_overflow", + Self::StartMenu => "start_menu", + Self::ActionCenter => "action_center", + } + } +} diff --git a/crates/core/src/snapshot_tests.rs b/crates/core/src/snapshot_tests.rs index 6e328bf..177be64 100644 --- a/crates/core/src/snapshot_tests.rs +++ b/crates/core/src/snapshot_tests.rs @@ -1,17 +1,12 @@ use super::*; -use crate::node::AccessibilityNode; +use crate::AccessibilityNode; fn node(role: &str) -> AccessibilityNode { AccessibilityNode { ref_id: None, role: role.into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: Default::default(), + presentation: Default::default(), children_count: None, children: vec![], } @@ -19,33 +14,41 @@ fn node(role: &str) -> AccessibilityNode { fn run_config(compact: bool, interactive_only: bool) -> RefAllocConfig<'static> { RefAllocConfig { - include_bounds: false, - interactive_only, - compact, - pid: 1, - source_app: Some("Test"), - source_window_id: None, - source_window_title: Some("Test Window"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: None, - path_prefix: &[], + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: false, + interactive_only, + compact, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(1), + app: Some("Test"), + window_id: None, + window_title: Some("Test Window"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, } } fn run_compact(tree: AccessibilityNode) -> AccessibilityNode { let mut refmap = RefMap::new(); - ref_alloc::allocate_refs(tree, &mut refmap, &run_config(true, false)) + ref_alloc::allocate_refs(tree, &mut refmap, &run_config(true, false)).unwrap() } fn run_compact_interactive(tree: AccessibilityNode) -> AccessibilityNode { let mut refmap = RefMap::new(); - ref_alloc::allocate_refs(tree, &mut refmap, &run_config(true, true)) + ref_alloc::allocate_refs(tree, &mut refmap, &run_config(true, true)).unwrap() } #[test] fn test_compact_collapses_single_child_chain() { let mut btn = node("button"); - btn.name = Some("Send".into()); + btn.identity.name = Some("Send".into()); let mut g1 = node("group"); g1.children = vec![btn]; let mut g2 = node("group"); @@ -57,14 +60,14 @@ fn test_compact_collapses_single_child_chain() { assert_eq!(result.role, "window"); assert_eq!(result.children.len(), 1); assert_eq!(result.children[0].role, "button"); - assert_eq!(result.children[0].name.as_deref(), Some("Send")); + assert_eq!(result.children[0].identity.name.as_deref(), Some("Send")); } #[test] fn test_compact_preserves_named_containers() { let btn = node("button"); let mut named = node("group"); - named.name = Some("Sidebar".into()); + named.identity.name = Some("Sidebar".into()); named.children = vec![btn]; let mut root = node("window"); root.children = vec![named]; @@ -72,14 +75,14 @@ fn test_compact_preserves_named_containers() { let result = run_compact(root); assert_eq!(result.children.len(), 1); assert_eq!(result.children[0].role, "group"); - assert_eq!(result.children[0].name.as_deref(), Some("Sidebar")); + assert_eq!(result.children[0].identity.name.as_deref(), Some("Sidebar")); } #[test] fn test_compact_preserves_description() { let btn = node("button"); let mut desc_node = node("group"); - desc_node.description = Some("toolbar".into()); + desc_node.identity.description = Some("toolbar".into()); desc_node.children = vec![btn]; let mut root = node("window"); root.children = vec![desc_node]; @@ -87,14 +90,17 @@ fn test_compact_preserves_description() { let result = run_compact(root); assert_eq!(result.children.len(), 1); assert_eq!(result.children[0].role, "group"); - assert_eq!(result.children[0].description.as_deref(), Some("toolbar")); + assert_eq!( + result.children[0].identity.description.as_deref(), + Some("toolbar") + ); } #[test] fn test_compact_preserves_states() { let btn = node("button"); let mut disabled = node("group"); - disabled.states = vec!["disabled".into()]; + disabled.presentation.states = vec!["disabled".into()]; disabled.children = vec![btn]; let mut root = node("window"); root.children = vec![disabled]; @@ -102,7 +108,7 @@ fn test_compact_preserves_states() { let result = run_compact(root); assert_eq!(result.children.len(), 1); assert_eq!(result.children[0].role, "group"); - assert_eq!(result.children[0].states, vec!["disabled"]); + assert_eq!(result.children[0].presentation.states, vec!["disabled"]); } #[test] @@ -123,7 +129,7 @@ fn test_compact_preserves_multi_child() { #[test] fn test_compact_with_interactive_only() { let mut btn = node("button"); - btn.name = Some("OK".into()); + btn.identity.name = Some("OK".into()); let text = node("statictext"); let mut g1 = node("group"); g1.children = vec![btn]; @@ -138,23 +144,47 @@ fn test_compact_with_interactive_only() { assert!(result.children[0].ref_id.is_some()); } +#[test] +fn zero_sized_actionable_node_remains_addressable() { + let mut button = node("button"); + button.identity.name = Some("zero-bounds-button".into()); + button.presentation.bounds = Some(crate::Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }); + let mut root = node("window"); + root.children = vec![button]; + let mut refmap = RefMap::new(); + + let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)).unwrap(); + + assert_eq!( + result.children[0].identity.name.as_deref(), + Some("zero-bounds-button") + ); + assert!(result.children[0].ref_id.is_some()); + assert_eq!(refmap.len(), 1); +} + #[test] fn test_skeleton_named_container_gets_ref() { let mut container = node("group"); - container.name = Some("Sidebar".into()); + container.identity.name = Some("Sidebar".into()); container.children_count = Some(5); let mut root = node("window"); root.children = vec![container]; let mut refmap = RefMap::new(); - let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)); + let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)).unwrap(); assert!(result.children[0].ref_id.is_some()); assert_eq!(refmap.len(), 1); let entry = refmap .get(result.children[0].ref_id.as_deref().unwrap()) .unwrap(); - assert!(entry.available_actions.is_empty()); + assert!(entry.capabilities.available_actions.is_empty()); } #[test] @@ -165,7 +195,7 @@ fn test_skeleton_unnamed_container_no_ref() { root.children = vec![container]; let mut refmap = RefMap::new(); - let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)); + let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)).unwrap(); assert!(result.children[0].ref_id.is_none()); assert_eq!(refmap.len(), 0); @@ -174,25 +204,33 @@ fn test_skeleton_unnamed_container_no_ref() { #[test] fn test_skeleton_anchor_suppressed_in_drilldown() { let mut anchor = node("group"); - anchor.name = Some("Channels".into()); + anchor.identity.name = Some("Channels".into()); anchor.children_count = Some(8); let mut root = node("group"); root.children = vec![anchor]; let mut refmap = RefMap::new(); let config = RefAllocConfig { - include_bounds: false, - interactive_only: false, - compact: false, - pid: 1, - source_app: Some("Test"), - source_window_id: None, - source_window_title: Some("Test Window"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: Some("@e3"), - path_prefix: &[], + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: false, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(1), + app: Some("Test"), + window_id: None, + window_title: Some("Test Window"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: Some("@e3"), + path_prefix: &[], + }, }; - let result = ref_alloc::allocate_refs(root, &mut refmap, &config); + let result = ref_alloc::allocate_refs(root, &mut refmap, &config).unwrap(); assert!( result.children[0].ref_id.is_none(), @@ -204,13 +242,13 @@ fn test_skeleton_anchor_suppressed_in_drilldown() { #[test] fn test_skeleton_described_container_gets_ref() { let mut container = node("group"); - container.description = Some("Channels and direct messages".into()); + container.identity.description = Some("Channels and direct messages".into()); container.children_count = Some(12); let mut root = node("window"); root.children = vec![container]; let mut refmap = RefMap::new(); - let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)); + let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, false)).unwrap(); assert!(result.children[0].ref_id.is_some()); assert_eq!(refmap.len(), 1); @@ -219,13 +257,13 @@ fn test_skeleton_described_container_gets_ref() { #[test] fn test_skeleton_truncated_node_survives_interactive_only() { let mut container = node("group"); - container.name = Some("Content".into()); + container.identity.name = Some("Content".into()); container.children_count = Some(10); let mut root = node("window"); root.children = vec![container]; let mut refmap = RefMap::new(); - let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, true)); + let result = ref_alloc::allocate_refs(root, &mut refmap, &run_config(false, true)).unwrap(); assert_eq!(result.children.len(), 1); assert_eq!(result.children[0].children_count, Some(10)); @@ -237,39 +275,47 @@ fn test_skeleton_fixture_matches_golden() { let golden_value: serde_json::Value = serde_json::from_str(golden).unwrap(); let mut sidebar = node("group"); - sidebar.name = Some("Sidebar".into()); + sidebar.identity.name = Some("Sidebar".into()); sidebar.children_count = Some(26); let mut described = node("group"); - described.description = Some("Channels and direct messages".into()); + described.identity.description = Some("Channels and direct messages".into()); described.children_count = Some(12); let mut send = node("button"); - send.name = Some("Send".into()); + send.identity.name = Some("Send".into()); let mut msg = node("textfield"); - msg.name = Some("Message".into()); + msg.identity.name = Some("Message".into()); let mut content = node("group"); - content.name = Some("Content".into()); + content.identity.name = Some("Content".into()); content.children = vec![send, msg]; let mut root = node("window"); - root.name = Some("Test Window".into()); + root.identity.name = Some("Test Window".into()); root.children = vec![sidebar, described, content]; let mut refmap = RefMap::new(); let config = RefAllocConfig { - include_bounds: false, - interactive_only: false, - compact: false, - pid: 42, - source_app: Some("Fixture"), - source_window_id: None, - source_window_title: Some("Fixture Window"), - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref_id: None, - path_prefix: &[], + options: crate::ref_alloc_options::RefAllocOptions { + include_bounds: false, + interactive_only: false, + compact: false, + }, + source: crate::ref_alloc_source::RefAllocSource { + pid: crate::ProcessId::new(42), + app: Some("Fixture"), + window_id: None, + window_title: Some("Fixture Window"), + window_bounds_hash: None, + process_instance: Some("test-instance"), + surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::ref_alloc_scope::RefAllocScope { + root_ref_id: None, + path_prefix: &[], + }, }; - let result = ref_alloc::allocate_refs(root, &mut refmap, &config); + let result = ref_alloc::allocate_refs(root, &mut refmap, &config).unwrap(); assert_eq!(refmap.len(), 4, "should allocate 4 refs total"); let result_value = serde_json::to_value(&result).unwrap(); diff --git a/crates/core/src/state.rs b/crates/core/src/state.rs new file mode 100644 index 0000000..911eff4 --- /dev/null +++ b/crates/core/src/state.rs @@ -0,0 +1,94 @@ +use crate::Rect; + +pub const FOCUSED: &str = "focused"; +pub const DISABLED: &str = "disabled"; +pub const SECURE: &str = "secure"; +pub const EXPANDED: &str = "expanded"; +pub const CHECKED: &str = "checked"; +pub const SELECTED: &str = "selected"; +pub const HIDDEN: &str = "hidden"; +pub const BUSY: &str = "busy"; +pub const MODAL: &str = "modal"; +pub const REQUIRED: &str = "required"; +pub const INDETERMINATE: &str = "indeterminate"; +pub const PRESSED: &str = "pressed"; +pub const READONLY: &str = "readonly"; +pub const OFFSCREEN: &str = "offscreen"; +/// Vocabulary member with no macOS AX producer today (per U2's producer +/// survey: no AX attribute maps cleanly to element-level validity). Reserved +/// for adapters/platforms that can emit it; `assert_states_in_vocabulary` +/// still accepts it so cross-platform consumers do not have to special-case +/// macOS. +pub const INVALID: &str = "invalid"; +/// Vocabulary member with no macOS AX producer today (per U2's producer +/// survey: AX exposes selection on the selectable child, not a +/// multi-select flag on the container). Reserved for adapters/platforms +/// that can emit it. +pub const MULTISELECTABLE: &str = "multiselectable"; +/// Vocabulary member with no macOS AX producer today (per U2's producer +/// survey: no direct AX attribute for "has a popup"). Reserved for +/// adapters/platforms that can emit it. +pub const HASPOPUP: &str = "haspopup"; + +pub const STATE_VOCABULARY: &[&str] = &[ + FOCUSED, + DISABLED, + SECURE, + EXPANDED, + CHECKED, + SELECTED, + HIDDEN, + BUSY, + MODAL, + REQUIRED, + INDETERMINATE, + PRESSED, + READONLY, + OFFSCREEN, + INVALID, + MULTISELECTABLE, + HASPOPUP, +]; + +pub fn has_state(states: &[String], token: &str) -> bool { + states.iter().any(|state| state == token) +} + +pub fn assert_states_in_vocabulary(states: &[String]) { + for state in states { + assert!( + STATE_VOCABULARY.contains(&state.as_str()), + "state token '{state}' is not in STATE_VOCABULARY" + ); + } +} + +pub fn is_visible(bounds: Option<Rect>, states: &[String]) -> bool { + crate::actionability::bounds_are_visible(bounds) + && !has_state(states, HIDDEN) + && !has_state(states, OFFSCREEN) +} + +pub struct VisibilityEvidence { + pub bounds: Option<Rect>, + pub states: Vec<String>, + pub bounds_from_live: bool, + pub states_from_live: bool, +} + +impl VisibilityEvidence { + pub fn applicable(&self) -> bool { + self.bounds_from_live && self.states_from_live + } + + pub fn result(&self) -> bool { + if !self.applicable() { + return false; + } + is_visible(self.bounds, &self.states) + } +} + +#[cfg(test)] +#[path = "state_tests.rs"] +mod tests; diff --git a/crates/core/src/state_predicate.rs b/crates/core/src/state_predicate.rs new file mode 100644 index 0000000..d5222f7 --- /dev/null +++ b/crates/core/src/state_predicate.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StatePredicate { + pub token: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected: Option<bool>, +} diff --git a/crates/core/src/state_tests.rs b/crates/core/src/state_tests.rs new file mode 100644 index 0000000..bc8a98d --- /dev/null +++ b/crates/core/src/state_tests.rs @@ -0,0 +1,62 @@ +use super::*; +use crate::Rect; + +#[test] +fn vocabulary_contains_seventeen_tokens() { + assert_eq!(STATE_VOCABULARY.len(), 17); +} + +#[test] +fn hidden_element_is_not_visible() { + let bounds = Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + assert!(!is_visible(bounds, &[HIDDEN.to_string()])); +} + +#[test] +fn zero_sized_bounds_are_not_visible() { + let bounds = Some(Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 10.0, + }); + assert!(!is_visible(bounds, &[])); +} + +#[test] +fn offscreen_element_is_not_visible() { + let bounds = Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + assert!(!is_visible(bounds, &[OFFSCREEN.to_string()])); +} + +#[test] +fn visible_element_with_live_evidence() { + let bounds = Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + assert!(is_visible(bounds, &[])); +} + +/// Proves `assert_states_in_vocabulary` is a real membership check and not a +/// tautology: fed a token that is not derived from any `state::` constant, +/// it must panic. Without this, a future refactor could gut the assertion +/// into a no-op and every other conformance test built on top of it would +/// keep passing silently. +#[test] +#[should_panic(expected = "is not in STATE_VOCABULARY")] +fn assert_states_in_vocabulary_panics_on_bogus_token() { + assert_states_in_vocabulary(&["zzz_not_a_real_state_token".to_string()]); +} diff --git a/crates/core/src/step_mechanism.rs b/crates/core/src/step_mechanism.rs new file mode 100644 index 0000000..aeb645d --- /dev/null +++ b/crates/core/src/step_mechanism.rs @@ -0,0 +1,42 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StepMechanism { + SemanticApi, + PhysicalSynthetic, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn semantic_api_serializes_to_snake_case() { + assert_eq!( + serde_json::to_value(StepMechanism::SemanticApi).unwrap(), + json!("semantic_api") + ); + } + + #[test] + fn physical_synthetic_serializes_to_snake_case() { + assert_eq!( + serde_json::to_value(StepMechanism::PhysicalSynthetic).unwrap(), + json!("physical_synthetic") + ); + } + + #[test] + fn semantic_api_round_trips_from_value() { + let value: StepMechanism = serde_json::from_value(json!("semantic_api")).unwrap(); + assert_eq!(value, StepMechanism::SemanticApi); + } + + #[test] + fn physical_synthetic_round_trips_from_value() { + let value: StepMechanism = serde_json::from_value(json!("physical_synthetic")).unwrap(); + assert_eq!(value, StepMechanism::PhysicalSynthetic); + } +} diff --git a/crates/core/src/surface_info.rs b/crates/core/src/surface_info.rs new file mode 100644 index 0000000..79e7cd0 --- /dev/null +++ b/crates/core/src/surface_info.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SurfaceInfo { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub item_count: Option<usize>, +} diff --git a/crates/core/src/surface_signal.rs b/crates/core/src/surface_signal.rs new file mode 100644 index 0000000..6970cff --- /dev/null +++ b/crates/core/src/surface_signal.rs @@ -0,0 +1,13 @@ +use crate::{ProcessId, snapshot_surface::SnapshotSurface}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SurfaceSignal { + pub kind: SnapshotSurface, + pub app: String, + pub pid: ProcessId, + pub process_instance: String, + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option<String>, +} diff --git a/crates/core/src/trace.rs b/crates/core/src/trace.rs index e5a3775..c40b30d 100644 --- a/crates/core/src/trace.rs +++ b/crates/core/src/trace.rs @@ -1,38 +1,16 @@ -use crate::error::AppError; +use crate::AppError; use crate::trace_sanitize::sanitize_trace_value; +use crate::trace_state::{TracePending, TraceState, TraceWriterState}; use serde_json::{Map, Value, json}; use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; -const MAX_TRACE_FILE_BYTES: u64 = 64 * 1024 * 1024; - +pub(crate) const MAX_TRACE_FILE_BYTES: u64 = 64 * 1024 * 1024; +const MAX_TRACE_EVENT_BYTES: usize = 1024 * 1024; static EVENT_SEQ: AtomicU64 = AtomicU64::new(0); -#[derive(Debug, Clone, Default)] -enum TracePending { - #[default] - None, - File(PathBuf), - SegmentDir(PathBuf), -} - -#[derive(Debug, Clone, Default)] -enum WriterState { - #[default] - Unopened, - Open(Arc<Mutex<std::fs::File>>), - Failed, -} - -#[derive(Debug, Default)] -struct TraceState { - pending: TracePending, - writer: Arc<Mutex<WriterState>>, - meta_written: AtomicBool, -} - #[derive(Debug, Clone, Default)] pub struct TraceConfig { strict: bool, @@ -55,17 +33,17 @@ impl TraceConfig { Some(path) => match open_trace_file(&path) { Ok(file) => ( TracePending::File(path), - WriterState::Open(Arc::new(Mutex::new(file))), + TraceWriterState::Open(Arc::new(Mutex::new(file))), ), Err(err) if strict || err.code() == "INVALID_ARGS" => return Err(err), Err(err) => { tracing::warn!("trace open failed: {err}"); - (TracePending::File(path), WriterState::Failed) + (TracePending::File(path), TraceWriterState::Failed) } }, None => match session_segment_dir { - Some(dir) => (TracePending::SegmentDir(dir), WriterState::Unopened), - None => (TracePending::None, WriterState::Unopened), + Some(dir) => (TracePending::SegmentDir(dir), TraceWriterState::Unopened), + None => (TracePending::None, TraceWriterState::Unopened), }, }; Ok(Self { @@ -97,12 +75,15 @@ impl TraceConfig { Some(writer) => writer, None => return Ok(()), }; - self.ensure_meta_if_needed(&writer, session_id)?; match writer .lock() .map_err(|_| AppError::Internal("trace writer lock poisoned".into())) - .and_then(|mut file| write_event(&mut file, event, session_id, fields())) - { + .and_then(|mut file| { + with_exclusive_file(&mut file, |file| { + self.ensure_meta_if_needed(file, session_id)?; + write_event_locked(file, event, session_id, fields()) + }) + }) { Ok(()) => Ok(()), Err(err) if self.strict => Err(err), Err(err) => { @@ -119,13 +100,13 @@ impl TraceConfig { .lock() .map_err(|_| AppError::Internal("trace writer lock poisoned".into()))?; match &*writer { - WriterState::Open(file) => return Ok(Some(file.clone())), - WriterState::Failed => return Ok(None), - WriterState::Unopened => {} + TraceWriterState::Open(file) => return Ok(Some(file.clone())), + TraceWriterState::Failed => return Ok(None), + TraceWriterState::Unopened => {} } let open_result = match &self.state.pending { TracePending::None => { - *writer = WriterState::Failed; + *writer = TraceWriterState::Failed; return Ok(None); } TracePending::File(path) => open_trace_file(path), @@ -134,14 +115,14 @@ impl TraceConfig { match open_result { Ok(file) => { let file = Arc::new(Mutex::new(file)); - *writer = WriterState::Open(file.clone()); + *writer = TraceWriterState::Open(file.clone()); Ok(Some(file)) } Err(err) if self.strict => Err(err), Err(err) if err.code() == "INVALID_ARGS" => Err(err), Err(err) => { tracing::warn!("trace open failed: {err}"); - *writer = WriterState::Failed; + *writer = TraceWriterState::Failed; Ok(None) } } @@ -152,7 +133,7 @@ impl TraceConfig { return false; } match self.state.writer.lock() { - Ok(writer) => !matches!(*writer, WriterState::Failed), + Ok(writer) => !matches!(*writer, TraceWriterState::Failed), Err(_) => true, } } @@ -182,20 +163,17 @@ impl TraceConfig { fn ensure_meta_if_needed( &self, - writer: &Arc<Mutex<std::fs::File>>, + file: &mut std::fs::File, session_id: Option<&str>, ) -> Result<(), AppError> { if self.state.meta_written.load(Ordering::Relaxed) { return Ok(()); } - let mut file = writer - .lock() - .map_err(|_| AppError::Internal("trace writer lock poisoned".into()))?; if file.metadata()?.len() > 0 { self.state.meta_written.store(true, Ordering::Relaxed); return Ok(()); } - write_meta_header(&mut file, session_id)?; + write_meta_header_locked(file, session_id)?; self.state.meta_written.store(true, Ordering::Relaxed); Ok(()) } @@ -231,8 +209,11 @@ pub(crate) fn process_start_ms() -> u64 { }) } -fn write_meta_header(file: &mut std::fs::File, session_id: Option<&str>) -> Result<(), AppError> { - write_event( +fn write_meta_header_locked( + file: &mut std::fs::File, + session_id: Option<&str>, +) -> Result<(), AppError> { + write_event_locked( file, "trace.meta", session_id, @@ -272,27 +253,52 @@ pub(crate) fn ensure_trace_dir(dir: &Path) -> Result<(), AppError> { } fn open_trace_file(path: &Path) -> Result<std::fs::File, AppError> { - let mut options = std::fs::OpenOptions::new(); - options.create(true).append(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - options.custom_flags(libc::O_NOFOLLOW); - } - let file = options.open(path).map_err(AppError::from)?; - reject_loose_trace_permissions(&file)?; - Ok(file) + crate::private_file::open_private_append(path).map_err(|error| { + if error.kind() == std::io::ErrorKind::PermissionDenied { + return AppError::invalid_input_with_suggestion( + "Trace path must be a private user-owned regular file with one link", + "Use a new --trace path or replace the existing unsafe file.", + ); + } + AppError::from(error) + }) } +#[cfg(test)] fn write_event( file: &mut std::fs::File, event: &str, session_id: Option<&str>, fields: Value, ) -> Result<(), AppError> { - reject_oversized_trace(file)?; - let mut body = Map::new(); + with_exclusive_file(file, |file| { + write_event_locked(file, event, session_id, fields) + }) +} + +fn write_event_locked( + file: &mut std::fs::File, + event: &str, + session_id: Option<&str>, + fields: Value, +) -> Result<(), AppError> { + let envelope_bytes = event + .len() + .saturating_mul(6) + .saturating_add(session_id.map_or(0, |session| session.len().saturating_mul(6))) + .saturating_add(512); + if envelope_bytes >= MAX_TRACE_EVENT_BYTES + || !value_fits(&fields, MAX_TRACE_EVENT_BYTES - envelope_bytes) + { + return Err(AppError::invalid_input_with_suggestion( + "Trace event exceeds the maximum supported event size", + "Emit bounded metadata and omit raw application content from trace fields.", + )); + } + let mut body = match sanitize_trace_value(fields) { + Value::Object(fields) => fields, + _ => Map::new(), + }; body.insert("event".to_string(), json!(event)); body.insert( "ts_ms".to_string(), @@ -307,23 +313,63 @@ fn write_event( "seq".to_string(), json!(EVENT_SEQ.fetch_add(1, Ordering::Relaxed)), ); - if let Value::Object(fields) = sanitize_trace_value(fields) { - for (key, value) in fields { - body.insert(key, value); - } - } + body.insert("writer_pid".to_string(), json!(std::process::id())); + body.insert( + "writer_proc_start_ms".to_string(), + json!(process_start_ms()), + ); if let Some(sid) = session_id { body.insert("session_id".to_string(), json!(sid)); } let mut line = Vec::new(); serde_json::to_writer(&mut line, &Value::Object(body))?; line.push(b'\n'); + reject_oversized_trace(file, line.len() as u64)?; file.write_all(&line).map_err(AppError::from) } -fn reject_oversized_trace(file: &std::fs::File) -> Result<(), AppError> { +fn value_fits(value: &Value, max_bytes: usize) -> bool { + fn visit(value: &Value, remaining: &mut usize) -> bool { + let fixed = match value { + Value::Null => 4, + Value::Bool(_) => 5, + Value::Number(_) => 32, + Value::String(string) => string.len().saturating_mul(6), + Value::Array(values) => { + if !take(remaining, values.len()) { + return false; + } + return values.iter().all(|value| visit(value, remaining)); + } + Value::Object(values) => { + for (key, value) in values { + if !take(remaining, key.len().saturating_mul(6).saturating_add(4)) + || !visit(value, remaining) + { + return false; + } + } + return true; + } + }; + take(remaining, fixed) + } + + fn take(remaining: &mut usize, amount: usize) -> bool { + let Some(next) = remaining.checked_sub(amount) else { + return false; + }; + *remaining = next; + true + } + + let mut remaining = max_bytes; + visit(value, &mut remaining) +} + +fn reject_oversized_trace(file: &std::fs::File, incoming: u64) -> Result<(), AppError> { let len = file.metadata()?.len(); - if len < MAX_TRACE_FILE_BYTES { + if len.saturating_add(incoming) <= MAX_TRACE_FILE_BYTES { return Ok(()); } Err(AppError::invalid_input_with_suggestion( @@ -332,23 +378,20 @@ fn reject_oversized_trace(file: &std::fs::File) -> Result<(), AppError> { )) } -#[cfg(unix)] -fn reject_loose_trace_permissions(file: &std::fs::File) -> Result<(), AppError> { - use std::os::unix::fs::PermissionsExt; - - let mode = file.metadata()?.permissions().mode() & 0o777; - if mode & 0o077 == 0 { - return Ok(()); +fn with_exclusive_file<T>( + file: &mut std::fs::File, + operation: impl FnOnce(&mut std::fs::File) -> Result<T, AppError>, +) -> Result<T, AppError> { + file.lock().map_err(AppError::from)?; + let result = operation(file); + let unlock = file.unlock().map_err(AppError::from); + match result { + Ok(value) => { + unlock?; + Ok(value) + } + Err(error) => Err(error), } - Err(AppError::invalid_input_with_suggestion( - "Trace file must not be readable or writable by group/other", - "Use a new --trace path or run chmod 600 on the existing trace file.", - )) -} - -#[cfg(not(unix))] -fn reject_loose_trace_permissions(_file: &std::fs::File) -> Result<(), AppError> { - Ok(()) } #[cfg(test)] diff --git a/crates/core/src/trace_artifact_budget.rs b/crates/core/src/trace_artifact_budget.rs new file mode 100644 index 0000000..f4668f4 --- /dev/null +++ b/crates/core/src/trace_artifact_budget.rs @@ -0,0 +1,374 @@ +use crate::{ + private_file::read_private_bounded, refs::write_private_file, refs_lock::RefStoreLock, +}; +use std::path::Path; + +const SCREENSHOT_BYTE_BUDGET: u64 = 128 * 1024 * 1024; +const SCREENSHOT_COUNT_BUDGET: u32 = 200; +const REFMAP_BYTE_BUDGET: u64 = 64 * 1024 * 1024; +const USAGE_LEDGER_FILE: &str = ".artifact-usage.json"; +const USAGE_LEDGER_MAX_BYTES: u64 = 256; +const ARTIFACT_LOCK_FILE: &str = ".artifact-budget.lock"; + +#[derive(Clone, Copy)] +struct ArtifactLimits { + screenshot_bytes: u64, + screenshot_count: u32, + refmap_bytes: u64, +} + +#[cfg(test)] +thread_local! { + static TEST_LIMITS: std::cell::Cell<Option<ArtifactLimits>> = const { std::cell::Cell::new(None) }; + static TEST_SCAN_COUNT: std::cell::Cell<u32> = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn set_test_limits(screenshot_bytes: u64, screenshot_count: u32, refmap_bytes: u64) { + TEST_LIMITS.with(|limits| { + limits.set(Some(ArtifactLimits { + screenshot_bytes, + screenshot_count, + refmap_bytes, + })); + }); +} + +#[cfg(test)] +pub(crate) fn clear_test_limits() { + TEST_LIMITS.with(|limits| limits.set(None)); +} + +#[cfg(test)] +fn reset_test_scan_count() { + TEST_SCAN_COUNT.with(|count| count.set(0)); +} + +#[cfg(test)] +fn test_scan_count() -> u32 { + TEST_SCAN_COUNT.with(std::cell::Cell::get) +} + +pub(crate) fn write_screenshot( + trace_dir: &Path, + path: &Path, + bytes: &[u8], + deadline: crate::Deadline, +) -> Result<(), &'static str> { + let _lock = artifact_lock_with_deadline(trace_dir, deadline)?; + let limits = limits(); + let usage = load_usage(trace_dir)?; + if usage.1 >= limits.screenshot_count { + return Err("count_budget"); + } + if usage.0.saturating_add(bytes.len() as u64) > limits.screenshot_bytes { + return Err("budget"); + } + let reserved = ( + usage.0.saturating_add(bytes.len() as u64), + usage.1.saturating_add(1), + usage.2, + ); + persist_usage(trace_dir, reserved)?; + if write_private_file(path, bytes).is_err() { + rollback_reservation_if_absent(trace_dir, path, usage); + return Err("write_failed"); + } + Ok(()) +} + +pub(crate) fn write_refmap_if_absent( + trace_dir: &Path, + path: &Path, + bytes: &[u8], +) -> Result<(), &'static str> { + let _lock = artifact_lock(trace_dir)?; + if path.is_file() { + return Ok(()); + } + let usage = load_usage(trace_dir)?; + if usage.2.saturating_add(bytes.len() as u64) > limits().refmap_bytes { + return Err("budget"); + } + let reserved = (usage.0, usage.1, usage.2.saturating_add(bytes.len() as u64)); + persist_usage(trace_dir, reserved)?; + if write_private_file(path, bytes).is_err() { + rollback_reservation_if_absent(trace_dir, path, usage); + return Err("write_failed"); + } + Ok(()) +} + +fn artifact_lock(trace_dir: &Path) -> Result<RefStoreLock, &'static str> { + RefStoreLock::acquire(&trace_dir.join(ARTIFACT_LOCK_FILE)).map_err(|_| "lock_failed") +} + +fn artifact_lock_with_deadline( + trace_dir: &Path, + deadline: crate::Deadline, +) -> Result<RefStoreLock, &'static str> { + RefStoreLock::acquire_with_deadline(&trace_dir.join(ARTIFACT_LOCK_FILE), deadline) + .map_err(|_| "lock_failed") +} + +fn load_usage(trace_dir: &Path) -> Result<(u64, u32, u64), &'static str> { + let ledger = trace_dir.join(USAGE_LEDGER_FILE); + if let Ok(bytes) = read_private_bounded(&ledger, USAGE_LEDGER_MAX_BYTES) + && let Some(usage) = decode_usage(&bytes) + { + return Ok(usage); + } + let (screenshot_bytes, screenshot_count) = directory_usage(&trace_dir.join("screens"))?; + let (refmap_bytes, _) = directory_usage(&trace_dir.join("refmaps"))?; + let usage = (screenshot_bytes, screenshot_count, refmap_bytes); + persist_usage(trace_dir, usage)?; + Ok(usage) +} + +fn decode_usage(bytes: &[u8]) -> Option<(u64, u32, u64)> { + let value: serde_json::Value = serde_json::from_slice(bytes).ok()?; + Some(( + value.get("screenshot_bytes")?.as_u64()?, + u32::try_from(value.get("screenshot_count")?.as_u64()?).ok()?, + value.get("refmap_bytes")?.as_u64()?, + )) +} + +fn persist_usage(trace_dir: &Path, usage: (u64, u32, u64)) -> Result<(), &'static str> { + let bytes = serde_json::to_vec(&serde_json::json!({ + "screenshot_bytes": usage.0, + "screenshot_count": usage.1, + "refmap_bytes": usage.2, + })) + .map_err(|_| "ledger_failed")?; + write_private_file(&trace_dir.join(USAGE_LEDGER_FILE), &bytes).map_err(|_| "ledger_failed") +} + +fn rollback_reservation_if_absent(trace_dir: &Path, path: &Path, usage: (u64, u32, u64)) { + let artifact_may_exist = path + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_file() || metadata.file_type().is_symlink()); + if !artifact_may_exist { + let _ = persist_usage(trace_dir, usage); + } +} + +fn directory_usage(dir: &Path) -> Result<(u64, u32), &'static str> { + #[cfg(test)] + TEST_SCAN_COUNT.with(|count| count.set(count.get().saturating_add(1))); + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok((0, 0)), + Err(_) => return Err("scan_failed"), + }; + let mut bytes = 0_u64; + let mut count = 0_u32; + for entry in entries { + let entry = entry.map_err(|_| "scan_failed")?; + let metadata = entry.path().symlink_metadata().map_err(|_| "scan_failed")?; + if metadata.file_type().is_symlink() { + return Err("scan_failed"); + } + if metadata.is_file() { + bytes = bytes.saturating_add(metadata.len()); + count = count.saturating_add(1); + } + } + Ok((bytes, count)) +} + +fn limits() -> ArtifactLimits { + #[cfg(test)] + if let Some(limits) = TEST_LIMITS.with(std::cell::Cell::get) { + return limits; + } + ArtifactLimits { + screenshot_bytes: SCREENSHOT_BYTE_BUDGET, + screenshot_count: SCREENSHOT_COUNT_BUDGET, + refmap_bytes: REFMAP_BYTE_BUDGET, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_dir(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!( + "agent-desktop-artifact-budget-{name}-{}", + std::process::id() + )) + } + + fn prepare_trace(name: &str) -> std::path::PathBuf { + let trace = temp_dir(name); + let _ = std::fs::remove_dir_all(&trace); + crate::trace::ensure_trace_dir(&trace.join("screens")).unwrap(); + crate::trace::ensure_trace_dir(&trace.join("refmaps")).unwrap(); + trace + } + + fn test_deadline() -> crate::Deadline { + crate::Deadline::standard().unwrap() + } + + #[test] + fn artifact_writes_scan_once_then_use_the_private_ledger() { + let trace = prepare_trace("scan-once"); + reset_test_scan_count(); + set_test_limits(100, 10, 100); + + write_screenshot( + &trace, + &trace.join("screens/a.png"), + &[1, 2], + test_deadline(), + ) + .unwrap(); + assert_eq!(test_scan_count(), 2); + write_screenshot(&trace, &trace.join("screens/b.png"), &[3], test_deadline()).unwrap(); + write_refmap_if_absent(&trace, &trace.join("refmaps/a.json"), &[4, 5]).unwrap(); + write_refmap_if_absent(&trace, &trace.join("refmaps/b.json"), &[6]).unwrap(); + + assert_eq!(test_scan_count(), 2); + let ledger = + read_private_bounded(&trace.join(USAGE_LEDGER_FILE), USAGE_LEDGER_MAX_BYTES).unwrap(); + assert_eq!(decode_usage(&ledger), Some((3, 2, 3))); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(trace.join(USAGE_LEDGER_FILE)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + } + + clear_test_limits(); + std::fs::remove_dir_all(trace).unwrap(); + } + + #[test] + fn missing_and_corrupt_ledgers_are_repaired_from_artifacts() { + let trace = prepare_trace("repair"); + set_test_limits(100, 10, 100); + write_screenshot(&trace, &trace.join("screens/a.png"), &[1], test_deadline()).unwrap(); + + std::fs::remove_file(trace.join(USAGE_LEDGER_FILE)).unwrap(); + reset_test_scan_count(); + write_screenshot(&trace, &trace.join("screens/b.png"), &[2], test_deadline()).unwrap(); + assert_eq!(test_scan_count(), 2); + write_private_file(&trace.join(USAGE_LEDGER_FILE), b"invalid").unwrap(); + reset_test_scan_count(); + write_refmap_if_absent(&trace, &trace.join("refmaps/a.json"), &[3]).unwrap(); + + assert_eq!(test_scan_count(), 2); + let ledger = + read_private_bounded(&trace.join(USAGE_LEDGER_FILE), USAGE_LEDGER_MAX_BYTES).unwrap(); + assert_eq!(decode_usage(&ledger), Some((2, 2, 1))); + clear_test_limits(); + std::fs::remove_dir_all(trace).unwrap(); + } + + #[test] + fn failed_artifact_write_rolls_back_its_reservation() { + let trace = prepare_trace("rollback"); + let blocked = trace.join("screens/blocked.png"); + std::fs::create_dir(&blocked).unwrap(); + set_test_limits(1, 1, 100); + + assert_eq!( + write_screenshot(&trace, &blocked, &[1], test_deadline()), + Err("write_failed") + ); + std::fs::remove_dir(&blocked).unwrap(); + assert!( + write_screenshot(&trace, &trace.join("screens/ok.png"), &[1], test_deadline(),).is_ok() + ); + + clear_test_limits(); + std::fs::remove_dir_all(trace).unwrap(); + } + + #[test] + fn reserved_usage_without_an_artifact_remains_fail_safe() { + let trace = prepare_trace("reserved"); + persist_usage(&trace, (1, 1, 0)).unwrap(); + set_test_limits(1, 1, 100); + + assert_eq!( + write_screenshot( + &trace, + &trace.join("screens/next.png"), + &[1], + test_deadline(), + ), + Err("count_budget") + ); + + clear_test_limits(); + std::fs::remove_dir_all(trace).unwrap(); + } + + #[test] + fn ambiguous_write_failure_with_a_file_keeps_the_reservation() { + let trace = prepare_trace("ambiguous-failure"); + let path = trace.join("screens/maybe-written.png"); + write_private_file(&path, &[1]).unwrap(); + persist_usage(&trace, (1, 1, 0)).unwrap(); + + rollback_reservation_if_absent(&trace, &path, (0, 0, 0)); + + let ledger = + read_private_bounded(&trace.join(USAGE_LEDGER_FILE), USAGE_LEDGER_MAX_BYTES).unwrap(); + assert_eq!(decode_usage(&ledger), Some((1, 1, 0))); + std::fs::remove_dir_all(trace).unwrap(); + } + + #[test] + fn preexisting_files_consume_the_persisted_session_budget() { + let trace = temp_dir("persisted"); + let screens = trace.join("screens"); + let _ = std::fs::remove_dir_all(&trace); + crate::trace::ensure_trace_dir(&screens).unwrap(); + std::fs::write(screens.join("existing.png"), [1]).unwrap(); + set_test_limits(100, 1, 100); + + let result = write_screenshot(&trace, &screens.join("next.png"), &[2], test_deadline()); + + assert_eq!(result, Err("count_budget")); + clear_test_limits(); + std::fs::remove_dir_all(trace).unwrap(); + } + + #[test] + fn separate_session_directories_have_independent_budgets() { + let first = temp_dir("first"); + let second = temp_dir("second"); + let _ = std::fs::remove_dir_all(&first); + let _ = std::fs::remove_dir_all(&second); + crate::trace::ensure_trace_dir(&first.join("screens")).unwrap(); + crate::trace::ensure_trace_dir(&second.join("screens")).unwrap(); + set_test_limits(100, 1, 100); + + assert!( + write_screenshot(&first, &first.join("screens/a.png"), &[1], test_deadline(),).is_ok() + ); + assert!( + write_screenshot( + &second, + &second.join("screens/a.png"), + &[1], + test_deadline(), + ) + .is_ok() + ); + + clear_test_limits(); + std::fs::remove_dir_all(first).unwrap(); + std::fs::remove_dir_all(second).unwrap(); + } +} diff --git a/crates/core/src/trace_artifacts.rs b/crates/core/src/trace_artifacts.rs index a6a776b..44c4592 100644 --- a/crates/core/src/trace_artifacts.rs +++ b/crates/core/src/trace_artifacts.rs @@ -1,128 +1,26 @@ use crate::{ + AppError, adapter::{PlatformAdapter, ScreenshotTarget}, context::CommandContext, - error::AppError, - refs::{RefMap, is_symlink, open_nofollow, write_private_file}, + refs::{RefEntry, RefMap, is_symlink}, refs_store::RefStore, trace::{ensure_trace_dir, process_start_ms}, }; use serde_json::json; -use std::io::Read; use std::path::{Component, Path, PathBuf}; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU32, Ordering}; -const SCREENSHOT_BYTE_BUDGET: u64 = 128 * 1024 * 1024; -const SCREENSHOT_COUNT_BUDGET: u32 = 200; -const REFMAP_BYTE_BUDGET: u64 = 64 * 1024 * 1024; - -// ponytail: per-process budget ceiling — a long multi-invocation session can exceed the intended per-session disk cap; session-scoped accounting (persisted counter under the refstore lock) is deferred to the Phase 4 daemon. static CAPTURE_SEQ: AtomicU32 = AtomicU32::new(0); -static SCREENSHOT_BYTES_USED: AtomicU64 = AtomicU64::new(0); -static SCREENSHOT_COUNT_USED: AtomicU32 = AtomicU32::new(0); -static REFMAP_BYTES_USED: AtomicU64 = AtomicU64::new(0); - -#[cfg(test)] -#[derive(Clone, Copy)] -struct LocalBudget { - screenshot_bytes: u64, - screenshot_count: u32, - refmap_bytes: u64, - screenshot_bytes_used: u64, - screenshot_count_used: u32, - refmap_bytes_used: u64, -} - -#[cfg(test)] -thread_local! { - static LOCAL_BUDGET: std::cell::RefCell<Option<LocalBudget>> = const { std::cell::RefCell::new(None) }; -} +const MAX_EMBED_SCREENSHOT_BYTES: u64 = 128 * 1024 * 1024; #[cfg(test)] pub(crate) fn set_test_budgets(screenshot_bytes: u64, screenshot_count: u32, refmap_bytes: u64) { - LOCAL_BUDGET.with(|cell| { - *cell.borrow_mut() = Some(LocalBudget { - screenshot_bytes, - screenshot_count, - refmap_bytes, - screenshot_bytes_used: 0, - screenshot_count_used: 0, - refmap_bytes_used: 0, - }); - }); + crate::trace_artifact_budget::set_test_limits(screenshot_bytes, screenshot_count, refmap_bytes); } #[cfg(test)] pub(crate) fn clear_test_budgets() { - LOCAL_BUDGET.with(|cell| { - *cell.borrow_mut() = None; - }); -} - -#[cfg(test)] -fn reserve_screenshot_local(byte_len: u64) -> Option<Result<(), &'static str>> { - let mut local = LOCAL_BUDGET.with(|cell| *cell.borrow())?; - if local.screenshot_count_used >= local.screenshot_count { - return Some(Err("count_budget")); - } - if local.screenshot_bytes_used.saturating_add(byte_len) > local.screenshot_bytes { - return Some(Err("budget")); - } - local.screenshot_count_used += 1; - local.screenshot_bytes_used = local.screenshot_bytes_used.saturating_add(byte_len); - LOCAL_BUDGET.with(|cell| *cell.borrow_mut() = Some(local)); - Some(Ok(())) -} - -#[cfg(test)] -fn reserve_refmap_local(byte_len: u64) -> Option<Result<(), &'static str>> { - let mut local = LOCAL_BUDGET.with(|cell| *cell.borrow())?; - if local.refmap_bytes_used.saturating_add(byte_len) > local.refmap_bytes { - return Some(Err("budget")); - } - local.refmap_bytes_used = local.refmap_bytes_used.saturating_add(byte_len); - LOCAL_BUDGET.with(|cell| *cell.borrow_mut() = Some(local)); - Some(Ok(())) -} - -fn reserve_atomic_bytes(used: &AtomicU64, limit: u64, byte_len: u64) -> Result<(), &'static str> { - used.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { - let next = cur.saturating_add(byte_len); - if next > limit { None } else { Some(next) } - }) - .map(|_| ()) - .map_err(|_| "budget") -} - -fn reserve_atomic_count(used: &AtomicU32, limit: u32) -> Result<(), &'static str> { - used.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { - let next = cur.saturating_add(1); - if next > limit { None } else { Some(next) } - }) - .map(|_| ()) - .map_err(|_| "count_budget") -} - -fn reserve_screenshot(byte_len: u64) -> Result<(), &'static str> { - #[cfg(test)] - if let Some(result) = reserve_screenshot_local(byte_len) { - return result; - } - reserve_atomic_count(&SCREENSHOT_COUNT_USED, SCREENSHOT_COUNT_BUDGET)?; - if let Err(reason) = - reserve_atomic_bytes(&SCREENSHOT_BYTES_USED, SCREENSHOT_BYTE_BUDGET, byte_len) - { - SCREENSHOT_COUNT_USED.fetch_sub(1, Ordering::Relaxed); - return Err(reason); - } - Ok(()) -} - -fn reserve_refmap(byte_len: u64) -> Result<(), &'static str> { - #[cfg(test)] - if let Some(result) = reserve_refmap_local(byte_len) { - return result; - } - reserve_atomic_bytes(&REFMAP_BYTES_USED, REFMAP_BYTE_BUDGET, byte_len) + crate::trace_artifact_budget::clear_test_limits(); } #[derive(Debug, Clone, PartialEq, Eq)] @@ -158,12 +56,32 @@ fn relative_to_trace(trace_dir: &Path, path: &Path) -> String { pub(crate) fn capture_action_screenshot( context: &CommandContext, adapter: &dyn PlatformAdapter, - pid: i32, + entry: &RefEntry, phase: &str, + deadline: crate::Deadline, ) -> ArtifactOutcome { if !artifacts_enabled(context) { return ArtifactOutcome::Skipped("disabled".into()); } + if deadline.is_expired() { + return ArtifactOutcome::Skipped("deadline".into()); + } + let Some(window_id) = entry + .source + .source_window_id + .as_deref() + .filter(|window_id| !window_id.is_empty()) + else { + return ArtifactOutcome::Skipped("exact_target_unavailable".into()); + }; + let Some(process_instance) = entry + .process + .process_instance + .as_deref() + .filter(|instance| !instance.is_empty()) + else { + return ArtifactOutcome::Skipped("exact_target_unavailable".into()); + }; let Some(trace_dir) = session_trace_dir(context) else { return ArtifactOutcome::Skipped("no_session".into()); }; @@ -172,22 +90,37 @@ pub(crate) fn capture_action_screenshot( return ArtifactOutcome::Skipped(format!("dir: {err}")); } - let buf = match adapter.screenshot(ScreenshotTarget::Window(pid)) { + let target = crate::WindowInfo { + id: window_id.into(), + title: entry.source.source_window_title.clone().unwrap_or_default(), + app: entry.source.source_app.clone().unwrap_or_default(), + pid: entry.process.pid, + process_instance: Some(process_instance.into()), + bounds: None, + state: crate::WindowState { + is_focused: false, + ..Default::default() + }, + }; + let buf = match adapter.screenshot(ScreenshotTarget::ExactWindow(target), deadline) { Ok(buf) => buf, Err(err) => { return ArtifactOutcome::Skipped(format!("adapter: {}", err.code.as_str())); } }; - let byte_len = buf.data.len() as u64; - if let Err(reason) = reserve_screenshot(byte_len) { - return ArtifactOutcome::Skipped(reason.into()); - } - let seq = CAPTURE_SEQ.fetch_add(1, Ordering::Relaxed); - let filename = format!("{}-{}-{}-{}.png", pid, process_start_ms(), seq, phase); + let filename = format!( + "{}-{}-{}-{}.png", + entry.process.pid, + process_start_ms(), + seq, + phase + ); let path = screens.join(&filename); - if write_private_file(&path, &buf.data).is_err() { - return ArtifactOutcome::Skipped("write_failed".into()); + if let Err(reason) = + crate::trace_artifact_budget::write_screenshot(&trace_dir, &path, &buf.data, deadline) + { + return ArtifactOutcome::Skipped(reason.into()); } ArtifactOutcome::Captured(relative_to_trace(&trace_dir, &path)) } @@ -208,9 +141,6 @@ pub(crate) fn copy_refmap_if_full( return Ok(()); } let dest = refmaps.join(format!("{snapshot_id}.json")); - if dest.is_file() { - return Ok(()); - } let json = match refmap.serialize_with_size_check() { Ok(json) => json, Err(err) => { @@ -218,15 +148,15 @@ pub(crate) fn copy_refmap_if_full( return Ok(()); } }; - let byte_len = json.len() as u64; - if reserve_refmap(byte_len).is_err() { + if crate::trace_artifact_budget::write_refmap_if_absent(&trace_dir, &dest, json.as_bytes()) + .is_err() + { let _ = context.trace_lazy( "action.artifacts.refmap_skipped", || json!({ "snapshot_id": snapshot_id }), ); return Ok(()); } - let _ = write_private_file(&dest, json.as_bytes()); Ok(()) } @@ -296,10 +226,7 @@ pub(crate) fn resolve_screenshot_path(trace_dir: &Path, relative: &str) -> Optio pub(crate) fn read_screenshot_for_embed(trace_dir: &Path, relative: &str) -> Option<Vec<u8>> { let path = resolve_screenshot_path(trace_dir, relative)?; - let mut file = open_nofollow(&path).ok()?; - let mut bytes = Vec::new(); - file.read_to_end(&mut bytes).ok()?; - Some(bytes) + crate::private_file::read_private_bounded(&path, MAX_EMBED_SCREENSHOT_BYTES).ok() } #[cfg(test)] @@ -309,3 +236,7 @@ mod tests; #[cfg(test)] #[path = "trace_artifacts_more_tests.rs"] mod more_tests; + +#[cfg(test)] +#[path = "trace_artifacts_toctou_tests.rs"] +mod toctou_tests; diff --git a/crates/core/src/trace_artifacts_more_tests.rs b/crates/core/src/trace_artifacts_more_tests.rs index a6f8904..18688f8 100644 --- a/crates/core/src/trace_artifacts_more_tests.rs +++ b/crates/core/src/trace_artifacts_more_tests.rs @@ -1,11 +1,79 @@ -use super::tests::{artifacts_session, entry, png_adapter, run_ref_action, setup_artifacts_test}; +use super::tests::{ + MINI_PNG, artifacts_session, deadline_png_adapter, entry, png_adapter, run_ref_action, + setup_artifacts_test, +}; use super::*; use crate::context::CommandContext; use crate::refs::RefMap; use crate::refs_store::RefStore; use crate::trace_artifacts::clear_test_budgets; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use crate::{ + Action, ActionRequest, ActionResult, AdapterError, ImageBuffer, + adapter::{ActionOps, InputOps, NativeHandle, ObservationOps, ScreenshotTarget, SystemOps}, +}; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; + +struct DeadlineConsumingScreenshotAdapter { + dispatch_calls: AtomicU32, + screenshot_calls: AtomicU32, + consume_pre_deadline: bool, +} + +impl ObservationOps for DeadlineConsumingScreenshotAdapter { + fn resolve_element_strict( + &self, + _entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) + } + + crate::adapter::complete_live_observation!("button", "Run", [crate::capability::CLICK]); +} + +impl ActionOps for DeadlineConsumingScreenshotAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<ActionResult, AdapterError> { + self.dispatch_calls.fetch_add(1, Ordering::SeqCst); + Ok(ActionResult::delivered_unverified("click")) + } +} + +impl InputOps for DeadlineConsumingScreenshotAdapter {} + +impl SystemOps for DeadlineConsumingScreenshotAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn screenshot( + &self, + _target: ScreenshotTarget, + deadline: crate::Deadline, + ) -> Result<ImageBuffer, AdapterError> { + if self.consume_pre_deadline && self.screenshot_calls.fetch_add(1, Ordering::SeqCst) == 0 { + let delay = if deadline.was_capped() { + std::time::Duration::from_millis(50) + } else { + deadline.remaining() + }; + std::thread::sleep(delay); + return Err(deadline.timeout_error()); + } + Ok(ImageBuffer { + data: MINI_PNG.to_vec(), + format: crate::ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + }) + } +} #[test] fn artifacts_full_captures_pre_and_post_pngs() { @@ -44,6 +112,83 @@ fn artifacts_full_captures_pre_and_post_pngs() { assert!(body.contains("screens/")); } +#[test] +fn artifact_capture_budget_supports_deadline_bound_screenshot_backends() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); + run_ref_action(&context, &deadline_png_adapter(250), 42).unwrap(); + let screens = RefStore::for_session(Some(&manifest.id)) + .unwrap() + .trace_dir() + .join("screens"); + + assert_eq!(std::fs::read_dir(screens).unwrap().count(), 2); +} + +#[test] +fn slow_pre_capture_preserves_dispatch_budget() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id), None, false).unwrap(); + let adapter = DeadlineConsumingScreenshotAdapter { + dispatch_calls: AtomicU32::new(0), + screenshot_calls: AtomicU32::new(0), + consume_pre_deadline: true, + }; + let target = entry(42); + + let result = crate::ref_action_wait::execute_with_auto_wait( + crate::ref_action_wait_context::RefActionWaitContext { + adapter: &adapter, + entry: &target, + ref_id: "@e1", + context: &context, + }, + ActionRequest::headless(Action::Click).with_timeout_ms(Some(500)), + crate::ref_action::dispatch_resolved, + ); + + assert!( + result.is_ok(), + "slow trace capture blocked dispatch: {result:?}" + ); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn contended_artifact_lock_preserves_dispatch_budget() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); + let trace_dir = RefStore::for_session(Some(&manifest.id)) + .unwrap() + .trace_dir(); + std::fs::create_dir_all(&trace_dir).unwrap(); + let _artifact_lock = + crate::refs_lock::RefStoreLock::acquire(&trace_dir.join(".artifact-budget.lock")).unwrap(); + let adapter = DeadlineConsumingScreenshotAdapter { + dispatch_calls: AtomicU32::new(0), + screenshot_calls: AtomicU32::new(0), + consume_pre_deadline: false, + }; + let target = entry(42); + + let result = crate::ref_action_wait::execute_with_auto_wait( + crate::ref_action_wait_context::RefActionWaitContext { + adapter: &adapter, + entry: &target, + ref_id: "@e1", + context: &context, + }, + ActionRequest::headless(Action::Click).with_timeout_ms(Some(250)), + crate::ref_action::dispatch_resolved, + ); + + assert!(result.is_ok(), "artifact lock blocked dispatch: {result:?}"); + assert_eq!(adapter.dispatch_calls.load(Ordering::SeqCst), 1); +} + #[test] fn byte_budget_exhaustion_skips_with_budget_reason() { let (_home, _lock) = setup_artifacts_test(); @@ -154,7 +299,7 @@ fn refmap_budget_skip_then_prune_leaves_prior_copy() { assert!(copied.is_file()); for i in 0..600 { let mut map = RefMap::new(); - map.allocate(entry(i)); + map.allocate(entry(i + 1)); let id = store.save_new_snapshot(&map).unwrap(); copy_refmap_if_full(&context, &store, &id, &map).unwrap(); } @@ -170,48 +315,6 @@ fn refmap_budget_skip_then_prune_leaves_prior_copy() { clear_test_budgets(); } -#[test] -fn reserve_atomic_bytes_never_overshoots_under_concurrency() { - let used = Arc::new(AtomicU64::new(0)); - let byte_len: u64 = 7; - let limit: u64 = byte_len * 10; - let mut handles = Vec::with_capacity(100); - for _ in 0..100 { - let used = used.clone(); - handles.push(std::thread::spawn(move || { - reserve_atomic_bytes(&used, limit, byte_len).is_ok() - })); - } - let successes = handles - .into_iter() - .map(|handle| handle.join().unwrap()) - .filter(|ok| *ok) - .count() as u64; - assert!(used.load(Ordering::Relaxed) <= limit); - assert!(successes * byte_len <= limit); - assert_eq!(successes, 10); -} - -#[test] -fn reserve_atomic_count_never_overshoots_under_concurrency() { - let used = Arc::new(AtomicU32::new(0)); - let limit: u32 = 10; - let mut handles = Vec::with_capacity(100); - for _ in 0..100 { - let used = used.clone(); - handles.push(std::thread::spawn(move || { - reserve_atomic_count(&used, limit).is_ok() - })); - } - let successes = handles - .into_iter() - .map(|handle| handle.join().unwrap()) - .filter(|ok| *ok) - .count() as u32; - assert!(used.load(Ordering::Relaxed) <= limit); - assert_eq!(successes, limit); -} - #[test] fn copy_refmap_internal_failure_is_best_effort() { let (_home, _lock) = setup_artifacts_test(); diff --git a/crates/core/src/trace_artifacts_outcome_tests.rs b/crates/core/src/trace_artifacts_outcome_tests.rs new file mode 100644 index 0000000..409ae56 --- /dev/null +++ b/crates/core/src/trace_artifacts_outcome_tests.rs @@ -0,0 +1,38 @@ +use super::*; + +#[test] +fn default_adapter_screenshot_skips_cleanly() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); + run_ref_action(&context, &DefaultScreenshotAdapter, 1).unwrap(); +} + +#[test] +fn failing_action_still_captures_post_screenshot() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); + let adapter = FailingActionAdapter { + screenshot_calls: AtomicU32::new(0), + }; + let err = run_ref_action(&context, &adapter, 1).unwrap_err(); + assert_eq!(err.code(), "INTERNAL"); + assert_eq!(adapter.screenshot_calls.load(Ordering::SeqCst), 2); +} + +#[test] +fn capture_targets_window_for_pid() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); + let adapter = png_adapter(); + run_ref_action(&context, &adapter, 99).unwrap(); + match adapter.target.lock().unwrap().take() { + Some(ScreenshotTarget::ExactWindow(window)) => { + assert_eq!(window.pid, 99); + assert_eq!(window.id, "w-99"); + } + _ => panic!("expected exact window screenshot target"), + } +} diff --git a/crates/core/src/trace_artifacts_tests.rs b/crates/core/src/trace_artifacts_tests.rs index 4ef6c42..a844e2a 100644 --- a/crates/core/src/trace_artifacts_tests.rs +++ b/crates/core/src/trace_artifacts_tests.rs @@ -1,20 +1,23 @@ use super::*; +use crate::AdapterError; use crate::action::Action; use crate::action_request::ActionRequest; use crate::action_result::ActionResult; -use crate::adapter::{ImageBuffer, ImageFormat, NativeHandle, PlatformAdapter, ScreenshotTarget}; +use crate::adapter::{ + ActionOps, InputOps, NativeHandle, ObservationOps, PlatformAdapter, ScreenshotTarget, SystemOps, +}; use crate::context::CommandContext; -use crate::error::AdapterError; use crate::ref_action::{ResolvedRefAction, execute_resolved}; use crate::refs_store::RefStore; use crate::refs_test_support::HomeGuard; use crate::session::{ArtifactsMode, SessionTraceMode, StartSessionOptions, start_session}; use crate::trace_artifacts::clear_test_budgets; +use crate::{ImageBuffer, ImageFormat}; use crate::{capability, refs::RefEntry}; use std::sync::Mutex; use std::sync::atomic::{AtomicU32, Ordering}; -const MINI_PNG: &[u8] = &[ +pub(super) const MINI_PNG: &[u8] = &[ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00, 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00, @@ -22,24 +25,45 @@ const MINI_PNG: &[u8] = &[ 0x42, 0x60, 0x82, ]; -pub(super) fn entry(pid: i32) -> RefEntry { +pub(super) fn entry(pid: u32) -> RefEntry { + let bounds = crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }; RefEntry { - pid, - role: "button".into(), - name: Some("Run".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: crate::adapter::SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), + process: crate::RefProcess { + pid: crate::ProcessId::new(pid), + process_instance: Some("test-instance".into()), + }, + identity: crate::RefEntryIdentity { + role: "button".into(), + name: Some("Run".into()), + value: None, + description: None, + native_id: None, + }, + geometry: crate::RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: crate::RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: crate::RefSource { + source_app: Some("FixtureApp".into()), + source_window_id: Some(format!("w-{pid}")), + source_window_title: Some("Fixture".into()), + source_window_bounds_hash: None, + source_surface: crate::adapter::SnapshotSurface::Window, + }, + scope: crate::RefScope { + root_ref: None, + path_is_absolute: false, + path: smallvec::SmallVec::new(), + }, } } @@ -54,28 +78,52 @@ pub(super) fn artifacts_session() -> crate::session::SessionManifest { pub(super) struct PngAdapter { target: Mutex<Option<ScreenshotTarget>>, + minimum_budget_ms: u64, } -impl PlatformAdapter for PngAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { +impl ObservationOps for PngAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for PngAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<ActionResult, AdapterError> { - Ok(ActionResult::new("ok")) + Ok(ActionResult::delivered_unverified("ok")) } +} - fn screenshot(&self, target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> { +impl InputOps for PngAdapter {} + +impl SystemOps for PngAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn screenshot( + &self, + target: ScreenshotTarget, + deadline: crate::Deadline, + ) -> Result<ImageBuffer, AdapterError> { *self.target.lock().unwrap() = Some(target); + if deadline.remaining_ms() < self.minimum_budget_ms { + return Err(deadline.timeout_error()); + } Ok(ImageBuffer { data: MINI_PNG.to_vec(), format: ImageFormat::Png, width: 1, height: 1, + scale_factor: 1.0, }) } } @@ -83,25 +131,52 @@ impl PlatformAdapter for PngAdapter { pub(super) fn png_adapter() -> PngAdapter { PngAdapter { target: Mutex::new(None), + minimum_budget_ms: 0, + } +} + +pub(super) fn deadline_png_adapter(minimum_budget_ms: u64) -> PngAdapter { + PngAdapter { + target: Mutex::new(None), + minimum_budget_ms, } } struct FailingScreenshotAdapter; -impl PlatformAdapter for FailingScreenshotAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { +impl ObservationOps for FailingScreenshotAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for FailingScreenshotAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<ActionResult, AdapterError> { - Ok(ActionResult::new("ok")) + Ok(ActionResult::delivered_unverified("ok")) } +} - fn screenshot(&self, _target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> { +impl InputOps for FailingScreenshotAdapter {} + +impl SystemOps for FailingScreenshotAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn screenshot( + &self, + _target: ScreenshotTarget, + _deadline: crate::Deadline, + ) -> Result<ImageBuffer, AdapterError> { Err(AdapterError::not_supported("screenshot")) } } @@ -110,61 +185,103 @@ struct FailingActionAdapter { screenshot_calls: AtomicU32, } -impl PlatformAdapter for FailingActionAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { +impl ObservationOps for FailingActionAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for FailingActionAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<ActionResult, AdapterError> { Err(AdapterError::internal("boom")) } +} - fn screenshot(&self, _target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> { +impl InputOps for FailingActionAdapter {} + +impl SystemOps for FailingActionAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn screenshot( + &self, + _target: ScreenshotTarget, + _deadline: crate::Deadline, + ) -> Result<ImageBuffer, AdapterError> { self.screenshot_calls.fetch_add(1, Ordering::SeqCst); Ok(ImageBuffer { data: MINI_PNG.to_vec(), format: ImageFormat::Png, width: 1, height: 1, + scale_factor: 1.0, }) } } struct DefaultScreenshotAdapter; -impl PlatformAdapter for DefaultScreenshotAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> { +impl ObservationOps for DefaultScreenshotAdapter { + fn resolve_element_strict( + &self, + _entry: &RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { Ok(NativeHandle::null()) } + crate::adapter::complete_live_observation!("button", "Run", [capability::CLICK]); +} + +impl ActionOps for DefaultScreenshotAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &crate::InteractionLease, ) -> Result<ActionResult, AdapterError> { - Ok(ActionResult::new("ok")) + Ok(ActionResult::delivered_unverified("ok")) } } +impl InputOps for DefaultScreenshotAdapter {} + +impl SystemOps for DefaultScreenshotAdapter { + crate::adapter::guarded_interaction_lease!(); +} + pub(super) fn run_ref_action( context: &CommandContext, adapter: &dyn PlatformAdapter, - pid: i32, -) -> Result<ActionResult, crate::error::AppError> { + pid: u32, +) -> Result<ActionResult, crate::AppError> { let entry = entry(pid); - execute_resolved( - ResolvedRefAction { + let deadline = crate::Deadline::standard()?; + let lease = adapter.acquire_interaction_lease(deadline)?; + let handle = NativeHandle::null(); + let target = crate::ref_action_context::RefActionContext::new( + crate::ref_action_wait_context::RefActionWaitContext { adapter, entry: &entry, - handle: &NativeHandle::null(), ref_id: "@e1", context, }, + deadline, + ); + execute_resolved( + ResolvedRefAction::new(target, &handle), ActionRequest::headless(Action::Click), + &lease, ) } @@ -270,36 +387,5 @@ fn symlinked_screens_dir_refuses_capture() { ); } -#[test] -fn default_adapter_screenshot_skips_cleanly() { - let (_home, _lock) = setup_artifacts_test(); - let manifest = artifacts_session(); - let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); - run_ref_action(&context, &DefaultScreenshotAdapter, 1).unwrap(); -} - -#[test] -fn failing_action_still_captures_post_screenshot() { - let (_home, _lock) = setup_artifacts_test(); - let manifest = artifacts_session(); - let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); - let adapter = FailingActionAdapter { - screenshot_calls: AtomicU32::new(0), - }; - let err = run_ref_action(&context, &adapter, 1).unwrap_err(); - assert_eq!(err.code(), "INTERNAL"); - assert_eq!(adapter.screenshot_calls.load(Ordering::SeqCst), 2); -} - -#[test] -fn capture_targets_window_for_pid() { - let (_home, _lock) = setup_artifacts_test(); - let manifest = artifacts_session(); - let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap(); - let adapter = png_adapter(); - run_ref_action(&context, &adapter, 99).unwrap(); - match adapter.target.lock().unwrap().take() { - Some(ScreenshotTarget::Window(pid)) => assert_eq!(pid, 99), - _ => panic!("expected Window screenshot target"), - } -} +#[path = "trace_artifacts_outcome_tests.rs"] +mod outcome_tests; diff --git a/crates/core/src/trace_artifacts_toctou_tests.rs b/crates/core/src/trace_artifacts_toctou_tests.rs new file mode 100644 index 0000000..fb02522 --- /dev/null +++ b/crates/core/src/trace_artifacts_toctou_tests.rs @@ -0,0 +1,199 @@ +use super::tests::{artifacts_session, entry, setup_artifacts_test}; +use crate::{ + AdapterError, ErrorCode, ImageBuffer, ImageFormat, + action::Action, + action_request::ActionRequest, + action_result::ActionResult, + adapter::{ + ActionOps, InputOps, LiveElement, NativeHandle, ObservationOps, ScreenshotTarget, SystemOps, + }, + context::CommandContext, + element_state::ElementState, + ref_action::{ResolvedRefAction, execute_resolved}, +}; +use std::sync::atomic::{AtomicU32, Ordering}; + +struct ScreenshotMutationAdapter { + live_reads: AtomicU32, + dispatches: AtomicU32, +} + +impl ObservationOps for ScreenshotMutationAdapter { + fn resolve_element_strict( + &self, + _entry: &crate::RefEntry, + _deadline: crate::Deadline, + ) -> Result<NativeHandle, AdapterError> { + Ok(NativeHandle::null()) + } + + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<Option<crate::Rect>, AdapterError> { + Ok(Some(crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + })) + } + + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: crate::Deadline, + ) -> Result<LiveElement, AdapterError> { + let read = self.live_reads.fetch_add(1, Ordering::SeqCst) + 1; + let states = if read == 1 { + Vec::new() + } else { + vec!["disabled".into()] + }; + Ok(LiveElement { + identity: crate::adapter::live_identity("Run"), + state: ElementState { + role: "button".into(), + states, + value: None, + enabled: Some(read == 1), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, + bounds: Some(crate::Rect { + x: 1.0, + y: 1.0, + width: 20.0, + height: 20.0, + }), + available_actions: vec![crate::capability::CLICK.into()], + }) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: crate::Point, + _deadline: crate::Deadline, + ) -> Result<crate::hit_test::HitTestResult, AdapterError> { + Ok(crate::hit_test::HitTestResult::ReachesTarget) + } +} + +impl ActionOps for ScreenshotMutationAdapter { + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + _lease: &crate::InteractionLease, + ) -> Result<ActionResult, AdapterError> { + self.dispatches.fetch_add(1, Ordering::SeqCst); + Ok(ActionResult::delivered_unverified("click")) + } +} + +impl InputOps for ScreenshotMutationAdapter {} + +impl SystemOps for ScreenshotMutationAdapter { + crate::adapter::guarded_interaction_lease!(); + + fn screenshot( + &self, + _target: ScreenshotTarget, + _deadline: crate::Deadline, + ) -> Result<ImageBuffer, AdapterError> { + Ok(ImageBuffer { + data: vec![1], + format: ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + }) + } +} + +#[test] +fn full_artifact_capture_is_followed_by_final_preflight() { + let (_home, _lock) = setup_artifacts_test(); + let manifest = artifacts_session(); + let context = CommandContext::new(Some(manifest.id), None, false).unwrap(); + let adapter = ScreenshotMutationAdapter { + live_reads: AtomicU32::new(0), + dispatches: AtomicU32::new(0), + }; + let entry = entry(1); + let deadline = crate::Deadline::standard().unwrap(); + let lease = adapter.acquire_interaction_lease(deadline).unwrap(); + let handle = NativeHandle::null(); + let target = crate::ref_action_context::RefActionContext::new( + crate::ref_action_wait_context::RefActionWaitContext { + adapter: &adapter, + entry: &entry, + ref_id: "@e1", + context: &context, + }, + deadline, + ); + let err = execute_resolved( + ResolvedRefAction::new(target, &handle), + ActionRequest::headless(Action::Click), + &lease, + ) + .unwrap_err(); + assert_eq!(err.code(), "ACTION_FAILED"); + assert_eq!(adapter.live_reads.load(Ordering::SeqCst), 2); + assert_eq!(adapter.dispatches.load(Ordering::SeqCst), 0); + let crate::AppError::Adapter(adapter_error) = err else { + panic!("expected adapter error"); + }; + assert_eq!(adapter_error.code, ErrorCode::ActionFailed); +} + +#[test] +fn oversized_screenshot_is_not_embedded() { + let directory = std::env::temp_dir().join(format!( + "agent-desktop-oversized-screen-{}", + crate::refs::new_snapshot_id() + )); + std::fs::create_dir_all(&directory).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let path = directory.join("screen.png"); + crate::refs::write_private_file(&path, b"png").unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(super::MAX_EMBED_SCREENSHOT_BYTES + 1) + .unwrap(); + + assert!(super::read_screenshot_for_embed(&directory, "screen.png").is_none()); + + std::fs::remove_dir_all(directory).unwrap(); +} + +#[cfg(unix)] +#[test] +fn screenshot_fifo_is_rejected_without_blocking() { + use std::ffi::CString; + + let directory = std::env::temp_dir().join(format!( + "agent-desktop-fifo-screen-{}", + crate::refs::new_snapshot_id() + )); + std::fs::create_dir_all(&directory).unwrap(); + let path = directory.join("screen.png"); + let c_path = CString::new(path.to_string_lossy().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + let started = std::time::Instant::now(); + + assert!(super::read_screenshot_for_embed(&directory, "screen.png").is_none()); + assert!(started.elapsed() < std::time::Duration::from_secs(1)); + + std::fs::remove_dir_all(directory).unwrap(); +} diff --git a/crates/core/src/trace_read/html.rs b/crates/core/src/trace_read/html.rs index 3a9d7bc..a8c41ac 100644 --- a/crates/core/src/trace_read/html.rs +++ b/crates/core/src/trace_read/html.rs @@ -1,5 +1,5 @@ use super::{ReadOptions, read_merged}; -use crate::error::AppError; +use crate::AppError; use crate::trace_artifacts::read_screenshot_for_embed; use base64::Engine; use serde_json::{Value, json}; diff --git a/crates/core/src/trace_read/html_tests.rs b/crates/core/src/trace_read/html_tests.rs index 1e9b4f7..aa21aeb 100644 --- a/crates/core/src/trace_read/html_tests.rs +++ b/crates/core/src/trace_read/html_tests.rs @@ -215,8 +215,8 @@ fn embed_budget_skips_later_screenshots() { let (_home, _lock, session_id, trace_dir) = setup_trace_session(); fs::create_dir_all(trace_dir.join("screens")).unwrap(); let big = vec![0u8; 60 * 1024 * 1024]; - fs::write(trace_dir.join("screens/a.png"), &big).unwrap(); - fs::write(trace_dir.join("screens/b.png"), &big).unwrap(); + write_screenshot_fixture(&trace_dir.join("screens/a.png"), &big); + write_screenshot_fixture(&trace_dir.join("screens/b.png"), &big); write_segment( &trace_dir, "100-1000.jsonl", @@ -336,7 +336,7 @@ fn embedded_screenshot_uses_base64_data_uri() { 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, ]; - fs::write(trace_dir.join("screens/shot.png"), png).unwrap(); + write_screenshot_fixture(&trace_dir.join("screens/shot.png"), &png); write_segment( &trace_dir, "100-1000.jsonl", @@ -354,3 +354,11 @@ fn embedded_screenshot_uses_base64_data_uri() { assert_eq!(stats.screenshots_embedded, 1); assert!(html.contains("data:image/png;base64,")); } +fn write_screenshot_fixture(path: &std::path::Path, bytes: &[u8]) { + fs::write(path, bytes).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap(); + } +} diff --git a/crates/core/src/trace_read/mod.rs b/crates/core/src/trace_read/mod.rs index b97b63b..eaa2cb1 100644 --- a/crates/core/src/trace_read/mod.rs +++ b/crates/core/src/trace_read/mod.rs @@ -4,7 +4,7 @@ mod segment; pub use html::{ExportOptions, ExportStats, TRACE_EXPORT_DEFAULT_LIMIT, export_html}; -use crate::error::AppError; +use crate::AppError; use merge::{ annotate_provenance, apply_tail_limit, detect_unpaired_commands, filter_by_event_prefix, merge_segments, @@ -88,6 +88,10 @@ pub fn read_merged(trace_dir: &Path, options: &ReadOptions) -> Result<MergedTrac continue; } + if is_managed_artifact_dir(&entry, name) { + continue; + } + let Some(parsed_name) = parse_segment_filename(name) else { if !name.starts_with('.') { warnings.push(TraceWarning { @@ -166,6 +170,13 @@ pub fn read_merged(trace_dir: &Path, options: &ReadOptions) -> Result<MergedTrac }) } +fn is_managed_artifact_dir(entry: &std::fs::DirEntry, name: &str) -> bool { + matches!(name, "screens" | "refmaps") + && entry + .file_type() + .is_ok_and(|file_type| file_type.is_dir() && !file_type.is_symlink()) +} + fn cap_list<T>(mut items: Vec<T>, cap: usize) -> (Vec<T>, bool) { if items.len() <= cap { return (items, false); diff --git a/crates/core/src/trace_read/mod_tests.rs b/crates/core/src/trace_read/mod_tests.rs index 4005db5..1ca7b17 100644 --- a/crates/core/src/trace_read/mod_tests.rs +++ b/crates/core/src/trace_read/mod_tests.rs @@ -51,6 +51,36 @@ fn foreign_file_produces_warning() { ); } +#[test] +fn managed_artifact_directories_do_not_produce_warnings() { + let dir = temp_dir("trace-mod-artifacts"); + fs::create_dir_all(dir.join("screens")).unwrap(); + fs::create_dir_all(dir.join("refmaps")).unwrap(); + + let result = read_merged(&dir, &ReadOptions::default()).unwrap(); + + assert!(result.warnings.is_empty()); +} + +#[cfg(unix)] +#[test] +fn symlinked_managed_artifact_name_produces_warning() { + let dir = temp_dir("trace-mod-artifact-link"); + let target = temp_dir("trace-mod-artifact-target"); + fs::create_dir_all(&dir).unwrap(); + fs::create_dir_all(&target).unwrap(); + std::os::unix::fs::symlink(&target, dir.join("screens")).unwrap(); + + let result = read_merged(&dir, &ReadOptions::default()).unwrap(); + + assert!( + result + .warnings + .iter() + .any(|warning| warning.kind == TraceWarningKind::ForeignFile) + ); +} + #[test] fn tmp_file_is_silently_ignored() { let dir = temp_dir("trace-mod-tmp"); diff --git a/crates/core/src/trace_read/segment.rs b/crates/core/src/trace_read/segment.rs index 79c9298..a951cfd 100644 --- a/crates/core/src/trace_read/segment.rs +++ b/crates/core/src/trace_read/segment.rs @@ -1,4 +1,4 @@ -use crate::error::AppError; +use crate::AppError; use serde_json::Value; use std::io::{BufRead, BufReader}; use std::path::Path; diff --git a/crates/core/src/trace_sanitize_tests.rs b/crates/core/src/trace_sanitize_tests.rs index db8aca7..c531cd1 100644 --- a/crates/core/src/trace_sanitize_tests.rs +++ b/crates/core/src/trace_sanitize_tests.rs @@ -56,3 +56,23 @@ fn trace_redaction_covers_nested_shapes_and_substring_keys() { assert!(value["action"]["password"].is_null()); assert_eq!(value["action"]["counter"], 3); } + +#[test] +fn trace_keeps_actionability_check_identifier_but_redacts_occluder_name() { + let value = sanitize_trace_value(json!({ + "checks": [ + { "check": "supported_action", "status": "fail", "reason": "Click is not available" }, + { + "check": "receives_events", + "status": "fail", + "occluder": { "role": "AXSheet", "name": "Save changes?" } + } + ] + })); + + assert_eq!(value["checks"][0]["check"], "supported_action"); + assert_eq!(value["checks"][0]["reason"], "Click is not available"); + assert_eq!(value["checks"][1]["check"], "receives_events"); + assert_eq!(value["checks"][1]["occluder"]["role"], "AXSheet"); + assert_eq!(value["checks"][1]["occluder"]["name"]["redacted"], true); +} diff --git a/crates/core/src/trace_state.rs b/crates/core/src/trace_state.rs new file mode 100644 index 0000000..1108b99 --- /dev/null +++ b/crates/core/src/trace_state.rs @@ -0,0 +1,27 @@ +use std::{ + path::PathBuf, + sync::{Arc, Mutex, atomic::AtomicBool}, +}; + +#[derive(Debug, Clone, Default)] +pub(crate) enum TracePending { + #[default] + None, + File(PathBuf), + SegmentDir(PathBuf), +} + +#[derive(Debug, Clone, Default)] +pub(crate) enum TraceWriterState { + #[default] + Unopened, + Open(Arc<Mutex<std::fs::File>>), + Failed, +} + +#[derive(Debug, Default)] +pub(crate) struct TraceState { + pub(crate) pending: TracePending, + pub(crate) writer: Arc<Mutex<TraceWriterState>>, + pub(crate) meta_written: AtomicBool, +} diff --git a/crates/core/src/trace_tests.rs b/crates/core/src/trace_tests.rs index d0fed1b..3afcc74 100644 --- a/crates/core/src/trace_tests.rs +++ b/crates/core/src/trace_tests.rs @@ -286,3 +286,101 @@ fn failed_writer_reports_no_sink() { "a trace writer whose open failed must not report an active sink" ); } + +#[test] +fn cross_process_writer_helper() { + let Some(path) = std::env::var_os("AGENT_DESKTOP_TRACE_HELPER_PATH") else { + return; + }; + let config = TraceConfig::build(Some(path.into()), None, true).unwrap(); + for index in 0..25 { + config + .emit("child.event", None, json!({ "index": index })) + .unwrap(); + } +} + +#[test] +fn explicit_trace_appends_are_cross_process_atomic_with_one_meta() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-cross-process-trace-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let mut children = Vec::new(); + for _ in 0..2 { + children.push( + std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("trace::tests::cross_process_writer_helper") + .arg("--nocapture") + .env("AGENT_DESKTOP_TRACE_HELPER_PATH", &path) + .spawn() + .unwrap(), + ); + } + for child in &mut children { + assert!(child.wait().unwrap().success()); + } + + let body = fs::read_to_string(&path).unwrap(); + let records = body + .lines() + .map(serde_json::from_str::<serde_json::Value>) + .collect::<Result<Vec<_>, _>>() + .unwrap(); + assert_eq!(records.len(), 51); + assert_eq!( + records + .iter() + .filter(|record| record["event"] == "trace.meta") + .count(), + 1 + ); + assert_eq!( + records + .iter() + .filter(|record| record["event"] == "child.event") + .count(), + 50 + ); + let child_records = records + .iter() + .filter(|record| record["event"] == "child.event") + .collect::<Vec<_>>(); + assert!(child_records.iter().all(|record| { + record["writer_pid"].as_u64().is_some() && record["writer_proc_start_ms"].as_u64().is_some() + })); + let identities = child_records + .iter() + .map(|record| { + ( + record["writer_pid"].as_u64().unwrap(), + record["writer_proc_start_ms"].as_u64().unwrap(), + record["seq"].as_u64().unwrap(), + ) + }) + .collect::<std::collections::HashSet<_>>(); + assert_eq!(identities.len(), 50); + fs::remove_file(path).unwrap(); +} + +#[test] +fn oversized_event_is_rejected_before_line_allocation() { + let path = std::env::temp_dir().join(format!( + "agent-desktop-oversized-event-{}.jsonl", + crate::refs::new_snapshot_id() + )); + let config = TraceConfig::build(Some(path.clone()), None, true).unwrap(); + + let error = config + .emit( + "event", + None, + json!({ "value": "x".repeat(MAX_TRACE_EVENT_BYTES) }), + ) + .unwrap_err(); + + assert_eq!(error.code(), "INVALID_ARGS"); + assert!(fs::metadata(&path).unwrap().len() < MAX_TRACE_EVENT_BYTES as u64); + fs::remove_file(path).unwrap(); +} diff --git a/crates/core/src/tree_options.rs b/crates/core/src/tree_options.rs new file mode 100644 index 0000000..cc46665 --- /dev/null +++ b/crates/core/src/tree_options.rs @@ -0,0 +1,31 @@ +use crate::snapshot_surface::SnapshotSurface; + +#[derive(Clone, Copy)] +pub struct TreeOptions { + pub max_depth: u8, + pub include_bounds: bool, + pub interactive_only: bool, + pub compact: bool, + pub surface: SnapshotSurface, + pub skeleton: bool, +} + +impl Default for TreeOptions { + fn default() -> Self { + Self { + max_depth: 10, + include_bounds: false, + interactive_only: false, + compact: false, + surface: SnapshotSurface::Window, + skeleton: false, + } + } +} + +impl TreeOptions { + pub(crate) fn with_ref_identity_bounds(mut self) -> Self { + self.include_bounds = true; + self + } +} diff --git a/crates/core/src/ui_event.rs b/crates/core/src/ui_event.rs new file mode 100644 index 0000000..043a9c0 --- /dev/null +++ b/crates/core/src/ui_event.rs @@ -0,0 +1,16 @@ +use crate::{EventKind, ProcessId}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct UiEvent { + #[serde(flatten)] + pub kind: EventKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub window_id: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub app: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub pid: Option<ProcessId>, +} diff --git a/crates/core/src/wait_budget.rs b/crates/core/src/wait_budget.rs new file mode 100644 index 0000000..2e112f5 --- /dev/null +++ b/crates/core/src/wait_budget.rs @@ -0,0 +1,28 @@ +use std::time::Duration; + +use serde_json::json; + +use crate::{AdapterError, ErrorCode}; + +pub const MAX_WAIT_TIMEOUT_MS: u64 = 24 * 60 * 60 * 1000; + +pub fn wait_timeout_duration(timeout_ms: u64) -> Result<Duration, AdapterError> { + if timeout_ms > MAX_WAIT_TIMEOUT_MS { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Timeout exceeds the {MAX_WAIT_TIMEOUT_MS}ms maximum"), + ) + .with_suggestion(format!( + "Choose a timeout between 0 and {MAX_WAIT_TIMEOUT_MS} milliseconds" + )) + .with_details(json!({ + "timeout_ms": timeout_ms, + "max_timeout_ms": MAX_WAIT_TIMEOUT_MS, + }))); + } + Ok(Duration::from_millis(timeout_ms)) +} + +#[cfg(test)] +#[path = "wait_budget_tests.rs"] +mod tests; diff --git a/crates/core/src/wait_budget_tests.rs b/crates/core/src/wait_budget_tests.rs new file mode 100644 index 0000000..4bf1b68 --- /dev/null +++ b/crates/core/src/wait_budget_tests.rs @@ -0,0 +1,24 @@ +use std::time::Instant; + +use crate::{Deadline, ErrorCode, MAX_WAIT_TIMEOUT_MS, wait_timeout_duration}; + +#[test] +fn maximum_timeout_is_accepted() { + assert_eq!( + wait_timeout_duration(MAX_WAIT_TIMEOUT_MS) + .unwrap() + .as_millis(), + u128::from(MAX_WAIT_TIMEOUT_MS) + ); +} + +#[test] +fn oversized_timeout_is_rejected_before_deadline_math() { + let error = wait_timeout_duration(u64::MAX).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert_eq!(error.details.unwrap()["timeout_ms"], u64::MAX); + assert_eq!( + Deadline::at(Instant::now(), u64::MAX).unwrap_err().code, + ErrorCode::InvalidArgs + ); +} diff --git a/crates/core/src/window_filter.rs b/crates/core/src/window_filter.rs new file mode 100644 index 0000000..75a6b6d --- /dev/null +++ b/crates/core/src/window_filter.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Clone, Default)] +pub struct WindowFilter { + pub focused_only: bool, + pub app: Option<String>, +} diff --git a/crates/core/src/window_focus.rs b/crates/core/src/window_focus.rs new file mode 100644 index 0000000..8df5b89 --- /dev/null +++ b/crates/core/src/window_focus.rs @@ -0,0 +1,80 @@ +use std::time::Duration; + +use crate::{AdapterError, AppError, ErrorCode, PlatformAdapter, WindowFilter, WindowInfo}; + +const FOCUS_SETTLE_TIMEOUT_MS: u64 = 750; +const FOCUS_POLL_INTERVAL_MS: u64 = 50; +const FOCUS_CONFIRMATIONS: u8 = 2; + +pub(crate) fn focus_and_confirm( + adapter: &dyn PlatformAdapter, + window: &WindowInfo, + lease: &crate::InteractionLease, +) -> Result<WindowInfo, AppError> { + adapter.focus_window(window, lease)?; + wait_for_focused_window_with_poll_interval( + adapter, + &window.id, + Some(&window.app), + Duration::from_millis(FOCUS_POLL_INTERVAL_MS), + lease.deadline(), + ) +} + +pub(crate) fn wait_for_focused_window_with_poll_interval( + adapter: &dyn PlatformAdapter, + window_id: &str, + app: Option<&str>, + poll_interval: Duration, + parent_deadline: crate::Deadline, +) -> Result<WindowInfo, AppError> { + let deadline = parent_deadline.capped(Duration::from_millis(FOCUS_SETTLE_TIMEOUT_MS)); + let mut confirmations = 0; + loop { + match observed_focused_window(adapter, app, deadline) { + Ok(Some(window)) if window.id == window_id => { + confirmations += 1; + if confirmations >= FOCUS_CONFIRMATIONS { + return Ok(window); + } + } + Ok(_) => confirmations = 0, + Err(AppError::Adapter(error)) if error.permits_retry_by_default() => { + confirmations = 0; + } + Err(error) => return Err(error), + } + if deadline.is_expired() { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "Window focus did not settle on the requested window", + ) + .with_suggestion("Run 'list-windows' to refresh window IDs, then retry.") + .into()); + } + if !poll_interval.is_zero() { + std::thread::sleep(poll_interval.min(deadline.remaining())); + } + } +} + +fn observed_focused_window( + adapter: &dyn PlatformAdapter, + app: Option<&str>, + deadline: crate::Deadline, +) -> Result<Option<WindowInfo>, AppError> { + match adapter.focused_window(deadline) { + Ok(window) => Ok(window), + Err(err) if err.code == ErrorCode::PlatformNotSupported => adapter + .list_windows( + &WindowFilter { + focused_only: true, + app: app.map(str::to_string), + }, + deadline, + ) + .map(|windows| windows.into_iter().next()) + .map_err(AppError::Adapter), + Err(err) => Err(AppError::Adapter(err)), + } +} diff --git a/crates/core/src/window_info.rs b/crates/core/src/window_info.rs new file mode 100644 index 0000000..aafc54c --- /dev/null +++ b/crates/core/src/window_info.rs @@ -0,0 +1,22 @@ +use serde::{Deserialize, Serialize}; + +use crate::{ProcessId, Rect, WindowState}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WindowInfo { + pub id: String, + pub title: String, + #[serde(rename = "app_name")] + pub app: String, + pub pid: ProcessId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub process_instance: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub bounds: Option<Rect>, + #[serde(flatten)] + pub state: WindowState, +} + +#[cfg(test)] +#[path = "window_info_tests.rs"] +mod tests; diff --git a/crates/core/src/window_info_tests.rs b/crates/core/src/window_info_tests.rs new file mode 100644 index 0000000..ab10e00 --- /dev/null +++ b/crates/core/src/window_info_tests.rs @@ -0,0 +1,40 @@ +use super::*; + +#[test] +fn legacy_window_json_defaults_optional_state() { + let window: WindowInfo = serde_json::from_value(serde_json::json!({ + "id": "w-1", + "title": "Document", + "app_name": "Editor", + "pid": 42, + "is_focused": true + })) + .expect("legacy window"); + + assert!(window.state.is_focused); + assert_eq!(window.state.minimized, None); + assert_eq!(window.state.visible, None); +} + +#[test] +fn optional_window_state_serializes_flat() { + let window = WindowInfo { + id: "w-1".into(), + title: "Document".into(), + app: "Editor".into(), + pid: crate::ProcessId::new(42), + process_instance: Some("42:1".into()), + bounds: None, + state: WindowState { + is_focused: false, + minimized: Some(true), + visible: Some(false), + }, + }; + let value = serde_json::to_value(window).expect("window json"); + + assert_eq!(value["is_focused"], false); + assert_eq!(value["minimized"], true); + assert_eq!(value["visible"], false); + assert!(value.get("state").is_none()); +} diff --git a/crates/core/src/window_lookup.rs b/crates/core/src/window_lookup.rs index e24a0d7..df92617 100644 --- a/crates/core/src/window_lookup.rs +++ b/crates/core/src/window_lookup.rs @@ -1,27 +1,105 @@ use crate::{ + AppError, ErrorCode, ProcessIdentity, WindowInfo, adapter::{PlatformAdapter, WindowFilter}, - error::AppError, - node::WindowInfo, }; +use serde_json::json; -pub(crate) fn find_window_for_pid( - pid: i32, +pub(crate) fn find_window_for_process( + process: ProcessIdentity, adapter: &dyn PlatformAdapter, + deadline: crate::Deadline, ) -> Result<WindowInfo, AppError> { let filter = WindowFilter { focused_only: false, app: None, }; - let windows = adapter.list_windows(&filter)?; - let mut candidates: Vec<_> = windows.into_iter().filter(|w| w.pid == pid).collect(); - if candidates.is_empty() { - return Err(AppError::invalid_input( - "No window found for this application", - )); - } - let selected = candidates + let same_pid: Vec<_> = adapter + .list_windows(&filter, deadline)? + .into_iter() + .filter(|window| window.pid == process.pid) + .collect(); + let incomplete_count = same_pid .iter() - .position(|w| w.is_focused) - .unwrap_or_default(); - Ok(candidates.swap_remove(selected)) + .filter(|window| window.process_instance.as_deref().is_none_or(str::is_empty)) + .count(); + if incomplete_count > 0 { + return Err(crate::AdapterError::new( + ErrorCode::ActionNotSupported, + "Window inventory has incomplete process identity", + ) + .with_details(json!({ + "candidate_count": same_pid.len(), + "incomplete_identity_count": incomplete_count, + })) + .into()); + } + let candidates: Vec<_> = same_pid + .into_iter() + .filter(|window| window.process_instance.as_deref() == Some(process.instance.as_str())) + .collect(); + select_window( + candidates, + crate::AdapterError::new( + ErrorCode::WindowNotFound, + "No window found for the selected process instance", + ), + "Multiple windows matched the selected process instance", + ) } + +pub(crate) fn select_window( + mut candidates: Vec<WindowInfo>, + empty_error: crate::AdapterError, + ambiguous_message: &str, +) -> Result<WindowInfo, AppError> { + if candidates.is_empty() { + return Err(empty_error.into()); + } + if candidates.len() == 1 { + return Ok(candidates.swap_remove(0)); + } + if candidates + .iter() + .any(|window| window.state.visible == Some(true)) + { + candidates.retain(|window| window.state.visible == Some(true)); + if candidates.len() == 1 { + return Ok(candidates.swap_remove(0)); + } + } + let focused = candidates + .iter() + .position(|window| window.state.is_focused) + .filter(|first| { + !candidates[*first + 1..] + .iter() + .any(|window| window.state.is_focused) + }); + if let Some(index) = focused { + return Ok(candidates.swap_remove(index)); + } + let summaries = candidates + .iter() + .take(10) + .map(|window| { + json!({ + "id": window.id, + "title": window.title, + "is_focused": window.state.is_focused, + "visible": window.state.visible, + }) + }) + .collect::<Vec<_>>(); + Err(crate::AdapterError::ambiguous_target(ambiguous_message) + .with_suggestion("Run 'list-windows' and retry with --window-id <id>.") + .with_details(json!({ + "candidate_count": candidates.len(), + "candidate_summaries_truncated": candidates.len() > summaries.len(), + "candidates": summaries, + })) + .into()) +} + +#[cfg(test)] +#[path = "window_lookup_tests.rs"] +mod tests; diff --git a/crates/core/src/window_lookup_tests.rs b/crates/core/src/window_lookup_tests.rs new file mode 100644 index 0000000..5c1ee13 --- /dev/null +++ b/crates/core/src/window_lookup_tests.rs @@ -0,0 +1,94 @@ +use super::*; +use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps}; + +struct WindowInventory(Vec<WindowInfo>); + +impl ObservationOps for WindowInventory { + fn list_windows( + &self, + _filter: &WindowFilter, + _deadline: crate::Deadline, + ) -> Result<Vec<WindowInfo>, crate::AdapterError> { + Ok(self.0.clone()) + } +} + +impl ActionOps for WindowInventory {} +impl InputOps for WindowInventory {} +impl SystemOps for WindowInventory {} + +fn window(instance: Option<&str>) -> WindowInfo { + WindowInfo { + id: "w-1".into(), + title: "Example".into(), + app: "Example".into(), + pid: crate::ProcessId::new(42), + process_instance: instance.map(str::to_string), + bounds: None, + state: crate::WindowState { + is_focused: true, + ..Default::default() + }, + } +} + +#[test] +fn matching_pid_with_unknown_generation_fails_closed() { + let adapter = WindowInventory(vec![window(Some("generation-a")), window(None)]); + + let error = find_window_for_process( + ProcessIdentity::new(42, "generation-a"), + &adapter, + crate::Deadline::standard().unwrap(), + ) + .unwrap_err(); + + assert_eq!(error.code(), "ACTION_NOT_SUPPORTED"); +} + +#[test] +fn shared_window_selector_emits_one_stable_ambiguity_shape() { + let mut first = window(Some("generation-a")); + first.state.is_focused = false; + let mut second = first.clone(); + second.id = "w-2".into(); + second.state.is_focused = false; + let error = select_window( + vec![first, second], + crate::AdapterError::new(ErrorCode::WindowNotFound, "none"), + "multiple", + ) + .unwrap_err(); + + let AppError::Adapter(error) = error else { + panic!("expected adapter error"); + }; + let details = error.details.as_ref().unwrap(); + assert_eq!(details["candidate_count"], 2); + assert_eq!(details["candidate_summaries_truncated"], false); + assert_eq!(details["candidates"][0]["id"], "w-1"); + assert_eq!(details["candidates"][0]["is_focused"], false); + assert!(details["candidates"][0].get("focused").is_none()); + assert!(details["candidates"][0].get("pid").is_none()); + assert!(details["candidates"][0].get("process_instance").is_none()); + assert!(error.suggestion.as_deref().unwrap().contains("--window-id")); +} + +#[test] +fn shared_window_selector_prefers_the_only_visible_window() { + let mut hidden = window(Some("generation-a")); + hidden.state.is_focused = false; + hidden.state.visible = Some(false); + let mut visible = hidden.clone(); + visible.id = "w-2".into(); + visible.state.visible = Some(true); + + let selected = select_window( + vec![hidden, visible], + crate::AdapterError::new(ErrorCode::WindowNotFound, "none"), + "multiple", + ) + .unwrap(); + + assert_eq!(selected.id, "w-2"); +} diff --git a/crates/core/src/window_op.rs b/crates/core/src/window_op.rs new file mode 100644 index 0000000..43fe79d --- /dev/null +++ b/crates/core/src/window_op.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum WindowOp { + Resize { width: f64, height: f64 }, + Move { x: f64, y: f64 }, + Minimize, + Maximize, + Restore, +} diff --git a/crates/core/src/window_state.rs b/crates/core/src/window_state.rs new file mode 100644 index 0000000..4bd40b8 --- /dev/null +++ b/crates/core/src/window_state.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct WindowState { + pub is_focused: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub minimized: Option<bool>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub visible: Option<bool>, +} diff --git a/crates/ffi/Cargo.toml b/crates/ffi/Cargo.toml index a615e00..3d09b28 100644 --- a/crates/ffi/Cargo.toml +++ b/crates/ffi/Cargo.toml @@ -7,6 +7,7 @@ publish = false [features] stub-adapter = [] +panic-injection = [] [lib] crate-type = ["cdylib", "rlib"] diff --git a/crates/ffi/README.md b/crates/ffi/README.md index 51d6fe9..cb8fbca 100644 --- a/crates/ffi/README.md +++ b/crates/ffi/README.md @@ -1,6 +1,6 @@ # agent-desktop-ffi -C-ABI cdylib over [`PlatformAdapter`](../../crates/core/src/adapter.rs). Exposes +C-ABI cdylib over [`PlatformAdapter`](../../crates/core/src/adapter/mod.rs). Exposes `libagent_desktop_ffi.{dylib,so,dll}` to Python (ctypes), Swift, Go (cgo), Node (ffi-napi), and C++ consumers without spawning a CLI subprocess per call. @@ -41,23 +41,30 @@ The full declaration list is in [`include/agent_desktop.h`](include/agent_deskto Consumers must call `ad_init(AD_ABI_VERSION_MAJOR)` after `dlopen` to verify the loaded dylib matches the header the consumer was built against. -## Codegen +Use additive versioned `AdExactRefEntry`, `AdExactWindowInfo`, and +`AdExactSurfaceInfo` APIs when round-tripping observed identities. Legacy ABI-v3 +layouts remain binary-compatible but cannot express process generation or a +surface ID; legacy direct window/ref targeting therefore fails closed. -The 5 command-backed JSON entrypoints (`ad_snapshot`, `ad_execute_by_ref`, -`ad_wait`, `ad_version`, `ad_status`) are **generated by `build.rs`** from the -template files in [`codegen_templates/`](codegen_templates/) into -`src/commands/generated.rs`. Do not hand-edit `generated.rs`; edit the -corresponding `.rs.in` template instead. The build panics if a template file -exists in `codegen_templates/` but is not registered in `build.rs`, so stale -templates cannot silently accumulate. +## Command wrappers + +Each command-backed JSON entrypoint has one canonical module under +`src/commands/`. The build script only configures the dylib install name and +never rewrites source files. ## Thread Safety (macOS) -Every adapter-touching entry point must be called from the process's **main -thread**. The guard fires at runtime in all build profiles; a worker-thread call -returns `AD_RESULT_ERR_INTERNAL` with a `'static` diagnostic string. Operations -safe from any thread: `ad_abi_version`, `ad_init`, adapter create/destroy, -`ad_last_error_*` readers, and all accessor / free helpers. +Adapter entrypoints may run on any host thread. AX and CG calls have no blanket +main-thread restriction, and Apple explicitly documents +[`NSWorkspace.shared`](https://developer.apple.com/documentation/appkit/nsworkspace/shared) +as safe from any thread. Cocoa-backed paths establish per-thread autorelease +pools; FFI initialization starts one `NSThread` once so Cocoa enables the locks +Apple documents for POSIX-threaded hosts. + +Native handles are stricter: resolve, use, and free each handle on the same +thread with the same adapter. Mutations are serialized across processes by the +canonical interaction lease. See the FFI skill's threading reference for the +full evidence and ordering contract. ## Error Model diff --git a/crates/ffi/build.rs b/crates/ffi/build.rs index a5fb811..fab710f 100644 --- a/crates/ffi/build.rs +++ b/crates/ffi/build.rs @@ -1,147 +1,5 @@ -//! Cargo's default install_name is the absolute build-machine path, which -//! breaks linking consumers (Swift/SPM, clang) once the dylib is extracted -//! from a release tarball. @rpath is the portable form. -//! -//! # panic=abort guard (Finding #14) -//! -//! `trap_panic` (the `catch_unwind` fence in `src/ffi.rs`) requires -//! `panic = "unwind"` to function; building with `panic = "abort"` silently -//! degrades hostile-input panics to process abort. The `release-ffi` profile -//! enforces `panic = "unwind"` and is the only profile used to ship the cdylib. -//! -//! A build-time check via `CARGO_CFG_PANIC` is not viable here: because this -//! crate declares both `cdylib` and `rlib` crate-types, Cargo always reports -//! `CARGO_CFG_PANIC = "unwind"` in the build script regardless of the active -//! profile's `panic` setting. The invariant is therefore documented rather -//! than machine-enforced; CI gates the cdylib via `--profile release-ffi` only. -//! -//! # FFI command codegen -//! -//! Family-B wrappers (command-backed, JSON-returning) are generated from -//! per-command template files under `codegen_templates/`. Each file is named -//! `<command>.rs.in` and loaded via `include_str!` in `command_templates()`. -//! The template set is the single source of truth for the command universe. -//! -//! Adding a new Family-B command requires exactly these steps: -//! 1. Create `codegen_templates/<name>.rs.in` with the wrapper body. -//! 2. Add `m.insert("<name>", include_str!("codegen_templates/<name>.rs.in"))` to -//! `command_templates()` in this file. -//! 3. Add `"<name>"` to `EXPECTED_COMMANDS` in -//! `tests/codegen_exhaustiveness.rs`. -//! -//! The build fails at step 3 if steps 1 and 2 are done without step 3. -//! An orphan `.rs.in` file (present in the dir but not in the map) triggers -//! a build-time panic so stale templates cannot silently accumulate. - -use std::collections::BTreeMap; -use std::path::PathBuf; - -const GENERATED_HEADER_STATIC: &str = "\ -//! @generated — produced by crates/ffi/build.rs codegen. -//! Edit the templates under crates/ffi/codegen_templates/, not this file. -"; - -/// Map from command name to its wrapper template (alphabetical by convention). -/// -/// This map is the single source of truth for which Family-B commands exist. -/// Every `.rs.in` file in `codegen_templates/` must appear here; the build -/// panics otherwise. -fn command_templates() -> BTreeMap<&'static str, &'static str> { - let mut m = BTreeMap::new(); - m.insert( - "execute_by_ref", - include_str!("codegen_templates/execute_by_ref.rs.in"), - ); - m.insert("snapshot", include_str!("codegen_templates/snapshot.rs.in")); - m.insert("status", include_str!("codegen_templates/status.rs.in")); - m.insert( - "trace_export", - include_str!("codegen_templates/trace_export.rs.in"), - ); - m.insert( - "trace_show", - include_str!("codegen_templates/trace_show.rs.in"), - ); - m.insert("version", include_str!("codegen_templates/version.rs.in")); - m.insert("wait", include_str!("codegen_templates/wait.rs.in")); - m -} - fn main() { if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { println!("cargo:rustc-cdylib-link-arg=-Wl,-install_name,@rpath/libagent_desktop_ffi.dylib"); } - - let manifest_dir = - PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")); - - let templates_dir = manifest_dir.join("codegen_templates"); - let generated_path = manifest_dir.join("src/commands/generated.rs"); - - let templates = command_templates(); - - for entry in std::fs::read_dir(&templates_dir) - .unwrap_or_else(|e| panic!("failed to read {}: {e}", templates_dir.display())) - { - let entry = entry.unwrap_or_else(|e| panic!("dir entry error in codegen_templates: {e}")); - let fname = entry.file_name(); - let fname_str = fname.to_string_lossy(); - if let Some(name) = fname_str.strip_suffix(".rs.in") { - if !templates.contains_key(name) { - panic!( - "orphan template file `{fname_str}` found in codegen_templates/ but not \ - registered in `command_templates()` in build.rs — add an entry or delete \ - the file." - ); - } - } - } - - for name in templates.keys() { - println!( - "cargo:rerun-if-changed={}", - templates_dir.join(format!("{name}.rs.in")).display() - ); - } - - let command_list = templates.keys().copied().collect::<Vec<_>>().join(", "); - let dynamic_comment = format!("//! Commands in alphabetical order: {command_list}.\n"); - - let mut output = String::from(GENERATED_HEADER_STATIC); - output.push_str(&dynamic_comment); - - let use_block = "\ -\nuse crate::AdAdapter;\ -\nuse crate::actions::conversion::action_from_c;\ -\nuse crate::commands::app_error_to_adapter;\ -\nuse crate::commands::envelope_out::write_command_envelope;\ -\nuse crate::convert::string::{\ -\n decode_optional_filter, optional_adapter_string, required_adapter_string,\ -\n};\ -\nuse crate::convert::surface::snapshot_surface_from_c;\ -\nuse crate::error::{self, AdResult, set_last_error};\ -\nuse crate::ffi_try::trap_panic;\ -\nuse crate::main_thread::require_main_thread;\ -\nuse crate::pointer_guard::guard_non_null;\ -\nuse crate::types::wait_args::AdWaitArgs;\ -\nuse crate::types::{AdAction, AdPolicyKind};\ -\nuse agent_desktop_core::commands::snapshot::SnapshotArgs;\ -\nuse agent_desktop_core::commands::status::execute_with_report_with_context;\ -\nuse agent_desktop_core::commands::wait::{WaitArgs, WaitModeArgs, WaitPredicateArgs};\ -\nuse agent_desktop_core::error::{AdapterError, AppError, ErrorCode};\ -\nuse agent_desktop_core::refs::validate_ref_id;\ -\nuse std::ffi::c_char;\ -\nuse std::ptr;\n"; - - output.push_str(use_block); - - for name in templates.keys() { - output.push_str(templates[name]); - } - - let existing = std::fs::read_to_string(&generated_path).unwrap_or_default(); - if existing != output { - std::fs::write(&generated_path, &output) - .unwrap_or_else(|e| panic!("failed to write {}: {e}", generated_path.display())); - } } diff --git a/crates/ffi/cbindgen.toml b/crates/ffi/cbindgen.toml index a8a3f94..9994058 100644 --- a/crates/ffi/cbindgen.toml +++ b/crates/ffi/cbindgen.toml @@ -1,4 +1,5 @@ language = "C" +cpp_compat = true include_guard = "AGENT_DESKTOP_H" no_includes = true sys_includes = ["stdint.h", "stdbool.h", "stddef.h"] @@ -20,14 +21,14 @@ after_includes = """ * AdAdapter *a = ad_adapter_create_with_session(session_id); // with session * * 3. Observe via ad_snapshot(). The returned JSON envelope contains the - * accessibility tree with @e-prefixed ref IDs (e.g. "@e5") that address + * accessibility tree with snapshot-qualified ref IDs (for example, + * "@s8f3k2p9:e5") that address * individual interactive elements. A refmap is written under * ~/.agent-desktop/ and is keyed to the session. The envelope carries - * data.snapshot_id — pass it back as the snapshot_id argument to - * ad_execute_by_ref to pin the action to that exact snapshot; pass NULL - * to target the latest snapshot for the session. + * data.snapshot_id. Qualified refs already pin the exact snapshot; legacy + * bare @eN refs require that ID as the snapshot_id argument. * - * 4. Act via ad_execute_by_ref(a, "@e5", snapshot_id, &action, policy, &out). + * 4. Act via ad_execute_by_ref(a, "@s8f3k2p9:e5", NULL, &action, policy, &out). * Build an AdAction by zero-initialising it and setting its kind field to * an AD_ACTION_KIND_* constant plus any kind-specific fields (e.g. .text * for AD_ACTION_KIND_TYPE_TEXT). policy=0 (Headless) keeps each action's @@ -38,8 +39,9 @@ after_includes = """ * 5. Ownership: every non-null *out string must be freed with ad_free_string(). * Destroy the adapter when done with ad_adapter_destroy(a). * - * macOS: all ad_* calls that interact with the AX API must be made from the - * main thread. ad_snapshot and ad_execute_by_ref enforce this with a guard. + * Calls may originate on any host thread. Native element handles remain + * thread-affine and must be used and released on the thread that resolved + * them. Desktop mutations are serialized by an interaction lease. */ """ @@ -68,6 +70,10 @@ _Static_assert(sizeof(AdActionStep) == AD_ACTION_STEP_SIZE, "AdActionStep ABI si _Static_assert(_Alignof(AdActionStep) == 8, "AdActionStep ABI alignment changed"); _Static_assert(offsetof(AdActionStep, label) == 0, "AdActionStep.label offset changed"); _Static_assert(offsetof(AdActionStep, outcome) == 8, "AdActionStep.outcome offset changed"); +_Static_assert(offsetof(AdActionStep, mechanism) == 16, "AdActionStep.mechanism offset changed"); +_Static_assert(offsetof(AdActionStep, has_mechanism) == 20, "AdActionStep.has_mechanism offset changed"); +_Static_assert(offsetof(AdActionStep, verified) == 21, "AdActionStep.verified offset changed"); +_Static_assert(offsetof(AdActionStep, has_verified) == 22, "AdActionStep.has_verified offset changed"); _Static_assert(sizeof(AdActionResult) == AD_ACTION_RESULT_SIZE, "AdActionResult ABI size changed"); _Static_assert(_Alignof(AdActionResult) == 8, "AdActionResult ABI alignment changed"); _Static_assert(offsetof(AdActionResult, action) == 0, "AdActionResult.action offset changed"); @@ -75,10 +81,87 @@ _Static_assert(offsetof(AdActionResult, ref_id) == 8, "AdActionResult.ref_id off _Static_assert(offsetof(AdActionResult, post_state) == 16, "AdActionResult.post_state offset changed"); _Static_assert(offsetof(AdActionResult, steps) == 24, "AdActionResult.steps offset changed"); _Static_assert(offsetof(AdActionResult, step_count) == 32, "AdActionResult.step_count offset changed"); +_Static_assert(offsetof(AdActionResult, details_json) == 40, "AdActionResult.details_json offset changed"); +_Static_assert(offsetof(AdActionResult, disposition) == 48, "AdActionResult.disposition offset changed"); +_Static_assert(sizeof(AdDeliverySemantics) == AD_DELIVERY_SEMANTICS_SIZE, "AdDeliverySemantics ABI size changed"); +_Static_assert(offsetof(AdDeliverySemantics, retry) == 4, "AdDeliverySemantics.retry offset changed"); _Static_assert(sizeof(AdRefEntry) == AD_REF_ENTRY_SIZE, "AdRefEntry ABI size changed"); _Static_assert(_Alignof(AdRefEntry) == 8, "AdRefEntry ABI alignment changed"); +_Static_assert(offsetof(AdRefEntry, process) == 0, "AdRefEntry.process offset changed"); +_Static_assert(offsetof(AdRefEntry, identity) == 8, "AdRefEntry.identity offset changed"); +_Static_assert(offsetof(AdRefEntry, geometry) == 48, "AdRefEntry.geometry offset changed"); +_Static_assert(offsetof(AdRefEntry, capabilities) == 96, "AdRefEntry.capabilities offset changed"); +_Static_assert(offsetof(AdRefEntry, source) == 128, "AdRefEntry.source offset changed"); +_Static_assert(offsetof(AdRefEntry, scope) == 168, "AdRefEntry.scope offset changed"); +_Static_assert(sizeof(AdExactRefEntry) == AD_EXACT_REF_ENTRY_SIZE, "AdExactRefEntry ABI size changed"); +_Static_assert(_Alignof(AdExactRefEntry) == 8, "AdExactRefEntry ABI alignment changed"); +_Static_assert(offsetof(AdExactRefEntry, version) == 0, "AdExactRefEntry.version offset changed"); +_Static_assert(offsetof(AdExactRefEntry, size) == 4, "AdExactRefEntry.size offset changed"); +_Static_assert(offsetof(AdExactRefEntry, entry) == 8, "AdExactRefEntry.entry offset changed"); +_Static_assert(offsetof(AdExactRefEntry, process_instance) == 208, "AdExactRefEntry.process_instance offset changed"); +_Static_assert(offsetof(AdExactRefEntry, identifier_kind) == 216, "AdExactRefEntry.identifier_kind offset changed"); +_Static_assert(sizeof(AdExactWindowInfo) == AD_EXACT_WINDOW_INFO_SIZE, "AdExactWindowInfo ABI size changed"); +_Static_assert(_Alignof(AdExactWindowInfo) == 8, "AdExactWindowInfo ABI alignment changed"); +_Static_assert(offsetof(AdExactWindowInfo, version) == 0, "AdExactWindowInfo.version offset changed"); +_Static_assert(offsetof(AdExactWindowInfo, size) == 4, "AdExactWindowInfo.size offset changed"); +_Static_assert(offsetof(AdExactWindowInfo, window) == 8, "AdExactWindowInfo.window offset changed"); +_Static_assert(offsetof(AdExactWindowInfo, process_instance) == 80, "AdExactWindowInfo.process_instance offset changed"); +_Static_assert(sizeof(AdExactSurfaceInfo) == AD_EXACT_SURFACE_INFO_SIZE, "AdExactSurfaceInfo ABI size changed"); +_Static_assert(_Alignof(AdExactSurfaceInfo) == 8, "AdExactSurfaceInfo ABI alignment changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, version) == 0, "AdExactSurfaceInfo.version offset changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, size) == 4, "AdExactSurfaceInfo.size offset changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, id) == 8, "AdExactSurfaceInfo.id offset changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, surface) == 16, "AdExactSurfaceInfo.surface offset changed"); +_Static_assert(sizeof(AdDisplayInfo) == AD_DISPLAY_INFO_SIZE, "AdDisplayInfo ABI size changed"); +_Static_assert(_Alignof(AdDisplayInfo) == 8, "AdDisplayInfo ABI alignment changed"); +_Static_assert(offsetof(AdDisplayInfo, version) == 0, "AdDisplayInfo.version offset changed"); +_Static_assert(offsetof(AdDisplayInfo, size) == 4, "AdDisplayInfo.size offset changed"); +_Static_assert(offsetof(AdDisplayInfo, id) == 8, "AdDisplayInfo.id offset changed"); +_Static_assert(offsetof(AdDisplayInfo, bounds) == 16, "AdDisplayInfo.bounds offset changed"); +_Static_assert(offsetof(AdDisplayInfo, is_primary) == 48, "AdDisplayInfo.is_primary offset changed"); +_Static_assert(offsetof(AdDisplayInfo, scale) == 56, "AdDisplayInfo.scale offset changed"); +_Static_assert(sizeof(AdRefProcess) == AD_REF_PROCESS_SIZE, "AdRefProcess ABI size changed"); +_Static_assert(sizeof(AdRefIdentity) == AD_REF_IDENTITY_SIZE, "AdRefIdentity ABI size changed"); +_Static_assert(offsetof(AdRefIdentity, native_id) == 32, "AdRefIdentity.native_id offset changed"); +_Static_assert(sizeof(AdStringSlice) == AD_STRING_SLICE_SIZE, "AdStringSlice ABI size changed"); +_Static_assert(sizeof(AdRefCapabilities) == AD_REF_CAPABILITIES_SIZE, "AdRefCapabilities ABI size changed"); +_Static_assert(sizeof(AdRefGeometry) == AD_REF_GEOMETRY_SIZE, "AdRefGeometry ABI size changed"); +_Static_assert(offsetof(AdRefGeometry, bounds_hash) == 32, "AdRefGeometry.bounds_hash offset changed"); +_Static_assert(sizeof(AdRefSource) == AD_REF_SOURCE_SIZE, "AdRefSource ABI size changed"); +_Static_assert(offsetof(AdRefSource, window_bounds_hash) == 24, "AdRefSource.window_bounds_hash offset changed"); +_Static_assert(sizeof(AdRefScope) == AD_REF_SCOPE_SIZE, "AdRefScope ABI size changed"); +_Static_assert(offsetof(AdRefScope, path) == 8, "AdRefScope.path offset changed"); _Static_assert(sizeof(struct AdWaitArgs) == AD_WAIT_ARGS_SIZE, "AdWaitArgs ABI size drift"); _Static_assert(_Alignof(struct AdWaitArgs) == 8, "AdWaitArgs ABI alignment changed"); +_Static_assert(offsetof(AdWaitArgs, mode) == 0, "AdWaitArgs.mode offset changed"); +_Static_assert(offsetof(AdWaitArgs, predicate) == 48, "AdWaitArgs.predicate offset changed"); +_Static_assert(offsetof(AdWaitArgs, scope) == 96, "AdWaitArgs.scope offset changed"); +_Static_assert(sizeof(AdOptionalU64) == AD_OPTIONAL_U64_SIZE, "AdOptionalU64 ABI size changed"); +_Static_assert(sizeof(AdOptionalUsize) == AD_OPTIONAL_USIZE_SIZE, "AdOptionalUsize ABI size changed"); +_Static_assert(sizeof(AdWaitSurfaceModes) == AD_WAIT_SURFACE_MODES_SIZE, "AdWaitSurfaceModes ABI size changed"); +_Static_assert(sizeof(AdWaitMode) == AD_WAIT_MODE_SIZE, "AdWaitMode ABI size changed"); +_Static_assert(sizeof(AdWaitPredicate) == AD_WAIT_PREDICATE_SIZE, "AdWaitPredicate ABI size changed"); +_Static_assert(sizeof(AdWaitScope) == AD_WAIT_SCOPE_SIZE, "AdWaitScope ABI size changed"); +_Static_assert(sizeof(AdNode) == AD_NODE_SIZE, "AdNode ABI size changed"); +_Static_assert(offsetof(AdNode, content) == 0, "AdNode.content offset changed"); +_Static_assert(offsetof(AdNode, presentation) == 48, "AdNode.presentation offset changed"); +_Static_assert(offsetof(AdNode, relation) == 96, "AdNode.relation offset changed"); +_Static_assert(sizeof(AdNodeContent) == AD_NODE_CONTENT_SIZE, "AdNodeContent ABI size changed"); +_Static_assert(sizeof(AdNodePresentation) == AD_NODE_PRESENTATION_SIZE, "AdNodePresentation ABI size changed"); +_Static_assert(sizeof(AdNodeRelation) == AD_NODE_RELATION_SIZE, "AdNodeRelation ABI size changed"); +_Static_assert(sizeof(AdNotificationIdentity) == AD_NOTIFICATION_IDENTITY_SIZE, "AdNotificationIdentity ABI size changed"); +_Static_assert(sizeof(AdNotificationActionRequest) == AD_NOTIFICATION_ACTION_REQUEST_SIZE, "AdNotificationActionRequest ABI size changed"); +_Static_assert(offsetof(AdNotificationActionRequest, identity) == 16, "AdNotificationActionRequest.identity offset changed"); +_Static_assert(sizeof(AdFindQuery) == AD_FIND_QUERY_SIZE, "AdFindQuery ABI size changed"); +_Static_assert(offsetof(AdFindQuery, filter) == 24, "AdFindQuery.filter offset changed"); +_Static_assert(sizeof(AdFindControl) == AD_FIND_CONTROL_SIZE, "AdFindControl ABI size changed"); +_Static_assert(offsetof(AdFindControl, timeout_ms) == 16, "AdFindControl.timeout_ms offset changed"); +_Static_assert(sizeof(AdFindSelection) == AD_FIND_SELECTION_SIZE, "AdFindSelection ABI size changed"); +_Static_assert(sizeof(AdFindIdentity) == AD_FIND_IDENTITY_SIZE, "AdFindIdentity ABI size changed"); +_Static_assert(sizeof(AdFindStatePredicate) == AD_FIND_STATE_PREDICATE_SIZE, "AdFindStatePredicate ABI size changed"); +_Static_assert(sizeof(AdFindStateSlice) == AD_FIND_STATE_SLICE_SIZE, "AdFindStateSlice ABI size changed"); +_Static_assert(sizeof(AdFindFilter) == AD_FIND_FILTER_SIZE, "AdFindFilter ABI size changed"); +_Static_assert(offsetof(AdFindFilter, exact) == 80, "AdFindFilter.exact offset changed"); #endif /* __STDC_VERSION__ >= 201112L */ #endif /* AGENT_DESKTOP_ABI_ASSERTS */ """ @@ -86,6 +169,9 @@ _Static_assert(_Alignof(struct AdWaitArgs) == 8, "AdWaitArgs ABI alignment chang [const] allow_static_const = false +[defines] +"feature = panic-injection" = "AGENT_DESKTOP_TEST_PANIC_INJECTION" + [export] # Force-emit the enums whose discriminants are the valid values for # public `int32_t` fields (AdAction.kind, AdScrollParams.direction, etc.). @@ -97,12 +183,17 @@ allow_static_const = false include = [ "AdActionKind", "AdDirection", + "AdDeliveryDisposition", + "AdFindSelectionKind", + "AdIdentifierKind", "AdModifier", "AdMouseButton", "AdMouseEventKind", "AdPolicyKind", + "AdRetryDisposition", "AdScreenshotKind", "AdSnapshotSurface", + "AdStepMechanism", "AdWindowOpKind", ] diff --git a/crates/ffi/codegen_templates/execute_by_ref.rs.in b/crates/ffi/codegen_templates/execute_by_ref.rs.in deleted file mode 100644 index 0186243..0000000 --- a/crates/ffi/codegen_templates/execute_by_ref.rs.in +++ /dev/null @@ -1,139 +0,0 @@ - -/// Drives a ref action (`@e5`, action) through the canonical ref-action -/// pipeline: `RefStore` load → `RefMap` lookup (→ `STALE_REF` on missing) → -/// strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live -/// actionability preflight → dispatch → handle release. -/// -/// Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type` -/// command); `PressKey` shares that `focus_fallback` base (a ref-targeted key -/// press may need the target focused); every other action defaults to -/// `headless`. An explicit `policy` discriminant may *elevate* to headed but -/// must not downgrade an action below its base. Base and elevation are computed -/// by `agent_desktop_core::commands::execute_by_ref::execute` via -/// `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and -/// FFI share a single source of policy truth. -/// -/// `ref_id` tri-state: null → `ErrInvalidArgs`; non-null invalid UTF-8 → -/// `ErrInvalidArgs`; valid UTF-8 but bad `@e{N}` format → `ErrInvalidArgs`. -/// -/// `snapshot_id` tri-state: null → use the latest snapshot for the session -/// (CLI `--snapshot` omitted); valid UTF-8 → pin to that snapshot id; non-null -/// invalid UTF-8 → `ErrInvalidArgs`. -/// -/// `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback, -/// 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)` -/// accepts the action's own CLI base (so `TypeText` still uses -/// `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks. -/// -/// On success `*out` is set to a NUL-terminated JSON envelope (command -/// `"execute_by_ref"`); free with `ad_free_string`. On guard or decode -/// failure (invalid args before the command runs) `*out` remains null. -/// On a command-level error (STALE_REF, AMBIGUOUS_TARGET, etc.) `*out` -/// holds the error JSON envelope and must still be freed with -/// `ad_free_string`. The last-error slot is populated on all failures. -/// -/// **Dispatch-before-serialize ordering**: the action is dispatched (and any -/// side effects committed) before the result JSON is serialized. In the -/// near-impossible event that serialization of an already-valid -/// `ActionResult` fails, `*out` is null and `ErrInternal` is returned while -/// the side effect has already occurred. No pre-validation machinery is -/// needed because serialization of a valid envelope effectively never fails. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer from `ad_adapter_create[_with_session]`. -/// `ref_id` must be a non-null pointer to a NUL-terminated C string within -/// `AD_MAX_STRING_BYTES + 1` bytes; null is **not** optional — it is defined -/// behaviour (no UB) but is rejected immediately with `ErrInvalidArgs`. -/// `snapshot_id` may be null (meaning: use the latest snapshot for this -/// session) or a non-null NUL-terminated C string within -/// `AD_MAX_STRING_BYTES + 1` bytes. `action` must be a non-null pointer to a -/// valid `AdAction`. `out` must be a non-null writable pointer. All pointers -/// must remain valid for the duration of the call. Must be called from the -/// main thread on macOS. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_execute_by_ref( - adapter: *const AdAdapter, - ref_id: *const c_char, - snapshot_id: *const c_char, - action: *const AdAction, - policy: i32, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - trap_panic(|| { - if let Err(rc) = require_main_thread() { - return rc; - } - guard_non_null!(adapter, c"adapter is null"); - guard_non_null!(action, c"action is null"); - - let ref_str = match required_adapter_string(ref_id, "ref_id") { - Ok(s) => s, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - if let Err(app_err) = validate_ref_id(&ref_str) { - let ae = app_error_to_adapter(app_err); - set_last_error(&ae); - return crate::error::last_error_code(); - } - - let snapshot_str = match optional_adapter_string(snapshot_id, "snapshot_id") { - Ok(opt) => opt, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - let caller_policy = match AdPolicyKind::from_c(policy) { - Some(p) => p, - None => { - set_last_error(&AdapterError::new( - ErrorCode::InvalidArgs, - "invalid policy kind discriminant", - )); - return AdResult::ErrInvalidArgs; - } - }; - - let core_action = match unsafe { action_from_c(&*action) } { - Ok(a) => a, - Err(msg) => { - set_last_error(&AdapterError::new(ErrorCode::InvalidArgs, msg)); - return AdResult::ErrInvalidArgs; - } - }; - - let caller_ip = caller_policy.to_interaction_policy(); - - let adapter_ref = unsafe { &*adapter }; - let context = match adapter_ref.command_context() { - Ok(ctx) => ctx, - Err(e) => { - let ae = app_error_to_adapter(e); - set_last_error(&ae); - return crate::error::last_error_code(); - } - }; - - let scope = context.command_scope("execute_by_ref"); - - let result = agent_desktop_core::commands::execute_by_ref::execute( - &ref_str, - snapshot_str.as_deref(), - core_action, - caller_ip, - adapter_ref.inner.as_ref(), - &context, - ); - scope.complete(&result); - - unsafe { write_command_envelope("execute_by_ref", result, out) } - }) -} diff --git a/crates/ffi/codegen_templates/wait.rs.in b/crates/ffi/codegen_templates/wait.rs.in deleted file mode 100644 index 0a7c214..0000000 --- a/crates/ffi/codegen_templates/wait.rs.in +++ /dev/null @@ -1,104 +0,0 @@ - -/// Runs `wait` with the given args, blocking the calling thread until the -/// condition is met or `timeout_ms` elapses. -/// -/// On success `*out` is set to a freshly allocated JSON string containing the -/// CLI-format wait envelope (`{version, ok, command, data}`). The caller must -/// release the string with `ad_free_string(*out)`. -/// -/// On a command-level failure (e.g. `TIMEOUT`, `ELEMENT_NOT_FOUND`) `*out` is -/// set to a freshly allocated JSON string with `"ok":false` and an `"error"` -/// payload. The caller must still release it with `ad_free_string(*out)`. The -/// last-error slot is also set. -/// -/// On an argument or infrastructure failure (null adapter, null args, null out, -/// off-main-thread, invalid UTF-8 field) `*out` is zeroed, the last-error slot -/// is set, and a negative `AdResult` code is returned. No allocation is made. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer returned by `ad_adapter_create` that -/// has not been destroyed. `args` must be non-null and point to a valid -/// zero-initialized `AdWaitArgs`. `out` must be non-null and point to a -/// writable `*mut c_char`. -/// -/// All `*const c_char` fields inside `AdWaitArgs` must be null or point to -/// readable, NUL-terminated memory within `AD_MAX_STRING_BYTES + 1` bytes. -/// -/// `ad_wait` blocks the calling thread for up to `timeout_ms` milliseconds -/// while it holds a live reference into the adapter's allocation. The adapter -/// must outlive the call: do not call `ad_adapter_destroy` on this handle from -/// another thread while `ad_wait` is running — that is a use-after-free. Ensure -/// the wait has returned before destroying the adapter. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_wait( - adapter: *const AdAdapter, - args: *const AdWaitArgs, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - guard_non_null!(args, c"args is null"); - - trap_panic(|| { - if let Err(rc) = require_main_thread() { - return rc; - } - guard_non_null!(adapter, c"adapter is null"); - - let args = unsafe { &*args }; - let adapter_ref = unsafe { &*adapter }; - - let ms = args.has_ms.then_some(args.ms); - - let element = unsafe { decode_optional_filter!(args.element, "element") }; - let window = unsafe { decode_optional_filter!(args.window, "window") }; - let text = unsafe { decode_optional_filter!(args.text, "text") }; - let snapshot_id = unsafe { decode_optional_filter!(args.snapshot_id, "snapshot_id") }; - let predicate = unsafe { decode_optional_filter!(args.predicate, "predicate") }; - let value = unsafe { decode_optional_filter!(args.value, "value") }; - let action_field = unsafe { decode_optional_filter!(args.action, "action") }; - let app = unsafe { decode_optional_filter!(args.app, "app") }; - - let wait_args = WaitArgs { - mode: WaitModeArgs { - ms, - element, - window, - text, - menu: args.menu, - menu_closed: args.menu_closed, - notification: args.notification, - }, - predicate: WaitPredicateArgs { - snapshot_id, - predicate, - value, - action: action_field, - count: args.has_count.then_some(args.count), - }, - timeout_ms: args.timeout_ms, - app, - }; - - let ctx = match adapter_ref.command_context() { - Ok(c) => c, - Err(app_err) => { - let adapter_err = app_error_to_adapter(app_err); - error::set_last_error(&adapter_err); - return error::last_error_code(); - } - }; - - let scope = ctx.command_scope("wait"); - - let result = agent_desktop_core::commands::wait::execute( - wait_args, - adapter_ref.inner.as_ref(), - &ctx, - ); - scope.complete(&result); - - unsafe { write_command_envelope("wait", result, out) } - }) -} diff --git a/crates/ffi/examples/panic_spike.rs b/crates/ffi/examples/panic_spike.rs deleted file mode 100644 index 402e258..0000000 --- a/crates/ffi/examples/panic_spike.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! Regression guard against a `panic = "abort"` regression in the `release-ffi` -//! Cargo profile. -//! -//! ## What this proves -//! -//! The `release-ffi` profile keeps `panic = "unwind"`, so -//! `std::panic::catch_unwind` — the same primitive that -//! `crate::ffi_try::trap_panic` relies on — remains effective under that -//! optimized profile. If `panic = "abort"` were accidentally restored, this -//! example would SIGABRT instead of catching: a loud, immediate regression -//! signal. -//! -//! ## What this does NOT prove -//! -//! This example does NOT dlopen the shipped cdylib and trigger a panic inside -//! a real `ad_*` entrypoint's `trap_panic` fence. A full cdylib-dlopen -//! panic-injection test is a tracked follow-up (Phase C+). -//! -//! ## Build and run -//! -//! cargo run --profile release-ffi --example panic_spike -p agent-desktop-ffi -//! -//! Expected output: `PANIC CAUGHT OK (code = -1)`, exit code 0. - -use std::panic::AssertUnwindSafe; - -#[unsafe(no_mangle)] -pub extern "C" fn spike_panicking_entry() -> i32 { - std::panic::catch_unwind(AssertUnwindSafe(|| -> i32 { - panic!("synthetic panic inside extern C fn"); - })) - .unwrap_or(-1) -} - -fn main() { - let code = spike_panicking_entry(); - if code == -1 { - println!("PANIC CAUGHT OK (code = -1)"); - std::process::exit(0); - } else { - println!("UNEXPECTED: got code {}", code); - std::process::exit(2); - } -} diff --git a/crates/ffi/include/agent_desktop.h b/crates/ffi/include/agent_desktop.h index 39c1312..39b4e42 100644 --- a/crates/ffi/include/agent_desktop.h +++ b/crates/ffi/include/agent_desktop.h @@ -16,14 +16,14 @@ * AdAdapter *a = ad_adapter_create_with_session(session_id); // with session * * 3. Observe via ad_snapshot(). The returned JSON envelope contains the - * accessibility tree with @e-prefixed ref IDs (e.g. "@e5") that address + * accessibility tree with snapshot-qualified ref IDs (for example, + * "@s8f3k2p9:e5") that address * individual interactive elements. A refmap is written under * ~/.agent-desktop/ and is keyed to the session. The envelope carries - * data.snapshot_id — pass it back as the snapshot_id argument to - * ad_execute_by_ref to pin the action to that exact snapshot; pass NULL - * to target the latest snapshot for the session. + * data.snapshot_id. Qualified refs already pin the exact snapshot; legacy + * bare @eN refs require that ID as the snapshot_id argument. * - * 4. Act via ad_execute_by_ref(a, "@e5", snapshot_id, &action, policy, &out). + * 4. Act via ad_execute_by_ref(a, "@s8f3k2p9:e5", NULL, &action, policy, &out). * Build an AdAction by zero-initialising it and setting its kind field to * an AD_ACTION_KIND_* constant plus any kind-specific fields (e.g. .text * for AD_ACTION_KIND_TYPE_TEXT). policy=0 (Headless) keeps each action's @@ -34,8 +34,9 @@ * 5. Ownership: every non-null *out string must be freed with ad_free_string(). * Destroy the adapter when done with ad_adapter_destroy(a). * - * macOS: all ad_* calls that interact with the AX API must be made from the - * main thread. ad_snapshot and ad_execute_by_ref enforce this with a guard. + * Calls may originate on any host thread. Native element handles remain + * thread-affine and must be used and released on the thread that resolved + * them. Desktop mutations are serialized by an interaction lease. */ @@ -51,7 +52,7 @@ * mismatch means the header and dylib are incompatible and the consumer should * refuse to proceed rather than risk undefined behaviour. */ -#define AD_ABI_VERSION_MAJOR 1 +#define AD_ABI_VERSION_MAJOR 3 /** * Maximum byte length (excluding the NUL terminator) accepted for any @@ -66,15 +67,69 @@ #define AD_ACTION_SIZE 96 -#define AD_ACTION_RESULT_SIZE 40 +#define AD_ACTION_RESULT_SIZE 56 -#define AD_ACTION_STEP_SIZE 16 +#define AD_ACTION_STEP_SIZE 32 + +#define AD_DELIVERY_SEMANTICS_SIZE 8 + +#define AD_DISPLAY_INFO_VERSION 1 + +#define AD_DISPLAY_INFO_SIZE 64 #define AD_DRAG_PARAMS_SIZE 48 #define AD_ELEMENT_STATE_SIZE 32 -#define AD_REF_ENTRY_SIZE 192 +#define AD_EXACT_REF_ENTRY_VERSION 1 + +#define AD_EXACT_REF_ENTRY_SIZE 224 + +#define AD_EXACT_SURFACE_INFO_VERSION 1 + +#define AD_EXACT_SURFACE_INFO_SIZE 40 + +#define AD_EXACT_WINDOW_INFO_VERSION 1 + +#define AD_EXACT_WINDOW_INFO_SIZE 88 + +#define AD_FIND_CONTROL_SIZE 24 + +#define AD_FIND_FILTER_SIZE 88 + +#define AD_FIND_IDENTITY_SIZE 40 + +#define AD_FIND_QUERY_VERSION 1 + +#define AD_FIND_QUERY_SIZE 112 + +#define AD_FIND_SELECTION_SIZE 8 + +#define AD_FIND_STATE_PREDICATE_SIZE 16 + +#define AD_FIND_STATE_SLICE_SIZE 16 + +#define AD_MODIFIER_CMD 0 + +#define AD_NODE_SIZE 112 + +#define AD_NODE_CONTENT_SIZE 48 + +#define AD_NODE_PRESENTATION_SIZE 48 + +#define AD_NODE_RELATION_SIZE 12 + +#define AD_NOTIFICATION_ACTION_REQUEST_SIZE 32 + +#define AD_NOTIFICATION_IDENTITY_SIZE 16 + +#define AD_OPTIONAL_U64_SIZE 16 + +#define AD_OPTIONAL_USIZE_SIZE 16 + +#define AD_REF_CAPABILITIES_SIZE 32 + +#define AD_REF_ENTRY_SIZE 200 /** * Per-field input caps enforced when converting an `AdRefEntry` at the C @@ -88,6 +143,18 @@ #define AD_MAX_REF_PATH_DEPTH 128 +#define AD_REF_GEOMETRY_SIZE 48 + +#define AD_REF_IDENTITY_SIZE 40 + +#define AD_REF_PROCESS_SIZE 4 + +#define AD_REF_SCOPE_SIZE 32 + +#define AD_REF_SOURCE_SIZE 40 + +#define AD_STRING_SLICE_SIZE 16 + /** * Pinned size of `AdWaitArgs` on 64-bit targets. The compile-time * assert below and the `ad_wait_args_size()` runtime getter form the @@ -96,14 +163,22 @@ */ #define AD_WAIT_ARGS_SIZE 112 +#define AD_WAIT_MODE_SIZE 48 + +#define AD_WAIT_PREDICATE_SIZE 48 + +#define AD_WAIT_SCOPE_SIZE 16 + +#define AD_WAIT_SURFACE_MODES_SIZE 3 + /** * New result codes may be appended in future releases. Always handle values * outside this list. */ enum AdResult -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_RESULT_OK = 0, AD_RESULT_ERR_PERM_DENIED = -1, @@ -121,31 +196,36 @@ enum AdResult AD_RESULT_ERR_SNAPSHOT_NOT_FOUND = -13, AD_RESULT_ERR_POLICY_DENIED = -14, AD_RESULT_ERR_AMBIGUOUS_TARGET = -15, + AD_RESULT_ERR_APP_UNRESPONSIVE = -16, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdResult AdResult; #else typedef int32_t AdResult; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdImageFormat -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_IMAGE_FORMAT_PNG = 0, AD_IMAGE_FORMAT_JPG = 1, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdImageFormat AdImageFormat; #else typedef int32_t AdImageFormat; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdActionKind -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_ACTION_KIND_CLICK = 0, AD_ACTION_KIND_DOUBLE_CLICK = 1, @@ -169,109 +249,196 @@ enum AdActionKind AD_ACTION_KIND_HOVER = 19, AD_ACTION_KIND_DRAG = 20, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdActionKind AdActionKind; #else typedef int32_t AdActionKind; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdDirection -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_DIRECTION_UP = 0, AD_DIRECTION_DOWN = 1, AD_DIRECTION_LEFT = 2, AD_DIRECTION_RIGHT = 3, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdDirection AdDirection; #else typedef int32_t AdDirection; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus + +enum AdDeliveryDisposition +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L + : int32_t +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L + { + AD_DELIVERY_DISPOSITION_UNKNOWN = 0, + AD_DELIVERY_DISPOSITION_NOT_DELIVERED = 1, + AD_DELIVERY_DISPOSITION_DELIVERY_UNCERTAIN = 2, + AD_DELIVERY_DISPOSITION_DELIVERED_UNVERIFIED = 3, + AD_DELIVERY_DISPOSITION_DELIVERED_VERIFIED = 4, +}; +#ifndef __cplusplus +#if __STDC_VERSION__ >= 202311L +typedef enum AdDeliveryDisposition AdDeliveryDisposition; +#else +typedef int32_t AdDeliveryDisposition; +#endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus + +enum AdFindSelectionKind +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L + : int32_t +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L + { + AD_FIND_SELECTION_KIND_STRICT = 0, + AD_FIND_SELECTION_KIND_FIRST = 1, + AD_FIND_SELECTION_KIND_LAST = 2, + AD_FIND_SELECTION_KIND_NTH = 3, +}; +#ifndef __cplusplus +#if __STDC_VERSION__ >= 202311L +typedef enum AdFindSelectionKind AdFindSelectionKind; +#else +typedef int32_t AdFindSelectionKind; +#endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus + +enum AdIdentifierKind +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L + : int32_t +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L + { + AD_IDENTIFIER_KIND_AX_IDENTIFIER = 0, + AD_IDENTIFIER_KIND_AX_DOM_IDENTIFIER = 1, + AD_IDENTIFIER_KIND_AUTOMATION_ID = 2, + AD_IDENTIFIER_KIND_RUNTIME_ID = 3, + AD_IDENTIFIER_KIND_ATSPI_OBJECT_PATH = 4, +}; +#ifndef __cplusplus +#if __STDC_VERSION__ >= 202311L +typedef enum AdIdentifierKind AdIdentifierKind; +#else +typedef int32_t AdIdentifierKind; +#endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdModifier -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { - AD_MODIFIER_CMD = 0, + AD_MODIFIER_META = 0, AD_MODIFIER_CTRL = 1, AD_MODIFIER_ALT = 2, AD_MODIFIER_SHIFT = 3, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdModifier AdModifier; #else typedef int32_t AdModifier; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdMouseButton -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_MOUSE_BUTTON_LEFT = 0, AD_MOUSE_BUTTON_RIGHT = 1, AD_MOUSE_BUTTON_MIDDLE = 2, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdMouseButton AdMouseButton; #else typedef int32_t AdMouseButton; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdMouseEventKind -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_MOUSE_EVENT_KIND_MOVE = 0, AD_MOUSE_EVENT_KIND_DOWN = 1, AD_MOUSE_EVENT_KIND_UP = 2, AD_MOUSE_EVENT_KIND_CLICK = 3, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdMouseEventKind AdMouseEventKind; #else typedef int32_t AdMouseEventKind; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdPolicyKind -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_POLICY_KIND_HEADLESS = 0, AD_POLICY_KIND_FOCUS_FALLBACK = 1, AD_POLICY_KIND_HEADED = 2, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdPolicyKind AdPolicyKind; #else typedef int32_t AdPolicyKind; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus + +enum AdRetryDisposition +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L + : int32_t +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L + { + AD_RETRY_DISPOSITION_UNKNOWN = 0, + AD_RETRY_DISPOSITION_SAFE = 1, + AD_RETRY_DISPOSITION_UNSAFE = 2, +}; +#ifndef __cplusplus +#if __STDC_VERSION__ >= 202311L +typedef enum AdRetryDisposition AdRetryDisposition; +#else +typedef int32_t AdRetryDisposition; +#endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdScreenshotKind -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_SCREENSHOT_KIND_SCREEN = 0, AD_SCREENSHOT_KIND_WINDOW = 1, AD_SCREENSHOT_KIND_FULL_SCREEN = 2, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdScreenshotKind AdScreenshotKind; #else typedef int32_t AdScreenshotKind; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdSnapshotSurface -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_SNAPSHOT_SURFACE_WINDOW = 0, AD_SNAPSHOT_SURFACE_FOCUSED = 1, @@ -280,17 +447,47 @@ enum AdSnapshotSurface AD_SNAPSHOT_SURFACE_SHEET = 4, AD_SNAPSHOT_SURFACE_POPOVER = 5, AD_SNAPSHOT_SURFACE_ALERT = 6, + AD_SNAPSHOT_SURFACE_DESKTOP = 7, + AD_SNAPSHOT_SURFACE_TASKBAR = 8, + AD_SNAPSHOT_SURFACE_SYSTEM_TRAY = 9, + AD_SNAPSHOT_SURFACE_QUICK_SETTINGS = 10, + AD_SNAPSHOT_SURFACE_NOTIFICATION_CENTER = 11, + AD_SNAPSHOT_SURFACE_TOOLBAR = 12, + AD_SNAPSHOT_SURFACE_DOCK = 13, + AD_SNAPSHOT_SURFACE_SPOTLIGHT = 14, + AD_SNAPSHOT_SURFACE_MENU_BAR_EXTRAS = 15, + AD_SNAPSHOT_SURFACE_SYSTEM_TRAY_OVERFLOW = 16, + AD_SNAPSHOT_SURFACE_START_MENU = 17, + AD_SNAPSHOT_SURFACE_ACTION_CENTER = 18, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdSnapshotSurface AdSnapshotSurface; #else typedef int32_t AdSnapshotSurface; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus + +enum AdStepMechanism +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L + : int32_t +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L + { + AD_STEP_MECHANISM_SEMANTIC_API = 1, + AD_STEP_MECHANISM_PHYSICAL_SYNTHETIC = 2, +}; +#ifndef __cplusplus +#if __STDC_VERSION__ >= 202311L +typedef enum AdStepMechanism AdStepMechanism; +#else +typedef int32_t AdStepMechanism; +#endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus enum AdWindowOpKind -#if __STDC_VERSION__ >= 202311L +#if defined(__cplusplus) || __STDC_VERSION__ >= 202311L : int32_t -#endif // __STDC_VERSION__ >= 202311L +#endif // defined(__cplusplus) || __STDC_VERSION__ >= 202311L { AD_WINDOW_OP_KIND_RESIZE = 0, AD_WINDOW_OP_KIND_MOVE = 1, @@ -298,11 +495,13 @@ enum AdWindowOpKind AD_WINDOW_OP_KIND_MAXIMIZE = 3, AD_WINDOW_OP_KIND_RESTORE = 4, }; +#ifndef __cplusplus #if __STDC_VERSION__ >= 202311L typedef enum AdWindowOpKind AdWindowOpKind; #else typedef int32_t AdWindowOpKind; #endif // __STDC_VERSION__ >= 202311L +#endif // __cplusplus typedef struct AdAdapter AdAdapter; @@ -312,6 +511,21 @@ typedef struct AdAdapter AdAdapter; */ typedef struct AdAppList AdAppList; +/** + * Opaque list handle emitted by `ad_list_displays`. + */ +typedef struct AdDisplayList AdDisplayList; + +/** + * Opaque list handle emitted by `ad_list_surfaces_exact`. + */ +typedef struct AdExactSurfaceList AdExactSurfaceList; + +/** + * Opaque list handle emitted by ad_list_windows_exact. + */ +typedef struct AdExactWindowList AdExactWindowList; + /** * Opaque image-buffer handle returned by `ad_screenshot`. The backing * byte buffer and its length live inside the Rust-owned struct — a @@ -345,6 +559,9 @@ typedef struct AdSurfaceList AdSurfaceList; typedef struct AdWindowList AdWindowList; typedef struct AdNativeHandle { + /** + * Opaque thread-affine registry token, never an allocation or OS pointer. + */ const void *ptr; } AdNativeHandle; @@ -427,16 +644,40 @@ typedef struct AdElementState { typedef struct AdActionStep { const char *label; const char *outcome; + int32_t mechanism; + bool has_mechanism; + bool verified; + bool has_verified; + uint64_t _reserved; } AdActionStep; +typedef struct AdDeliverySemantics { + int32_t delivery; + int32_t retry; +} AdDeliverySemantics; + typedef struct AdActionResult { const char *action; const char *ref_id; struct AdElementState *post_state; struct AdActionStep *steps; uint32_t step_count; + const char *details_json; + struct AdDeliverySemantics disposition; } AdActionResult; +typedef struct AdRefProcess { + uint32_t pid; +} AdRefProcess; + +typedef struct AdRefIdentity { + const char *role; + const char *name; + const char *value; + const char *description; + const char *native_id; +} AdRefIdentity; + typedef struct AdRect { double x; double y; @@ -444,118 +685,164 @@ typedef struct AdRect { double height; } AdRect; -typedef struct AdRefEntry { - int32_t pid; - const char *role; - const char *name; - const char *value; - const char *description; - const char *const *states; - size_t state_count; - const char *const *available_actions; - size_t available_action_count; +typedef struct AdRefGeometry { struct AdRect bounds; - bool has_bounds; uint64_t bounds_hash; + bool has_bounds; bool has_bounds_hash; - const char *source_app; - const char *source_window_id; - const char *source_window_title; - int32_t source_surface; +} AdRefGeometry; + +typedef struct AdStringSlice { + const char *const *items; + size_t count; +} AdStringSlice; + +typedef struct AdRefCapabilities { + struct AdStringSlice states; + struct AdStringSlice available_actions; +} AdRefCapabilities; + +typedef struct AdRefSource { + const char *app; + const char *window_id; + const char *window_title; + uint64_t window_bounds_hash; + int32_t surface; + bool has_window_bounds_hash; +} AdRefSource; + +typedef struct AdRefScope { const char *root_ref; - bool path_is_absolute; const uint32_t *path; size_t path_count; + bool path_is_absolute; +} AdRefScope; + +typedef struct AdRefEntry { + struct AdRefProcess process; + struct AdRefIdentity identity; + struct AdRefGeometry geometry; + struct AdRefCapabilities capabilities; + struct AdRefSource source; + struct AdRefScope scope; } AdRefEntry; +/** + * Additive exact-identity payload for low-level struct-based ref actions. + * + * Callers must set `version` to `AD_EXACT_REF_ENTRY_VERSION`, `size` to + * `AD_EXACT_REF_ENTRY_SIZE`, and `process_instance` to the generation token + * emitted by the snapshot. When `entry.identity.native_id` is non-null, + * `identifier_kind` must name its exact platform identifier namespace. + */ +typedef struct AdExactRefEntry { + uint32_t version; + uint32_t size; + struct AdRefEntry entry; + const char *process_instance; + int32_t identifier_kind; +} AdExactRefEntry; + typedef struct AdWindowInfo { + /** + * Legacy observation-only window ID. This struct has no process-generation + * evidence and is rejected by targeting APIs; use `AdExactWindowInfo` for + * any operation that sends a previously observed window back to the library. + */ const char *id; const char *title; const char *app_name; - int32_t pid; + uint32_t pid; struct AdRect bounds; bool has_bounds; bool is_focused; } AdWindowInfo; +/** + * Additive generation-pinned window identity for operations that target a + * previously observed live window. + */ +typedef struct AdExactWindowInfo { + uint32_t version; + uint32_t size; + struct AdWindowInfo window; + const char *process_instance; +} AdExactWindowInfo; + typedef struct AdAppInfo { const char *name; - int32_t pid; + uint32_t pid; const char *bundle_id; } AdAppInfo; +typedef struct AdOptionalU64 { + uint64_t value; + bool present; +} AdOptionalU64; + +typedef struct AdWaitSurfaceModes { + bool menu; + bool menu_closed; + bool notification; +} AdWaitSurfaceModes; + +typedef struct AdWaitMode { + struct AdOptionalU64 pause; + const char *element; + const char *window; + const char *text; + struct AdWaitSurfaceModes surfaces; +} AdWaitMode; + +typedef struct AdOptionalUsize { + size_t value; + bool present; +} AdOptionalUsize; + +typedef struct AdWaitPredicate { + const char *snapshot_id; + const char *predicate; + const char *value; + const char *action; + struct AdOptionalUsize count; +} AdWaitPredicate; + +typedef struct AdWaitScope { + uint64_t timeout_ms; + const char *app; +} AdWaitScope; + /** - * Arguments for `ad_wait`, mirroring `core::commands::wait::WaitArgs`. + * Arguments for `ad_wait`, mirroring `core::commands::wait::WaitArgs` for + * the pause/element/text/surface wait modes and predicates. * - * Fields map as follows: - * - `Option<u64>` → `u64` value + `bool has_*` sentinel (ms, count). - * - `Option<String>` → nullable `*const c_char` (null = absent). - * - `bool` → `bool`. + * The core event-wait mode (`--event` / `--window-id`) is intentionally not + * exposed over FFI in this release; `wait_args_from_ffi` always forwards + * `event: None` and `window_id: None` to core. `mode.window` here is a + * title-appearance wait (poll until a window with the given title exists), + * which is a distinct semantic from the deferred event-wait mode. + * + * Mode, predicate, and scope fields are grouped into named PODs. Optional + * numbers use `AdOptional*`; optional strings are nullable pointers. * * Callers must zero-initialize before use and verify layout via * `AD_WAIT_ARGS_SIZE` / `ad_wait_args_size()`. */ typedef struct AdWaitArgs { - /** - * Milliseconds to sleep (WaitMode::ms). - */ - uint64_t ms; - bool has_ms; - /** - * Element ref id to wait for (WaitMode::element). - */ - const char *element; - /** - * Window title to wait for (WaitMode::window). - */ - const char *window; - /** - * Text to wait for (WaitMode::text / WaitMode::notification text). - */ - const char *text; - /** - * Wait for menu to open (true) or close (false via menu_closed). - */ - bool menu; - /** - * Wait for menu to close. - */ - bool menu_closed; - /** - * Wait for a notification. - */ - bool notification; - /** - * Snapshot id for element predicate (WaitPredicateArgs::snapshot_id). - */ - const char *snapshot_id; - /** - * Predicate kind string (WaitPredicateArgs::predicate). - */ - const char *predicate; - /** - * Expected value for value-predicate (WaitPredicateArgs::value). - */ - const char *value; - /** - * Action name for actionability-predicate (WaitPredicateArgs::action). - */ - const char *action; - /** - * Expected match count for text waits (WaitPredicateArgs::count). - */ - size_t count; - bool has_count; - /** - * Timeout in milliseconds. - */ - uint64_t timeout_ms; - /** - * App name filter (null = any). Maps to WaitArgs::app. - */ - const char *app; + struct AdWaitMode mode; + struct AdWaitPredicate predicate; + struct AdWaitScope scope; } AdWaitArgs; +typedef struct AdDisplayInfo { + uint32_t version; + uint32_t size; + const char *id; + struct AdRect bounds; + bool is_primary; + double scale; +} AdDisplayInfo; + /** * Mouse event dispatched by `ad_mouse_event`. * @@ -571,6 +858,18 @@ typedef struct AdMouseEvent { uint32_t click_count; } AdMouseEvent; +typedef struct AdNotificationIdentity { + const char *app; + const char *title; +} AdNotificationIdentity; + +typedef struct AdNotificationActionRequest { + uint32_t index; + int32_t policy; + const char *action_name; + struct AdNotificationIdentity identity; +} AdNotificationActionRequest; + typedef struct AdNotificationFilter { const char *app; const char *text; @@ -587,10 +886,47 @@ typedef struct AdNotificationInfo { uint32_t action_count; } AdNotificationInfo; -typedef struct AdFindQuery { +typedef struct AdFindSelection { + int32_t kind; + uint32_t nth; +} AdFindSelection; + +typedef struct AdFindControl { + uint32_t version; + struct AdFindSelection selection; + uint64_t timeout_ms; +} AdFindControl; + +typedef struct AdFindIdentity { const char *role; - const char *name_substring; - const char *value_substring; + const char *name; + const char *description; + const char *native_id; + const char *value; +} AdFindIdentity; + +typedef struct AdFindStatePredicate { + const char *token; + int32_t expected; +} AdFindStatePredicate; + +typedef struct AdFindStateSlice { + const struct AdFindStatePredicate *items; + size_t count; +} AdFindStateSlice; + +typedef struct AdFindFilter { + struct AdFindIdentity identity; + const char *has_text; + struct AdFindStateSlice states; + const struct AdFindQuery *has; + const struct AdFindQuery *has_not; + bool exact; +} AdFindFilter; + +typedef struct AdFindQuery { + struct AdFindControl control; + struct AdFindFilter filter; } AdFindQuery; /** @@ -604,7 +940,7 @@ typedef struct AdFindQuery { typedef struct AdScreenshotTarget { int32_t kind; uint64_t screen_index; - int32_t pid; + uint32_t pid; } AdScreenshotTarget; typedef struct AdSurfaceInfo { @@ -613,20 +949,42 @@ typedef struct AdSurfaceInfo { int64_t item_count; } AdSurfaceInfo; -typedef struct AdNode { +/** + * Additive surface observation that preserves the core surface ID. + */ +typedef struct AdExactSurfaceInfo { + uint32_t version; + uint32_t size; + const char *id; + struct AdSurfaceInfo surface; +} AdExactSurfaceInfo; + +typedef struct AdNodeContent { const char *ref_id; const char *role; const char *name; const char *value; const char *description; const char *hint; +} AdNodeContent; + +typedef struct AdNodePresentation { char **states; - uint32_t state_count; struct AdRect bounds; + uint32_t state_count; bool has_bounds; +} AdNodePresentation; + +typedef struct AdNodeRelation { int32_t parent_index; uint32_t child_start; uint32_t child_count; +} AdNodeRelation; + +typedef struct AdNode { + struct AdNodeContent content; + struct AdNodePresentation presentation; + struct AdNodeRelation relation; } AdNode; typedef struct AdNodeTree { @@ -667,6 +1025,10 @@ typedef struct AdWindowOp { double y; } AdWindowOp; +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + /** * Returns the packed ABI major version of this dylib build. * @@ -704,6 +1066,10 @@ AdResult ad_init(uint32_t expected_major); * the same live adapter. Free the handle before destroying that adapter. * `action` must be a non-null pointer to a valid `AdAction`. * `out` must be a non-null pointer to an `AdActionResult` to write the result into. + * + * Handles come from exact resolvers and already carry process-generation + * evidence, so this executes under the same policy as + * `ad_execute_action_with_policy`. */ AdResult ad_execute_action(const struct AdAdapter *adapter, const struct AdNativeHandle *handle, @@ -763,30 +1129,40 @@ AdResult ad_execute_ref_action_with_policy(const struct AdAdapter *adapter, struct AdActionResult *out); /** - * Releases a handle previously returned by `ad_resolve_element` and - * zeroes the caller's struct so accidentally calling this twice is - * a deterministic no-op instead of a double-free on the underlying - * `CFRelease`. - * - * On macOS this calls `CFRelease` on the underlying `AXUIElementRef`, - * balancing the `CFRetain` that happened during `ad_resolve_element`. - * On Windows/Linux the call is a no-op that returns `AD_RESULT_OK` - * (platform adapters inherit the default `not_supported` impl; the - * FFI surface translates it so callers apply the same release - * pattern everywhere). - * - * Ownership contract: the FFI owns the handle from the moment - * `ad_resolve_element` writes `ptr`. Copying the struct after that - * point and calling `ad_free_handle` on either copy is undefined — - * there is no way for the library to detect forged non-null pointers. - * Callers that legitimately need a "copy" should re-resolve. + * Executes a struct-based ref action with exact process-generation and typed + * native-id evidence. * * # Safety * - * `adapter` must be a non-null pointer returned by `ad_adapter_create` - * and must be the same live adapter that produced `handle`. - * `handle` must be null or a `*mut AdNativeHandle` previously - * populated by `ad_resolve_element`. On return `(*handle).ptr` is + * All pointers must be valid. `entry` must carry the current exact-entry + * version and size. `out` is zeroed before any fallible operation. + */ +AdResult ad_execute_ref_action_exact_with_policy(const struct AdAdapter *adapter, + const struct AdExactRefEntry *entry, + const struct AdAction *action, + int32_t policy, + struct AdActionResult *out); + +/** + * Releases a handle previously returned by an exact resolver and + * zeroes the caller's struct so accidentally calling this twice is + * a deterministic no-op instead of dropping its owned payload twice. + * + * `AdNativeHandle.ptr` is an opaque registry token, not an operating-system + * or Rust allocation address. Removing it releases the platform payload. + * + * Ownership contract: the FFI owns the handle from the moment a resolver + * writes `ptr`. Copying the struct after that point is unsupported. Releasing + * the original zeroes it and makes a second release of that same struct a + * no-op; releasing an unzeroed copy is rejected. + * + * # Safety + * + * `adapter` must be a non-null pointer returned by `ad_adapter_create`. + * It must identify the same adapter that created the handle. The adapter may + * already have been destroyed; handles remain independently owned until freed. + * `handle` must be null or a `*mut AdNativeHandle` previously populated by an + * exact resolver on the calling thread. On return `(*handle).ptr` is * `NULL` so a double-call is a no-op instead of a double-free. */ AdResult ad_free_handle(const struct AdAdapter *adapter, struct AdNativeHandle *handle); @@ -797,11 +1173,24 @@ AdResult ad_free_handle(const struct AdAdapter *adapter, struct AdNativeHandle * * `adapter` must be a non-null pointer returned by `ad_adapter_create`. * `entry` must be a non-null pointer to a valid `AdRefEntry`. * `out` must be a non-null pointer to an `AdNativeHandle` to write the result into. + * + * This legacy entrypoint lacks exact identity evidence and fails closed. Use ad_resolve_element_exact. */ AdResult ad_resolve_element(const struct AdAdapter *adapter, const struct AdRefEntry *entry, struct AdNativeHandle *out); +/** + * Resolves an element using process-generation and typed native-id evidence. + * + * # Safety + * + * `adapter` and `entry` must be live and valid; `out` must be writable. + */ +AdResult ad_resolve_element_exact(const struct AdAdapter *adapter, + const struct AdExactRefEntry *entry, + struct AdNativeHandle *out); + /** * # Safety * @@ -849,13 +1238,8 @@ struct AdAdapter *ad_adapter_create_with_session(const char *session); * `ad_adapter_create_with_session`, or null. After this call the pointer * is invalid and must not be used. * - * The adapter must not be destroyed while any other call on it is still in - * flight on another thread. Destroying the handle concurrently with an - * in-flight call (e.g. `ad_wait` blocking on the main thread while this - * function is called from a worker thread) is undefined behaviour — the - * `Box` is freed while the blocked call still dereferences it. The caller - * owns this synchronisation: ensure all calls on the handle have returned - * before calling `ad_adapter_destroy`. + * Calls that acquired the adapter before destruction retain it until they + * return. Calls beginning after destruction fail with `ErrInvalidArgs`. */ void ad_adapter_destroy(struct AdAdapter *adapter); @@ -902,6 +1286,18 @@ AdResult ad_launch_app(const struct AdAdapter *adapter, uint64_t timeout_ms, struct AdWindowInfo *out); +/** + * Launches an application and returns a generation-pinned exact window. + * + * # Safety + * `adapter`, `id`, and `out` must satisfy the same requirements as + * `ad_launch_app`. Release the result with `ad_release_exact_window_fields`. + */ +AdResult ad_launch_app_exact(const struct AdAdapter *adapter, + const char *id, + uint64_t timeout_ms, + struct AdExactWindowInfo *out); + /** * # Safety * `adapter` must be a valid pointer from `ad_adapter_create`. @@ -936,31 +1332,36 @@ const struct AdAppInfo *ad_app_list_get(const struct AdAppList *list, uint32_t i void ad_app_list_free(struct AdAppList *list); /** - * Drives a ref action (`@e5`, action) through the canonical ref-action + * Drives a snapshot-qualified ref action (`@<snapshot_id>:e5`, action) + * through the canonical ref-action * pipeline: `RefStore` load → `RefMap` lookup (→ `STALE_REF` on missing) → * strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live - * actionability preflight → dispatch → handle release. + * actionability preflight → dispatch → owned-handle drop. * - * Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type` - * command); `PressKey` shares that `focus_fallback` base (a ref-targeted key - * press may need the target focused); every other action defaults to - * `headless`. An explicit `policy` discriminant may *elevate* to headed but - * must not downgrade an action below its base. Base and elevation are computed - * by `agent_desktop_core::commands::execute_by_ref::execute` via + * Policy: semantic actions, including `TypeText`, default to strict + * `headless`. Explicit `PressKey` defaults to `focus_fallback`. A policy + * discriminant may elevate to focus fallback or headed. Base and elevation + * are computed by `agent_desktop_core::commands::execute_by_ref::execute` via * `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and * FFI share a single source of policy truth. * * `ref_id` tri-state: null → `ErrInvalidArgs`; non-null invalid UTF-8 → * `ErrInvalidArgs`; valid UTF-8 but bad `@e{N}` format → `ErrInvalidArgs`. * - * `snapshot_id` tri-state: null → use the latest snapshot for the session - * (CLI `--snapshot` omitted); valid UTF-8 → pin to that snapshot id; non-null - * invalid UTF-8 → `ErrInvalidArgs`. + * `snapshot_id` tri-state: null is valid only when `ref_id` embeds its + * snapshot; valid UTF-8 pins a legacy bare `@eN` ref or must match the + * snapshot embedded in a qualified ref; invalid UTF-8 returns `ErrInvalidArgs`. * * `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback, * 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)` - * accepts the action's own CLI base (so `TypeText` still uses - * `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks. + * accepts the action's base policy. `FocusFallback (1)` explicitly permits + * focus without cursor movement. `Headed (2)` opts in to physical cursor and + * keyboard delivery. + * + * Uses a fixed 5000ms auto-wait budget (`DEFAULT_ACTION_TIMEOUT_MS`) before + * the actionability preflight, matching the CLI default. Call + * `ad_execute_by_ref_timeout` with an explicit `timeout_ms` (-1 = default, + * 0 = single-shot with no auto-wait) to control this. * * On success `*out` is set to a NUL-terminated JSON envelope (command * `"execute_by_ref"`); free with `ad_free_string`. On guard or decode @@ -982,12 +1383,12 @@ void ad_app_list_free(struct AdAppList *list); * `ref_id` must be a non-null pointer to a NUL-terminated C string within * `AD_MAX_STRING_BYTES + 1` bytes; null is **not** optional — it is defined * behaviour (no UB) but is rejected immediately with `ErrInvalidArgs`. - * `snapshot_id` may be null (meaning: use the latest snapshot for this - * session) or a non-null NUL-terminated C string within - * `AD_MAX_STRING_BYTES + 1` bytes. `action` must be a non-null pointer to a + * `snapshot_id` may be null only for a snapshot-qualified ref, or a non-null + * NUL-terminated C string within `AD_MAX_STRING_BYTES + 1` bytes. `action` + * must be a non-null pointer to a * valid `AdAction`. `out` must be a non-null writable pointer. All pointers * must remain valid for the duration of the call. Must be called from the - * main thread on macOS. + * calling thread. */ AdResult ad_execute_by_ref(const struct AdAdapter *adapter, const char *ref_id, @@ -996,13 +1397,30 @@ AdResult ad_execute_by_ref(const struct AdAdapter *adapter, int32_t policy, char **out); +/** + * Same as `ad_execute_by_ref` but with an explicit pre-action auto-wait + * budget in milliseconds. `timeout_ms == -1` uses the 5000ms default and + * `timeout_ms == 0` disables auto-wait for a single-shot preflight. + * + * # Safety + * + * Same pointer and threading requirements as `ad_execute_by_ref`. + */ +AdResult ad_execute_by_ref_timeout(const struct AdAdapter *adapter, + const char *ref_id, + const char *snapshot_id, + const struct AdAction *action, + int32_t policy, + int64_t timeout_ms, + char **out); + /** * Takes a full CLI-format snapshot of the target application window, * allocates `@e` refs for all interactive elements, persists the refmap * to disk, and writes the JSON envelope into `*out`. * * The JSON shape matches `agent-desktop snapshot`: - * `{"version":"2.0","ok":true,"command":"snapshot","data":{"app":"...","window":{...},"ref_count":N,"snapshot_id":"...","tree":{...}}}`. + * `{"version":"2.1","ok":true,"command":"snapshot","data":{"app":"...","window":{...},"ref_count":N,"snapshot_id":"...","tree":{...}}}`. * * **`*out` ownership and error behaviour:** * - On success (`AD_RESULT_OK`): `*out` is a heap-allocated JSON string with `"ok":true`. @@ -1010,7 +1428,7 @@ AdResult ad_execute_by_ref(const struct AdAdapter *adapter, * - On a command-level error (e.g. app not found, snapshot failure): `*out` is a * heap-allocated JSON string with `"ok":false` and an `"error"` payload. Caller * must still free it with `ad_free_string`. The last-error slot is also set. - * - On an argument or infrastructure error (null adapter, off-main-thread, invalid + * - On an argument or infrastructure error (null adapter, invalid * UTF-8, bad surface discriminant, context failure): `*out` is set to null and no * allocation is made. Only the last-error slot is set. * @@ -1039,7 +1457,7 @@ AdResult ad_execute_by_ref(const struct AdAdapter *adapter, * `ad_adapter_create_with_session`. `out` must be a non-null writable * `*mut *mut c_char`. `app` must be null or a NUL-terminated string within * `AD_MAX_STRING_BYTES + 1` bytes. All pointers must remain valid for the - * duration of the call. `adapter` must be used from the main thread on macOS. + * duration of the call. */ AdResult ad_snapshot(const struct AdAdapter *adapter, const char *app, @@ -1054,10 +1472,9 @@ AdResult ad_snapshot(const struct AdAdapter *adapter, * envelope matching the `agent-desktop status` CLI output. * * `ad_status` does not query the accessibility tree; it reads the - * permission report and ref-store metadata only, so it is safe to call - * from any thread (unlike tree-traversal commands that require the - * macOS main thread). On success `*out` is a NUL-terminated, - * heap-allocated JSON string freed with `ad_free_string`. + * permission report and ref-store metadata only. Like other adapter + * entrypoints, it may be called from any host thread. On success `*out` is a + * NUL-terminated, heap-allocated JSON string freed with `ad_free_string`. * * On a command-level failure `*out` is set to a heap-allocated JSON string * with `"ok":false` and an `"error"` payload. The caller must still release @@ -1154,7 +1571,7 @@ AdResult ad_version(char **out); * last-error slot is also set. * * On an argument or infrastructure failure (null adapter, null args, null out, - * off-main-thread, invalid UTF-8 field) `*out` is zeroed, the last-error slot + * invalid UTF-8 field) `*out` is zeroed, the last-error slot * is set, and a negative `AdResult` code is returned. No allocation is made. * * # Safety @@ -1167,14 +1584,40 @@ AdResult ad_version(char **out); * All `*const c_char` fields inside `AdWaitArgs` must be null or point to * readable, NUL-terminated memory within `AD_MAX_STRING_BYTES + 1` bytes. * - * `ad_wait` blocks the calling thread for up to `timeout_ms` milliseconds - * while it holds a live reference into the adapter's allocation. The adapter - * must outlive the call: do not call `ad_adapter_destroy` on this handle from - * another thread while `ad_wait` is running — that is a use-after-free. Ensure - * the wait has returned before destroying the adapter. + * `ad_wait` retains the adapter while blocked. Concurrent destruction revokes + * the opaque adapter token for new calls without invalidating this call. */ AdResult ad_wait(const struct AdAdapter *adapter, const struct AdWaitArgs *args, char **out); +/** + * Lists displays in screenshot screen-index order. + * + * # Safety + * `adapter` must be valid and `out` must be writable. Success produces an + * opaque list freed with `ad_display_list_free`. + */ +AdResult ad_list_displays(const struct AdAdapter *adapter, struct AdDisplayList **out); + +/** + * # Safety + * `list` must be null or returned by `ad_list_displays`. + */ +uint32_t ad_display_list_count(const struct AdDisplayList *list); + +/** + * Returns a borrowed display entry, or null when `index` is out of range. + * + * # Safety + * `list` must be null or returned by `ad_list_displays`. + */ +const struct AdDisplayInfo *ad_display_list_get(const struct AdDisplayList *list, uint32_t index); + +/** + * # Safety + * `list` must be null or returned by `ad_list_displays`. + */ +void ad_display_list_free(struct AdDisplayList *list); + /** * Last-error lifetime — errno-style. * @@ -1228,6 +1671,17 @@ const char *ad_last_error_platform_detail(void); */ const char *ad_last_error_details(void); +/** + * Writes the delivery and retry semantics associated with the calling + * thread's last error. If no error has been recorded, both values are + * `UNKNOWN`. This successful read does not clear or replace last-error state. + * + * # Safety + * + * `out` must point to writable `AdDeliverySemantics` storage. + */ +AdResult ad_last_error_delivery_semantics(struct AdDeliverySemantics *out); + /** * Reads the current clipboard text and writes an owned C string into * `*out`. The caller must free the returned pointer with @@ -1260,11 +1714,11 @@ AdResult ad_clear_clipboard(const struct AdAdapter *adapter); /** * Frees a C string previously returned by `ad_get_clipboard` or any * other FFI call documented as allocating a C string for the caller. - * Null-tolerant — safe to call on `NULL`. Double-free is undefined. + * Null-tolerant. Unknown pointers and repeated frees are ignored. * * # Safety - * `s` must be null or a pointer previously handed out by this crate. - * After this call the pointer is invalid and must not be used. + * `s` may be null or a pointer previously handed out by this crate. + * After a successful free the pointer is invalid and must not be used. */ void ad_free_string(char *s); @@ -1285,6 +1739,8 @@ AdResult ad_drag(const struct AdAdapter *adapter, const struct AdDragParams *par * at the given screen point. Click count is only consulted when `event.kind` * is `CLICK` (e.g., `click_count == 2` for a double-click). Callers that * need headless policy enforcement should use ref actions with policy. + * Carries no modifier chord — use [`ad_mouse_event_with_modifiers`] for + * meta/ctrl/alt/shift-held clicks. * * # Safety * `adapter` must be a non-null pointer returned by `ad_adapter_create`. @@ -1293,47 +1749,48 @@ AdResult ad_drag(const struct AdAdapter *adapter, const struct AdDragParams *par AdResult ad_mouse_event(const struct AdAdapter *adapter, const struct AdMouseEvent *event); /** - * Registers a callback to receive `tracing` events, or unregisters the - * current callback when `cb` is `NULL`. - * - * # Install semantics - * - * The subscriber layer is installed exactly once — on the first call with a - * non-null `cb`. If a foreign global subscriber already owns the process at - * that point, the install fails and this function returns - * `AD_RESULT_ERR_INTERNAL` with a diagnostic last-error. No callback pointer - * is stored in that case; events will never be delivered until the consumer - * remedies the conflict. Subsequent calls with a non-null `cb` after a - * **successful** install only swap the stored pointer and always return - * `AD_RESULT_OK`. - * - * `NULL` always returns `AD_RESULT_OK` — unregistering cannot fail. - * - * # Callback contract - * - * - `level` — 1 (ERROR) … 5 (TRACE) - * - `msg` — a NUL-terminated JSON string; valid only for the call's duration - * - * Sensitive field values (password, token, text, …) are replaced with - * `{"redacted":true}` before the message is formatted. - * - * Invocations are best-effort. A panicking callback is caught and silently - * discarded; no command fails because of a trace delivery error. A callback - * that emits `tracing` events is safe: the recursive `on_event` is dropped - * by a per-thread guard before it reaches the callback again. + * Additive counterpart to [`ad_mouse_event`] that also carries a held + * modifier chord (meta/ctrl/alt/shift) — e.g. Meta-click for additive + * selection, shift-click for range selection. `AdMouseEvent`'s layout is + * unchanged; modifiers travel as a separate array + count, mirroring + * `AdKeyCombo::modifiers`/`modifier_count`. * * # Safety - * - * `cb` must be null or a valid function pointer with the declared signature. - * The pointer is stored atomically; the subscriber may call it from threads - * other than the registering thread. - * - * A callback unregistered via `NULL` may still be invoked from another thread - * for a brief window after this call returns. The callback (and any data it - * captures) must remain valid for the process lifetime, or the caller must - * quiesce all tracing sources before unregistering. + * `adapter` must be a non-null pointer returned by `ad_adapter_create`. + * `event` must be a non-null pointer to a valid `AdMouseEvent`. + * `modifiers` must point to `modifier_count` valid `int32_t` values, or be + * null when `modifier_count` is 0. */ -AdResult ad_set_log_callback(void (*cb)(int32_t level, const char *msg)); +AdResult ad_mouse_event_with_modifiers(const struct AdAdapter *adapter, + const struct AdMouseEvent *event, + const int32_t *modifiers, + uint32_t modifier_count); + +/** + * Dispatches a physical wheel event using platform-neutral line deltas. + * Positive `delta_y` scrolls up and negative scrolls down; positive + * `delta_x` scrolls left and negative scrolls right. `modifier_mask` uses + * bits 0-3 for meta, ctrl, alt, and shift respectively. + * + * # Safety + * `adapter` must be a non-null pointer returned by `ad_adapter_create`. + */ +AdResult ad_mouse_wheel(const struct AdAdapter *adapter, + struct AdPoint point, + double delta_x, + double delta_y, + uint32_t modifier_mask); + +/** + * Registers or clears the callback used for events emitted synchronously + * inside later `ad_*` calls on the same thread. + * + * The callback may be invoked concurrently by different host threads. The + * message pointer is valid only until the callback returns. The callback must + * not unwind across this C ABI boundary; C++ exceptions and Rust panics must + * be caught inside the callback. Violating that contract may abort the host. + */ +AdResult ad_set_log_callback(void (*callback)(int32_t level, const char *msg)); /** * Triggers the named action on the notification at `index`. Typical @@ -1349,40 +1806,35 @@ AdResult ad_set_log_callback(void (*cb)(int32_t level, const char *msg)); * press the action button on a different notification than the host * intended. * - * `expected_app` and `expected_title` let the host pin the targeted - * notification to an observed fingerprint. If either pointer is - * non-null, the row currently at `index` must match that field or the - * call fails closed with `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`. Both - * null preserves the legacy index-only behavior for hosts that do - * their own reconciliation. + * `request.identity` pins the target to an observed fingerprint. At least one + * identity field is required; a mismatch fails closed with + * `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`. * * # Safety - * `adapter` must be valid. `action_name` must be a non-null UTF-8 - * C string. `expected_app` and `expected_title` must each be null - * or a NUL-terminated UTF-8 C string. Invalid UTF-8 in either field + * `adapter` and `request` must be valid. `request.action_name` must be a + * non-null UTF-8 C string. Identity fields must each be null or a + * NUL-terminated UTF-8 C string. Invalid UTF-8 in either field * is rejected with `AD_RESULT_ERR_INVALID_ARGS` rather than silently * treated as "no fingerprint". `out` must be a valid writable * `*mut AdActionResult`; on error it is zero-initialized. */ AdResult ad_notification_action(const struct AdAdapter *adapter, - uint32_t index, - const char *expected_app, - const char *expected_title, - const char *action_name, + const struct AdNotificationActionRequest *request, struct AdActionResult *out); /** - * Dismisses the notification at `index`. Indexes are only valid within - * the response to the most recent `ad_list_notifications` call on this - * thread — the adapter re-queries internally, so dismissing by a stale - * index returns `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`. + * Dismisses a notification only when the current row matches an identity + * observed in the same listing. At least one expected field is required. * * # Safety - * `adapter` must be valid. `app_filter` may be null. + * `adapter` must be valid. String pointers may be null and otherwise must be + * NUL-terminated UTF-8. */ AdResult ad_dismiss_notification(const struct AdAdapter *adapter, uint32_t index, - const char *app_filter); + const char *app_filter, + const char *expected_app, + const char *expected_title); /** * Dismisses every notification matching `app_filter` (null = all apps). @@ -1421,10 +1873,9 @@ void ad_dismiss_all_notifications_free(struct AdNotificationList *dismissed, /** * Lists the notifications currently on-screen. * - * Notification indexes are only stable within a single list response. - * Pass them straight to `ad_dismiss_notification` / - * `ad_notification_action` without caching across ticks — the adapter - * re-queries Notification Center internally on every call. + * Notification indexes are only stable within a single list response. Pass + * the entry's app or title fingerprint to the checked mutation functions; + * index-only mutations are rejected. * * # Safety * `adapter` must be valid. `filter` may be null. `out` must be a valid @@ -1459,21 +1910,9 @@ const struct AdNotificationInfo *ad_notification_list_get(const struct AdNotific void ad_notification_list_free(struct AdNotificationList *list); /** - * Finds the first element in `win`'s accessibility tree matching the - * query and resolves it to an opaque `AdNativeHandle`. The caller owns - * the handle and must release it with `ad_free_handle(adapter, handle)` - * once done. - * - * Matching is DFS order, first hit wins. All query fields are optional - * (null = "don't care") and case-insensitive substring matches: - * - `role` against `AccessibilityNode.role` - * - `name_substring` against `AccessibilityNode.name` - * - `value_substring` against `AccessibilityNode.value` - * - * The internal tree fetch always sets `include_bounds: true` so - * `resolve_element_strict` can disambiguate duplicate-label siblings via - * `bounds_hash`; without bounds on the matched node the resolver falls - * back to role+name alone and may pick the wrong element. + * Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process + * generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. + * Use `ad_find_exact`. * * # Safety * `adapter`, `win`, and `query` must be valid pointers. `out_handle` @@ -1485,6 +1924,21 @@ AdResult ad_find(const struct AdAdapter *adapter, const struct AdFindQuery *query, struct AdNativeHandle *out_handle); +/** + * Finds and strictly resolves one element within a generation-pinned window. + * `AdFindQuery.control.selection` must explicitly request first, last, or nth + * behavior when duplicate matches are acceptable. The returned native handle + * is adapter-bound and thread-affine; release it with `ad_free_handle` on the + * resolving thread. + * + * # Safety + * All pointers must be valid and `out_handle` must be writable. + */ +AdResult ad_find_exact(const struct AdAdapter *adapter, + const struct AdExactWindowInfo *win, + const struct AdFindQuery *query, + struct AdNativeHandle *out_handle); + /** * Reads a single property off a previously-resolved element handle. * @@ -1508,35 +1962,13 @@ AdResult ad_get(const struct AdAdapter *adapter, char **out); /** - * Checks whether a named boolean state is set on the first element - * matching `query` inside `win`'s accessibility tree. Intended for - * the common agent idiom `find → is("focused") → if yes, act`. - * - * Supported property names reflect the strings the macOS tree - * builder actually emits in `AccessibilityNode.states`: - * - * - `"focused"` — true when the node carries the `focused` state. - * - `"disabled"` — true when the adapter surfaced `disabled`. - * - `"enabled"` — derived: true iff `disabled` is NOT present. There - * is no `enabled` string in the adapter output; asking for it - * returns the logical negation so agents don't have to invert - * themselves. - * - * `"selected"`, `"checked"`, and `"expanded"` are not currently - * emitted by any platform adapter; asking for them returns - * `AD_RESULT_ERR_INVALID_ARGS` with a diagnostic last-error rather - * than silently answering `false`. The set will widen as adapters - * grow support; future additions stay backwards-compatible - * (unknown → InvalidArgs, known → deterministic answer). - * - * On entry `*out` is always cleared to `false` so a caller inspecting - * the slot after an error sees a predictable sentinel, not whatever - * was there before. If the query matches nothing, returns - * `AD_RESULT_ERR_ELEMENT_NOT_FOUND` with `*out` still `false`. + * Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process + * generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. + * Use `ad_is_exact`. * * # Safety - * All pointers must be valid. `property` must be a non-null UTF-8 - * C string. `out` must be a valid writable `*mut bool`. + * All pointers must be valid. `property` must be a non-null UTF-8 C string. + * `out` must be a valid writable `*mut bool`. */ AdResult ad_is(const struct AdAdapter *adapter, const struct AdWindowInfo *win, @@ -1544,6 +1976,22 @@ AdResult ad_is(const struct AdAdapter *adapter, const char *property, bool *out); +/** + * Checks a boolean state within a generation-pinned exact window. + * + * # Safety + * All pointers must be valid and `out` must be writable. + */ +AdResult ad_is_exact(const struct AdAdapter *adapter, + const struct AdExactWindowInfo *win, + const struct AdFindQuery *query, + const char *property, + bool *out); + +#if defined(AGENT_DESKTOP_TEST_PANIC_INJECTION) +AdResult ad_test_panic_boundary(void); +#endif + /** * Borrowed pointer to the image bytes; valid until the buffer is freed. * Returns null if `buf` is null. @@ -1587,6 +2035,14 @@ uint32_t ad_image_buffer_height(const struct AdImageBuffer *buf); */ AdImageFormat ad_image_buffer_format(const struct AdImageBuffer *buf); +/** + * Point-to-pixel scale factor for the captured display or window. + * + * # Safety + * `buf` must be null or returned by `ad_screenshot`. + */ +double ad_image_buffer_scale_factor(const struct AdImageBuffer *buf); + /** * Allocates and returns an opaque `AdImageBuffer`. The handle owns its * byte buffer; inspect it through `ad_image_buffer_data` / @@ -1602,6 +2058,17 @@ AdResult ad_screenshot(const struct AdAdapter *adapter, const struct AdScreenshotTarget *target, struct AdImageBuffer **out); +/** + * Captures one generation-pinned exact window. + * + * # Safety + * `adapter`, `window`, and `out` must be valid pointers. The returned image + * must be freed with `ad_image_buffer_free`. + */ +AdResult ad_screenshot_window_exact(const struct AdAdapter *adapter, + const struct AdExactWindowInfo *window, + struct AdImageBuffer **out); + /** * Frees the image buffer allocated by `ad_screenshot`. * @@ -1617,7 +2084,9 @@ void ad_image_buffer_free(struct AdImageBuffer *buf); * `*mut *mut AdSurfaceList`. Success produces a list handle freed via * `ad_surface_list_free`. */ -AdResult ad_list_surfaces(const struct AdAdapter *adapter, int32_t pid, struct AdSurfaceList **out); +AdResult ad_list_surfaces(const struct AdAdapter *adapter, + uint32_t pid, + struct AdSurfaceList **out); /** * # Safety @@ -1641,6 +2110,37 @@ const struct AdSurfaceInfo *ad_surface_list_get(const struct AdSurfaceList *list */ void ad_surface_list_free(struct AdSurfaceList *list); +/** + * Lists surfaces without dropping their core surface IDs. + * + * # Safety + * `adapter` and `out` must be valid. The returned list must be freed with + * `ad_exact_surface_list_free`. + */ +AdResult ad_list_surfaces_exact(const struct AdAdapter *adapter, + uint32_t pid, + struct AdExactSurfaceList **out); + +/** + * # Safety + * `list` must be null or returned by `ad_list_surfaces_exact`. + */ +uint32_t ad_exact_surface_list_count(const struct AdExactSurfaceList *list); + +/** + * # Safety + * `list` must be null or returned by `ad_list_surfaces_exact`. The result is + * borrowed until the list is freed. + */ +const struct AdExactSurfaceInfo *ad_exact_surface_list_get(const struct AdExactSurfaceList *list, + uint32_t index); + +/** + * # Safety + * `list` must be null or returned by `ad_list_surfaces_exact`. + */ +void ad_exact_surface_list_free(struct AdExactSurfaceList *list); + /** * # Safety * `tree` must be null or point to a valid `AdNodeTree` previously returned @@ -1649,67 +2149,56 @@ void ad_surface_list_free(struct AdSurfaceList *list); void ad_free_tree(struct AdNodeTree *tree); /** - * Snapshots `win`'s accessibility tree into the flat BFS layout - * described in the types module. The result is written into `*out` - * and must be freed with `ad_free_tree`. Direct children of any node - * live contiguously at `nodes[child_start..child_start + child_count]`. - * - * `opts.max_depth` caps tree depth. `opts.surface` selects which - * surface to snapshot (window body, menu, menubar, sheet, popover, - * alert, or focused subtree); see `AdSnapshotSurface`. - * `opts.interactive_only` prunes non-interactive nodes; `opts.compact` - * collapses containers with no semantic payload. - * - * # Raw-tree contract - * - * This is a **raw adapter tree** — ref-less, no refmap persistence, and - * no JSON envelope. Differences the caller must know about: - * - * - `ref_id` is always null on every `AdNode`. `ref_alloc::allocate_refs` - * is not run; `@e` ref assignment is a snapshot-pipeline concern. - * - `include_bounds`, `interactive_only`, and `compact` are honoured via - * `ref_alloc::transform_tree` after the adapter returns. Because refs are - * not allocated, the `interactive_only` cut is role-based rather than - * ref-based; otherwise the semantics match the snapshot path. - * - No skeleton/drill-down pipeline is wired through — `skeleton` is - * always false on the underlying `TreeOptions`. - * - * # When to use this function vs `ad_snapshot` - * - * **Observe–act agents** that need `@e` refs and refmap persistence should - * call `ad_snapshot` instead. `ad_snapshot` runs the full snapshot pipeline - * (ref allocation, refmap write to disk, JSON envelope with - * `{"version":"2.0","ok":true,...}`) and is the correct starting point for - * any workflow that drives subsequent ref-based actions via - * `ad_execute_by_ref` (with an `AdAction`). - * - * Use `ad_get_tree` when you need the raw flat BFS layout without refs — - * for example, to drive your own traversal logic or to populate a UI - * inspector that does not use the ref-based action API. For point lookups - * that bypass tree shape entirely, `ad_find` + `ad_get` / `ad_is` are - * another alternative. - * - * On error `*out` is zeroed so `ad_free_tree` on it is a safe no-op. + * Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process + * generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. + * Use `ad_get_tree_exact`. * * # Safety - * All pointers must be non-null. `win.id` and `win.title` must be - * valid UTF-8 C strings. `out` must be writable. + * All pointers must be non-null and `out` must be writable. */ AdResult ad_get_tree(const struct AdAdapter *adapter, const struct AdWindowInfo *win, const struct AdTreeOptions *opts, struct AdNodeTree *out); +/** + * Snapshots a generation-pinned window into the flat, owned, breadth-first C + * tree layout. Direct children are contiguous at + * `nodes[child_start..child_start + child_count]`; free the result with + * `ad_free_tree`. + * + * This is a raw adapter tree: nodes do not receive refs, no refmap is + * persisted, and no JSON envelope is produced. `max_depth`, `surface`, + * `include_bounds`, `interactive_only`, and `compact` are applied; skeleton + * and drill-down behavior are not. Use `ad_snapshot` for the canonical + * observe-act workflow with snapshot-qualified refs. + * + * # Safety + * All pointers must be valid and `out` must be writable. + */ +AdResult ad_get_tree_exact(const struct AdAdapter *adapter, + const struct AdExactWindowInfo *win, + const struct AdTreeOptions *opts, + struct AdNodeTree *out); + size_t ad_action_size(void); size_t ad_action_result_size(void); size_t ad_action_step_size(void); +size_t ad_display_info_size(void); + size_t ad_drag_params_size(void); size_t ad_element_state_size(void); +size_t ad_exact_ref_entry_size(void); + +size_t ad_exact_surface_info_size(void); + +size_t ad_exact_window_info_size(void); + size_t ad_ref_entry_size(void); /** @@ -1720,17 +2209,26 @@ size_t ad_ref_entry_size(void); size_t ad_wait_args_size(void); /** - * Brings `win` to the foreground on the current space. Returns - * `AD_RESULT_ERR_WINDOW_NOT_FOUND` when the referenced window no longer - * exists (the caller should re-list and retry). + * Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process + * generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. + * Use `ad_focus_window_exact`. * * # Safety * `adapter` must be a non-null pointer from `ad_adapter_create`. `win` - * must be a non-null pointer to an `AdWindowInfo` whose `id` and - * `title` fields are non-null, valid UTF-8 C strings. + * must be a non-null pointer to an `AdWindowInfo`. */ AdResult ad_focus_window(const struct AdAdapter *adapter, const struct AdWindowInfo *win); +/** + * Focuses a generation-pinned exact window. + * + * # Safety + * `adapter` and `win` must be valid pointers. `win` must carry the current + * exact-window version and size. + */ +AdResult ad_focus_window_exact(const struct AdAdapter *adapter, + const struct AdExactWindowInfo *win); + /** * Releases the heap-allocated string fields (`id`, `title`, `app_name`) * inside a single `AdWindowInfo` previously written by `ad_launch_app` @@ -1749,6 +2247,14 @@ AdResult ad_focus_window(const struct AdAdapter *adapter, const struct AdWindowI */ void ad_release_window_fields(struct AdWindowInfo *win); +/** + * Releases every owned string inside one exact window value. + * + * # Safety + * `win` must be null or point to a value written by `ad_launch_app_exact`. + */ +void ad_release_exact_window_fields(struct AdExactWindowInfo *win); + /** * # Safety * `adapter` must be valid. `out` must be a valid writable @@ -1783,21 +2289,66 @@ const struct AdWindowInfo *ad_window_list_get(const struct AdWindowList *list, u void ad_window_list_free(struct AdWindowList *list); /** - * Performs a window-manager operation (`Resize`, `Move`, `Minimize`, - * `Maximize`, `Restore`) on `win`. Width / height / x / y are consulted - * only for the variants that use them; other kinds ignore them. - * - * An invalid `op.kind` discriminant is rejected with - * `AD_RESULT_ERR_INVALID_ARGS` before any adapter call. + * Lists windows with explicit process-generation evidence. * * # Safety - * `adapter` and `win` must be non-null pointers. `win.id` and - * `win.title` must be non-null valid UTF-8 C strings. + * `adapter` and `out` must be valid. `app_filter` may be null or a valid + * bounded UTF-8 C string. The returned list must be freed with + * `ad_exact_window_list_free`. + */ +AdResult ad_list_windows_exact(const struct AdAdapter *adapter, + const char *app_filter, + bool focused_only, + struct AdExactWindowList **out); + +/** + * # Safety + * `list` must be null or returned by `ad_list_windows_exact`. + */ +uint32_t ad_exact_window_list_count(const struct AdExactWindowList *list); + +/** + * # Safety + * `list` must be null or returned by `ad_list_windows_exact`. The returned + * pointer is borrowed until the list is freed. + */ +const struct AdExactWindowInfo *ad_exact_window_list_get(const struct AdExactWindowList *list, + uint32_t index); + +/** + * # Safety + * `list` must be null or returned by `ad_list_windows_exact`. + */ +void ad_exact_window_list_free(struct AdExactWindowList *list); + +/** + * Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process + * generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. + * Use `ad_window_op_exact`. + * + * # Safety + * `adapter` and `win` must be non-null pointers. */ AdResult ad_window_op(const struct AdAdapter *adapter, const struct AdWindowInfo *win, struct AdWindowOp op); +/** + * Performs a window-manager operation against an exact generation-pinned + * window identity. + * + * # Safety + * `adapter` and `win` must be valid pointers. `win` must carry the current + * exact-window version and size. + */ +AdResult ad_window_op_exact(const struct AdAdapter *adapter, + const struct AdExactWindowInfo *win, + struct AdWindowOp op); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + #endif /* AGENT_DESKTOP_H */ /* C11 ABI layout guards — auto-generated; do not hand-edit. @@ -1818,6 +2369,10 @@ _Static_assert(sizeof(AdActionStep) == AD_ACTION_STEP_SIZE, "AdActionStep ABI si _Static_assert(_Alignof(AdActionStep) == 8, "AdActionStep ABI alignment changed"); _Static_assert(offsetof(AdActionStep, label) == 0, "AdActionStep.label offset changed"); _Static_assert(offsetof(AdActionStep, outcome) == 8, "AdActionStep.outcome offset changed"); +_Static_assert(offsetof(AdActionStep, mechanism) == 16, "AdActionStep.mechanism offset changed"); +_Static_assert(offsetof(AdActionStep, has_mechanism) == 20, "AdActionStep.has_mechanism offset changed"); +_Static_assert(offsetof(AdActionStep, verified) == 21, "AdActionStep.verified offset changed"); +_Static_assert(offsetof(AdActionStep, has_verified) == 22, "AdActionStep.has_verified offset changed"); _Static_assert(sizeof(AdActionResult) == AD_ACTION_RESULT_SIZE, "AdActionResult ABI size changed"); _Static_assert(_Alignof(AdActionResult) == 8, "AdActionResult ABI alignment changed"); _Static_assert(offsetof(AdActionResult, action) == 0, "AdActionResult.action offset changed"); @@ -1825,9 +2380,86 @@ _Static_assert(offsetof(AdActionResult, ref_id) == 8, "AdActionResult.ref_id off _Static_assert(offsetof(AdActionResult, post_state) == 16, "AdActionResult.post_state offset changed"); _Static_assert(offsetof(AdActionResult, steps) == 24, "AdActionResult.steps offset changed"); _Static_assert(offsetof(AdActionResult, step_count) == 32, "AdActionResult.step_count offset changed"); +_Static_assert(offsetof(AdActionResult, details_json) == 40, "AdActionResult.details_json offset changed"); +_Static_assert(offsetof(AdActionResult, disposition) == 48, "AdActionResult.disposition offset changed"); +_Static_assert(sizeof(AdDeliverySemantics) == AD_DELIVERY_SEMANTICS_SIZE, "AdDeliverySemantics ABI size changed"); +_Static_assert(offsetof(AdDeliverySemantics, retry) == 4, "AdDeliverySemantics.retry offset changed"); _Static_assert(sizeof(AdRefEntry) == AD_REF_ENTRY_SIZE, "AdRefEntry ABI size changed"); _Static_assert(_Alignof(AdRefEntry) == 8, "AdRefEntry ABI alignment changed"); +_Static_assert(offsetof(AdRefEntry, process) == 0, "AdRefEntry.process offset changed"); +_Static_assert(offsetof(AdRefEntry, identity) == 8, "AdRefEntry.identity offset changed"); +_Static_assert(offsetof(AdRefEntry, geometry) == 48, "AdRefEntry.geometry offset changed"); +_Static_assert(offsetof(AdRefEntry, capabilities) == 96, "AdRefEntry.capabilities offset changed"); +_Static_assert(offsetof(AdRefEntry, source) == 128, "AdRefEntry.source offset changed"); +_Static_assert(offsetof(AdRefEntry, scope) == 168, "AdRefEntry.scope offset changed"); +_Static_assert(sizeof(AdExactRefEntry) == AD_EXACT_REF_ENTRY_SIZE, "AdExactRefEntry ABI size changed"); +_Static_assert(_Alignof(AdExactRefEntry) == 8, "AdExactRefEntry ABI alignment changed"); +_Static_assert(offsetof(AdExactRefEntry, version) == 0, "AdExactRefEntry.version offset changed"); +_Static_assert(offsetof(AdExactRefEntry, size) == 4, "AdExactRefEntry.size offset changed"); +_Static_assert(offsetof(AdExactRefEntry, entry) == 8, "AdExactRefEntry.entry offset changed"); +_Static_assert(offsetof(AdExactRefEntry, process_instance) == 208, "AdExactRefEntry.process_instance offset changed"); +_Static_assert(offsetof(AdExactRefEntry, identifier_kind) == 216, "AdExactRefEntry.identifier_kind offset changed"); +_Static_assert(sizeof(AdExactWindowInfo) == AD_EXACT_WINDOW_INFO_SIZE, "AdExactWindowInfo ABI size changed"); +_Static_assert(_Alignof(AdExactWindowInfo) == 8, "AdExactWindowInfo ABI alignment changed"); +_Static_assert(offsetof(AdExactWindowInfo, version) == 0, "AdExactWindowInfo.version offset changed"); +_Static_assert(offsetof(AdExactWindowInfo, size) == 4, "AdExactWindowInfo.size offset changed"); +_Static_assert(offsetof(AdExactWindowInfo, window) == 8, "AdExactWindowInfo.window offset changed"); +_Static_assert(offsetof(AdExactWindowInfo, process_instance) == 80, "AdExactWindowInfo.process_instance offset changed"); +_Static_assert(sizeof(AdExactSurfaceInfo) == AD_EXACT_SURFACE_INFO_SIZE, "AdExactSurfaceInfo ABI size changed"); +_Static_assert(_Alignof(AdExactSurfaceInfo) == 8, "AdExactSurfaceInfo ABI alignment changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, version) == 0, "AdExactSurfaceInfo.version offset changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, size) == 4, "AdExactSurfaceInfo.size offset changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, id) == 8, "AdExactSurfaceInfo.id offset changed"); +_Static_assert(offsetof(AdExactSurfaceInfo, surface) == 16, "AdExactSurfaceInfo.surface offset changed"); +_Static_assert(sizeof(AdDisplayInfo) == AD_DISPLAY_INFO_SIZE, "AdDisplayInfo ABI size changed"); +_Static_assert(_Alignof(AdDisplayInfo) == 8, "AdDisplayInfo ABI alignment changed"); +_Static_assert(offsetof(AdDisplayInfo, version) == 0, "AdDisplayInfo.version offset changed"); +_Static_assert(offsetof(AdDisplayInfo, size) == 4, "AdDisplayInfo.size offset changed"); +_Static_assert(offsetof(AdDisplayInfo, id) == 8, "AdDisplayInfo.id offset changed"); +_Static_assert(offsetof(AdDisplayInfo, bounds) == 16, "AdDisplayInfo.bounds offset changed"); +_Static_assert(offsetof(AdDisplayInfo, is_primary) == 48, "AdDisplayInfo.is_primary offset changed"); +_Static_assert(offsetof(AdDisplayInfo, scale) == 56, "AdDisplayInfo.scale offset changed"); +_Static_assert(sizeof(AdRefProcess) == AD_REF_PROCESS_SIZE, "AdRefProcess ABI size changed"); +_Static_assert(sizeof(AdRefIdentity) == AD_REF_IDENTITY_SIZE, "AdRefIdentity ABI size changed"); +_Static_assert(offsetof(AdRefIdentity, native_id) == 32, "AdRefIdentity.native_id offset changed"); +_Static_assert(sizeof(AdStringSlice) == AD_STRING_SLICE_SIZE, "AdStringSlice ABI size changed"); +_Static_assert(sizeof(AdRefCapabilities) == AD_REF_CAPABILITIES_SIZE, "AdRefCapabilities ABI size changed"); +_Static_assert(sizeof(AdRefGeometry) == AD_REF_GEOMETRY_SIZE, "AdRefGeometry ABI size changed"); +_Static_assert(offsetof(AdRefGeometry, bounds_hash) == 32, "AdRefGeometry.bounds_hash offset changed"); +_Static_assert(sizeof(AdRefSource) == AD_REF_SOURCE_SIZE, "AdRefSource ABI size changed"); +_Static_assert(offsetof(AdRefSource, window_bounds_hash) == 24, "AdRefSource.window_bounds_hash offset changed"); +_Static_assert(sizeof(AdRefScope) == AD_REF_SCOPE_SIZE, "AdRefScope ABI size changed"); +_Static_assert(offsetof(AdRefScope, path) == 8, "AdRefScope.path offset changed"); _Static_assert(sizeof(struct AdWaitArgs) == AD_WAIT_ARGS_SIZE, "AdWaitArgs ABI size drift"); _Static_assert(_Alignof(struct AdWaitArgs) == 8, "AdWaitArgs ABI alignment changed"); +_Static_assert(offsetof(AdWaitArgs, mode) == 0, "AdWaitArgs.mode offset changed"); +_Static_assert(offsetof(AdWaitArgs, predicate) == 48, "AdWaitArgs.predicate offset changed"); +_Static_assert(offsetof(AdWaitArgs, scope) == 96, "AdWaitArgs.scope offset changed"); +_Static_assert(sizeof(AdOptionalU64) == AD_OPTIONAL_U64_SIZE, "AdOptionalU64 ABI size changed"); +_Static_assert(sizeof(AdOptionalUsize) == AD_OPTIONAL_USIZE_SIZE, "AdOptionalUsize ABI size changed"); +_Static_assert(sizeof(AdWaitSurfaceModes) == AD_WAIT_SURFACE_MODES_SIZE, "AdWaitSurfaceModes ABI size changed"); +_Static_assert(sizeof(AdWaitMode) == AD_WAIT_MODE_SIZE, "AdWaitMode ABI size changed"); +_Static_assert(sizeof(AdWaitPredicate) == AD_WAIT_PREDICATE_SIZE, "AdWaitPredicate ABI size changed"); +_Static_assert(sizeof(AdWaitScope) == AD_WAIT_SCOPE_SIZE, "AdWaitScope ABI size changed"); +_Static_assert(sizeof(AdNode) == AD_NODE_SIZE, "AdNode ABI size changed"); +_Static_assert(offsetof(AdNode, content) == 0, "AdNode.content offset changed"); +_Static_assert(offsetof(AdNode, presentation) == 48, "AdNode.presentation offset changed"); +_Static_assert(offsetof(AdNode, relation) == 96, "AdNode.relation offset changed"); +_Static_assert(sizeof(AdNodeContent) == AD_NODE_CONTENT_SIZE, "AdNodeContent ABI size changed"); +_Static_assert(sizeof(AdNodePresentation) == AD_NODE_PRESENTATION_SIZE, "AdNodePresentation ABI size changed"); +_Static_assert(sizeof(AdNodeRelation) == AD_NODE_RELATION_SIZE, "AdNodeRelation ABI size changed"); +_Static_assert(sizeof(AdNotificationIdentity) == AD_NOTIFICATION_IDENTITY_SIZE, "AdNotificationIdentity ABI size changed"); +_Static_assert(sizeof(AdNotificationActionRequest) == AD_NOTIFICATION_ACTION_REQUEST_SIZE, "AdNotificationActionRequest ABI size changed"); +_Static_assert(offsetof(AdNotificationActionRequest, identity) == 16, "AdNotificationActionRequest.identity offset changed"); +_Static_assert(sizeof(AdFindQuery) == AD_FIND_QUERY_SIZE, "AdFindQuery ABI size changed"); +_Static_assert(offsetof(AdFindQuery, filter) == 24, "AdFindQuery.filter offset changed"); +_Static_assert(sizeof(AdFindControl) == AD_FIND_CONTROL_SIZE, "AdFindControl ABI size changed"); +_Static_assert(offsetof(AdFindControl, timeout_ms) == 16, "AdFindControl.timeout_ms offset changed"); +_Static_assert(sizeof(AdFindSelection) == AD_FIND_SELECTION_SIZE, "AdFindSelection ABI size changed"); +_Static_assert(sizeof(AdFindIdentity) == AD_FIND_IDENTITY_SIZE, "AdFindIdentity ABI size changed"); +_Static_assert(sizeof(AdFindStatePredicate) == AD_FIND_STATE_PREDICATE_SIZE, "AdFindStatePredicate ABI size changed"); +_Static_assert(sizeof(AdFindStateSlice) == AD_FIND_STATE_SLICE_SIZE, "AdFindStateSlice ABI size changed"); +_Static_assert(sizeof(AdFindFilter) == AD_FIND_FILTER_SIZE, "AdFindFilter ABI size changed"); +_Static_assert(offsetof(AdFindFilter, exact) == 80, "AdFindFilter.exact offset changed"); #endif /* __STDC_VERSION__ >= 201112L */ #endif /* AGENT_DESKTOP_ABI_ASSERTS */ diff --git a/crates/ffi/src/abi_version.rs b/crates/ffi/src/abi_version.rs index c2c50ee..fa631b2 100644 --- a/crates/ffi/src/abi_version.rs +++ b/crates/ffi/src/abi_version.rs @@ -12,7 +12,7 @@ use std::ffi::CStr; /// with the major compiled against the header to verify ABI compatibility; a /// mismatch means the header and dylib are incompatible and the consumer should /// refuse to proceed rather than risk undefined behaviour. -pub const AD_ABI_VERSION_MAJOR: u32 = 1; +pub const AD_ABI_VERSION_MAJOR: u32 = 3; static MISMATCH_MESSAGE: &CStr = c"ABI major version mismatch: recompile against the installed header"; @@ -48,3 +48,7 @@ pub extern "C" fn ad_init(expected_major: u32) -> AdResult { } }) } + +#[cfg(test)] +#[path = "abi_version_tests.rs"] +mod tests; diff --git a/crates/ffi/src/abi_version_tests.rs b/crates/ffi/src/abi_version_tests.rs new file mode 100644 index 0000000..ddd0ab8 --- /dev/null +++ b/crates/ffi/src/abi_version_tests.rs @@ -0,0 +1,24 @@ +use super::*; + +/// ABI 2 shipped the expanded result/ref layouts. Replacing the three-argument +/// `ad_dismiss_notification` with the identity-required five-argument contract +/// breaks callers compiled against ABI 2, so this branch must advertise ABI 3. +#[test] +fn abi_major_covers_the_checked_dismiss_signature() { + let major = std::hint::black_box(AD_ABI_VERSION_MAJOR); + assert!( + major >= 3, + "identity-required ad_dismiss_notification is incompatible with ABI 2" + ); +} + +#[test] +fn ad_init_accepts_current_major_and_rejects_stale_major() { + assert_eq!(ad_init(AD_ABI_VERSION_MAJOR), AdResult::Ok); + assert_eq!(ad_init(AD_ABI_VERSION_MAJOR - 1), AdResult::ErrInvalidArgs); +} + +#[test] +fn ad_abi_version_reports_the_compiled_major() { + assert_eq!(ad_abi_version(), AD_ABI_VERSION_MAJOR); +} diff --git a/crates/ffi/src/actions/conversion.rs b/crates/ffi/src/actions/conversion.rs index 8d35bfa..82fa4db 100644 --- a/crates/ffi/src/actions/conversion.rs +++ b/crates/ffi/src/actions/conversion.rs @@ -1,8 +1,8 @@ use crate::convert::string::c_to_string; use crate::types::{AdAction, AdActionKind, AdDirection, AdKeyCombo, AdModifier}; -use agent_desktop_core::action::{Action, Direction, KeyCombo as CoreKeyCombo, Modifier}; +use agent_desktop_core::{Action, Direction, KeyCombo as CoreKeyCombo, Modifier}; -/// Four modifier keys exist (`AdModifier::{Cmd, Ctrl, Alt, Shift}`), +/// Four modifier keys exist (`AdModifier::{Meta, Ctrl, Alt, Shift}`), /// so a combo can name at most four. Anything larger must be bogus /// input — bail out instead of trusting it into `from_raw_parts`. const MAX_MODIFIERS_PER_COMBO: u32 = 4; @@ -24,7 +24,7 @@ pub(crate) unsafe fn key_combo_from_c(k: &AdKeyCombo) -> Result<CoreKeyCombo, &' for raw_modifier in slice { let m = AdModifier::from_c(*raw_modifier).ok_or("invalid modifier discriminant")?; let modifier = match m { - AdModifier::Cmd => Modifier::Cmd, + AdModifier::Meta => Modifier::Meta, AdModifier::Ctrl => Modifier::Ctrl, AdModifier::Alt => Modifier::Alt, AdModifier::Shift => Modifier::Shift, @@ -97,7 +97,7 @@ pub(crate) unsafe fn action_from_c(action: &AdAction) -> Result<Action, &'static mod tests { use super::*; use crate::convert::string::{free_c_string, string_to_c}; - use crate::types::{AdDragParams, AdPoint, AdScrollParams}; + use crate::{AdDragParams, AdPoint, AdScrollParams}; use std::ptr; fn make_scroll_params() -> AdScrollParams { @@ -200,7 +200,7 @@ mod tests { #[test] fn key_combo_accepts_valid_modifier_slice() { let key = string_to_c("s"); - let mods: [i32; 2] = [AdModifier::Cmd as i32, AdModifier::Shift as i32]; + let mods: [i32; 2] = [AdModifier::Meta as i32, AdModifier::Shift as i32]; let combo = AdKeyCombo { key, modifiers: mods.as_ptr(), diff --git a/crates/ffi/src/actions/execute.rs b/crates/ffi/src/actions/execute.rs index e4ea888..2730e89 100644 --- a/crates/ffi/src/actions/execute.rs +++ b/crates/ffi/src/actions/execute.rs @@ -4,8 +4,10 @@ use crate::actions::result::action_result_to_c; use crate::commands::app_error_to_adapter; use crate::error::{self, AdResult}; use crate::ffi_try::trap_panic; -use crate::types::{AdAction, AdActionResult, AdNativeHandle, AdPolicyKind, AdRefEntry}; -use agent_desktop_core::{action::Action, action_request::ActionRequest, adapter::NativeHandle}; +use crate::types::{ + AdAction, AdActionResult, AdExactRefEntry, AdNativeHandle, AdPolicyKind, AdRefEntry, +}; +use agent_desktop_core::{Action, ActionRequest}; /// Low-level native-handle action. Dispatches directly to the platform adapter /// without strict ref re-identification or actionability preflight. This is a @@ -20,6 +22,10 @@ use agent_desktop_core::{action::Action, action_request::ActionRequest, adapter: /// the same live adapter. Free the handle before destroying that adapter. /// `action` must be a non-null pointer to a valid `AdAction`. /// `out` must be a non-null pointer to an `AdActionResult` to write the result into. +/// +/// Handles come from exact resolvers and already carry process-generation +/// evidence, so this executes under the same policy as +/// `ad_execute_action_with_policy`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_execute_action( adapter: *const AdAdapter, @@ -58,17 +64,13 @@ pub unsafe extern "C" fn ad_execute_action_with_policy( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = std::mem::zeroed(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(handle, c"handle is null"); crate::pointer_guard::guard_non_null!(action, c"action is null"); - let adapter = &*adapter; let handle_ref = &*handle; if handle_ref.ptr.is_null() { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, "handle.ptr is null — the handle has already been freed or was never resolved", )); return AdResult::ErrInvalidArgs; @@ -77,13 +79,32 @@ pub unsafe extern "C" fn ad_execute_action_with_policy( Ok(action) => action, Err(result) => return result, }; - let native_handle = NativeHandle::from_ptr(handle_ref.ptr); let policy = match decode_policy(policy) { Ok(policy) => policy, Err(result) => return result, }; - let request = action_request(policy, core_action); - match adapter.inner.execute_action(&native_handle, request) { + let adapter_id = adapter.addr(); + let adapter = crate::adapter::acquire_adapter!(adapter); + let (native_handle, process) = + match crate::actions::native_handle::acquire_ffi_handle(adapter_id, handle_ref) { + Ok(handle) => handle, + Err(error) => { + error::set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let request = action_request(policy, core_action).with_expected_process(process); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + if let Action::Drag(params) = &request.action + && let Err(error) = params.validate(lease.deadline()) + { + error::set_last_error(&error); + return error::last_error_code(); + } + match adapter + .inner + .execute_action(native_handle.as_ref(), request, &lease) + { Ok(result) => { *out = action_result_to_c(&result); AdResult::Ok @@ -128,13 +149,9 @@ pub unsafe extern "C" fn ad_execute_ref_action_with_policy( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = std::mem::zeroed(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(entry, c"entry is null"); crate::pointer_guard::guard_non_null!(action, c"action is null"); - let adapter_ref = &*adapter; let entry_ref = &*entry; let core_entry = match crate::actions::resolve::core_ref_entry_from_ffi(entry_ref) { Ok(entry) => entry, @@ -143,44 +160,87 @@ pub unsafe extern "C" fn ad_execute_ref_action_with_policy( return error::last_error_code(); } }; - let core_action = match decode_action(&*action) { - Ok(action) => action, - Err(result) => return result, - }; - let policy = match decode_policy(policy) { - Ok(policy) => policy, - Err(result) => return result, - }; - let request = action_request(policy, core_action); - let context = match adapter_ref.command_context() { - Ok(ctx) => ctx, - Err(err) => { - error::set_last_error(&app_error_to_adapter(err)); + execute_core_ref_action(adapter, core_entry, &*action, policy, out) + }) +} + +/// Executes a struct-based ref action with exact process-generation and typed +/// native-id evidence. +/// +/// # Safety +/// +/// All pointers must be valid. `entry` must carry the current exact-entry +/// version and size. `out` is zeroed before any fallible operation. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_execute_ref_action_exact_with_policy( + adapter: *const AdAdapter, + entry: *const AdExactRefEntry, + action: *const AdAction, + policy: i32, + out: *mut AdActionResult, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = std::mem::zeroed(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(entry, c"entry is null"); + crate::pointer_guard::guard_non_null!(action, c"action is null"); + let core_entry = match crate::actions::resolve::core_ref_entry_from_exact(&*entry) { + Ok(entry) => entry, + Err(error) => { + error::set_last_error(&error); return error::last_error_code(); } }; - match agent_desktop_core::ref_action::execute_entry_with_context( - adapter_ref.inner.as_ref(), - &core_entry, - request, - &context, - ) { - Ok(result) => { - *out = action_result_to_c(&result); - AdResult::Ok - } - Err(err) => { - error::set_last_error(&err); - error::last_error_code() - } - } + execute_core_ref_action(adapter, core_entry, &*action, policy, out) }) } +unsafe fn execute_core_ref_action( + adapter: *const AdAdapter, + entry: agent_desktop_core::RefEntry, + action: &AdAction, + policy: i32, + out: *mut AdActionResult, +) -> AdResult { + let core_action = match decode_action(action) { + Ok(action) => action, + Err(result) => return result, + }; + let policy = match decode_policy(policy) { + Ok(policy) => policy, + Err(result) => return result, + }; + let adapter_ref = crate::adapter::acquire_adapter!(adapter); + let request = action_request(policy, core_action); + let context = match adapter_ref.command_context() { + Ok(context) => context, + Err(error) => { + error::set_last_error(&app_error_to_adapter(error)); + return error::last_error_code(); + } + }; + match agent_desktop_core::ref_action::execute_entry_with_context( + adapter_ref.inner.as_ref(), + &entry, + request, + &context, + ) { + Ok(result) => { + unsafe { *out = action_result_to_c(&result) }; + AdResult::Ok + } + Err(error) => { + error::set_last_error(&error); + error::last_error_code() + } + } +} + fn decode_action(action: &AdAction) -> Result<Action, AdResult> { unsafe { action_from_c(action) }.map_err(|msg| { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, msg, )); AdResult::ErrInvalidArgs @@ -189,8 +249,8 @@ fn decode_action(action: &AdAction) -> Result<Action, AdResult> { fn decode_policy(policy: i32) -> Result<AdPolicyKind, AdResult> { AdPolicyKind::from_c(policy).ok_or_else(|| { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, "invalid policy kind discriminant", )); AdResult::ErrInvalidArgs @@ -198,16 +258,18 @@ fn decode_policy(policy: i32) -> Result<AdPolicyKind, AdResult> { } fn action_request(policy: AdPolicyKind, action: Action) -> ActionRequest { - ActionRequest { - action, - policy: policy.to_interaction_policy(), + match policy { + AdPolicyKind::Headless => ActionRequest::headless(action), + AdPolicyKind::FocusFallback => ActionRequest::focus_fallback(action), + AdPolicyKind::Headed => ActionRequest::headed(action), } + .with_timeout_ms(Some(5000)) } #[cfg(test)] mod tests { use super::*; - use agent_desktop_core::{action::Action, interaction_policy::InteractionPolicy}; + use agent_desktop_core::{Action, interaction_policy::InteractionPolicy}; #[test] fn ffi_policy_kind_maps_to_core_interaction_policy() { diff --git a/crates/ffi/src/actions/native_handle.rs b/crates/ffi/src/actions/native_handle.rs index 7bf1ba2..3b2caa3 100644 --- a/crates/ffi/src/actions/native_handle.rs +++ b/crates/ffi/src/actions/native_handle.rs @@ -2,32 +2,43 @@ use crate::AdAdapter; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; use crate::types::AdNativeHandle; -use agent_desktop_core::adapter::NativeHandle; +use agent_desktop_core::{NativeHandle, ProcessIdentity}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::atomic::AtomicUsize; -/// Releases a handle previously returned by `ad_resolve_element` and +static NEXT_HANDLE_ID: AtomicUsize = AtomicUsize::new(1); + +struct NativeHandleRecord { + owner_adapter_id: usize, + handle: Rc<NativeHandle>, + process: ProcessIdentity, +} + +thread_local! { + static HANDLES: RefCell<HashMap<usize, NativeHandleRecord>> = RefCell::new(HashMap::new()); +} + +/// Releases a handle previously returned by an exact resolver and /// zeroes the caller's struct so accidentally calling this twice is -/// a deterministic no-op instead of a double-free on the underlying -/// `CFRelease`. +/// a deterministic no-op instead of dropping its owned payload twice. /// -/// On macOS this calls `CFRelease` on the underlying `AXUIElementRef`, -/// balancing the `CFRetain` that happened during `ad_resolve_element`. -/// On Windows/Linux the call is a no-op that returns `AD_RESULT_OK` -/// (platform adapters inherit the default `not_supported` impl; the -/// FFI surface translates it so callers apply the same release -/// pattern everywhere). +/// `AdNativeHandle.ptr` is an opaque registry token, not an operating-system +/// or Rust allocation address. Removing it releases the platform payload. /// -/// Ownership contract: the FFI owns the handle from the moment -/// `ad_resolve_element` writes `ptr`. Copying the struct after that -/// point and calling `ad_free_handle` on either copy is undefined — -/// there is no way for the library to detect forged non-null pointers. -/// Callers that legitimately need a "copy" should re-resolve. +/// Ownership contract: the FFI owns the handle from the moment a resolver +/// writes `ptr`. Copying the struct after that point is unsupported. Releasing +/// the original zeroes it and makes a second release of that same struct a +/// no-op; releasing an unzeroed copy is rejected. /// /// # Safety /// -/// `adapter` must be a non-null pointer returned by `ad_adapter_create` -/// and must be the same live adapter that produced `handle`. -/// `handle` must be null or a `*mut AdNativeHandle` previously -/// populated by `ad_resolve_element`. On return `(*handle).ptr` is +/// `adapter` must be a non-null pointer returned by `ad_adapter_create`. +/// It must identify the same adapter that created the handle. The adapter may +/// already have been destroyed; handles remain independently owned until freed. +/// `handle` must be null or a `*mut AdNativeHandle` previously populated by an +/// exact resolver on the calling thread. On return `(*handle).ptr` is /// `NULL` so a double-call is a no-op instead of a double-free. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_free_handle( @@ -35,36 +46,169 @@ pub unsafe extern "C" fn ad_free_handle( handle: *mut AdNativeHandle, ) -> AdResult { trap_panic(|| unsafe { - if adapter.is_null() { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "adapter is null", - )); - return AdResult::ErrInvalidArgs; - } + let adapter_id = match crate::adapter::adapter_id(adapter) { + Ok(id) => id, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; if handle.is_null() { return AdResult::Ok; } - let raw = (*handle).ptr; - if raw.is_null() { + let token = (*handle).ptr; + if token.is_null() { return AdResult::Ok; } - (*handle).ptr = std::ptr::null(); - let adapter = &*adapter; - let native = NativeHandle::from_ptr(raw); - match adapter.inner.release_handle(&native) { - Ok(()) => AdResult::Ok, - Err(e) => { - if matches!( - e.code, - agent_desktop_core::error::ErrorCode::ActionNotSupported - | agent_desktop_core::error::ErrorCode::PlatformNotSupported - ) { - return AdResult::Ok; - } - set_last_error(&e); - crate::error::last_error_code() + let result = release_ffi_handle(adapter_id, token.addr()); + match result { + Ok(()) => { + (*handle).ptr = std::ptr::null(); + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + AdResult::ErrInvalidArgs } } }) } + +pub(crate) fn into_ffi_handle( + owner_adapter_id: usize, + handle: NativeHandle, + process: ProcessIdentity, +) -> Result<*const std::ffi::c_void, agent_desktop_core::AdapterError> { + let id = crate::opaque_id::allocate(&NEXT_HANDLE_ID, "Native handle")?; + HANDLES.with(|handles| { + handles.borrow_mut().insert( + id, + NativeHandleRecord { + owner_adapter_id, + handle: Rc::new(handle), + process, + }, + ); + }); + Ok(std::ptr::with_exposed_provenance(id)) +} + +pub(crate) fn acquire_ffi_handle( + owner_adapter_id: usize, + handle: &AdNativeHandle, +) -> Result<(Rc<NativeHandle>, ProcessIdentity), agent_desktop_core::AdapterError> { + if handle.ptr.is_null() { + return Err(agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "handle.ptr is null — the handle has already been freed or was never resolved", + )); + } + HANDLES + .with(|handles| { + handles.borrow().get(&handle.ptr.addr()).and_then(|record| { + (record.owner_adapter_id == owner_adapter_id) + .then(|| (Rc::clone(&record.handle), record.process.clone())) + }) + }) + .ok_or_else(|| { + agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "native handle is invalid, freed, belongs to another adapter, or belongs to another thread", + ) + }) +} + +fn release_ffi_handle( + owner_adapter_id: usize, + handle_id: usize, +) -> Result<(), agent_desktop_core::AdapterError> { + HANDLES.with(|handles| { + let mut handles = handles.borrow_mut(); + match handles.get(&handle_id) { + Some(record) if record.owner_adapter_id == owner_adapter_id => { + handles.remove(&handle_id); + Ok(()) + } + _ => Err(agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "native handle is invalid, freed, belongs to another adapter, or belongs to another thread", + )), + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + struct DropProbe(Rc<Cell<u32>>); + + fn process() -> ProcessIdentity { + ProcessIdentity::new(42, "generation-1") + } + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + #[test] + fn handle_cannot_cross_adapter_identity() { + let token = into_ffi_handle(41, NativeHandle::null(), process()).unwrap(); + let handle = AdNativeHandle { ptr: token }; + + assert!(acquire_ffi_handle(42, &handle).is_err()); + assert!(release_ffi_handle(42, token.addr()).is_err()); + assert!(acquire_ffi_handle(41, &handle).is_ok()); + assert!(release_ffi_handle(41, token.addr()).is_ok()); + } + + #[test] + fn successful_release_drops_payload_once() { + let drops = Rc::new(Cell::new(0)); + let token = into_ffi_handle( + 71, + NativeHandle::new(DropProbe(Rc::clone(&drops))), + process(), + ) + .unwrap(); + + assert!(release_ffi_handle(71, token.addr()).is_ok()); + assert_eq!(drops.get(), 1); + assert!(release_ffi_handle(71, token.addr()).is_err()); + assert_eq!(drops.get(), 1); + } + + #[test] + fn public_free_rejects_wrong_adapter_without_consuming_handle() { + let owner = crate::adapter::ad_adapter_create(); + let other = crate::adapter::ad_adapter_create(); + let token = into_ffi_handle(owner.addr(), NativeHandle::null(), process()).unwrap(); + let mut handle = AdNativeHandle { ptr: token }; + + let wrong = unsafe { ad_free_handle(other, &mut handle) }; + assert_eq!(wrong, AdResult::ErrInvalidArgs); + assert_eq!(handle.ptr, token); + assert_eq!(unsafe { ad_free_handle(owner, &mut handle) }, AdResult::Ok); + assert!(handle.ptr.is_null()); + + unsafe { + crate::adapter::ad_adapter_destroy(owner); + crate::adapter::ad_adapter_destroy(other); + } + } + + #[test] + fn handle_can_be_freed_after_adapter_destruction() { + let owner = crate::adapter::ad_adapter_create(); + let token = into_ffi_handle(owner.addr(), NativeHandle::null(), process()).unwrap(); + let mut handle = AdNativeHandle { ptr: token }; + + unsafe { crate::adapter::ad_adapter_destroy(owner) }; + + assert_eq!(unsafe { ad_free_handle(owner, &mut handle) }, AdResult::Ok); + assert!(handle.ptr.is_null()); + } +} diff --git a/crates/ffi/src/actions/resolve.rs b/crates/ffi/src/actions/resolve.rs index b9a11f6..6b51f1b 100644 --- a/crates/ffi/src/actions/resolve.rs +++ b/crates/ffi/src/actions/resolve.rs @@ -3,15 +3,22 @@ use crate::convert::string::optional_adapter_string; use crate::convert::surface::snapshot_surface_from_c; use crate::error::{self, AdResult}; use crate::ffi_try::trap_panic; -use crate::types::{AdNativeHandle, AdRefEntry}; -use agent_desktop_core::refs::RefEntry as CoreRefEntry; -use std::mem::ManuallyDrop; +use crate::types::{AdExactRefEntry, AdIdentifierKind, AdNativeHandle, AdRefEntry}; +use agent_desktop_core::{ + AdapterError, ElementIdentifier, ErrorCode, Rect, RefCapabilities, RefEntry as CoreRefEntry, + RefEntryIdentity, RefGeometry, RefProcess, RefScope, RefSource, +}; + +const MAX_REF_FIELD_BYTES: usize = 65_536; +const MAX_REF_TOKEN_BYTES: usize = 256; /// # Safety /// /// `adapter` must be a non-null pointer returned by `ad_adapter_create`. /// `entry` must be a non-null pointer to a valid `AdRefEntry`. /// `out` must be a non-null pointer to an `AdNativeHandle` to write the result into. +/// +/// This legacy entrypoint lacks exact identity evidence and fails closed. Use ad_resolve_element_exact. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_resolve_element( adapter: *const AdAdapter, @@ -23,7 +30,6 @@ pub unsafe extern "C" fn ad_resolve_element( (*out).ptr = std::ptr::null(); crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(entry, c"entry is null"); - let adapter = &*adapter; let entry = &*entry; let core_entry = match core_ref_entry_from_ffi(entry) { Ok(entry) => entry, @@ -32,39 +38,149 @@ pub unsafe extern "C" fn ad_resolve_element( return error::last_error_code(); } }; - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } - match adapter.inner.resolve_element_strict(&core_entry) { - Ok(handle) => { - let handle = ManuallyDrop::new(handle); - (*out).ptr = handle.as_raw(); - AdResult::Ok - } - Err(e) => { - error::set_last_error(&e); - error::last_error_code() - } - } + resolve_core_entry(adapter, &core_entry, out) }) } +/// Resolves an element using process-generation and typed native-id evidence. +/// +/// # Safety +/// +/// `adapter` and `entry` must be live and valid; `out` must be writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_resolve_element_exact( + adapter: *const AdAdapter, + entry: *const AdExactRefEntry, + out: *mut AdNativeHandle, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + (*out).ptr = std::ptr::null(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(entry, c"entry is null"); + let core_entry = match core_ref_entry_from_exact(&*entry) { + Ok(entry) => entry, + Err(error) => { + error::set_last_error(&error); + return error::last_error_code(); + } + }; + resolve_core_entry(adapter, &core_entry, out) + }) +} + +unsafe fn resolve_core_entry( + adapter: *const AdAdapter, + entry: &CoreRefEntry, + out: *mut AdNativeHandle, +) -> AdResult { + let adapter_id = adapter.addr(); + let adapter = crate::adapter::acquire_adapter!(adapter); + let process = match process_identity(entry) { + Ok(process) => process, + Err(error) => { + error::set_last_error(&error); + return error::last_error_code(); + } + }; + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.resolve_element_strict(entry, deadline) { + Ok(handle) => { + match crate::actions::native_handle::into_ffi_handle(adapter_id, handle, process) { + Ok(token) => { + unsafe { (*out).ptr = token }; + AdResult::Ok + } + Err(error) => { + error::set_last_error(&error); + error::last_error_code() + } + } + } + Err(error) => { + error::set_last_error(&error); + error::last_error_code() + } + } +} + pub(crate) unsafe fn core_ref_entry_from_ffi( + _entry: &AdRefEntry, +) -> Result<CoreRefEntry, AdapterError> { + Err(AdapterError::new( + ErrorCode::InvalidArgs, + "legacy AdRefEntry lacks process-generation and typed identifier evidence; use AdExactRefEntry", + )) +} + +pub(crate) unsafe fn core_ref_entry_from_exact( + exact: &AdExactRefEntry, +) -> Result<CoreRefEntry, AdapterError> { + if exact.version != crate::types::exact_ref_entry::AD_EXACT_REF_ENTRY_VERSION + || exact.size as usize != crate::types::exact_ref_entry::AD_EXACT_REF_ENTRY_SIZE + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "AdExactRefEntry version or size does not match this library", + )); + } + let process_instance = unsafe { optional_string(exact.process_instance, "process_instance") }? + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + AdapterError::new(ErrorCode::InvalidArgs, "process_instance is null or empty") + })?; + let native_id = + match unsafe { optional_string(exact.entry.identity.native_id, "identity.native_id") }? { + Some(value) if value.trim().is_empty() => { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "identity.native_id is empty", + )); + } + Some(value) => { + let kind = AdIdentifierKind::from_c(exact.identifier_kind).ok_or_else(|| { + AdapterError::new( + ErrorCode::InvalidArgs, + "identity.identifier_kind has an invalid discriminant", + ) + })?; + Some(ElementIdentifier { + kind: kind.to_core(), + value, + }) + } + None => None, + }; + unsafe { decode_core_ref_entry(&exact.entry, process_instance, native_id) } +} + +unsafe fn decode_core_ref_entry( entry: &AdRefEntry, -) -> Result<CoreRefEntry, agent_desktop_core::error::AdapterError> { - let role = unsafe { optional_string(entry.role, "role") }?.ok_or_else(|| { - agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "role is null", - ) - })?; - let name = unsafe { optional_string(entry.name, "name") }?; - let value = unsafe { optional_string(entry.value, "value") }?; - let description = unsafe { optional_string(entry.description, "description") }?; + process_instance: String, + native_id: Option<ElementIdentifier>, +) -> Result<CoreRefEntry, AdapterError> { + if entry.process.pid == 0 { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "identity.pid must be positive", + )); + } + let role = unsafe { optional_string(entry.identity.role, "identity.role") }? + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "role is null"))?; + if !agent_desktop_core::Role::is_canonical(&role) || role == "unknown" { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "identity.role is not canonical", + )); + } + let name = unsafe { optional_string(entry.identity.name, "identity.name") }?; + let value = unsafe { optional_string(entry.identity.value, "identity.value") }?; + let description = + unsafe { optional_string(entry.identity.description, "identity.description") }?; let states = unsafe { string_array( - entry.states, - entry.state_count, + entry.capabilities.states.items, + entry.capabilities.states.count, "states", "AD_MAX_REF_STATES", crate::types::ref_entry::AD_MAX_REF_STATES, @@ -72,57 +188,141 @@ pub(crate) unsafe fn core_ref_entry_from_ffi( }?; let available_actions = unsafe { string_array( - entry.available_actions, - entry.available_action_count, + entry.capabilities.available_actions.items, + entry.capabilities.available_actions.count, "available_actions", "AD_MAX_REF_ACTIONS", crate::types::ref_entry::AD_MAX_REF_ACTIONS, ) }?; - let bounds = if entry.has_bounds { - Some(agent_desktop_core::node::Rect { - x: entry.bounds.x, - y: entry.bounds.y, - width: entry.bounds.width, - height: entry.bounds.height, - }) + let bounds = if entry.geometry.has_bounds { + Some( + Rect { + x: entry.geometry.bounds.x, + y: entry.geometry.bounds.y, + width: entry.geometry.bounds.width, + height: entry.geometry.bounds.height, + } + .validate()?, + ) } else { None }; - let bounds_hash = if entry.has_bounds_hash { - Some(entry.bounds_hash) + let supplied_bounds_hash = if entry.geometry.has_bounds_hash { + Some(entry.geometry.bounds_hash) } else { None }; - let source_surface = snapshot_surface_from_c(entry.source_surface, "source_surface")?; - let path = unsafe { ref_path(entry.path, entry.path_count)? }; + let derived_bounds_hash = bounds.and_then(|value| value.bounds_hash()); + let source_surface = snapshot_surface_from_c(entry.source.surface, "source.surface")?; + let path = unsafe { ref_path(entry.scope.path, entry.scope.path_count)? }; + if let (Some(supplied), Some(derived)) = (supplied_bounds_hash, derived_bounds_hash) + && supplied != derived + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "geometry.bounds_hash does not match geometry.bounds", + )); + } + let bounds_hash = supplied_bounds_hash.or(derived_bounds_hash); + let source_app = unsafe { optional_string(entry.source.app, "source.app") }?; + let source_window_id = unsafe { optional_string(entry.source.window_id, "source.window_id") }?; + let source_window_title = + unsafe { optional_string(entry.source.window_title, "source.window_title") }?; + let source_window_bounds_hash = entry + .source + .has_window_bounds_hash + .then_some(entry.source.window_bounds_hash); + let root_ref = unsafe { optional_string(entry.scope.root_ref, "scope.root_ref") }?; + for field in [ + Some(role.as_str()), + Some(process_instance.as_str()), + name.as_deref(), + value.as_deref(), + description.as_deref(), + native_id + .as_ref() + .map(|identifier| identifier.value.as_str()), + source_app.as_deref(), + source_window_id.as_deref(), + source_window_title.as_deref(), + root_ref.as_deref(), + ] + .into_iter() + .flatten() + { + if field.len() > MAX_REF_FIELD_BYTES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "ref identity evidence exceeds the field limit", + )); + } + } + if states + .iter() + .chain(available_actions.iter()) + .any(|value| value.len() > MAX_REF_TOKEN_BYTES) + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "ref state or action evidence exceeds the token limit", + )); + } Ok(CoreRefEntry { - pid: entry.pid, - role, - name, - value, - description, - states, - bounds, - bounds_hash, - available_actions, - source_app: unsafe { optional_string(entry.source_app, "source_app") }?, - source_window_id: unsafe { optional_string(entry.source_window_id, "source_window_id") }?, - source_window_title: unsafe { - optional_string(entry.source_window_title, "source_window_title") - }?, - source_surface, - root_ref: unsafe { optional_string(entry.root_ref, "root_ref") }?, - path_is_absolute: entry.path_is_absolute, - path, + process: RefProcess { + pid: agent_desktop_core::ProcessId::new(entry.process.pid), + process_instance: Some(process_instance), + }, + identity: RefEntryIdentity { + role, + name, + value, + description, + native_id, + }, + geometry: RefGeometry { + bounds, + bounds_hash, + }, + capabilities: RefCapabilities { + states, + available_actions, + }, + source: RefSource { + source_app, + source_window_id, + source_window_title, + source_window_bounds_hash, + source_surface, + }, + scope: RefScope { + root_ref, + path_is_absolute: entry.scope.path_is_absolute, + path, + }, }) } +fn process_identity( + entry: &CoreRefEntry, +) -> Result<agent_desktop_core::ProcessIdentity, AdapterError> { + let instance = entry.process.process_instance.clone().ok_or_else(|| { + AdapterError::new( + ErrorCode::InvalidArgs, + "process_instance is required for a native handle", + ) + })?; + Ok(agent_desktop_core::ProcessIdentity::new( + entry.process.pid, + instance, + )) +} + unsafe fn optional_string( ptr: *const std::os::raw::c_char, field: &str, -) -> Result<Option<String>, agent_desktop_core::error::AdapterError> { +) -> Result<Option<String>, AdapterError> { optional_adapter_string(ptr, field) } @@ -132,16 +332,16 @@ fn check_array_len( field: &str, constant: &str, max: usize, -) -> Result<(), agent_desktop_core::error::AdapterError> { +) -> Result<(), AdapterError> { if len > max { - return Err(agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + return Err(AdapterError::new( + ErrorCode::InvalidArgs, format!("{field} count {len} exceeds {constant} ({max})"), )); } if is_null { - return Err(agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + return Err(AdapterError::new( + ErrorCode::InvalidArgs, format!("{field} count is nonzero but pointer is null"), )); } @@ -154,7 +354,7 @@ unsafe fn string_array( field: &str, constant: &str, max: usize, -) -> Result<Vec<String>, agent_desktop_core::error::AdapterError> { +) -> Result<Vec<String>, AdapterError> { if len == 0 { return Ok(Vec::new()); } @@ -166,10 +366,7 @@ unsafe fn string_array( .map(|(index, item)| { let element = format!("{field}[{index}]"); unsafe { optional_string(*item, &element) }?.ok_or_else(|| { - agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - format!("{element} is null"), - ) + AdapterError::new(ErrorCode::InvalidArgs, format!("{element} is null")) }) }) .collect() @@ -178,7 +375,7 @@ unsafe fn string_array( unsafe fn ref_path( ptr: *const u32, len: usize, -) -> Result<smallvec::SmallVec<[usize; 8]>, agent_desktop_core::error::AdapterError> { +) -> Result<smallvec::SmallVec<[usize; 8]>, AdapterError> { if len == 0 { return Ok(smallvec::SmallVec::new()); } diff --git a/crates/ffi/src/actions/resolve_tests.rs b/crates/ffi/src/actions/resolve_tests.rs index a266374..b7637da 100644 --- a/crates/ffi/src/actions/resolve_tests.rs +++ b/crates/ffi/src/actions/resolve_tests.rs @@ -1,36 +1,22 @@ use super::*; -use agent_desktop_core::error::ErrorCode; +use agent_desktop_core::ErrorCode; use std::ffi::CString; fn test_ref_entry() -> AdRefEntry { - AdRefEntry { - pid: 0, - role: std::ptr::null(), - name: std::ptr::null(), - value: std::ptr::null(), - description: std::ptr::null(), - states: std::ptr::null(), - state_count: 0, - available_actions: std::ptr::null(), - available_action_count: 0, - bounds: crate::types::AdRect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 0.0, - }, - has_bounds: false, - bounds_hash: 0, - has_bounds_hash: false, - source_app: std::ptr::null(), - source_window_id: std::ptr::null(), - source_window_title: std::ptr::null(), - source_surface: 0, - root_ref: std::ptr::null(), - path_is_absolute: false, - path: std::ptr::null(), - path_count: 0, - } + let mut entry: AdRefEntry = unsafe { std::mem::zeroed() }; + entry.process.pid = 42; + entry +} + +fn decode(entry: AdRefEntry) -> Result<CoreRefEntry, AdapterError> { + let exact = AdExactRefEntry { + version: crate::types::exact_ref_entry::AD_EXACT_REF_ENTRY_VERSION, + size: crate::types::exact_ref_entry::AD_EXACT_REF_ENTRY_SIZE as u32, + entry, + process_instance: c"42:100".as_ptr(), + identifier_kind: AdIdentifierKind::AxIdentifier as i32, + }; + unsafe { core_ref_entry_from_exact(&exact) } } #[test] @@ -49,54 +35,97 @@ fn ffi_ref_entry_preserves_description_identity() { let actions = [action.as_ptr()]; let path = [1_u32, 2, 3]; let mut entry = test_ref_entry(); - entry.pid = 42; - entry.role = role.as_ptr(); - entry.name = name.as_ptr(); - entry.value = value.as_ptr(); - entry.description = description.as_ptr(); - entry.states = states.as_ptr(); - entry.state_count = states.len(); - entry.available_actions = actions.as_ptr(); - entry.available_action_count = actions.len(); - entry.bounds = crate::types::AdRect { + entry.process.pid = 42; + entry.identity.role = role.as_ptr(); + entry.identity.name = name.as_ptr(); + entry.identity.value = value.as_ptr(); + entry.identity.description = description.as_ptr(); + entry.capabilities.states.items = states.as_ptr(); + entry.capabilities.states.count = states.len(); + entry.capabilities.available_actions.items = actions.as_ptr(); + entry.capabilities.available_actions.count = actions.len(); + entry.geometry.bounds = crate::types::AdRect { x: 1.0, y: 2.0, width: 3.0, height: 4.0, }; - entry.has_bounds = true; - entry.bounds_hash = 123; - entry.has_bounds_hash = true; - entry.source_app = source_app.as_ptr(); - entry.source_window_id = window_id.as_ptr(); - entry.source_window_title = window_title.as_ptr(); - entry.source_surface = 5; - entry.root_ref = root_ref.as_ptr(); - entry.path_is_absolute = true; - entry.path = path.as_ptr(); - entry.path_count = path.len(); + entry.geometry.has_bounds = true; + entry.geometry.bounds_hash = Rect { + x: 1.0, + y: 2.0, + width: 3.0, + height: 4.0, + } + .bounds_hash() + .unwrap(); + entry.geometry.has_bounds_hash = true; + entry.source.app = source_app.as_ptr(); + entry.source.window_id = window_id.as_ptr(); + entry.source.window_title = window_title.as_ptr(); + entry.source.window_bounds_hash = 777; + entry.source.has_window_bounds_hash = true; + entry.source.surface = 5; + entry.scope.root_ref = root_ref.as_ptr(); + entry.scope.path_is_absolute = true; + entry.scope.path = path.as_ptr(); + entry.scope.path_count = path.len(); - let core_entry = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap(); + let core_entry = decode(entry).unwrap(); - assert_eq!(core_entry.pid, 42); - assert_eq!(core_entry.role, "button"); - assert_eq!(core_entry.name.as_deref(), Some("Primary")); - assert_eq!(core_entry.value.as_deref(), Some("On")); - assert_eq!(core_entry.description.as_deref(), Some("Insert Shape")); - assert_eq!(core_entry.states, ["focused"]); - assert_eq!(core_entry.available_actions, ["Click"]); - assert_eq!(core_entry.bounds.unwrap().width, 3.0); - assert_eq!(core_entry.bounds_hash, Some(123)); - assert_eq!(core_entry.source_app.as_deref(), Some("Finder")); - assert_eq!(core_entry.source_window_id.as_deref(), Some("w-1")); - assert_eq!(core_entry.source_window_title.as_deref(), Some("Documents")); + assert_eq!(core_entry.process.pid, 42); + assert_eq!(core_entry.identity.role, "button"); + assert_eq!(core_entry.identity.name.as_deref(), Some("Primary")); + assert_eq!(core_entry.identity.value.as_deref(), Some("On")); assert_eq!( - core_entry.source_surface, - agent_desktop_core::adapter::SnapshotSurface::Popover + core_entry.identity.description.as_deref(), + Some("Insert Shape") + ); + assert_eq!(core_entry.capabilities.states, ["focused"]); + assert_eq!(core_entry.capabilities.available_actions, ["Click"]); + assert_eq!(core_entry.geometry.bounds.unwrap().width, 3.0); + assert_eq!( + core_entry.geometry.bounds_hash, + core_entry.geometry.bounds.unwrap().bounds_hash() + ); + assert_eq!(core_entry.source.source_app.as_deref(), Some("Finder")); + assert_eq!(core_entry.source.source_window_id.as_deref(), Some("w-1")); + assert_eq!( + core_entry.source.source_window_title.as_deref(), + Some("Documents") + ); + assert_eq!(core_entry.source.source_window_bounds_hash, Some(777)); + assert_eq!( + core_entry.source.source_surface, + agent_desktop_core::SnapshotSurface::Popover + ); + assert_eq!(core_entry.scope.root_ref.as_deref(), Some("@e1")); + assert!(core_entry.scope.path_is_absolute); + assert_eq!(core_entry.scope.path.as_slice(), &[1, 2, 3]); +} + +#[test] +fn ffi_ref_entry_derives_bounds_hash_when_caller_omits_it() { + let role = CString::new("button").unwrap(); + let mut entry = test_ref_entry(); + entry.identity.role = role.as_ptr(); + entry.geometry.bounds = crate::types::AdRect { + x: 1.0, + y: 2.0, + width: 3.0, + height: 4.0, + }; + entry.geometry.has_bounds = true; + + let core_entry = decode(entry).unwrap(); + + assert_eq!( + core_entry.geometry.bounds_hash, + core_entry + .geometry + .bounds + .and_then(|bounds| bounds.bounds_hash()) ); - assert_eq!(core_entry.root_ref.as_deref(), Some("@e1")); - assert!(core_entry.path_is_absolute); - assert_eq!(core_entry.path.as_slice(), &[1, 2, 3]); } #[test] @@ -104,24 +133,24 @@ fn ffi_ref_entry_rejects_invalid_description_identity() { let role = CString::new("button").unwrap(); let bad_description: [u8; 2] = [0xC3, 0x00]; let mut entry = test_ref_entry(); - entry.pid = 42; - entry.role = role.as_ptr(); - entry.description = bad_description.as_ptr().cast(); + entry.process.pid = 42; + entry.identity.role = role.as_ptr(); + entry.identity.description = bad_description.as_ptr().cast(); - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); - assert_eq!(err.message, "description is not valid UTF-8"); + assert_eq!(err.message, "identity.description is not valid UTF-8"); } #[test] fn ffi_ref_entry_rejects_invalid_array_pointer() { let role = CString::new("button").unwrap(); let mut entry = test_ref_entry(); - entry.role = role.as_ptr(); - entry.state_count = 1; + entry.identity.role = role.as_ptr(); + entry.capabilities.states.count = 1; - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert_eq!(err.message, "states count is nonzero but pointer is null"); @@ -131,13 +160,13 @@ fn ffi_ref_entry_rejects_invalid_array_pointer() { fn ffi_ref_entry_rejects_unknown_surface() { let role = CString::new("button").unwrap(); let mut entry = test_ref_entry(); - entry.role = role.as_ptr(); - entry.source_surface = 99; + entry.identity.role = role.as_ptr(); + entry.source.surface = 99; - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); - assert_eq!(err.message, "invalid source_surface discriminant"); + assert_eq!(err.message, "invalid source.surface discriminant"); } fn string_array_of(len: usize) -> (Vec<CString>, Vec<*const std::os::raw::c_char>) { @@ -153,11 +182,11 @@ fn ffi_ref_entry_rejects_oversized_state_count() { let role = CString::new("button").unwrap(); let (_owned, ptrs) = string_array_of(crate::types::ref_entry::AD_MAX_REF_STATES + 1); let mut entry = test_ref_entry(); - entry.role = role.as_ptr(); - entry.states = ptrs.as_ptr(); - entry.state_count = ptrs.len(); + entry.identity.role = role.as_ptr(); + entry.capabilities.states.items = ptrs.as_ptr(); + entry.capabilities.states.count = ptrs.len(); - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert!(err.message.contains("AD_MAX_REF_STATES")); @@ -168,11 +197,11 @@ fn ffi_ref_entry_rejects_oversized_action_count() { let role = CString::new("button").unwrap(); let (_owned, ptrs) = string_array_of(crate::types::ref_entry::AD_MAX_REF_ACTIONS + 1); let mut entry = test_ref_entry(); - entry.role = role.as_ptr(); - entry.available_actions = ptrs.as_ptr(); - entry.available_action_count = ptrs.len(); + entry.identity.role = role.as_ptr(); + entry.capabilities.available_actions.items = ptrs.as_ptr(); + entry.capabilities.available_actions.count = ptrs.len(); - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert!(err.message.contains("AD_MAX_REF_ACTIONS")); @@ -183,11 +212,11 @@ fn ffi_ref_entry_rejects_oversized_path_count() { let role = CString::new("button").unwrap(); let path: Vec<u32> = (0..(crate::types::ref_entry::AD_MAX_REF_PATH_DEPTH as u32 + 1)).collect(); let mut entry = test_ref_entry(); - entry.role = role.as_ptr(); - entry.path = path.as_ptr(); - entry.path_count = path.len(); + entry.identity.role = role.as_ptr(); + entry.scope.path = path.as_ptr(); + entry.scope.path_count = path.len(); - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert!(err.message.contains("AD_MAX_REF_PATH_DEPTH")); @@ -198,11 +227,48 @@ fn ffi_ref_entry_rejects_unterminated_name_within_byte_cap() { let role = CString::new("button").unwrap(); let unterminated = vec![b'a'; crate::convert::string::AD_MAX_STRING_BYTES + 1]; let mut entry = test_ref_entry(); - entry.role = role.as_ptr(); - entry.name = unterminated.as_ptr().cast(); + entry.identity.role = role.as_ptr(); + entry.identity.name = unterminated.as_ptr().cast(); - let err = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + let err = decode(entry).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert!(err.message.contains("name exceeds AD_MAX_STRING_BYTES")); } + +#[test] +fn legacy_ref_entry_fails_closed_without_exact_identity() { + let entry = test_ref_entry(); + let error = unsafe { core_ref_entry_from_ffi(&entry) }.unwrap_err(); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert!(error.message.contains("AdExactRefEntry")); +} + +#[test] +fn exact_ref_entry_preserves_typed_identifier_atomically() { + let role = CString::new("button").unwrap(); + let native_id = CString::new("checkout").unwrap(); + let mut entry = test_ref_entry(); + entry.process.pid = 42; + entry.identity.role = role.as_ptr(); + entry.identity.native_id = native_id.as_ptr(); + let exact = AdExactRefEntry { + version: crate::types::exact_ref_entry::AD_EXACT_REF_ENTRY_VERSION, + size: crate::types::exact_ref_entry::AD_EXACT_REF_ENTRY_SIZE as u32, + entry, + process_instance: c"42:100".as_ptr(), + identifier_kind: AdIdentifierKind::AxDomIdentifier as i32, + }; + + let decoded = unsafe { core_ref_entry_from_exact(&exact) }.unwrap(); + + assert_eq!(decoded.process.process_instance.as_deref(), Some("42:100")); + assert_eq!( + decoded.identity.native_id, + Some(ElementIdentifier { + kind: agent_desktop_core::IdentifierKind::AxDomIdentifier, + value: "checkout".into(), + }) + ); +} diff --git a/crates/ffi/src/actions/result.rs b/crates/ffi/src/actions/result.rs index 1941b5e..114579a 100644 --- a/crates/ffi/src/actions/result.rs +++ b/crates/ffi/src/actions/result.rs @@ -1,7 +1,10 @@ use crate::convert::string::{free_c_string, opt_string_to_c, string_to_c_lossy}; -use crate::types::{AdActionResult, AdElementState, action_step::AdActionStep}; +use crate::types::{ + AdActionResult, AdElementState, action_step::AdActionStep, step_mechanism::AdStepMechanism, +}; use agent_desktop_core::action_result::ActionResult as CoreActionResult; use agent_desktop_core::action_step_outcome::ActionStepOutcome; +use agent_desktop_core::step_mechanism::StepMechanism; use std::ptr; pub(crate) fn action_result_to_c(r: &CoreActionResult) -> AdActionResult { @@ -15,12 +18,17 @@ pub(crate) fn action_result_to_c(r: &CoreActionResult) -> AdActionResult { let states = if state.states.is_empty() { ptr::null_mut() } else { - let mut ptrs: Vec<*mut std::os::raw::c_char> = + let ptrs: Vec<*mut std::os::raw::c_char> = state.states.iter().map(|s| string_to_c_lossy(s)).collect(); - ptrs.push(ptr::null_mut()); + let len = ptrs.len(); let mut boxed = ptrs.into_boxed_slice(); let raw = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation( + crate::resource::AllocationKind::ActionStateStrings, + raw, + len, + ); raw }; let elem = Box::new(AdElementState { @@ -29,7 +37,13 @@ pub(crate) fn action_result_to_c(r: &CoreActionResult) -> AdActionResult { state_count, value, }); - Box::into_raw(elem) + let raw = Box::into_raw(elem); + crate::resource::register_allocation( + crate::resource::AllocationKind::ActionPostState, + raw, + 1, + ); + raw } }; AdActionResult { @@ -38,6 +52,12 @@ pub(crate) fn action_result_to_c(r: &CoreActionResult) -> AdActionResult { post_state, steps: action_steps_to_c(r), step_count: r.steps.len() as u32, + details_json: r + .details + .as_ref() + .map(|details| string_to_c_lossy(&details.to_string())) + .unwrap_or(ptr::null_mut()), + disposition: crate::types::AdDeliverySemantics::from_core(r.disposition()), } } @@ -57,10 +77,15 @@ pub unsafe extern "C" fn ad_free_action_result(result: *mut AdActionResult) { let r = &mut *result; free_c_string(r.action as *mut _); free_c_string(r.ref_id as *mut _); + free_c_string(r.details_json as *mut _); if !r.steps.is_null() { free_step_array(r.steps); } - if !r.post_state.is_null() { + if crate::resource::take_allocation( + crate::resource::AllocationKind::ActionPostState, + r.post_state, + ) == Some(1) + { let state = &mut *r.post_state; free_c_string(state.role as *mut _); free_c_string(state.value as *mut _); @@ -74,6 +99,8 @@ pub unsafe extern "C" fn ad_free_action_result(result: *mut AdActionResult) { r.ref_id = ptr::null(); r.steps = ptr::null_mut(); r.step_count = 0; + r.details_json = ptr::null(); + r.disposition = crate::types::AdDeliverySemantics::unknown(); }) } @@ -81,18 +108,28 @@ fn action_steps_to_c(r: &CoreActionResult) -> *mut AdActionStep { if r.steps.is_empty() { return ptr::null_mut(); } - let mut steps = r + let steps = r .steps .iter() - .map(|step| AdActionStep { - label: string_to_c_lossy(step.label()), - outcome: string_to_c_lossy(step_outcome_name(&step.outcome)), + .map(|step| { + let mechanism = step.mechanism().map(core_mechanism_to_c); + let verified = step.verified(); + AdActionStep { + label: string_to_c_lossy(step.label()), + outcome: string_to_c_lossy(step_outcome_name(&step.outcome)), + mechanism: mechanism.unwrap_or(AdStepMechanism::SemanticApi) as i32, + has_mechanism: mechanism.is_some(), + verified: verified.unwrap_or(false), + has_verified: verified.is_some(), + _reserved: 0, + } }) .collect::<Vec<_>>(); - steps.push(step_sentinel()); + let len = steps.len(); let mut boxed = steps.into_boxed_slice(); let raw = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation(crate::resource::AllocationKind::ActionSteps, raw, len); raw } @@ -104,51 +141,48 @@ fn step_outcome_name(outcome: &ActionStepOutcome) -> &'static str { } } -fn step_sentinel() -> AdActionStep { - AdActionStep { - label: ptr::null(), - outcome: ptr::null(), +fn core_mechanism_to_c(mechanism: StepMechanism) -> AdStepMechanism { + match mechanism { + StepMechanism::SemanticApi => AdStepMechanism::SemanticApi, + StepMechanism::PhysicalSynthetic => AdStepMechanism::PhysicalSynthetic, } } unsafe fn free_state_array(states: *mut *mut std::os::raw::c_char) { unsafe { - let mut len = 0; - while !(*states.add(len)).is_null() { - len += 1; - } + let Some(len) = crate::resource::take_allocation( + crate::resource::AllocationKind::ActionStateStrings, + states, + ) else { + return; + }; for index in 0..len { free_c_string(*states.add(index)); } drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( - states, - len + 1, + states, len, ))); } } unsafe fn free_step_array(steps: *mut AdActionStep) { unsafe { - let mut len = 0; - while !step_is_sentinel(&*steps.add(len)) { - len += 1; - } + let Some(len) = + crate::resource::take_allocation(crate::resource::AllocationKind::ActionSteps, steps) + else { + return; + }; for index in 0..len { let step = &mut *steps.add(index); free_c_string(step.label as *mut _); free_c_string(step.outcome as *mut _); } drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( - steps, - len + 1, + steps, len, ))); } } -fn step_is_sentinel(step: &AdActionStep) -> bool { - step.label.is_null() && step.outcome.is_null() -} - #[cfg(test)] mod tests { use super::*; @@ -157,15 +191,15 @@ mod tests { #[test] fn test_action_result_to_c_with_state() { - let core_result = CoreActionResult { - action: "click".to_owned(), - post_state: Some(ElementState { + let core_result = + CoreActionResult::delivered_unverified("click").with_state(ElementState { role: "button".to_owned(), states: vec!["focused".to_owned(), "enabled".to_owned()], value: Some("OK".to_owned()), - }), - steps: Vec::new(), - }; + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }); let c_result = action_result_to_c(&core_result); unsafe { assert_eq!(c_to_string(c_result.action).as_deref(), Some("click")); @@ -190,20 +224,35 @@ mod tests { state_count: u32::MAX, value: ptr::null(), }); - let mut steps = vec![ - AdActionStep { - label: crate::convert::string::string_to_c_lossy("AXPress"), - outcome: crate::convert::string::string_to_c_lossy("succeeded"), - }, - step_sentinel(), - ] + let mut steps = vec![AdActionStep { + label: crate::convert::string::string_to_c_lossy("AXPress"), + outcome: crate::convert::string::string_to_c_lossy("succeeded"), + mechanism: AdStepMechanism::SemanticApi as i32, + has_mechanism: true, + verified: false, + has_verified: false, + _reserved: 0, + }] .into_boxed_slice(); + let post_state = Box::into_raw(post_state); + crate::resource::register_allocation( + crate::resource::AllocationKind::ActionPostState, + post_state, + 1, + ); + crate::resource::register_allocation( + crate::resource::AllocationKind::ActionSteps, + steps.as_mut_ptr(), + steps.len(), + ); let mut c_result = AdActionResult { action: crate::convert::string::string_to_c_lossy("click"), ref_id: ptr::null(), - post_state: Box::into_raw(post_state), + post_state, steps: steps.as_mut_ptr(), step_count: u32::MAX, + details_json: crate::convert::string::string_to_c_lossy("{\"uncertain\":true}"), + disposition: crate::types::AdDeliverySemantics::unknown(), }; std::mem::forget(steps); unsafe { ad_free_action_result(&mut c_result) }; @@ -211,13 +260,17 @@ mod tests { assert!(c_result.post_state.is_null()); assert!(c_result.steps.is_null()); assert_eq!(c_result.step_count, 0); + assert!(c_result.details_json.is_null()); } #[test] fn action_result_to_c_preserves_steps() { - let core_result = CoreActionResult::new("click").with_steps(vec![ - agent_desktop_core::action_step::ActionStep::attempted("AXScrollToVisible"), - agent_desktop_core::action_step::ActionStep::succeeded("AXPress"), + let core_result = CoreActionResult::delivered_unverified("click").with_steps(vec![ + agent_desktop_core::action_step::ActionStep::attempted("AXScrollToVisible") + .with_mechanism(StepMechanism::SemanticApi), + agent_desktop_core::action_step::ActionStep::succeeded("AXPress") + .with_mechanism(StepMechanism::SemanticApi) + .with_verified(true), ]); let mut c_result = action_result_to_c(&core_result); @@ -233,6 +286,12 @@ mod tests { c_to_string((*c_result.steps.add(0)).outcome).as_deref(), Some("attempted") ); + assert!((*c_result.steps.add(0)).has_mechanism); + assert_eq!( + (*c_result.steps.add(0)).mechanism, + AdStepMechanism::SemanticApi as i32 + ); + assert!(!(*c_result.steps.add(0)).has_verified); assert_eq!( c_to_string((*c_result.steps.add(1)).label).as_deref(), Some("AXPress") @@ -241,6 +300,13 @@ mod tests { c_to_string((*c_result.steps.add(1)).outcome).as_deref(), Some("succeeded") ); + assert!((*c_result.steps.add(1)).has_mechanism); + assert_eq!( + (*c_result.steps.add(1)).mechanism, + AdStepMechanism::SemanticApi as i32 + ); + assert!((*c_result.steps.add(1)).has_verified); + assert!((*c_result.steps.add(1)).verified); } unsafe { ad_free_action_result(&mut c_result) }; @@ -248,15 +314,75 @@ mod tests { assert_eq!(c_result.step_count, 0); } + #[test] + fn action_result_to_c_preserves_details_json() { + let core_result = + CoreActionResult::delivered_unverified("click").with_details(serde_json::json!({ + "transient_ambiguity": true + })); + + let mut c_result = action_result_to_c(&core_result); + + unsafe { + assert_eq!( + c_to_string(c_result.details_json).as_deref(), + Some("{\"transient_ambiguity\":true}") + ); + } + unsafe { ad_free_action_result(&mut c_result) }; + assert!(c_result.details_json.is_null()); + } + + #[test] + fn action_result_to_c_preserves_delivery_and_retry_semantics() { + let core_result = CoreActionResult::delivered_unverified("click"); + let mut result = action_result_to_c(&core_result); + + assert_eq!( + result.disposition.delivery, + crate::types::AdDeliveryDisposition::DeliveredUnverified as i32 + ); + assert_eq!( + result.disposition.retry, + crate::types::AdRetryDisposition::Unsafe as i32 + ); + unsafe { ad_free_action_result(&mut result) }; + } + + #[test] + fn free_action_result_rejects_a_mutated_nested_string_pointer() { + let core_result = CoreActionResult::delivered_unverified("click") + .with_steps(vec![agent_desktop_core::ActionStep::succeeded("AXPress")]); + let mut result = action_result_to_c(&core_result); + let original = unsafe { (*result.steps).label as *mut std::os::raw::c_char }; + let foreign = std::ffi::CString::new("foreign").unwrap().into_raw(); + unsafe { (*result.steps).label = foreign }; + + unsafe { ad_free_action_result(&mut result) }; + assert_eq!( + unsafe { std::ffi::CStr::from_ptr(foreign) }.to_bytes(), + b"foreign" + ); + unsafe { + drop(std::ffi::CString::from_raw(foreign)); + free_c_string(original); + } + } + fn state_array(states: &[&str]) -> *mut *mut std::os::raw::c_char { - let mut ptrs: Vec<*mut std::os::raw::c_char> = states + let ptrs: Vec<*mut std::os::raw::c_char> = states .iter() .map(|state| crate::convert::string::string_to_c_lossy(state)) .collect(); - ptrs.push(ptr::null_mut()); + let len = ptrs.len(); let mut boxed = ptrs.into_boxed_slice(); let raw = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation( + crate::resource::AllocationKind::ActionStateStrings, + raw, + len, + ); raw } diff --git a/crates/ffi/src/adapter.rs b/crates/ffi/src/adapter.rs index d33baee..8f5c183 100644 --- a/crates/ffi/src/adapter.rs +++ b/crates/ffi/src/adapter.rs @@ -1,16 +1,82 @@ use crate::convert::string::optional_adapter_string; use crate::error::{self, AdResult}; use crate::ffi_try::{trap_panic, trap_panic_ptr, trap_panic_void}; -use agent_desktop_core::context::{CommandContext, validate_session_id}; -use agent_desktop_core::error::{AdapterError, AppError}; -use agent_desktop_core::{PermissionState, adapter::PlatformAdapter}; +use agent_desktop_core::PlatformAdapter; +#[cfg(any(feature = "stub-adapter", test))] +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; +use agent_desktop_core::{ + AdapterError, AppError, PermissionState, + context::{CommandContext, validate_session_id}, +}; +use std::collections::HashMap; use std::ffi::c_char; +use std::sync::atomic::AtomicUsize; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; + +static ADAPTERS: OnceLock<Mutex<HashMap<usize, Arc<AdAdapter>>>> = OnceLock::new(); +static NEXT_ADAPTER_ID: AtomicUsize = AtomicUsize::new(1); pub struct AdAdapter { pub(crate) inner: Box<dyn PlatformAdapter>, pub(crate) session_id: Option<String>, + pub(crate) _session_lease: Option<agent_desktop_core::session::SessionLivenessLease>, } +fn adapters() -> &'static Mutex<HashMap<usize, Arc<AdAdapter>>> { + ADAPTERS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn lock_adapters() -> MutexGuard<'static, HashMap<usize, Arc<AdAdapter>>> { + match adapters().lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +pub(crate) fn register_adapter(adapter: AdAdapter) -> Result<*mut AdAdapter, AdapterError> { + let id = crate::opaque_id::allocate(&NEXT_ADAPTER_ID, "Adapter handle")?; + if lock_adapters().insert(id, Arc::new(adapter)).is_some() { + return Err(AdapterError::internal( + "Adapter handle identifier collision", + )); + } + Ok(std::ptr::with_exposed_provenance_mut(id)) +} + +pub(crate) fn adapter_id(adapter: *const AdAdapter) -> Result<usize, AdapterError> { + if adapter.is_null() { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "adapter is null", + )); + } + Ok(adapter.addr()) +} + +pub(crate) fn lookup_adapter(adapter: *const AdAdapter) -> Result<Arc<AdAdapter>, AdapterError> { + let id = adapter_id(adapter)?; + lock_adapters().get(&id).cloned().ok_or_else(|| { + AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "adapter handle is invalid or has already been destroyed", + ) + }) +} + +macro_rules! acquire_adapter { + ($adapter:expr) => {{ + match $crate::adapter::lookup_adapter($adapter) { + Ok(adapter) => adapter, + Err(error) => { + $crate::error::set_last_error(&error); + return $crate::error::last_error_code(); + } + } + }}; +} + +pub(crate) use acquire_adapter; + /// A no-op adapter used under the `stub-adapter` Cargo feature. /// /// Every method delegates to the `PlatformAdapter` trait defaults, all of @@ -22,34 +88,72 @@ pub struct AdAdapter { struct StubAdapter; #[cfg(feature = "stub-adapter")] -impl PlatformAdapter for StubAdapter {} +impl ObservationOps for StubAdapter {} #[cfg(feature = "stub-adapter")] -fn build_adapter() -> Box<dyn PlatformAdapter> { - Box::new(StubAdapter) +impl ActionOps for StubAdapter {} + +#[cfg(feature = "stub-adapter")] +impl InputOps for StubAdapter {} + +#[cfg(feature = "stub-adapter")] +impl SystemOps for StubAdapter {} + +#[cfg(feature = "stub-adapter")] +fn build_adapter() -> Result<Box<dyn PlatformAdapter>, AdapterError> { + Ok(Box::new(StubAdapter)) } #[cfg(not(feature = "stub-adapter"))] -fn build_adapter() -> Box<dyn PlatformAdapter> { +fn build_adapter() -> Result<Box<dyn PlatformAdapter>, AdapterError> { #[cfg(target_os = "macos")] { - Box::new(agent_desktop_macos::MacOSAdapter::new()) + agent_desktop_macos::ensure_cocoa_multithreaded()?; + Ok(Box::new(agent_desktop_macos::MacOSAdapter::new())) } #[cfg(target_os = "windows")] { - Box::new(agent_desktop_windows::WindowsAdapter::new()) + Ok(Box::new(agent_desktop_windows::WindowsAdapter::new())) } #[cfg(target_os = "linux")] { - Box::new(agent_desktop_linux::LinuxAdapter::new()) + Ok(Box::new(agent_desktop_linux::LinuxAdapter::new())) } #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] compile_error!("Unsupported platform") } +fn create_adapter(session_id: Option<String>) -> *mut AdAdapter { + let inner = match build_adapter() { + Ok(inner) => inner, + Err(error) => { + error::set_last_error(&error); + return std::ptr::null_mut(); + } + }; + let session_lease = match crate::session_lease::acquire(session_id.as_deref()) { + Ok(lease) => lease, + Err(error) => { + error::set_last_error(&error); + return std::ptr::null_mut(); + } + }; + match register_adapter(AdAdapter { + inner, + session_id, + _session_lease: session_lease, + }) { + Ok(handle) => handle, + Err(error) => { + error::set_last_error(&error); + std::ptr::null_mut() + } + } +} + /// Builds a platform adapter for the current OS and returns an opaque /// handle. Returns null on allocation failure or if a Rust panic is /// caught at the FFI boundary (inspect `ad_last_error_*` for details). @@ -59,13 +163,7 @@ fn build_adapter() -> Box<dyn PlatformAdapter> { /// common pattern is one adapter per process lifetime. #[unsafe(no_mangle)] pub extern "C" fn ad_adapter_create() -> *mut AdAdapter { - trap_panic_ptr(|| { - let adapter = AdAdapter { - inner: build_adapter(), - session_id: None, - }; - Box::into_raw(Box::new(adapter)) - }) + trap_panic_ptr(|| create_adapter(None)) } /// Builds a session-scoped platform adapter. `session` may be: @@ -97,7 +195,7 @@ pub unsafe extern "C" fn ad_adapter_create_with_session(session: *const c_char) let adapter_err = match app_err { AppError::Adapter(e) => e, other => AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + agent_desktop_core::ErrorCode::InvalidArgs, other.to_string(), ), }; @@ -105,11 +203,7 @@ pub unsafe extern "C" fn ad_adapter_create_with_session(session: *const c_char) return std::ptr::null_mut(); } } - let adapter = AdAdapter { - inner: build_adapter(), - session_id, - }; - Box::into_raw(Box::new(adapter)) + create_adapter(session_id) }) } @@ -119,18 +213,19 @@ pub unsafe extern "C" fn ad_adapter_create_with_session(session: *const c_char) /// `ad_adapter_create_with_session`, or null. After this call the pointer /// is invalid and must not be used. /// -/// The adapter must not be destroyed while any other call on it is still in -/// flight on another thread. Destroying the handle concurrently with an -/// in-flight call (e.g. `ad_wait` blocking on the main thread while this -/// function is called from a worker thread) is undefined behaviour — the -/// `Box` is freed while the blocked call still dereferences it. The caller -/// owns this synchronisation: ensure all calls on the handle have returned -/// before calling `ad_adapter_destroy`. +/// Calls that acquired the adapter before destruction retain it until they +/// return. Calls beginning after destruction fail with `ErrInvalidArgs`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_adapter_destroy(adapter: *mut AdAdapter) { trap_panic_void(|| { if !adapter.is_null() { - drop(unsafe { Box::from_raw(adapter) }); + let id = adapter.addr(); + if lock_adapters().remove(&id).is_none() { + error::set_last_error(&AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "adapter handle is invalid or has already been destroyed", + )); + } } }) } @@ -142,14 +237,21 @@ pub unsafe extern "C" fn ad_adapter_destroy(adapter: *mut AdAdapter) { #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_check_permissions(adapter: *const AdAdapter) -> AdResult { trap_panic(|| { - crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = unsafe { &*adapter }; - match adapter.inner.permission_report().accessibility { + let adapter = acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + let report = match adapter.inner.permission_report(deadline) { + Ok(report) => report, + Err(error) => { + error::set_last_error(&error); + return error::last_error_code(); + } + }; + match report.accessibility { PermissionState::Granted => AdResult::Ok, PermissionState::Denied { suggestion } => { error::set_last_error( - &agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::PermDenied, + &agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::PermDenied, "Accessibility permission not granted", ) .with_suggestion(suggestion), @@ -166,7 +268,7 @@ impl AdAdapter { /// Builds a `CommandContext` from this adapter's session. Callers that /// need a context for context-taking commands (snapshot, status, wait) /// call this at the FFI entry boundary. - pub fn command_context(&self) -> Result<CommandContext, AppError> { + pub(crate) fn command_context(&self) -> Result<CommandContext, AppError> { CommandContext::new(self.session_id.clone(), None, false) } } @@ -174,19 +276,18 @@ impl AdAdapter { fn unknown_permission_result(adapter: &dyn PlatformAdapter) -> AdResult { let (code, message, suggestion) = if adapter.unknown_accessibility_means_unsupported() { ( - agent_desktop_core::error::ErrorCode::PlatformNotSupported, + agent_desktop_core::ErrorCode::PlatformNotSupported, "Accessibility permission state is unknown because this platform adapter does not support permission probing", "Use a platform adapter with implemented permission probing before executing desktop actions.", ) } else { ( - agent_desktop_core::error::ErrorCode::Internal, + agent_desktop_core::ErrorCode::Internal, "Accessibility permission state is unknown", "Run the platform-specific permission report before executing desktop actions.", ) }; - let err = - agent_desktop_core::error::AdapterError::new(code, message).with_suggestion(suggestion); + let err = agent_desktop_core::AdapterError::new(code, message).with_suggestion(suggestion); error::set_last_error(&err); crate::error::last_error_code() } @@ -195,6 +296,15 @@ fn unknown_permission_result(adapter: &dyn PlatformAdapter) -> AdResult { mod tests { use super::*; + fn registered_adapter(inner: Box<dyn PlatformAdapter>) -> *mut AdAdapter { + register_adapter(AdAdapter { + inner, + session_id: None, + _session_lease: None, + }) + .unwrap() + } + #[test] fn test_adapter_create_destroy() { let ptr = ad_adapter_create(); @@ -207,39 +317,69 @@ mod tests { unsafe { ad_adapter_destroy(std::ptr::null_mut()) }; } + #[test] + fn destroy_revokes_new_calls_without_invalidating_in_flight_owners() { + let handle = ad_adapter_create(); + let retained = lookup_adapter(handle).unwrap(); + + unsafe { ad_adapter_destroy(handle) }; + + assert!(lookup_adapter(handle).is_err()); + let _ = retained + .inner + .permission_report(agent_desktop_core::Deadline::standard().unwrap()); + } + struct UnknownPermissionAdapter; - impl PlatformAdapter for UnknownPermissionAdapter { - fn permission_report(&self) -> agent_desktop_core::PermissionReport { - agent_desktop_core::PermissionReport { + impl ObservationOps for UnknownPermissionAdapter {} + + impl ActionOps for UnknownPermissionAdapter {} + + impl InputOps for UnknownPermissionAdapter {} + + impl SystemOps for UnknownPermissionAdapter { + fn permission_report( + &self, + _deadline: agent_desktop_core::Deadline, + ) -> Result<agent_desktop_core::PermissionReport, agent_desktop_core::AdapterError> + { + Ok(agent_desktop_core::PermissionReport { accessibility: PermissionState::Unknown, screen_recording: PermissionState::Unknown, automation: PermissionState::NotRequired, - } + }) } } #[test] fn check_permissions_maps_default_unknown_accessibility_to_platform_unsupported() { - let adapter = AdAdapter { - inner: Box::new(UnknownPermissionAdapter), - session_id: None, - }; - - let result = unsafe { ad_check_permissions(&adapter) }; + let adapter = registered_adapter(Box::new(UnknownPermissionAdapter)); + let result = unsafe { ad_check_permissions(adapter) }; + unsafe { ad_adapter_destroy(adapter) }; assert_eq!(result, AdResult::ErrPlatformNotSupported); } struct AmbiguousPermissionAdapter; - impl PlatformAdapter for AmbiguousPermissionAdapter { - fn permission_report(&self) -> agent_desktop_core::PermissionReport { - agent_desktop_core::PermissionReport { + impl ObservationOps for AmbiguousPermissionAdapter {} + + impl ActionOps for AmbiguousPermissionAdapter {} + + impl InputOps for AmbiguousPermissionAdapter {} + + impl SystemOps for AmbiguousPermissionAdapter { + fn permission_report( + &self, + _deadline: agent_desktop_core::Deadline, + ) -> Result<agent_desktop_core::PermissionReport, agent_desktop_core::AdapterError> + { + Ok(agent_desktop_core::PermissionReport { accessibility: PermissionState::Unknown, screen_recording: PermissionState::Unknown, automation: PermissionState::NotRequired, - } + }) } fn unknown_accessibility_means_unsupported(&self) -> bool { @@ -249,12 +389,9 @@ mod tests { #[test] fn check_permissions_preserves_ambiguous_unknown_accessibility_as_internal() { - let adapter = AdAdapter { - inner: Box::new(AmbiguousPermissionAdapter), - session_id: None, - }; - - let result = unsafe { ad_check_permissions(&adapter) }; + let adapter = registered_adapter(Box::new(AmbiguousPermissionAdapter)); + let result = unsafe { ad_check_permissions(adapter) }; + unsafe { ad_adapter_destroy(adapter) }; assert_eq!(result, AdResult::ErrInternal); } diff --git a/crates/ffi/src/apps/close.rs b/crates/ffi/src/apps/close.rs index 4cb0c6b..f6ecd72 100644 --- a/crates/ffi/src/apps/close.rs +++ b/crates/ffi/src/apps/close.rs @@ -20,10 +20,7 @@ pub unsafe extern "C" fn ad_close_app( id: *const c_char, force: bool, ) -> AdResult { - trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } + trap_panic(|| { crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); let id_str = match super::decode_app_id(id) { Ok(id) => id, @@ -32,12 +29,14 @@ pub unsafe extern "C" fn ad_close_app( return crate::error::last_error_code(); } }; - let adapter = &*adapter; - - match adapter.inner.close_app(&id_str, force) { - Ok(()) => AdResult::Ok, + let adapter = crate::adapter::acquire_adapter!(adapter); + match agent_desktop_core::commands::close_app::execute( + agent_desktop_core::commands::close_app::CloseAppArgs { app: id_str, force }, + adapter.inner.as_ref(), + ) { + Ok(_) => AdResult::Ok, Err(e) => { - set_last_error(&e); + set_last_error(&crate::commands::app_error_to_adapter(e)); crate::error::last_error_code() } } diff --git a/crates/ffi/src/apps/launch.rs b/crates/ffi/src/apps/launch.rs index 3977274..4395000 100644 --- a/crates/ffi/src/apps/launch.rs +++ b/crates/ffi/src/apps/launch.rs @@ -1,8 +1,10 @@ use crate::AdAdapter; -use crate::convert::window::window_info_to_c; +use crate::convert::window::{ + exact_window_info_to_c, validate_exact_window_info, window_info_to_c, +}; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::types::AdWindowInfo; +use crate::types::{AdExactWindowInfo, AdWindowInfo}; use std::os::raw::c_char; /// Launches the application identified by `id` (bundle id on macOS, @@ -28,9 +30,6 @@ pub unsafe extern "C" fn ad_launch_app( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = std::mem::zeroed(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); let id_str = match super::decode_app_id(id) { Ok(id) => id, @@ -39,9 +38,26 @@ pub unsafe extern "C" fn ad_launch_app( return crate::error::last_error_code(); } }; - - let adapter = &*adapter; - match adapter.inner.launch_app(&id_str, timeout_ms) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let options = agent_desktop_core::launch_options::LaunchOptions { + timeout_ms, + ..Default::default() + }; + let deadline = match launch_deadline(timeout_ms) { + Ok(deadline) => deadline, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + let lease = match adapter.inner.acquire_interaction_lease(deadline) { + Ok(lease) => lease, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + match adapter.inner.launch_app(&id_str, &options, &lease) { Ok(win) => { *out = window_info_to_c(&win); AdResult::Ok @@ -53,3 +69,162 @@ pub unsafe extern "C" fn ad_launch_app( } }) } + +/// Launches an application and returns a generation-pinned exact window. +/// +/// # Safety +/// `adapter`, `id`, and `out` must satisfy the same requirements as +/// `ad_launch_app`. Release the result with `ad_release_exact_window_fields`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_launch_app_exact( + adapter: *const AdAdapter, + id: *const c_char, + timeout_ms: u64, + out: *mut AdExactWindowInfo, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = std::mem::zeroed(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + let id = match super::decode_app_id(id) { + Ok(id) => id, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + let adapter = crate::adapter::acquire_adapter!(adapter); + let options = agent_desktop_core::launch_options::LaunchOptions { + timeout_ms, + ..Default::default() + }; + let deadline = match launch_deadline(timeout_ms) { + Ok(deadline) => deadline, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + let lease = match adapter.inner.acquire_interaction_lease(deadline) { + Ok(lease) => lease, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + match adapter.inner.launch_app(&id, &options, &lease) { + Ok(window) => match validate_exact_window_info(&window) { + Ok(()) => { + *out = exact_window_info_to_c(&window); + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + }, + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } + }) +} + +fn launch_deadline( + timeout_ms: u64, +) -> Result<agent_desktop_core::Deadline, agent_desktop_core::AdapterError> { + if timeout_ms == 0 { + crate::operation::deadline() + } else { + agent_desktop_core::Deadline::after(timeout_ms) + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + + use agent_desktop_core::launch_options::LaunchOptions; + use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; + use agent_desktop_core::{ + AdapterError, Deadline, InteractionLease, ProcessId, WindowInfo, WindowState, + }; + + use super::*; + + struct LaunchProbe { + calls: AtomicUsize, + timeout_ms: AtomicU64, + } + + struct LaunchAdapter { + probe: Arc<LaunchProbe>, + } + + impl ObservationOps for LaunchAdapter {} + impl ActionOps for LaunchAdapter {} + impl InputOps for LaunchAdapter {} + + impl SystemOps for LaunchAdapter { + fn acquire_interaction_lease( + &self, + deadline: Deadline, + ) -> Result<InteractionLease, AdapterError> { + InteractionLease::guarded(deadline, ()) + } + + fn launch_app( + &self, + _id: &str, + options: &LaunchOptions, + _lease: &InteractionLease, + ) -> Result<WindowInfo, AdapterError> { + self.probe.calls.fetch_add(1, Ordering::SeqCst); + self.probe + .timeout_ms + .store(options.timeout_ms, Ordering::SeqCst); + Ok(WindowInfo { + id: "w-launch".into(), + title: "Launched".into(), + app: "Fixture".into(), + pid: ProcessId::new(42), + process_instance: Some("fixture-42".into()), + bounds: None, + state: WindowState::default(), + }) + } + } + + #[test] + fn zero_timeout_reaches_launch_once_without_defaulting() { + let probe = Arc::new(LaunchProbe { + calls: AtomicUsize::new(0), + timeout_ms: AtomicU64::new(u64::MAX), + }); + let adapter = crate::adapter::register_adapter(crate::AdAdapter { + inner: Box::new(LaunchAdapter { + probe: Arc::clone(&probe), + }), + session_id: None, + _session_lease: None, + }) + .unwrap(); + let id = CString::new("Fixture").unwrap(); + let mut out: AdWindowInfo = unsafe { std::mem::zeroed() }; + + assert_eq!( + unsafe { ad_launch_app(adapter, id.as_ptr(), 0, &mut out) }, + AdResult::Ok + ); + assert_eq!(probe.calls.load(Ordering::SeqCst), 1); + assert_eq!(probe.timeout_ms.load(Ordering::SeqCst), 0); + + unsafe { + crate::windows::free_one::ad_release_window_fields(&mut out); + crate::adapter::ad_adapter_destroy(adapter); + } + } +} diff --git a/crates/ffi/src/apps/list.rs b/crates/ffi/src/apps/list.rs index e85280d..3f6c57b 100644 --- a/crates/ffi/src/apps/list.rs +++ b/crates/ffi/src/apps/list.rs @@ -18,13 +18,15 @@ pub unsafe extern "C" fn ad_list_apps( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; - match adapter.inner.list_apps() { + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.list_apps(deadline) { Ok(apps) => { + if let Err(error) = crate::resource::validate_list_len(apps.len(), "App list") { + set_last_error(&error); + return crate::error::last_error_code(); + } let items: Vec<AdAppInfo> = apps.iter().map(app_info_to_c).collect(); let list = Box::new(AdAppList { items: items.into_boxed_slice(), diff --git a/crates/ffi/src/apps/mod.rs b/crates/ffi/src/apps/mod.rs index 7a5a9ef..d472220 100644 --- a/crates/ffi/src/apps/mod.rs +++ b/crates/ffi/src/apps/mod.rs @@ -2,7 +2,7 @@ pub(crate) mod close; pub(crate) mod launch; pub(crate) mod list; -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::AdapterError; use std::os::raw::c_char; fn decode_app_id(id: *const c_char) -> Result<String, AdapterError> { diff --git a/crates/ffi/src/commands/envelope_out.rs b/crates/ffi/src/commands/envelope_out.rs index ecba38d..d926cf1 100644 --- a/crates/ffi/src/commands/envelope_out.rs +++ b/crates/ffi/src/commands/envelope_out.rs @@ -1,8 +1,8 @@ use crate::commands::app_error_to_adapter; use crate::convert::string::string_to_c; use crate::error::{AdResult, set_last_error}; -use agent_desktop_core::error::{AdapterError, AppError, ErrorCode}; use agent_desktop_core::output::{ErrorPayload, Response}; +use agent_desktop_core::{AdapterError, AppError, ErrorCode}; use serde_json::Value; use std::ffi::c_char; @@ -91,7 +91,7 @@ mod tests { use super::*; use crate::convert::string::free_c_string; use crate::error::{clear_last_error, last_error_code}; - use agent_desktop_core::error::AdapterError; + use agent_desktop_core::AdapterError; use std::ffi::CStr; unsafe fn read_and_free(out: *mut c_char) -> String { diff --git a/crates/ffi/src/commands/execute_by_ref.rs b/crates/ffi/src/commands/execute_by_ref.rs new file mode 100644 index 0000000..b56e8b8 --- /dev/null +++ b/crates/ffi/src/commands/execute_by_ref.rs @@ -0,0 +1,83 @@ +use crate::AdAdapter; +use crate::error::AdResult; +use crate::types::AdAction; +use std::ffi::c_char; + +/// Drives a snapshot-qualified ref action (`@<snapshot_id>:e5`, action) +/// through the canonical ref-action +/// pipeline: `RefStore` load → `RefMap` lookup (→ `STALE_REF` on missing) → +/// strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live +/// actionability preflight → dispatch → owned-handle drop. +/// +/// Policy: semantic actions, including `TypeText`, default to strict +/// `headless`. Explicit `PressKey` defaults to `focus_fallback`. A policy +/// discriminant may elevate to focus fallback or headed. Base and elevation +/// are computed by `agent_desktop_core::commands::execute_by_ref::execute` via +/// `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and +/// FFI share a single source of policy truth. +/// +/// `ref_id` tri-state: null → `ErrInvalidArgs`; non-null invalid UTF-8 → +/// `ErrInvalidArgs`; valid UTF-8 but bad `@e{N}` format → `ErrInvalidArgs`. +/// +/// `snapshot_id` tri-state: null is valid only when `ref_id` embeds its +/// snapshot; valid UTF-8 pins a legacy bare `@eN` ref or must match the +/// snapshot embedded in a qualified ref; invalid UTF-8 returns `ErrInvalidArgs`. +/// +/// `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback, +/// 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)` +/// accepts the action's base policy. `FocusFallback (1)` explicitly permits +/// focus without cursor movement. `Headed (2)` opts in to physical cursor and +/// keyboard delivery. +/// +/// Uses a fixed 5000ms auto-wait budget (`DEFAULT_ACTION_TIMEOUT_MS`) before +/// the actionability preflight, matching the CLI default. Call +/// `ad_execute_by_ref_timeout` with an explicit `timeout_ms` (-1 = default, +/// 0 = single-shot with no auto-wait) to control this. +/// +/// On success `*out` is set to a NUL-terminated JSON envelope (command +/// `"execute_by_ref"`); free with `ad_free_string`. On guard or decode +/// failure (invalid args before the command runs) `*out` remains null. +/// On a command-level error (STALE_REF, AMBIGUOUS_TARGET, etc.) `*out` +/// holds the error JSON envelope and must still be freed with +/// `ad_free_string`. The last-error slot is populated on all failures. +/// +/// **Dispatch-before-serialize ordering**: the action is dispatched (and any +/// side effects committed) before the result JSON is serialized. In the +/// near-impossible event that serialization of an already-valid +/// `ActionResult` fails, `*out` is null and `ErrInternal` is returned while +/// the side effect has already occurred. No pre-validation machinery is +/// needed because serialization of a valid envelope effectively never fails. +/// +/// # Safety +/// +/// `adapter` must be a non-null pointer from `ad_adapter_create[_with_session]`. +/// `ref_id` must be a non-null pointer to a NUL-terminated C string within +/// `AD_MAX_STRING_BYTES + 1` bytes; null is **not** optional — it is defined +/// behaviour (no UB) but is rejected immediately with `ErrInvalidArgs`. +/// `snapshot_id` may be null only for a snapshot-qualified ref, or a non-null +/// NUL-terminated C string within `AD_MAX_STRING_BYTES + 1` bytes. `action` +/// must be a non-null pointer to a +/// valid `AdAction`. `out` must be a non-null writable pointer. All pointers +/// must remain valid for the duration of the call. Must be called from the +/// calling thread. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_execute_by_ref( + adapter: *const AdAdapter, + ref_id: *const c_char, + snapshot_id: *const c_char, + action: *const AdAction, + policy: i32, + out: *mut *mut c_char, +) -> AdResult { + unsafe { + super::execute_by_ref_timeout::ad_execute_by_ref_timeout( + adapter, + ref_id, + snapshot_id, + action, + policy, + -1, + out, + ) + } +} diff --git a/crates/ffi/src/commands/execute_by_ref_timeout.rs b/crates/ffi/src/commands/execute_by_ref_timeout.rs new file mode 100644 index 0000000..5754fed --- /dev/null +++ b/crates/ffi/src/commands/execute_by_ref_timeout.rs @@ -0,0 +1,123 @@ +use crate::AdAdapter; +use crate::actions::conversion::action_from_c; +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::commands::timeout::decode_ref_action_timeout; +use crate::convert::string::{optional_adapter_string, required_adapter_string}; +use crate::error::{AdResult, set_last_error}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use crate::types::{AdAction, AdPolicyKind}; +use agent_desktop_core::refs::validate_ref_id; +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::ffi::c_char; +use std::ptr; + +/// Same as `ad_execute_by_ref` but with an explicit pre-action auto-wait +/// budget in milliseconds. `timeout_ms == -1` uses the 5000ms default and +/// `timeout_ms == 0` disables auto-wait for a single-shot preflight. +/// +/// # Safety +/// +/// Same pointer and threading requirements as `ad_execute_by_ref`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_execute_by_ref_timeout( + adapter: *const AdAdapter, + ref_id: *const c_char, + snapshot_id: *const c_char, + action: *const AdAction, + policy: i32, + timeout_ms: i64, + out: *mut *mut c_char, +) -> AdResult { + guard_non_null!(out, c"out is null"); + unsafe { *out = ptr::null_mut() }; + trap_panic(|| { + guard_non_null!(adapter, c"adapter is null"); + guard_non_null!(action, c"action is null"); + let timeout_ms = match decode_ref_action_timeout(timeout_ms) { + Ok(timeout_ms) => timeout_ms, + Err(err) => { + set_last_error(&err); + return AdResult::ErrInvalidArgs; + } + }; + + let ref_str = match required_adapter_string(ref_id, "ref_id") { + Ok(s) => s, + Err(e) => { + set_last_error(&e); + return AdResult::ErrInvalidArgs; + } + }; + + if let Err(app_err) = validate_ref_id(&ref_str) { + let ae = app_error_to_adapter(app_err); + set_last_error(&ae); + return crate::error::last_error_code(); + } + + let snapshot_str = match optional_adapter_string(snapshot_id, "snapshot_id") { + Ok(opt) => opt, + Err(e) => { + set_last_error(&e); + return AdResult::ErrInvalidArgs; + } + }; + if ref_str.starts_with("@e") && snapshot_str.is_none() { + set_last_error(&AdapterError::new( + ErrorCode::InvalidArgs, + "Bare refs require an explicit snapshot_id", + )); + return AdResult::ErrInvalidArgs; + } + + let caller_policy = match AdPolicyKind::from_c(policy) { + Some(p) => p, + None => { + set_last_error(&AdapterError::new( + ErrorCode::InvalidArgs, + "invalid policy kind discriminant", + )); + return AdResult::ErrInvalidArgs; + } + }; + + let core_action = match unsafe { action_from_c(&*action) } { + Ok(a) => a, + Err(msg) => { + set_last_error(&AdapterError::new(ErrorCode::InvalidArgs, msg)); + return AdResult::ErrInvalidArgs; + } + }; + + let caller_ip = caller_policy.to_interaction_policy(); + + let adapter_ref = crate::adapter::acquire_adapter!(adapter); + let context = match adapter_ref.command_context() { + Ok(ctx) => ctx, + Err(e) => { + let ae = app_error_to_adapter(e); + set_last_error(&ae); + return crate::error::last_error_code(); + } + }; + + let scope = crate::commands::mutating_command_scope!(context, "execute_by_ref"); + + let result = agent_desktop_core::commands::execute_by_ref::execute_with_timeout( + agent_desktop_core::commands::execute_by_ref::ExecuteByRefArgs { + ref_id: &ref_str, + snapshot_id: snapshot_str.as_deref(), + action: core_action, + caller_policy: caller_ip, + }, + timeout_ms, + adapter_ref.inner.as_ref(), + &context, + ); + crate::commands::complete_scope!(scope, &result); + + unsafe { write_command_envelope("execute_by_ref", result, out) } + }) +} diff --git a/crates/ffi/src/commands/generated.rs b/crates/ffi/src/commands/generated.rs deleted file mode 100644 index 424dc4c..0000000 --- a/crates/ffi/src/commands/generated.rs +++ /dev/null @@ -1,597 +0,0 @@ -//! @generated — produced by crates/ffi/build.rs codegen. -//! Edit the templates under crates/ffi/codegen_templates/, not this file. -//! Commands in alphabetical order: execute_by_ref, snapshot, status, trace_export, trace_show, version, wait. - -use crate::AdAdapter; -use crate::actions::conversion::action_from_c; -use crate::commands::app_error_to_adapter; -use crate::commands::envelope_out::write_command_envelope; -use crate::convert::string::{ - decode_optional_filter, optional_adapter_string, required_adapter_string, -}; -use crate::convert::surface::snapshot_surface_from_c; -use crate::error::{self, AdResult, set_last_error}; -use crate::ffi_try::trap_panic; -use crate::main_thread::require_main_thread; -use crate::pointer_guard::guard_non_null; -use crate::types::wait_args::AdWaitArgs; -use crate::types::{AdAction, AdPolicyKind}; -use agent_desktop_core::commands::snapshot::SnapshotArgs; -use agent_desktop_core::commands::status::execute_with_report_with_context; -use agent_desktop_core::commands::wait::{WaitArgs, WaitModeArgs, WaitPredicateArgs}; -use agent_desktop_core::error::{AdapterError, AppError, ErrorCode}; -use agent_desktop_core::refs::validate_ref_id; -use std::ffi::c_char; -use std::ptr; - -/// Drives a ref action (`@e5`, action) through the canonical ref-action -/// pipeline: `RefStore` load → `RefMap` lookup (→ `STALE_REF` on missing) → -/// strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live -/// actionability preflight → dispatch → handle release. -/// -/// Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type` -/// command); `PressKey` shares that `focus_fallback` base (a ref-targeted key -/// press may need the target focused); every other action defaults to -/// `headless`. An explicit `policy` discriminant may *elevate* to headed but -/// must not downgrade an action below its base. Base and elevation are computed -/// by `agent_desktop_core::commands::execute_by_ref::execute` via -/// `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and -/// FFI share a single source of policy truth. -/// -/// `ref_id` tri-state: null → `ErrInvalidArgs`; non-null invalid UTF-8 → -/// `ErrInvalidArgs`; valid UTF-8 but bad `@e{N}` format → `ErrInvalidArgs`. -/// -/// `snapshot_id` tri-state: null → use the latest snapshot for the session -/// (CLI `--snapshot` omitted); valid UTF-8 → pin to that snapshot id; non-null -/// invalid UTF-8 → `ErrInvalidArgs`. -/// -/// `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback, -/// 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)` -/// accepts the action's own CLI base (so `TypeText` still uses -/// `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks. -/// -/// On success `*out` is set to a NUL-terminated JSON envelope (command -/// `"execute_by_ref"`); free with `ad_free_string`. On guard or decode -/// failure (invalid args before the command runs) `*out` remains null. -/// On a command-level error (STALE_REF, AMBIGUOUS_TARGET, etc.) `*out` -/// holds the error JSON envelope and must still be freed with -/// `ad_free_string`. The last-error slot is populated on all failures. -/// -/// **Dispatch-before-serialize ordering**: the action is dispatched (and any -/// side effects committed) before the result JSON is serialized. In the -/// near-impossible event that serialization of an already-valid -/// `ActionResult` fails, `*out` is null and `ErrInternal` is returned while -/// the side effect has already occurred. No pre-validation machinery is -/// needed because serialization of a valid envelope effectively never fails. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer from `ad_adapter_create[_with_session]`. -/// `ref_id` must be a non-null pointer to a NUL-terminated C string within -/// `AD_MAX_STRING_BYTES + 1` bytes; null is **not** optional — it is defined -/// behaviour (no UB) but is rejected immediately with `ErrInvalidArgs`. -/// `snapshot_id` may be null (meaning: use the latest snapshot for this -/// session) or a non-null NUL-terminated C string within -/// `AD_MAX_STRING_BYTES + 1` bytes. `action` must be a non-null pointer to a -/// valid `AdAction`. `out` must be a non-null writable pointer. All pointers -/// must remain valid for the duration of the call. Must be called from the -/// main thread on macOS. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_execute_by_ref( - adapter: *const AdAdapter, - ref_id: *const c_char, - snapshot_id: *const c_char, - action: *const AdAction, - policy: i32, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - trap_panic(|| { - if let Err(rc) = require_main_thread() { - return rc; - } - guard_non_null!(adapter, c"adapter is null"); - guard_non_null!(action, c"action is null"); - - let ref_str = match required_adapter_string(ref_id, "ref_id") { - Ok(s) => s, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - if let Err(app_err) = validate_ref_id(&ref_str) { - let ae = app_error_to_adapter(app_err); - set_last_error(&ae); - return crate::error::last_error_code(); - } - - let snapshot_str = match optional_adapter_string(snapshot_id, "snapshot_id") { - Ok(opt) => opt, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - let caller_policy = match AdPolicyKind::from_c(policy) { - Some(p) => p, - None => { - set_last_error(&AdapterError::new( - ErrorCode::InvalidArgs, - "invalid policy kind discriminant", - )); - return AdResult::ErrInvalidArgs; - } - }; - - let core_action = match unsafe { action_from_c(&*action) } { - Ok(a) => a, - Err(msg) => { - set_last_error(&AdapterError::new(ErrorCode::InvalidArgs, msg)); - return AdResult::ErrInvalidArgs; - } - }; - - let caller_ip = caller_policy.to_interaction_policy(); - - let adapter_ref = unsafe { &*adapter }; - let context = match adapter_ref.command_context() { - Ok(ctx) => ctx, - Err(e) => { - let ae = app_error_to_adapter(e); - set_last_error(&ae); - return crate::error::last_error_code(); - } - }; - - let scope = context.command_scope("execute_by_ref"); - - let result = agent_desktop_core::commands::execute_by_ref::execute( - &ref_str, - snapshot_str.as_deref(), - core_action, - caller_ip, - adapter_ref.inner.as_ref(), - &context, - ); - scope.complete(&result); - - unsafe { write_command_envelope("execute_by_ref", result, out) } - }) -} - -/// Takes a full CLI-format snapshot of the target application window, -/// allocates `@e` refs for all interactive elements, persists the refmap -/// to disk, and writes the JSON envelope into `*out`. -/// -/// The JSON shape matches `agent-desktop snapshot`: -/// `{"version":"2.0","ok":true,"command":"snapshot","data":{"app":"...","window":{...},"ref_count":N,"snapshot_id":"...","tree":{...}}}`. -/// -/// **`*out` ownership and error behaviour:** -/// - On success (`AD_RESULT_OK`): `*out` is a heap-allocated JSON string with `"ok":true`. -/// Caller must free it with `ad_free_string`. -/// - On a command-level error (e.g. app not found, snapshot failure): `*out` is a -/// heap-allocated JSON string with `"ok":false` and an `"error"` payload. Caller -/// must still free it with `ad_free_string`. The last-error slot is also set. -/// - On an argument or infrastructure error (null adapter, off-main-thread, invalid -/// UTF-8, bad surface discriminant, context failure): `*out` is set to null and no -/// allocation is made. Only the last-error slot is set. -/// -/// `app` is tri-state: -/// - null — snapshot the currently focused window (same as running the command with no `--app`). -/// - valid UTF-8 string — snapshot the named application's focused window. -/// - non-null but invalid UTF-8 or exceeding `AD_MAX_STRING_BYTES` — returns `ErrInvalidArgs`. -/// -/// `surface` is an `AdSnapshotSurface` discriminant (0 = Window, 1 = Focused, …). -/// An out-of-range value returns `ErrInvalidArgs`. -/// -/// This entrypoint always targets the active focused window of the requested -/// application; explicit window targeting (`window_id`) is not yet exposed -/// over the ABI. Progressive traversal (skeleton mode and `--root` drill-down) -/// is likewise not exposed here. Both are planned fast-follows to this -/// entrypoint — agents needing them should use the CLI in the meantime. -/// -/// **Dispatch-before-serialize ordering**: the snapshot and refmap persistence -/// occur before the result JSON is serialised. In the near-impossible event -/// that serialisation of an already-valid result fails, `*out` is set to null -/// and `ErrInternal` is returned while the refmap is already written. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer from `ad_adapter_create` or -/// `ad_adapter_create_with_session`. `out` must be a non-null writable -/// `*mut *mut c_char`. `app` must be null or a NUL-terminated string within -/// `AD_MAX_STRING_BYTES + 1` bytes. All pointers must remain valid for the -/// duration of the call. `adapter` must be used from the main thread on macOS. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_snapshot( - adapter: *const AdAdapter, - app: *const c_char, - surface: i32, - max_depth: u8, - interactive_only: bool, - compact: bool, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - trap_panic(|| { - if let Err(rc) = require_main_thread() { - return rc; - } - guard_non_null!(adapter, c"adapter is null"); - - let app_filter = unsafe { decode_optional_filter!(app, "app") }; - - let core_surface = match snapshot_surface_from_c(surface, "surface") { - Ok(s) => s, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - let adapter_ref = unsafe { &*adapter }; - let context = match adapter_ref.command_context() { - Ok(ctx) => ctx, - Err(e) => { - let ae = app_error_to_adapter(e); - set_last_error(&ae); - return crate::error::last_error_code(); - } - }; - - let args = SnapshotArgs { - app: app_filter, - window_id: None, - max_depth, - include_bounds: false, - interactive_only, - compact, - surface: core_surface, - skeleton: false, - root_ref: None, - snapshot_id: None, - }; - - let scope = context.command_scope("snapshot"); - - let result = agent_desktop_core::commands::snapshot::execute( - args, - adapter_ref.inner.as_ref(), - &context, - ); - scope.complete(&result); - - unsafe { write_command_envelope("snapshot", result, out) } - }) -} - -/// Returns the adapter's current health and permission state as a JSON -/// envelope matching the `agent-desktop status` CLI output. -/// -/// `ad_status` does not query the accessibility tree; it reads the -/// permission report and ref-store metadata only, so it is safe to call -/// from any thread (unlike tree-traversal commands that require the -/// macOS main thread). On success `*out` is a NUL-terminated, -/// heap-allocated JSON string freed with `ad_free_string`. -/// -/// On a command-level failure `*out` is set to a heap-allocated JSON string -/// with `"ok":false` and an `"error"` payload. The caller must still release -/// it with `ad_free_string(*out)`. The last-error slot is also set. -/// -/// On an argument or infrastructure failure (null adapter, null out, context -/// error) `*out` is zeroed and only the last-error slot is populated. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer returned by `ad_adapter_create` -/// that has not been destroyed. `out` must be a non-null writable -/// `*mut *mut c_char`. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_status( - adapter: *const crate::AdAdapter, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - guard_non_null!(adapter, c"adapter is null"); - - trap_panic(|| { - let adapter = unsafe { &*adapter }; - - let ctx = match adapter.command_context() { - Ok(c) => c, - Err(app_err) => { - let ae = app_error_to_adapter(app_err); - error::set_last_error(&ae); - return error::last_error_code(); - } - }; - - let report = adapter.inner.permission_report(); - - let scope = ctx.command_scope("status"); - - let result: Result<serde_json::Value, AppError> = - execute_with_report_with_context(&*adapter.inner, &report, &ctx); - scope.complete(&result); - - unsafe { write_command_envelope("status", result, out) } - }) -} - -/// Exports the merged trace timeline for the adapter's active session as a -/// single self-contained HTML file matching `agent-desktop trace export`. -/// -/// `limit` controls tail semantics: `0` embeds all events; the default `5000` -/// matches the CLI. Pass `-1` to use the CLI default explicitly. -/// -/// `out_path` may be null; when set it must be a NUL-terminated UTF-8 path -/// within `AD_MAX_STRING_BYTES + 1` bytes. -/// -/// On success `*out` is a heap-allocated JSON envelope freed with -/// `ad_free_string`. On command-level failure `*out` still holds an error -/// envelope that must be freed. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer from `ad_adapter_create` or -/// `ad_adapter_create_with_session`. `out` must be non-null. `out_path` -/// may be null or a NUL-terminated UTF-8 string within `AD_MAX_STRING_BYTES + 1` -/// bytes. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_trace_export( - adapter: *const AdAdapter, - limit: i32, - out_path: *const c_char, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - trap_panic(|| { - guard_non_null!(adapter, c"adapter is null"); - - let path = match optional_adapter_string(out_path, "out_path") { - Ok(value) => value, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - let effective_limit = if limit < 0 { - agent_desktop_core::trace_read::TRACE_EXPORT_DEFAULT_LIMIT - } else { - limit as usize - }; - - let adapter_ref = unsafe { &*adapter }; - let context = match adapter_ref.command_context() { - Ok(ctx) => ctx, - Err(e) => { - let ae = app_error_to_adapter(e); - set_last_error(&ae); - return crate::error::last_error_code(); - } - }; - - let scope = context.command_scope("trace"); - let result = agent_desktop_core::commands::trace::execute( - agent_desktop_core::commands::trace::TraceAction::Export { - limit: effective_limit, - out: path.map(std::path::PathBuf::from), - }, - &context, - ); - scope.complete(&result); - - unsafe { write_command_envelope("trace", result, out) } - }) -} - -/// Returns the merged trace timeline for the adapter's active session as a -/// JSON envelope matching `agent-desktop trace show`. -/// -/// `limit` controls tail semantics: `0` embeds all events; the default `500` -/// matches the CLI. Pass `-1` to use the CLI default explicitly. -/// -/// `event_prefix` may be null; when set, only events whose name starts with the -/// prefix are returned before the tail limit is applied. -/// -/// On success `*out` is a heap-allocated JSON envelope freed with -/// `ad_free_string`. On command-level failure `*out` still holds an error -/// envelope that must be freed. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer from `ad_adapter_create` or -/// `ad_adapter_create_with_session`. `out` must be non-null. `event_prefix` -/// may be null or a NUL-terminated UTF-8 string within `AD_MAX_STRING_BYTES + 1` -/// bytes. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_trace_show( - adapter: *const AdAdapter, - limit: i32, - event_prefix: *const c_char, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - trap_panic(|| { - guard_non_null!(adapter, c"adapter is null"); - - let event = match optional_adapter_string(event_prefix, "event_prefix") { - Ok(value) => value, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - - let effective_limit = if limit < 0 { - agent_desktop_core::commands::trace::TRACE_SHOW_DEFAULT_LIMIT - } else { - limit as usize - }; - - let adapter_ref = unsafe { &*adapter }; - let context = match adapter_ref.command_context() { - Ok(ctx) => ctx, - Err(e) => { - let ae = app_error_to_adapter(e); - set_last_error(&ae); - return crate::error::last_error_code(); - } - }; - - let scope = context.command_scope("trace"); - let result = agent_desktop_core::commands::trace::execute( - agent_desktop_core::commands::trace::TraceAction::Show { - limit: effective_limit, - event, - }, - &context, - ); - scope.complete(&result); - - unsafe { write_command_envelope("trace", result, out) } - }) -} - -/// Returns the `agent-desktop` version envelope as an owned JSON C string. -/// -/// The returned string has the same `{version, ok, command, data}` shape -/// as `agent-desktop version` on the CLI. Free it with `ad_free_string`. -/// -/// On success `*out` points to the envelope JSON. -/// On error `*out` is null and the last-error slot is populated. -/// -/// # Safety -/// `out` must be a non-null writable `*mut *mut c_char`. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_version(out: *mut *mut c_char) -> AdResult { - trap_panic(|| unsafe { - guard_non_null!(out, c"out is null"); - *out = ptr::null_mut(); - let context = match agent_desktop_core::context::CommandContext::new(None, None, false) { - Ok(ctx) => ctx, - Err(app_err) => { - let ae = app_error_to_adapter(app_err); - set_last_error(&ae); - return crate::error::last_error_code(); - } - }; - let scope = context.command_scope("version"); - let result = agent_desktop_core::commands::version::execute(); - scope.complete(&result); - write_command_envelope("version", result, out) - }) -} - -/// Runs `wait` with the given args, blocking the calling thread until the -/// condition is met or `timeout_ms` elapses. -/// -/// On success `*out` is set to a freshly allocated JSON string containing the -/// CLI-format wait envelope (`{version, ok, command, data}`). The caller must -/// release the string with `ad_free_string(*out)`. -/// -/// On a command-level failure (e.g. `TIMEOUT`, `ELEMENT_NOT_FOUND`) `*out` is -/// set to a freshly allocated JSON string with `"ok":false` and an `"error"` -/// payload. The caller must still release it with `ad_free_string(*out)`. The -/// last-error slot is also set. -/// -/// On an argument or infrastructure failure (null adapter, null args, null out, -/// off-main-thread, invalid UTF-8 field) `*out` is zeroed, the last-error slot -/// is set, and a negative `AdResult` code is returned. No allocation is made. -/// -/// # Safety -/// -/// `adapter` must be a non-null pointer returned by `ad_adapter_create` that -/// has not been destroyed. `args` must be non-null and point to a valid -/// zero-initialized `AdWaitArgs`. `out` must be non-null and point to a -/// writable `*mut c_char`. -/// -/// All `*const c_char` fields inside `AdWaitArgs` must be null or point to -/// readable, NUL-terminated memory within `AD_MAX_STRING_BYTES + 1` bytes. -/// -/// `ad_wait` blocks the calling thread for up to `timeout_ms` milliseconds -/// while it holds a live reference into the adapter's allocation. The adapter -/// must outlive the call: do not call `ad_adapter_destroy` on this handle from -/// another thread while `ad_wait` is running — that is a use-after-free. Ensure -/// the wait has returned before destroying the adapter. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn ad_wait( - adapter: *const AdAdapter, - args: *const AdWaitArgs, - out: *mut *mut c_char, -) -> AdResult { - guard_non_null!(out, c"out is null"); - unsafe { *out = ptr::null_mut() }; - guard_non_null!(args, c"args is null"); - - trap_panic(|| { - if let Err(rc) = require_main_thread() { - return rc; - } - guard_non_null!(adapter, c"adapter is null"); - - let args = unsafe { &*args }; - let adapter_ref = unsafe { &*adapter }; - - let ms = args.has_ms.then_some(args.ms); - - let element = unsafe { decode_optional_filter!(args.element, "element") }; - let window = unsafe { decode_optional_filter!(args.window, "window") }; - let text = unsafe { decode_optional_filter!(args.text, "text") }; - let snapshot_id = unsafe { decode_optional_filter!(args.snapshot_id, "snapshot_id") }; - let predicate = unsafe { decode_optional_filter!(args.predicate, "predicate") }; - let value = unsafe { decode_optional_filter!(args.value, "value") }; - let action_field = unsafe { decode_optional_filter!(args.action, "action") }; - let app = unsafe { decode_optional_filter!(args.app, "app") }; - - let wait_args = WaitArgs { - mode: WaitModeArgs { - ms, - element, - window, - text, - menu: args.menu, - menu_closed: args.menu_closed, - notification: args.notification, - }, - predicate: WaitPredicateArgs { - snapshot_id, - predicate, - value, - action: action_field, - count: args.has_count.then_some(args.count), - }, - timeout_ms: args.timeout_ms, - app, - }; - - let ctx = match adapter_ref.command_context() { - Ok(c) => c, - Err(app_err) => { - let adapter_err = app_error_to_adapter(app_err); - error::set_last_error(&adapter_err); - return error::last_error_code(); - } - }; - - let scope = ctx.command_scope("wait"); - - let result = agent_desktop_core::commands::wait::execute( - wait_args, - adapter_ref.inner.as_ref(), - &ctx, - ); - scope.complete(&result); - - unsafe { write_command_envelope("wait", result, out) } - }) -} diff --git a/crates/ffi/src/commands/mod.rs b/crates/ffi/src/commands/mod.rs index 5b3d093..4c5d731 100644 --- a/crates/ffi/src/commands/mod.rs +++ b/crates/ffi/src/commands/mod.rs @@ -1,7 +1,15 @@ pub(crate) mod envelope_out; -pub(crate) mod generated; +pub(crate) mod execute_by_ref; +pub(crate) mod execute_by_ref_timeout; +pub(crate) mod snapshot; +pub(crate) mod status; +pub(crate) mod timeout; +pub(crate) mod trace_export; +pub(crate) mod trace_show; +pub(crate) mod version; +pub(crate) mod wait; -use agent_desktop_core::error::{AdapterError, AppError, ErrorCode}; +use agent_desktop_core::{AdapterError, AppError, ErrorCode}; /// Converts a core `AppError` into an `AdapterError` for use with /// `set_last_error`. `AppError::Adapter` is already an `AdapterError`; @@ -15,6 +23,46 @@ pub(crate) fn app_error_to_adapter(err: AppError) -> AdapterError { } } +macro_rules! command_scope { + ($context:expr, $name:expr) => {{ + match $context.command_scope($name) { + Ok(scope) => scope, + Err(error) => { + let error = $crate::commands::app_error_to_adapter(error); + $crate::error::set_last_error(&error); + return $crate::error::last_error_code(); + } + } + }}; +} + +macro_rules! mutating_command_scope { + ($context:expr, $name:expr) => {{ + match $context.mutating_command_scope($name) { + Ok(scope) => scope, + Err(error) => { + let error = $crate::commands::app_error_to_adapter(error); + $crate::error::set_last_error(&error); + return $crate::error::last_error_code(); + } + } + }}; +} + +macro_rules! complete_scope { + ($scope:expr, $result:expr) => {{ + if let Err(error) = $scope.complete($result) { + let error = $crate::commands::app_error_to_adapter(error); + $crate::error::set_last_error(&error); + return $crate::error::last_error_code(); + } + }}; +} + +pub(crate) use command_scope; +pub(crate) use complete_scope; +pub(crate) use mutating_command_scope; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ffi/codegen_templates/snapshot.rs.in b/crates/ffi/src/commands/snapshot.rs similarity index 83% rename from crates/ffi/codegen_templates/snapshot.rs.in rename to crates/ffi/src/commands/snapshot.rs index 210666b..fe88be9 100644 --- a/crates/ffi/codegen_templates/snapshot.rs.in +++ b/crates/ffi/src/commands/snapshot.rs @@ -1,10 +1,21 @@ +use crate::AdAdapter; +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::convert::string::decode_optional_filter; +use crate::convert::surface::snapshot_surface_from_c; +use crate::error::{AdResult, set_last_error}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use agent_desktop_core::commands::snapshot::SnapshotArgs; +use std::ffi::c_char; +use std::ptr; /// Takes a full CLI-format snapshot of the target application window, /// allocates `@e` refs for all interactive elements, persists the refmap /// to disk, and writes the JSON envelope into `*out`. /// /// The JSON shape matches `agent-desktop snapshot`: -/// `{"version":"2.0","ok":true,"command":"snapshot","data":{"app":"...","window":{...},"ref_count":N,"snapshot_id":"...","tree":{...}}}`. +/// `{"version":"2.1","ok":true,"command":"snapshot","data":{"app":"...","window":{...},"ref_count":N,"snapshot_id":"...","tree":{...}}}`. /// /// **`*out` ownership and error behaviour:** /// - On success (`AD_RESULT_OK`): `*out` is a heap-allocated JSON string with `"ok":true`. @@ -12,7 +23,7 @@ /// - On a command-level error (e.g. app not found, snapshot failure): `*out` is a /// heap-allocated JSON string with `"ok":false` and an `"error"` payload. Caller /// must still free it with `ad_free_string`. The last-error slot is also set. -/// - On an argument or infrastructure error (null adapter, off-main-thread, invalid +/// - On an argument or infrastructure error (null adapter, invalid /// UTF-8, bad surface discriminant, context failure): `*out` is set to null and no /// allocation is made. Only the last-error slot is set. /// @@ -41,7 +52,7 @@ /// `ad_adapter_create_with_session`. `out` must be a non-null writable /// `*mut *mut c_char`. `app` must be null or a NUL-terminated string within /// `AD_MAX_STRING_BYTES + 1` bytes. All pointers must remain valid for the -/// duration of the call. `adapter` must be used from the main thread on macOS. +/// duration of the call. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_snapshot( adapter: *const AdAdapter, @@ -55,9 +66,6 @@ pub unsafe extern "C" fn ad_snapshot( guard_non_null!(out, c"out is null"); unsafe { *out = ptr::null_mut() }; trap_panic(|| { - if let Err(rc) = require_main_thread() { - return rc; - } guard_non_null!(adapter, c"adapter is null"); let app_filter = unsafe { decode_optional_filter!(app, "app") }; @@ -69,8 +77,7 @@ pub unsafe extern "C" fn ad_snapshot( return AdResult::ErrInvalidArgs; } }; - - let adapter_ref = unsafe { &*adapter }; + let adapter_ref = crate::adapter::acquire_adapter!(adapter); let context = match adapter_ref.command_context() { Ok(ctx) => ctx, Err(e) => { @@ -93,14 +100,14 @@ pub unsafe extern "C" fn ad_snapshot( snapshot_id: None, }; - let scope = context.command_scope("snapshot"); + let scope = crate::commands::command_scope!(context, "snapshot"); let result = agent_desktop_core::commands::snapshot::execute( args, adapter_ref.inner.as_ref(), &context, ); - scope.complete(&result); + crate::commands::complete_scope!(scope, &result); unsafe { write_command_envelope("snapshot", result, out) } }) diff --git a/crates/ffi/codegen_templates/status.rs.in b/crates/ffi/src/commands/status.rs similarity index 58% rename from crates/ffi/codegen_templates/status.rs.in rename to crates/ffi/src/commands/status.rs index a617786..68ead08 100644 --- a/crates/ffi/codegen_templates/status.rs.in +++ b/crates/ffi/src/commands/status.rs @@ -1,12 +1,20 @@ +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::error::{self, AdResult}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use agent_desktop_core::AppError; +use agent_desktop_core::commands::status::execute_with_report_with_context; +use std::ffi::c_char; +use std::ptr; /// Returns the adapter's current health and permission state as a JSON /// envelope matching the `agent-desktop status` CLI output. /// /// `ad_status` does not query the accessibility tree; it reads the -/// permission report and ref-store metadata only, so it is safe to call -/// from any thread (unlike tree-traversal commands that require the -/// macOS main thread). On success `*out` is a NUL-terminated, -/// heap-allocated JSON string freed with `ad_free_string`. +/// permission report and ref-store metadata only. Like other adapter +/// entrypoints, it may be called from any host thread. On success `*out` is a +/// NUL-terminated, heap-allocated JSON string freed with `ad_free_string`. /// /// On a command-level failure `*out` is set to a heap-allocated JSON string /// with `"ok":false` and an `"error"` payload. The caller must still release @@ -30,7 +38,7 @@ pub unsafe extern "C" fn ad_status( guard_non_null!(adapter, c"adapter is null"); trap_panic(|| { - let adapter = unsafe { &*adapter }; + let adapter = crate::adapter::acquire_adapter!(adapter); let ctx = match adapter.command_context() { Ok(c) => c, @@ -41,13 +49,20 @@ pub unsafe extern "C" fn ad_status( } }; - let report = adapter.inner.permission_report(); + let deadline = crate::operation::operation_deadline!(); + let report = match adapter.inner.permission_report(deadline) { + Ok(report) => report, + Err(error) => { + error::set_last_error(&error); + return error::last_error_code(); + } + }; - let scope = ctx.command_scope("status"); + let scope = crate::commands::command_scope!(ctx, "status"); let result: Result<serde_json::Value, AppError> = execute_with_report_with_context(&*adapter.inner, &report, &ctx); - scope.complete(&result); + crate::commands::complete_scope!(scope, &result); unsafe { write_command_envelope("status", result, out) } }) diff --git a/crates/ffi/src/commands/timeout.rs b/crates/ffi/src/commands/timeout.rs new file mode 100644 index 0000000..1903d25 --- /dev/null +++ b/crates/ffi/src/commands/timeout.rs @@ -0,0 +1,30 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; + +pub(crate) fn decode_ref_action_timeout(timeout_ms: i64) -> Result<u64, AdapterError> { + match timeout_ms { + -1 => Ok(5_000), + 0.. => Ok(timeout_ms as u64), + _ => Err(AdapterError::new( + ErrorCode::InvalidArgs, + "timeout_ms must be -1 for the default, 0 for single-shot, or a positive millisecond budget", + )), + } +} + +#[cfg(test)] +mod tests { + use super::decode_ref_action_timeout; + + #[test] + fn signed_timeout_preserves_default_and_single_shot_sentinels() { + assert_eq!(decode_ref_action_timeout(-1).unwrap(), 5_000); + assert_eq!(decode_ref_action_timeout(0).unwrap(), 0); + assert_eq!(decode_ref_action_timeout(250).unwrap(), 250); + } + + #[test] + fn signed_timeout_rejects_values_below_default_sentinel() { + let err = decode_ref_action_timeout(-2).unwrap_err(); + assert_eq!(err.code.as_str(), "INVALID_ARGS"); + } +} diff --git a/crates/ffi/codegen_templates/trace_export.rs.in b/crates/ffi/src/commands/trace_export.rs similarity index 80% rename from crates/ffi/codegen_templates/trace_export.rs.in rename to crates/ffi/src/commands/trace_export.rs index ae5288b..b1db7f4 100644 --- a/crates/ffi/codegen_templates/trace_export.rs.in +++ b/crates/ffi/src/commands/trace_export.rs @@ -1,3 +1,12 @@ +use crate::AdAdapter; +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::convert::string::optional_adapter_string; +use crate::error::{AdResult, set_last_error}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use std::ffi::c_char; +use std::ptr; /// Exports the merged trace timeline for the adapter's active session as a /// single self-contained HTML file matching `agent-desktop trace export`. @@ -44,7 +53,7 @@ pub unsafe extern "C" fn ad_trace_export( limit as usize }; - let adapter_ref = unsafe { &*adapter }; + let adapter_ref = crate::adapter::acquire_adapter!(adapter); let context = match adapter_ref.command_context() { Ok(ctx) => ctx, Err(e) => { @@ -54,7 +63,7 @@ pub unsafe extern "C" fn ad_trace_export( } }; - let scope = context.command_scope("trace"); + let scope = crate::commands::mutating_command_scope!(context, "trace"); let result = agent_desktop_core::commands::trace::execute( agent_desktop_core::commands::trace::TraceAction::Export { limit: effective_limit, @@ -62,7 +71,7 @@ pub unsafe extern "C" fn ad_trace_export( }, &context, ); - scope.complete(&result); + crate::commands::complete_scope!(scope, &result); unsafe { write_command_envelope("trace", result, out) } }) diff --git a/crates/ffi/codegen_templates/trace_show.rs.in b/crates/ffi/src/commands/trace_show.rs similarity index 80% rename from crates/ffi/codegen_templates/trace_show.rs.in rename to crates/ffi/src/commands/trace_show.rs index e36fe53..4c58a99 100644 --- a/crates/ffi/codegen_templates/trace_show.rs.in +++ b/crates/ffi/src/commands/trace_show.rs @@ -1,3 +1,12 @@ +use crate::AdAdapter; +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::convert::string::optional_adapter_string; +use crate::error::{AdResult, set_last_error}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use std::ffi::c_char; +use std::ptr; /// Returns the merged trace timeline for the adapter's active session as a /// JSON envelope matching `agent-desktop trace show`. @@ -44,7 +53,7 @@ pub unsafe extern "C" fn ad_trace_show( limit as usize }; - let adapter_ref = unsafe { &*adapter }; + let adapter_ref = crate::adapter::acquire_adapter!(adapter); let context = match adapter_ref.command_context() { Ok(ctx) => ctx, Err(e) => { @@ -54,7 +63,7 @@ pub unsafe extern "C" fn ad_trace_show( } }; - let scope = context.command_scope("trace"); + let scope = crate::commands::command_scope!(context, "trace"); let result = agent_desktop_core::commands::trace::execute( agent_desktop_core::commands::trace::TraceAction::Show { limit: effective_limit, @@ -62,7 +71,7 @@ pub unsafe extern "C" fn ad_trace_show( }, &context, ); - scope.complete(&result); + crate::commands::complete_scope!(scope, &result); unsafe { write_command_envelope("trace", result, out) } }) diff --git a/crates/ffi/codegen_templates/version.rs.in b/crates/ffi/src/commands/version.rs similarity index 73% rename from crates/ffi/codegen_templates/version.rs.in rename to crates/ffi/src/commands/version.rs index 9a6b159..2e4a98d 100644 --- a/crates/ffi/codegen_templates/version.rs.in +++ b/crates/ffi/src/commands/version.rs @@ -1,3 +1,10 @@ +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::error::{AdResult, set_last_error}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use std::ffi::c_char; +use std::ptr; /// Returns the `agent-desktop` version envelope as an owned JSON C string. /// @@ -22,9 +29,9 @@ pub unsafe extern "C" fn ad_version(out: *mut *mut c_char) -> AdResult { return crate::error::last_error_code(); } }; - let scope = context.command_scope("version"); + let scope = crate::commands::command_scope!(context, "version"); let result = agent_desktop_core::commands::version::execute(); - scope.complete(&result); + crate::commands::complete_scope!(scope, &result); write_command_envelope("version", result, out) }) } diff --git a/crates/ffi/src/commands/wait.rs b/crates/ffi/src/commands/wait.rs new file mode 100644 index 0000000..33ccf5b --- /dev/null +++ b/crates/ffi/src/commands/wait.rs @@ -0,0 +1,120 @@ +use crate::AdAdapter; +use crate::commands::app_error_to_adapter; +use crate::commands::envelope_out::write_command_envelope; +use crate::convert::string::optional_adapter_string; +use crate::error::{self, AdResult}; +use crate::ffi_try::trap_panic; +use crate::pointer_guard::guard_non_null; +use crate::types::wait_args::AdWaitArgs; +use agent_desktop_core::AdapterError; +use agent_desktop_core::commands::wait::{WaitArgs, WaitModeArgs, WaitPredicateArgs}; +use agent_desktop_core::commands::wait_surface::SurfaceWait; +use std::ffi::c_char; +use std::ptr; + +/// Runs `wait` with the given args, blocking the calling thread until the +/// condition is met or `timeout_ms` elapses. +/// +/// On success `*out` is set to a freshly allocated JSON string containing the +/// CLI-format wait envelope (`{version, ok, command, data}`). The caller must +/// release the string with `ad_free_string(*out)`. +/// +/// On a command-level failure (e.g. `TIMEOUT`, `ELEMENT_NOT_FOUND`) `*out` is +/// set to a freshly allocated JSON string with `"ok":false` and an `"error"` +/// payload. The caller must still release it with `ad_free_string(*out)`. The +/// last-error slot is also set. +/// +/// On an argument or infrastructure failure (null adapter, null args, null out, +/// invalid UTF-8 field) `*out` is zeroed, the last-error slot +/// is set, and a negative `AdResult` code is returned. No allocation is made. +/// +/// # Safety +/// +/// `adapter` must be a non-null pointer returned by `ad_adapter_create` that +/// has not been destroyed. `args` must be non-null and point to a valid +/// zero-initialized `AdWaitArgs`. `out` must be non-null and point to a +/// writable `*mut c_char`. +/// +/// All `*const c_char` fields inside `AdWaitArgs` must be null or point to +/// readable, NUL-terminated memory within `AD_MAX_STRING_BYTES + 1` bytes. +/// +/// `ad_wait` retains the adapter while blocked. Concurrent destruction revokes +/// the opaque adapter token for new calls without invalidating this call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_wait( + adapter: *const AdAdapter, + args: *const AdWaitArgs, + out: *mut *mut c_char, +) -> AdResult { + guard_non_null!(out, c"out is null"); + unsafe { *out = ptr::null_mut() }; + guard_non_null!(args, c"args is null"); + + trap_panic(|| { + guard_non_null!(adapter, c"adapter is null"); + + let ffi_args = unsafe { &*args }; + let mut wait_args = match wait_args_from_ffi(ffi_args) { + Ok(args) => args, + Err(err) => { + error::set_last_error(&err); + return error::last_error_code(); + } + }; + let adapter_ref = crate::adapter::acquire_adapter!(adapter); + + let ctx = match adapter_ref.command_context() { + Ok(c) => c, + Err(app_err) => { + let adapter_err = app_error_to_adapter(app_err); + error::set_last_error(&adapter_err); + return error::last_error_code(); + } + }; + + let scope = crate::commands::command_scope!(ctx, "wait"); + + let result = SurfaceWait::from_flags( + ffi_args.mode.surfaces.menu, + ffi_args.mode.surfaces.menu_closed, + ffi_args.mode.surfaces.notification, + ) + .and_then(|surface| { + wait_args.mode.surface = surface; + agent_desktop_core::commands::wait::execute(wait_args, adapter_ref.inner.as_ref(), &ctx) + }); + crate::commands::complete_scope!(scope, &result); + + unsafe { write_command_envelope("wait", result, out) } + }) +} + +fn wait_args_from_ffi(args: &AdWaitArgs) -> Result<WaitArgs, AdapterError> { + Ok(WaitArgs { + mode: WaitModeArgs { + ms: args.mode.pause.present.then_some(args.mode.pause.value), + element: optional_adapter_string(args.mode.element, "mode.element")?, + window: optional_adapter_string(args.mode.window, "mode.window")?, + text: optional_adapter_string(args.mode.text, "mode.text")?, + surface: None, + event: None, + window_id: None, + }, + predicate: WaitPredicateArgs { + snapshot_id: optional_adapter_string( + args.predicate.snapshot_id, + "predicate.snapshot_id", + )?, + predicate: optional_adapter_string(args.predicate.predicate, "predicate.kind")?, + value: optional_adapter_string(args.predicate.value, "predicate.value")?, + action: optional_adapter_string(args.predicate.action, "predicate.action")?, + count: args + .predicate + .count + .present + .then_some(args.predicate.count.value), + }, + timeout_ms: args.scope.timeout_ms, + app: optional_adapter_string(args.scope.app, "scope.app")?, + }) +} diff --git a/crates/ffi/src/convert/app.rs b/crates/ffi/src/convert/app.rs index 1bef12b..bad502d 100644 --- a/crates/ffi/src/convert/app.rs +++ b/crates/ffi/src/convert/app.rs @@ -1,13 +1,13 @@ use crate::convert::string::{free_c_string, opt_string_to_c, string_to_c_lossy}; use crate::types::AdAppInfo; -use agent_desktop_core::node::AppInfo; +use agent_desktop_core::AppInfo; use std::os::raw::c_char; use std::ptr; pub(crate) fn app_info_to_c(a: &AppInfo) -> AdAppInfo { AdAppInfo { name: string_to_c_lossy(&a.name), - pid: a.pid, + pid: a.pid.get(), bundle_id: opt_string_to_c(a.bundle_id.as_deref()), } } @@ -30,8 +30,9 @@ mod tests { fn test_app_info_roundtrip() { let a = AppInfo { name: "Finder".into(), - pid: 42, + pid: agent_desktop_core::ProcessId::new(42), bundle_id: Some("com.apple.finder".into()), + process_instance: Some("42:100".into()), }; let c = app_info_to_c(&a); assert_eq!(unsafe { c_to_string(c.name) }.as_deref(), Some("Finder")); diff --git a/crates/ffi/src/convert/display.rs b/crates/ffi/src/convert/display.rs new file mode 100644 index 0000000..a1bb1aa --- /dev/null +++ b/crates/ffi/src/convert/display.rs @@ -0,0 +1,105 @@ +use crate::convert::string::{free_c_string, string_to_c_lossy}; +use crate::types::AdDisplayInfo; +use agent_desktop_core::{AdapterError, DisplayInfo, ErrorCode}; +use std::os::raw::c_char; +use std::ptr; + +pub(crate) fn validate_display_info(display: &DisplayInfo) -> Result<(), AdapterError> { + if display.id.is_empty() { + return Err(AdapterError::new( + ErrorCode::Internal, + "Display id is empty", + )); + } + crate::resource::validate_output_string(&display.id, "Display id")?; + display.bounds.validate().map_err(|error| { + AdapterError::new( + ErrorCode::Internal, + format!("Display has invalid bounds: {}", error.message), + ) + })?; + if !display.scale.is_finite() || display.scale <= 0.0 { + return Err(AdapterError::new( + ErrorCode::Internal, + "Display has invalid scale", + )); + } + Ok(()) +} + +pub(crate) fn display_info_to_c(display: &DisplayInfo) -> AdDisplayInfo { + AdDisplayInfo { + version: crate::types::display_info::AD_DISPLAY_INFO_VERSION, + size: crate::types::display_info::AD_DISPLAY_INFO_SIZE as u32, + id: string_to_c_lossy(&display.id), + bounds: crate::convert::rect_to_c(&display.bounds), + is_primary: display.is_primary, + scale: display.scale, + } +} + +pub(crate) unsafe fn free_display_info_fields(display: &mut AdDisplayInfo) { + unsafe { + free_c_string(display.id as *mut c_char); + display.id = ptr::null(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::convert::string::c_to_string; + use agent_desktop_core::Rect; + + fn display() -> DisplayInfo { + DisplayInfo { + id: "main".into(), + bounds: Rect { + x: 0.0, + y: 0.0, + width: 1920.0, + height: 1080.0, + }, + is_primary: true, + scale: 2.0, + } + } + + #[test] + fn display_conversion_preserves_targeting_fields() { + let source = display(); + validate_display_info(&source).expect("valid display"); + let mut converted = display_info_to_c(&source); + + assert_eq!( + unsafe { c_to_string(converted.id) }.as_deref(), + Some("main") + ); + assert_eq!(converted.bounds.width, 1920.0); + assert!(converted.is_primary); + assert_eq!(converted.scale, 2.0); + unsafe { free_display_info_fields(&mut converted) }; + assert!(converted.id.is_null()); + } + + #[test] + fn display_validation_rejects_invalid_platform_output() { + let mut source = display(); + source.scale = f64::NAN; + assert_eq!( + validate_display_info(&source) + .expect_err("invalid scale") + .code, + ErrorCode::Internal + ); + + source.scale = 2.0; + source.bounds.width = -1.0; + assert_eq!( + validate_display_info(&source) + .expect_err("invalid bounds") + .code, + ErrorCode::Internal + ); + } +} diff --git a/crates/ffi/src/convert/mod.rs b/crates/ffi/src/convert/mod.rs index deece16..677b014 100644 --- a/crates/ffi/src/convert/mod.rs +++ b/crates/ffi/src/convert/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod app; +pub(crate) mod display; pub(crate) mod notification; pub(crate) mod rect; pub(crate) mod string; diff --git a/crates/ffi/src/convert/notification.rs b/crates/ffi/src/convert/notification.rs index 44cfb3c..28b640f 100644 --- a/crates/ffi/src/convert/notification.rs +++ b/crates/ffi/src/convert/notification.rs @@ -1,6 +1,6 @@ use crate::convert::string::{free_c_string, opt_string_to_c, string_to_c_lossy}; use crate::types::AdNotificationInfo; -use agent_desktop_core::notification::NotificationInfo; +use agent_desktop_core::NotificationInfo; use std::os::raw::c_char; use std::ptr; @@ -21,7 +21,7 @@ pub(crate) unsafe fn free_notification_info_fields(info: &mut AdNotificationInfo free_c_string(info.app_name as *mut c_char); free_c_string(info.title as *mut c_char); free_c_string(info.body as *mut c_char); - free_c_string_array(info.actions, info.action_count); + free_c_string_array(info.actions); info.app_name = ptr::null(); info.title = ptr::null(); info.body = ptr::null(); @@ -39,21 +39,28 @@ fn strings_to_c_array(strings: &[String]) -> (*mut *mut c_char, u32) { let mut boxed = ptrs.into_boxed_slice(); let ptr = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation( + crate::resource::AllocationKind::NotificationActions, + ptr, + count as usize, + ); (ptr, count) } -unsafe fn free_c_string_array(arr: *mut *mut c_char, count: u32) { +unsafe fn free_c_string_array(arr: *mut *mut c_char) { unsafe { - if arr.is_null() { + let Some(count) = crate::resource::take_allocation( + crate::resource::AllocationKind::NotificationActions, + arr, + ) else { return; - } - let slice = std::slice::from_raw_parts_mut(arr, count as usize); + }; + let slice = std::slice::from_raw_parts_mut(arr, count); for p in slice.iter_mut() { free_c_string(*p); } drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( - arr, - count as usize, + arr, count, ))); } } @@ -62,7 +69,7 @@ unsafe fn free_c_string_array(arr: *mut *mut c_char, count: u32) { mod tests { use super::*; use crate::convert::string::c_to_string; - use agent_desktop_core::notification::NotificationInfo; + use agent_desktop_core::NotificationInfo; use std::os::raw::c_char; fn make_info(body: Option<&str>, actions: &[&str]) -> NotificationInfo { diff --git a/crates/ffi/src/convert/rect.rs b/crates/ffi/src/convert/rect.rs index 33131da..1852777 100644 --- a/crates/ffi/src/convert/rect.rs +++ b/crates/ffi/src/convert/rect.rs @@ -1,5 +1,5 @@ use crate::types::AdRect; -use agent_desktop_core::node::Rect; +use agent_desktop_core::Rect; pub(crate) fn rect_to_c(r: &Rect) -> AdRect { AdRect { diff --git a/crates/ffi/src/convert/string.rs b/crates/ffi/src/convert/string.rs index 8e86c1d..740541d 100644 --- a/crates/ffi/src/convert/string.rs +++ b/crates/ffi/src/convert/string.rs @@ -2,11 +2,11 @@ use std::ffi::CString; use std::os::raw::c_char; use std::ptr; -use agent_desktop_core::error::{AdapterError, ErrorCode}; +use agent_desktop_core::{AdapterError, ErrorCode}; pub(crate) fn string_to_c(s: &str) -> *mut c_char { match CString::new(s) { - Ok(cs) => cs.into_raw(), + Ok(cs) => into_registered_c_string(cs), Err(_) => ptr::null_mut(), } } @@ -25,26 +25,34 @@ pub(crate) fn string_to_c_lossy(s: &str) -> *mut c_char { .map(|c| if c == '\0' { '\u{FFFD}' } else { c }) .collect(); match CString::new(cleaned) { - Ok(cs) => cs.into_raw(), + Ok(cs) => into_registered_c_string(cs), Err(_) => ptr::null_mut(), } } pub(crate) fn opt_string_to_c(s: Option<&str>) -> *mut c_char { match s { - Some(s) => string_to_c(s), + Some(s) => string_to_c_lossy(s), None => ptr::null_mut(), } } pub(crate) unsafe fn free_c_string(ptr: *mut c_char) { unsafe { - if !ptr.is_null() { + if crate::resource::take_allocation(crate::resource::AllocationKind::CString, ptr) + == Some(1) + { drop(CString::from_raw(ptr)); } } } +fn into_registered_c_string(value: CString) -> *mut c_char { + let ptr = value.into_raw(); + crate::resource::register_allocation(crate::resource::AllocationKind::CString, ptr, 1); + ptr +} + /// Maximum byte length (excluding the NUL terminator) accepted for any /// foreign C string. Bounds both the terminator scan and the resulting /// allocation, so a missing NUL or a hostile caller cannot walk arbitrary @@ -137,8 +145,8 @@ macro_rules! decode_optional_filter { match $crate::convert::string::try_c_to_string($ptr) { Ok(value) => value, Err(err) => { - $crate::error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + $crate::error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, err.describe($label), )); return $crate::error::AdResult::ErrInvalidArgs; @@ -170,11 +178,39 @@ mod tests { assert_eq!(back, None); } + #[test] + fn optional_string_with_interior_nul_stays_present() { + let c = opt_string_to_c(Some("before\0after")); + assert!(!c.is_null()); + assert_eq!( + unsafe { c_to_string(c) }.as_deref(), + Some("before\u{FFFD}after") + ); + unsafe { free_c_string(c) }; + } + #[test] fn test_free_null_is_noop() { unsafe { free_c_string(ptr::null_mut()) }; } + #[test] + fn free_rejects_unowned_and_double_freed_pointers() { + let foreign = CString::new("foreign").unwrap().into_raw(); + unsafe { free_c_string(foreign) }; + assert_eq!( + unsafe { std::ffi::CStr::from_ptr(foreign) }.to_bytes(), + b"foreign" + ); + unsafe { drop(CString::from_raw(foreign)) }; + + let owned = string_to_c("owned"); + unsafe { + free_c_string(owned); + free_c_string(owned); + } + } + #[test] fn test_lossy_no_nul_same_as_string_to_c() { let c = string_to_c_lossy("hello"); @@ -280,7 +316,7 @@ mod tests { #[test] fn required_adapter_string_null_ptr_returns_err_naming_the_field() { let err = required_adapter_string(ptr::null(), "app_name").unwrap_err(); - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::InvalidArgs); + assert_eq!(err.code, agent_desktop_core::ErrorCode::InvalidArgs); assert_eq!(err.message, "app_name is null"); } @@ -288,7 +324,7 @@ mod tests { fn optional_adapter_string_invalid_utf8_returns_err_naming_the_field() { let bad: [u8; 3] = [0xFF, 0xFE, 0x00]; let err = optional_adapter_string(bad.as_ptr() as *const c_char, "role").unwrap_err(); - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::InvalidArgs); + assert_eq!(err.code, agent_desktop_core::ErrorCode::InvalidArgs); assert_eq!(err.message, "role is not valid UTF-8"); } } diff --git a/crates/ffi/src/convert/surface.rs b/crates/ffi/src/convert/surface.rs index 2125866..53602a3 100644 --- a/crates/ffi/src/convert/surface.rs +++ b/crates/ffi/src/convert/surface.rs @@ -1,10 +1,6 @@ use crate::convert::string::{free_c_string, opt_string_to_c, string_to_c_lossy}; -use crate::types::{AdSnapshotSurface, AdSurfaceInfo}; -use agent_desktop_core::{ - adapter::SnapshotSurface, - error::{AdapterError, ErrorCode}, - node::SurfaceInfo, -}; +use crate::types::{AdExactSurfaceInfo, AdSnapshotSurface, AdSurfaceInfo}; +use agent_desktop_core::{AdapterError, ErrorCode, SnapshotSurface, SurfaceInfo}; use std::os::raw::c_char; use std::ptr; @@ -16,6 +12,30 @@ pub(crate) fn surface_info_to_c(s: &SurfaceInfo) -> AdSurfaceInfo { } } +pub(crate) fn exact_surface_info_to_c(surface: &SurfaceInfo) -> AdExactSurfaceInfo { + AdExactSurfaceInfo { + version: crate::types::exact_surface_info::AD_EXACT_SURFACE_INFO_VERSION, + size: crate::types::exact_surface_info::AD_EXACT_SURFACE_INFO_SIZE as u32, + id: string_to_c_lossy(&surface.id), + surface: surface_info_to_c(surface), + } +} + +pub(crate) fn validate_surface_info(surface: &SurfaceInfo) -> Result<(), AdapterError> { + if surface.id.is_empty() { + return Err(AdapterError::new( + ErrorCode::Internal, + "Surface id is empty", + )); + } + crate::resource::validate_output_string(&surface.id, "Surface id")?; + crate::resource::validate_output_string(&surface.kind, "Surface kind")?; + if let Some(title) = &surface.title { + crate::resource::validate_output_string(title, "Surface title")?; + } + Ok(()) +} + pub(crate) fn snapshot_surface_from_c( raw: i32, field: &str, @@ -29,6 +49,18 @@ pub(crate) fn snapshot_surface_from_c( AdSnapshotSurface::Sheet => SnapshotSurface::Sheet, AdSnapshotSurface::Popover => SnapshotSurface::Popover, AdSnapshotSurface::Alert => SnapshotSurface::Alert, + AdSnapshotSurface::Desktop => SnapshotSurface::Desktop, + AdSnapshotSurface::Taskbar => SnapshotSurface::Taskbar, + AdSnapshotSurface::SystemTray => SnapshotSurface::SystemTray, + AdSnapshotSurface::QuickSettings => SnapshotSurface::QuickSettings, + AdSnapshotSurface::NotificationCenter => SnapshotSurface::NotificationCenter, + AdSnapshotSurface::Toolbar => SnapshotSurface::Toolbar, + AdSnapshotSurface::Dock => SnapshotSurface::Dock, + AdSnapshotSurface::Spotlight => SnapshotSurface::Spotlight, + AdSnapshotSurface::MenuBarExtras => SnapshotSurface::MenuBarExtras, + AdSnapshotSurface::SystemTrayOverflow => SnapshotSurface::SystemTrayOverflow, + AdSnapshotSurface::StartMenu => SnapshotSurface::StartMenu, + AdSnapshotSurface::ActionCenter => SnapshotSurface::ActionCenter, }) .ok_or_else(|| { AdapterError::new( @@ -47,6 +79,14 @@ pub(crate) unsafe fn free_surface_info_fields(s: &mut AdSurfaceInfo) { } } +pub(crate) unsafe fn free_exact_surface_info_fields(surface: &mut AdExactSurfaceInfo) { + unsafe { + free_c_string(surface.id as *mut c_char); + free_surface_info_fields(&mut surface.surface); + surface.id = ptr::null(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -55,6 +95,7 @@ mod tests { #[test] fn test_surface_info_no_title() { let s = SurfaceInfo { + id: "menu-1".into(), kind: "menu".into(), title: None, item_count: Some(3), @@ -67,6 +108,39 @@ mod tests { unsafe { free_surface_info_fields(&mut c) }; } + #[test] + fn exact_surface_info_preserves_id_and_releases_every_string() { + let surface = SurfaceInfo { + id: "ax-window:42".into(), + kind: "window".into(), + title: Some("Preferences".into()), + item_count: Some(8), + }; + let mut exact = exact_surface_info_to_c(&surface); + + assert_eq!( + exact.version, + crate::types::exact_surface_info::AD_EXACT_SURFACE_INFO_VERSION + ); + assert_eq!( + exact.size as usize, + crate::types::exact_surface_info::AD_EXACT_SURFACE_INFO_SIZE + ); + assert_eq!( + unsafe { c_to_string(exact.id) }.as_deref(), + Some("ax-window:42") + ); + assert_eq!( + unsafe { c_to_string(exact.surface.title) }.as_deref(), + Some("Preferences") + ); + + unsafe { free_exact_surface_info_fields(&mut exact) }; + assert!(exact.id.is_null()); + assert!(exact.surface.kind.is_null()); + assert!(exact.surface.title.is_null()); + } + #[test] fn snapshot_surface_from_c_uses_shared_enum_validation() { assert_eq!( @@ -82,6 +156,7 @@ mod tests { #[test] fn item_count_none_maps_to_sentinel_minus_one() { let s = SurfaceInfo { + id: "menu-1".into(), kind: "menu".into(), title: None, item_count: None, @@ -95,6 +170,7 @@ mod tests { #[test] fn item_count_some_zero_maps_to_zero_not_to_absent_sentinel() { let s = SurfaceInfo { + id: "popover-1".into(), kind: "popover".into(), title: None, item_count: Some(0), @@ -111,6 +187,7 @@ mod tests { #[test] fn title_some_maps_to_non_null_c_string_with_correct_value() { let s = SurfaceInfo { + id: "sheet-1".into(), kind: "sheet".into(), title: Some("Save Panel".into()), item_count: None, @@ -126,8 +203,8 @@ mod tests { } #[test] - fn snapshot_surface_from_c_maps_all_seven_variants_exactly() { - let cases: [(i32, SnapshotSurface); 7] = [ + fn snapshot_surface_from_c_maps_all_variants_exactly() { + let cases: [(i32, SnapshotSurface); 19] = [ (0, SnapshotSurface::Window), (1, SnapshotSurface::Focused), (2, SnapshotSurface::Menu), @@ -135,6 +212,18 @@ mod tests { (4, SnapshotSurface::Sheet), (5, SnapshotSurface::Popover), (6, SnapshotSurface::Alert), + (7, SnapshotSurface::Desktop), + (8, SnapshotSurface::Taskbar), + (9, SnapshotSurface::SystemTray), + (10, SnapshotSurface::QuickSettings), + (11, SnapshotSurface::NotificationCenter), + (12, SnapshotSurface::Toolbar), + (13, SnapshotSurface::Dock), + (14, SnapshotSurface::Spotlight), + (15, SnapshotSurface::MenuBarExtras), + (16, SnapshotSurface::SystemTrayOverflow), + (17, SnapshotSurface::StartMenu), + (18, SnapshotSurface::ActionCenter), ]; for (raw, expected) in cases { assert_eq!( diff --git a/crates/ffi/src/convert/window.rs b/crates/ffi/src/convert/window.rs index 601acd9..7065eca 100644 --- a/crates/ffi/src/convert/window.rs +++ b/crates/ffi/src/convert/window.rs @@ -1,7 +1,7 @@ use crate::convert::rect::rect_to_c; -use crate::convert::string::{free_c_string, string_to_c_lossy}; -use crate::types::{AdRect, AdWindowInfo}; -use agent_desktop_core::node::WindowInfo; +use crate::convert::string::{free_c_string, opt_string_to_c, string_to_c_lossy}; +use crate::types::{AdExactWindowInfo, AdRect, AdWindowInfo}; +use agent_desktop_core::{AdapterError, ErrorCode, WindowInfo}; use std::os::raw::c_char; use std::ptr; @@ -22,13 +22,60 @@ pub(crate) fn window_info_to_c(w: &WindowInfo) -> AdWindowInfo { id: string_to_c_lossy(&w.id), title: string_to_c_lossy(&w.title), app_name: string_to_c_lossy(&w.app), - pid: w.pid, + pid: w.pid.get(), bounds, has_bounds, - is_focused: w.is_focused, + is_focused: w.state.is_focused, } } +pub(crate) fn exact_window_info_to_c(w: &WindowInfo) -> AdExactWindowInfo { + AdExactWindowInfo { + version: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_VERSION, + size: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_SIZE as u32, + window: window_info_to_c(w), + process_instance: opt_string_to_c(w.process_instance.as_deref()), + } +} + +pub(crate) fn validate_exact_window_info(window: &WindowInfo) -> Result<(), AdapterError> { + if window.pid.get() == 0 { + return Err(AdapterError::new( + ErrorCode::Internal, + "Exact window pid is not positive", + )); + } + if window.id.is_empty() { + return Err(AdapterError::new( + ErrorCode::Internal, + "Exact window id is empty", + )); + } + let process_instance = window + .process_instance + .as_deref() + .filter(|identity| !identity.is_empty()) + .ok_or_else(|| { + AdapterError::new( + ErrorCode::Internal, + "Exact window lacks process-generation evidence", + ) + })?; + crate::resource::validate_output_string(&window.id, "Window id")?; + crate::resource::validate_output_string(&window.title, "Window title")?; + crate::resource::validate_output_string(&window.app, "Window app")?; + crate::resource::validate_output_string(process_instance, "Window process instance")?; + if let Some(bounds) = window.bounds { + bounds.validate().map_err(|error| { + AdapterError::new( + ErrorCode::Internal, + format!("Exact window has invalid bounds: {}", error.message), + ) + })?; + } + Ok(()) +} + pub(crate) unsafe fn free_window_info_fields(w: &mut AdWindowInfo) { unsafe { free_c_string(w.id as *mut c_char); @@ -40,11 +87,19 @@ pub(crate) unsafe fn free_window_info_fields(w: &mut AdWindowInfo) { } } +pub(crate) unsafe fn free_exact_window_info_fields(w: &mut AdExactWindowInfo) { + unsafe { + free_window_info_fields(&mut w.window); + free_c_string(w.process_instance as *mut c_char); + w.process_instance = ptr::null(); + } +} + #[cfg(test)] mod tests { use super::*; use crate::convert::string::c_to_string; - use agent_desktop_core::node::Rect; + use agent_desktop_core::Rect; #[test] fn test_window_info_roundtrip() { @@ -52,14 +107,19 @@ mod tests { id: "w-123".into(), title: "Documents".into(), app: "Finder".into(), - pid: 42, + pid: agent_desktop_core::ProcessId::new(42), + process_instance: Some("42:100".into()), bounds: Some(Rect { x: 10.0, y: 20.0, width: 800.0, height: 600.0, }), - is_focused: true, + state: agent_desktop_core::WindowState { + is_focused: true, + minimized: None, + visible: None, + }, }; let c = window_info_to_c(&w); assert_eq!(unsafe { c_to_string(c.id) }.as_deref(), Some("w-123")); @@ -85,9 +145,10 @@ mod tests { id: "w-7".into(), title: "Untitled".into(), app: "TextEdit".into(), - pid: 99, + pid: agent_desktop_core::ProcessId::new(99), + process_instance: Some("99:200".into()), bounds: None, - is_focused: false, + state: agent_desktop_core::WindowState::default(), }; let c = window_info_to_c(&w); assert!( @@ -102,4 +163,43 @@ mod tests { let mut c = c; unsafe { free_window_info_fields(&mut c) }; } + + #[test] + fn exact_window_info_carries_owned_process_generation() { + let window = WindowInfo { + id: "w-9".into(), + title: "Main".into(), + app: "Fixture".into(), + pid: agent_desktop_core::ProcessId::new(9), + process_instance: Some("9:123".into()), + bounds: None, + state: agent_desktop_core::WindowState::default(), + }; + let mut exact = exact_window_info_to_c(&window); + + assert_eq!( + unsafe { c_to_string(exact.process_instance) }.as_deref(), + Some("9:123") + ); + unsafe { free_exact_window_info_fields(&mut exact) }; + assert!(exact.process_instance.is_null()); + } + + #[test] + fn exact_window_validation_rejects_missing_process_generation() { + let window = WindowInfo { + id: "w-9".into(), + title: "Main".into(), + app: "Fixture".into(), + pid: agent_desktop_core::ProcessId::new(9), + process_instance: None, + bounds: None, + state: agent_desktop_core::WindowState::default(), + }; + + let error = validate_exact_window_info(&window).unwrap_err(); + + assert_eq!(error.code, ErrorCode::Internal); + assert!(error.message.contains("process-generation")); + } } diff --git a/crates/ffi/src/displays/list.rs b/crates/ffi/src/displays/list.rs new file mode 100644 index 0000000..62a6257 --- /dev/null +++ b/crates/ffi/src/displays/list.rs @@ -0,0 +1,106 @@ +use crate::AdAdapter; +use crate::convert::display::{display_info_to_c, free_display_info_fields, validate_display_info}; +use crate::error::{AdResult, set_last_error}; +use crate::ffi_try::{trap_panic, trap_panic_void}; +use crate::types::{AdDisplayInfo, AdDisplayList}; +use std::ptr; + +/// Lists displays in screenshot screen-index order. +/// +/// # Safety +/// `adapter` must be valid and `out` must be writable. Success produces an +/// opaque list freed with `ad_display_list_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_list_displays( + adapter: *const AdAdapter, + out: *mut *mut AdDisplayList, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = ptr::null_mut(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.list_displays(deadline) { + Ok(displays) => { + if let Err(error) = + crate::resource::validate_list_len(displays.len(), "Display list") + .and_then(|_| displays.iter().try_for_each(validate_display_info)) + { + set_last_error(&error); + return crate::error::last_error_code(); + } + let items = displays + .iter() + .map(display_info_to_c) + .collect::<Vec<_>>() + .into_boxed_slice(); + *out = Box::into_raw(Box::new(AdDisplayList { items })); + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } + }) +} + +/// # Safety +/// `list` must be null or returned by `ad_list_displays`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_display_list_count(list: *const AdDisplayList) -> u32 { + if list.is_null() { + return 0; + } + unsafe { &*list }.items.len() as u32 +} + +/// Returns a borrowed display entry, or null when `index` is out of range. +/// +/// # Safety +/// `list` must be null or returned by `ad_list_displays`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_display_list_get( + list: *const AdDisplayList, + index: u32, +) -> *const AdDisplayInfo { + if list.is_null() { + return ptr::null(); + } + unsafe { &*list } + .items + .get(index as usize) + .map_or(ptr::null(), std::ptr::from_ref) +} + +/// # Safety +/// `list` must be null or returned by `ad_list_displays`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_display_list_free(list: *mut AdDisplayList) { + trap_panic_void(|| unsafe { + if list.is_null() { + return; + } + let mut list = Box::from_raw(list); + for item in &mut list.items { + free_display_info_fields(item); + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn list_accessors_bound_borrowed_entries() { + let list = Box::into_raw(Box::new(AdDisplayList { + items: Vec::new().into_boxed_slice(), + })); + + assert_eq!(unsafe { ad_display_list_count(list) }, 0); + assert!(unsafe { ad_display_list_get(list, 0) }.is_null()); + unsafe { ad_display_list_free(list) }; + } +} diff --git a/crates/ffi/src/displays/mod.rs b/crates/ffi/src/displays/mod.rs new file mode 100644 index 0000000..c5ea173 --- /dev/null +++ b/crates/ffi/src/displays/mod.rs @@ -0,0 +1 @@ +pub(crate) mod list; diff --git a/crates/ffi/src/enum_validation.rs b/crates/ffi/src/enum_validation.rs index fd11ed9..413ce30 100644 --- a/crates/ffi/src/enum_validation.rs +++ b/crates/ffi/src/enum_validation.rs @@ -52,7 +52,7 @@ try_from_c_enum! { try_from_c_enum! { AdModifier { - Cmd = 0, Ctrl = 1, Alt = 2, Shift = 3, + Meta = 0, Ctrl = 1, Alt = 2, Shift = 3, } } @@ -83,7 +83,11 @@ try_from_c_enum! { try_from_c_enum! { AdSnapshotSurface { Window = 0, Focused = 1, Menu = 2, Menubar = 3, - Sheet = 4, Popover = 5, Alert = 6, + Sheet = 4, Popover = 5, Alert = 6, Desktop = 7, + Taskbar = 8, SystemTray = 9, QuickSettings = 10, + NotificationCenter = 11, + Toolbar = 12, Dock = 13, Spotlight = 14, MenuBarExtras = 15, + SystemTrayOverflow = 16, StartMenu = 17, ActionCenter = 18, } } @@ -178,6 +182,7 @@ mod tests { let _ = AdMouseEventKind::from_c(raw); let _ = AdWindowOpKind::from_c(raw); let _ = AdScreenshotKind::from_c(raw); + let _ = AdSnapshotSurface::from_c(raw); } } @@ -201,7 +206,7 @@ mod tests { #[test] fn modifier_boundary_discriminants_map_to_exact_variants() { - assert_eq!(AdModifier::from_c(0), Some(AdModifier::Cmd)); + assert_eq!(AdModifier::from_c(0), Some(AdModifier::Meta)); assert_eq!(AdModifier::from_c(3), Some(AdModifier::Shift)); } @@ -238,6 +243,9 @@ mod tests { AdSnapshotSurface::from_c(0), Some(AdSnapshotSurface::Window) ); - assert_eq!(AdSnapshotSurface::from_c(6), Some(AdSnapshotSurface::Alert)); + assert_eq!( + AdSnapshotSurface::from_c(18), + Some(AdSnapshotSurface::ActionCenter) + ); } } diff --git a/crates/ffi/src/error.rs b/crates/ffi/src/error.rs index e3521d1..aff3ce3 100644 --- a/crates/ffi/src/error.rs +++ b/crates/ffi/src/error.rs @@ -1,4 +1,5 @@ -use agent_desktop_core::error::{AdapterError, ErrorCode}; +use crate::types::AdDeliverySemantics; +use agent_desktop_core::{AdapterError, DeliverySemantics, ErrorCode}; use std::cell::RefCell; use std::ffi::{CStr, CString, c_char}; @@ -23,6 +24,7 @@ pub enum AdResult { ErrSnapshotNotFound = -13, ErrPolicyDenied = -14, ErrAmbiguousTarget = -15, + ErrAppUnresponsive = -16, } const _: () = assert!(AdResult::ErrPermDenied as i32 == -1); @@ -40,6 +42,7 @@ const _: () = assert!(AdResult::ErrInternal as i32 == -12); const _: () = assert!(AdResult::ErrSnapshotNotFound as i32 == -13); const _: () = assert!(AdResult::ErrPolicyDenied as i32 == -14); const _: () = assert!(AdResult::ErrAmbiguousTarget as i32 == -15); +const _: () = assert!(AdResult::ErrAppUnresponsive as i32 == -16); enum MessageSource { Owned(CString), @@ -69,6 +72,7 @@ struct StoredError { suggestion: Option<CString>, platform_detail: Option<CString>, details: Option<CString>, + disposition: DeliverySemantics, } static NUL_BYTE_FALLBACK: &CStr = c"(message contained null byte)"; @@ -77,6 +81,15 @@ thread_local! { static LAST_ERROR: RefCell<Option<StoredError>> = const { RefCell::new(None) }; } +fn replace_last_error(error: StoredError) { + let mut error = Some(error); + let _ = LAST_ERROR.try_with(|cell| { + if let Ok(mut slot) = cell.try_borrow_mut() { + *slot = error.take(); + } + }); +} + /// Maps a core `ErrorCode` to its stable C-ABI `AdResult`. `ErrorCode` and the /// error variants of `AdResult` are a bijection: each maps to exactly one of the /// other. This match is exhaustive over `ErrorCode`, so a new `ErrorCode` cannot @@ -100,6 +113,7 @@ fn error_code_to_result(code: &ErrorCode) -> AdResult { ErrorCode::Internal => AdResult::ErrInternal, ErrorCode::SnapshotNotFound => AdResult::ErrSnapshotNotFound, ErrorCode::PolicyDenied => AdResult::ErrPolicyDenied, + ErrorCode::AppUnresponsive => AdResult::ErrAppUnresponsive, } } @@ -119,45 +133,66 @@ pub(crate) fn set_last_error(err: &AdapterError) { .as_ref() .and_then(|details| serde_json::to_string(details).ok()) .and_then(|details| CString::new(details).ok()); - LAST_ERROR.with(|cell| { - *cell.borrow_mut() = Some(StoredError { - code, - message, - suggestion, - platform_detail, - details, - }); + replace_last_error(StoredError { + code, + message, + suggestion, + platform_detail, + details, + disposition: err.disposition, }); } #[cfg(test)] pub(crate) fn clear_last_error() { - LAST_ERROR.with(|cell| { - *cell.borrow_mut() = None; + let _ = LAST_ERROR.try_with(|cell| { + if let Ok(mut slot) = cell.try_borrow_mut() { + *slot = None; + } }); } +#[cfg(test)] +pub(crate) fn with_last_error_mutably_borrowed<R>(body: impl FnOnce() -> R) -> R { + LAST_ERROR.with(|cell| { + let _borrow = cell.borrow_mut(); + body() + }) +} + /// Sets the last-error using a `'static CStr` message. Never allocates, /// never panics — safe to call from a panic handler. pub(crate) fn set_last_error_static(code: AdResult, message: &'static CStr) { - LAST_ERROR.with(|cell| { - *cell.borrow_mut() = Some(StoredError { - code, - message: MessageSource::Static(message), - suggestion: None, - platform_detail: None, - details: None, - }); + replace_last_error(StoredError { + code, + message: MessageSource::Static(message), + suggestion: None, + platform_detail: None, + details: None, + disposition: DeliverySemantics::unknown(), }); } pub(crate) fn last_error_code() -> AdResult { - LAST_ERROR.with(|cell| { - cell.borrow() - .as_ref() - .map(|e| e.code) - .unwrap_or(AdResult::Ok) - }) + LAST_ERROR + .try_with(|cell| { + cell.try_borrow() + .ok() + .and_then(|error| error.as_ref().map(|error| error.code)) + .unwrap_or(AdResult::Ok) + }) + .unwrap_or(AdResult::Ok) +} + +fn last_error_disposition() -> DeliverySemantics { + LAST_ERROR + .try_with(|cell| { + cell.try_borrow() + .ok() + .and_then(|error| error.as_ref().map(|error| error.disposition)) + .unwrap_or_else(DeliverySemantics::unknown) + }) + .unwrap_or_else(|_| DeliverySemantics::unknown()) } /// Last-error lifetime — errno-style. @@ -189,12 +224,14 @@ pub extern "C" fn ad_last_error_code() -> AdResult { #[unsafe(no_mangle)] pub extern "C" fn ad_last_error_message() -> *const c_char { crate::ffi_try::trap_panic_const_ptr(|| { - LAST_ERROR.with(|cell| { - cell.borrow() - .as_ref() - .map(|e| e.message.as_ptr()) - .unwrap_or(std::ptr::null()) - }) + LAST_ERROR + .try_with(|cell| { + cell.try_borrow() + .ok() + .and_then(|error| error.as_ref().map(|error| error.message.as_ptr())) + .unwrap_or(std::ptr::null()) + }) + .unwrap_or(std::ptr::null()) }) } @@ -204,12 +241,18 @@ pub extern "C" fn ad_last_error_message() -> *const c_char { #[unsafe(no_mangle)] pub extern "C" fn ad_last_error_suggestion() -> *const c_char { crate::ffi_try::trap_panic_const_ptr(|| { - LAST_ERROR.with(|cell| { - cell.borrow() - .as_ref() - .and_then(|e| e.suggestion.as_ref().map(|s| s.as_ptr())) - .unwrap_or(std::ptr::null()) - }) + LAST_ERROR + .try_with(|cell| { + cell.try_borrow() + .ok() + .and_then(|error| { + error + .as_ref() + .and_then(|error| error.suggestion.as_ref().map(|value| value.as_ptr())) + }) + .unwrap_or(std::ptr::null()) + }) + .unwrap_or(std::ptr::null()) }) } @@ -220,12 +263,18 @@ pub extern "C" fn ad_last_error_suggestion() -> *const c_char { #[unsafe(no_mangle)] pub extern "C" fn ad_last_error_platform_detail() -> *const c_char { crate::ffi_try::trap_panic_const_ptr(|| { - LAST_ERROR.with(|cell| { - cell.borrow() - .as_ref() - .and_then(|e| e.platform_detail.as_ref().map(|s| s.as_ptr())) - .unwrap_or(std::ptr::null()) - }) + LAST_ERROR + .try_with(|cell| { + cell.try_borrow() + .ok() + .and_then(|error| { + error.as_ref().and_then(|error| { + error.platform_detail.as_ref().map(|value| value.as_ptr()) + }) + }) + .unwrap_or(std::ptr::null()) + }) + .unwrap_or(std::ptr::null()) }) } @@ -237,12 +286,36 @@ pub extern "C" fn ad_last_error_platform_detail() -> *const c_char { #[unsafe(no_mangle)] pub extern "C" fn ad_last_error_details() -> *const c_char { crate::ffi_try::trap_panic_const_ptr(|| { - LAST_ERROR.with(|cell| { - cell.borrow() - .as_ref() - .and_then(|e| e.details.as_ref().map(|s| s.as_ptr())) - .unwrap_or(std::ptr::null()) - }) + LAST_ERROR + .try_with(|cell| { + cell.try_borrow() + .ok() + .and_then(|error| { + error + .as_ref() + .and_then(|error| error.details.as_ref().map(|value| value.as_ptr())) + }) + .unwrap_or(std::ptr::null()) + }) + .unwrap_or(std::ptr::null()) + }) +} + +/// Writes the delivery and retry semantics associated with the calling +/// thread's last error. If no error has been recorded, both values are +/// `UNKNOWN`. This successful read does not clear or replace last-error state. +/// +/// # Safety +/// +/// `out` must point to writable `AdDeliverySemantics` storage. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_last_error_delivery_semantics( + out: *mut AdDeliverySemantics, +) -> AdResult { + crate::ffi_try::trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = AdDeliverySemantics::from_core(last_error_disposition()); + AdResult::Ok }) } diff --git a/crates/ffi/src/error_tests.rs b/crates/ffi/src/error_tests.rs index 377612b..b6b49a6 100644 --- a/crates/ffi/src/error_tests.rs +++ b/crates/ffi/src/error_tests.rs @@ -44,6 +44,7 @@ fn result_discriminants_preserve_existing_abi_values() { assert_eq!(AdResult::ErrSnapshotNotFound as i32, -13); assert_eq!(AdResult::ErrPolicyDenied as i32, -14); assert_eq!(AdResult::ErrAmbiguousTarget as i32, -15); + assert_eq!(AdResult::ErrAppUnresponsive as i32, -16); } /// Reverse of `error_code_to_result`, kept solely to guard the bijection. The @@ -70,6 +71,7 @@ fn error_code_origin(result: AdResult) -> Option<ErrorCode> { AdResult::ErrSnapshotNotFound => ErrorCode::SnapshotNotFound, AdResult::ErrPolicyDenied => ErrorCode::PolicyDenied, AdResult::ErrAmbiguousTarget => ErrorCode::AmbiguousTarget, + AdResult::ErrAppUnresponsive => ErrorCode::AppUnresponsive, }) } @@ -91,6 +93,7 @@ fn error_code_and_ad_result_error_variants_stay_in_bijection() { AdResult::ErrSnapshotNotFound, AdResult::ErrPolicyDenied, AdResult::ErrAmbiguousTarget, + AdResult::ErrAppUnresponsive, ] { let code = error_code_origin(result).expect("error variant must have an ErrorCode origin"); assert_eq!( @@ -113,6 +116,27 @@ fn test_set_and_get_error() { assert!(last_error_details_str().is_none()); } +#[test] +fn last_error_preserves_delivery_and_retry_semantics() { + let err = AdapterError::timeout("clipboard helper timed out") + .with_disposition(DeliverySemantics::uncertain()); + set_last_error(&err); + let mut semantics = AdDeliverySemantics::unknown(); + + assert_eq!( + unsafe { ad_last_error_delivery_semantics(&mut semantics) }, + AdResult::Ok + ); + assert_eq!( + semantics.delivery, + crate::types::AdDeliveryDisposition::DeliveryUncertain as i32 + ); + assert_eq!( + semantics.retry, + crate::types::AdRetryDisposition::Unsafe as i32 + ); +} + #[test] fn test_set_and_get_structured_details() { let err = AdapterError::new(ErrorCode::AmbiguousTarget, "ambiguous").with_details( diff --git a/crates/ffi/src/ffi_try.rs b/crates/ffi/src/ffi_try.rs index b25ad56..98afd8b 100644 --- a/crates/ffi/src/ffi_try.rs +++ b/crates/ffi/src/ffi_try.rs @@ -18,7 +18,9 @@ pub(crate) fn trap_panic<F>(body: F) -> AdResult where F: FnOnce() -> AdResult, { - match catch_unwind(AssertUnwindSafe(body)) { + match catch_unwind(AssertUnwindSafe(|| { + crate::log_callback::with_dispatch(body) + })) { Ok(result) => result, Err(_) => { set_last_error_static(AdResult::ErrInternal, PANIC_MESSAGE); @@ -33,7 +35,9 @@ pub(crate) fn trap_panic_ptr<T, F>(body: F) -> *mut T where F: FnOnce() -> *mut T, { - match catch_unwind(AssertUnwindSafe(body)) { + match catch_unwind(AssertUnwindSafe(|| { + crate::log_callback::with_dispatch(body) + })) { Ok(ptr) => ptr, Err(_) => { set_last_error_static(AdResult::ErrInternal, PANIC_MESSAGE); @@ -48,7 +52,9 @@ pub(crate) fn trap_panic_const_ptr<T, F>(body: F) -> *const T where F: FnOnce() -> *const T, { - match catch_unwind(AssertUnwindSafe(body)) { + match catch_unwind(AssertUnwindSafe(|| { + crate::log_callback::with_dispatch(body) + })) { Ok(ptr) => ptr, Err(_) => { set_last_error_static(AdResult::ErrInternal, PANIC_MESSAGE); @@ -65,7 +71,11 @@ pub(crate) fn trap_panic_void<F>(body: F) where F: FnOnce(), { - if catch_unwind(AssertUnwindSafe(body)).is_err() { + if catch_unwind(AssertUnwindSafe(|| { + crate::log_callback::with_dispatch(body) + })) + .is_err() + { set_last_error_static(AdResult::ErrInternal, PANIC_MESSAGE); } } @@ -116,4 +126,12 @@ mod tests { fn test_void_panic_is_swallowed() { trap_panic_void(|| panic!("boom")); } + + #[test] + fn panic_handler_survives_reentrant_last_error_borrow() { + let result = crate::error::with_last_error_mutably_borrowed(|| { + trap_panic(|| -> AdResult { panic!("reentrant") }) + }); + assert_eq!(result, AdResult::ErrInternal); + } } diff --git a/crates/ffi/src/input/clipboard.rs b/crates/ffi/src/input/clipboard.rs index f36f219..48e465d 100644 --- a/crates/ffi/src/input/clipboard.rs +++ b/crates/ffi/src/input/clipboard.rs @@ -1,7 +1,8 @@ use crate::AdAdapter; -use crate::convert::string::{c_to_string, free_c_string, string_to_c}; +use crate::convert::string::{free_c_string, required_adapter_string, string_to_c}; use crate::error::{self, AdResult}; use crate::ffi_try::{trap_panic, trap_panic_void}; +use agent_desktop_core::{ClipboardContent, ClipboardFormat}; use std::os::raw::c_char; /// Reads the current clipboard text and writes an owned C string into @@ -19,17 +20,22 @@ pub unsafe extern "C" fn ad_get_clipboard( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = std::ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; - match adapter.inner.get_clipboard() { - Ok(text) => { + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter + .inner + .get_clipboard_content(ClipboardFormat::Text, deadline) + { + Ok(content) => { + let text = match content { + Some(ClipboardContent::Text(text)) => text, + _ => String::new(), + }; let c = string_to_c(&text); if c.is_null() { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::Internal, + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::Internal, "clipboard text contains an interior NUL and cannot be represented as a C string", )); return AdResult::ErrInternal; @@ -56,23 +62,21 @@ pub unsafe extern "C" fn ad_set_clipboard( adapter: *const AdAdapter, text: *const c_char, ) -> AdResult { - trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } + trap_panic(|| { crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; - let text = match c_to_string(text) { - Some(s) => s, - None => { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "text is null or invalid UTF-8", - )); + let text = match required_adapter_string(text, "text") { + Ok(text) => text, + Err(error) => { + error::set_last_error(&error); return AdResult::ErrInvalidArgs; } }; - match adapter.inner.set_clipboard(&text) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter + .inner + .set_clipboard_content(&ClipboardContent::Text(text), &lease) + { Ok(()) => AdResult::Ok, Err(e) => { error::set_last_error(&e); @@ -88,13 +92,11 @@ pub unsafe extern "C" fn ad_set_clipboard( /// `adapter` must be a non-null pointer returned by `ad_adapter_create`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_clear_clipboard(adapter: *const AdAdapter) -> AdResult { - trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } + trap_panic(|| { crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; - match adapter.inner.clear_clipboard() { + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter.inner.clear_clipboard(&lease) { Ok(()) => AdResult::Ok, Err(e) => { error::set_last_error(&e); @@ -106,11 +108,11 @@ pub unsafe extern "C" fn ad_clear_clipboard(adapter: *const AdAdapter) -> AdResu /// Frees a C string previously returned by `ad_get_clipboard` or any /// other FFI call documented as allocating a C string for the caller. -/// Null-tolerant — safe to call on `NULL`. Double-free is undefined. +/// Null-tolerant. Unknown pointers and repeated frees are ignored. /// /// # Safety -/// `s` must be null or a pointer previously handed out by this crate. -/// After this call the pointer is invalid and must not be used. +/// `s` may be null or a pointer previously handed out by this crate. +/// After a successful free the pointer is invalid and must not be used. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_free_string(s: *mut c_char) { trap_panic_void(|| unsafe { free_c_string(s) }) diff --git a/crates/ffi/src/input/drag.rs b/crates/ffi/src/input/drag.rs index 69f18b8..bbe9d24 100644 --- a/crates/ffi/src/input/drag.rs +++ b/crates/ffi/src/input/drag.rs @@ -17,15 +17,17 @@ pub unsafe extern "C" fn ad_drag( params: *const AdDragParams, ) -> AdResult { trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(params, c"params is null"); - let adapter = &*adapter; let p = &*params; let core_params = p.to_core(); - match adapter.inner.drag(core_params) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + if let Err(error) = core_params.validate(lease.deadline()) { + error::set_last_error(&error); + return error::last_error_code(); + } + match adapter.inner.drag(core_params, &lease) { Ok(()) => AdResult::Ok, Err(e) => { error::set_last_error(&e); diff --git a/crates/ffi/src/input/mouse.rs b/crates/ffi/src/input/mouse.rs index 273cf20..2f7bd8a 100644 --- a/crates/ffi/src/input/mouse.rs +++ b/crates/ffi/src/input/mouse.rs @@ -1,12 +1,18 @@ use crate::AdAdapter; use crate::error::{self, AdResult}; use crate::ffi_try::trap_panic; -use crate::types::{AdMouseButton, AdMouseEvent, AdMouseEventKind}; -use agent_desktop_core::action::{ - MouseButton as CoreMouseButton, MouseEvent as CoreMouseEvent, +use crate::types::{AdModifier, AdMouseButton, AdMouseEvent, AdMouseEventKind}; +use agent_desktop_core::{ + Modifier as CoreModifier, MouseButton as CoreMouseButton, MouseEvent as CoreMouseEvent, MouseEventKind as CoreMouseEventKind, Point as CorePoint, }; +/// Four modifier keys exist (`AdModifier::{Meta, Ctrl, Alt, Shift}`), so a +/// chord can name at most four. Anything larger must be bogus input — bail +/// out instead of trusting it into `from_raw_parts`. +const MAX_MOUSE_MODIFIERS: u32 = 4; +const ALL_MODIFIER_BITS: u32 = 0b1111; + pub(crate) fn mouse_button_from_c(b: AdMouseButton) -> CoreMouseButton { match b { AdMouseButton::Left => CoreMouseButton::Left, @@ -15,10 +21,99 @@ pub(crate) fn mouse_button_from_c(b: AdMouseButton) -> CoreMouseButton { } } +/// Parses a `modifiers` array + count pair into a `Vec<Modifier>`, mirroring +/// `AdKeyCombo`'s `modifiers`/`modifier_count` contract so mouse chords and +/// key chords validate identically at the C boundary. +/// +/// # Safety +/// `modifiers` must point to `count` valid `int32_t` values, or be null when +/// `count` is 0. +pub(crate) unsafe fn modifiers_from_c( + modifiers: *const i32, + count: u32, +) -> Result<Vec<CoreModifier>, &'static str> { + if count > MAX_MOUSE_MODIFIERS { + return Err("modifier_count exceeds MAX_MOUSE_MODIFIERS (4)"); + } + if count > 0 && modifiers.is_null() { + return Err("modifier_count > 0 but modifiers pointer is null"); + } + let mut out = Vec::with_capacity(count as usize); + if count > 0 { + let slice = unsafe { std::slice::from_raw_parts(modifiers, count as usize) }; + for raw_modifier in slice { + let m = AdModifier::from_c(*raw_modifier).ok_or("invalid modifier discriminant")?; + out.push(match m { + AdModifier::Meta => CoreModifier::Meta, + AdModifier::Ctrl => CoreModifier::Ctrl, + AdModifier::Alt => CoreModifier::Alt, + AdModifier::Shift => CoreModifier::Shift, + }); + } + } + Ok(out) +} + +fn modifiers_from_mask(mask: u32) -> Result<Vec<CoreModifier>, &'static str> { + if mask & !ALL_MODIFIER_BITS != 0 { + return Err("modifier mask contains unknown bits"); + } + let mut modifiers = Vec::new(); + for (bit, modifier) in [ + (0, CoreModifier::Meta), + (1, CoreModifier::Ctrl), + (2, CoreModifier::Alt), + (3, CoreModifier::Shift), + ] { + if mask & (1 << bit) != 0 { + modifiers.push(modifier); + } + } + Ok(modifiers) +} + +fn build_mouse_event( + ev: &AdMouseEvent, + modifiers: Vec<CoreModifier>, +) -> Result<CoreMouseEvent, &'static str> { + let validated_button = + AdMouseButton::from_c(ev.button).ok_or("invalid mouse button discriminant")?; + let validated_kind = + AdMouseEventKind::from_c(ev.kind).ok_or("invalid mouse event kind discriminant")?; + let point = CorePoint { + x: ev.point.x, + y: ev.point.y, + }; + point + .validate() + .map_err(|_| "mouse coordinates exceed supported geometry bounds")?; + let button = mouse_button_from_c(validated_button); + let kind = match validated_kind { + AdMouseEventKind::Move => CoreMouseEventKind::Move, + AdMouseEventKind::Down => CoreMouseEventKind::Down, + AdMouseEventKind::Up => CoreMouseEventKind::Up, + AdMouseEventKind::Click => { + agent_desktop_core::validate_mouse_click_count(ev.click_count) + .map_err(|_| "click_count must be between 1 and 100")?; + CoreMouseEventKind::Click { + count: ev.click_count, + } + } + }; + Ok(CoreMouseEvent { + kind, + point, + button, + modifiers, + }) +} + /// Dispatches an explicit physical mouse event (move / down / up / click) /// at the given screen point. Click count is only consulted when `event.kind` /// is `CLICK` (e.g., `click_count == 2` for a double-click). Callers that /// need headless policy enforcement should use ref actions with policy. +/// Carries no modifier chord — use [`ad_mouse_event_with_modifiers`] for +/// meta/ctrl/alt/shift-held clicks. /// /// # Safety /// `adapter` must be a non-null pointer returned by `ad_adapter_create`. @@ -29,52 +124,22 @@ pub unsafe extern "C" fn ad_mouse_event( event: *const AdMouseEvent, ) -> AdResult { trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(event, c"event is null"); - let adapter = &*adapter; let ev = &*event; - let validated_button = match AdMouseButton::from_c(ev.button) { - Some(b) => b, - None => { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "invalid mouse button discriminant", + let core_event = match build_mouse_event(ev, Vec::new()) { + Ok(e) => e, + Err(msg) => { + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + msg, )); return AdResult::ErrInvalidArgs; } }; - let validated_kind = match AdMouseEventKind::from_c(ev.kind) { - Some(k) => k, - None => { - error::set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "invalid mouse event kind discriminant", - )); - return AdResult::ErrInvalidArgs; - } - }; - let point = CorePoint { - x: ev.point.x, - y: ev.point.y, - }; - let button = mouse_button_from_c(validated_button); - let kind = match validated_kind { - AdMouseEventKind::Move => CoreMouseEventKind::Move, - AdMouseEventKind::Down => CoreMouseEventKind::Down, - AdMouseEventKind::Up => CoreMouseEventKind::Up, - AdMouseEventKind::Click => CoreMouseEventKind::Click { - count: ev.click_count, - }, - }; - let core_event = CoreMouseEvent { - kind, - point, - button, - }; - match adapter.inner.mouse_event(core_event) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter.inner.mouse_event(core_event, &lease) { Ok(()) => AdResult::Ok, Err(e) => { error::set_last_error(&e); @@ -84,54 +149,118 @@ pub unsafe extern "C" fn ad_mouse_event( }) } -#[cfg(test)] -mod tests { - use super::*; - use crate::types::AdPoint; - - #[test] - fn test_mouse_button_mapping() { - assert!(matches!( - mouse_button_from_c(AdMouseButton::Left), - CoreMouseButton::Left - )); - assert!(matches!( - mouse_button_from_c(AdMouseButton::Right), - CoreMouseButton::Right - )); - assert!(matches!( - mouse_button_from_c(AdMouseButton::Middle), - CoreMouseButton::Middle - )); - } - - #[test] - fn valid_discriminants_convert_to_typed_enums() { - let ev = AdMouseEvent { - kind: AdMouseEventKind::Click as i32, - point: AdPoint { x: 10.0, y: 20.0 }, - button: AdMouseButton::Left as i32, - click_count: 2, +/// Additive counterpart to [`ad_mouse_event`] that also carries a held +/// modifier chord (meta/ctrl/alt/shift) — e.g. Meta-click for additive +/// selection, shift-click for range selection. `AdMouseEvent`'s layout is +/// unchanged; modifiers travel as a separate array + count, mirroring +/// `AdKeyCombo::modifiers`/`modifier_count`. +/// +/// # Safety +/// `adapter` must be a non-null pointer returned by `ad_adapter_create`. +/// `event` must be a non-null pointer to a valid `AdMouseEvent`. +/// `modifiers` must point to `modifier_count` valid `int32_t` values, or be +/// null when `modifier_count` is 0. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_mouse_event_with_modifiers( + adapter: *const AdAdapter, + event: *const AdMouseEvent, + modifiers: *const i32, + modifier_count: u32, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(event, c"event is null"); + let ev = &*event; + let mods = match modifiers_from_c(modifiers, modifier_count) { + Ok(m) => m, + Err(msg) => { + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + msg, + )); + return AdResult::ErrInvalidArgs; + } }; - assert!(matches!( - AdMouseButton::from_c(ev.button), - Some(AdMouseButton::Left) - )); - assert!(matches!( - AdMouseEventKind::from_c(ev.kind), - Some(AdMouseEventKind::Click) - )); - } - - #[test] - fn invalid_discriminants_reject_without_ub() { - let ev = AdMouseEvent { - kind: 999, - point: AdPoint { x: 0.0, y: 0.0 }, - button: -5, - click_count: 0, + let core_event = match build_mouse_event(ev, mods) { + Ok(e) => e, + Err(msg) => { + error::set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + msg, + )); + return AdResult::ErrInvalidArgs; + } }; - assert!(AdMouseButton::from_c(ev.button).is_none()); - assert!(AdMouseEventKind::from_c(ev.kind).is_none()); - } + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter.inner.mouse_event(core_event, &lease) { + Ok(()) => AdResult::Ok, + Err(e) => { + error::set_last_error(&e); + error::last_error_code() + } + } + }) } + +/// Dispatches a physical wheel event using platform-neutral line deltas. +/// Positive `delta_y` scrolls up and negative scrolls down; positive +/// `delta_x` scrolls left and negative scrolls right. `modifier_mask` uses +/// bits 0-3 for meta, ctrl, alt, and shift respectively. +/// +/// # Safety +/// `adapter` must be a non-null pointer returned by `ad_adapter_create`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_mouse_wheel( + adapter: *const AdAdapter, + point: crate::types::AdPoint, + delta_x: f64, + delta_y: f64, + modifier_mask: u32, +) -> AdResult { + trap_panic(|| { + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + let point = CorePoint { + x: point.x, + y: point.y, + }; + if point.validate().is_err() || !delta_x.is_finite() || !delta_y.is_finite() { + let err = agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "wheel coordinates and line deltas must be finite", + ); + error::set_last_error(&err); + return AdResult::ErrInvalidArgs; + } + let modifiers = match modifiers_from_mask(modifier_mask) { + Ok(modifiers) => modifiers, + Err(message) => { + let err = agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + message, + ); + error::set_last_error(&err); + return AdResult::ErrInvalidArgs; + } + }; + let event = CoreMouseEvent { + kind: CoreMouseEventKind::Wheel { delta_x, delta_y }, + point, + button: CoreMouseButton::Left, + modifiers, + }; + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter.inner.mouse_event(event, &lease) { + Ok(()) => AdResult::Ok, + Err(err) => { + error::set_last_error(&err); + error::last_error_code() + } + } + }) +} + +#[cfg(test)] +#[path = "mouse_tests.rs"] +mod tests; diff --git a/crates/ffi/src/input/mouse_tests.rs b/crates/ffi/src/input/mouse_tests.rs new file mode 100644 index 0000000..b5c7501 --- /dev/null +++ b/crates/ffi/src/input/mouse_tests.rs @@ -0,0 +1,169 @@ +use super::*; +use crate::types::AdPoint; + +#[test] +fn test_mouse_button_mapping() { + assert!(matches!( + mouse_button_from_c(AdMouseButton::Left), + CoreMouseButton::Left + )); + assert!(matches!( + mouse_button_from_c(AdMouseButton::Right), + CoreMouseButton::Right + )); + assert!(matches!( + mouse_button_from_c(AdMouseButton::Middle), + CoreMouseButton::Middle + )); +} + +#[test] +fn valid_discriminants_convert_to_typed_enums() { + let ev = AdMouseEvent { + kind: AdMouseEventKind::Click as i32, + point: AdPoint { x: 10.0, y: 20.0 }, + button: AdMouseButton::Left as i32, + click_count: 2, + }; + assert!(matches!( + AdMouseButton::from_c(ev.button), + Some(AdMouseButton::Left) + )); + assert!(matches!( + AdMouseEventKind::from_c(ev.kind), + Some(AdMouseEventKind::Click) + )); +} + +#[test] +fn invalid_discriminants_reject_without_ub() { + let ev = AdMouseEvent { + kind: 999, + point: AdPoint { x: 0.0, y: 0.0 }, + button: -5, + click_count: 0, + }; + assert!(AdMouseButton::from_c(ev.button).is_none()); + assert!(AdMouseEventKind::from_c(ev.kind).is_none()); +} + +/// F10 regression: `ad_mouse_event` hardcoded `modifiers: Vec::new()` with no +/// way for an FFI caller to supply a chord at all. These cover the additive +/// `modifiers_from_c` parser and `build_mouse_event`'s threading of its +/// output, which `ad_mouse_event_with_modifiers` relies on. +#[test] +fn modifiers_from_c_empty_when_count_zero() { + let result = unsafe { modifiers_from_c(std::ptr::null(), 0) }.unwrap(); + assert!(result.is_empty()); +} + +#[test] +fn modifiers_from_c_maps_all_four_discriminants_in_order() { + let raw: [i32; 4] = [ + AdModifier::Meta as i32, + AdModifier::Ctrl as i32, + AdModifier::Alt as i32, + AdModifier::Shift as i32, + ]; + let result = unsafe { modifiers_from_c(raw.as_ptr(), 4) }.unwrap(); + assert_eq!( + result, + vec![ + CoreModifier::Meta, + CoreModifier::Ctrl, + CoreModifier::Alt, + CoreModifier::Shift, + ] + ); +} + +#[test] +fn modifiers_from_c_rejects_null_pointer_with_positive_count() { + let result = unsafe { modifiers_from_c(std::ptr::null(), 2) }; + assert!(result.is_err()); +} + +#[test] +fn modifiers_from_c_rejects_count_exceeding_cap() { + let raw: [i32; 5] = [0, 1, 2, 3, 0]; + let result = unsafe { modifiers_from_c(raw.as_ptr(), 5) }; + assert!(result.is_err()); +} + +#[test] +fn modifiers_from_c_rejects_invalid_discriminant() { + let raw: [i32; 1] = [999]; + let result = unsafe { modifiers_from_c(raw.as_ptr(), 1) }; + assert!(result.is_err()); +} + +#[test] +fn wheel_modifier_mask_maps_all_documented_bits() { + assert_eq!( + modifiers_from_mask(0b1111).unwrap(), + vec![ + CoreModifier::Meta, + CoreModifier::Ctrl, + CoreModifier::Alt, + CoreModifier::Shift, + ] + ); +} + +#[test] +fn wheel_modifier_mask_rejects_unknown_bits() { + assert!(modifiers_from_mask(0b1_0000).is_err()); +} + +/// The core regression check: `build_mouse_event` must carry whatever +/// modifiers it was given through unchanged, not silently drop them the way +/// this function's predecessor did. +#[test] +fn build_mouse_event_carries_requested_modifiers_through_unchanged() { + let ev = AdMouseEvent { + kind: AdMouseEventKind::Click as i32, + point: AdPoint { x: 1.0, y: 2.0 }, + button: AdMouseButton::Left as i32, + click_count: 1, + }; + let core_event = build_mouse_event(&ev, vec![CoreModifier::Meta, CoreModifier::Shift]).unwrap(); + assert_eq!( + core_event.modifiers, + vec![CoreModifier::Meta, CoreModifier::Shift] + ); +} + +#[test] +fn build_mouse_event_rejects_invalid_button_discriminant() { + let ev = AdMouseEvent { + kind: AdMouseEventKind::Click as i32, + point: AdPoint { x: 1.0, y: 2.0 }, + button: -5, + click_count: 1, + }; + assert!(build_mouse_event(&ev, Vec::new()).is_err()); +} + +#[test] +fn build_mouse_event_rejects_zero_click_count() { + let ev = AdMouseEvent { + kind: AdMouseEventKind::Click as i32, + point: AdPoint { x: 1.0, y: 2.0 }, + button: AdMouseButton::Left as i32, + click_count: 0, + }; + + assert!(build_mouse_event(&ev, Vec::new()).is_err()); +} + +#[test] +fn build_mouse_event_rejects_excessive_click_count() { + let ev = AdMouseEvent { + kind: AdMouseEventKind::Click as i32, + point: AdPoint { x: 1.0, y: 2.0 }, + button: AdMouseButton::Left as i32, + click_count: agent_desktop_core::MAX_MOUSE_CLICK_COUNT + 1, + }; + + assert!(build_mouse_event(&ev, Vec::new()).is_err()); +} diff --git a/crates/ffi/src/lib.rs b/crates/ffi/src/lib.rs index 4db892a..20705c6 100644 --- a/crates/ffi/src/lib.rs +++ b/crates/ffi/src/lib.rs @@ -1,27 +1,17 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + //! # agent-desktop FFI //! //! C-ABI surface over `PlatformAdapter`. Exposes //! `libagent_desktop_ffi.{dylib,so,dll}` to Python / Swift / Go / Node / //! C++ consumers. //! -//! ## ⚠ Thread safety (macOS) +//! ## Thread safety //! -//! Every adapter-touching FFI entry must be invoked on the process's -//! main thread. The guard runs at **runtime in every build profile**: -//! a worker-thread call returns `AD_RESULT_ERR_INTERNAL` with a -//! `'static` diagnostic message — no silent UB even under -//! `--profile release-ffi`. -//! -//! Operations exempt from the guard (safe from any thread): -//! -//! - `ad_abi_version` / `ad_init` (no adapter, no AX/Cocoa state) -//! - `ad_adapter_create` / `ad_adapter_destroy` -//! - `ad_last_error_*` readers -//! - `ad_check_permissions`, `ad_status` (permission + ref-store reads, no AX tree traversal) -//! - All `ad_*_list_{count,get,free}` accessors and -//! `ad_image_buffer_*` accessors -//! - `ad_release_window_fields`, `ad_free_handle`, `ad_free_tree`, -//! `ad_free_action_result`, `ad_free_string` +//! Adapters are safe to call from multiple host threads. Native element handles +//! are thread-affine registry capabilities and must be used and released on the +//! thread that resolved them. Desktop mutations are serialized by the adapter's +//! cross-process interaction lease. //! //! ## Build profile //! @@ -42,65 +32,128 @@ //! any number of subsequent successful calls on the same thread; only //! the next *failing* call rotates it. Matches POSIX `errno` semantics. +#[cfg(panic = "abort")] +compile_error!( + "agent-desktop-ffi requires panic=unwind; build with --profile release-ffi, not --release" +); + pub(crate) mod abi_version; pub(crate) mod actions; pub(crate) mod adapter; pub(crate) mod apps; pub(crate) mod commands; pub(crate) mod convert; +pub(crate) mod displays; pub(crate) mod enum_validation; pub mod error; pub(crate) mod ffi_try; pub(crate) mod input; pub(crate) mod log_callback; -pub(crate) mod main_thread; pub(crate) mod notifications; pub(crate) mod observation; +pub(crate) mod opaque_id; +pub(crate) mod operation; +#[cfg(feature = "panic-injection")] +pub(crate) mod panic_injection; pub(crate) mod pointer_guard; +pub(crate) mod resource; pub(crate) mod screenshot; +pub(crate) mod session_lease; pub(crate) mod surfaces; pub(crate) mod tree; -pub mod types; +pub(crate) mod types; pub(crate) mod windows; pub use abi_version::AD_ABI_VERSION_MAJOR; pub use adapter::AdAdapter; pub use error::AdResult; -pub use types::action::AdAction; +pub use types::action::{AD_ACTION_SIZE, AdAction, ad_action_size}; pub use types::action_kind::AdActionKind; -pub use types::action_result::AdActionResult; -pub use types::action_step::AdActionStep; +pub use types::action_result::{AD_ACTION_RESULT_SIZE, AdActionResult, ad_action_result_size}; +pub use types::action_step::{AD_ACTION_STEP_SIZE, AdActionStep, ad_action_step_size}; pub use types::app_info::AdAppInfo; pub use types::app_list::AdAppList; +pub use types::delivery_disposition::AdDeliveryDisposition; +pub use types::delivery_semantics::{AD_DELIVERY_SEMANTICS_SIZE, AdDeliverySemantics}; pub use types::direction::AdDirection; -pub use types::drag_params::AdDragParams; -pub use types::element_state::AdElementState; -pub use types::find_query::AdFindQuery; +pub use types::display_info::{ + AD_DISPLAY_INFO_SIZE, AD_DISPLAY_INFO_VERSION, AdDisplayInfo, ad_display_info_size, +}; +pub use types::display_list::AdDisplayList; +pub use types::drag_params::{AD_DRAG_PARAMS_SIZE, AdDragParams, ad_drag_params_size}; +pub use types::element_state::{AD_ELEMENT_STATE_SIZE, AdElementState, ad_element_state_size}; +pub use types::exact_ref_entry::{ + AD_EXACT_REF_ENTRY_SIZE, AD_EXACT_REF_ENTRY_VERSION, AdExactRefEntry, ad_exact_ref_entry_size, +}; +pub use types::exact_surface_info::{ + AD_EXACT_SURFACE_INFO_SIZE, AD_EXACT_SURFACE_INFO_VERSION, AdExactSurfaceInfo, + ad_exact_surface_info_size, +}; +pub use types::exact_surface_list::AdExactSurfaceList; +pub use types::exact_window_info::{ + AD_EXACT_WINDOW_INFO_SIZE, AD_EXACT_WINDOW_INFO_VERSION, AdExactWindowInfo, + ad_exact_window_info_size, +}; +pub use types::exact_window_list::AdExactWindowList; +pub use types::find_control::{AD_FIND_CONTROL_SIZE, AdFindControl}; +pub use types::find_filter::{AD_FIND_FILTER_SIZE, AdFindFilter}; +pub use types::find_identity::{AD_FIND_IDENTITY_SIZE, AdFindIdentity}; +pub use types::find_query::{AD_FIND_QUERY_SIZE, AD_FIND_QUERY_VERSION, AdFindQuery}; +pub use types::find_selection::{AD_FIND_SELECTION_SIZE, AdFindSelection}; +pub use types::find_selection_kind::AdFindSelectionKind; +pub use types::find_state_predicate::{AD_FIND_STATE_PREDICATE_SIZE, AdFindStatePredicate}; +pub use types::find_state_slice::{AD_FIND_STATE_SLICE_SIZE, AdFindStateSlice}; +pub use types::identifier_kind::AdIdentifierKind; pub use types::image_buffer::AdImageBuffer; pub use types::image_format::AdImageFormat; pub use types::key_combo::AdKeyCombo; -pub use types::modifier::AdModifier; +pub use types::modifier::{AD_MODIFIER_CMD, AdModifier}; pub use types::mouse_button::AdMouseButton; pub use types::mouse_event::AdMouseEvent; pub use types::mouse_event_kind::AdMouseEventKind; pub use types::native_handle::AdNativeHandle; -pub use types::node::AdNode; +pub use types::node::{AD_NODE_SIZE, AdNode}; +pub use types::node_content::{AD_NODE_CONTENT_SIZE, AdNodeContent}; +pub use types::node_presentation::{AD_NODE_PRESENTATION_SIZE, AdNodePresentation}; +pub use types::node_relation::{AD_NODE_RELATION_SIZE, AdNodeRelation}; pub use types::node_tree::AdNodeTree; +pub use types::notification_action_request::{ + AD_NOTIFICATION_ACTION_REQUEST_SIZE, AdNotificationActionRequest, +}; pub use types::notification_filter::AdNotificationFilter; +pub use types::notification_identity::{AD_NOTIFICATION_IDENTITY_SIZE, AdNotificationIdentity}; pub use types::notification_info::AdNotificationInfo; pub use types::notification_list::AdNotificationList; +pub use types::optional_u64::{AD_OPTIONAL_U64_SIZE, AdOptionalU64}; +pub use types::optional_usize::{AD_OPTIONAL_USIZE_SIZE, AdOptionalUsize}; pub use types::point::AdPoint; pub use types::policy_kind::AdPolicyKind; pub use types::rect::AdRect; -pub use types::ref_entry::AdRefEntry; +pub use types::ref_capabilities::{AD_REF_CAPABILITIES_SIZE, AdRefCapabilities}; +pub use types::ref_entry::{ + AD_MAX_REF_ACTIONS, AD_MAX_REF_PATH_DEPTH, AD_MAX_REF_STATES, AD_REF_ENTRY_SIZE, AdRefEntry, + ad_ref_entry_size, +}; +pub use types::ref_geometry::{AD_REF_GEOMETRY_SIZE, AdRefGeometry}; +pub use types::ref_identity::{AD_REF_IDENTITY_SIZE, AdRefIdentity}; +pub use types::ref_process::{AD_REF_PROCESS_SIZE, AdRefProcess}; +pub use types::ref_scope::{AD_REF_SCOPE_SIZE, AdRefScope}; +pub use types::ref_source::{AD_REF_SOURCE_SIZE, AdRefSource}; +pub use types::retry_disposition::AdRetryDisposition; pub use types::screenshot_kind::AdScreenshotKind; pub use types::screenshot_target::AdScreenshotTarget; pub use types::scroll_params::AdScrollParams; pub use types::snapshot_surface::AdSnapshotSurface; +pub use types::step_mechanism::AdStepMechanism; +pub use types::string_slice::{AD_STRING_SLICE_SIZE, AdStringSlice}; pub use types::surface_info::AdSurfaceInfo; pub use types::surface_list::AdSurfaceList; pub use types::tree_options::AdTreeOptions; -pub use types::wait_args::AdWaitArgs; +pub use types::wait_args::{AD_WAIT_ARGS_SIZE, AdWaitArgs, ad_wait_args_size}; +pub use types::wait_mode::{AD_WAIT_MODE_SIZE, AdWaitMode}; +pub use types::wait_predicate::{AD_WAIT_PREDICATE_SIZE, AdWaitPredicate}; +pub use types::wait_scope::{AD_WAIT_SCOPE_SIZE, AdWaitScope}; +pub use types::wait_surface_modes::{AD_WAIT_SURFACE_MODES_SIZE, AdWaitSurfaceModes}; pub use types::window_info::AdWindowInfo; pub use types::window_list::AdWindowList; pub use types::window_op::AdWindowOp; diff --git a/crates/ffi/src/log_callback.rs b/crates/ffi/src/log_callback.rs index 3262189..c2ef8f6 100644 --- a/crates/ffi/src/log_callback.rs +++ b/crates/ffi/src/log_callback.rs @@ -1,103 +1,38 @@ -//! `ad_set_log_callback` — forward `tracing` events to a consumer callback. +//! Scoped forwarding of Rust tracing events to a foreign callback. //! -//! # Thread-safety -//! -//! `tracing` events fire from arbitrary threads. The callback pointer is -//! stored in a global `AtomicPtr` (lock-free). A `tracing_subscriber` layer -//! is installed exactly once (via [`OnceLock`]) when the first non-null callback -//! is registered; subsequent registrations only swap the pointer. If a foreign -//! global subscriber already owns the process at the time of the first -//! registration, the install fails and `AD_RESULT_ERR_INTERNAL` is returned — -//! the callback is never stored and events will not be delivered. -//! -//! Re-entrancy is prevented by a per-thread flag: if a consumer callback -//! itself emits a `tracing` event, the recursive `on_event` invocation is -//! silently discarded rather than overflowing the stack. -//! -//! # Level mapping -//! -//! | `tracing` level | `level` passed to callback | -//! |-----------------|---------------------------| -//! | ERROR | 1 | -//! | WARN | 2 | -//! | INFO | 3 | -//! | DEBUG | 4 | -//! | TRACE | 5 | -//! -//! # Pointer lifetime -//! -//! The `msg` pointer passed to the callback is valid **only for the duration -//! of the call**. The consumer must copy the string before returning. -//! -//! # Redaction -//! -//! Fields whose keys match `SENSITIVE_KEYS` (password, token, text, …) are -//! replaced with `{"redacted":true}` before the message is formatted, using -//! the same logic as the file-trace writer. +//! A callback is active only while an `ad_*` entrypoint executes on the +//! calling thread. The FFI never installs or replaces the host process's +//! global tracing subscriber. use std::cell::Cell; use std::ffi::{CString, c_char}; -use std::os::raw::c_void; use std::panic::{AssertUnwindSafe, catch_unwind}; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::{Mutex, MutexGuard}; use agent_desktop_core::sanitize_trace_value; use serde_json::{Map, Value}; use tracing::field::{Field, Visit}; -use tracing::{Event, Level, Subscriber}; +use tracing::{Dispatch, Event, Level, Subscriber}; use tracing_subscriber::Layer; use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; use crate::error::AdResult; -/// Raw storage for the consumer callback. Null means no callback registered. -/// Stored as `*mut c_void` to avoid `fn` pointer restrictions on `AtomicPtr`. -static CALLBACK_SLOT: AtomicPtr<c_void> = AtomicPtr::new(std::ptr::null_mut()); - -/// Memoises the outcome of the one-time subscriber install attempt. -/// -/// `true` — our `CallbackLayer` subscriber is installed and callback delivery works. -/// `false` — `try_init` failed because a foreign global subscriber already owns the -/// process; our layer was never installed. -static INSTALL_RESULT: OnceLock<bool> = OnceLock::new(); +static CALLBACK: Mutex<Option<unsafe extern "C" fn(level: i32, msg: *const c_char)>> = + Mutex::new(None); thread_local! { static IN_CALLBACK: Cell<bool> = const { Cell::new(false) }; } -/// RAII guard that resets [`IN_CALLBACK`] on drop, so a panicking callback -/// does not permanently poison the flag on its thread. struct CallbackGuard; impl Drop for CallbackGuard { fn drop(&mut self) { - IN_CALLBACK.with(|g| g.set(false)); + IN_CALLBACK.set(false); } } -/// The concrete type of the consumer callback. -type LogCb = unsafe extern "C" fn(level: i32, msg: *const c_char); - -/// Soundness guard: `fn` pointer and data pointer must be the same size so -/// the `transmute` in `on_event` is layout-safe on every target. -const _: () = assert!( - std::mem::size_of::<LogCb>() == std::mem::size_of::<*mut c_void>(), - "fn pointer size must equal data pointer size for LogCb transmute" -); - -fn level_to_i32(level: &Level) -> i32 { - match *level { - Level::ERROR => 1, - Level::WARN => 2, - Level::INFO => 3, - Level::DEBUG => 4, - Level::TRACE => 5, - } -} - -/// Collects event fields into a `serde_json::Map`. struct FieldCollector { map: Map<String, Value>, } @@ -126,126 +61,79 @@ impl Visit for FieldCollector { } } -/// `tracing_subscriber` layer that forwards events to the registered callback. struct CallbackLayer; impl<S: Subscriber> Layer<S> for CallbackLayer { fn on_event(&self, event: &Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) { - let ptr = CALLBACK_SLOT.load(Ordering::Acquire); - if ptr.is_null() { + if IN_CALLBACK.get() { return; } - - if IN_CALLBACK.with(Cell::get) { + let Some(callback) = current_callback() else { return; - } - - let _ = catch_unwind(AssertUnwindSafe(|| { - let level_i32 = level_to_i32(event.metadata().level()); - - let mut collector = FieldCollector { map: Map::new() }; - event.record(&mut collector); - - let sanitized = sanitize_trace_value(Value::Object(collector.map)); - let msg_str = serde_json::to_string(&sanitized).unwrap_or_default(); - - let Ok(c_msg) = CString::new(msg_str) else { - return; - }; - - IN_CALLBACK.with(|g| g.set(true)); - let cb: LogCb = unsafe { std::mem::transmute(ptr) }; - let _reset = CallbackGuard; - unsafe { cb(level_i32, c_msg.as_ptr()) }; - })); + }; + let message = catch_unwind(AssertUnwindSafe(|| format_event(event))) + .ok() + .flatten(); + let Some(message) = message else { + return; + }; + IN_CALLBACK.set(true); + let _guard = CallbackGuard; + unsafe { callback(level_number(event.metadata().level()), message.as_ptr()) }; } } -fn install_layer_once() -> bool { - *INSTALL_RESULT.get_or_init(|| { - tracing_subscriber::registry() - .with(CallbackLayer) - .try_init() - .is_ok() - }) -} - -/// Maps an install outcome to the `AdResult` that `ad_set_log_callback` should -/// return when registering a non-null callback. -/// -/// Extracted as a named function so the routing logic can be unit-tested -/// independently of global subscriber state. -#[cfg(test)] -fn result_for_install(succeeded: bool) -> AdResult { - if succeeded { - AdResult::Ok - } else { - AdResult::ErrInternal +fn callback_slot() +-> MutexGuard<'static, Option<unsafe extern "C" fn(level: i32, msg: *const c_char)>> { + match CALLBACK.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), } } -/// Registers a callback to receive `tracing` events, or unregisters the -/// current callback when `cb` is `NULL`. +fn current_callback() -> Option<unsafe extern "C" fn(level: i32, msg: *const c_char)> { + *callback_slot() +} + +fn level_number(level: &Level) -> i32 { + match *level { + Level::ERROR => 1, + Level::WARN => 2, + Level::INFO => 3, + Level::DEBUG => 4, + Level::TRACE => 5, + } +} + +fn format_event(event: &Event<'_>) -> Option<CString> { + let mut collector = FieldCollector { map: Map::new() }; + event.record(&mut collector); + let sanitized = sanitize_trace_value(Value::Object(collector.map)); + CString::new(serde_json::to_string(&sanitized).ok()?).ok() +} + +pub(crate) fn with_dispatch<R>(body: impl FnOnce() -> R) -> R { + if current_callback().is_none() { + return body(); + } + let subscriber = tracing_subscriber::registry().with(CallbackLayer); + let dispatch = Dispatch::new(subscriber); + tracing::dispatcher::with_default(&dispatch, body) +} + +/// Registers or clears the callback used for events emitted synchronously +/// inside later `ad_*` calls on the same thread. /// -/// # Install semantics -/// -/// The subscriber layer is installed exactly once — on the first call with a -/// non-null `cb`. If a foreign global subscriber already owns the process at -/// that point, the install fails and this function returns -/// `AD_RESULT_ERR_INTERNAL` with a diagnostic last-error. No callback pointer -/// is stored in that case; events will never be delivered until the consumer -/// remedies the conflict. Subsequent calls with a non-null `cb` after a -/// **successful** install only swap the stored pointer and always return -/// `AD_RESULT_OK`. -/// -/// `NULL` always returns `AD_RESULT_OK` — unregistering cannot fail. -/// -/// # Callback contract -/// -/// - `level` — 1 (ERROR) … 5 (TRACE) -/// - `msg` — a NUL-terminated JSON string; valid only for the call's duration -/// -/// Sensitive field values (password, token, text, …) are replaced with -/// `{"redacted":true}` before the message is formatted. -/// -/// Invocations are best-effort. A panicking callback is caught and silently -/// discarded; no command fails because of a trace delivery error. A callback -/// that emits `tracing` events is safe: the recursive `on_event` is dropped -/// by a per-thread guard before it reaches the callback again. -/// -/// # Safety -/// -/// `cb` must be null or a valid function pointer with the declared signature. -/// The pointer is stored atomically; the subscriber may call it from threads -/// other than the registering thread. -/// -/// A callback unregistered via `NULL` may still be invoked from another thread -/// for a brief window after this call returns. The callback (and any data it -/// captures) must remain valid for the process lifetime, or the caller must -/// quiesce all tracing sources before unregistering. +/// The callback may be invoked concurrently by different host threads. The +/// message pointer is valid only until the callback returns. The callback must +/// not unwind across this C ABI boundary; C++ exceptions and Rust panics must +/// be caught inside the callback. Violating that contract may abort the host. #[unsafe(no_mangle)] pub extern "C" fn ad_set_log_callback( - cb: Option<unsafe extern "C" fn(level: i32, msg: *const c_char)>, + callback: Option<unsafe extern "C" fn(level: i32, msg: *const c_char)>, ) -> AdResult { crate::ffi_try::trap_panic(|| { - match cb { - Some(f) => { - if !install_layer_once() { - crate::error::set_last_error_static( - AdResult::ErrInternal, - c"ad_set_log_callback: a foreign tracing subscriber already owns this \ - process; our CallbackLayer was not installed and events will not be \ - delivered", - ); - return AdResult::ErrInternal; - } - let raw = f as *mut c_void; - CALLBACK_SLOT.store(raw, Ordering::Release); - } - None => { - CALLBACK_SLOT.store(std::ptr::null_mut(), Ordering::Release); - } - } + *callback_slot() = callback; AdResult::Ok }) } @@ -253,24 +141,41 @@ pub extern "C" fn ad_set_log_callback( #[cfg(test)] mod tests { use super::*; + use std::ffi::CStr; + use std::sync::Mutex; - #[test] - fn result_for_install_success_returns_ok() { - assert_eq!(result_for_install(true), AdResult::Ok); + static MESSAGES: Mutex<Vec<String>> = Mutex::new(Vec::new()); + static LOG_TEST_LOCK: Mutex<()> = Mutex::new(()); + + unsafe extern "C" fn record(_level: i32, message: *const c_char) { + let message = unsafe { CStr::from_ptr(message) } + .to_string_lossy() + .into_owned(); + MESSAGES.lock().unwrap().push(message); } #[test] - fn result_for_install_failure_returns_err_internal() { - assert_eq!(result_for_install(false), AdResult::ErrInternal); + fn scoped_dispatch_delivers_and_redacts() { + let _guard = LOG_TEST_LOCK.lock().unwrap(); + MESSAGES.lock().unwrap().clear(); + assert_eq!(ad_set_log_callback(Some(record)), AdResult::Ok); + with_dispatch(|| tracing::error!(password = "secret", operation = "login")); + assert_eq!(ad_set_log_callback(None), AdResult::Ok); + + let messages = MESSAGES.lock().unwrap(); + assert_eq!(messages.len(), 1); + assert!(messages[0].contains("redacted")); + assert!(messages[0].contains("login")); + assert!(!messages[0].contains("secret")); } #[test] - fn null_unregister_always_ok_independent_of_install_state() { - let result = ad_set_log_callback(None); - assert_eq!( - result, - AdResult::Ok, - "NULL unregister must return Ok regardless of install state" - ); + fn callback_does_not_capture_outside_scope() { + let _guard = LOG_TEST_LOCK.lock().unwrap(); + MESSAGES.lock().unwrap().clear(); + assert_eq!(ad_set_log_callback(Some(record)), AdResult::Ok); + tracing::error!(operation = "outside"); + assert_eq!(ad_set_log_callback(None), AdResult::Ok); + assert!(MESSAGES.lock().unwrap().is_empty()); } } diff --git a/crates/ffi/src/main_thread.rs b/crates/ffi/src/main_thread.rs deleted file mode 100644 index bba2b30..0000000 --- a/crates/ffi/src/main_thread.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! Main-thread enforcement for macOS-sensitive FFI entrypoints. -//! -//! macOS accessibility (AX) and Cocoa APIs must run on the process's -//! main thread. Off-thread calls silently corrupt state or crash, and -//! the crash looks like memory corruption from the consumer side — -//! impossible to debug from a Python / Node / Swift host. -//! -//! `require_main_thread()` performs a **runtime** check (always, in -//! every build profile) and returns `AD_RESULT_ERR_INTERNAL` with a -//! `'static` diagnostic last-error when a worker-thread call is -//! detected. The check compiles away on non-macOS targets — AT-SPI -//! and UIA don't impose the same affinity rule. -//! -//! Exempt from the rule: `ad_adapter_create`, `ad_adapter_destroy`, -//! `ad_last_error_*`, and the entire `ad_free_*` / `ad_*_list_free` / -//! `ad_image_buffer_free` / `ad_release_window_fields` / `ad_free_handle` -//! / `ad_free_string` / `ad_free_action_result` / `ad_free_tree` family. -//! Those paths touch no AX/Cocoa state and are safe from any thread. - -use crate::error::{AdResult, set_last_error_static}; -use std::ffi::CStr; - -static OFF_MAIN_THREAD_MESSAGE: &CStr = - c"agent_desktop FFI entry called off the main thread (macOS requires main-thread AX/Cocoa calls)"; - -#[cfg(target_os = "macos")] -pub(crate) fn is_main_thread() -> bool { - unsafe { libc::pthread_main_np() != 0 } -} - -#[cfg(not(target_os = "macos"))] -pub(crate) fn is_main_thread() -> bool { - true -} - -/// Fail-closed runtime main-thread check. Returns -/// `Ok(())` on the main thread (always, on non-macOS). Returns -/// `Err(AdResult::ErrInternal)` on a worker thread with the last-error -/// slot populated with a `'static` diagnostic message. -/// -/// Unlike a `debug_assert!`, this variant still fires in -/// `--profile release-ffi` and other optimized builds. -#[inline] -pub(crate) fn require_main_thread() -> Result<(), AdResult> { - if is_main_thread() { - Ok(()) - } else { - set_last_error_static(AdResult::ErrInternal, OFF_MAIN_THREAD_MESSAGE); - Err(AdResult::ErrInternal) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn is_main_thread_call_is_always_safe_even_on_workers() { - let _ = is_main_thread(); - } - - #[test] - fn require_main_thread_returns_err_on_worker() { - let outcome = std::thread::spawn(require_main_thread).join().unwrap(); - #[cfg(target_os = "macos")] - assert!(matches!(outcome, Err(AdResult::ErrInternal))); - #[cfg(not(target_os = "macos"))] - assert!(outcome.is_ok()); - } -} diff --git a/crates/ffi/src/notifications/action.rs b/crates/ffi/src/notifications/action.rs index 8942585..465917c 100644 --- a/crates/ffi/src/notifications/action.rs +++ b/crates/ffi/src/notifications/action.rs @@ -1,11 +1,9 @@ use crate::AdAdapter; use crate::actions::result::action_result_to_c; -use crate::convert::string::{c_to_string, decode_optional_filter}; +use crate::convert::string::required_adapter_string; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::types::AdActionResult; -use agent_desktop_core::notification::NotificationIdentity; -use std::os::raw::c_char; +use crate::types::{AdActionResult, AdNotificationActionRequest, AdPolicyKind}; /// Triggers the named action on the notification at `index`. Typical /// action names are those reported in `AdNotificationInfo.actions` @@ -20,61 +18,76 @@ use std::os::raw::c_char; /// press the action button on a different notification than the host /// intended. /// -/// `expected_app` and `expected_title` let the host pin the targeted -/// notification to an observed fingerprint. If either pointer is -/// non-null, the row currently at `index` must match that field or the -/// call fails closed with `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`. Both -/// null preserves the legacy index-only behavior for hosts that do -/// their own reconciliation. +/// `request.identity` pins the target to an observed fingerprint. At least one +/// identity field is required; a mismatch fails closed with +/// `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`. /// /// # Safety -/// `adapter` must be valid. `action_name` must be a non-null UTF-8 -/// C string. `expected_app` and `expected_title` must each be null -/// or a NUL-terminated UTF-8 C string. Invalid UTF-8 in either field +/// `adapter` and `request` must be valid. `request.action_name` must be a +/// non-null UTF-8 C string. Identity fields must each be null or a +/// NUL-terminated UTF-8 C string. Invalid UTF-8 in either field /// is rejected with `AD_RESULT_ERR_INVALID_ARGS` rather than silently /// treated as "no fingerprint". `out` must be a valid writable /// `*mut AdActionResult`; on error it is zero-initialized. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_notification_action( adapter: *const AdAdapter, - index: u32, - expected_app: *const c_char, - expected_title: *const c_char, - action_name: *const c_char, + request: *const AdNotificationActionRequest, out: *mut AdActionResult, ) -> AdResult { trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = std::mem::zeroed(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; - let action = match c_to_string(action_name) { - Some(s) => s, + crate::pointer_guard::guard_non_null!(request, c"request is null"); + let request = &*request; + let index = match super::index::notification_index(request.index) { + Ok(index) => index, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let action = match required_adapter_string(request.action_name, "action_name") { + Ok(action) => action, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let identity = match super::identity::decode(request.identity.app, request.identity.title) { + Ok(identity) => identity, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let policy = match AdPolicyKind::from_c(request.policy) { + Some(policy) => policy.to_interaction_policy(), None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "action_name is null or invalid UTF-8", + set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Invalid notification action policy", )); return AdResult::ErrInvalidArgs; } }; - let expected_app = decode_optional_filter!(expected_app, "expected_app"); - let expected_title = decode_optional_filter!(expected_title, "expected_title"); - let identity = if expected_app.is_some() || expected_title.is_some() { - Some(NotificationIdentity { - expected_app, - expected_title, - }) - } else { - None + if !policy.allow_focus_steal { + set_last_error(&agent_desktop_core::AdapterError::policy_denied_for_policy( + "Notification actions open and focus the operating system notification surface", + policy, + )); + return crate::error::last_error_code(); + } + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + let request = agent_desktop_core::NotificationActionRequest { + index, + identity: &identity, + action_name: &action, + policy, }; - match adapter - .inner - .notification_action(index as usize, identity.as_ref(), &action) - { + match adapter.inner.notification_action(request, &lease) { Ok(result) => { *out = action_result_to_c(&result); AdResult::Ok diff --git a/crates/ffi/src/notifications/dismiss.rs b/crates/ffi/src/notifications/dismiss.rs index 65f00af..0fabc8f 100644 --- a/crates/ffi/src/notifications/dismiss.rs +++ b/crates/ffi/src/notifications/dismiss.rs @@ -4,34 +4,49 @@ use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; use std::os::raw::c_char; -/// Dismisses the notification at `index`. Indexes are only valid within -/// the response to the most recent `ad_list_notifications` call on this -/// thread — the adapter re-queries internally, so dismissing by a stale -/// index returns `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`. +/// Dismisses a notification only when the current row matches an identity +/// observed in the same listing. At least one expected field is required. /// /// # Safety -/// `adapter` must be valid. `app_filter` may be null. +/// `adapter` must be valid. String pointers may be null and otherwise must be +/// NUL-terminated UTF-8. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_dismiss_notification( adapter: *const AdAdapter, index: u32, app_filter: *const c_char, + expected_app: *const c_char, + expected_title: *const c_char, ) -> AdResult { trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; + let index = match super::index::notification_index(index) { + Ok(index) => index, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; let filter = decode_optional_filter!(app_filter, "app_filter"); - let filter_ref = filter.as_deref(); - match adapter - .inner - .dismiss_notification(index as usize, filter_ref) - { + let identity = match super::identity::decode(expected_app, expected_title) { + Ok(identity) => identity, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let adapter = crate::adapter::acquire_adapter!(adapter); + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + let request = agent_desktop_core::DismissNotificationRequest { + index, + app_filter: filter.as_deref(), + identity: &identity, + policy: agent_desktop_core::interaction_policy::InteractionPolicy::headless(), + }; + match adapter.inner.dismiss_notification(request, &lease) { Ok(_) => AdResult::Ok, - Err(e) => { - set_last_error(&e); + Err(error) => { + set_last_error(&error); crate::error::last_error_code() } } diff --git a/crates/ffi/src/notifications/dismiss_all.rs b/crates/ffi/src/notifications/dismiss_all.rs index d5839b3..faf2d30 100644 --- a/crates/ffi/src/notifications/dismiss_all.rs +++ b/crates/ffi/src/notifications/dismiss_all.rs @@ -35,15 +35,24 @@ pub unsafe extern "C" fn ad_dismiss_all_notifications( crate::pointer_guard::guard_non_null!(failed_out, c"failed_out is null"); *dismissed_out = ptr::null_mut(); *failed_out = ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; let filter = decode_optional_filter!(app_filter, "app_filter"); + let adapter = crate::adapter::acquire_adapter!(adapter); let filter_ref = filter.as_deref(); - match adapter.inner.dismiss_all_notifications(filter_ref) { + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + let request = agent_desktop_core::DismissAllNotificationsRequest { + app_filter: filter_ref, + policy: agent_desktop_core::interaction_policy::InteractionPolicy::headless(), + }; + match adapter.inner.dismiss_all_notifications(request, &lease) { Ok((dismissed, failed_messages)) => { + if let Err(error) = crate::resource::validate_list_len( + dismissed.len().max(failed_messages.len()), + "Notification mutation result", + ) { + set_last_error(&error); + return crate::error::last_error_code(); + } let dismissed_items: Vec<AdNotificationInfo> = dismissed.iter().map(notification_info_to_c).collect(); let dismissed_list = Box::new(AdNotificationList { @@ -55,7 +64,7 @@ pub unsafe extern "C" fn ad_dismiss_all_notifications( .into_iter() .enumerate() .map(|(i, msg)| { - let info = agent_desktop_core::notification::NotificationInfo { + let info = agent_desktop_core::NotificationInfo { index: i, app_name: String::new(), title: String::from("dismiss failed"), diff --git a/crates/ffi/src/notifications/filter.rs b/crates/ffi/src/notifications/filter.rs index 2d0a681..183b1db 100644 --- a/crates/ffi/src/notifications/filter.rs +++ b/crates/ffi/src/notifications/filter.rs @@ -1,7 +1,7 @@ use crate::convert::string::try_c_to_string; use crate::types::AdNotificationFilter; -use agent_desktop_core::error::{AdapterError, ErrorCode}; -use agent_desktop_core::notification::NotificationFilter; +use agent_desktop_core::NotificationFilter; +use agent_desktop_core::{AdapterError, ErrorCode}; /// Converts a C `AdNotificationFilter` into the core filter type. /// diff --git a/crates/ffi/src/notifications/identity.rs b/crates/ffi/src/notifications/identity.rs new file mode 100644 index 0000000..dc97c46 --- /dev/null +++ b/crates/ffi/src/notifications/identity.rs @@ -0,0 +1,19 @@ +use crate::convert::string::optional_adapter_string; +use agent_desktop_core::{AdapterError, ErrorCode, NotificationIdentity}; + +pub(crate) fn decode( + expected_app: *const std::os::raw::c_char, + expected_title: *const std::os::raw::c_char, +) -> Result<NotificationIdentity, AdapterError> { + let identity = NotificationIdentity { + expected_app: optional_adapter_string(expected_app, "expected_app")?, + expected_title: optional_adapter_string(expected_title, "expected_title")?, + }; + if identity.is_empty() { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "expected_app or expected_title is required", + )); + } + Ok(identity) +} diff --git a/crates/ffi/src/notifications/index.rs b/crates/ffi/src/notifications/index.rs new file mode 100644 index 0000000..15d8703 --- /dev/null +++ b/crates/ffi/src/notifications/index.rs @@ -0,0 +1,22 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; + +pub(crate) fn notification_index(index: u32) -> Result<usize, AdapterError> { + if index == 0 { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Notification index is 1-based and must be greater than zero", + )); + } + Ok(index as usize) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn index_is_one_based() { + assert!(notification_index(0).is_err()); + assert_eq!(notification_index(1).unwrap(), 1); + } +} diff --git a/crates/ffi/src/notifications/list.rs b/crates/ffi/src/notifications/list.rs index c9d3fe4..992c6eb 100644 --- a/crates/ffi/src/notifications/list.rs +++ b/crates/ffi/src/notifications/list.rs @@ -8,10 +8,9 @@ use std::ptr; /// Lists the notifications currently on-screen. /// -/// Notification indexes are only stable within a single list response. -/// Pass them straight to `ad_dismiss_notification` / -/// `ad_notification_action` without caching across ticks — the adapter -/// re-queries Notification Center internally on every call. +/// Notification indexes are only stable within a single list response. Pass +/// the entry's app or title fingerprint to the checked mutation functions; +/// index-only mutations are rejected. /// /// # Safety /// `adapter` must be valid. `filter` may be null. `out` must be a valid @@ -26,11 +25,7 @@ pub unsafe extern "C" fn ad_list_notifications( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; let core_filter = match filter_from_c(filter) { Ok(f) => f, Err(e) => { @@ -38,8 +33,21 @@ pub unsafe extern "C" fn ad_list_notifications( return crate::error::last_error_code(); } }; - match adapter.inner.list_notifications(&core_filter) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.list_notifications( + &core_filter, + agent_desktop_core::InteractionPolicy::headless(), + deadline, + None, + ) { Ok(notifications) => { + if let Err(error) = + crate::resource::validate_list_len(notifications.len(), "Notification list") + { + set_last_error(&error); + return crate::error::last_error_code(); + } let items: Vec<AdNotificationInfo> = notifications.iter().map(notification_info_to_c).collect(); let list = Box::new(AdNotificationList { diff --git a/crates/ffi/src/notifications/mod.rs b/crates/ffi/src/notifications/mod.rs index 4468e29..ea4befa 100644 --- a/crates/ffi/src/notifications/mod.rs +++ b/crates/ffi/src/notifications/mod.rs @@ -2,4 +2,6 @@ pub(crate) mod action; pub(crate) mod dismiss; pub(crate) mod dismiss_all; pub(crate) mod filter; +mod identity; +mod index; pub(crate) mod list; diff --git a/crates/ffi/src/observation/find.rs b/crates/ffi/src/observation/find.rs index 1dabdbd..17c5bba 100644 --- a/crates/ffi/src/observation/find.rs +++ b/crates/ffi/src/observation/find.rs @@ -1,28 +1,24 @@ use crate::AdAdapter; -use crate::convert::string::decode_optional_filter; +use crate::convert::string::optional_adapter_string; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::observation::walk::find_first_match; -use crate::types::{AdFindQuery, AdNativeHandle, AdWindowInfo}; -use agent_desktop_core::adapter::{SnapshotSurface, TreeOptions}; -use agent_desktop_core::refs::RefEntry; -use std::mem::ManuallyDrop; +use crate::types::{ + AdExactWindowInfo, AdFindQuery, AdFindSelectionKind, AdNativeHandle, AdWindowInfo, +}; +use agent_desktop_core::{ + AdapterError, ContainmentPredicate, ErrorCode, IdentityPredicate, LocatorMaterialization, + LocatorQuery, LocatorResolveRequest, LocatorSelection, ObservationRoot, StatePredicate, + resolve_query, +}; +use std::collections::HashSet; -/// Finds the first element in `win`'s accessibility tree matching the -/// query and resolves it to an opaque `AdNativeHandle`. The caller owns -/// the handle and must release it with `ad_free_handle(adapter, handle)` -/// once done. -/// -/// Matching is DFS order, first hit wins. All query fields are optional -/// (null = "don't care") and case-insensitive substring matches: -/// - `role` against `AccessibilityNode.role` -/// - `name_substring` against `AccessibilityNode.name` -/// - `value_substring` against `AccessibilityNode.value` -/// -/// The internal tree fetch always sets `include_bounds: true` so -/// `resolve_element_strict` can disambiguate duplicate-label siblings via -/// `bounds_hash`; without bounds on the matched node the resolver falls -/// back to role+name alone and may pick the wrong element. +const DEFAULT_FIND_TIMEOUT_MS: u64 = 5_000; +const MAX_FIND_STATES: usize = 64; +const MAX_CONTAINMENT_DEPTH: usize = 8; + +/// Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process +/// generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. +/// Use `ad_find_exact`. /// /// # Safety /// `adapter`, `win`, and `query` must be valid pointers. `out_handle` @@ -38,13 +34,9 @@ pub unsafe extern "C" fn ad_find( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out_handle, c"out_handle is null"); (*out_handle).ptr = std::ptr::null(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(win, c"win is null"); crate::pointer_guard::guard_non_null!(query, c"query is null"); - let adapter = &*adapter; let core_win = match crate::windows::ad_window_to_core(&*win) { Ok(w) => w, Err(e) => { @@ -52,74 +44,334 @@ pub unsafe extern "C" fn ad_find( return crate::error::last_error_code(); } }; - let q = &*query; - let role_filter = decode_optional_filter!(q.role, "query.role"); - let name_filter = decode_optional_filter!(q.name_substring, "query.name_substring"); - let value_filter = decode_optional_filter!(q.value_substring, "query.value_substring"); + find_in_window(adapter, &core_win, &*query, out_handle) + }) +} - let tree = match adapter.inner.get_tree( - &core_win, - &TreeOptions { - max_depth: 50, - include_bounds: true, - interactive_only: false, - compact: false, - surface: SnapshotSurface::Window, - skeleton: false, - }, - ) { - Ok(t) => t, - Err(e) => { - set_last_error(&e); +/// Finds and strictly resolves one element within a generation-pinned window. +/// `AdFindQuery.control.selection` must explicitly request first, last, or nth +/// behavior when duplicate matches are acceptable. The returned native handle +/// is adapter-bound and thread-affine; release it with `ad_free_handle` on the +/// resolving thread. +/// +/// # Safety +/// All pointers must be valid and `out_handle` must be writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_find_exact( + adapter: *const AdAdapter, + win: *const AdExactWindowInfo, + query: *const AdFindQuery, + out_handle: *mut AdNativeHandle, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out_handle, c"out_handle is null"); + (*out_handle).ptr = std::ptr::null(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(win, c"win is null"); + crate::pointer_guard::guard_non_null!(query, c"query is null"); + let window = match crate::windows::ad_exact_window_to_core(&*win) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); return crate::error::last_error_code(); } }; - - let matched = match find_first_match( - &tree, - role_filter.as_deref(), - name_filter.as_deref(), - value_filter.as_deref(), - ) { - Some(n) => n, - None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::ElementNotFound, - "no element matched the find query", - )); - return AdResult::ErrElementNotFound; - } - }; - - let bounds_hash = matched.bounds.as_ref().map(|r| r.bounds_hash()); - let ref_entry = RefEntry { - pid: core_win.pid, - role: matched.role.clone(), - name: matched.name.clone(), - value: matched.value.clone(), - description: matched.description.clone(), - states: matched.states.clone(), - bounds: matched.bounds, - bounds_hash, - available_actions: Vec::new(), - source_app: None, - source_window_id: Some(core_win.id.clone()), - source_window_title: Some(core_win.title.clone()), - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: false, - path: smallvec::SmallVec::new(), - }; - match adapter.inner.resolve_element_strict(&ref_entry) { - Ok(handle) => { - let handle = ManuallyDrop::new(handle); - (*out_handle).ptr = handle.as_raw(); - AdResult::Ok - } - Err(e) => { - set_last_error(&e); - crate::error::last_error_code() - } - } + find_in_window(adapter, &window, &*query, out_handle) }) } + +unsafe fn find_in_window( + adapter: *const AdAdapter, + window: &agent_desktop_core::WindowInfo, + query: &AdFindQuery, + out_handle: *mut AdNativeHandle, +) -> AdResult { + let (locator, request) = match unsafe { decode_query(query) } { + Ok(decoded) => decoded, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let adapter_id = adapter.addr(); + let adapter = crate::adapter::acquire_adapter!(adapter); + let resolution = match resolve_query( + adapter.inner.as_ref(), + &locator, + ObservationRoot::Window(window), + &request, + ) { + Ok(resolution) => resolution, + Err(e) => { + set_last_error(&crate::commands::app_error_to_adapter(e)); + return crate::error::last_error_code(); + } + }; + let selected = if request.selection == LocatorSelection::Strict { + match agent_desktop_core::require_unique(resolution) { + Ok(selected) => selected, + Err(error) => { + set_last_error(&crate::commands::app_error_to_adapter(error)); + return crate::error::last_error_code(); + } + } + } else { + if !resolution.meta.selection_complete { + set_last_error( + &AdapterError::timeout("Locator traversal could not prove the selected result") + .with_details(serde_json::json!({ + "kind": "locator_incomplete", + "observed_matches": resolution.meta.total_matches, + "query_stats": resolution.stats, + })), + ); + return AdResult::ErrTimeout; + } + let Some(selected) = resolution.matches.into_iter().next() else { + set_last_error(&AdapterError::new( + ErrorCode::ElementNotFound, + "Locator query matched no elements", + )); + return AdResult::ErrElementNotFound; + }; + selected + }; + let entry = selected.into_entry(); + let Some(process_instance) = entry.process.process_instance.clone() else { + set_last_error(&AdapterError::new( + ErrorCode::InvalidArgs, + "resolved element has no process-generation identity", + )); + return AdResult::ErrInvalidArgs; + }; + let process = agent_desktop_core::ProcessIdentity::new(entry.process.pid, process_instance); + match adapter + .inner + .resolve_element_strict(&entry, request.deadline) + { + Ok(handle) => { + match crate::actions::native_handle::into_ffi_handle(adapter_id, handle, process) { + Ok(token) => { + unsafe { (*out_handle).ptr = token }; + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } + } + Err(e) => { + set_last_error(&e); + crate::error::last_error_code() + } + } +} + +pub(crate) unsafe fn decode_query( + query: &AdFindQuery, +) -> Result<(LocatorQuery, LocatorResolveRequest), AdapterError> { + let timeout_ms = if query.control.timeout_ms == 0 { + DEFAULT_FIND_TIMEOUT_MS + } else { + query.control.timeout_ms + }; + let deadline = agent_desktop_core::Deadline::after(timeout_ms)?; + let selection = match AdFindSelectionKind::from_c(query.control.selection.kind) { + Some(AdFindSelectionKind::Strict) => LocatorSelection::Strict, + Some(AdFindSelectionKind::First) => LocatorSelection::First, + Some(AdFindSelectionKind::Last) => LocatorSelection::Last, + Some(AdFindSelectionKind::Nth) => LocatorSelection::Nth(query.control.selection.nth), + None => { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Invalid find selection kind", + )); + } + }; + let mut ancestors = HashSet::new(); + let locator = unsafe { decode_locator(query, 0, &mut ancestors)? }; + Ok(( + locator, + LocatorResolveRequest { + selection, + deadline, + max_raw_depth: 50, + materialization: LocatorMaterialization::None, + }, + )) +} + +unsafe fn decode_locator( + query: &AdFindQuery, + depth: usize, + ancestors: &mut HashSet<usize>, +) -> Result<LocatorQuery, AdapterError> { + if query.control.version != crate::types::find_query::AD_FIND_QUERY_VERSION { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Unsupported find query version", + )); + } + if depth > MAX_CONTAINMENT_DEPTH { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Find containment exceeds maximum depth", + )); + } + let address = std::ptr::from_ref(query).addr(); + if !ancestors.insert(address) { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Find containment contains a pointer cycle", + )); + } + let result = unsafe { decode_locator_fields(query, depth, ancestors) }; + ancestors.remove(&address); + result +} + +unsafe fn decode_locator_fields( + query: &AdFindQuery, + depth: usize, + ancestors: &mut HashSet<usize>, +) -> Result<LocatorQuery, AdapterError> { + let filter = &query.filter; + let states = unsafe { decode_states(filter.states.items, filter.states.count)? }; + let has = unsafe { decode_containment(filter.has, depth, ancestors)? }; + let has_not = unsafe { decode_containment(filter.has_not, depth, ancestors)? }; + Ok(LocatorQuery { + identity: IdentityPredicate { + role: optional_adapter_string(filter.identity.role, "filter.identity.role")?, + name: optional_adapter_string(filter.identity.name, "filter.identity.name")?, + description: optional_adapter_string( + filter.identity.description, + "filter.identity.description", + )?, + native_id: optional_adapter_string( + filter.identity.native_id, + "filter.identity.native_id", + )?, + value: optional_adapter_string(filter.identity.value, "filter.identity.value")?, + }, + has_text: optional_adapter_string(filter.has_text, "filter.has_text")?, + exact: filter.exact, + states, + containment: ContainmentPredicate { has, has_not }, + }) +} + +unsafe fn decode_containment( + nested: *const AdFindQuery, + depth: usize, + ancestors: &mut HashSet<usize>, +) -> Result<Option<Box<LocatorQuery>>, AdapterError> { + if nested.is_null() { + return Ok(None); + } + unsafe { decode_locator(&*nested, depth + 1, ancestors) } + .map(Box::new) + .map(Some) +} + +unsafe fn decode_states( + items: *const crate::types::AdFindStatePredicate, + count: usize, +) -> Result<Vec<StatePredicate>, AdapterError> { + if count > MAX_FIND_STATES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Find state predicate count exceeds maximum", + )); + } + if count == 0 { + return Ok(Vec::new()); + } + if items.is_null() { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Find states pointer is null while count is nonzero", + )); + } + unsafe { std::slice::from_raw_parts(items, count) } + .iter() + .enumerate() + .map(|(index, state)| { + let expected = match state.expected { + -1 => None, + 0 => Some(false), + 1 => Some(true), + _ => { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("filter.states[{index}].expected must be -1, 0, or 1"), + )); + } + }; + Ok(StatePredicate { + token: crate::convert::string::required_adapter_string( + state.token, + &format!("filter.states[{index}].token"), + )?, + expected, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::AdFindStatePredicate; + use std::ffi::CString; + + fn query() -> AdFindQuery { + let mut query = unsafe { std::mem::zeroed::<AdFindQuery>() }; + query.control.version = crate::types::find_query::AD_FIND_QUERY_VERSION; + query + } + + #[test] + fn zeroed_selection_is_strict_and_timeout_is_bounded_default() { + let decoded = unsafe { decode_query(&query()) }.unwrap(); + assert_eq!(decoded.1.selection, LocatorSelection::Strict); + assert_eq!(decoded.1.deadline.timeout_ms(), DEFAULT_FIND_TIMEOUT_MS); + } + + #[test] + fn explicit_nth_selection_is_preserved() { + let mut query = query(); + query.control.selection.kind = AdFindSelectionKind::Nth as i32; + query.control.selection.nth = 7; + let decoded = unsafe { decode_query(&query) }.unwrap(); + assert_eq!(decoded.1.selection, LocatorSelection::Nth(7)); + } + + #[test] + fn invalid_version_and_state_encoding_fail_closed() { + let mut invalid_version = query(); + invalid_version.control.version = u32::MAX; + assert!(unsafe { decode_query(&invalid_version) }.is_err()); + + let token = CString::new("focused").unwrap(); + let state = AdFindStatePredicate { + token: token.as_ptr(), + expected: 2, + }; + let mut invalid_state = query(); + invalid_state.filter.states.items = &state; + invalid_state.filter.states.count = 1; + assert!(unsafe { decode_query(&invalid_state) }.is_err()); + } + + #[test] + fn containment_pointer_cycles_fail_closed() { + let mut query = query(); + query.filter.has = std::ptr::from_ref(&query); + assert!(unsafe { decode_query(&query) }.is_err()); + } +} + +#[cfg(test)] +#[path = "find_abi_tests.rs"] +mod abi_tests; diff --git a/crates/ffi/src/observation/find_abi_tests.rs b/crates/ffi/src/observation/find_abi_tests.rs new file mode 100644 index 0000000..b76fccb --- /dev/null +++ b/crates/ffi/src/observation/find_abi_tests.rs @@ -0,0 +1,134 @@ +use super::*; +use crate::adapter::AdAdapter; +use crate::types::{ + AdExactWindowInfo, AdFindQuery, AdFindSelectionKind, AdNativeHandle, AdRect, AdWindowInfo, +}; +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; +use agent_desktop_core::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, ObservationRequest, + ObservationRoot, ObservationSource, ObservedSubtree, ObservedTree, +}; + +const TEST_TIMEOUT_MS: u64 = 1_000; + +struct CardinalityAdapter { + duplicate: bool, + complete: bool, +} + +impl ActionOps for CardinalityAdapter {} +impl InputOps for CardinalityAdapter {} +impl SystemOps for CardinalityAdapter {} + +impl ObservationOps for CardinalityAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result<ObservedTree, agent_desktop_core::AdapterError> { + let ObservationRoot::Window(window) = root else { + return Err(agent_desktop_core::AdapterError::internal( + "expected window root", + )); + }; + let mut roots = vec![subtree(self.complete)]; + if self.duplicate { + roots.push(subtree(self.complete)); + } + ObservedTree::from_roots( + roots, + ObservationSource::Window(window.clone()), + Default::default(), + self.complete, + ) + } +} + +fn subtree(complete: bool) -> ObservedSubtree { + ObservedSubtree::new( + LocatorEvidence { + role: LocatorField::Known("button".into()), + name: LocatorField::Known("Save".into()), + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::absent(), + states: LocatorField::Known(Vec::new()), + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Known(vec!["Click".into()]), + }, + }, + Vec::new(), + complete, + None, + ) +} + +fn ffi_window() -> AdExactWindowInfo { + AdExactWindowInfo { + version: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_VERSION, + size: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_SIZE as u32, + window: AdWindowInfo { + id: c"w-1".as_ptr(), + title: c"Fixture".as_ptr(), + app_name: c"Fixture".as_ptr(), + pid: 42, + bounds: AdRect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }, + has_bounds: false, + is_focused: false, + }, + process_instance: c"42:100".as_ptr(), + } +} + +fn query() -> AdFindQuery { + let mut query = unsafe { std::mem::zeroed::<AdFindQuery>() }; + query.control.version = crate::types::find_query::AD_FIND_QUERY_VERSION; + query.control.selection.kind = AdFindSelectionKind::Strict as i32; + query.control.timeout_ms = TEST_TIMEOUT_MS; + query.filter.identity.name = c"Save".as_ptr(); + query +} + +fn call(adapter: CardinalityAdapter) -> AdResult { + let handle = crate::adapter::register_adapter(AdAdapter { + inner: Box::new(adapter), + session_id: None, + _session_lease: None, + }) + .unwrap(); + let mut out = AdNativeHandle { + ptr: std::ptr::null(), + }; + let result = unsafe { ad_find_exact(handle, &ffi_window(), &query(), &mut out) }; + unsafe { crate::adapter::ad_adapter_destroy(handle) }; + assert!(out.ptr.is_null()); + result +} + +#[test] +fn strict_duplicate_is_ambiguous_at_the_c_boundary() { + assert_eq!( + call(CardinalityAdapter { + duplicate: true, + complete: true, + }), + AdResult::ErrAmbiguousTarget + ); +} + +#[test] +fn strict_incomplete_observation_times_out_at_the_c_boundary() { + assert_eq!( + call(CardinalityAdapter { + duplicate: false, + complete: false, + }), + AdResult::ErrTimeout + ); +} diff --git a/crates/ffi/src/observation/get.rs b/crates/ffi/src/observation/get.rs index b75c99d..f57c104 100644 --- a/crates/ffi/src/observation/get.rs +++ b/crates/ffi/src/observation/get.rs @@ -3,9 +3,23 @@ use crate::convert::string::{c_to_string, string_to_c_lossy}; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; use crate::types::AdNativeHandle; -use agent_desktop_core::adapter::NativeHandle; use std::os::raw::c_char; +enum Property { + Value, + Bounds, +} + +impl Property { + fn parse(name: &str) -> Option<Self> { + match name { + "value" => Some(Self::Value), + "bounds" => Some(Self::Bounds), + _ => None, + } + } +} + /// Reads a single property off a previously-resolved element handle. /// /// Supported properties: @@ -31,34 +45,42 @@ pub unsafe extern "C" fn ad_get( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = std::ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(handle, c"handle is null"); - let adapter = &*adapter; - let raw = (*handle).ptr; - if raw.is_null() { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "handle.ptr is null — the handle has already been freed or was never resolved", - )); - return AdResult::ErrInvalidArgs; - } - let native = NativeHandle::from_ptr(raw); let prop = match c_to_string(property) { Some(s) => s, None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, "property is null or invalid UTF-8", )); return AdResult::ErrInvalidArgs; } }; + let property = match Property::parse(&prop) { + Some(property) => property, + None => { + set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "unknown property — expected one of: value, bounds", + )); + return AdResult::ErrInvalidArgs; + } + }; + let adapter_id = adapter.addr(); + let adapter = crate::adapter::acquire_adapter!(adapter); + let (native, _) = + match crate::actions::native_handle::acquire_ffi_handle(adapter_id, &*handle) { + Ok(native) => native, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let deadline = crate::operation::operation_deadline!(); - match prop.as_str() { - "value" => match adapter.inner.get_live_value(&native) { + match property { + Property::Value => match adapter.inner.get_live_value(native.as_ref(), deadline) { Ok(Some(v)) => { *out = string_to_c_lossy(&v); AdResult::Ok @@ -69,7 +91,7 @@ pub unsafe extern "C" fn ad_get( crate::error::last_error_code() } }, - "bounds" => match adapter.inner.get_element_bounds(&native) { + Property::Bounds => match adapter.inner.get_element_bounds(native.as_ref(), deadline) { Ok(Some(r)) => { let json = format!( "{{\"x\":{},\"y\":{},\"width\":{},\"height\":{}}}", @@ -84,13 +106,6 @@ pub unsafe extern "C" fn ad_get( crate::error::last_error_code() } }, - _ => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "unknown property — expected one of: value, bounds", - )); - AdResult::ErrInvalidArgs - } } }) } diff --git a/crates/ffi/src/observation/is.rs b/crates/ffi/src/observation/is.rs index 181bddf..f4e78c2 100644 --- a/crates/ffi/src/observation/is.rs +++ b/crates/ffi/src/observation/is.rs @@ -1,41 +1,48 @@ use crate::AdAdapter; -use crate::convert::string::{c_to_string, decode_optional_filter}; +use crate::convert::string::required_adapter_string; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::types::{AdFindQuery, AdWindowInfo}; -use agent_desktop_core::adapter::{SnapshotSurface, TreeOptions}; -use agent_desktop_core::node::AccessibilityNode; +use crate::types::{AdExactWindowInfo, AdFindQuery, AdWindowInfo}; +use agent_desktop_core::{ObservationRoot, resolve_query}; use std::os::raw::c_char; -/// Checks whether a named boolean state is set on the first element -/// matching `query` inside `win`'s accessibility tree. Intended for -/// the common agent idiom `find → is("focused") → if yes, act`. -/// -/// Supported property names reflect the strings the macOS tree -/// builder actually emits in `AccessibilityNode.states`: -/// -/// - `"focused"` — true when the node carries the `focused` state. -/// - `"disabled"` — true when the adapter surfaced `disabled`. -/// - `"enabled"` — derived: true iff `disabled` is NOT present. There -/// is no `enabled` string in the adapter output; asking for it -/// returns the logical negation so agents don't have to invert -/// themselves. -/// -/// `"selected"`, `"checked"`, and `"expanded"` are not currently -/// emitted by any platform adapter; asking for them returns -/// `AD_RESULT_ERR_INVALID_ARGS` with a diagnostic last-error rather -/// than silently answering `false`. The set will widen as adapters -/// grow support; future additions stay backwards-compatible -/// (unknown → InvalidArgs, known → deterministic answer). -/// -/// On entry `*out` is always cleared to `false` so a caller inspecting -/// the slot after an error sees a predictable sentinel, not whatever -/// was there before. If the query matches nothing, returns -/// `AD_RESULT_ERR_ELEMENT_NOT_FOUND` with `*out` still `false`. +enum SupportedProperty { + Focused, + Disabled, + Enabled, +} + +impl SupportedProperty { + fn parse(name: &str) -> Option<Self> { + match name { + "focused" => Some(Self::Focused), + "disabled" => Some(Self::Disabled), + "enabled" => Some(Self::Enabled), + _ => None, + } + } + + fn evaluate(&self, states: &[String]) -> bool { + let contains = |wanted: &str| { + states + .iter() + .any(|state| state.eq_ignore_ascii_case(wanted)) + }; + match self { + Self::Focused => contains("focused"), + Self::Disabled => contains("disabled"), + Self::Enabled => !contains("disabled"), + } + } +} + +/// Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process +/// generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. +/// Use `ad_is_exact`. /// /// # Safety -/// All pointers must be valid. `property` must be a non-null UTF-8 -/// C string. `out` must be a valid writable `*mut bool`. +/// All pointers must be valid. `property` must be a non-null UTF-8 C string. +/// `out` must be a valid writable `*mut bool`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_is( adapter: *const AdAdapter, @@ -47,164 +54,142 @@ pub unsafe extern "C" fn ad_is( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = false; - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(win, c"win is null"); crate::pointer_guard::guard_non_null!(query, c"query is null"); - let adapter = &*adapter; - let core_win = match crate::windows::ad_window_to_core(&*win) { - Ok(w) => w, - Err(e) => { - set_last_error(&e); + let core_window = match crate::windows::ad_window_to_core(&*win) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); return crate::error::last_error_code(); } }; - let q = &*query; - let role_filter = decode_optional_filter!(q.role, "query.role"); - let name_filter = decode_optional_filter!(q.name_substring, "query.name_substring"); - let value_filter = decode_optional_filter!(q.value_substring, "query.value_substring"); - let prop = match c_to_string(property) { - Some(s) => s, - None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "property is null or invalid UTF-8", - )); - return AdResult::ErrInvalidArgs; - } - }; - let property = match SupportedProperty::from_name(&prop) { - Some(p) => p, - None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "unknown property — expected one of: focused, disabled, enabled", - )); - return AdResult::ErrInvalidArgs; - } - }; - - let tree = match adapter.inner.get_tree( - &core_win, - &TreeOptions { - max_depth: 50, - include_bounds: false, - interactive_only: false, - compact: false, - surface: SnapshotSurface::Window, - skeleton: false, - }, - ) { - Ok(t) => t, - Err(e) => { - set_last_error(&e); - return crate::error::last_error_code(); - } - }; - - let matched = match crate::observation::walk::find_first_match( - &tree, - role_filter.as_deref(), - name_filter.as_deref(), - value_filter.as_deref(), - ) { - Some(n) => n, - None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::ElementNotFound, - "no element matched the find query", - )); - return AdResult::ErrElementNotFound; - } - }; - - *out = property.evaluate(matched); - AdResult::Ok + is_in_window(adapter, &core_window, &*query, property, out) }) } -/// Compile-time set of property names `ad_is` answers. Each variant -/// encodes how the answer is derived from `AccessibilityNode.states`, -/// which keeps the documented contract and the implementation in -/// lockstep — an addition here is the only way to add a supported name. -enum SupportedProperty { - Focused, - Disabled, - EnabledDerivedFromDisabled, +/// Checks a boolean state within a generation-pinned exact window. +/// +/// # Safety +/// All pointers must be valid and `out` must be writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_is_exact( + adapter: *const AdAdapter, + win: *const AdExactWindowInfo, + query: *const AdFindQuery, + property: *const c_char, + out: *mut bool, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = false; + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(win, c"win is null"); + crate::pointer_guard::guard_non_null!(query, c"query is null"); + let window = match crate::windows::ad_exact_window_to_core(&*win) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + is_in_window(adapter, &window, &*query, property, out) + }) } -impl SupportedProperty { - fn from_name(name: &str) -> Option<Self> { - match name { - "focused" => Some(SupportedProperty::Focused), - "disabled" => Some(SupportedProperty::Disabled), - "enabled" => Some(SupportedProperty::EnabledDerivedFromDisabled), - _ => None, +unsafe fn is_in_window( + adapter: *const AdAdapter, + window: &agent_desktop_core::WindowInfo, + query: &AdFindQuery, + property: *const c_char, + out: *mut bool, +) -> AdResult { + let (locator, request) = match unsafe { super::find::decode_query(query) } { + Ok(decoded) => decoded, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; } - } - - fn evaluate(&self, node: &AccessibilityNode) -> bool { - match self { - SupportedProperty::Focused => has_state(node, "focused"), - SupportedProperty::Disabled => has_state(node, "disabled"), - SupportedProperty::EnabledDerivedFromDisabled => !has_state(node, "disabled"), + }; + let property_name = match required_adapter_string(property, "property") { + Ok(property) => property, + Err(error) => { + set_last_error(&error); + return AdResult::ErrInvalidArgs; } - } -} - -fn has_state(node: &AccessibilityNode, state: &str) -> bool { - node.states.iter().any(|s| s.eq_ignore_ascii_case(state)) + }; + let property = match SupportedProperty::parse(&property_name) { + Some(property) => property, + None => { + let error = agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "unknown property — expected one of: focused, disabled, enabled", + ); + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }; + let adapter = crate::adapter::acquire_adapter!(adapter); + let resolution = match resolve_query( + adapter.inner.as_ref(), + &locator, + ObservationRoot::Window(window), + &request, + ) { + Ok(resolution) => resolution, + Err(error) => { + set_last_error(&crate::commands::app_error_to_adapter(error)); + return crate::error::last_error_code(); + } + }; + let selected = if request.selection == agent_desktop_core::LocatorSelection::Strict { + match agent_desktop_core::require_unique(resolution) { + Ok(selected) => selected, + Err(error) => { + set_last_error(&crate::commands::app_error_to_adapter(error)); + return crate::error::last_error_code(); + } + } + } else { + if !resolution.meta.selection_complete { + set_last_error(&agent_desktop_core::AdapterError::timeout( + "Locator traversal could not prove the selected result", + )); + return AdResult::ErrTimeout; + } + let Some(selected) = resolution.matches.into_iter().next() else { + set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::ElementNotFound, + "Locator query matched no elements", + )); + return crate::error::last_error_code(); + }; + selected + }; + let entry = selected.into_entry(); + unsafe { *out = property.evaluate(&entry.capabilities.states) }; + AdResult::Ok } +#[cfg(test)] +#[path = "is_abi_tests.rs"] +mod abi_tests; + #[cfg(test)] mod tests { - use super::*; + use super::SupportedProperty; - fn node_with_states(states: &[&str]) -> AccessibilityNode { - AccessibilityNode { - ref_id: None, - role: "button".into(), - name: None, - value: None, - description: None, - hint: None, - states: states.iter().map(|s| s.to_string()).collect(), - available_actions: vec![], - bounds: None, - children: vec![], - children_count: None, - } + #[test] + fn properties_evaluate_normalized_states() { + let states = vec![String::from("focused")]; + assert!(SupportedProperty::Focused.evaluate(&states)); + assert!(!SupportedProperty::Disabled.evaluate(&states)); + assert!(SupportedProperty::Enabled.evaluate(&states)); + assert!(!SupportedProperty::Enabled.evaluate(&[String::from("disabled")])); } #[test] - fn focused_mirrors_state_presence() { - let prop = SupportedProperty::from_name("focused").unwrap(); - assert!(prop.evaluate(&node_with_states(&["focused"]))); - assert!(!prop.evaluate(&node_with_states(&[]))); - } - - #[test] - fn disabled_mirrors_state_presence() { - let prop = SupportedProperty::from_name("disabled").unwrap(); - assert!(prop.evaluate(&node_with_states(&["disabled"]))); - assert!(!prop.evaluate(&node_with_states(&["focused"]))); - } - - #[test] - fn enabled_is_derived_negation_of_disabled() { - let prop = SupportedProperty::from_name("enabled").unwrap(); - assert!(!prop.evaluate(&node_with_states(&["disabled"]))); - assert!(prop.evaluate(&node_with_states(&[]))); - assert!(prop.evaluate(&node_with_states(&["focused"]))); - } - - #[test] - fn unsupported_names_do_not_resolve() { - assert!(SupportedProperty::from_name("selected").is_none()); - assert!(SupportedProperty::from_name("checked").is_none()); - assert!(SupportedProperty::from_name("expanded").is_none()); - assert!(SupportedProperty::from_name("bogus").is_none()); + fn rejects_unknown_properties() { + assert!(SupportedProperty::parse("selected").is_none()); } } diff --git a/crates/ffi/src/observation/is_abi_tests.rs b/crates/ffi/src/observation/is_abi_tests.rs new file mode 100644 index 0000000..dfe19cf --- /dev/null +++ b/crates/ffi/src/observation/is_abi_tests.rs @@ -0,0 +1,129 @@ +use super::*; +use crate::adapter::AdAdapter; +use crate::types::{AdExactWindowInfo, AdFindQuery, AdFindSelectionKind, AdRect, AdWindowInfo}; +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; +use agent_desktop_core::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, ObservationRequest, + ObservationRoot, ObservationSource, ObservedSubtree, ObservedTree, +}; + +struct DuplicateAdapter; + +impl ActionOps for DuplicateAdapter {} +impl InputOps for DuplicateAdapter {} +impl SystemOps for DuplicateAdapter {} + +impl ObservationOps for DuplicateAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + _request: &ObservationRequest, + ) -> Result<ObservedTree, agent_desktop_core::AdapterError> { + let ObservationRoot::Window(window) = root else { + return Err(agent_desktop_core::AdapterError::internal( + "expected window root", + )); + }; + ObservedTree::from_roots( + vec![button(Vec::new()), button(vec!["disabled".into()])], + ObservationSource::Window(window.clone()), + Default::default(), + true, + ) + } +} + +fn button(states: Vec<String>) -> ObservedSubtree { + ObservedSubtree::new( + LocatorEvidence { + role: LocatorField::Known("button".into()), + name: LocatorField::Known("Save".into()), + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::absent(), + states: LocatorField::Known(states), + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Known(vec!["Click".into()]), + }, + }, + Vec::new(), + true, + None, + ) +} + +fn window() -> AdExactWindowInfo { + AdExactWindowInfo { + version: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_VERSION, + size: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_SIZE as u32, + window: AdWindowInfo { + id: c"w-1".as_ptr(), + title: c"Fixture".as_ptr(), + app_name: c"Fixture".as_ptr(), + pid: 42, + bounds: AdRect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }, + has_bounds: false, + is_focused: false, + }, + process_instance: c"42:100".as_ptr(), + } +} + +fn query() -> AdFindQuery { + let mut query = unsafe { std::mem::zeroed::<AdFindQuery>() }; + query.control.version = crate::types::find_query::AD_FIND_QUERY_VERSION; + query.control.selection.kind = AdFindSelectionKind::Strict as i32; + query.control.timeout_ms = 1_000; + query.filter.identity.name = c"Save".as_ptr(); + query +} + +#[test] +fn strict_duplicate_is_ambiguous_at_the_c_boundary() { + let adapter = crate::adapter::register_adapter(AdAdapter { + inner: Box::new(DuplicateAdapter), + session_id: None, + _session_lease: None, + }) + .unwrap(); + let mut out = false; + + let result = + unsafe { ad_is_exact(adapter, &window(), &query(), c"enabled".as_ptr(), &mut out) }; + + unsafe { crate::adapter::ad_adapter_destroy(adapter) }; + assert_eq!(result, AdResult::ErrAmbiguousTarget); + assert!(!out); +} + +#[test] +fn last_and_nth_selection_are_applied_at_the_c_boundary() { + let adapter = crate::adapter::register_adapter(AdAdapter { + inner: Box::new(DuplicateAdapter), + session_id: None, + _session_lease: None, + }) + .unwrap(); + for (kind, nth) in [ + (AdFindSelectionKind::Last, 0), + (AdFindSelectionKind::Nth, 1), + ] { + let mut query = query(); + query.control.selection.kind = kind as i32; + query.control.selection.nth = nth; + let mut out = false; + + let result = + unsafe { ad_is_exact(adapter, &window(), &query, c"disabled".as_ptr(), &mut out) }; + + assert_eq!(result, AdResult::Ok); + assert!(out); + } + unsafe { crate::adapter::ad_adapter_destroy(adapter) }; +} diff --git a/crates/ffi/src/observation/mod.rs b/crates/ffi/src/observation/mod.rs index 43a9942..a37c0c0 100644 --- a/crates/ffi/src/observation/mod.rs +++ b/crates/ffi/src/observation/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod find; pub(crate) mod get; pub(crate) mod is; -pub(crate) mod walk; diff --git a/crates/ffi/src/observation/walk.rs b/crates/ffi/src/observation/walk.rs deleted file mode 100644 index 10d92df..0000000 --- a/crates/ffi/src/observation/walk.rs +++ /dev/null @@ -1,125 +0,0 @@ -use agent_desktop_core::node::AccessibilityNode; - -/// Finds the first node in DFS order that matches every provided filter. -/// Filters are ANDed; a `None` filter means "don't care". Substring -/// matching is case-insensitive to tolerate platform-specific casing -/// (macOS AX strings vary between "AXButton" and "button"). -pub(crate) fn find_first_match<'a>( - node: &'a AccessibilityNode, - role: Option<&str>, - name: Option<&str>, - value: Option<&str>, -) -> Option<&'a AccessibilityNode> { - if matches_all(node, role, name, value) { - return Some(node); - } - for child in &node.children { - if let Some(hit) = find_first_match(child, role, name, value) { - return Some(hit); - } - } - None -} - -fn matches_all( - node: &AccessibilityNode, - role: Option<&str>, - name: Option<&str>, - value: Option<&str>, -) -> bool { - if let Some(r) = role { - if !contains_ignore_case(&node.role, r) { - return false; - } - } - if let Some(n) = name { - match node.name.as_deref() { - Some(actual) if contains_ignore_case(actual, n) => {} - _ => return false, - } - } - if let Some(v) = value { - match node.value.as_deref() { - Some(actual) if contains_ignore_case(actual, v) => {} - _ => return false, - } - } - true -} - -fn contains_ignore_case(haystack: &str, needle: &str) -> bool { - haystack.to_lowercase().contains(&needle.to_lowercase()) -} - -#[cfg(test)] -mod tests { - use super::*; - use agent_desktop_core::node::Rect; - - fn node(role: &str, name: Option<&str>, value: Option<&str>) -> AccessibilityNode { - AccessibilityNode { - ref_id: None, - role: role.into(), - name: name.map(str::to_string), - value: value.map(str::to_string), - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, - children: vec![], - children_count: None, - } - } - - #[test] - fn finds_first_matching_role() { - let mut root = node("window", None, None); - root.children.push(node("button", Some("OK"), None)); - root.children.push(node("button", Some("Cancel"), None)); - - let hit = find_first_match(&root, Some("button"), None, None).unwrap(); - assert_eq!(hit.name.as_deref(), Some("OK")); - } - - #[test] - fn value_substring_disambiguates_duplicate_labels() { - let mut root = node("window", None, None); - root.children - .push(node("button", Some("Save"), Some("pressed"))); - root.children - .push(node("button", Some("Save"), Some("default"))); - - let hit = find_first_match(&root, Some("button"), Some("Save"), Some("default")).unwrap(); - assert_eq!(hit.value.as_deref(), Some("default")); - } - - #[test] - fn matched_node_bounds_hash_stable_across_calls() { - let mut matched = node("button", Some("OK"), None); - matched.bounds = Some(Rect { - x: 10.0, - y: 20.0, - width: 80.0, - height: 24.0, - }); - let first = matched.bounds.as_ref().map(|r| r.bounds_hash()).unwrap(); - let second = matched.bounds.as_ref().map(|r| r.bounds_hash()).unwrap(); - assert_eq!(first, second); - } - - #[test] - fn missing_filter_field_treats_as_dont_care() { - let root = node("button", Some("OK"), Some("enabled")); - assert!(find_first_match(&root, None, None, None).is_some()); - assert!(find_first_match(&root, Some("button"), None, None).is_some()); - assert!(find_first_match(&root, None, Some("OK"), None).is_some()); - } - - #[test] - fn no_match_returns_none() { - let root = node("window", Some("App"), None); - assert!(find_first_match(&root, Some("slider"), None, None).is_none()); - assert!(find_first_match(&root, None, Some("nonexistent"), None).is_none()); - } -} diff --git a/crates/ffi/src/opaque_id.rs b/crates/ffi/src/opaque_id.rs new file mode 100644 index 0000000..91e775a --- /dev/null +++ b/crates/ffi/src/opaque_id.rs @@ -0,0 +1,31 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use agent_desktop_core::{AdapterError, ErrorCode}; + +pub(crate) fn allocate( + counter: &AtomicUsize, + resource: &'static str, +) -> Result<usize, AdapterError> { + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next| { + next.checked_add(1) + }) + .map_err(|_| { + AdapterError::new( + ErrorCode::Internal, + format!("{resource} identifier space exhausted"), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn allocation_never_wraps_or_reuses_zero() { + let counter = AtomicUsize::new(usize::MAX); + assert!(allocate(&counter, "Test").is_err()); + assert_eq!(counter.load(Ordering::Relaxed), usize::MAX); + } +} diff --git a/crates/ffi/src/operation.rs b/crates/ffi/src/operation.rs new file mode 100644 index 0000000..cc2cf2a --- /dev/null +++ b/crates/ffi/src/operation.rs @@ -0,0 +1,69 @@ +use agent_desktop_core::{AdapterError, Deadline, InteractionLease, PlatformAdapter}; + +pub(crate) const DEFAULT_FFI_TIMEOUT_MS: u64 = 5_000; + +pub(crate) fn deadline() -> Result<Deadline, AdapterError> { + Deadline::after(DEFAULT_FFI_TIMEOUT_MS) +} + +pub(crate) fn lease(adapter: &dyn PlatformAdapter) -> Result<InteractionLease, AdapterError> { + let deadline = deadline()?; + #[cfg(unix)] + if let Some(raw_fd) = inherited_lease_fd()? { + return agent_desktop_core::adopt_inherited_unix_interaction_lease(raw_fd, deadline); + } + adapter.acquire_interaction_lease(deadline) +} + +#[cfg(unix)] +fn inherited_lease_fd() -> Result<Option<std::os::fd::RawFd>, AdapterError> { + let Some(value) = std::env::var_os(agent_desktop_core::INTERACTION_LEASE_FD_ENV) else { + return Ok(None); + }; + let value = value.into_string().map_err(|_| { + AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Inherited interaction lease FD is not valid UTF-8", + ) + })?; + let raw_fd = value.parse::<std::os::fd::RawFd>().map_err(|_| { + AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Inherited interaction lease FD must be a nonnegative decimal integer", + ) + })?; + if raw_fd < 0 { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Inherited interaction lease FD must be nonnegative", + )); + } + Ok(Some(raw_fd)) +} + +macro_rules! operation_deadline { + () => {{ + match $crate::operation::deadline() { + Ok(deadline) => deadline, + Err(error) => { + $crate::error::set_last_error(&error); + return $crate::error::last_error_code(); + } + } + }}; +} + +macro_rules! interaction_lease { + ($adapter:expr) => {{ + match $crate::operation::lease($adapter) { + Ok(lease) => lease, + Err(error) => { + $crate::error::set_last_error(&error); + return $crate::error::last_error_code(); + } + } + }}; +} + +pub(crate) use interaction_lease; +pub(crate) use operation_deadline; diff --git a/crates/ffi/src/panic_injection.rs b/crates/ffi/src/panic_injection.rs new file mode 100644 index 0000000..5b8de85 --- /dev/null +++ b/crates/ffi/src/panic_injection.rs @@ -0,0 +1,6 @@ +use crate::error::AdResult; + +#[unsafe(no_mangle)] +pub extern "C" fn ad_test_panic_boundary() -> AdResult { + crate::ffi_try::trap_panic(|| panic!("synthetic panic at the exported cdylib boundary")) +} diff --git a/crates/ffi/src/resource.rs b/crates/ffi/src/resource.rs new file mode 100644 index 0000000..7c14a1f --- /dev/null +++ b/crates/ffi/src/resource.rs @@ -0,0 +1,106 @@ +use agent_desktop_core::{AdapterError, ErrorCode, ImageBuffer}; +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub(crate) enum AllocationKind { + ActionPostState, + ActionStateStrings, + ActionSteps, + CString, + NotificationActions, + TreeNodes, + TreeStateStrings, +} + +static ALLOCATIONS: OnceLock<Mutex<HashMap<(AllocationKind, usize), usize>>> = OnceLock::new(); + +pub(crate) const MAX_FFI_LIST_ITEMS: usize = 100_000; +pub(crate) const MAX_FFI_IMAGE_BYTES: usize = 512 * 1024 * 1024; + +fn allocations() -> MutexGuard<'static, HashMap<(AllocationKind, usize), usize>> { + match ALLOCATIONS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +pub(crate) fn register_allocation<T>(kind: AllocationKind, ptr: *mut T, len: usize) { + if !ptr.is_null() { + allocations().insert((kind, ptr.addr()), len); + } +} + +pub(crate) fn take_allocation<T>(kind: AllocationKind, ptr: *mut T) -> Option<usize> { + (!ptr.is_null()) + .then(|| allocations().remove(&(kind, ptr.addr()))) + .flatten() +} + +pub(crate) fn validate_list_len(len: usize, label: &str) -> Result<(), AdapterError> { + if len <= MAX_FFI_LIST_ITEMS && u32::try_from(len).is_ok() { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::Internal, + format!("{label} exceeds the FFI output item limit"), + )) +} + +pub(crate) fn validate_output_string(value: &str, label: &str) -> Result<(), AdapterError> { + if value.len() <= crate::convert::string::AD_MAX_STRING_BYTES { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::Internal, + format!("{label} exceeds the FFI output string limit"), + )) +} + +pub(crate) fn validate_image(image: &ImageBuffer) -> Result<(), AdapterError> { + validate_image_parts( + image.data.len(), + image.width, + image.height, + image.scale_factor, + ) +} + +fn validate_image_parts( + byte_len: usize, + width: u32, + height: u32, + scale_factor: f64, +) -> Result<(), AdapterError> { + if byte_len > MAX_FFI_IMAGE_BYTES { + return Err(image_validation_error( + "Screenshot exceeds the FFI image byte limit", + "bytes", + )); + } + if width == 0 || height == 0 || width > 100_000 || height > 100_000 { + return Err(image_validation_error( + "Screenshot dimensions are outside the FFI image limits", + "dimensions", + )); + } + if !scale_factor.is_finite() || scale_factor <= 0.0 || scale_factor > 16.0 { + return Err(image_validation_error( + "Screenshot scale factor is outside the FFI image limits", + "scale_factor", + )); + } + Ok(()) +} + +fn image_validation_error(message: &str, reason: &str) -> AdapterError { + AdapterError::new(ErrorCode::Internal, message) + .with_details(serde_json::json!({ "reason": reason })) +} + +#[cfg(test)] +#[path = "resource_tests.rs"] +mod tests; diff --git a/crates/ffi/src/resource_tests.rs b/crates/ffi/src/resource_tests.rs new file mode 100644 index 0000000..b586228 --- /dev/null +++ b/crates/ffi/src/resource_tests.rs @@ -0,0 +1,28 @@ +use super::*; +use agent_desktop_core::ImageFormat; + +fn image() -> ImageBuffer { + ImageBuffer { + data: vec![0; 4], + format: ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + } +} + +#[test] +fn image_validation_distinguishes_dimensions_scale_and_bytes() { + let mut invalid_dimensions = image(); + invalid_dimensions.width = 0; + let dimensions = validate_image(&invalid_dimensions).unwrap_err(); + assert!(dimensions.message.contains("dimensions")); + + let mut invalid_scale = image(); + invalid_scale.scale_factor = f64::NAN; + let scale = validate_image(&invalid_scale).unwrap_err(); + assert!(scale.message.contains("scale factor")); + + let bytes = validate_image_parts(MAX_FFI_IMAGE_BYTES + 1, 1, 1, 1.0).unwrap_err(); + assert!(bytes.message.contains("byte limit")); +} diff --git a/crates/ffi/src/screenshot/accessors.rs b/crates/ffi/src/screenshot/accessors.rs index 0a1eb28..cf2928d 100644 --- a/crates/ffi/src/screenshot/accessors.rs +++ b/crates/ffi/src/screenshot/accessors.rs @@ -68,3 +68,16 @@ pub unsafe extern "C" fn ad_image_buffer_format(buf: *const AdImageBuffer) -> Ad let buf_ref: &AdImageBuffer = unsafe { &*buf }; buf_ref.format } + +/// Point-to-pixel scale factor for the captured display or window. +/// +/// # Safety +/// `buf` must be null or returned by `ad_screenshot`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_image_buffer_scale_factor(buf: *const AdImageBuffer) -> f64 { + if buf.is_null() { + return 1.0; + } + let buf_ref: &AdImageBuffer = unsafe { &*buf }; + buf_ref.scale_factor +} diff --git a/crates/ffi/src/screenshot/capture.rs b/crates/ffi/src/screenshot/capture.rs index cabf179..547f9bf 100644 --- a/crates/ffi/src/screenshot/capture.rs +++ b/crates/ffi/src/screenshot/capture.rs @@ -1,8 +1,10 @@ use crate::AdAdapter; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::types::{AdImageBuffer, AdImageFormat, AdScreenshotKind, AdScreenshotTarget}; -use agent_desktop_core::adapter::{ImageFormat, ScreenshotTarget as CoreScreenshotTarget}; +use crate::types::{ + AdExactWindowInfo, AdImageBuffer, AdImageFormat, AdScreenshotKind, AdScreenshotTarget, +}; +use agent_desktop_core::{ImageFormat, ScreenshotTarget as CoreScreenshotTarget}; use std::ptr; /// Allocates and returns an opaque `AdImageBuffer`. The handle owns its @@ -23,47 +25,101 @@ pub unsafe extern "C" fn ad_screenshot( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(target, c"target is null"); - let adapter = &*adapter; let t = &*target; let kind = match AdScreenshotKind::from_c(t.kind) { Some(k) => k, None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + set_last_error(&agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, "invalid screenshot kind discriminant", )); return AdResult::ErrInvalidArgs; } }; let core_target = match kind { - AdScreenshotKind::Screen => CoreScreenshotTarget::Screen(t.screen_index as usize), - AdScreenshotKind::Window => CoreScreenshotTarget::Window(t.pid), + AdScreenshotKind::Screen => match usize::try_from(t.screen_index) { + Ok(index) if index <= 10_000 => CoreScreenshotTarget::Screen(index), + _ => { + let error = agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "screen_index exceeds the supported display limit", + ); + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } + }, + AdScreenshotKind::Window => { + let error = agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "legacy window screenshot targeting lacks process-generation identity; use ad_screenshot_window_exact", + ); + set_last_error(&error); + return AdResult::ErrInvalidArgs; + } AdScreenshotKind::FullScreen => CoreScreenshotTarget::FullScreen, }; - - match adapter.inner.screenshot(core_target) { - Ok(img) => { - let buffer = Box::new(AdImageBuffer { - data: img.data.into_boxed_slice(), - width: img.width, - height: img.height, - format: match img.format { - ImageFormat::Png => AdImageFormat::Png, - ImageFormat::Jpg => AdImageFormat::Jpg, - }, - }); - *out = Box::into_raw(buffer); - AdResult::Ok - } - Err(e) => { - set_last_error(&e); - crate::error::last_error_code() - } - } + capture(adapter, core_target, out) }) } + +/// Captures one generation-pinned exact window. +/// +/// # Safety +/// `adapter`, `window`, and `out` must be valid pointers. The returned image +/// must be freed with `ad_image_buffer_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_screenshot_window_exact( + adapter: *const AdAdapter, + window: *const AdExactWindowInfo, + out: *mut *mut AdImageBuffer, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = ptr::null_mut(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(window, c"window is null"); + let window = match crate::windows::ad_exact_window_to_core(&*window) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + capture(adapter, CoreScreenshotTarget::ExactWindow(window), out) + }) +} + +unsafe fn capture( + adapter: *const AdAdapter, + target: CoreScreenshotTarget, + out: *mut *mut AdImageBuffer, +) -> AdResult { + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.screenshot(target, deadline) { + Ok(image) => { + if let Err(error) = crate::resource::validate_image(&image) { + set_last_error(&error); + return crate::error::last_error_code(); + } + let buffer = Box::new(AdImageBuffer { + data: image.data.into_boxed_slice(), + width: image.width, + height: image.height, + format: match image.format { + ImageFormat::Png => AdImageFormat::Png, + ImageFormat::Jpg => AdImageFormat::Jpg, + }, + scale_factor: image.scale_factor, + }); + unsafe { *out = Box::into_raw(buffer) }; + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } +} diff --git a/crates/ffi/src/session_lease.rs b/crates/ffi/src/session_lease.rs new file mode 100644 index 0000000..6ec3e76 --- /dev/null +++ b/crates/ffi/src/session_lease.rs @@ -0,0 +1,13 @@ +use agent_desktop_core::{AdapterError, AppError, session::SessionLivenessLease}; + +pub(crate) fn acquire( + session_id: Option<&str>, +) -> Result<Option<SessionLivenessLease>, AdapterError> { + let Some(session_id) = session_id else { + return Ok(None); + }; + agent_desktop_core::session::acquire_liveness_lease(session_id).map_err(|error| match error { + AppError::Adapter(error) => error, + other => AdapterError::internal(other.to_string()), + }) +} diff --git a/crates/ffi/src/surfaces/list.rs b/crates/ffi/src/surfaces/list.rs index 4e22692..6bf7a8a 100644 --- a/crates/ffi/src/surfaces/list.rs +++ b/crates/ffi/src/surfaces/list.rs @@ -1,8 +1,11 @@ use crate::AdAdapter; -use crate::convert::surface::{free_surface_info_fields, surface_info_to_c}; +use crate::convert::surface::{ + exact_surface_info_to_c, free_exact_surface_info_fields, free_surface_info_fields, + surface_info_to_c, validate_surface_info, +}; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::{trap_panic, trap_panic_void}; -use crate::types::{AdSurfaceInfo, AdSurfaceList}; +use crate::types::{AdExactSurfaceInfo, AdExactSurfaceList, AdSurfaceInfo, AdSurfaceList}; use std::ptr; /// # Safety @@ -12,19 +15,34 @@ use std::ptr; #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_list_surfaces( adapter: *const AdAdapter, - pid: i32, + pid: u32, out: *mut *mut AdSurfaceList, ) -> AdResult { trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; - match adapter.inner.list_surfaces(pid) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + let process = match process_identity_for_pid(adapter.inner.as_ref(), pid, deadline) { + Ok(process) => process, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + match adapter.inner.list_surfaces(process, deadline) { Ok(surfaces) => { + if let Err(error) = + crate::resource::validate_list_len(surfaces.len(), "Surface list") + { + set_last_error(&error); + return crate::error::last_error_code(); + } + if let Err(error) = surfaces.iter().try_for_each(validate_surface_info) { + set_last_error(&error); + return crate::error::last_error_code(); + } let items: Vec<AdSurfaceInfo> = surfaces.iter().map(surface_info_to_c).collect(); let list = Box::new(AdSurfaceList { items: items.into_boxed_slice(), @@ -86,3 +104,213 @@ pub unsafe extern "C" fn ad_surface_list_free(list: *mut AdSurfaceList) { } }) } + +/// Lists surfaces without dropping their core surface IDs. +/// +/// # Safety +/// `adapter` and `out` must be valid. The returned list must be freed with +/// `ad_exact_surface_list_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_list_surfaces_exact( + adapter: *const AdAdapter, + pid: u32, + out: *mut *mut AdExactSurfaceList, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = ptr::null_mut(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + let process = match process_identity_for_pid(adapter.inner.as_ref(), pid, deadline) { + Ok(process) => process, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + match adapter.inner.list_surfaces(process, deadline) { + Ok(surfaces) => { + if let Err(error) = + crate::resource::validate_list_len(surfaces.len(), "Exact surface list") + { + set_last_error(&error); + return crate::error::last_error_code(); + } + if let Err(error) = surfaces.iter().try_for_each(validate_surface_info) { + set_last_error(&error); + return crate::error::last_error_code(); + } + let items: Vec<AdExactSurfaceInfo> = + surfaces.iter().map(exact_surface_info_to_c).collect(); + *out = Box::into_raw(Box::new(AdExactSurfaceList { + items: items.into_boxed_slice(), + })); + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } + }) +} + +fn process_identity_for_pid( + adapter: &dyn agent_desktop_core::PlatformAdapter, + pid: u32, + deadline: agent_desktop_core::Deadline, +) -> Result<agent_desktop_core::ProcessIdentity, agent_desktop_core::AdapterError> { + if pid == 0 { + return Err(agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "surface pid must be positive", + )); + } + let mut matches = adapter + .list_apps(deadline)? + .into_iter() + .filter(|app| app.pid.get() == pid) + .collect::<Vec<_>>(); + if matches.is_empty() { + return Err(agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::AppNotFound, + "surface pid did not identify a live application instance", + )); + } + if matches.len() > 1 { + return Err(agent_desktop_core::AdapterError::ambiguous_target( + "surface pid identified multiple live application instances", + )); + } + let app = matches.swap_remove(0); + let instance = app + .process_instance + .filter(|instance| !instance.is_empty()) + .ok_or_else(|| { + agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::ActionNotSupported, + "surface application has no process-generation identity", + ) + })?; + Ok(agent_desktop_core::ProcessIdentity::new( + agent_desktop_core::ProcessId::new(pid), + instance, + )) +} + +/// # Safety +/// `list` must be null or returned by `ad_list_surfaces_exact`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_exact_surface_list_count(list: *const AdExactSurfaceList) -> u32 { + if list.is_null() { + return 0; + } + unsafe { &*list }.items.len() as u32 +} + +/// # Safety +/// `list` must be null or returned by `ad_list_surfaces_exact`. The result is +/// borrowed until the list is freed. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_exact_surface_list_get( + list: *const AdExactSurfaceList, + index: u32, +) -> *const AdExactSurfaceInfo { + if list.is_null() { + return ptr::null(); + } + unsafe { &*list } + .items + .get(index as usize) + .map_or(ptr::null(), std::ptr::from_ref) +} + +/// # Safety +/// `list` must be null or returned by `ad_list_surfaces_exact`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_exact_surface_list_free(list: *mut AdExactSurfaceList) { + trap_panic_void(|| unsafe { + if list.is_null() { + return; + } + let mut list = Box::from_raw(list); + for item in &mut list.items { + free_exact_surface_info_fields(item); + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; + use agent_desktop_core::{AppInfo, SurfaceInfo}; + + struct AppInventory(Vec<AppInfo>); + + impl ActionOps for AppInventory {} + impl InputOps for AppInventory {} + impl SystemOps for AppInventory {} + + impl ObservationOps for AppInventory { + fn list_apps( + &self, + _deadline: agent_desktop_core::Deadline, + ) -> Result<Vec<AppInfo>, agent_desktop_core::AdapterError> { + Ok(self.0.clone()) + } + } + + fn app(pid: u32, instance: &str) -> AppInfo { + AppInfo { + name: "Fixture".into(), + pid: agent_desktop_core::ProcessId::new(pid), + bundle_id: None, + process_instance: Some(instance.into()), + } + } + + #[test] + fn missing_surface_pid_is_app_not_found() { + let error = process_identity_for_pid( + &AppInventory(Vec::new()), + 42, + agent_desktop_core::Deadline::standard().unwrap(), + ) + .unwrap_err(); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::AppNotFound); + } + + #[test] + fn duplicate_surface_pid_is_ambiguous() { + let error = process_identity_for_pid( + &AppInventory(vec![app(42, "a"), app(42, "b")]), + 42, + agent_desktop_core::Deadline::standard().unwrap(), + ) + .unwrap_err(); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::AmbiguousTarget); + } + + #[test] + fn exact_surface_list_owns_borrowed_entries_until_explicit_free() { + let item = exact_surface_info_to_c(&SurfaceInfo { + id: "surface-1".into(), + kind: "window".into(), + title: None, + item_count: Some(3), + }); + let list = Box::into_raw(Box::new(AdExactSurfaceList { + items: vec![item].into_boxed_slice(), + })); + + assert_eq!(unsafe { ad_exact_surface_list_count(list) }, 1); + assert!(!unsafe { ad_exact_surface_list_get(list, 0) }.is_null()); + assert!(unsafe { ad_exact_surface_list_get(list, 1) }.is_null()); + unsafe { ad_exact_surface_list_free(list) }; + unsafe { ad_exact_surface_list_free(ptr::null_mut()) }; + } +} diff --git a/crates/ffi/src/tree/flatten.rs b/crates/ffi/src/tree/flatten.rs index 0a1907c..33751d6 100644 --- a/crates/ffi/src/tree/flatten.rs +++ b/crates/ffi/src/tree/flatten.rs @@ -1,6 +1,6 @@ use crate::convert::string::{opt_string_to_c, string_to_c_lossy}; -use crate::types::{AdNode, AdNodeTree, AdRect}; -use agent_desktop_core::node::AccessibilityNode; +use crate::types::{AdNode, AdNodeContent, AdNodePresentation, AdNodeRelation, AdNodeTree, AdRect}; +use agent_desktop_core::AccessibilityNode; use std::collections::VecDeque; use std::os::raw::c_char; use std::ptr; @@ -20,8 +20,10 @@ use std::ptr; /// `a`, overlapping with `a`'s siblings — the range /// `a.child_start..a.child_start + a.child_count` therefore stepped into /// grandchildren. BFS keeps siblings contiguous by construction. -pub(crate) fn flatten_tree(root: &AccessibilityNode) -> AdNodeTree { - let total = count_nodes(root); +pub(crate) fn flatten_tree( + root: &AccessibilityNode, +) -> Result<AdNodeTree, agent_desktop_core::AdapterError> { + let total = count_nodes_bounded(root, crate::resource::MAX_FFI_LIST_ITEMS)?; let mut flat: Vec<AdNode> = Vec::with_capacity(total); flat.push(to_ad_node(root, -1)); @@ -34,8 +36,8 @@ pub(crate) fn flatten_tree(root: &AccessibilityNode) -> AdNodeTree { } let child_start = flat.len() as u32; let child_count = node.children.len() as u32; - flat[node_idx].child_start = child_start; - flat[node_idx].child_count = child_count; + flat[node_idx].relation.child_start = child_start; + flat[node_idx].relation.child_count = child_count; for child in &node.children { let child_idx = flat.len(); flat.push(to_ad_node(child, node_idx as i32)); @@ -44,57 +46,48 @@ pub(crate) fn flatten_tree(root: &AccessibilityNode) -> AdNodeTree { } let count = flat.len() as u32; - flat.push(sentinel_node()); let nodes = if flat.is_empty() { ptr::null_mut() } else { let mut boxed = flat.into_boxed_slice(); let ptr = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation( + crate::resource::AllocationKind::TreeNodes, + ptr, + count as usize, + ); ptr }; - AdNodeTree { nodes, count } + Ok(AdNodeTree { nodes, count }) } -fn sentinel_node() -> AdNode { - AdNode { - ref_id: ptr::null(), - role: ptr::null(), - name: ptr::null(), - value: ptr::null(), - description: ptr::null(), - hint: ptr::null(), - states: ptr::null_mut(), - state_count: 0, - bounds: AdRect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 0.0, - }, - has_bounds: false, - parent_index: -1, - child_start: 0, - child_count: 0, - } -} - -fn count_nodes(node: &AccessibilityNode) -> usize { +fn count_nodes_bounded( + node: &AccessibilityNode, + limit: usize, +) -> Result<usize, agent_desktop_core::AdapterError> { let mut total: usize = 0; let mut queue: VecDeque<&AccessibilityNode> = VecDeque::new(); queue.push_back(node); while let Some(n) = queue.pop_front() { - total += 1; + total = total.saturating_add(1); + crate::resource::validate_list_len(total, "Accessibility tree")?; + if total > limit { + return Err(agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::Internal, + "Accessibility tree exceeds the FFI output item limit", + )); + } for c in &n.children { queue.push_back(c); } } - total + Ok(total) } fn to_ad_node(node: &AccessibilityNode, parent_index: i32) -> AdNode { - let (states_ptr, state_count) = strings_to_c_array(&node.states); - let (bounds, has_bounds) = match &node.bounds { + let (states_ptr, state_count) = strings_to_c_array(&node.presentation.states); + let (bounds, has_bounds) = match &node.presentation.bounds { Some(r) => (crate::convert::rect_to_c(r), true), None => ( AdRect { @@ -107,19 +100,25 @@ fn to_ad_node(node: &AccessibilityNode, parent_index: i32) -> AdNode { ), }; AdNode { - ref_id: opt_string_to_c(node.ref_id.as_deref()), - role: string_to_c_lossy(&node.role), - name: opt_string_to_c(node.name.as_deref()), - value: opt_string_to_c(node.value.as_deref()), - description: opt_string_to_c(node.description.as_deref()), - hint: opt_string_to_c(node.hint.as_deref()), - states: states_ptr, - state_count, - bounds, - has_bounds, - parent_index, - child_start: 0, - child_count: 0, + content: AdNodeContent { + ref_id: opt_string_to_c(node.ref_id.as_deref()), + role: string_to_c_lossy(&node.role), + name: opt_string_to_c(node.identity.name.as_deref()), + value: opt_string_to_c(node.identity.value.as_deref()), + description: opt_string_to_c(node.identity.description.as_deref()), + hint: opt_string_to_c(node.presentation.hint.as_deref()), + }, + presentation: AdNodePresentation { + states: states_ptr, + bounds, + state_count, + has_bounds, + }, + relation: AdNodeRelation { + parent_index, + child_start: 0, + child_count: 0, + }, } } @@ -127,12 +126,16 @@ fn strings_to_c_array(strings: &[String]) -> (*mut *mut c_char, u32) { if strings.is_empty() { return (ptr::null_mut(), 0); } - let mut ptrs: Vec<*mut c_char> = strings.iter().map(|s| string_to_c_lossy(s)).collect(); + let ptrs: Vec<*mut c_char> = strings.iter().map(|s| string_to_c_lossy(s)).collect(); let count = ptrs.len() as u32; - ptrs.push(ptr::null_mut()); let mut boxed = ptrs.into_boxed_slice(); let ptr = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation( + crate::resource::AllocationKind::TreeStateStrings, + ptr, + count as usize, + ); (ptr, count) } @@ -146,13 +149,8 @@ mod tests { AccessibilityNode { ref_id: None, role: role.into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: agent_desktop_core::NodeIdentity::default(), + presentation: agent_desktop_core::NodePresentation::default(), children: vec![], children_count: None, } @@ -160,20 +158,20 @@ mod tests { fn direct_children(nodes: &[AdNode], idx: usize) -> Vec<&AdNode> { let n = &nodes[idx]; - let start = n.child_start as usize; - let end = start + n.child_count as usize; + let start = n.relation.child_start as usize; + let end = start + n.relation.child_count as usize; nodes[start..end].iter().collect() } #[test] fn test_flatten_single_node() { let root = node("window"); - let tree = flatten_tree(&root); + let tree = flatten_tree(&root).unwrap(); assert_eq!(tree.count, 1); let nodes = unsafe { std::slice::from_raw_parts(tree.nodes, 1) }; - assert_eq!(nodes[0].parent_index, -1); - assert_eq!(nodes[0].child_count, 0); - let role = unsafe { c_to_string(nodes[0].role) }; + assert_eq!(nodes[0].relation.parent_index, -1); + assert_eq!(nodes[0].relation.child_count, 0); + let role = unsafe { c_to_string(nodes[0].content.role) }; assert_eq!(role.as_deref(), Some("window")); unsafe { ad_free_tree(&tree as *const _ as *mut _) }; } @@ -184,17 +182,17 @@ mod tests { let mut root = node("window"); root.children = vec![btn]; - let tree = flatten_tree(&root); + let tree = flatten_tree(&root).unwrap(); assert_eq!(tree.count, 2); let nodes = unsafe { std::slice::from_raw_parts(tree.nodes, 2) }; - assert_eq!(nodes[0].parent_index, -1); - assert_eq!(nodes[0].child_start, 1); - assert_eq!(nodes[0].child_count, 1); + assert_eq!(nodes[0].relation.parent_index, -1); + assert_eq!(nodes[0].relation.child_start, 1); + assert_eq!(nodes[0].relation.child_count, 1); - assert_eq!(nodes[1].parent_index, 0); - assert_eq!(nodes[1].child_count, 0); - let role = unsafe { c_to_string(nodes[1].role) }; + assert_eq!(nodes[1].relation.parent_index, 0); + assert_eq!(nodes[1].relation.child_count, 0); + let role = unsafe { c_to_string(nodes[1].content.role) }; assert_eq!(role.as_deref(), Some("button")); unsafe { ad_free_tree(&tree as *const _ as *mut _) }; @@ -210,35 +208,35 @@ mod tests { let mut root = node("root"); root.children = vec![a, b]; - let tree = flatten_tree(&root); + let tree = flatten_tree(&root).unwrap(); assert_eq!(tree.count, 5); let nodes = unsafe { std::slice::from_raw_parts(tree.nodes, 5) }; let roles: Vec<String> = nodes .iter() - .map(|n| unsafe { c_to_string(n.role).unwrap() }) + .map(|n| unsafe { c_to_string(n.content.role).unwrap() }) .collect(); assert_eq!(roles, vec!["root", "a", "b", "a1", "a2"]); let root_children: Vec<String> = direct_children(nodes, 0) .iter() - .map(|n| unsafe { c_to_string(n.role).unwrap() }) + .map(|n| unsafe { c_to_string(n.content.role).unwrap() }) .collect(); assert_eq!(root_children, vec!["a", "b"]); let a_idx = nodes .iter() - .position(|n| unsafe { c_to_string(n.role).unwrap() } == "a") + .position(|n| unsafe { c_to_string(n.content.role).unwrap() } == "a") .unwrap(); let a_children: Vec<String> = direct_children(nodes, a_idx) .iter() - .map(|n| unsafe { c_to_string(n.role).unwrap() }) + .map(|n| unsafe { c_to_string(n.content.role).unwrap() }) .collect(); assert_eq!(a_children, vec!["a1", "a2"]); let b_idx = nodes .iter() - .position(|n| unsafe { c_to_string(n.role).unwrap() } == "b") + .position(|n| unsafe { c_to_string(n.content.role).unwrap() } == "b") .unwrap(); assert!(direct_children(nodes, b_idx).is_empty()); @@ -253,18 +251,18 @@ mod tests { parent.children = vec![leaf]; leaf = parent; } - let tree = flatten_tree(&leaf); + let tree = flatten_tree(&leaf).unwrap(); assert_eq!(tree.count, 11); let nodes = unsafe { std::slice::from_raw_parts(tree.nodes, 11) }; let mut cursor = 0usize; for expected in 0..11 { - let role = unsafe { c_to_string(nodes[cursor].role).unwrap() }; + let role = unsafe { c_to_string(nodes[cursor].content.role).unwrap() }; assert_eq!(role, format!("l{}", expected)); let children = direct_children(nodes, cursor); if expected < 10 { assert_eq!(children.len(), 1); - cursor = nodes[cursor].child_start as usize; + cursor = nodes[cursor].relation.child_start as usize; } else { assert!(children.is_empty()); } @@ -278,26 +276,37 @@ mod tests { for i in 0..100 { root.children.push(node(&format!("child_{}", i))); } - let tree = flatten_tree(&root); + let tree = flatten_tree(&root).unwrap(); assert_eq!(tree.count, 101); let nodes = unsafe { std::slice::from_raw_parts(tree.nodes, 101) }; let children = direct_children(nodes, 0); assert_eq!(children.len(), 100); for (i, c) in children.iter().enumerate() { - let role = unsafe { c_to_string(c.role).unwrap() }; + let role = unsafe { c_to_string(c.content.role).unwrap() }; assert_eq!(role, format!("child_{}", i)); } unsafe { ad_free_tree(&tree as *const _ as *mut _) }; } + #[test] + fn node_count_fails_before_flattening_past_the_resource_limit() { + let mut root = node("root"); + root.children = vec![node("a"), node("b")]; + + let error = count_nodes_bounded(&root, 2).unwrap_err(); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::Internal); + assert!(error.message.contains("item limit")); + } + #[test] fn test_flatten_with_states() { let mut btn = node("button"); - btn.states = vec!["focused".into(), "enabled".into()]; - let tree = flatten_tree(&btn); + btn.presentation.states = vec!["focused".into(), "enabled".into()]; + let tree = flatten_tree(&btn).unwrap(); let nodes = unsafe { std::slice::from_raw_parts(tree.nodes, 1) }; - assert_eq!(nodes[0].state_count, 2); - let states = unsafe { std::slice::from_raw_parts(nodes[0].states, 2) }; + assert_eq!(nodes[0].presentation.state_count, 2); + let states = unsafe { std::slice::from_raw_parts(nodes[0].presentation.states, 2) }; let s0 = unsafe { c_to_string(states[0]) }; let s1 = unsafe { c_to_string(states[1]) }; assert_eq!(s0.as_deref(), Some("focused")); diff --git a/crates/ffi/src/tree/free.rs b/crates/ffi/src/tree/free.rs index 132915f..56ebad0 100644 --- a/crates/ffi/src/tree/free.rs +++ b/crates/ffi/src/tree/free.rs @@ -3,43 +3,38 @@ use crate::types::{AdNode, AdNodeTree}; use std::os::raw::c_char; use std::ptr; -const MAX_NODE_STATE_STRINGS_TO_FREE: usize = 1024; -const MAX_TREE_NODES_TO_FREE: usize = 1_000_000; - unsafe fn free_c_string_array(arr: *mut *mut c_char) { unsafe { - if arr.is_null() { - return; - } - let mut len = 0; - while len < MAX_NODE_STATE_STRINGS_TO_FREE && !(*arr.add(len)).is_null() { - free_c_string(*arr.add(len)); - len += 1; - } - drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( + let Some(len) = crate::resource::take_allocation( + crate::resource::AllocationKind::TreeStateStrings, arr, - len + 1, - ))); + ) else { + return; + }; + for index in 0..len { + free_c_string(*arr.add(index)); + } + drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut(arr, len))); } } unsafe fn free_node_fields(node: &mut AdNode) { unsafe { - free_c_string(node.ref_id as *mut c_char); - free_c_string(node.role as *mut c_char); - free_c_string(node.name as *mut c_char); - free_c_string(node.value as *mut c_char); - free_c_string(node.description as *mut c_char); - free_c_string(node.hint as *mut c_char); - free_c_string_array(node.states); - node.ref_id = ptr::null(); - node.role = ptr::null(); - node.name = ptr::null(); - node.value = ptr::null(); - node.description = ptr::null(); - node.hint = ptr::null(); - node.states = ptr::null_mut(); - node.state_count = 0; + free_c_string(node.content.ref_id as *mut c_char); + free_c_string(node.content.role as *mut c_char); + free_c_string(node.content.name as *mut c_char); + free_c_string(node.content.value as *mut c_char); + free_c_string(node.content.description as *mut c_char); + free_c_string(node.content.hint as *mut c_char); + free_c_string_array(node.presentation.states); + node.content.ref_id = ptr::null(); + node.content.role = ptr::null(); + node.content.name = ptr::null(); + node.content.value = ptr::null(); + node.content.description = ptr::null(); + node.content.hint = ptr::null(); + node.presentation.states = ptr::null_mut(); + node.presentation.state_count = 0; } } @@ -56,33 +51,30 @@ pub unsafe extern "C" fn ad_free_tree(tree: *mut AdNodeTree) { if tree.nodes.is_null() { return; } - let node_count = sentinel_node_count(tree.nodes); + let Some(node_count) = crate::resource::take_allocation( + crate::resource::AllocationKind::TreeNodes, + tree.nodes, + ) else { + tree.nodes = ptr::null_mut(); + tree.count = 0; + return; + }; let nodes = std::slice::from_raw_parts_mut(tree.nodes, node_count); for node in nodes.iter_mut() { free_node_fields(node); } drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( - tree.nodes, - node_count + 1, + tree.nodes, node_count, ))); tree.nodes = ptr::null_mut(); tree.count = 0; }) } -unsafe fn sentinel_node_count(nodes: *mut AdNode) -> usize { - unsafe { - let mut count = 0; - while count < MAX_TREE_NODES_TO_FREE && !(*nodes.add(count)).role.is_null() { - count += 1; - } - count - } -} - #[cfg(test)] mod tests { use super::*; + use crate::types::{AdNodeContent, AdNodePresentation, AdNodeRelation}; #[test] fn test_free_null_tree_is_noop() { @@ -98,8 +90,13 @@ mod tests { } fn tree_with_node(node: AdNode) -> AdNodeTree { - let mut nodes = vec![node, sentinel_node()].into_boxed_slice(); + let mut nodes = vec![node].into_boxed_slice(); let raw = nodes.as_mut_ptr(); + crate::resource::register_allocation( + crate::resource::AllocationKind::TreeNodes, + raw, + nodes.len(), + ); std::mem::forget(nodes); AdNodeTree { nodes: raw, @@ -109,78 +106,61 @@ mod tests { fn node_with_states(states: &[&str], state_count: u32) -> AdNode { AdNode { - ref_id: ptr::null(), - role: crate::convert::string::string_to_c_lossy("button"), - name: ptr::null(), - value: ptr::null(), - description: ptr::null(), - hint: ptr::null(), - states: state_array(states), - state_count, - bounds: crate::types::AdRect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 0.0, + content: AdNodeContent { + ref_id: ptr::null(), + role: crate::convert::string::string_to_c_lossy("button"), + name: ptr::null(), + value: ptr::null(), + description: ptr::null(), + hint: ptr::null(), }, - has_bounds: false, - parent_index: -1, - child_start: 0, - child_count: 0, - } - } - - fn sentinel_node() -> AdNode { - AdNode { - ref_id: ptr::null(), - role: ptr::null(), - name: ptr::null(), - value: ptr::null(), - description: ptr::null(), - hint: ptr::null(), - states: ptr::null_mut(), - state_count: 0, - bounds: crate::types::AdRect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 0.0, + presentation: AdNodePresentation { + states: state_array(states), + bounds: crate::types::AdRect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }, + state_count, + has_bounds: false, + }, + relation: AdNodeRelation { + parent_index: -1, + child_start: 0, + child_count: 0, }, - has_bounds: false, - parent_index: -1, - child_start: 0, - child_count: 0, } } fn state_array(states: &[&str]) -> *mut *mut c_char { - let mut ptrs: Vec<*mut c_char> = states + let ptrs: Vec<*mut c_char> = states .iter() .map(|state| crate::convert::string::string_to_c_lossy(state)) .collect(); - ptrs.push(ptr::null_mut()); + let len = ptrs.len(); let mut boxed = ptrs.into_boxed_slice(); let raw = boxed.as_mut_ptr(); std::mem::forget(boxed); + crate::resource::register_allocation( + crate::resource::AllocationKind::TreeStateStrings, + raw, + len, + ); raw } #[test] fn free_tree_ignores_mutated_tree_count() { - let root = agent_desktop_core::node::AccessibilityNode { + let root = agent_desktop_core::AccessibilityNode { ref_id: None, role: "button".into(), - name: None, - value: None, - description: None, - hint: None, - states: vec![], - available_actions: vec![], - bounds: None, + identity: agent_desktop_core::NodeIdentity::default(), + presentation: agent_desktop_core::NodePresentation::default(), children: vec![], children_count: None, }; - let mut tree = crate::tree::flatten::flatten_tree(&root); + let mut tree = crate::tree::flatten::flatten_tree(&root).unwrap(); tree.count = u32::MAX; unsafe { ad_free_tree(&mut tree) }; diff --git a/crates/ffi/src/tree/get.rs b/crates/ffi/src/tree/get.rs index 1d1eff4..3bdea5f 100644 --- a/crates/ffi/src/tree/get.rs +++ b/crates/ffi/src/tree/get.rs @@ -3,54 +3,15 @@ use crate::convert::surface::snapshot_surface_from_c; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; use crate::tree::flatten::flatten_tree; -use crate::types::{AdNodeTree, AdTreeOptions, AdWindowInfo}; +use crate::types::{AdExactWindowInfo, AdNodeTree, AdTreeOptions, AdWindowInfo}; use std::ptr; -/// Snapshots `win`'s accessibility tree into the flat BFS layout -/// described in the types module. The result is written into `*out` -/// and must be freed with `ad_free_tree`. Direct children of any node -/// live contiguously at `nodes[child_start..child_start + child_count]`. -/// -/// `opts.max_depth` caps tree depth. `opts.surface` selects which -/// surface to snapshot (window body, menu, menubar, sheet, popover, -/// alert, or focused subtree); see `AdSnapshotSurface`. -/// `opts.interactive_only` prunes non-interactive nodes; `opts.compact` -/// collapses containers with no semantic payload. -/// -/// # Raw-tree contract -/// -/// This is a **raw adapter tree** — ref-less, no refmap persistence, and -/// no JSON envelope. Differences the caller must know about: -/// -/// - `ref_id` is always null on every `AdNode`. `ref_alloc::allocate_refs` -/// is not run; `@e` ref assignment is a snapshot-pipeline concern. -/// - `include_bounds`, `interactive_only`, and `compact` are honoured via -/// `ref_alloc::transform_tree` after the adapter returns. Because refs are -/// not allocated, the `interactive_only` cut is role-based rather than -/// ref-based; otherwise the semantics match the snapshot path. -/// - No skeleton/drill-down pipeline is wired through — `skeleton` is -/// always false on the underlying `TreeOptions`. -/// -/// # When to use this function vs `ad_snapshot` -/// -/// **Observe–act agents** that need `@e` refs and refmap persistence should -/// call `ad_snapshot` instead. `ad_snapshot` runs the full snapshot pipeline -/// (ref allocation, refmap write to disk, JSON envelope with -/// `{"version":"2.0","ok":true,...}`) and is the correct starting point for -/// any workflow that drives subsequent ref-based actions via -/// `ad_execute_by_ref` (with an `AdAction`). -/// -/// Use `ad_get_tree` when you need the raw flat BFS layout without refs — -/// for example, to drive your own traversal logic or to populate a UI -/// inspector that does not use the ref-based action API. For point lookups -/// that bypass tree shape entirely, `ad_find` + `ad_get` / `ad_is` are -/// another alternative. -/// -/// On error `*out` is zeroed so `ad_free_tree` on it is a safe no-op. +/// Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process +/// generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. +/// Use `ad_get_tree_exact`. /// /// # Safety -/// All pointers must be non-null. `win.id` and `win.title` must be -/// valid UTF-8 C strings. `out` must be writable. +/// All pointers must be non-null and `out` must be writable. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_get_tree( adapter: *const AdAdapter, @@ -64,14 +25,10 @@ pub unsafe extern "C" fn ad_get_tree( (*out).nodes = ptr::null_mut(); (*out).count = 0; } - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(win, c"win is null"); crate::pointer_guard::guard_non_null!(opts, c"opts is null"); - let adapter = unsafe { &*adapter }; let opts_ref = unsafe { &*opts }; let core_win = match crate::windows::ad_window_to_core(unsafe { &*win }) { Ok(w) => w, @@ -80,37 +37,94 @@ pub unsafe extern "C" fn ad_get_tree( return crate::error::last_error_code(); } }; - let surface = match snapshot_surface_from_c(opts_ref.surface, "snapshot surface") { - Ok(surface) => surface, - Err(e) => { - set_last_error(&e); - return AdResult::ErrInvalidArgs; - } - }; - let core_opts = agent_desktop_core::adapter::TreeOptions { - max_depth: opts_ref.max_depth, - include_bounds: opts_ref.include_bounds, - interactive_only: opts_ref.interactive_only, - compact: opts_ref.compact, - surface, - skeleton: false, - }; - - match adapter.inner.get_tree(&core_win, &core_opts) { - Ok(tree) => { - let shaped = agent_desktop_core::ref_alloc::transform_tree( - tree, - core_opts.include_bounds, - core_opts.interactive_only, - core_opts.compact, - ); - unsafe { *out = flatten_tree(&shaped) }; - AdResult::Ok - } - Err(e) => { - set_last_error(&e); - crate::error::last_error_code() - } - } + unsafe { get_core_tree(adapter, &core_win, opts_ref, out) } }) } + +/// Snapshots a generation-pinned window into the flat, owned, breadth-first C +/// tree layout. Direct children are contiguous at +/// `nodes[child_start..child_start + child_count]`; free the result with +/// `ad_free_tree`. +/// +/// This is a raw adapter tree: nodes do not receive refs, no refmap is +/// persisted, and no JSON envelope is produced. `max_depth`, `surface`, +/// `include_bounds`, `interactive_only`, and `compact` are applied; skeleton +/// and drill-down behavior are not. Use `ad_snapshot` for the canonical +/// observe-act workflow with snapshot-qualified refs. +/// +/// # Safety +/// All pointers must be valid and `out` must be writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_get_tree_exact( + adapter: *const AdAdapter, + win: *const AdExactWindowInfo, + opts: *const AdTreeOptions, + out: *mut AdNodeTree, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + (*out).nodes = ptr::null_mut(); + (*out).count = 0; + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(win, c"win is null"); + crate::pointer_guard::guard_non_null!(opts, c"opts is null"); + let window = match crate::windows::ad_exact_window_to_core(&*win) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + get_core_tree(adapter, &window, &*opts, out) + }) +} + +unsafe fn get_core_tree( + adapter: *const AdAdapter, + window: &agent_desktop_core::WindowInfo, + options: &AdTreeOptions, + out: *mut AdNodeTree, +) -> AdResult { + let surface = match snapshot_surface_from_c(options.surface, "snapshot surface") { + Ok(surface) => surface, + Err(e) => { + set_last_error(&e); + return AdResult::ErrInvalidArgs; + } + }; + let core_opts = agent_desktop_core::TreeOptions { + max_depth: options.max_depth, + include_bounds: options.include_bounds, + interactive_only: options.interactive_only, + compact: options.compact, + surface, + skeleton: false, + }; + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + + match adapter.inner.get_tree(window, &core_opts, deadline) { + Ok(tree) => { + let shaped = agent_desktop_core::ref_alloc::transform_tree( + tree, + core_opts.include_bounds, + core_opts.interactive_only, + core_opts.compact, + ); + match flatten_tree(&shaped) { + Ok(tree) => { + unsafe { *out = tree }; + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } + } + Err(e) => { + set_last_error(&e); + crate::error::last_error_code() + } + } +} diff --git a/crates/ffi/src/types/action_result.rs b/crates/ffi/src/types/action_result.rs index 6c8d772..9e76761 100644 --- a/crates/ffi/src/types/action_result.rs +++ b/crates/ffi/src/types/action_result.rs @@ -1,4 +1,5 @@ use crate::types::action_step::AdActionStep; +use crate::types::delivery_semantics::AdDeliverySemantics; use crate::types::element_state::AdElementState; use std::os::raw::c_char; @@ -9,9 +10,11 @@ pub struct AdActionResult { pub post_state: *mut AdElementState, pub steps: *mut AdActionStep, pub step_count: u32, + pub details_json: *const c_char, + pub disposition: AdDeliverySemantics, } -pub const AD_ACTION_RESULT_SIZE: usize = 40; +pub const AD_ACTION_RESULT_SIZE: usize = 56; const _: () = assert!(std::mem::size_of::<AdActionResult>() == AD_ACTION_RESULT_SIZE); diff --git a/crates/ffi/src/types/action_step.rs b/crates/ffi/src/types/action_step.rs index d11dc90..d3835a9 100644 --- a/crates/ffi/src/types/action_step.rs +++ b/crates/ffi/src/types/action_step.rs @@ -4,9 +4,14 @@ use std::os::raw::c_char; pub struct AdActionStep { pub label: *const c_char, pub outcome: *const c_char, + pub mechanism: i32, + pub has_mechanism: bool, + pub verified: bool, + pub has_verified: bool, + pub _reserved: u64, } -pub const AD_ACTION_STEP_SIZE: usize = 16; +pub const AD_ACTION_STEP_SIZE: usize = 32; const _: () = assert!(std::mem::size_of::<AdActionStep>() == AD_ACTION_STEP_SIZE); @@ -14,3 +19,21 @@ const _: () = assert!(std::mem::size_of::<AdActionStep>() == AD_ACTION_STEP_SIZE pub extern "C" fn ad_action_step_size() -> usize { std::mem::size_of::<AdActionStep>() } + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{offset_of, size_of}; + + #[test] + fn layout_matches_published_abi() { + assert_eq!(size_of::<AdActionStep>(), AD_ACTION_STEP_SIZE); + assert_eq!(offset_of!(AdActionStep, label), 0); + assert_eq!(offset_of!(AdActionStep, outcome), 8); + assert_eq!(offset_of!(AdActionStep, mechanism), 16); + assert_eq!(offset_of!(AdActionStep, has_mechanism), 20); + assert_eq!(offset_of!(AdActionStep, verified), 21); + assert_eq!(offset_of!(AdActionStep, has_verified), 22); + assert_eq!(offset_of!(AdActionStep, _reserved), 24); + } +} diff --git a/crates/ffi/src/types/app_info.rs b/crates/ffi/src/types/app_info.rs index 2f168bb..6c3a057 100644 --- a/crates/ffi/src/types/app_info.rs +++ b/crates/ffi/src/types/app_info.rs @@ -3,6 +3,6 @@ use std::os::raw::c_char; #[repr(C)] pub struct AdAppInfo { pub name: *const c_char, - pub pid: i32, + pub pid: u32, pub bundle_id: *const c_char, } diff --git a/crates/ffi/src/types/delivery_disposition.rs b/crates/ffi/src/types/delivery_disposition.rs new file mode 100644 index 0000000..73aec38 --- /dev/null +++ b/crates/ffi/src/types/delivery_disposition.rs @@ -0,0 +1,23 @@ +#[repr(i32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdDeliveryDisposition { + Unknown = 0, + NotDelivered = 1, + DeliveryUncertain = 2, + DeliveredUnverified = 3, + DeliveredVerified = 4, +} + +impl From<agent_desktop_core::DeliveryDisposition> for AdDeliveryDisposition { + fn from(value: agent_desktop_core::DeliveryDisposition) -> Self { + match value { + agent_desktop_core::DeliveryDisposition::Unknown => Self::Unknown, + agent_desktop_core::DeliveryDisposition::NotDelivered => Self::NotDelivered, + agent_desktop_core::DeliveryDisposition::DeliveryUncertain => Self::DeliveryUncertain, + agent_desktop_core::DeliveryDisposition::DeliveredUnverified => { + Self::DeliveredUnverified + } + agent_desktop_core::DeliveryDisposition::DeliveredVerified => Self::DeliveredVerified, + } + } +} diff --git a/crates/ffi/src/types/delivery_semantics.rs b/crates/ffi/src/types/delivery_semantics.rs new file mode 100644 index 0000000..587eb17 --- /dev/null +++ b/crates/ffi/src/types/delivery_semantics.rs @@ -0,0 +1,28 @@ +use crate::types::{AdDeliveryDisposition, AdRetryDisposition}; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AdDeliverySemantics { + pub delivery: i32, + pub retry: i32, +} + +pub const AD_DELIVERY_SEMANTICS_SIZE: usize = 8; + +impl AdDeliverySemantics { + pub(crate) fn from_core(value: agent_desktop_core::DeliverySemantics) -> Self { + Self { + delivery: AdDeliveryDisposition::from(value.delivery()) as i32, + retry: AdRetryDisposition::from(value.retry()) as i32, + } + } + + pub(crate) const fn unknown() -> Self { + Self { + delivery: AdDeliveryDisposition::Unknown as i32, + retry: AdRetryDisposition::Unknown as i32, + } + } +} + +const _: () = assert!(std::mem::size_of::<AdDeliverySemantics>() == AD_DELIVERY_SEMANTICS_SIZE); diff --git a/crates/ffi/src/types/display_info.rs b/crates/ffi/src/types/display_info.rs new file mode 100644 index 0000000..f4e9607 --- /dev/null +++ b/crates/ffi/src/types/display_info.rs @@ -0,0 +1,22 @@ +use crate::types::AdRect; +use std::os::raw::c_char; + +pub const AD_DISPLAY_INFO_VERSION: u32 = 1; +pub const AD_DISPLAY_INFO_SIZE: usize = 64; + +#[repr(C)] +pub struct AdDisplayInfo { + pub version: u32, + pub size: u32, + pub id: *const c_char, + pub bounds: AdRect, + pub is_primary: bool, + pub scale: f64, +} + +const _: () = assert!(std::mem::size_of::<AdDisplayInfo>() == AD_DISPLAY_INFO_SIZE); + +#[unsafe(no_mangle)] +pub extern "C" fn ad_display_info_size() -> usize { + std::mem::size_of::<AdDisplayInfo>() +} diff --git a/crates/ffi/src/types/display_list.rs b/crates/ffi/src/types/display_list.rs new file mode 100644 index 0000000..96b9468 --- /dev/null +++ b/crates/ffi/src/types/display_list.rs @@ -0,0 +1,6 @@ +use crate::types::AdDisplayInfo; + +/// Opaque list handle emitted by `ad_list_displays`. +pub struct AdDisplayList { + pub(crate) items: Box<[AdDisplayInfo]>, +} diff --git a/crates/ffi/src/types/drag_params.rs b/crates/ffi/src/types/drag_params.rs index 4b63880..9103d5e 100644 --- a/crates/ffi/src/types/drag_params.rs +++ b/crates/ffi/src/types/drag_params.rs @@ -1,5 +1,5 @@ use crate::types::point::AdPoint; -use agent_desktop_core::action::{DragParams as CoreDragParams, Point as CorePoint}; +use agent_desktop_core::{DragParams as CoreDragParams, Point as CorePoint}; /// Caller-allocated drag parameters. Callers must zero-initialize the whole /// struct before setting fields so unset numeric fields read as the `0` diff --git a/crates/ffi/src/types/exact_ref_entry.rs b/crates/ffi/src/types/exact_ref_entry.rs new file mode 100644 index 0000000..8ca858e --- /dev/null +++ b/crates/ffi/src/types/exact_ref_entry.rs @@ -0,0 +1,28 @@ +use std::os::raw::c_char; + +use crate::types::AdRefEntry; + +pub const AD_EXACT_REF_ENTRY_VERSION: u32 = 1; +pub const AD_EXACT_REF_ENTRY_SIZE: usize = 224; + +/// Additive exact-identity payload for low-level struct-based ref actions. +/// +/// Callers must set `version` to `AD_EXACT_REF_ENTRY_VERSION`, `size` to +/// `AD_EXACT_REF_ENTRY_SIZE`, and `process_instance` to the generation token +/// emitted by the snapshot. When `entry.identity.native_id` is non-null, +/// `identifier_kind` must name its exact platform identifier namespace. +#[repr(C)] +pub struct AdExactRefEntry { + pub version: u32, + pub size: u32, + pub entry: AdRefEntry, + pub process_instance: *const c_char, + pub identifier_kind: i32, +} + +const _: () = assert!(std::mem::size_of::<AdExactRefEntry>() == AD_EXACT_REF_ENTRY_SIZE); + +#[unsafe(no_mangle)] +pub extern "C" fn ad_exact_ref_entry_size() -> usize { + std::mem::size_of::<AdExactRefEntry>() +} diff --git a/crates/ffi/src/types/exact_surface_info.rs b/crates/ffi/src/types/exact_surface_info.rs new file mode 100644 index 0000000..a84f087 --- /dev/null +++ b/crates/ffi/src/types/exact_surface_info.rs @@ -0,0 +1,22 @@ +use std::os::raw::c_char; + +use crate::types::AdSurfaceInfo; + +pub const AD_EXACT_SURFACE_INFO_VERSION: u32 = 1; +pub const AD_EXACT_SURFACE_INFO_SIZE: usize = 40; + +/// Additive surface observation that preserves the core surface ID. +#[repr(C)] +pub struct AdExactSurfaceInfo { + pub version: u32, + pub size: u32, + pub id: *const c_char, + pub surface: AdSurfaceInfo, +} + +const _: () = assert!(std::mem::size_of::<AdExactSurfaceInfo>() == AD_EXACT_SURFACE_INFO_SIZE); + +#[unsafe(no_mangle)] +pub extern "C" fn ad_exact_surface_info_size() -> usize { + std::mem::size_of::<AdExactSurfaceInfo>() +} diff --git a/crates/ffi/src/types/exact_surface_list.rs b/crates/ffi/src/types/exact_surface_list.rs new file mode 100644 index 0000000..721ef5a --- /dev/null +++ b/crates/ffi/src/types/exact_surface_list.rs @@ -0,0 +1,6 @@ +use crate::types::AdExactSurfaceInfo; + +/// Opaque list handle emitted by `ad_list_surfaces_exact`. +pub struct AdExactSurfaceList { + pub(crate) items: Box<[AdExactSurfaceInfo]>, +} diff --git a/crates/ffi/src/types/exact_window_info.rs b/crates/ffi/src/types/exact_window_info.rs new file mode 100644 index 0000000..274a15a --- /dev/null +++ b/crates/ffi/src/types/exact_window_info.rs @@ -0,0 +1,23 @@ +use std::os::raw::c_char; + +use crate::types::AdWindowInfo; + +pub const AD_EXACT_WINDOW_INFO_VERSION: u32 = 1; +pub const AD_EXACT_WINDOW_INFO_SIZE: usize = 88; + +/// Additive generation-pinned window identity for operations that target a +/// previously observed live window. +#[repr(C)] +pub struct AdExactWindowInfo { + pub version: u32, + pub size: u32, + pub window: AdWindowInfo, + pub process_instance: *const c_char, +} + +const _: () = assert!(std::mem::size_of::<AdExactWindowInfo>() == AD_EXACT_WINDOW_INFO_SIZE); + +#[unsafe(no_mangle)] +pub extern "C" fn ad_exact_window_info_size() -> usize { + std::mem::size_of::<AdExactWindowInfo>() +} diff --git a/crates/ffi/src/types/exact_window_list.rs b/crates/ffi/src/types/exact_window_list.rs new file mode 100644 index 0000000..adf2b03 --- /dev/null +++ b/crates/ffi/src/types/exact_window_list.rs @@ -0,0 +1,6 @@ +use crate::types::AdExactWindowInfo; + +/// Opaque list handle emitted by ad_list_windows_exact. +pub struct AdExactWindowList { + pub(crate) items: Box<[AdExactWindowInfo]>, +} diff --git a/crates/ffi/src/types/find_control.rs b/crates/ffi/src/types/find_control.rs new file mode 100644 index 0000000..a6a3078 --- /dev/null +++ b/crates/ffi/src/types/find_control.rs @@ -0,0 +1,12 @@ +use crate::types::AdFindSelection; + +#[repr(C)] +pub struct AdFindControl { + pub version: u32, + pub selection: AdFindSelection, + pub timeout_ms: u64, +} + +pub const AD_FIND_CONTROL_SIZE: usize = 24; + +const _: () = assert!(std::mem::size_of::<AdFindControl>() == AD_FIND_CONTROL_SIZE); diff --git a/crates/ffi/src/types/find_filter.rs b/crates/ffi/src/types/find_filter.rs new file mode 100644 index 0000000..f6d2f10 --- /dev/null +++ b/crates/ffi/src/types/find_filter.rs @@ -0,0 +1,16 @@ +use crate::types::{AdFindIdentity, AdFindQuery, AdFindStateSlice}; +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdFindFilter { + pub identity: AdFindIdentity, + pub has_text: *const c_char, + pub states: AdFindStateSlice, + pub has: *const AdFindQuery, + pub has_not: *const AdFindQuery, + pub exact: bool, +} + +pub const AD_FIND_FILTER_SIZE: usize = 88; + +const _: () = assert!(std::mem::size_of::<AdFindFilter>() == AD_FIND_FILTER_SIZE); diff --git a/crates/ffi/src/types/find_identity.rs b/crates/ffi/src/types/find_identity.rs new file mode 100644 index 0000000..abe1b9c --- /dev/null +++ b/crates/ffi/src/types/find_identity.rs @@ -0,0 +1,14 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdFindIdentity { + pub role: *const c_char, + pub name: *const c_char, + pub description: *const c_char, + pub native_id: *const c_char, + pub value: *const c_char, +} + +pub const AD_FIND_IDENTITY_SIZE: usize = 40; + +const _: () = assert!(std::mem::size_of::<AdFindIdentity>() == AD_FIND_IDENTITY_SIZE); diff --git a/crates/ffi/src/types/find_query.rs b/crates/ffi/src/types/find_query.rs index 6f80efb..a4f2fe1 100644 --- a/crates/ffi/src/types/find_query.rs +++ b/crates/ffi/src/types/find_query.rs @@ -1,8 +1,12 @@ -use std::os::raw::c_char; +use crate::types::{AdFindControl, AdFindFilter}; #[repr(C)] pub struct AdFindQuery { - pub role: *const c_char, - pub name_substring: *const c_char, - pub value_substring: *const c_char, + pub control: AdFindControl, + pub filter: AdFindFilter, } + +pub const AD_FIND_QUERY_VERSION: u32 = 1; +pub const AD_FIND_QUERY_SIZE: usize = 112; + +const _: () = assert!(std::mem::size_of::<AdFindQuery>() == AD_FIND_QUERY_SIZE); diff --git a/crates/ffi/src/types/find_selection.rs b/crates/ffi/src/types/find_selection.rs new file mode 100644 index 0000000..5cf263d --- /dev/null +++ b/crates/ffi/src/types/find_selection.rs @@ -0,0 +1,9 @@ +#[repr(C)] +pub struct AdFindSelection { + pub kind: i32, + pub nth: u32, +} + +pub const AD_FIND_SELECTION_SIZE: usize = 8; + +const _: () = assert!(std::mem::size_of::<AdFindSelection>() == AD_FIND_SELECTION_SIZE); diff --git a/crates/ffi/src/types/find_selection_kind.rs b/crates/ffi/src/types/find_selection_kind.rs new file mode 100644 index 0000000..2c2468f --- /dev/null +++ b/crates/ffi/src/types/find_selection_kind.rs @@ -0,0 +1,20 @@ +#[repr(i32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdFindSelectionKind { + Strict = 0, + First = 1, + Last = 2, + Nth = 3, +} + +impl AdFindSelectionKind { + pub(crate) fn from_c(value: i32) -> Option<Self> { + match value { + 0 => Some(Self::Strict), + 1 => Some(Self::First), + 2 => Some(Self::Last), + 3 => Some(Self::Nth), + _ => None, + } + } +} diff --git a/crates/ffi/src/types/find_state_predicate.rs b/crates/ffi/src/types/find_state_predicate.rs new file mode 100644 index 0000000..672ac11 --- /dev/null +++ b/crates/ffi/src/types/find_state_predicate.rs @@ -0,0 +1,11 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdFindStatePredicate { + pub token: *const c_char, + pub expected: i32, +} + +pub const AD_FIND_STATE_PREDICATE_SIZE: usize = 16; + +const _: () = assert!(std::mem::size_of::<AdFindStatePredicate>() == AD_FIND_STATE_PREDICATE_SIZE); diff --git a/crates/ffi/src/types/find_state_slice.rs b/crates/ffi/src/types/find_state_slice.rs new file mode 100644 index 0000000..0ccf513 --- /dev/null +++ b/crates/ffi/src/types/find_state_slice.rs @@ -0,0 +1,11 @@ +use crate::types::AdFindStatePredicate; + +#[repr(C)] +pub struct AdFindStateSlice { + pub items: *const AdFindStatePredicate, + pub count: usize, +} + +pub const AD_FIND_STATE_SLICE_SIZE: usize = 16; + +const _: () = assert!(std::mem::size_of::<AdFindStateSlice>() == AD_FIND_STATE_SLICE_SIZE); diff --git a/crates/ffi/src/types/identifier_kind.rs b/crates/ffi/src/types/identifier_kind.rs new file mode 100644 index 0000000..de02fa5 --- /dev/null +++ b/crates/ffi/src/types/identifier_kind.rs @@ -0,0 +1,32 @@ +#[repr(i32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdIdentifierKind { + AxIdentifier = 0, + AxDomIdentifier = 1, + AutomationId = 2, + RuntimeId = 3, + AtspiObjectPath = 4, +} + +impl AdIdentifierKind { + pub(crate) fn from_c(value: i32) -> Option<Self> { + match value { + 0 => Some(Self::AxIdentifier), + 1 => Some(Self::AxDomIdentifier), + 2 => Some(Self::AutomationId), + 3 => Some(Self::RuntimeId), + 4 => Some(Self::AtspiObjectPath), + _ => None, + } + } + + pub(crate) fn to_core(self) -> agent_desktop_core::IdentifierKind { + match self { + Self::AxIdentifier => agent_desktop_core::IdentifierKind::AxIdentifier, + Self::AxDomIdentifier => agent_desktop_core::IdentifierKind::AxDomIdentifier, + Self::AutomationId => agent_desktop_core::IdentifierKind::AutomationId, + Self::RuntimeId => agent_desktop_core::IdentifierKind::RuntimeId, + Self::AtspiObjectPath => agent_desktop_core::IdentifierKind::AtspiObjectPath, + } + } +} diff --git a/crates/ffi/src/types/image_buffer.rs b/crates/ffi/src/types/image_buffer.rs index f225a89..0c66c7b 100644 --- a/crates/ffi/src/types/image_buffer.rs +++ b/crates/ffi/src/types/image_buffer.rs @@ -10,4 +10,5 @@ pub struct AdImageBuffer { pub(crate) width: u32, pub(crate) height: u32, pub(crate) format: AdImageFormat, + pub(crate) scale_factor: f64, } diff --git a/crates/ffi/src/types/mod.rs b/crates/ffi/src/types/mod.rs index 515c76f..1eb5dc9 100644 --- a/crates/ffi/src/types/mod.rs +++ b/crates/ffi/src/types/mod.rs @@ -1,79 +1,149 @@ -pub mod action; -pub mod action_kind; -pub mod action_result; -pub mod action_step; -pub mod app_info; -pub mod app_list; -pub mod direction; -pub mod drag_params; -pub mod element_state; -pub mod find_query; -pub mod image_buffer; -pub mod image_format; -pub mod key_combo; -pub mod modifier; -pub mod mouse_button; -pub mod mouse_event; -pub mod mouse_event_kind; -pub mod native_handle; -pub mod node; -pub mod node_tree; -pub mod notification_filter; -pub mod notification_info; -pub mod notification_list; -pub mod point; -pub mod policy_kind; -pub mod rect; -pub mod ref_entry; -pub mod screenshot_kind; -pub mod screenshot_target; -pub mod scroll_params; -pub mod snapshot_surface; -pub mod surface_info; -pub mod surface_list; -pub mod tree_options; -pub mod wait_args; -pub mod window_info; -pub mod window_list; -pub mod window_op; -pub mod window_op_kind; +pub(crate) mod action; +pub(crate) mod action_kind; +pub(crate) mod action_result; +pub(crate) mod action_step; +pub(crate) mod app_info; +pub(crate) mod app_list; +pub(crate) mod delivery_disposition; +pub(crate) mod delivery_semantics; +pub(crate) mod direction; +pub(crate) mod display_info; +pub(crate) mod display_list; +pub(crate) mod drag_params; +pub(crate) mod element_state; +pub(crate) mod exact_ref_entry; +pub(crate) mod exact_surface_info; +pub(crate) mod exact_surface_list; +pub(crate) mod exact_window_info; +pub(crate) mod exact_window_list; +pub(crate) mod find_control; +pub(crate) mod find_filter; +pub(crate) mod find_identity; +pub(crate) mod find_query; +pub(crate) mod find_selection; +pub(crate) mod find_selection_kind; +pub(crate) mod find_state_predicate; +pub(crate) mod find_state_slice; +pub(crate) mod identifier_kind; +pub(crate) mod image_buffer; +pub(crate) mod image_format; +pub(crate) mod key_combo; +pub(crate) mod modifier; +pub(crate) mod mouse_button; +pub(crate) mod mouse_event; +pub(crate) mod mouse_event_kind; +pub(crate) mod native_handle; +pub(crate) mod node; +pub(crate) mod node_content; +pub(crate) mod node_presentation; +pub(crate) mod node_relation; +pub(crate) mod node_tree; +pub(crate) mod notification_action_request; +pub(crate) mod notification_filter; +pub(crate) mod notification_identity; +pub(crate) mod notification_info; +pub(crate) mod notification_list; +pub(crate) mod optional_u64; +pub(crate) mod optional_usize; +pub(crate) mod point; +pub(crate) mod policy_kind; +pub(crate) mod rect; +pub(crate) mod ref_capabilities; +pub(crate) mod ref_entry; +pub(crate) mod ref_geometry; +pub(crate) mod ref_identity; +pub(crate) mod ref_process; +pub(crate) mod ref_scope; +pub(crate) mod ref_source; +pub(crate) mod retry_disposition; +pub(crate) mod screenshot_kind; +pub(crate) mod screenshot_target; +pub(crate) mod scroll_params; +pub(crate) mod snapshot_surface; +pub(crate) mod step_mechanism; +pub(crate) mod string_slice; +pub(crate) mod surface_info; +pub(crate) mod surface_list; +pub(crate) mod tree_options; +pub(crate) mod wait_args; +pub(crate) mod wait_mode; +pub(crate) mod wait_predicate; +pub(crate) mod wait_scope; +pub(crate) mod wait_surface_modes; +pub(crate) mod window_info; +pub(crate) mod window_list; +pub(crate) mod window_op; +pub(crate) mod window_op_kind; -pub use action::AdAction; -pub use action_kind::AdActionKind; -pub use action_result::AdActionResult; -pub use action_step::AdActionStep; -pub use app_info::AdAppInfo; -pub use app_list::AdAppList; -pub use direction::AdDirection; -pub use drag_params::AdDragParams; -pub use element_state::AdElementState; -pub use find_query::AdFindQuery; -pub use image_buffer::AdImageBuffer; -pub use image_format::AdImageFormat; -pub use key_combo::AdKeyCombo; -pub use modifier::AdModifier; -pub use mouse_button::AdMouseButton; -pub use mouse_event::AdMouseEvent; -pub use mouse_event_kind::AdMouseEventKind; -pub use native_handle::AdNativeHandle; -pub use node::AdNode; -pub use node_tree::AdNodeTree; -pub use notification_filter::AdNotificationFilter; -pub use notification_info::AdNotificationInfo; -pub use notification_list::AdNotificationList; -pub use point::AdPoint; -pub use policy_kind::AdPolicyKind; -pub use rect::AdRect; -pub use ref_entry::AdRefEntry; -pub use screenshot_kind::AdScreenshotKind; -pub use screenshot_target::AdScreenshotTarget; -pub use scroll_params::AdScrollParams; -pub use snapshot_surface::AdSnapshotSurface; -pub use surface_info::AdSurfaceInfo; -pub use surface_list::AdSurfaceList; -pub use tree_options::AdTreeOptions; -pub use wait_args::AdWaitArgs; -pub use window_info::AdWindowInfo; -pub use window_list::AdWindowList; -pub use window_op::AdWindowOp; -pub use window_op_kind::AdWindowOpKind; +pub(crate) use action::AdAction; +pub(crate) use action_kind::AdActionKind; +pub(crate) use action_result::AdActionResult; +pub(crate) use app_info::AdAppInfo; +pub(crate) use app_list::AdAppList; +pub(crate) use delivery_disposition::AdDeliveryDisposition; +pub(crate) use delivery_semantics::AdDeliverySemantics; +pub(crate) use direction::AdDirection; +pub(crate) use display_info::AdDisplayInfo; +pub(crate) use display_list::AdDisplayList; +pub(crate) use drag_params::AdDragParams; +pub(crate) use element_state::AdElementState; +pub(crate) use exact_ref_entry::AdExactRefEntry; +pub(crate) use exact_surface_info::AdExactSurfaceInfo; +pub(crate) use exact_surface_list::AdExactSurfaceList; +pub(crate) use exact_window_info::AdExactWindowInfo; +pub(crate) use exact_window_list::AdExactWindowList; +pub(crate) use find_control::AdFindControl; +pub(crate) use find_filter::AdFindFilter; +pub(crate) use find_identity::AdFindIdentity; +pub(crate) use find_query::AdFindQuery; +pub(crate) use find_selection::AdFindSelection; +pub(crate) use find_selection_kind::AdFindSelectionKind; +pub(crate) use find_state_predicate::AdFindStatePredicate; +pub(crate) use find_state_slice::AdFindStateSlice; +pub(crate) use identifier_kind::AdIdentifierKind; +pub(crate) use image_buffer::AdImageBuffer; +pub(crate) use image_format::AdImageFormat; +pub(crate) use key_combo::AdKeyCombo; +pub(crate) use modifier::AdModifier; +pub(crate) use mouse_button::AdMouseButton; +pub(crate) use mouse_event::AdMouseEvent; +pub(crate) use mouse_event_kind::AdMouseEventKind; +pub(crate) use native_handle::AdNativeHandle; +pub(crate) use node::AdNode; +pub(crate) use node_content::AdNodeContent; +pub(crate) use node_presentation::AdNodePresentation; +pub(crate) use node_relation::AdNodeRelation; +pub(crate) use node_tree::AdNodeTree; +pub(crate) use notification_action_request::AdNotificationActionRequest; +pub(crate) use notification_filter::AdNotificationFilter; +pub(crate) use notification_identity::AdNotificationIdentity; +pub(crate) use notification_info::AdNotificationInfo; +pub(crate) use notification_list::AdNotificationList; +pub(crate) use optional_u64::AdOptionalU64; +pub(crate) use optional_usize::AdOptionalUsize; +pub(crate) use point::AdPoint; +pub(crate) use policy_kind::AdPolicyKind; +pub(crate) use rect::AdRect; +pub(crate) use ref_capabilities::AdRefCapabilities; +pub(crate) use ref_entry::AdRefEntry; +pub(crate) use ref_geometry::AdRefGeometry; +pub(crate) use ref_identity::AdRefIdentity; +pub(crate) use ref_process::AdRefProcess; +pub(crate) use ref_scope::AdRefScope; +pub(crate) use ref_source::AdRefSource; +pub(crate) use retry_disposition::AdRetryDisposition; +pub(crate) use screenshot_kind::AdScreenshotKind; +pub(crate) use screenshot_target::AdScreenshotTarget; +pub(crate) use snapshot_surface::AdSnapshotSurface; +pub(crate) use string_slice::AdStringSlice; +pub(crate) use surface_info::AdSurfaceInfo; +pub(crate) use surface_list::AdSurfaceList; +pub(crate) use tree_options::AdTreeOptions; +pub(crate) use wait_mode::AdWaitMode; +pub(crate) use wait_predicate::AdWaitPredicate; +pub(crate) use wait_scope::AdWaitScope; +pub(crate) use wait_surface_modes::AdWaitSurfaceModes; +pub(crate) use window_info::AdWindowInfo; +pub(crate) use window_list::AdWindowList; +pub(crate) use window_op::AdWindowOp; +pub(crate) use window_op_kind::AdWindowOpKind; diff --git a/crates/ffi/src/types/modifier.rs b/crates/ffi/src/types/modifier.rs index c62fdf5..96c53f7 100644 --- a/crates/ffi/src/types/modifier.rs +++ b/crates/ffi/src/types/modifier.rs @@ -1,8 +1,10 @@ #[repr(i32)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AdModifier { - Cmd = 0, + Meta = 0, Ctrl = 1, Alt = 2, Shift = 3, } + +pub const AD_MODIFIER_CMD: i32 = 0; diff --git a/crates/ffi/src/types/native_handle.rs b/crates/ffi/src/types/native_handle.rs index 95b7873..a7b562f 100644 --- a/crates/ffi/src/types/native_handle.rs +++ b/crates/ffi/src/types/native_handle.rs @@ -1,4 +1,5 @@ #[repr(C)] pub struct AdNativeHandle { + /// Opaque thread-affine registry token, never an allocation or OS pointer. pub ptr: *const std::ffi::c_void, } diff --git a/crates/ffi/src/types/node.rs b/crates/ffi/src/types/node.rs index a2f7d21..5dec65b 100644 --- a/crates/ffi/src/types/node.rs +++ b/crates/ffi/src/types/node.rs @@ -1,19 +1,12 @@ -use crate::types::rect::AdRect; -use std::os::raw::c_char; +use crate::types::{AdNodeContent, AdNodePresentation, AdNodeRelation}; #[repr(C)] pub struct AdNode { - pub ref_id: *const c_char, - pub role: *const c_char, - pub name: *const c_char, - pub value: *const c_char, - pub description: *const c_char, - pub hint: *const c_char, - pub states: *mut *mut c_char, - pub state_count: u32, - pub bounds: AdRect, - pub has_bounds: bool, - pub parent_index: i32, - pub child_start: u32, - pub child_count: u32, + pub content: AdNodeContent, + pub presentation: AdNodePresentation, + pub relation: AdNodeRelation, } + +pub const AD_NODE_SIZE: usize = 112; + +const _: () = assert!(std::mem::size_of::<AdNode>() == AD_NODE_SIZE); diff --git a/crates/ffi/src/types/node_content.rs b/crates/ffi/src/types/node_content.rs new file mode 100644 index 0000000..a119782 --- /dev/null +++ b/crates/ffi/src/types/node_content.rs @@ -0,0 +1,15 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdNodeContent { + pub ref_id: *const c_char, + pub role: *const c_char, + pub name: *const c_char, + pub value: *const c_char, + pub description: *const c_char, + pub hint: *const c_char, +} + +pub const AD_NODE_CONTENT_SIZE: usize = 48; + +const _: () = assert!(std::mem::size_of::<AdNodeContent>() == AD_NODE_CONTENT_SIZE); diff --git a/crates/ffi/src/types/node_presentation.rs b/crates/ffi/src/types/node_presentation.rs new file mode 100644 index 0000000..ed860b5 --- /dev/null +++ b/crates/ffi/src/types/node_presentation.rs @@ -0,0 +1,14 @@ +use crate::types::AdRect; +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdNodePresentation { + pub states: *mut *mut c_char, + pub bounds: AdRect, + pub state_count: u32, + pub has_bounds: bool, +} + +pub const AD_NODE_PRESENTATION_SIZE: usize = 48; + +const _: () = assert!(std::mem::size_of::<AdNodePresentation>() == AD_NODE_PRESENTATION_SIZE); diff --git a/crates/ffi/src/types/node_relation.rs b/crates/ffi/src/types/node_relation.rs new file mode 100644 index 0000000..e1d3e3f --- /dev/null +++ b/crates/ffi/src/types/node_relation.rs @@ -0,0 +1,10 @@ +#[repr(C)] +pub struct AdNodeRelation { + pub parent_index: i32, + pub child_start: u32, + pub child_count: u32, +} + +pub const AD_NODE_RELATION_SIZE: usize = 12; + +const _: () = assert!(std::mem::size_of::<AdNodeRelation>() == AD_NODE_RELATION_SIZE); diff --git a/crates/ffi/src/types/notification_action_request.rs b/crates/ffi/src/types/notification_action_request.rs new file mode 100644 index 0000000..2fd0416 --- /dev/null +++ b/crates/ffi/src/types/notification_action_request.rs @@ -0,0 +1,16 @@ +use crate::types::AdNotificationIdentity; +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdNotificationActionRequest { + pub index: u32, + pub policy: i32, + pub action_name: *const c_char, + pub identity: AdNotificationIdentity, +} + +pub const AD_NOTIFICATION_ACTION_REQUEST_SIZE: usize = 32; + +const _: () = assert!( + std::mem::size_of::<AdNotificationActionRequest>() == AD_NOTIFICATION_ACTION_REQUEST_SIZE +); diff --git a/crates/ffi/src/types/notification_identity.rs b/crates/ffi/src/types/notification_identity.rs new file mode 100644 index 0000000..06a68c6 --- /dev/null +++ b/crates/ffi/src/types/notification_identity.rs @@ -0,0 +1,12 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdNotificationIdentity { + pub app: *const c_char, + pub title: *const c_char, +} + +pub const AD_NOTIFICATION_IDENTITY_SIZE: usize = 16; + +const _: () = + assert!(std::mem::size_of::<AdNotificationIdentity>() == AD_NOTIFICATION_IDENTITY_SIZE); diff --git a/crates/ffi/src/types/optional_u64.rs b/crates/ffi/src/types/optional_u64.rs new file mode 100644 index 0000000..162eef2 --- /dev/null +++ b/crates/ffi/src/types/optional_u64.rs @@ -0,0 +1,9 @@ +#[repr(C)] +pub struct AdOptionalU64 { + pub value: u64, + pub present: bool, +} + +pub const AD_OPTIONAL_U64_SIZE: usize = 16; + +const _: () = assert!(std::mem::size_of::<AdOptionalU64>() == AD_OPTIONAL_U64_SIZE); diff --git a/crates/ffi/src/types/optional_usize.rs b/crates/ffi/src/types/optional_usize.rs new file mode 100644 index 0000000..c94823f --- /dev/null +++ b/crates/ffi/src/types/optional_usize.rs @@ -0,0 +1,9 @@ +#[repr(C)] +pub struct AdOptionalUsize { + pub value: usize, + pub present: bool, +} + +pub const AD_OPTIONAL_USIZE_SIZE: usize = 16; + +const _: () = assert!(std::mem::size_of::<AdOptionalUsize>() == AD_OPTIONAL_USIZE_SIZE); diff --git a/crates/ffi/src/types/ref_capabilities.rs b/crates/ffi/src/types/ref_capabilities.rs new file mode 100644 index 0000000..5321581 --- /dev/null +++ b/crates/ffi/src/types/ref_capabilities.rs @@ -0,0 +1,11 @@ +use crate::types::AdStringSlice; + +#[repr(C)] +pub struct AdRefCapabilities { + pub states: AdStringSlice, + pub available_actions: AdStringSlice, +} + +pub const AD_REF_CAPABILITIES_SIZE: usize = 32; + +const _: () = assert!(std::mem::size_of::<AdRefCapabilities>() == AD_REF_CAPABILITIES_SIZE); diff --git a/crates/ffi/src/types/ref_entry.rs b/crates/ffi/src/types/ref_entry.rs index 6d9bb4d..b9ead73 100644 --- a/crates/ffi/src/types/ref_entry.rs +++ b/crates/ffi/src/types/ref_entry.rs @@ -1,32 +1,18 @@ -use crate::types::AdRect; -use std::os::raw::c_char; +use crate::types::{ + AdRefCapabilities, AdRefGeometry, AdRefIdentity, AdRefProcess, AdRefScope, AdRefSource, +}; #[repr(C)] pub struct AdRefEntry { - pub pid: i32, - pub role: *const c_char, - pub name: *const c_char, - pub value: *const c_char, - pub description: *const c_char, - pub states: *const *const c_char, - pub state_count: usize, - pub available_actions: *const *const c_char, - pub available_action_count: usize, - pub bounds: AdRect, - pub has_bounds: bool, - pub bounds_hash: u64, - pub has_bounds_hash: bool, - pub source_app: *const c_char, - pub source_window_id: *const c_char, - pub source_window_title: *const c_char, - pub source_surface: i32, - pub root_ref: *const c_char, - pub path_is_absolute: bool, - pub path: *const u32, - pub path_count: usize, + pub process: AdRefProcess, + pub identity: AdRefIdentity, + pub geometry: AdRefGeometry, + pub capabilities: AdRefCapabilities, + pub source: AdRefSource, + pub scope: AdRefScope, } -pub const AD_REF_ENTRY_SIZE: usize = 192; +pub const AD_REF_ENTRY_SIZE: usize = 200; /// Per-field input caps enforced when converting an `AdRefEntry` at the C /// boundary, sized from what real accessibility trees produce (a handful of diff --git a/crates/ffi/src/types/ref_geometry.rs b/crates/ffi/src/types/ref_geometry.rs new file mode 100644 index 0000000..67afcac --- /dev/null +++ b/crates/ffi/src/types/ref_geometry.rs @@ -0,0 +1,13 @@ +use crate::types::AdRect; + +#[repr(C)] +pub struct AdRefGeometry { + pub bounds: AdRect, + pub bounds_hash: u64, + pub has_bounds: bool, + pub has_bounds_hash: bool, +} + +pub const AD_REF_GEOMETRY_SIZE: usize = 48; + +const _: () = assert!(std::mem::size_of::<AdRefGeometry>() == AD_REF_GEOMETRY_SIZE); diff --git a/crates/ffi/src/types/ref_identity.rs b/crates/ffi/src/types/ref_identity.rs new file mode 100644 index 0000000..4461a34 --- /dev/null +++ b/crates/ffi/src/types/ref_identity.rs @@ -0,0 +1,14 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdRefIdentity { + pub role: *const c_char, + pub name: *const c_char, + pub value: *const c_char, + pub description: *const c_char, + pub native_id: *const c_char, +} + +pub const AD_REF_IDENTITY_SIZE: usize = 40; + +const _: () = assert!(std::mem::size_of::<AdRefIdentity>() == AD_REF_IDENTITY_SIZE); diff --git a/crates/ffi/src/types/ref_process.rs b/crates/ffi/src/types/ref_process.rs new file mode 100644 index 0000000..7074f3c --- /dev/null +++ b/crates/ffi/src/types/ref_process.rs @@ -0,0 +1,8 @@ +#[repr(C)] +pub struct AdRefProcess { + pub pid: u32, +} + +pub const AD_REF_PROCESS_SIZE: usize = 4; + +const _: () = assert!(std::mem::size_of::<AdRefProcess>() == AD_REF_PROCESS_SIZE); diff --git a/crates/ffi/src/types/ref_scope.rs b/crates/ffi/src/types/ref_scope.rs new file mode 100644 index 0000000..2352ed1 --- /dev/null +++ b/crates/ffi/src/types/ref_scope.rs @@ -0,0 +1,13 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdRefScope { + pub root_ref: *const c_char, + pub path: *const u32, + pub path_count: usize, + pub path_is_absolute: bool, +} + +pub const AD_REF_SCOPE_SIZE: usize = 32; + +const _: () = assert!(std::mem::size_of::<AdRefScope>() == AD_REF_SCOPE_SIZE); diff --git a/crates/ffi/src/types/ref_source.rs b/crates/ffi/src/types/ref_source.rs new file mode 100644 index 0000000..e6d7465 --- /dev/null +++ b/crates/ffi/src/types/ref_source.rs @@ -0,0 +1,15 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdRefSource { + pub app: *const c_char, + pub window_id: *const c_char, + pub window_title: *const c_char, + pub window_bounds_hash: u64, + pub surface: i32, + pub has_window_bounds_hash: bool, +} + +pub const AD_REF_SOURCE_SIZE: usize = 40; + +const _: () = assert!(std::mem::size_of::<AdRefSource>() == AD_REF_SOURCE_SIZE); diff --git a/crates/ffi/src/types/retry_disposition.rs b/crates/ffi/src/types/retry_disposition.rs new file mode 100644 index 0000000..81975b5 --- /dev/null +++ b/crates/ffi/src/types/retry_disposition.rs @@ -0,0 +1,17 @@ +#[repr(i32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdRetryDisposition { + Unknown = 0, + Safe = 1, + Unsafe = 2, +} + +impl From<agent_desktop_core::RetryDisposition> for AdRetryDisposition { + fn from(value: agent_desktop_core::RetryDisposition) -> Self { + match value { + agent_desktop_core::RetryDisposition::Unknown => Self::Unknown, + agent_desktop_core::RetryDisposition::Safe => Self::Safe, + agent_desktop_core::RetryDisposition::Unsafe => Self::Unsafe, + } + } +} diff --git a/crates/ffi/src/types/screenshot_target.rs b/crates/ffi/src/types/screenshot_target.rs index 580e455..c0ac497 100644 --- a/crates/ffi/src/types/screenshot_target.rs +++ b/crates/ffi/src/types/screenshot_target.rs @@ -8,5 +8,5 @@ pub struct AdScreenshotTarget { pub kind: i32, pub screen_index: u64, - pub pid: i32, + pub pid: u32, } diff --git a/crates/ffi/src/types/snapshot_surface.rs b/crates/ffi/src/types/snapshot_surface.rs index d925045..16aef09 100644 --- a/crates/ffi/src/types/snapshot_surface.rs +++ b/crates/ffi/src/types/snapshot_surface.rs @@ -8,4 +8,16 @@ pub enum AdSnapshotSurface { Sheet = 4, Popover = 5, Alert = 6, + Desktop = 7, + Taskbar = 8, + SystemTray = 9, + QuickSettings = 10, + NotificationCenter = 11, + Toolbar = 12, + Dock = 13, + Spotlight = 14, + MenuBarExtras = 15, + SystemTrayOverflow = 16, + StartMenu = 17, + ActionCenter = 18, } diff --git a/crates/ffi/src/types/step_mechanism.rs b/crates/ffi/src/types/step_mechanism.rs new file mode 100644 index 0000000..40ee9b4 --- /dev/null +++ b/crates/ffi/src/types/step_mechanism.rs @@ -0,0 +1,17 @@ +#[repr(i32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdStepMechanism { + SemanticApi = 1, + PhysicalSynthetic = 2, +} + +#[cfg(test)] +mod tests { + use super::AdStepMechanism; + + #[test] + fn discriminants_are_abi_stable() { + assert_eq!(AdStepMechanism::SemanticApi as i32, 1); + assert_eq!(AdStepMechanism::PhysicalSynthetic as i32, 2); + } +} diff --git a/crates/ffi/src/types/string_slice.rs b/crates/ffi/src/types/string_slice.rs new file mode 100644 index 0000000..df5d227 --- /dev/null +++ b/crates/ffi/src/types/string_slice.rs @@ -0,0 +1,11 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdStringSlice { + pub items: *const *const c_char, + pub count: usize, +} + +pub const AD_STRING_SLICE_SIZE: usize = 16; + +const _: () = assert!(std::mem::size_of::<AdStringSlice>() == AD_STRING_SLICE_SIZE); diff --git a/crates/ffi/src/types/wait_args.rs b/crates/ffi/src/types/wait_args.rs index ef925c6..fbc5868 100644 --- a/crates/ffi/src/types/wait_args.rs +++ b/crates/ffi/src/types/wait_args.rs @@ -1,57 +1,24 @@ -use std::os::raw::c_char; +use crate::types::{AdWaitMode, AdWaitPredicate, AdWaitScope}; -/// Arguments for `ad_wait`, mirroring `core::commands::wait::WaitArgs`. +/// Arguments for `ad_wait`, mirroring `core::commands::wait::WaitArgs` for +/// the pause/element/text/surface wait modes and predicates. /// -/// Fields map as follows: -/// - `Option<u64>` → `u64` value + `bool has_*` sentinel (ms, count). -/// - `Option<String>` → nullable `*const c_char` (null = absent). -/// - `bool` → `bool`. +/// The core event-wait mode (`--event` / `--window-id`) is intentionally not +/// exposed over FFI in this release; `wait_args_from_ffi` always forwards +/// `event: None` and `window_id: None` to core. `mode.window` here is a +/// title-appearance wait (poll until a window with the given title exists), +/// which is a distinct semantic from the deferred event-wait mode. +/// +/// Mode, predicate, and scope fields are grouped into named PODs. Optional +/// numbers use `AdOptional*`; optional strings are nullable pointers. /// /// Callers must zero-initialize before use and verify layout via /// `AD_WAIT_ARGS_SIZE` / `ad_wait_args_size()`. #[repr(C)] pub struct AdWaitArgs { - /// Milliseconds to sleep (WaitMode::ms). - pub ms: u64, - pub has_ms: bool, - - /// Element ref id to wait for (WaitMode::element). - pub element: *const c_char, - - /// Window title to wait for (WaitMode::window). - pub window: *const c_char, - - /// Text to wait for (WaitMode::text / WaitMode::notification text). - pub text: *const c_char, - - /// Wait for menu to open (true) or close (false via menu_closed). - pub menu: bool, - /// Wait for menu to close. - pub menu_closed: bool, - /// Wait for a notification. - pub notification: bool, - - /// Snapshot id for element predicate (WaitPredicateArgs::snapshot_id). - pub snapshot_id: *const c_char, - - /// Predicate kind string (WaitPredicateArgs::predicate). - pub predicate: *const c_char, - - /// Expected value for value-predicate (WaitPredicateArgs::value). - pub value: *const c_char, - - /// Action name for actionability-predicate (WaitPredicateArgs::action). - pub action: *const c_char, - - /// Expected match count for text waits (WaitPredicateArgs::count). - pub count: usize, - pub has_count: bool, - - /// Timeout in milliseconds. - pub timeout_ms: u64, - - /// App name filter (null = any). Maps to WaitArgs::app. - pub app: *const c_char, + pub mode: AdWaitMode, + pub predicate: AdWaitPredicate, + pub scope: AdWaitScope, } /// Pinned size of `AdWaitArgs` on 64-bit targets. The compile-time diff --git a/crates/ffi/src/types/wait_mode.rs b/crates/ffi/src/types/wait_mode.rs new file mode 100644 index 0000000..68ace62 --- /dev/null +++ b/crates/ffi/src/types/wait_mode.rs @@ -0,0 +1,15 @@ +use crate::types::{AdOptionalU64, AdWaitSurfaceModes}; +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdWaitMode { + pub pause: AdOptionalU64, + pub element: *const c_char, + pub window: *const c_char, + pub text: *const c_char, + pub surfaces: AdWaitSurfaceModes, +} + +pub const AD_WAIT_MODE_SIZE: usize = 48; + +const _: () = assert!(std::mem::size_of::<AdWaitMode>() == AD_WAIT_MODE_SIZE); diff --git a/crates/ffi/src/types/wait_predicate.rs b/crates/ffi/src/types/wait_predicate.rs new file mode 100644 index 0000000..7215452 --- /dev/null +++ b/crates/ffi/src/types/wait_predicate.rs @@ -0,0 +1,15 @@ +use crate::types::AdOptionalUsize; +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdWaitPredicate { + pub snapshot_id: *const c_char, + pub predicate: *const c_char, + pub value: *const c_char, + pub action: *const c_char, + pub count: AdOptionalUsize, +} + +pub const AD_WAIT_PREDICATE_SIZE: usize = 48; + +const _: () = assert!(std::mem::size_of::<AdWaitPredicate>() == AD_WAIT_PREDICATE_SIZE); diff --git a/crates/ffi/src/types/wait_scope.rs b/crates/ffi/src/types/wait_scope.rs new file mode 100644 index 0000000..5a37422 --- /dev/null +++ b/crates/ffi/src/types/wait_scope.rs @@ -0,0 +1,11 @@ +use std::os::raw::c_char; + +#[repr(C)] +pub struct AdWaitScope { + pub timeout_ms: u64, + pub app: *const c_char, +} + +pub const AD_WAIT_SCOPE_SIZE: usize = 16; + +const _: () = assert!(std::mem::size_of::<AdWaitScope>() == AD_WAIT_SCOPE_SIZE); diff --git a/crates/ffi/src/types/wait_surface_modes.rs b/crates/ffi/src/types/wait_surface_modes.rs new file mode 100644 index 0000000..48ead51 --- /dev/null +++ b/crates/ffi/src/types/wait_surface_modes.rs @@ -0,0 +1,10 @@ +#[repr(C)] +pub struct AdWaitSurfaceModes { + pub menu: bool, + pub menu_closed: bool, + pub notification: bool, +} + +pub const AD_WAIT_SURFACE_MODES_SIZE: usize = 3; + +const _: () = assert!(std::mem::size_of::<AdWaitSurfaceModes>() == AD_WAIT_SURFACE_MODES_SIZE); diff --git a/crates/ffi/src/types/window_info.rs b/crates/ffi/src/types/window_info.rs index 4b99aaa..31bda8d 100644 --- a/crates/ffi/src/types/window_info.rs +++ b/crates/ffi/src/types/window_info.rs @@ -3,10 +3,13 @@ use std::os::raw::c_char; #[repr(C)] pub struct AdWindowInfo { + /// Legacy observation-only window ID. This struct has no process-generation + /// evidence and is rejected by targeting APIs; use `AdExactWindowInfo` for + /// any operation that sends a previously observed window back to the library. pub id: *const c_char, pub title: *const c_char, pub app_name: *const c_char, - pub pid: i32, + pub pid: u32, pub bounds: AdRect, pub has_bounds: bool, pub is_focused: bool, diff --git a/crates/ffi/src/windows/focus.rs b/crates/ffi/src/windows/focus.rs index 177e859..7e289c4 100644 --- a/crates/ffi/src/windows/focus.rs +++ b/crates/ffi/src/windows/focus.rs @@ -1,29 +1,24 @@ use crate::AdAdapter; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::types::AdWindowInfo; -use crate::windows::to_core::ad_window_to_core; +use crate::types::{AdExactWindowInfo, AdWindowInfo}; +use crate::windows::to_core::{ad_exact_window_to_core, ad_window_to_core}; -/// Brings `win` to the foreground on the current space. Returns -/// `AD_RESULT_ERR_WINDOW_NOT_FOUND` when the referenced window no longer -/// exists (the caller should re-list and retry). +/// Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process +/// generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. +/// Use `ad_focus_window_exact`. /// /// # Safety /// `adapter` must be a non-null pointer from `ad_adapter_create`. `win` -/// must be a non-null pointer to an `AdWindowInfo` whose `id` and -/// `title` fields are non-null, valid UTF-8 C strings. +/// must be a non-null pointer to an `AdWindowInfo`. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_focus_window( adapter: *const AdAdapter, win: *const AdWindowInfo, ) -> AdResult { trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(win, c"win is null"); - let adapter = &*adapter; let core_win = match ad_window_to_core(&*win) { Ok(w) => w, Err(e) => { @@ -31,12 +26,51 @@ pub unsafe extern "C" fn ad_focus_window( return crate::error::last_error_code(); } }; - match adapter.inner.focus_window(&core_win) { - Ok(()) => AdResult::Ok, - Err(e) => { - set_last_error(&e); - crate::error::last_error_code() - } - } + focus_core_window(adapter, &core_win) }) } + +/// Focuses a generation-pinned exact window. +/// +/// # Safety +/// `adapter` and `win` must be valid pointers. `win` must carry the current +/// exact-window version and size. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_focus_window_exact( + adapter: *const AdAdapter, + win: *const AdExactWindowInfo, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(win, c"win is null"); + let window = match ad_exact_window_to_core(&*win) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + focus_core_window(adapter, &window) + }) +} + +fn focus_core_window( + adapter: *const AdAdapter, + window: &agent_desktop_core::WindowInfo, +) -> AdResult { + let adapter = match crate::adapter::lookup_adapter(adapter) { + Ok(adapter) => adapter, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter.inner.focus_window(window, &lease) { + Ok(()) => AdResult::Ok, + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } +} diff --git a/crates/ffi/src/windows/free_one.rs b/crates/ffi/src/windows/free_one.rs index cbb3c37..a450e7b 100644 --- a/crates/ffi/src/windows/free_one.rs +++ b/crates/ffi/src/windows/free_one.rs @@ -1,6 +1,6 @@ use crate::convert::window::free_window_info_fields; use crate::ffi_try::trap_panic_void; -use crate::types::AdWindowInfo; +use crate::types::{AdExactWindowInfo, AdWindowInfo}; /// Releases the heap-allocated string fields (`id`, `title`, `app_name`) /// inside a single `AdWindowInfo` previously written by `ad_launch_app` @@ -25,3 +25,16 @@ pub unsafe extern "C" fn ad_release_window_fields(win: *mut AdWindowInfo) { free_window_info_fields(&mut *win); }) } + +/// Releases every owned string inside one exact window value. +/// +/// # Safety +/// `win` must be null or point to a value written by `ad_launch_app_exact`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_release_exact_window_fields(win: *mut AdExactWindowInfo) { + trap_panic_void(|| unsafe { + if let Some(window) = win.as_mut() { + crate::convert::window::free_exact_window_info_fields(window); + } + }) +} diff --git a/crates/ffi/src/windows/list.rs b/crates/ffi/src/windows/list.rs index 41694f8..c4d3dcf 100644 --- a/crates/ffi/src/windows/list.rs +++ b/crates/ffi/src/windows/list.rs @@ -1,10 +1,13 @@ use crate::AdAdapter; use crate::convert::string::decode_optional_filter; -use crate::convert::window::{free_window_info_fields, window_info_to_c}; +use crate::convert::window::{ + exact_window_info_to_c, free_exact_window_info_fields, free_window_info_fields, + validate_exact_window_info, window_info_to_c, +}; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::{trap_panic, trap_panic_void}; -use crate::types::{AdWindowInfo, AdWindowList}; -use agent_desktop_core::adapter::WindowFilter; +use crate::types::{AdExactWindowInfo, AdExactWindowList, AdWindowInfo, AdWindowList}; +use agent_desktop_core::WindowFilter; use std::os::raw::c_char; use std::ptr; @@ -22,17 +25,20 @@ pub unsafe extern "C" fn ad_list_windows( trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); *out = ptr::null_mut(); - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); - let adapter = &*adapter; let filter = WindowFilter { focused_only, app: decode_optional_filter!(app_filter, "app_filter"), }; - match adapter.inner.list_windows(&filter) { + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.list_windows(&filter, deadline) { Ok(windows) => { + if let Err(error) = crate::resource::validate_list_len(windows.len(), "Window list") + { + set_last_error(&error); + return crate::error::last_error_code(); + } let items: Vec<AdWindowInfo> = windows.iter().map(window_info_to_c).collect(); let list = Box::new(AdWindowList { items: items.into_boxed_slice(), @@ -94,3 +100,95 @@ pub unsafe extern "C" fn ad_window_list_free(list: *mut AdWindowList) { } }) } + +/// Lists windows with explicit process-generation evidence. +/// +/// # Safety +/// `adapter` and `out` must be valid. `app_filter` may be null or a valid +/// bounded UTF-8 C string. The returned list must be freed with +/// `ad_exact_window_list_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_list_windows_exact( + adapter: *const AdAdapter, + app_filter: *const c_char, + focused_only: bool, + out: *mut *mut AdExactWindowList, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(out, c"out is null"); + *out = ptr::null_mut(); + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + let filter = WindowFilter { + focused_only, + app: decode_optional_filter!(app_filter, "app_filter"), + }; + let adapter = crate::adapter::acquire_adapter!(adapter); + let deadline = crate::operation::operation_deadline!(); + match adapter.inner.list_windows(&filter, deadline) { + Ok(windows) => { + if let Err(error) = + crate::resource::validate_list_len(windows.len(), "Exact window list") + { + set_last_error(&error); + return crate::error::last_error_code(); + } + if let Err(error) = windows.iter().try_for_each(validate_exact_window_info) { + set_last_error(&error); + return crate::error::last_error_code(); + } + let items: Vec<AdExactWindowInfo> = + windows.iter().map(exact_window_info_to_c).collect(); + *out = Box::into_raw(Box::new(AdExactWindowList { + items: items.into_boxed_slice(), + })); + AdResult::Ok + } + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } + }) +} + +/// # Safety +/// `list` must be null or returned by `ad_list_windows_exact`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_exact_window_list_count(list: *const AdExactWindowList) -> u32 { + if list.is_null() { + return 0; + } + unsafe { &*list }.items.len() as u32 +} + +/// # Safety +/// `list` must be null or returned by `ad_list_windows_exact`. The returned +/// pointer is borrowed until the list is freed. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_exact_window_list_get( + list: *const AdExactWindowList, + index: u32, +) -> *const AdExactWindowInfo { + if list.is_null() { + return ptr::null(); + } + unsafe { &*list } + .items + .get(index as usize) + .map_or(ptr::null(), std::ptr::from_ref) +} + +/// # Safety +/// `list` must be null or returned by `ad_list_windows_exact`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_exact_window_list_free(list: *mut AdExactWindowList) { + trap_panic_void(|| unsafe { + if list.is_null() { + return; + } + let mut list = Box::from_raw(list); + for item in &mut list.items { + free_exact_window_info_fields(item); + } + }) +} diff --git a/crates/ffi/src/windows/mod.rs b/crates/ffi/src/windows/mod.rs index f9829e7..03698b9 100644 --- a/crates/ffi/src/windows/mod.rs +++ b/crates/ffi/src/windows/mod.rs @@ -4,4 +4,4 @@ pub(crate) mod list; pub(crate) mod op; pub(crate) mod to_core; -pub(crate) use to_core::ad_window_to_core; +pub(crate) use to_core::{ad_exact_window_to_core, ad_window_to_core}; diff --git a/crates/ffi/src/windows/op.rs b/crates/ffi/src/windows/op.rs index 5a9fbe4..96e41f3 100644 --- a/crates/ffi/src/windows/op.rs +++ b/crates/ffi/src/windows/op.rs @@ -1,20 +1,16 @@ use crate::AdAdapter; use crate::error::{AdResult, set_last_error}; use crate::ffi_try::trap_panic; -use crate::types::{AdWindowInfo, AdWindowOp, AdWindowOpKind}; -use crate::windows::to_core::ad_window_to_core; -use agent_desktop_core::action::WindowOp; +use crate::types::{AdExactWindowInfo, AdWindowInfo, AdWindowOp, AdWindowOpKind}; +use crate::windows::to_core::{ad_exact_window_to_core, ad_window_to_core}; +use agent_desktop_core::WindowOp; -/// Performs a window-manager operation (`Resize`, `Move`, `Minimize`, -/// `Maximize`, `Restore`) on `win`. Width / height / x / y are consulted -/// only for the variants that use them; other kinds ignore them. -/// -/// An invalid `op.kind` discriminant is rejected with -/// `AD_RESULT_ERR_INVALID_ARGS` before any adapter call. +/// Legacy ABI compatibility entrypoint. `AdWindowInfo` cannot carry process +/// generation, so this function fails closed with `AD_RESULT_ERR_INVALID_ARGS`. +/// Use `ad_window_op_exact`. /// /// # Safety -/// `adapter` and `win` must be non-null pointers. `win.id` and -/// `win.title` must be non-null valid UTF-8 C strings. +/// `adapter` and `win` must be non-null pointers. #[unsafe(no_mangle)] pub unsafe extern "C" fn ad_window_op( adapter: *const AdAdapter, @@ -22,12 +18,8 @@ pub unsafe extern "C" fn ad_window_op( op: AdWindowOp, ) -> AdResult { trap_panic(|| unsafe { - if let Err(rc) = crate::main_thread::require_main_thread() { - return rc; - } crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); crate::pointer_guard::guard_non_null!(win, c"win is null"); - let adapter = &*adapter; let core_win = match ad_window_to_core(&*win) { Ok(w) => w, Err(e) => { @@ -35,32 +27,110 @@ pub unsafe extern "C" fn ad_window_op( return crate::error::last_error_code(); } }; - let kind = match AdWindowOpKind::from_c(op.kind) { - Some(k) => k, - None => { - set_last_error(&agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - "invalid window op kind discriminant", - )); - return AdResult::ErrInvalidArgs; + let core_op = match decode_window_op(op) { + Ok(op) => op, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); } }; - let core_op = match kind { - AdWindowOpKind::Resize => WindowOp::Resize { - width: op.width, - height: op.height, - }, - AdWindowOpKind::Move => WindowOp::Move { x: op.x, y: op.y }, - AdWindowOpKind::Minimize => WindowOp::Minimize, - AdWindowOpKind::Maximize => WindowOp::Maximize, - AdWindowOpKind::Restore => WindowOp::Restore, - }; - match adapter.inner.window_op(&core_win, core_op) { - Ok(()) => AdResult::Ok, - Err(e) => { - set_last_error(&e); - crate::error::last_error_code() - } - } + perform_window_op(adapter, &core_win, core_op) }) } + +/// Performs a window-manager operation against an exact generation-pinned +/// window identity. +/// +/// # Safety +/// `adapter` and `win` must be valid pointers. `win` must carry the current +/// exact-window version and size. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ad_window_op_exact( + adapter: *const AdAdapter, + win: *const AdExactWindowInfo, + op: AdWindowOp, +) -> AdResult { + trap_panic(|| unsafe { + crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); + crate::pointer_guard::guard_non_null!(win, c"win is null"); + let window = match ad_exact_window_to_core(&*win) { + Ok(window) => window, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + let op = match decode_window_op(op) { + Ok(op) => op, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + perform_window_op(adapter, &window, op) + }) +} + +fn decode_window_op(op: AdWindowOp) -> Result<WindowOp, agent_desktop_core::AdapterError> { + let kind = AdWindowOpKind::from_c(op.kind).ok_or_else(|| { + agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "invalid window op kind discriminant", + ) + })?; + let invalid_geometry = match kind { + AdWindowOpKind::Resize => { + !op.width.is_finite() + || !op.height.is_finite() + || op.width <= 0.0 + || op.height <= 0.0 + || op.width > 10_000_000.0 + || op.height > 10_000_000.0 + } + AdWindowOpKind::Move => { + !op.x.is_finite() + || !op.y.is_finite() + || op.x.abs() > 10_000_000.0 + || op.y.abs() > 10_000_000.0 + } + _ => false, + }; + if invalid_geometry { + return Err(agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "window geometry must be finite, bounded, and positive for resize", + )); + } + Ok(match kind { + AdWindowOpKind::Resize => WindowOp::Resize { + width: op.width, + height: op.height, + }, + AdWindowOpKind::Move => WindowOp::Move { x: op.x, y: op.y }, + AdWindowOpKind::Minimize => WindowOp::Minimize, + AdWindowOpKind::Maximize => WindowOp::Maximize, + AdWindowOpKind::Restore => WindowOp::Restore, + }) +} + +fn perform_window_op( + adapter: *const AdAdapter, + window: &agent_desktop_core::WindowInfo, + op: WindowOp, +) -> AdResult { + let adapter = match crate::adapter::lookup_adapter(adapter) { + Ok(adapter) => adapter, + Err(error) => { + set_last_error(&error); + return crate::error::last_error_code(); + } + }; + let lease = crate::operation::interaction_lease!(adapter.inner.as_ref()); + match adapter.inner.window_op(window, op, &lease) { + Ok(()) => AdResult::Ok, + Err(error) => { + set_last_error(&error); + crate::error::last_error_code() + } + } +} diff --git a/crates/ffi/src/windows/to_core.rs b/crates/ffi/src/windows/to_core.rs index e7d3572..0b15e8f 100644 --- a/crates/ffi/src/windows/to_core.rs +++ b/crates/ffi/src/windows/to_core.rs @@ -1,6 +1,6 @@ use crate::convert::string::{optional_adapter_string, required_adapter_string}; -use crate::types::AdWindowInfo; -use agent_desktop_core::error::AdapterError; +use crate::types::{AdExactWindowInfo, AdWindowInfo}; +use agent_desktop_core::{AdapterError, Rect, WindowInfo}; /// Converts an `AdWindowInfo` from C into the core `WindowInfo`. /// @@ -11,28 +11,68 @@ use agent_desktop_core::error::AdapterError; /// /// `app_name` is allowed to be empty (some Electron apps report blank /// window owners) and is filled in from the platform adapter as needed. -pub(crate) fn ad_window_to_core( - w: &AdWindowInfo, -) -> Result<agent_desktop_core::node::WindowInfo, AdapterError> { +pub(crate) fn ad_window_to_core(_w: &AdWindowInfo) -> Result<WindowInfo, AdapterError> { + Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "legacy AdWindowInfo lacks process-generation evidence; use AdExactWindowInfo", + )) +} + +pub(crate) fn ad_exact_window_to_core( + exact: &AdExactWindowInfo, +) -> Result<WindowInfo, AdapterError> { + if exact.version != crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_VERSION + || exact.size as usize != crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_SIZE + { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "AdExactWindowInfo version or size does not match this library", + )); + } + let process_instance = required_adapter_string(exact.process_instance, "process_instance")?; + if process_instance.is_empty() { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "process_instance is empty", + )); + } + decode_window(&exact.window, process_instance) +} + +fn decode_window(w: &AdWindowInfo, process_instance: String) -> Result<WindowInfo, AdapterError> { + if w.pid == 0 { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "window pid must be positive", + )); + } let id = required_adapter_string(w.id, "window id")?; let title = required_adapter_string(w.title, "window title")?; let app = optional_adapter_string(w.app_name, "window app_name")?.unwrap_or_default(); - Ok(agent_desktop_core::node::WindowInfo { + let bounds = if w.has_bounds { + let bounds = Rect { + x: w.bounds.x, + y: w.bounds.y, + width: w.bounds.width, + height: w.bounds.height, + }; + bounds.validate()?; + Some(bounds) + } else { + None + }; + Ok(WindowInfo { id, title, app, - pid: w.pid, - bounds: if w.has_bounds { - Some(agent_desktop_core::node::Rect { - x: w.bounds.x, - y: w.bounds.y, - width: w.bounds.width, - height: w.bounds.height, - }) - } else { - None + pid: agent_desktop_core::ProcessId::new(w.pid), + process_instance: Some(process_instance), + bounds, + state: agent_desktop_core::WindowState { + is_focused: w.is_focused, + minimized: None, + visible: None, }, - is_focused: w.is_focused, }) } @@ -41,7 +81,7 @@ mod tests { use super::*; use crate::convert::string::AD_MAX_STRING_BYTES; use crate::types::AdRect; - use agent_desktop_core::error::ErrorCode; + use agent_desktop_core::ErrorCode; use std::ffi::CString; #[test] @@ -52,7 +92,7 @@ mod tests { app.push(0); let win = window(id.as_ptr(), title.as_ptr(), app.as_ptr().cast()); - let err = ad_window_to_core(&win).unwrap_err(); + let err = ad_exact_window_to_core(&win).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert!(err.message.contains("window app_name exceeds")); @@ -65,30 +105,56 @@ mod tests { let app = [0xff_u8, 0x00]; let win = window(id.as_ptr(), title.as_ptr(), app.as_ptr().cast()); - let err = ad_window_to_core(&win).unwrap_err(); + let err = ad_exact_window_to_core(&win).unwrap_err(); assert_eq!(err.code, ErrorCode::InvalidArgs); assert!(err.message.contains("window app_name is not valid UTF-8")); } + #[test] + fn legacy_window_fails_closed_without_process_generation() { + let window = unsafe { std::mem::zeroed::<AdWindowInfo>() }; + let error = ad_window_to_core(&window).unwrap_err(); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert!(error.message.contains("AdExactWindowInfo")); + } + + #[test] + fn exact_window_rejects_unknown_layout_version() { + let mut exact = unsafe { std::mem::zeroed::<AdExactWindowInfo>() }; + exact.version = u32::MAX; + exact.size = crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_SIZE as u32; + + let error = ad_exact_window_to_core(&exact).unwrap_err(); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert!(error.message.contains("version or size")); + } + fn window( id: *const std::os::raw::c_char, title: *const std::os::raw::c_char, app_name: *const std::os::raw::c_char, - ) -> AdWindowInfo { - AdWindowInfo { - id, - title, - app_name, - pid: 7, - bounds: AdRect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 0.0, + ) -> AdExactWindowInfo { + AdExactWindowInfo { + version: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_VERSION, + size: crate::types::exact_window_info::AD_EXACT_WINDOW_INFO_SIZE as u32, + window: AdWindowInfo { + id, + title, + app_name, + pid: 7, + bounds: AdRect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }, + has_bounds: false, + is_focused: false, }, - has_bounds: false, - is_focused: false, + process_instance: c"7:100".as_ptr(), } } } diff --git a/crates/ffi/tests/c_abi_actions.rs b/crates/ffi/tests/c_abi_actions.rs index 4a9909a..1da1ba2 100644 --- a/crates/ffi/tests/c_abi_actions.rs +++ b/crates/ffi/tests/c_abi_actions.rs @@ -18,11 +18,7 @@ fn enum_fuzz_invalid_discriminant_rejected() { }; let mut out: AdActionResult = std::mem::zeroed(); let rc = ad_execute_action(adapter, &handle, &action, &mut out); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "arbitrary enum bit pattern must be rejected, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } @@ -41,10 +37,7 @@ fn invalid_policy_discriminant_rejected_without_ub() { AdPolicyKind::Headed as i32 + 1, &mut out, ); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } @@ -57,19 +50,16 @@ fn execute_action_rejects_null_handle_ptr() { }; let mut out: AdActionResult = std::mem::zeroed(); let rc = ad_execute_action(adapter, &handle, &action, &mut out); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } #[test] -fn execute_ref_action_uses_strict_resolution_before_dispatch() { +fn legacy_ref_action_fails_closed_without_exact_identity() { with_adapter(|adapter| unsafe { let role = std::ffi::CString::new("button").unwrap(); let mut entry = default_ref_entry(); - entry.role = role.as_ptr(); + entry.identity.role = role.as_ptr(); let action = default_action(); let mut out: AdActionResult = std::mem::zeroed(); @@ -81,15 +71,12 @@ fn execute_ref_action_uses_strict_resolution_before_dispatch() { &mut out, ); - assert!(matches!( - rc, - AdResult::ErrStaleRef | AdResult::ErrElementNotFound | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } #[test] -fn execute_action_policy_requires_main_thread_on_macos() { +fn execute_action_policy_rejects_null_adapter_on_worker_thread() { let rc = std::thread::spawn(|| unsafe { let action = default_action(); let handle = AdNativeHandle { @@ -107,16 +94,11 @@ fn execute_action_policy_requires_main_thread_on_macos() { .join() .unwrap(); - #[cfg(target_os = "macos")] - assert_eq!(rc, AdResult::ErrInternal); - #[cfg(not(target_os = "macos"))] assert_eq!(rc, AdResult::ErrInvalidArgs); } -/// Verifies that `ad_execute_ref_action_with_policy` uses the adapter's session -/// context rather than a default one. The observable contract is that resolution -/// fails (stale ref) identically whether a session id is present or absent — -/// the session id is wired into trace emission, not into the error path. +/// Session context must not weaken the legacy ref entry's fail-closed identity +/// check. #[test] fn execute_ref_action_with_session_adapter_wires_context() { unsafe { @@ -126,7 +108,7 @@ fn execute_ref_action_with_session_adapter_wires_context() { let role = CString::new("button").unwrap(); let mut entry = default_ref_entry(); - entry.role = role.as_ptr(); + entry.identity.role = role.as_ptr(); let action = default_action(); let mut out: AdActionResult = std::mem::zeroed(); @@ -138,42 +120,57 @@ fn execute_ref_action_with_session_adapter_wires_context() { &mut out, ); - assert!( - matches!( - rc, - AdResult::ErrStaleRef | AdResult::ErrElementNotFound | AdResult::ErrInternal - ), - "session adapter must still reject unresolvable entry, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); ad_adapter_destroy(adapter); } } #[test] -fn free_action_result_releases_non_empty_steps_array() { +fn free_action_result_never_scans_an_unowned_steps_array() { let mut steps = vec![ AdActionStep { label: CString::new("AXScrollToVisible").unwrap().into_raw(), outcome: CString::new("attempted").unwrap().into_raw(), + mechanism: 1, + has_mechanism: true, + verified: false, + has_verified: false, + _reserved: 0, }, AdActionStep { label: CString::new("AXPress").unwrap().into_raw(), outcome: CString::new("succeeded").unwrap().into_raw(), + mechanism: 1, + has_mechanism: true, + verified: true, + has_verified: true, + _reserved: 0, }, AdActionStep { label: std::ptr::null(), outcome: std::ptr::null(), + mechanism: 0, + has_mechanism: false, + verified: false, + has_verified: false, + _reserved: 0, }, ] .into_boxed_slice(); + let steps_ptr = steps.as_mut_ptr(); + let action_ptr = CString::new("click").unwrap().into_raw(); let mut result = AdActionResult { - action: CString::new("click").unwrap().into_raw(), + action: action_ptr, ref_id: std::ptr::null(), post_state: std::ptr::null_mut(), steps: steps.as_mut_ptr(), step_count: 2, + details_json: std::ptr::null(), + disposition: agent_desktop_ffi::AdDeliverySemantics { + delivery: agent_desktop_ffi::AdDeliveryDisposition::Unknown as i32, + retry: agent_desktop_ffi::AdRetryDisposition::Unknown as i32, + }, }; std::mem::forget(steps); @@ -182,4 +179,14 @@ fn free_action_result_releases_non_empty_steps_array() { assert!(result.action.is_null()); assert!(result.steps.is_null()); assert_eq!(result.step_count, 0); + assert!(result.details_json.is_null()); + + let mut steps = unsafe { Box::from_raw(std::ptr::slice_from_raw_parts_mut(steps_ptr, 3)) }; + for step in steps.iter_mut().take(2) { + unsafe { + drop(CString::from_raw(step.label as *mut _)); + drop(CString::from_raw(step.outcome as *mut _)); + } + } + unsafe { drop(CString::from_raw(action_ptr)) }; } diff --git a/crates/ffi/tests/c_abi_displays.rs b/crates/ffi/tests/c_abi_displays.rs new file mode 100644 index 0000000..b49961a --- /dev/null +++ b/crates/ffi/tests/c_abi_displays.rs @@ -0,0 +1,47 @@ +#![allow(improper_ctypes)] + +use agent_desktop_ffi::{ + AD_DISPLAY_INFO_SIZE, AD_DISPLAY_INFO_VERSION, AdAdapter, AdDisplayInfo, AdDisplayList, + AdResult, +}; +use std::mem::{offset_of, size_of}; + +unsafe extern "C" { + fn ad_list_displays(adapter: *const AdAdapter, out: *mut *mut AdDisplayList) -> AdResult; + fn ad_display_list_count(list: *const AdDisplayList) -> u32; + fn ad_display_list_get(list: *const AdDisplayList, index: u32) -> *const AdDisplayInfo; + fn ad_display_list_free(list: *mut AdDisplayList); +} + +#[test] +fn list_displays_rejects_null_adapter_and_clears_out() { + unsafe { + let mut list = std::ptr::dangling_mut::<AdDisplayList>(); + let result = ad_list_displays(std::ptr::null(), &mut list); + + assert_eq!(result, AdResult::ErrInvalidArgs); + assert!(list.is_null()); + } +} + +#[test] +fn display_info_layout_is_pinned() { + assert_eq!(AD_DISPLAY_INFO_VERSION, 1); + assert_eq!(AD_DISPLAY_INFO_SIZE, 64); + assert_eq!(size_of::<AdDisplayInfo>(), AD_DISPLAY_INFO_SIZE); + assert_eq!(offset_of!(AdDisplayInfo, version), 0); + assert_eq!(offset_of!(AdDisplayInfo, size), 4); + assert_eq!(offset_of!(AdDisplayInfo, id), 8); + assert_eq!(offset_of!(AdDisplayInfo, bounds), 16); + assert_eq!(offset_of!(AdDisplayInfo, is_primary), 48); + assert_eq!(offset_of!(AdDisplayInfo, scale), 56); +} + +#[test] +fn display_list_accessors_are_null_tolerant() { + unsafe { + assert_eq!(ad_display_list_count(std::ptr::null()), 0); + assert!(ad_display_list_get(std::ptr::null(), 0).is_null()); + ad_display_list_free(std::ptr::null_mut()); + } +} diff --git a/crates/ffi/tests/c_abi_execute_by_ref.rs b/crates/ffi/tests/c_abi_execute_by_ref.rs index 47cdd1a..2c6ff58 100644 --- a/crates/ffi/tests/c_abi_execute_by_ref.rs +++ b/crates/ffi/tests/c_abi_execute_by_ref.rs @@ -1,8 +1,8 @@ mod common; use common::{ - AdResult, CStr, ad_execute_by_ref, ad_free_string, ad_last_error_code, default_action, - with_adapter, + AdResult, CStr, ad_execute_by_ref, ad_execute_by_ref_timeout, ad_free_string, + ad_last_error_code, ad_last_error_message, default_action, with_adapter, }; #[test] @@ -40,10 +40,7 @@ fn execute_by_ref_null_adapter_rejected() { 0, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "null adapter must fail — got {rc:?} (ErrInternal on macOS off-main-thread is expected)" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on failure"); } } @@ -61,14 +58,30 @@ fn execute_by_ref_null_ref_id_returns_invalid_args() { 0, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "null ref_id must fail — got {rc:?}" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on null ref_id"); }); } +#[test] +fn execute_by_ref_bare_ref_requires_explicit_snapshot() { + with_adapter(|adapter| unsafe { + let ref_id = std::ffi::CString::new("@e1").unwrap(); + let action = default_action(); + let mut out = std::ptr::null_mut(); + let result = ad_execute_by_ref( + adapter, + ref_id.as_ptr(), + std::ptr::null(), + &action, + 0, + &mut out, + ); + assert_eq!(result, AdResult::ErrInvalidArgs); + assert!(out.is_null()); + }); +} + #[test] fn execute_by_ref_invalid_utf8_ref_id_returns_invalid_args() { with_adapter(|adapter| unsafe { @@ -83,10 +96,7 @@ fn execute_by_ref_invalid_utf8_ref_id_returns_invalid_args() { 0, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "invalid UTF-8 ref_id must fail — got {rc:?}" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on invalid UTF-8 ref_id"); }); } @@ -104,10 +114,7 @@ fn execute_by_ref_null_action_rejected() { 0, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "null action must fail — got {rc:?}" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on null action"); }); } @@ -126,10 +133,7 @@ fn execute_by_ref_invalid_ref_format_returns_invalid_args() { 0, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "bad ref format must fail — got {rc:?}" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on bad ref format"); }); } @@ -183,18 +187,63 @@ fn execute_by_ref_out_of_range_policy_returns_invalid_args() { 99, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "out-of-range policy discriminant must fail — got {rc:?} \ - (ErrInternal on macOS off-main-thread is expected)" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on invalid policy"); - if rc == AdResult::ErrInvalidArgs { - assert_eq!( - ad_last_error_code(), - AdResult::ErrInvalidArgs, - "last-error code must reflect the invalid-args rejection" + assert_eq!( + ad_last_error_code(), + AdResult::ErrInvalidArgs, + "last-error code must reflect the invalid-args rejection" + ); + }); +} + +#[test] +fn execute_by_ref_timeout_rejects_values_below_default_sentinel_at_the_boundary() { + with_adapter(|adapter| unsafe { + let ref_id = std::ffi::CString::new("@e1").unwrap(); + let action = default_action(); + let mut out = std::ptr::null_mut(); + + let rc = ad_execute_by_ref_timeout( + adapter, + ref_id.as_ptr(), + std::ptr::null(), + &action, + 0, + -2, + &mut out, + ); + + assert_eq!(rc, AdResult::ErrInvalidArgs); + assert!(out.is_null()); + let message = CStr::from_ptr(ad_last_error_message()).to_string_lossy(); + assert!(message.contains("-1")); + assert!(message.contains("0")); + }); +} + +#[test] +fn execute_by_ref_timeout_accepts_default_and_single_shot_sentinels_before_ref_validation() { + with_adapter(|adapter| unsafe { + let invalid_ref = std::ffi::CString::new("@e0").unwrap(); + let action = default_action(); + for timeout_ms in [-1, 0] { + let mut out = std::ptr::null_mut(); + let rc = ad_execute_by_ref_timeout( + adapter, + invalid_ref.as_ptr(), + std::ptr::null(), + &action, + 0, + timeout_ms, + &mut out, ); + + assert_eq!(rc, AdResult::ErrInvalidArgs); + assert!(out.is_null()); + let message = CStr::from_ptr(ad_last_error_message()).to_string_lossy(); + assert!(message.contains("ref")); + assert!(!message.contains("timeout_ms must")); } }); } diff --git a/crates/ffi/tests/c_abi_inputs.rs b/crates/ffi/tests/c_abi_inputs.rs index 0d4eaf7..213c22a 100644 --- a/crates/ffi/tests/c_abi_inputs.rs +++ b/crates/ffi/tests/c_abi_inputs.rs @@ -1,8 +1,9 @@ mod common; use common::{ - AdNativeHandle, AdResult, AdWindowInfo, AdWindowList, ad_launch_app, ad_list_windows, - ad_resolve_element, c_char, default_ref_entry, with_adapter, + AdNativeHandle, AdResult, AdWindowInfo, AdWindowList, CStr, ad_last_error_message, + ad_launch_app, ad_list_windows, ad_resolve_element, ad_resolve_element_exact, c_char, + default_exact_ref_entry, default_ref_entry, with_adapter, }; #[test] @@ -11,10 +12,7 @@ fn invalid_utf8_filter_rejected_not_silently_widened() { let bad: [u8; 2] = [0xC3, 0x00]; let mut list: *mut AdWindowList = std::ptr::null_mut(); let rc = ad_list_windows(adapter, bad.as_ptr() as *const c_char, false, &mut list); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(list.is_null()); }); } @@ -25,16 +23,12 @@ fn invalid_utf8_app_id_rejected() { let bad: [u8; 2] = [0xC3, 0]; let mut out: AdWindowInfo = std::mem::zeroed(); let rc = ad_launch_app(adapter, bad.as_ptr() as *const c_char, 0, &mut out); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "must reject without UB, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } #[test] -fn resolve_element_rejects_null_role() { +fn legacy_resolve_fails_closed_before_role_validation() { with_adapter(|adapter| unsafe { let mut out = AdNativeHandle { ptr: std::ptr::null(), @@ -42,39 +36,51 @@ fn resolve_element_rejects_null_role() { let rc = ad_resolve_element(adapter, &default_ref_entry(), &mut out); assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.ptr.is_null()); + let message = CStr::from_ptr(ad_last_error_message()).to_string_lossy(); + assert!(message.contains("legacy AdRefEntry lacks")); }); } #[test] -fn resolve_element_rejects_invalid_utf8_name() { +fn exact_resolve_rejects_invalid_utf8_name_at_the_name_boundary() { with_adapter(|adapter| unsafe { let role = std::ffi::CString::new("button").unwrap(); + let process_instance = std::ffi::CString::new("generation-1").unwrap(); let bad_name: [u8; 2] = [0xC3, 0x00]; - let mut entry = default_ref_entry(); - entry.role = role.as_ptr(); - entry.name = bad_name.as_ptr() as *const c_char; + let mut exact = default_exact_ref_entry(); + exact.process_instance = process_instance.as_ptr(); + exact.entry.process.pid = 1; + exact.entry.identity.role = role.as_ptr(); + exact.entry.identity.name = bad_name.as_ptr() as *const c_char; let mut out = AdNativeHandle { ptr: std::ptr::null(), }; - let rc = ad_resolve_element(adapter, &entry, &mut out); + let rc = ad_resolve_element_exact(adapter, &exact, &mut out); assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.ptr.is_null()); + let message = CStr::from_ptr(ad_last_error_message()).to_string_lossy(); + assert!(message.contains("identity.name")); }); } #[test] -fn resolve_element_rejects_invalid_utf8_description() { +fn exact_resolve_rejects_invalid_utf8_description_at_the_description_boundary() { with_adapter(|adapter| unsafe { let role = std::ffi::CString::new("button").unwrap(); + let process_instance = std::ffi::CString::new("generation-1").unwrap(); let bad_description: [u8; 2] = [0xC3, 0x00]; - let mut entry = default_ref_entry(); - entry.role = role.as_ptr(); - entry.description = bad_description.as_ptr() as *const c_char; + let mut exact = default_exact_ref_entry(); + exact.process_instance = process_instance.as_ptr(); + exact.entry.process.pid = 1; + exact.entry.identity.role = role.as_ptr(); + exact.entry.identity.description = bad_description.as_ptr() as *const c_char; let mut out = AdNativeHandle { ptr: std::ptr::null(), }; - let rc = ad_resolve_element(adapter, &entry, &mut out); + let rc = ad_resolve_element_exact(adapter, &exact, &mut out); assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.ptr.is_null()); + let message = CStr::from_ptr(ad_last_error_message()).to_string_lossy(); + assert!(message.contains("identity.description")); }); } diff --git a/crates/ffi/tests/c_abi_json_commands.rs b/crates/ffi/tests/c_abi_json_commands.rs index 34395d7..c84e45f 100644 --- a/crates/ffi/tests/c_abi_json_commands.rs +++ b/crates/ffi/tests/c_abi_json_commands.rs @@ -61,6 +61,18 @@ fn ad_version_null_out_returns_invalid_args() { } } +#[test] +fn ad_free_string_ignores_a_second_free() { + unsafe { + let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); + assert_eq!(ad_version(&mut out), AdResult::Ok); + assert!(!out.is_null()); + + ad_free_string(out); + ad_free_string(out); + } +} + #[test] fn ad_version_success_preserves_prior_last_error() { unsafe { diff --git a/crates/ffi/tests/c_abi_layout.rs b/crates/ffi/tests/c_abi_layout.rs index eec240f..667443d 100644 --- a/crates/ffi/tests/c_abi_layout.rs +++ b/crates/ffi/tests/c_abi_layout.rs @@ -1,16 +1,33 @@ mod common; +use agent_desktop_ffi::{ + AdDeliverySemantics, AdExactRefEntry, AdExactSurfaceInfo, AdExactWindowInfo, AdFindControl, + AdFindFilter, AdFindIdentity, AdFindQuery, AdFindSelection, AdFindStatePredicate, + AdFindStateSlice, AdNode, AdNodeContent, AdNodePresentation, AdNodeRelation, AdOptionalU64, + AdOptionalUsize, AdRefCapabilities, AdRefGeometry, AdRefIdentity, AdRefProcess, AdRefScope, + AdRefSource, AdScreenshotTarget, AdStringSlice, AdWaitMode, AdWaitPredicate, AdWaitScope, + AdWaitSurfaceModes, +}; use common::{ AdAction, AdActionResult, AdActionStep, AdElementState, AdPoint, AdRect, AdRefEntry, AdWaitArgs, }; use std::mem::{MaybeUninit, align_of, offset_of, size_of}; +#[test] +fn screenshot_target_layout_is_guarded_for_c_consumers() { + assert_eq!(size_of::<AdScreenshotTarget>(), 24); + assert_eq!(align_of::<AdScreenshotTarget>(), 8); + assert_eq!(offset_of!(AdScreenshotTarget, kind), 0); + assert_eq!(offset_of!(AdScreenshotTarget, screen_index), 8); + assert_eq!(offset_of!(AdScreenshotTarget, pid), 16); +} + #[test] fn action_layout_is_guarded_for_c_consumers() { - assert_eq!(agent_desktop_ffi::types::action::AD_ACTION_SIZE, 96); + assert_eq!(agent_desktop_ffi::AD_ACTION_SIZE, 96); assert_eq!( unsafe { common::ad_action_size() }, - agent_desktop_ffi::types::action::AD_ACTION_SIZE + agent_desktop_ffi::AD_ACTION_SIZE ); assert_eq!(size_of::<AdAction>(), 96); assert_eq!(align_of::<AdAction>(), align_of::<usize>()); @@ -35,47 +52,60 @@ fn action_layout_is_guarded_for_c_consumers() { #[test] fn action_result_layout_is_guarded_for_c_consumers() { - assert_eq!( - agent_desktop_ffi::types::action_result::AD_ACTION_RESULT_SIZE, - 40 - ); + assert_eq!(agent_desktop_ffi::AD_ACTION_RESULT_SIZE, 56); assert_eq!( unsafe { common::ad_action_result_size() }, - agent_desktop_ffi::types::action_result::AD_ACTION_RESULT_SIZE - ); - assert_eq!( - agent_desktop_ffi::types::action_step::AD_ACTION_STEP_SIZE, - 16 + agent_desktop_ffi::AD_ACTION_RESULT_SIZE ); + assert_eq!(agent_desktop_ffi::AD_ACTION_STEP_SIZE, 32); assert_eq!( unsafe { common::ad_action_step_size() }, - agent_desktop_ffi::types::action_step::AD_ACTION_STEP_SIZE + agent_desktop_ffi::AD_ACTION_STEP_SIZE ); assert_eq!( size_of::<AdActionStep>(), - agent_desktop_ffi::types::action_step::AD_ACTION_STEP_SIZE + agent_desktop_ffi::AD_ACTION_STEP_SIZE ); assert_eq!(align_of::<AdActionStep>(), align_of::<usize>()); - assert_eq!(size_of::<AdActionResult>(), 40); + assert_eq!(size_of::<AdActionResult>(), 56); assert_eq!(align_of::<AdActionResult>(), align_of::<usize>()); assert_eq!(offset_of!(AdActionResult, action), 0); assert_eq!(offset_of!(AdActionResult, ref_id), 8); assert_eq!(offset_of!(AdActionResult, post_state), 16); assert_eq!(offset_of!(AdActionResult, steps), 24); assert_eq!(offset_of!(AdActionResult, step_count), 32); + assert_eq!(offset_of!(AdActionResult, details_json), 40); + assert_eq!(offset_of!(AdActionResult, disposition), 48); + assert_eq!(size_of::<AdDeliverySemantics>(), 8); + assert_eq!(offset_of!(AdDeliverySemantics, delivery), 0); + assert_eq!(offset_of!(AdDeliverySemantics, retry), 4); assert_eq!(offset_of!(AdActionStep, label), 0); assert_eq!(offset_of!(AdActionStep, outcome), 8); + assert_eq!(offset_of!(AdActionStep, mechanism), 16); + assert_eq!(offset_of!(AdActionStep, has_mechanism), 20); + assert_eq!(offset_of!(AdActionStep, verified), 21); + assert_eq!(offset_of!(AdActionStep, has_verified), 22); + assert_eq!(offset_of!(AdActionStep, _reserved), 24); + + let copied = unsafe { + let step = MaybeUninit::<AdActionStep>::zeroed().assume_init(); + std::ptr::read(&step as *const AdActionStep) + }; + assert!(copied.label.is_null()); + assert!(copied.outcome.is_null()); + assert_eq!(copied.mechanism, 0); + assert!(!copied.has_mechanism); + assert!(!copied.verified); + assert!(!copied.has_verified); + assert_eq!(copied._reserved, 0); } #[test] fn element_state_layout_is_guarded_for_c_consumers() { - assert_eq!( - agent_desktop_ffi::types::element_state::AD_ELEMENT_STATE_SIZE, - 32 - ); + assert_eq!(agent_desktop_ffi::AD_ELEMENT_STATE_SIZE, 32); assert_eq!( unsafe { common::ad_element_state_size() }, - agent_desktop_ffi::types::element_state::AD_ELEMENT_STATE_SIZE + agent_desktop_ffi::AD_ELEMENT_STATE_SIZE ); assert_eq!(size_of::<AdElementState>(), 32); assert_eq!(align_of::<AdElementState>(), align_of::<usize>()); @@ -102,88 +132,184 @@ fn rect_and_point_layouts_are_memcpyable() { assert_eq!(copied.y, 4.0); } +#[test] +fn find_query_layout_is_guarded_for_c_consumers() { + assert_eq!(agent_desktop_ffi::AD_FIND_SELECTION_SIZE, 8); + assert_eq!(size_of::<AdFindSelection>(), 8); + assert_eq!(offset_of!(AdFindSelection, kind), 0); + assert_eq!(offset_of!(AdFindSelection, nth), 4); + assert_eq!(agent_desktop_ffi::AD_FIND_CONTROL_SIZE, 24); + assert_eq!(size_of::<AdFindControl>(), 24); + assert_eq!(offset_of!(AdFindControl, version), 0); + assert_eq!(offset_of!(AdFindControl, selection), 4); + assert_eq!(offset_of!(AdFindControl, timeout_ms), 16); + assert_eq!(agent_desktop_ffi::AD_FIND_IDENTITY_SIZE, 40); + assert_eq!(size_of::<AdFindIdentity>(), 40); + assert_eq!(offset_of!(AdFindIdentity, role), 0); + assert_eq!(offset_of!(AdFindIdentity, value), 32); + assert_eq!(agent_desktop_ffi::AD_FIND_STATE_PREDICATE_SIZE, 16); + assert_eq!(size_of::<AdFindStatePredicate>(), 16); + assert_eq!(offset_of!(AdFindStatePredicate, token), 0); + assert_eq!(offset_of!(AdFindStatePredicate, expected), 8); + assert_eq!(agent_desktop_ffi::AD_FIND_STATE_SLICE_SIZE, 16); + assert_eq!(size_of::<AdFindStateSlice>(), 16); + assert_eq!(offset_of!(AdFindStateSlice, items), 0); + assert_eq!(offset_of!(AdFindStateSlice, count), 8); + assert_eq!(agent_desktop_ffi::AD_FIND_FILTER_SIZE, 88); + assert_eq!(size_of::<AdFindFilter>(), 88); + assert_eq!(offset_of!(AdFindFilter, identity), 0); + assert_eq!(offset_of!(AdFindFilter, has_text), 40); + assert_eq!(offset_of!(AdFindFilter, states), 48); + assert_eq!(offset_of!(AdFindFilter, has), 64); + assert_eq!(offset_of!(AdFindFilter, has_not), 72); + assert_eq!(offset_of!(AdFindFilter, exact), 80); + assert_eq!(agent_desktop_ffi::AD_FIND_QUERY_VERSION, 1); + assert_eq!(agent_desktop_ffi::AD_FIND_QUERY_SIZE, 112); + assert_eq!(size_of::<AdFindQuery>(), 112); + assert_eq!(align_of::<AdFindQuery>(), align_of::<usize>()); + assert_eq!(offset_of!(AdFindQuery, control), 0); + assert_eq!(offset_of!(AdFindQuery, filter), 24); +} + #[test] fn ref_entry_input_caps_match_the_published_header_values() { - assert_eq!(agent_desktop_ffi::types::ref_entry::AD_MAX_REF_STATES, 64); - assert_eq!(agent_desktop_ffi::types::ref_entry::AD_MAX_REF_ACTIONS, 32); - assert_eq!( - agent_desktop_ffi::types::ref_entry::AD_MAX_REF_PATH_DEPTH, - 128 - ); + assert_eq!(agent_desktop_ffi::AD_MAX_REF_STATES, 64); + assert_eq!(agent_desktop_ffi::AD_MAX_REF_ACTIONS, 32); + assert_eq!(agent_desktop_ffi::AD_MAX_REF_PATH_DEPTH, 128); } #[test] fn ref_entry_layout_is_guarded_for_c_consumers() { - assert_eq!(agent_desktop_ffi::types::ref_entry::AD_REF_ENTRY_SIZE, 192); + assert_eq!(agent_desktop_ffi::AD_REF_ENTRY_SIZE, 200); assert_eq!( unsafe { common::ad_ref_entry_size() }, - agent_desktop_ffi::types::ref_entry::AD_REF_ENTRY_SIZE + agent_desktop_ffi::AD_REF_ENTRY_SIZE ); - assert_eq!(size_of::<AdRefEntry>(), 192); + assert_eq!(size_of::<AdRefEntry>(), 200); assert_eq!(align_of::<AdRefEntry>(), align_of::<usize>()); - assert_eq!(offset_of!(AdRefEntry, pid), 0); - let offsets = [ - offset_of!(AdRefEntry, pid), - offset_of!(AdRefEntry, role), - offset_of!(AdRefEntry, name), - offset_of!(AdRefEntry, value), - offset_of!(AdRefEntry, description), - offset_of!(AdRefEntry, states), - offset_of!(AdRefEntry, state_count), - offset_of!(AdRefEntry, available_actions), - offset_of!(AdRefEntry, available_action_count), - offset_of!(AdRefEntry, bounds), - offset_of!(AdRefEntry, has_bounds), - offset_of!(AdRefEntry, bounds_hash), - offset_of!(AdRefEntry, has_bounds_hash), - offset_of!(AdRefEntry, source_app), - offset_of!(AdRefEntry, source_window_id), - offset_of!(AdRefEntry, source_window_title), - offset_of!(AdRefEntry, source_surface), - offset_of!(AdRefEntry, root_ref), - offset_of!(AdRefEntry, path_is_absolute), - offset_of!(AdRefEntry, path), - offset_of!(AdRefEntry, path_count), - ]; - assert!(offsets.windows(2).all(|pair| pair[0] < pair[1])); + assert_eq!(offset_of!(AdRefEntry, process), 0); + assert_eq!(offset_of!(AdRefEntry, identity), 8); + assert_eq!(offset_of!(AdRefEntry, geometry), 48); + assert_eq!(offset_of!(AdRefEntry, capabilities), 96); + assert_eq!(offset_of!(AdRefEntry, source), 128); + assert_eq!(offset_of!(AdRefEntry, scope), 168); + assert_eq!(size_of::<AdRefProcess>(), 4); + assert_eq!(size_of::<AdRefIdentity>(), 40); + assert_eq!(offset_of!(AdRefIdentity, name), 8); + assert_eq!(offset_of!(AdRefIdentity, native_id), 32); + assert_eq!(size_of::<AdStringSlice>(), 16); + assert_eq!(offset_of!(AdStringSlice, count), 8); + assert_eq!(size_of::<AdRefCapabilities>(), 32); + assert_eq!(offset_of!(AdRefCapabilities, available_actions), 16); + assert_eq!(size_of::<AdRefGeometry>(), 48); + assert_eq!(offset_of!(AdRefGeometry, bounds_hash), 32); + assert_eq!(offset_of!(AdRefGeometry, has_bounds), 40); + assert_eq!(offset_of!(AdRefGeometry, has_bounds_hash), 41); + assert_eq!(size_of::<AdRefSource>(), 40); + assert_eq!(offset_of!(AdRefSource, window_bounds_hash), 24); + assert_eq!(offset_of!(AdRefSource, surface), 32); + assert_eq!(offset_of!(AdRefSource, has_window_bounds_hash), 36); + assert_eq!(size_of::<AdRefScope>(), 32); + assert_eq!(offset_of!(AdRefScope, path), 8); + assert_eq!(offset_of!(AdRefScope, path_count), 16); + assert_eq!(offset_of!(AdRefScope, path_is_absolute), 24); let copied = unsafe { let entry = MaybeUninit::<AdRefEntry>::zeroed().assume_init(); std::ptr::read(&entry as *const AdRefEntry) }; - assert_eq!(copied.pid, 0); - assert_eq!(copied.path_count, 0); + assert_eq!(copied.process.pid, 0); + assert_eq!(copied.scope.path_count, 0); + assert!(copied.identity.native_id.is_null()); +} + +#[test] +fn exact_ref_entry_is_additive_versioned_and_layout_pinned() { + assert_eq!(agent_desktop_ffi::AD_EXACT_REF_ENTRY_VERSION, 1); + assert_eq!(agent_desktop_ffi::AD_EXACT_REF_ENTRY_SIZE, 224); + assert_eq!(unsafe { common::ad_exact_ref_entry_size() }, 224); + assert_eq!(size_of::<AdExactRefEntry>(), 224); + assert_eq!(align_of::<AdExactRefEntry>(), align_of::<usize>()); + assert_eq!(offset_of!(AdExactRefEntry, version), 0); + assert_eq!(offset_of!(AdExactRefEntry, size), 4); + assert_eq!(offset_of!(AdExactRefEntry, entry), 8); + assert_eq!(offset_of!(AdExactRefEntry, process_instance), 208); + assert_eq!(offset_of!(AdExactRefEntry, identifier_kind), 216); +} + +#[test] +fn exact_window_info_is_additive_versioned_and_layout_pinned() { + assert_eq!(agent_desktop_ffi::AD_EXACT_WINDOW_INFO_VERSION, 1); + assert_eq!(agent_desktop_ffi::AD_EXACT_WINDOW_INFO_SIZE, 88); + assert_eq!(unsafe { common::ad_exact_window_info_size() }, 88); + assert_eq!(size_of::<AdExactWindowInfo>(), 88); + assert_eq!(align_of::<AdExactWindowInfo>(), align_of::<usize>()); + assert_eq!(offset_of!(AdExactWindowInfo, version), 0); + assert_eq!(offset_of!(AdExactWindowInfo, size), 4); + assert_eq!(offset_of!(AdExactWindowInfo, window), 8); + assert_eq!(offset_of!(AdExactWindowInfo, process_instance), 80); +} + +#[test] +fn exact_surface_info_is_additive_versioned_and_layout_pinned() { + assert_eq!(agent_desktop_ffi::AD_EXACT_SURFACE_INFO_VERSION, 1); + assert_eq!(agent_desktop_ffi::AD_EXACT_SURFACE_INFO_SIZE, 40); + assert_eq!(unsafe { common::ad_exact_surface_info_size() }, 40); + assert_eq!(size_of::<AdExactSurfaceInfo>(), 40); + assert_eq!(align_of::<AdExactSurfaceInfo>(), align_of::<usize>()); + assert_eq!(offset_of!(AdExactSurfaceInfo, version), 0); + assert_eq!(offset_of!(AdExactSurfaceInfo, size), 4); + assert_eq!(offset_of!(AdExactSurfaceInfo, id), 8); + assert_eq!(offset_of!(AdExactSurfaceInfo, surface), 16); } #[test] fn wait_args_layout_is_guarded_for_c_consumers() { - assert_eq!(agent_desktop_ffi::types::wait_args::AD_WAIT_ARGS_SIZE, 112); + assert_eq!(agent_desktop_ffi::AD_WAIT_ARGS_SIZE, 112); assert_eq!( unsafe { common::ad_wait_args_size() }, - agent_desktop_ffi::types::wait_args::AD_WAIT_ARGS_SIZE + agent_desktop_ffi::AD_WAIT_ARGS_SIZE ); assert_eq!(size_of::<AdWaitArgs>(), 112); assert_eq!(align_of::<AdWaitArgs>(), align_of::<usize>()); - let offsets = [ - offset_of!(AdWaitArgs, ms), - offset_of!(AdWaitArgs, has_ms), - offset_of!(AdWaitArgs, element), - offset_of!(AdWaitArgs, window), - offset_of!(AdWaitArgs, text), - offset_of!(AdWaitArgs, menu), - offset_of!(AdWaitArgs, snapshot_id), - offset_of!(AdWaitArgs, count), - offset_of!(AdWaitArgs, timeout_ms), - offset_of!(AdWaitArgs, app), - ]; - assert!(offsets.windows(2).all(|pair| pair[0] < pair[1])); + assert_eq!(offset_of!(AdWaitArgs, mode), 0); + assert_eq!(offset_of!(AdWaitArgs, predicate), 48); + assert_eq!(offset_of!(AdWaitArgs, scope), 96); + assert_eq!(size_of::<AdOptionalU64>(), 16); + assert_eq!(offset_of!(AdOptionalU64, present), 8); + assert_eq!(size_of::<AdWaitSurfaceModes>(), 3); + assert_eq!(size_of::<AdWaitMode>(), 48); + assert_eq!(offset_of!(AdWaitMode, element), 16); + assert_eq!(offset_of!(AdWaitMode, surfaces), 40); + assert_eq!(size_of::<AdOptionalUsize>(), 16); + assert_eq!(offset_of!(AdOptionalUsize, present), 8); + assert_eq!(size_of::<AdWaitPredicate>(), 48); + assert_eq!(offset_of!(AdWaitPredicate, count), 32); + assert_eq!(size_of::<AdWaitScope>(), 16); + assert_eq!(offset_of!(AdWaitScope, app), 8); let zeroed = unsafe { MaybeUninit::<AdWaitArgs>::zeroed().assume_init() }; - assert_eq!(zeroed.ms, 0); - assert!(!zeroed.has_ms); - assert!(zeroed.element.is_null()); - assert!(!zeroed.menu); - assert_eq!(zeroed.timeout_ms, 0); + assert_eq!(zeroed.mode.pause.value, 0); + assert!(!zeroed.mode.pause.present); + assert!(zeroed.mode.element.is_null()); + assert!(!zeroed.mode.surfaces.menu); + assert_eq!(zeroed.scope.timeout_ms, 0); +} + +#[test] +fn node_layout_is_grouped_and_pinned() { + assert_eq!(size_of::<AdNode>(), 112); + assert_eq!(offset_of!(AdNode, content), 0); + assert_eq!(offset_of!(AdNode, presentation), 48); + assert_eq!(offset_of!(AdNode, relation), 96); + assert_eq!(size_of::<AdNodeContent>(), 48); + assert_eq!(offset_of!(AdNodeContent, hint), 40); + assert_eq!(size_of::<AdNodePresentation>(), 48); + assert_eq!(offset_of!(AdNodePresentation, bounds), 8); + assert_eq!(offset_of!(AdNodePresentation, state_count), 40); + assert_eq!(offset_of!(AdNodePresentation, has_bounds), 44); + assert_eq!(size_of::<AdNodeRelation>(), 12); + assert_eq!(offset_of!(AdNodeRelation, child_count), 8); } diff --git a/crates/ffi/tests/c_abi_lifecycle.rs b/crates/ffi/tests/c_abi_lifecycle.rs index 295dd44..59e51f9 100644 --- a/crates/ffi/tests/c_abi_lifecycle.rs +++ b/crates/ffi/tests/c_abi_lifecycle.rs @@ -1,21 +1,34 @@ mod common; +use agent_desktop_core::NativeHandle; use common::{ - AdFindQuery, AdNativeHandle, AdResult, AdWindowInfo, AdWindowList, CStr, ad_adapter_create, - ad_adapter_destroy, ad_app_list_count, ad_app_list_free, ad_app_list_get, ad_check_permissions, - ad_find, ad_free_handle, ad_last_error_code, ad_last_error_message, ad_list_apps, - ad_list_windows, ad_window_list_count, ad_window_list_free, with_adapter, + AdExactSurfaceList, AdExactWindowList, AdFindQuery, AdNativeHandle, AdResult, AdWindowInfo, + AdWindowList, CStr, ad_adapter_create, ad_adapter_destroy, ad_app_list_count, ad_app_list_free, + ad_app_list_get, ad_check_permissions, ad_exact_surface_list_count, ad_exact_surface_list_free, + ad_exact_surface_list_get, ad_exact_window_list_count, ad_exact_window_list_free, + ad_exact_window_list_get, ad_find, ad_free_handle, ad_last_error_code, ad_last_error_message, + ad_list_apps, ad_list_surfaces_exact, ad_list_windows, ad_list_windows_exact, + ad_window_list_count, ad_window_list_free, with_adapter, }; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; + +struct DropProbe(Arc<AtomicU32>); + +impl Drop for DropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} #[test] fn null_adapter_rejected_without_ub() { unsafe { let mut list = std::ptr::null_mut(); let rc = ad_list_apps(std::ptr::null(), &mut list); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(list.is_null(), "out-param must stay null on failure"); let rc2 = ad_check_permissions(std::ptr::null()); @@ -27,10 +40,7 @@ fn null_adapter_rejected_without_ub() { fn null_out_param_rejected_before_write() { with_adapter(|adapter| unsafe { let rc = ad_list_apps(adapter, std::ptr::null_mut()); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } @@ -43,6 +53,14 @@ fn null_tolerance_on_list_accessors_and_free() { assert_eq!(ad_window_list_count(std::ptr::null()), 0); ad_window_list_free(std::ptr::null_mut()); + + assert_eq!(ad_exact_window_list_count(std::ptr::null()), 0); + assert!(ad_exact_window_list_get(std::ptr::null(), 0).is_null()); + ad_exact_window_list_free(std::ptr::null_mut()); + + assert_eq!(ad_exact_surface_list_count(std::ptr::null()), 0); + assert!(ad_exact_surface_list_get(std::ptr::null(), 0).is_null()); + ad_exact_surface_list_free(std::ptr::null_mut()); } } @@ -97,24 +115,40 @@ fn list_windows_focused_only_runs() { }); } +#[test] +fn exact_list_out_params_are_zeroed_on_stub_or_platform_failure() { + with_adapter(|adapter| unsafe { + let mut windows = std::ptr::dangling_mut::<AdExactWindowList>(); + let windows_rc = ad_list_windows_exact(adapter, std::ptr::null(), true, &mut windows); + if windows_rc == AdResult::Ok { + assert!(!windows.is_null()); + ad_exact_window_list_free(windows); + } else { + assert!(windows.is_null()); + } + + let mut surfaces = std::ptr::dangling_mut::<AdExactSurfaceList>(); + let surfaces_rc = ad_list_surfaces_exact(adapter, 1, &mut surfaces); + if surfaces_rc == AdResult::Ok { + assert!(!surfaces.is_null()); + ad_exact_surface_list_free(surfaces); + } else { + assert!(surfaces.is_null()); + } + }); +} + #[test] fn find_returns_not_found_on_empty_query_against_no_window() { with_adapter(|adapter| unsafe { let bad_win: AdWindowInfo = std::mem::zeroed(); - let query = AdFindQuery { - role: std::ptr::null(), - name_substring: std::ptr::null(), - value_substring: std::ptr::null(), - }; + let mut query: AdFindQuery = std::mem::zeroed(); + query.control.version = agent_desktop_ffi::AD_FIND_QUERY_VERSION; let mut handle = AdNativeHandle { ptr: std::ptr::null(), }; let rc = ad_find(adapter, &bad_win, &query, &mut handle); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "zeroed window must not succeed, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); }); } @@ -133,18 +167,20 @@ fn free_handle_null_is_noop() { }); } -#[cfg(not(target_os = "macos"))] #[test] -fn free_handle_zeroes_ptr_so_double_free_is_noop() { +fn free_handle_rejects_forged_allocation_pointer_without_dereferencing_it() { with_adapter(|adapter| unsafe { - let fake_live_ptr = 0x1234 as *const std::ffi::c_void; - let mut handle = AdNativeHandle { ptr: fake_live_ptr }; - - let _ = ad_free_handle(adapter, &mut handle); - assert!(handle.ptr.is_null()); + let drops = Arc::new(AtomicU32::new(0)); + let native = Box::new(NativeHandle::new(DropProbe(Arc::clone(&drops)))); + let raw = Box::into_raw(native); + let mut handle = AdNativeHandle { ptr: raw.cast() }; let rc = ad_free_handle(adapter, &mut handle); - assert_eq!(rc, AdResult::Ok); + assert_eq!(rc, AdResult::ErrInvalidArgs); + assert_eq!(handle.ptr, raw.cast()); + assert_eq!(drops.load(Ordering::SeqCst), 0); + drop(Box::from_raw(raw)); + assert_eq!(drops.load(Ordering::SeqCst), 1); }); } @@ -155,11 +191,7 @@ fn last_error_survives_successful_calls() { assert!(!adapter.is_null()); let mut out: AdWindowInfo = std::mem::zeroed(); let rc = common::ad_launch_app(adapter, std::ptr::null(), 0, &mut out); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "must fail, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); let msg_ptr = ad_last_error_message(); assert!(!msg_ptr.is_null()); diff --git a/crates/ffi/tests/c_abi_log_callback.rs b/crates/ffi/tests/c_abi_log_callback.rs index c3dcc9c..390ed8d 100644 --- a/crates/ffi/tests/c_abi_log_callback.rs +++ b/crates/ffi/tests/c_abi_log_callback.rs @@ -1,6 +1,6 @@ mod common; -use common::{AdResult, CStr, ad_set_log_callback}; +use common::{AdResult, CStr, ad_free_string, ad_set_log_callback, ad_snapshot, with_adapter}; use std::os::raw::c_char; use std::sync::Mutex; @@ -37,6 +37,14 @@ fn drain_recorder() -> Vec<Delivery> { .unwrap_or_default() } +fn emit_scoped_ffi_event() { + with_adapter(|adapter| unsafe { + let mut out = std::ptr::null_mut(); + let _ = ad_snapshot(adapter, std::ptr::null(), 0, 1, false, false, &mut out); + ad_free_string(out); + }); +} + #[test] fn log_callback_register_delivers_event() { let _guard = LOG_TEST_LOCK.lock().unwrap(); @@ -47,10 +55,7 @@ fn log_callback_register_delivers_event() { assert_eq!(rc, AdResult::Ok, "register must succeed"); } - tracing::error!( - test_marker = "deliver_event", - "log_callback_register_delivers_event" - ); + emit_scoped_ffi_event(); let deliveries = drain_recorder(); assert!( @@ -59,9 +64,9 @@ fn log_callback_register_delivers_event() { ); let d = deliveries .iter() - .find(|d| d.message.contains("deliver_event")) + .find(|d| d.message.contains("tree: snapshot")) .unwrap_or(&deliveries[0]); - assert_eq!(d.level, 1, "ERROR maps to level 1"); + assert_eq!(d.level, 4, "DEBUG maps to level 4"); assert!(!d.message.is_empty(), "message must be non-empty"); unsafe { @@ -79,12 +84,7 @@ fn log_callback_spawned_thread_does_not_panic() { assert_eq!(rc, AdResult::Ok); } - let handle = std::thread::spawn(|| { - tracing::warn!( - source = "spawned_thread", - "log_callback_spawned_thread_does_not_panic" - ); - }); + let handle = std::thread::spawn(emit_scoped_ffi_event); handle.join().expect("spawned thread must not panic"); let deliveries = drain_recorder(); @@ -110,7 +110,7 @@ fn log_callback_null_unregisters() { } clear_recorder(); - tracing::error!(test_marker = "after_null", "log_callback_null_unregisters"); + emit_scoped_ffi_event(); let deliveries = drain_recorder(); assert!( @@ -121,7 +121,7 @@ fn log_callback_null_unregisters() { } #[test] -fn log_callback_redacts_sensitive_fields() { +fn log_callback_does_not_capture_host_global_events() { let _guard = LOG_TEST_LOCK.lock().unwrap(); clear_recorder(); @@ -130,38 +130,10 @@ fn log_callback_redacts_sensitive_fields() { assert_eq!(rc, AdResult::Ok); } - tracing::error!( - password = "super_secret_password", - token = "bearer_xyz", - operation = "login_attempt", - "log_callback_redacts_sensitive_fields" - ); + tracing::error!(operation = "host_global_event"); let deliveries = drain_recorder(); - assert!(!deliveries.is_empty(), "expected at least one delivery"); - - let combined: String = deliveries - .iter() - .map(|d| d.message.as_str()) - .collect::<Vec<_>>() - .join(" "); - - assert!( - !combined.contains("super_secret_password"), - "raw password must not appear in callback message; got: {combined}" - ); - assert!( - !combined.contains("bearer_xyz"), - "raw token must not appear in callback message; got: {combined}" - ); - assert!( - combined.contains("redacted"), - "redaction marker must appear; got: {combined}" - ); - assert!( - combined.contains("login_attempt"), - "non-sensitive field value must be preserved; got: {combined}" - ); + assert!(deliveries.is_empty()); unsafe { let _ = ad_set_log_callback(None); diff --git a/crates/ffi/tests/c_abi_notifications.rs b/crates/ffi/tests/c_abi_notifications.rs new file mode 100644 index 0000000..7560ca3 --- /dev/null +++ b/crates/ffi/tests/c_abi_notifications.rs @@ -0,0 +1,39 @@ +mod common; + +use common::{ + AdActionResult, AdNotificationActionRequest, AdNotificationIdentity, AdPolicyKind, AdResult, + ad_last_error_code, ad_notification_action, with_adapter, +}; +use std::ffi::CString; + +#[test] +fn notification_action_rejects_null_request_before_platform_work() { + with_adapter(|adapter| unsafe { + let mut out = std::mem::zeroed::<AdActionResult>(); + let result = ad_notification_action(adapter, std::ptr::null(), &mut out); + assert_eq!(result, AdResult::ErrInvalidArgs); + assert_eq!(ad_last_error_code(), result); + }); +} + +#[test] +fn notification_action_headless_policy_fails_closed() { + with_adapter(|adapter| unsafe { + let app = CString::new("Slack").unwrap(); + let action = CString::new("Reply").unwrap(); + let request = AdNotificationActionRequest { + index: 1, + policy: AdPolicyKind::Headless as i32, + action_name: action.as_ptr(), + identity: AdNotificationIdentity { + app: app.as_ptr(), + title: std::ptr::null(), + }, + }; + let mut out = std::mem::zeroed::<AdActionResult>(); + let result = ad_notification_action(adapter, &request, &mut out); + assert_eq!(result, AdResult::ErrPolicyDenied); + assert_eq!(ad_last_error_code(), result); + assert!(out.action.is_null()); + }); +} diff --git a/crates/ffi/tests/c_abi_passthrough.rs b/crates/ffi/tests/c_abi_passthrough.rs index e31febe..361dea5 100644 --- a/crates/ffi/tests/c_abi_passthrough.rs +++ b/crates/ffi/tests/c_abi_passthrough.rs @@ -30,18 +30,6 @@ /// both `ErrPermDenied` and `ErrPlatformNotSupported` as "adapter not /// operational here". /// -/// # Main-thread tolerance -/// -/// `ad_snapshot`, `ad_wait`, and `ad_execute_by_ref` each call -/// `require_main_thread()` before touching the adapter. The libtest harness -/// spawns each `#[test]` on a worker thread, so on macOS those guards fire -/// first and return `ErrInternal` with no JSON envelope. The tests accept -/// either outcome: -/// - `ErrInternal` (off-main-thread on macOS, no envelope produced) — ok. -/// - `ErrPlatformNotSupported` + valid JSON envelope — ok. -/// -/// Both outcomes are correct; neither is a regression. -/// /// Commands gated by `#[cfg(feature = "stub-adapter")]` so they compile only /// when the feature is active. The normal test build (`cargo test -p /// agent-desktop-ffi --tests`) never compiles or runs this file. @@ -51,20 +39,16 @@ mod common; #[cfg(feature = "stub-adapter")] use common::{ AdResult, AdWaitArgs, CStr, ad_adapter_create, ad_adapter_destroy, ad_check_permissions, - ad_execute_by_ref, ad_free_string, ad_last_error_code, ad_last_error_message, ad_snapshot, - ad_status, ad_version, ad_wait, default_action, with_adapter, + ad_clear_clipboard, ad_execute_by_ref, ad_free_string, ad_get_clipboard, ad_last_error_code, + ad_last_error_message, ad_list_apps, ad_set_clipboard, ad_snapshot, ad_status, ad_version, + ad_wait, default_action, with_adapter, }; /// A helper that parses the JSON envelope written to `*out` and asserts -/// `PLATFORM_NOT_SUPPORTED` shape. Returns `true` when the envelope was -/// present and verified; `false` when `*out` is null (meaning the function -/// returned early — e.g. due to the macOS off-main-thread guard firing before -/// the adapter was reached). +/// `PLATFORM_NOT_SUPPORTED` shape. #[cfg(feature = "stub-adapter")] -unsafe fn assert_platform_not_supported_envelope(out: *mut std::os::raw::c_char) -> bool { - if out.is_null() { - return false; - } +unsafe fn assert_platform_not_supported_envelope(out: *mut std::os::raw::c_char) { + assert!(!out.is_null(), "command failure must produce an envelope"); let json_str = unsafe { CStr::from_ptr(out) } .to_str() .expect("envelope must be valid UTF-8"); @@ -85,7 +69,6 @@ unsafe fn assert_platform_not_supported_envelope(out: *mut std::os::raw::c_char) !suggestion.is_empty(), "error.suggestion must be non-empty — got: {json_str}" ); - true } /// `ad_version` has no adapter dependency and must always succeed even on a @@ -134,8 +117,8 @@ fn stub_ad_check_permissions_returns_err_perm_denied() { }); } -/// `ad_status` is not main-thread gated. Under the stub adapter it returns a -/// valid JSON envelope with `ok:true` because `execute_with_report_with_context` +/// Under the stub adapter `ad_status` returns a valid JSON envelope with +/// `ok:true` because `execute_with_report_with_context` /// reports the Denied permission state as a valid (non-error) status payload. /// /// This test asserts the envelope is produced and is valid JSON — the specific @@ -169,45 +152,23 @@ fn stub_ad_status_returns_valid_envelope() { }); } -/// `ad_snapshot` calls `require_main_thread()` before reaching the adapter. -/// On macOS the libtest worker thread fires the guard first, returning -/// `ErrInternal` with no envelope. On non-macOS the call proceeds to the stub -/// adapter and must produce a `PLATFORM_NOT_SUPPORTED` error envelope. +/// The stub adapter must produce a `PLATFORM_NOT_SUPPORTED` error envelope. #[cfg(feature = "stub-adapter")] #[test] -fn stub_ad_snapshot_platform_not_supported_or_off_main_thread() { +fn stub_ad_snapshot_returns_platform_not_supported_envelope() { with_adapter(|adapter| unsafe { let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_snapshot(adapter, std::ptr::null(), 0, 6, false, false, &mut out); - match rc { - AdResult::ErrInternal => { - assert!( - out.is_null(), - "ErrInternal (off-main-thread guard) must leave out null" - ); - } - AdResult::ErrPlatformNotSupported => { - let had_envelope = assert_platform_not_supported_envelope(out); - assert!( - had_envelope, - "ErrPlatformNotSupported must be accompanied by an error envelope" - ); - ad_free_string(out); - } - other => { - panic!( - "stub ad_snapshot must return ErrInternal (macOS off-main-thread) or \ - ErrPlatformNotSupported, got {other:?}" - ); - } - } + assert_eq!(rc, AdResult::ErrPlatformNotSupported); + assert_platform_not_supported_envelope(out); + ad_free_string(out); }); } /// `ad_wait` is exercised in its adapter-free `ms` mode: it sleeps for `ms` /// and returns an Ok envelope without touching the adapter, proving the -/// entrypoint is callable and structured under the stub without crashing (or -/// `ErrInternal` off the main thread on macOS). ad_wait's adapter-dependent +/// entrypoint is callable and structured under the stub without crashing. +/// ad_wait's adapter-dependent /// element/predicate modes need a real refmap from a successful snapshot, /// which the stub cannot produce, so PLATFORM_NOT_SUPPORTED parity for those /// modes is covered by the real E2E harness rather than this stub gate. @@ -216,55 +177,50 @@ fn stub_ad_snapshot_platform_not_supported_or_off_main_thread() { fn stub_ad_wait_ms_mode_callable_under_stub() { with_adapter(|adapter| unsafe { let args = AdWaitArgs { - ms: 1, - has_ms: true, - element: std::ptr::null(), - window: std::ptr::null(), - text: std::ptr::null(), - menu: false, - menu_closed: false, - notification: false, - snapshot_id: std::ptr::null(), - predicate: std::ptr::null(), - value: std::ptr::null(), - action: std::ptr::null(), - count: 0, - has_count: false, - timeout_ms: 200, - app: std::ptr::null(), + mode: common::AdWaitMode { + pause: common::AdOptionalU64 { + value: 1, + present: true, + }, + element: std::ptr::null(), + window: std::ptr::null(), + text: std::ptr::null(), + surfaces: common::AdWaitSurfaceModes { + menu: false, + menu_closed: false, + notification: false, + }, + }, + predicate: common::AdWaitPredicate { + snapshot_id: std::ptr::null(), + predicate: std::ptr::null(), + value: std::ptr::null(), + action: std::ptr::null(), + count: common::AdOptionalUsize { + value: 0, + present: false, + }, + }, + scope: common::AdWaitScope { + timeout_ms: 200, + app: std::ptr::null(), + }, }; let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_wait(adapter, &args, &mut out); - match rc { - AdResult::Ok => { - if !out.is_null() { - ad_free_string(out); - } - } - AdResult::ErrInternal => { - assert!( - out.is_null(), - "ErrInternal (off-main-thread guard) must leave out null" - ); - } - other => { - panic!( - "stub ad_wait (ms mode) must return Ok (adapter-free timer) or \ - ErrInternal (macOS off-main-thread), got {other:?}" - ); - } - } + assert_eq!(rc, AdResult::Ok); + assert!(!out.is_null()); + ad_free_string(out); }); } -/// `ad_execute_by_ref` calls `require_main_thread()` before reaching the -/// adapter. Same main-thread tolerance applies. Uses a syntactically valid -/// ref-id so arg-decode succeeds before the thread check. +/// A qualified ref reaches the ref-store path without relying on ambient +/// snapshot state. #[cfg(feature = "stub-adapter")] #[test] -fn stub_ad_execute_by_ref_platform_not_supported_or_off_main_thread() { +fn stub_ad_execute_by_ref_returns_structured_ref_error() { with_adapter(|adapter| unsafe { - let ref_id = std::ffi::CString::new("@e1").unwrap(); + let ref_id = std::ffi::CString::new("@stub-snapshot:e1").unwrap(); let action = default_action(); let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_execute_by_ref( @@ -276,29 +232,22 @@ fn stub_ad_execute_by_ref_platform_not_supported_or_off_main_thread() { &mut out, ); match rc { - AdResult::ErrInternal => { - assert!( - out.is_null(), - "ErrInternal (off-main-thread guard) must leave out null" - ); - } AdResult::ErrPlatformNotSupported | AdResult::ErrSnapshotNotFound | AdResult::ErrStaleRef => { - if !out.is_null() { - let json_str = CStr::from_ptr(out) - .to_str() - .expect("envelope must be valid UTF-8"); - let parsed: serde_json::Value = - serde_json::from_str(json_str).expect("envelope must be valid JSON"); - assert_eq!(parsed["ok"].as_bool(), Some(false)); - assert!(!parsed["error"]["code"].as_str().unwrap_or("").is_empty()); - ad_free_string(out); - } + assert!(!out.is_null()); + let json_str = CStr::from_ptr(out) + .to_str() + .expect("envelope must be valid UTF-8"); + let parsed: serde_json::Value = + serde_json::from_str(json_str).expect("envelope must be valid JSON"); + assert_eq!(parsed["ok"].as_bool(), Some(false)); + assert!(!parsed["error"]["code"].as_str().unwrap_or("").is_empty()); + ad_free_string(out); } other => { panic!( - "stub ad_execute_by_ref must return ErrInternal, ErrPlatformNotSupported, \ + "stub ad_execute_by_ref must return ErrPlatformNotSupported, \ ErrSnapshotNotFound, or ErrStaleRef, got {other:?}" ); } @@ -320,3 +269,34 @@ fn stub_adapter_create_and_destroy_round_trip() { ad_adapter_destroy(adapter); } } + +/// AppKit-backed platform implementations remain callable from worker threads; +/// the macOS adapter supplies its own autorelease pools. The stub proves that +/// the FFI boundary reaches the adapter rather than rejecting the thread. +#[cfg(feature = "stub-adapter")] +#[test] +fn app_and_clipboard_families_reach_stub_from_worker_thread() { + let results = std::thread::spawn(|| unsafe { + let adapter = ad_adapter_create(); + assert!(!adapter.is_null()); + + let mut apps = std::ptr::null_mut(); + let list_apps = ad_list_apps(adapter, &mut apps); + assert!(apps.is_null()); + + let mut clipboard = std::ptr::null_mut(); + let get_clipboard = ad_get_clipboard(adapter, &mut clipboard); + assert!(clipboard.is_null()); + + let text = std::ffi::CString::new("worker-thread").unwrap(); + let set_clipboard = ad_set_clipboard(adapter, text.as_ptr()); + let clear_clipboard = ad_clear_clipboard(adapter); + + ad_adapter_destroy(adapter); + [list_apps, get_clipboard, set_clipboard, clear_clipboard] + }) + .join() + .unwrap(); + + assert_eq!(results, [AdResult::ErrPlatformNotSupported; 4]); +} diff --git a/crates/ffi/tests/c_abi_roundtrip.rs b/crates/ffi/tests/c_abi_roundtrip.rs index 07894b7..5621a25 100644 --- a/crates/ffi/tests/c_abi_roundtrip.rs +++ b/crates/ffi/tests/c_abi_roundtrip.rs @@ -6,23 +6,16 @@ /// Sets a temp HOME (empty refmap), calls `ad_execute_by_ref`, and asserts /// that the returned envelope has `ok:false` with both `error.code` (string) /// and `error.message` (string), and that `command` equals `"execute_by_ref"`. -/// On macOS the main-thread guard fires before the command path is reached -/// (libtest runs bodies off the main thread), returning `ErrInternal` with a -/// null `*out` — that path is tolerated. -/// /// - `snapshot_execute_by_ref_live_roundtrip` — marked `#[ignore]`. /// The full observe→act loop (real `ad_snapshot` → `@e` ref → -/// `ad_execute_by_ref` against a live app) cannot run under the default -/// libtest harness on macOS: libtest schedules test bodies on worker threads, -/// and the AX/main-thread guard blocks every AX call made off the main -/// thread. CI proof of the full loop is tracked as the deferred -/// external-consumer smoke harness (Python ctypes) — plan unit U9 / Phase B. +/// `ad_execute_by_ref` against a live app) requires Accessibility permission, +/// a stable target app, and an interactive macOS runner. CI proof is tracked +/// by the external-consumer smoke harness. /// To run the roundtrip manually: /// ```text /// cargo test -p agent-desktop-ffi --tests c_abi_roundtrip \ /// snapshot_execute_by_ref_live_roundtrip -- --ignored /// ``` -/// Run from a process that owns the main thread (e.g. the E2E harness). /// AX accessibility permission must be granted and a target app must be open. mod common; @@ -71,14 +64,11 @@ impl Drop for TestHome { /// Verifies the error-envelope contract at the C boundary when no refmap exists. /// /// Sets a temp HOME so the refmap store is empty. Calls `ad_execute_by_ref` -/// with a well-formed `@e1` ref. On macOS the main-thread guard fires first -/// (libtest runs off the main thread), returning `ErrInternal` with `out` -/// remaining null — that is tolerated. When the command path does execute -/// (non-macOS or future main-thread execution), the envelope must have +/// with a well-formed qualified ref. The envelope must have /// `ok:false` and carry both `error.code` (string) and `error.message` /// (string) — the latter guaranteed by the CLAUDE.md error contract — proving /// the unified error-envelope contract holds at the ABI boundary. The exact -/// `error.code` value is not pinned: an empty refmap with an `@e1` ref may +/// `error.code` value is not pinned: an empty refmap may /// surface either `SNAPSHOT_NOT_FOUND` or `STALE_REF` depending on the load /// path; pinning either would make this test flaky. #[test] @@ -86,7 +76,7 @@ fn stale_ref_returns_ok_false_error_envelope() { let _home = TestHome::new(); with_adapter_raw(|adapter| unsafe { - let ref_id = std::ffi::CString::new("@e1").unwrap(); + let ref_id = std::ffi::CString::new("@smissing:e1").unwrap(); let action = default_action(); let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); @@ -105,14 +95,7 @@ fn stale_ref_returns_ok_false_error_envelope() { "must return a valid AdResult (<=0), got {rc_i32}" ); - if out.is_null() { - assert_eq!( - rc, - AdResult::ErrInternal, - "null *out is only expected when ErrInternal (main-thread guard on macOS), got {rc:?}" - ); - return; - } + assert!(!out.is_null(), "command failure must return an envelope"); let s = CStr::from_ptr(out).to_string_lossy(); let val: serde_json::Value = serde_json::from_str(&s).expect("response must be valid JSON"); @@ -152,28 +135,23 @@ fn stale_ref_returns_ok_false_error_envelope() { /// /// This test is `#[ignore]` by design and must remain so in headless CI. /// -/// **Why ignored**: libtest schedules test bodies on worker threads. On macOS -/// every AX API call (including `ad_snapshot` and `ad_execute_by_ref`) must -/// originate on the main thread; the AX guard returns `ErrInternal` immediately -/// when called off it. There is no libtest API to pin a test to the main -/// thread, so the full observe→act loop cannot be verified inside a libtest -/// integration test on macOS. +/// **Why ignored**: the test needs Accessibility permission, a live interactive +/// app, and exclusive control over mutable UI state. Those conditions are not +/// available on generic headless CI. /// /// **Deferred CI proof**: the full-loop proof — real `ad_snapshot` producing /// `@e` refs consumed by `ad_execute_by_ref` against a live app at the C /// boundary — is tracked as plan unit U9 / Phase B: an external-consumer smoke -/// harness (Python ctypes) that runs in the E2E environment where the process -/// itself owns the main thread. +/// harness (Python ctypes) that runs in the macOS E2E environment. /// /// **Manual execution** (requires AX permission + a running target app): /// ```text /// cargo test -p agent-desktop-ffi --tests c_abi_roundtrip \ /// snapshot_execute_by_ref_live_roundtrip -- --ignored /// ``` -/// Run from a process that owns the main thread (e.g. the E2E harness). Do -/// NOT un-ignore this test — it will fail in any headless CI that uses libtest. +/// Do not un-ignore this test on generic CI. #[test] -#[ignore = "requires AX permission, a live app, and main-thread execution — run via E2E harness"] +#[ignore = "requires AX permission, a live app, and exclusive interactive UI control"] fn snapshot_execute_by_ref_live_roundtrip() { let _home = TestHome::new(); @@ -251,7 +229,7 @@ fn search_refs(val: &serde_json::Value) -> Option<String> { serde_json::Value::Object(map) => { if let Some(r) = map.get("ref") { if let Some(s) = r.as_str() { - if s.starts_with("@e") { + if s.starts_with("@e") || (s.starts_with("@s") && s.contains(":e")) { return Some(s.to_owned()); } } diff --git a/crates/ffi/tests/c_abi_session.rs b/crates/ffi/tests/c_abi_session.rs index 2ee2bfa..f83363d 100644 --- a/crates/ffi/tests/c_abi_session.rs +++ b/crates/ffi/tests/c_abi_session.rs @@ -2,53 +2,45 @@ mod common; use common::{ AdResult, CStr, ad_adapter_create, ad_adapter_create_with_session, ad_adapter_destroy, - ad_last_error_code, ad_last_error_message, + ad_free_string, ad_last_error_code, ad_last_error_message, ad_status, with_isolated_home, }; #[test] -fn sessionless_adapter_has_no_session_id() { - unsafe { - let ptr = ad_adapter_create(); - assert!(!ptr.is_null(), "ad_adapter_create must not return null"); - let ctx = (*ptr) - .command_context() - .expect("command_context must succeed"); - assert_eq!(ctx.session_id(), None); - ad_adapter_destroy(ptr); - } -} +fn adapter_creation_accepts_sessionless_null_and_valid_session_forms() { + with_isolated_home(|| unsafe { + let sessionless = ad_adapter_create(); + assert!(!sessionless.is_null()); + assert_eq!(status_session_id(sessionless), None); + ad_adapter_destroy(sessionless); + + let null_session = ad_adapter_create_with_session(std::ptr::null()); + assert!(!null_session.is_null()); + assert_eq!(status_session_id(null_session), None); + ad_adapter_destroy(null_session); -#[test] -fn session_adapter_carries_session_id() { - unsafe { let session = std::ffi::CString::new("agent-a").unwrap(); let ptr = ad_adapter_create_with_session(session.as_ptr()); assert!( !ptr.is_null(), "ad_adapter_create_with_session must not return null" ); - let ctx = (*ptr) - .command_context() - .expect("command_context must succeed"); - assert_eq!(ctx.session_id(), Some("agent-a")); + assert_eq!(status_session_id(ptr).as_deref(), Some("agent-a")); ad_adapter_destroy(ptr); - } + }); } -#[test] -fn null_session_adapter_is_sessionless() { - unsafe { - let ptr = ad_adapter_create_with_session(std::ptr::null()); - assert!( - !ptr.is_null(), - "null session must yield a sessionless adapter" - ); - let ctx = (*ptr) - .command_context() - .expect("command_context must succeed"); - assert_eq!(ctx.session_id(), None); - ad_adapter_destroy(ptr); - } +unsafe fn status_session_id(adapter: *const common::AdAdapter) -> Option<String> { + let mut out = std::ptr::null_mut(); + assert_eq!(unsafe { ad_status(adapter, &mut out) }, AdResult::Ok); + assert!(!out.is_null()); + let envelope: serde_json::Value = serde_json::from_str( + unsafe { CStr::from_ptr(out) } + .to_str() + .expect("status envelope must be UTF-8"), + ) + .expect("status envelope must be JSON"); + unsafe { ad_free_string(out) }; + envelope["data"]["session_id"].as_str().map(str::to_owned) } #[test] diff --git a/crates/ffi/tests/c_abi_session_trace.rs b/crates/ffi/tests/c_abi_session_trace.rs index 44016ca..76fb8a5 100644 --- a/crates/ffi/tests/c_abi_session_trace.rs +++ b/crates/ffi/tests/c_abi_session_trace.rs @@ -1,22 +1,23 @@ mod common; use agent_desktop_core::session::{ - SessionTraceMode, StartSessionOptions, start_session, trace_dir, + GcOptions, SessionTraceMode, StartSessionOptions, gc, start_session, trace_dir, write_manifest, }; use common::{ - AdResult, CStr, ad_adapter_create_with_session, ad_adapter_destroy, ad_check_permissions, + AdResult, ad_adapter_create_with_session, ad_adapter_destroy, ad_check_permissions, ad_free_string, ad_status, }; use std::ffi::CString; use std::fs; use std::sync::Mutex; +use std::time::Duration; static HOME_LOCK: Mutex<()> = Mutex::new(()); struct TestHome { _lock: std::sync::MutexGuard<'static, ()>, dir: std::path::PathBuf, - prev: Option<std::ffi::OsString>, + previous: Option<std::ffi::OsString>, } impl TestHome { @@ -30,174 +31,132 @@ impl TestHome { .as_nanos() )); fs::create_dir_all(&dir).unwrap(); - let prev = std::env::var_os("HOME"); + let previous = std::env::var_os("HOME"); unsafe { std::env::set_var("HOME", &dir) }; Self { _lock: lock, dir, - prev, + previous, } } } impl Drop for TestHome { fn drop(&mut self) { - match self.prev.as_ref() { - Some(prev) => unsafe { std::env::set_var("HOME", prev) }, + match self.previous.as_ref() { + Some(previous) => unsafe { std::env::set_var("HOME", previous) }, None => unsafe { std::env::remove_var("HOME") }, } let _ = fs::remove_dir_all(&self.dir); } } -fn trace_dir_for(session_id: &str) -> std::path::PathBuf { - trace_dir(session_id).unwrap() -} - -#[test] -fn ffi_trace_on_session_writes_segment() { - let _home = TestHome::new(); - let manifest = start_session(StartSessionOptions { - name: None, - trace: SessionTraceMode::On, - force: false, - ..Default::default() - }) - .unwrap(); - for call in 0..2 { - unsafe { - let session = CString::new(manifest.id.as_str()).unwrap(); - let ptr = ad_adapter_create_with_session(session.as_ptr()); - assert!(!ptr.is_null()); - let ctx = (*ptr) - .command_context() - .expect("command_context must succeed"); - ctx.trace("ffi.event", serde_json::json!({ "call": call })) - .unwrap(); - ad_adapter_destroy(ptr); - } - } - let trace_dir = trace_dir_for(&manifest.id); - let segments: Vec<_> = fs::read_dir(trace_dir) +fn trace_segments(session_id: &str) -> Vec<std::path::PathBuf> { + fs::read_dir(trace_dir(session_id).unwrap()) .unwrap() .flatten() - .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "jsonl")) - .collect(); - assert_eq!( - segments.len(), - 1, - "a long-lived process must write one segment across many FFI calls" - ); - let body = fs::read_to_string(segments[0].path()).unwrap(); - assert!(body.contains("\"call\":0")); - assert!(body.contains("\"call\":1")); + .map(|entry| entry.path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "jsonl") + }) + .collect() } -#[test] -fn ffi_plain_session_writes_no_trace_files() { - let _home = TestHome::new(); +unsafe fn call_status(session_id: &str) { + let session = CString::new(session_id).unwrap(); + let adapter = unsafe { ad_adapter_create_with_session(session.as_ptr()) }; + assert!(!adapter.is_null()); + let mut out = std::ptr::null_mut(); + let result = unsafe { ad_status(adapter, &mut out) }; + assert_eq!(result, AdResult::Ok); + assert!(!out.is_null()); unsafe { - let session = CString::new("plain-session").unwrap(); - let ptr = ad_adapter_create_with_session(session.as_ptr()); - assert!(!ptr.is_null()); - let ctx = (*ptr) - .command_context() - .expect("command_context must succeed"); - ctx.trace("ffi.event", serde_json::json!({})).unwrap(); - ad_check_permissions(ptr); - ad_adapter_destroy(ptr); - } - assert!(!trace_dir_for("plain-session").exists()); -} - -/// R8 claims `command.start`/`command.end` boundary events fire over FFI via -/// the generated entrypoints. The two tests above only prove the trace -/// plumbing itself works by hand-crafting a `CommandContext::trace` call — -/// they never go through a real `ad_*` entrypoint's codegen-injected -/// `context.command_scope(...)` / `scope.complete(...)` pair. -/// -/// This test drives the real `ad_status` entrypoint end-to-end (chosen -/// because it needs no accessibility permission and no main-thread affinity, -/// so it runs headless in CI — see `crates/ffi/src/commands/generated.rs`) -/// under a trace-enabled session, then reads the on-disk trace segment and -/// asserts both boundary events were actually written by the generated code, -/// in the right order. -#[test] -fn ffi_real_ad_status_entrypoint_emits_command_start_and_end_trace_events() { - let _home = TestHome::new(); - let manifest = start_session(StartSessionOptions { - name: None, - trace: SessionTraceMode::On, - force: false, - ..Default::default() - }) - .unwrap(); - - unsafe { - let session = CString::new(manifest.id.as_str()).unwrap(); - let ptr = ad_adapter_create_with_session(session.as_ptr()); - assert!(!ptr.is_null()); - - let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); - let rc = ad_status(ptr, &mut out); - assert_eq!( - rc, - AdResult::Ok, - "the real ad_status entrypoint must succeed under a trace-enabled session" - ); - assert!( - !out.is_null(), - "ad_status must produce an envelope on success" - ); - let body = CStr::from_ptr(out).to_string_lossy().into_owned(); - assert!( - body.contains("\"command\":\"status\""), - "envelope command must be 'status', got: {body}" - ); ad_free_string(out); - ad_adapter_destroy(ptr); + ad_adapter_destroy(adapter); + } +} + +#[test] +fn traced_ffi_commands_reuse_one_process_segment_and_emit_ordered_boundaries() { + let _home = TestHome::new(); + let manifest = start_session(StartSessionOptions { + name: None, + trace: SessionTraceMode::On, + ..Default::default() + }) + .unwrap(); + + unsafe { + call_status(&manifest.id); + call_status(&manifest.id); } - let trace_dir = trace_dir_for(&manifest.id); - let segments: Vec<_> = fs::read_dir(trace_dir) + let segments = trace_segments(&manifest.id); + assert_eq!(segments.len(), 1); + let events: Vec<serde_json::Value> = fs::read_to_string(&segments[0]) .unwrap() - .flatten() - .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "jsonl")) - .collect(); - assert_eq!( - segments.len(), - 1, - "a single FFI call must write exactly one trace segment" - ); - - let trace_body = fs::read_to_string(segments[0].path()).unwrap(); - let events: Vec<serde_json::Value> = trace_body .lines() .map(|line| serde_json::from_str(line).unwrap()) + .filter(|event: &serde_json::Value| event["command"].as_str() == Some("status")) .collect(); - - let start_idx = events.iter().position(|event| { - event["event"].as_str() == Some("command.start") - && event["command"].as_str() == Some("status") - }); - let end_idx = events.iter().position(|event| { - event["event"].as_str() == Some("command.end") - && event["command"].as_str() == Some("status") - && event["ok"].as_bool() == Some(true) - }); - - assert!( - start_idx.is_some(), - "trace segment must contain a command.start event fired by ad_status's real \ - command_scope(\"status\") call, got: {trace_body}" - ); - assert!( - end_idx.is_some(), - "trace segment must contain a command.end event fired by ad_status's real \ - scope.complete(...) call, got: {trace_body}" - ); - assert!( - start_idx < end_idx, - "command.start must be recorded before command.end for the same invocation" + let boundaries: Vec<_> = events + .iter() + .filter_map(|event| event["event"].as_str()) + .filter(|event| matches!(*event, "command.start" | "command.end")) + .collect(); + assert_eq!( + boundaries, + [ + "command.start", + "command.end", + "command.start", + "command.end" + ] ); } + +#[test] +fn manifestless_session_does_not_create_trace_files() { + let _home = TestHome::new(); + let session_id = "plain-session"; + unsafe { + let session = CString::new(session_id).unwrap(); + let adapter = ad_adapter_create_with_session(session.as_ptr()); + assert!(!adapter.is_null()); + let _ = ad_check_permissions(adapter); + ad_adapter_destroy(adapter); + } + assert!(!trace_dir(session_id).unwrap().exists()); +} + +#[test] +fn session_scoped_adapter_holds_liveness_until_destroyed() { + let _home = TestHome::new(); + let mut manifest = start_session(StartSessionOptions { + name: None, + trace: SessionTraceMode::Off, + ..Default::default() + }) + .unwrap(); + manifest.created_at = 0; + write_manifest(&manifest).unwrap(); + let session = CString::new(manifest.id.as_str()).unwrap(); + let adapter = unsafe { ad_adapter_create_with_session(session.as_ptr()) }; + assert!(!adapter.is_null()); + + let retained = gc(GcOptions { + ended_only: false, + older_than: Some(Duration::ZERO), + }) + .unwrap(); + assert!(!retained.removed.contains(&manifest.id)); + + unsafe { ad_adapter_destroy(adapter) }; + let removed = gc(GcOptions { + ended_only: false, + older_than: Some(Duration::ZERO), + }) + .unwrap(); + assert!(removed.removed.contains(&manifest.id)); +} diff --git a/crates/ffi/tests/c_abi_snapshot.rs b/crates/ffi/tests/c_abi_snapshot.rs index 9992463..c527903 100644 --- a/crates/ffi/tests/c_abi_snapshot.rs +++ b/crates/ffi/tests/c_abi_snapshot.rs @@ -17,7 +17,7 @@ fn snapshot_null_out_returns_invalid_args() { assert_eq!( rc, AdResult::ErrInvalidArgs, - "null out is rejected by the outer guard before any adapter or thread check" + "null out is rejected before adapter work" ); }); } @@ -35,10 +35,7 @@ fn snapshot_null_adapter_rejected() { false, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "null adapter must fail — got {rc:?} (ErrInternal on macOS off-main-thread is expected)" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on null-adapter failure"); } } @@ -57,10 +54,7 @@ fn snapshot_invalid_utf8_app_rejected() { false, &mut out, ); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "invalid UTF-8 app must fail — got {rc:?}" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!( out.is_null(), "out must stay null on arg validation failure" @@ -73,10 +67,7 @@ fn snapshot_invalid_surface_rejected() { with_adapter(|adapter| unsafe { let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_snapshot(adapter, std::ptr::null(), 99, 6, false, false, &mut out); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "out-of-range surface must fail — got {rc:?}" - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on invalid surface"); }); } diff --git a/crates/ffi/tests/c_abi_wait.rs b/crates/ffi/tests/c_abi_wait.rs index 89778fb..f79b78e 100644 --- a/crates/ffi/tests/c_abi_wait.rs +++ b/crates/ffi/tests/c_abi_wait.rs @@ -1,20 +1,49 @@ mod common; use common::{ - AdResult, AdWaitArgs, CStr, ad_free_string, ad_last_error_code, ad_last_error_message, ad_wait, - with_adapter, + AdOptionalU64, AdOptionalUsize, AdResult, AdWaitArgs, AdWaitMode, AdWaitPredicate, AdWaitScope, + AdWaitSurfaceModes, CStr, ad_free_string, ad_last_error_code, ad_wait, with_adapter, }; +fn wait_args(ms: Option<u64>, element: *const std::os::raw::c_char, timeout_ms: u64) -> AdWaitArgs { + AdWaitArgs { + mode: AdWaitMode { + pause: AdOptionalU64 { + value: ms.unwrap_or_default(), + present: ms.is_some(), + }, + element, + window: std::ptr::null(), + text: std::ptr::null(), + surfaces: AdWaitSurfaceModes { + menu: false, + menu_closed: false, + notification: false, + }, + }, + predicate: AdWaitPredicate { + snapshot_id: std::ptr::null(), + predicate: std::ptr::null(), + value: std::ptr::null(), + action: std::ptr::null(), + count: AdOptionalUsize { + value: 0, + present: false, + }, + }, + scope: AdWaitScope { + timeout_ms, + app: std::ptr::null(), + }, + } +} + #[test] fn ad_wait_null_args_rejected() { with_adapter(|adapter| unsafe { let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_wait(adapter, std::ptr::null(), &mut out); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "null args must be rejected, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert!(out.is_null(), "out must stay null on null-args rejection"); assert_eq!( ad_last_error_code(), @@ -27,30 +56,9 @@ fn ad_wait_null_args_rejected() { #[test] fn ad_wait_null_out_rejected() { with_adapter(|adapter| unsafe { - let args = AdWaitArgs { - ms: 1, - has_ms: true, - element: std::ptr::null(), - window: std::ptr::null(), - text: std::ptr::null(), - menu: false, - menu_closed: false, - notification: false, - snapshot_id: std::ptr::null(), - predicate: std::ptr::null(), - value: std::ptr::null(), - action: std::ptr::null(), - count: 0, - has_count: false, - timeout_ms: 500, - app: std::ptr::null(), - }; + let args = wait_args(Some(1), std::ptr::null(), 500); let rc = ad_wait(adapter, &args, std::ptr::null_mut()); - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "null out must be rejected, got {:?}", - rc - ); + assert_eq!(rc, AdResult::ErrInvalidArgs); assert_eq!( ad_last_error_code(), rc, @@ -60,51 +68,19 @@ fn ad_wait_null_out_rejected() { } #[test] -fn ad_wait_ms_mode_returns_ok_or_off_thread_error() { +fn ad_wait_ms_mode_returns_ok_from_worker_thread() { with_adapter(|adapter| unsafe { - let args = AdWaitArgs { - ms: 50, - has_ms: true, - element: std::ptr::null(), - window: std::ptr::null(), - text: std::ptr::null(), - menu: false, - menu_closed: false, - notification: false, - snapshot_id: std::ptr::null(), - predicate: std::ptr::null(), - value: std::ptr::null(), - action: std::ptr::null(), - count: 0, - has_count: false, - timeout_ms: 500, - app: std::ptr::null(), - }; + let args = wait_args(Some(50), std::ptr::null(), 500); let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_wait(adapter, &args, &mut out); - match rc { - AdResult::Ok => { - assert!(!out.is_null(), "Ok result must set out"); - let json_cstr = CStr::from_ptr(out); - let json: serde_json::Value = - serde_json::from_str(json_cstr.to_str().unwrap()).unwrap(); - assert_eq!(json["ok"], serde_json::Value::Bool(true)); - assert_eq!(json["command"], "wait"); - ad_free_string(out); - } - AdResult::ErrInternal => { - assert!(out.is_null(), "ErrInternal must leave out null"); - let msg = ad_last_error_message(); - assert!(!msg.is_null(), "error message must be set on failure"); - assert_eq!( - ad_last_error_code(), - AdResult::ErrInternal, - "last-error code must match returned AdResult (errno invariant)" - ); - } - other => panic!("unexpected result from ms-mode ad_wait: {:?}", other), - } + assert_eq!(rc, AdResult::Ok); + assert!(!out.is_null(), "Ok result must set out"); + let json_cstr = CStr::from_ptr(out); + let json: serde_json::Value = serde_json::from_str(json_cstr.to_str().unwrap()).unwrap(); + assert_eq!(json["ok"], serde_json::Value::Bool(true)); + assert_eq!(json["command"], "wait"); + ad_free_string(out); }); } @@ -112,24 +88,7 @@ fn ad_wait_ms_mode_returns_ok_or_off_thread_error() { fn ad_wait_command_error_writes_error_envelope_into_out() { with_adapter(|adapter| unsafe { let elem = std::ffi::CString::new("__nonexistent_element__").unwrap(); - let args = AdWaitArgs { - ms: 0, - has_ms: false, - element: elem.as_ptr(), - window: std::ptr::null(), - text: std::ptr::null(), - menu: false, - menu_closed: false, - notification: false, - snapshot_id: std::ptr::null(), - predicate: std::ptr::null(), - value: std::ptr::null(), - action: std::ptr::null(), - count: 0, - has_count: false, - timeout_ms: 0, - app: std::ptr::null(), - }; + let args = wait_args(None, elem.as_ptr(), 0); let mut out: *mut std::os::raw::c_char = std::ptr::null_mut(); let rc = ad_wait(adapter, &args, &mut out); @@ -138,12 +97,6 @@ fn ad_wait_command_error_writes_error_envelope_into_out() { assert!(!out.is_null(), "Ok result must set out"); ad_free_string(out); } - AdResult::ErrInternal => { - assert!( - out.is_null(), - "ErrInternal from off-main-thread guard must leave out null" - ); - } _ => { assert!( !out.is_null(), diff --git a/crates/ffi/tests/c_header_compile.rs b/crates/ffi/tests/c_header_compile.rs index 2c8ded5..4dc6525 100644 --- a/crates/ffi/tests/c_header_compile.rs +++ b/crates/ffi/tests/c_header_compile.rs @@ -49,14 +49,26 @@ fn committed_header_compiles_with_every_public_enum_constant() { #include <stddef.h> #include "agent_desktop.h" +static void log_callback(int32_t level, const char *message) { + (void)level; + (void)message; +} + int main(void) { (void)AD_ACTION_KIND_CLICK; (void)AD_DIRECTION_UP; + (void)AD_DELIVERY_DISPOSITION_DELIVERED_VERIFIED; + (void)AD_FIND_SELECTION_KIND_STRICT; + (void)AD_IDENTIFIER_KIND_AX_IDENTIFIER; + (void)AD_MODIFIER_META; (void)AD_MODIFIER_CMD; (void)AD_MOUSE_BUTTON_LEFT; (void)AD_MOUSE_EVENT_KIND_MOVE; + (void)AD_POLICY_KIND_HEADLESS; + (void)AD_RETRY_DISPOSITION_SAFE; (void)AD_SCREENSHOT_KIND_FULL_SCREEN; (void)AD_SNAPSHOT_SURFACE_WINDOW; + (void)AD_STEP_MECHANISM_SEMANTIC_API; (void)AD_WINDOW_OP_KIND_RESIZE; (void)AD_IMAGE_FORMAT_PNG; (void)AD_RESULT_OK; @@ -66,13 +78,46 @@ int main(void) { _Static_assert(AD_ACTION_RESULT_SIZE == sizeof(AdActionResult), "AdActionResult size macro drifted"); _Static_assert(offsetof(AdActionResult, steps) == 24, "AdActionResult.steps offset changed"); _Static_assert(offsetof(AdActionResult, step_count) == 32, "AdActionResult.step_count offset changed"); + _Static_assert(offsetof(AdActionResult, details_json) == 40, "AdActionResult.details_json offset changed"); + _Static_assert(offsetof(AdActionResult, disposition) == 48, "AdActionResult.disposition offset changed"); + _Static_assert(AD_DELIVERY_SEMANTICS_SIZE == sizeof(AdDeliverySemantics), "AdDeliverySemantics size macro drifted"); _Static_assert(offsetof(AdActionStep, outcome) == 8, "AdActionStep.outcome offset changed"); + _Static_assert(offsetof(AdActionStep, mechanism) == 16, "AdActionStep.mechanism offset changed"); + _Static_assert(offsetof(AdActionStep, has_mechanism) == 20, "AdActionStep.has_mechanism offset changed"); + _Static_assert(offsetof(AdActionStep, verified) == 21, "AdActionStep.verified offset changed"); + _Static_assert(offsetof(AdActionStep, has_verified) == 22, "AdActionStep.has_verified offset changed"); _Static_assert(AD_ELEMENT_STATE_SIZE == sizeof(AdElementState), "AdElementState size macro drifted"); + _Static_assert(AD_DISPLAY_INFO_SIZE == sizeof(AdDisplayInfo), "AdDisplayInfo size macro drifted"); + _Static_assert(offsetof(AdDisplayInfo, id) == 8, "AdDisplayInfo.id offset changed"); + _Static_assert(offsetof(AdDisplayInfo, scale) == 56, "AdDisplayInfo.scale offset changed"); + _Static_assert(_Generic(((AdAppInfo){0}).pid, uint32_t: 1, default: 0), "AdAppInfo.pid must be uint32_t"); + _Static_assert(_Generic(((AdWindowInfo){0}).pid, uint32_t: 1, default: 0), "AdWindowInfo.pid must be uint32_t"); + _Static_assert(_Generic(((AdRefProcess){0}).pid, uint32_t: 1, default: 0), "AdRefProcess.pid must be uint32_t"); + _Static_assert(_Generic(((AdScreenshotTarget){0}).pid, uint32_t: 1, default: 0), "AdScreenshotTarget.pid must be uint32_t"); + AdResult (*list_surfaces)(const struct AdAdapter *, uint32_t, struct AdSurfaceList **) = ad_list_surfaces; + AdResult (*list_surfaces_exact)(const struct AdAdapter *, uint32_t, struct AdExactSurfaceList **) = ad_list_surfaces_exact; + AdResult (*list_displays)(const struct AdAdapter *, struct AdDisplayList **) = ad_list_displays; + AdResult callback_result = ad_set_log_callback(log_callback); + AdResult clear_callback_result = ad_set_log_callback(NULL); + (void)list_surfaces; + (void)list_surfaces_exact; + (void)list_displays; + (void)callback_result; + (void)clear_callback_result; (void)ad_action_step_size; (void)ad_ref_entry_size; + (void)ad_exact_ref_entry_size; + (void)ad_exact_surface_info_size; + (void)ad_exact_window_info_size; + (void)ad_display_info_size; (void)ad_last_error_details; + (void)ad_last_error_delivery_semantics; _Static_assert(AD_REF_ENTRY_SIZE == sizeof(AdRefEntry), "AdRefEntry size macro drifted"); - _Static_assert(AD_REF_ENTRY_SIZE == 192, "AdRefEntry ABI size changed"); + _Static_assert(AD_REF_ENTRY_SIZE == 200, "AdRefEntry ABI size changed"); + _Static_assert(AD_EXACT_REF_ENTRY_SIZE == sizeof(AdExactRefEntry), "AdExactRefEntry size macro drifted"); + _Static_assert(AD_EXACT_REF_ENTRY_SIZE == 224, "AdExactRefEntry ABI size changed"); + _Static_assert(AD_EXACT_SURFACE_INFO_SIZE == sizeof(AdExactSurfaceInfo), "AdExactSurfaceInfo size macro drifted"); + _Static_assert(AD_EXACT_WINDOW_INFO_SIZE == sizeof(AdExactWindowInfo), "AdExactWindowInfo size macro drifted"); return 0; } "#; @@ -80,6 +125,8 @@ int main(void) { let include = header_include_dir(); let status = Command::new(cc) + .arg("-std=c11") + .arg("-pedantic-errors") .arg("-Wall") .arg("-Werror") .arg("-I") diff --git a/crates/ffi/tests/cdylib_panic_probe.c b/crates/ffi/tests/cdylib_panic_probe.c new file mode 100644 index 0000000..816bb71 --- /dev/null +++ b/crates/ffi/tests/cdylib_panic_probe.c @@ -0,0 +1,29 @@ +#include <dlfcn.h> +#include <stdint.h> +#include <stdio.h> + +typedef int32_t (*panic_boundary_fn)(void); + +int main(int argc, char **argv) { + if (argc != 2) { + return 2; + } + void *library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (library == NULL) { + fprintf(stderr, "%s\n", dlerror()); + return 3; + } + panic_boundary_fn boundary = (panic_boundary_fn)dlsym(library, "ad_test_panic_boundary"); + if (boundary == NULL) { + fprintf(stderr, "%s\n", dlerror()); + dlclose(library); + return 4; + } + int32_t result = boundary(); + dlclose(library); + if (result != -12) { + fprintf(stderr, "unexpected result: %d\n", result); + return 5; + } + return 0; +} diff --git a/crates/ffi/tests/codegen_exhaustiveness.rs b/crates/ffi/tests/codegen_exhaustiveness.rs deleted file mode 100644 index 1017674..0000000 --- a/crates/ffi/tests/codegen_exhaustiveness.rs +++ /dev/null @@ -1,135 +0,0 @@ -/// Codegen exhaustiveness + per-command policy pin tests. -/// -/// Independently verifies that: -/// 1. Every expected Family-B command has a generated `ad_<name>` wrapper in -/// `src/commands/generated.rs` (the committed output). -/// 2. Each command's interaction-policy pin is preserved across refactors. -/// -/// Adding a new Family-B command requires updating EXPECTED_COMMANDS here -/// and adding a template to build.rs. This test fails the build if either -/// side is out of sync with the generated file. -mod common; - -use agent_desktop_core::action::Action; -use agent_desktop_core::interaction_policy::InteractionPolicy; - -/// Known Family-B commands — the exhaustive set of command-backed JSON -/// wrappers that must appear in `src/commands/generated.rs`. -const EXPECTED_COMMANDS: &[&str] = &[ - "execute_by_ref", - "snapshot", - "status", - "trace_export", - "trace_show", - "version", - "wait", -]; - -#[test] -fn generated_file_contains_all_expected_wrappers() { - let src = include_str!("../src/commands/generated.rs"); - - for name in EXPECTED_COMMANDS { - let fn_sig = format!("pub unsafe extern \"C\" fn ad_{name}("); - assert!( - src.contains(&fn_sig), - "generated src/commands/generated.rs is missing `ad_{name}` — \ - check templates in build.rs and run cargo build to regenerate" - ); - } -} - -#[test] -fn expected_command_count_matches_generated_wrapper_count() { - let src = include_str!("../src/commands/generated.rs"); - let actual_count = src - .lines() - .filter(|l| l.contains("pub unsafe extern \"C\" fn ad_")) - .count(); - assert_eq!( - actual_count, - EXPECTED_COMMANDS.len(), - "generated file has {actual_count} wrappers but EXPECTED_COMMANDS has {} — \ - update EXPECTED_COMMANDS to match the templates in build.rs", - EXPECTED_COMMANDS.len() - ); -} - -#[test] -fn generated_wrappers_are_in_alphabetical_order() { - let src = include_str!("../src/commands/generated.rs"); - let fn_names: Vec<&str> = src - .lines() - .filter(|l| l.contains("pub unsafe extern \"C\" fn ad_")) - .filter_map(|l| { - l.split("fn ad_") - .nth(1) - .and_then(|rest| rest.split('(').next()) - }) - .collect(); - let sorted = { - let mut s = fn_names.clone(); - s.sort_unstable(); - s - }; - assert_eq!( - fn_names, sorted, - "generated wrappers must appear in alphabetical order" - ); -} - -#[test] -fn policy_type_text_base_is_focus_fallback() { - let base = Action::TypeText("hi".into()).base_interaction_policy(); - assert_eq!( - base, - InteractionPolicy::focus_fallback(), - "TypeText base policy must be focus_fallback (KTD6)" - ); -} - -#[test] -fn policy_click_base_is_headless() { - let base = Action::Click.base_interaction_policy(); - assert_eq!( - base, - InteractionPolicy::headless(), - "Click base policy must be headless (KTD6)" - ); -} - -#[test] -fn policy_headless_caller_cannot_downgrade_type_text() { - let base = Action::TypeText("x".into()).base_interaction_policy(); - let effective = base.join(InteractionPolicy::headless()); - assert_eq!( - effective, - InteractionPolicy::focus_fallback(), - "headless caller must not downgrade TypeText below focus_fallback" - ); -} - -#[test] -fn policy_headed_caller_elevates_click_to_headed() { - let base = Action::Click.base_interaction_policy(); - let effective = base.join(InteractionPolicy::headed()); - assert_eq!( - effective, - InteractionPolicy::headed(), - "headed caller must elevate Click to headed" - ); -} - -#[test] -fn click_base_plus_focus_fallback_caller_gives_focus_fallback() { - let base = Action::Click.base_interaction_policy(); - let effective = base.join(InteractionPolicy::focus_fallback()); - assert_eq!(effective, InteractionPolicy::focus_fallback()); -} - -#[test] -fn type_text_base_plus_headed_caller_becomes_headed() { - let base = Action::TypeText("x".into()).base_interaction_policy(); - let effective = base.join(InteractionPolicy::headed()); - assert_eq!(effective, InteractionPolicy::headed()); -} diff --git a/crates/ffi/tests/common/mod.rs b/crates/ffi/tests/common/mod.rs index d52e2f8..16a783f 100644 --- a/crates/ffi/tests/common/mod.rs +++ b/crates/ffi/tests/common/mod.rs @@ -4,22 +4,77 @@ pub use agent_desktop_ffi::error::AdResult; pub use agent_desktop_ffi::{ AdAction, AdActionResult, AdActionStep, AdAdapter, AdAppList, AdDirection, AdDragParams, - AdElementState, AdFindQuery, AdKeyCombo, AdNativeHandle, AdPoint, AdPolicyKind, AdRect, - AdRefEntry, AdScrollParams, AdWaitArgs, AdWindowInfo, AdWindowList, + AdElementState, AdExactRefEntry, AdExactSurfaceInfo, AdExactSurfaceList, AdExactWindowInfo, + AdExactWindowList, AdFindQuery, AdIdentifierKind, AdKeyCombo, AdNativeHandle, + AdNotificationActionRequest, AdNotificationIdentity, AdOptionalU64, AdOptionalUsize, AdPoint, + AdPolicyKind, AdRect, AdRefEntry, AdScrollParams, AdWaitArgs, AdWaitMode, AdWaitPredicate, + AdWaitScope, AdWaitSurfaceModes, AdWindowInfo, AdWindowList, }; pub use std::ffi::CStr; pub use std::os::raw::c_char; +use std::sync::{ + Mutex, + atomic::{AtomicU64, Ordering}, +}; + +static HOME_LOCK: Mutex<()> = Mutex::new(()); +static HOME_ID: AtomicU64 = AtomicU64::new(1); + +struct IsolatedHome { + _lock: std::sync::MutexGuard<'static, ()>, + path: std::path::PathBuf, + previous: Option<std::ffi::OsString>, +} + +impl IsolatedHome { + fn enter() -> Self { + let lock = HOME_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let id = HOME_ID.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "agent-desktop-ffi-test-{}-{id}", + std::process::id() + )); + std::fs::create_dir_all(&path).expect("create isolated FFI test HOME"); + let previous = std::env::var_os("HOME"); + unsafe { std::env::set_var("HOME", &path) }; + Self { + _lock: lock, + path, + previous, + } + } +} + +impl Drop for IsolatedHome { + fn drop(&mut self) { + match self.previous.as_ref() { + Some(previous) => unsafe { std::env::set_var("HOME", previous) }, + None => unsafe { std::env::remove_var("HOME") }, + } + let _ = std::fs::remove_dir_all(&self.path); + } +} unsafe extern "C" { pub fn ad_abi_version() -> u32; pub fn ad_init(expected_major: u32) -> AdResult; pub fn ad_version(out: *mut *mut c_char) -> AdResult; pub fn ad_free_string(s: *mut c_char); + pub fn ad_notification_action( + adapter: *const AdAdapter, + request: *const AdNotificationActionRequest, + out: *mut AdActionResult, + ) -> AdResult; pub fn ad_set_log_callback( cb: Option<unsafe extern "C" fn(level: i32, msg: *const c_char)>, ) -> AdResult; pub fn ad_ref_entry_size() -> usize; + pub fn ad_exact_ref_entry_size() -> usize; + pub fn ad_exact_surface_info_size() -> usize; + pub fn ad_exact_window_info_size() -> usize; pub fn ad_action_size() -> usize; pub fn ad_action_step_size() -> usize; pub fn ad_action_result_size() -> usize; @@ -45,6 +100,9 @@ unsafe extern "C" { pub fn ad_app_list_count(list: *const AdAppList) -> u32; pub fn ad_app_list_get(list: *const AdAppList, index: u32) -> *const u8; pub fn ad_app_list_free(list: *mut AdAppList); + pub fn ad_get_clipboard(adapter: *const AdAdapter, out: *mut *mut c_char) -> AdResult; + pub fn ad_set_clipboard(adapter: *const AdAdapter, text: *const c_char) -> AdResult; + pub fn ad_clear_clipboard(adapter: *const AdAdapter) -> AdResult; pub fn ad_list_windows( adapter: *const AdAdapter, @@ -54,6 +112,29 @@ unsafe extern "C" { ) -> AdResult; pub fn ad_window_list_count(list: *const AdWindowList) -> u32; pub fn ad_window_list_free(list: *mut AdWindowList); + pub fn ad_list_windows_exact( + adapter: *const AdAdapter, + app_filter: *const c_char, + focused_only: bool, + out: *mut *mut AdExactWindowList, + ) -> AdResult; + pub fn ad_exact_window_list_count(list: *const AdExactWindowList) -> u32; + pub fn ad_exact_window_list_get( + list: *const AdExactWindowList, + index: u32, + ) -> *const AdExactWindowInfo; + pub fn ad_exact_window_list_free(list: *mut AdExactWindowList); + pub fn ad_list_surfaces_exact( + adapter: *const AdAdapter, + pid: u32, + out: *mut *mut AdExactSurfaceList, + ) -> AdResult; + pub fn ad_exact_surface_list_count(list: *const AdExactSurfaceList) -> u32; + pub fn ad_exact_surface_list_get( + list: *const AdExactSurfaceList, + index: u32, + ) -> *const AdExactSurfaceInfo; + pub fn ad_exact_surface_list_free(list: *mut AdExactSurfaceList); pub fn ad_launch_app( adapter: *const AdAdapter, @@ -98,6 +179,11 @@ unsafe extern "C" { entry: *const AdRefEntry, out: *mut AdNativeHandle, ) -> AdResult; + pub fn ad_resolve_element_exact( + adapter: *const AdAdapter, + entry: *const AdExactRefEntry, + out: *mut AdNativeHandle, + ) -> AdResult; pub fn ad_snapshot( adapter: *const AdAdapter, @@ -118,10 +204,20 @@ unsafe extern "C" { policy: i32, out: *mut *mut c_char, ) -> AdResult; + pub fn ad_execute_by_ref_timeout( + adapter: *const AdAdapter, + ref_id: *const c_char, + snapshot_id: *const c_char, + action: *const AdAction, + policy: i32, + timeout_ms: i64, + out: *mut *mut c_char, + ) -> AdResult; } pub fn with_adapter<F: FnOnce(*mut AdAdapter)>(body: F) { + let _home = IsolatedHome::enter(); unsafe { let adapter = ad_adapter_create(); assert!(!adapter.is_null(), "ad_adapter_create must not return null"); @@ -130,35 +226,20 @@ pub fn with_adapter<F: FnOnce(*mut AdAdapter)>(body: F) { } } +pub fn with_isolated_home<F: FnOnce()>(body: F) { + let _home = IsolatedHome::enter(); + body(); +} + pub fn default_ref_entry() -> AdRefEntry { - AdRefEntry { - pid: 0, - role: std::ptr::null(), - name: std::ptr::null(), - value: std::ptr::null(), - description: std::ptr::null(), - states: std::ptr::null(), - state_count: 0, - available_actions: std::ptr::null(), - available_action_count: 0, - bounds: AdRect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 0.0, - }, - has_bounds: false, - bounds_hash: 0, - has_bounds_hash: false, - source_app: std::ptr::null(), - source_window_id: std::ptr::null(), - source_window_title: std::ptr::null(), - source_surface: 0, - root_ref: std::ptr::null(), - path_is_absolute: false, - path: std::ptr::null(), - path_count: 0, - } + unsafe { std::mem::zeroed() } +} + +pub fn default_exact_ref_entry() -> AdExactRefEntry { + let mut entry: AdExactRefEntry = unsafe { std::mem::zeroed() }; + entry.version = 1; + entry.size = std::mem::size_of::<AdExactRefEntry>() as u32; + entry } pub fn default_action() -> AdAction { diff --git a/crates/ffi/tests/cpp_header_compile.rs b/crates/ffi/tests/cpp_header_compile.rs new file mode 100644 index 0000000..f623133 --- /dev/null +++ b/crates/ffi/tests/cpp_header_compile.rs @@ -0,0 +1,146 @@ +//! Verifies the committed C header is also a valid C++17 interface. + +use std::path::PathBuf; +use std::process::Command; + +fn system_cxx() -> Option<&'static str> { + ["c++", "clang++", "g++"].into_iter().find(|compiler| { + Command::new(compiler) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + }) +} + +fn header_include_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("include") +} + +#[test] +fn committed_header_compiles_as_cpp17_with_portable_abi_types() { + let Some(cxx) = system_cxx() else { + eprintln!("skipping: system C++ compiler not found"); + return; + }; + + let stem = format!("agent_desktop_header_cpp17_{}", std::process::id()); + let tmp = std::env::temp_dir().join(format!("{stem}.cpp")); + let obj = std::env::temp_dir().join(format!("{stem}.o")); + let src = r#" +#include <cstdint> +#include <limits> +#include <type_traits> +#include "agent_desktop.h" + +static void log_callback(std::int32_t level, const char *message) { + (void)level; + (void)message; +} + +static_assert(std::is_enum_v<AdModifier>); +static_assert(std::is_same_v<std::underlying_type_t<AdModifier>, std::int32_t>); +static_assert(AD_MODIFIER_META == 0); +static_assert(AD_MODIFIER_CTRL == 1); +static_assert(AD_MODIFIER_ALT == 2); +static_assert(AD_MODIFIER_SHIFT == 3); +static_assert(AD_MODIFIER_CMD == AD_MODIFIER_META); + +static_assert(std::is_enum_v<AdSnapshotSurface>); +static_assert(std::is_same_v<std::underlying_type_t<AdSnapshotSurface>, std::int32_t>); +static_assert(AD_SNAPSHOT_SURFACE_WINDOW == 0); +static_assert(AD_SNAPSHOT_SURFACE_FOCUSED == 1); +static_assert(AD_SNAPSHOT_SURFACE_MENU == 2); +static_assert(AD_SNAPSHOT_SURFACE_MENUBAR == 3); +static_assert(AD_SNAPSHOT_SURFACE_SHEET == 4); +static_assert(AD_SNAPSHOT_SURFACE_POPOVER == 5); +static_assert(AD_SNAPSHOT_SURFACE_ALERT == 6); +static_assert(AD_SNAPSHOT_SURFACE_DESKTOP == 7); +static_assert(AD_SNAPSHOT_SURFACE_TASKBAR == 8); +static_assert(AD_SNAPSHOT_SURFACE_SYSTEM_TRAY == 9); +static_assert(AD_SNAPSHOT_SURFACE_QUICK_SETTINGS == 10); +static_assert(AD_SNAPSHOT_SURFACE_NOTIFICATION_CENTER == 11); +static_assert(AD_SNAPSHOT_SURFACE_TOOLBAR == 12); +static_assert(AD_SNAPSHOT_SURFACE_DOCK == 13); +static_assert(AD_SNAPSHOT_SURFACE_SPOTLIGHT == 14); +static_assert(AD_SNAPSHOT_SURFACE_MENU_BAR_EXTRAS == 15); +static_assert(AD_SNAPSHOT_SURFACE_SYSTEM_TRAY_OVERFLOW == 16); +static_assert(AD_SNAPSHOT_SURFACE_START_MENU == 17); +static_assert(AD_SNAPSHOT_SURFACE_ACTION_CENTER == 18); + +static_assert(AD_DELIVERY_DISPOSITION_UNKNOWN == 0); +static_assert(AD_DELIVERY_DISPOSITION_NOT_DELIVERED == 1); +static_assert(AD_DELIVERY_DISPOSITION_DELIVERY_UNCERTAIN == 2); +static_assert(AD_DELIVERY_DISPOSITION_DELIVERED_UNVERIFIED == 3); +static_assert(AD_DELIVERY_DISPOSITION_DELIVERED_VERIFIED == 4); +static_assert(AD_FIND_SELECTION_KIND_STRICT == 0); +static_assert(AD_FIND_SELECTION_KIND_FIRST == 1); +static_assert(AD_FIND_SELECTION_KIND_LAST == 2); +static_assert(AD_FIND_SELECTION_KIND_NTH == 3); +static_assert(AD_IDENTIFIER_KIND_AX_IDENTIFIER == 0); +static_assert(AD_IDENTIFIER_KIND_AX_DOM_IDENTIFIER == 1); +static_assert(AD_IDENTIFIER_KIND_AUTOMATION_ID == 2); +static_assert(AD_IDENTIFIER_KIND_RUNTIME_ID == 3); +static_assert(AD_IDENTIFIER_KIND_ATSPI_OBJECT_PATH == 4); +static_assert(AD_POLICY_KIND_HEADLESS == 0); +static_assert(AD_POLICY_KIND_FOCUS_FALLBACK == 1); +static_assert(AD_POLICY_KIND_HEADED == 2); +static_assert(AD_RETRY_DISPOSITION_UNKNOWN == 0); +static_assert(AD_RETRY_DISPOSITION_SAFE == 1); +static_assert(AD_RETRY_DISPOSITION_UNSAFE == 2); +static_assert(AD_STEP_MECHANISM_SEMANTIC_API == 1); +static_assert(AD_STEP_MECHANISM_PHYSICAL_SYNTHETIC == 2); + +using AppPid = decltype(AdAppInfo{}.pid); +using WindowPid = decltype(AdWindowInfo{}.pid); +using RefPid = decltype(AdRefProcess{}.pid); +using ScreenshotPid = decltype(AdScreenshotTarget{}.pid); +static_assert(std::is_same_v<AppPid, std::uint32_t>); +static_assert(std::is_same_v<WindowPid, std::uint32_t>); +static_assert(std::is_same_v<RefPid, std::uint32_t>); +static_assert(std::is_same_v<ScreenshotPid, std::uint32_t>); +static_assert(std::numeric_limits<AppPid>::max() == UINT32_MAX); + +using ListSurfacesFn = AdResult (*)(const AdAdapter *, std::uint32_t, AdSurfaceList **); +using ListExactSurfacesFn = AdResult (*)(const AdAdapter *, std::uint32_t, AdExactSurfaceList **); +using ListDisplaysFn = AdResult (*)(const AdAdapter *, AdDisplayList **); +static_assert(std::is_same_v<decltype(&ad_list_surfaces), ListSurfacesFn>); +static_assert(std::is_same_v<decltype(&ad_list_surfaces_exact), ListExactSurfacesFn>); +static_assert(std::is_same_v<decltype(&ad_list_displays), ListDisplaysFn>); +static_assert(sizeof(AdDisplayInfo) == AD_DISPLAY_INFO_SIZE); +static_assert(offsetof(AdDisplayInfo, id) == 8); +static_assert(offsetof(AdDisplayInfo, scale) == 56); + +int main() { + const auto callback_result = ad_set_log_callback(log_callback); + const auto clear_callback_result = ad_set_log_callback(nullptr); + (void)callback_result; + (void)clear_callback_result; + return 0; +} +"#; + std::fs::write(&tmp, src).expect("write C++17 test translation unit"); + + let output = Command::new(cxx) + .arg("-std=c++17") + .arg("-pedantic-errors") + .arg("-Wall") + .arg("-Wextra") + .arg("-Werror") + .arg("-I") + .arg(header_include_dir()) + .arg("-c") + .arg(&tmp) + .arg("-o") + .arg(&obj) + .output() + .expect("C++ compiler invocation failed"); + + let _ = std::fs::remove_file(&tmp); + let _ = std::fs::remove_file(&obj); + + assert!( + output.status.success(), + "C++17 compile of agent_desktop.h failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/ffi/tests/error_lifetime.rs b/crates/ffi/tests/error_lifetime.rs index 5eb7477..0a7faad 100644 --- a/crates/ffi/tests/error_lifetime.rs +++ b/crates/ffi/tests/error_lifetime.rs @@ -16,9 +16,8 @@ unsafe extern "C" { fn ad_last_error_code() -> AdResult; } -/// Worker-thread cargo tests hit the main-thread guard first (`ErrInternal`); -/// main-thread callers would see `ErrInvalidArgs`. The contract under test is -/// that any failure populates last-error and the pointer stays stable. +/// Any failure populates last-error and the pointer stays stable across later +/// successful calls on the same thread. #[test] fn last_error_pointer_survives_across_successful_calls() { unsafe { @@ -28,10 +27,7 @@ fn last_error_pointer_survives_across_successful_calls() { let bad_id = std::ptr::null(); let mut out_win: agent_desktop_ffi::AdWindowInfo = std::mem::zeroed(); let rc = ad_launch_app(adapter, bad_id, 0, &mut out_win); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); + assert_eq!(rc, AdResult::ErrInvalidArgs); let first_msg_ptr = ad_last_error_message(); let first_details_ptr = ad_last_error_details(); diff --git a/crates/ffi/tests/run_cdylib_panic_probe.sh b/crates/ffi/tests/run_cdylib_panic_probe.sh new file mode 100644 index 0000000..564852f --- /dev/null +++ b/crates/ffi/tests/run_cdylib_panic_probe.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(git rev-parse --show-toplevel) +cd "$ROOT" + +TARGET_ROOT=${CARGO_TARGET_DIR:-target} +PROBE=${TMPDIR:-/tmp}/agent-desktop-cdylib-panic-probe + +cargo build --locked --profile release-ffi -p agent-desktop-ffi --features panic-injection +cc crates/ffi/tests/cdylib_panic_probe.c -o "$PROBE" +DYLIB=$(find "$TARGET_ROOT/release-ffi" -name 'libagent_desktop_ffi.dylib' -print -quit) +if [[ -z "$DYLIB" ]]; then + echo "FAIL: release-ffi dylib was not produced" >&2 + exit 1 +fi +"$PROBE" "$DYLIB" diff --git a/crates/linux/src/adapter.rs b/crates/linux/src/adapter.rs index 608122f..8fb4650 100644 --- a/crates/linux/src/adapter.rs +++ b/crates/linux/src/adapter.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::adapter::PlatformAdapter; +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; pub struct LinuxAdapter; @@ -14,4 +14,50 @@ impl Default for LinuxAdapter { } } -impl PlatformAdapter for LinuxAdapter {} +impl ObservationOps for LinuxAdapter {} +impl ActionOps for LinuxAdapter {} +impl InputOps for LinuxAdapter {} +impl SystemOps for LinuxAdapter {} + +#[cfg(test)] +mod tests { + use super::*; + use agent_desktop_core::{AppError, CommandContext, ErrorCode, SnapshotSurface}; + + #[test] + fn snapshot_surfaces_fail_closed_until_linux_implements_them() { + let adapter = LinuxAdapter::new(); + assert!(adapter.supported_surfaces().is_empty()); + + let error = agent_desktop_core::commands::snapshot::execute( + agent_desktop_core::commands::snapshot::SnapshotArgs { + app: None, + window_id: None, + max_depth: 1, + include_bounds: false, + interactive_only: false, + compact: true, + surface: SnapshotSurface::Window, + skeleton: false, + root_ref: None, + snapshot_id: None, + }, + &adapter, + &CommandContext::default(), + ) + .expect_err("an unimplemented surface must fail at validation"); + + let AppError::Adapter(error) = error else { + panic!("surface validation must return an adapter error") + }; + assert_eq!(error.code, ErrorCode::PlatformNotSupported); + assert!( + error + .details + .as_ref() + .and_then(|details| details.get("supported_surfaces")) + .and_then(|surfaces| surfaces.as_array()) + .is_some_and(Vec::is_empty) + ); + } +} diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index 38e38c1..e8d41e3 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -1,3 +1,5 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + mod actions; mod adapter; mod input; diff --git a/crates/macos/Cargo.toml b/crates/macos/Cargo.toml index c652a4c..55445bd 100644 --- a/crates/macos/Cargo.toml +++ b/crates/macos/Cargo.toml @@ -14,15 +14,21 @@ serde_json.workspace = true tracing.workspace = true base64.workspace = true rustc-hash.workspace = true +libc.workspace = true [target.'cfg(target_os = "macos")'.dependencies] accessibility-sys = "0.2.0" core-foundation = "0.10.1" core-foundation-sys = "0.8.7" -core-graphics = { version = "0.25.0", features = ["highsierra"] } +core-graphics = { version = "0.25.0", features = ["elcapitan", "highsierra"] } [features] dev-tools = [] +interactive-tests = [] + +[[bin]] +name = "agent-desktop-macos-helper" +path = "src/bin/agent-desktop-macos-helper.rs" [[example]] name = "ax_probe" diff --git a/crates/macos/build.rs b/crates/macos/build.rs index 59c1fae..10196c4 100644 --- a/crates/macos/build.rs +++ b/crates/macos/build.rs @@ -1,6 +1,134 @@ +use std::path::PathBuf; +use std::process::Command; + fn main() { - println!("cargo:rustc-link-lib=framework=ApplicationServices"); - println!("cargo:rustc-link-lib=framework=CoreFoundation"); - println!("cargo:rustc-link-lib=framework=CoreGraphics"); + println!("cargo:rerun-if-changed=src/system/launch_bridge.m"); + println!("cargo:rerun-if-changed=src/system/appkit_bridge.m"); + println!("cargo:rerun-if-changed=src/system/screen_bridge.m"); + println!("cargo:rerun-if-env-changed=TARGET"); + println!("cargo:rerun-if-env-changed=MACOSX_DEPLOYMENT_TARGET"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") { + return; + } + let target = required_target(); + let deployment = deployment_target(); + let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "unknown".into()); + println!("cargo:rustc-env=AGENT_DESKTOP_MACOS_HELPER_BUILD_ID={version}:{target}:{deployment}"); + let Some(out_dir) = std::env::var_os("OUT_DIR").map(PathBuf::from) else { + eprintln!("OUT_DIR is required"); + std::process::exit(1); + }; + let object = out_dir.join("launch_bridge.o"); + let appkit_object = out_dir.join("appkit_bridge.o"); + let screen_object = out_dir.join("screen_bridge.o"); + let archive = out_dir.join("libagent_desktop_launch_bridge.a"); + run( + Command::new("xcrun") + .args(["--sdk", "macosx", "clang"]) + .args(["-fobjc-arc", "-fblocks", "-target", &target]) + .arg(format!("-mmacosx-version-min={deployment}")) + .args([ + "-Wall", + "-Wextra", + "-Werror", + "-c", + "src/system/appkit_bridge.m", + "-o", + ]) + .arg(&appkit_object), + "compile Objective-C AppKit bridge", + ); + run( + Command::new("xcrun") + .args(["--sdk", "macosx", "clang"]) + .args(["-fobjc-arc", "-fblocks", "-target", &target]) + .arg(format!("-mmacosx-version-min={deployment}")) + .args([ + "-Wall", + "-Wextra", + "-Werror", + "-c", + "src/system/launch_bridge.m", + "-o", + ]) + .arg(&object), + "compile Objective-C launch bridge", + ); + run( + Command::new("xcrun") + .args(["--sdk", "macosx", "clang"]) + .args(["-fobjc-arc", "-target", &target]) + .arg(format!("-mmacosx-version-min={deployment}")) + .args([ + "-Wall", + "-Wextra", + "-Werror", + "-c", + "src/system/screen_bridge.m", + "-o", + ]) + .arg(&screen_object), + "compile Objective-C screen bridge", + ); + run( + Command::new("xcrun") + .args(["--sdk", "macosx", "ar", "rcs"]) + .arg(&archive) + .arg(&object) + .arg(&appkit_object) + .arg(&screen_object), + "archive Objective-C launch bridge", + ); + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-link-lib=static=agent_desktop_launch_bridge"); println!("cargo:rustc-link-lib=framework=AppKit"); } + +fn required_target() -> String { + match std::env::var("TARGET") { + Ok(target) + if matches!( + target.as_str(), + "aarch64-apple-darwin" | "x86_64-apple-darwin" + ) => + { + target + } + Ok(target) => { + eprintln!("unsupported macOS launch bridge target: {target}"); + std::process::exit(1); + } + Err(error) => { + eprintln!("TARGET is required: {error}"); + std::process::exit(1); + } + } +} + +fn deployment_target() -> String { + let deployment = std::env::var("MACOSX_DEPLOYMENT_TARGET").unwrap_or_else(|_| "10.15".into()); + let supported = deployment + .split_once('.') + .and_then(|(major, minor)| Some((major.parse::<u32>().ok()?, minor.parse::<u32>().ok()?))) + .is_some_and(|version| version >= (10, 15)); + if supported { + deployment + } else { + eprintln!("MACOSX_DEPLOYMENT_TARGET must be 10.15 or newer"); + std::process::exit(1); + } +} + +fn run(command: &mut Command, label: &str) { + match command.status() { + Ok(status) if status.success() => {} + Ok(status) => { + eprintln!("{label} failed with {status}"); + std::process::exit(1); + } + Err(error) => { + eprintln!("{label}: {error}"); + std::process::exit(1); + } + } +} diff --git a/crates/macos/src/actions/adapter.rs b/crates/macos/src/actions/adapter.rs new file mode 100644 index 0000000..3abaa98 --- /dev/null +++ b/crates/macos/src/actions/adapter.rs @@ -0,0 +1,56 @@ +use agent_desktop_core::{ + Action, ActionOps, ActionResult, ActionStep, AdapterError, InteractionLease, NativeHandle, + StepMechanism, action_request::ActionRequest, +}; + +use crate::adapter::{MacOSAdapter, ax_element}; + +impl ActionOps for MacOSAdapter { + fn execute_action( + &self, + handle: &NativeHandle, + request: ActionRequest, + lease: &InteractionLease, + ) -> Result<ActionResult, AdapterError> { + if handle.is_null() { + return execute_global_action(request, lease); + } + execute_action_impl(handle, request, lease) + } + + fn scroll_into_view( + &self, + handle: &NativeHandle, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + crate::actions::scroll_into_view::scroll_into_view_impl( + ax_element(handle)?, + lease.deadline(), + ) + } +} + +fn execute_action_impl( + handle: &NativeHandle, + request: ActionRequest, + lease: &InteractionLease, +) -> Result<ActionResult, AdapterError> { + crate::actions::perform_action(ax_element(handle)?, &request, lease.deadline()) +} + +fn execute_global_action( + request: ActionRequest, + lease: &InteractionLease, +) -> Result<ActionResult, AdapterError> { + let Action::PressKey(combo) = request.action else { + return Err(AdapterError::not_supported("global element action")); + }; + crate::input::keyboard::synthesize_key(&combo, None, lease.deadline())?; + Ok( + ActionResult::delivered_unverified("press_key").with_steps(vec![ + ActionStep::succeeded("CGEventPost") + .with_mechanism(StepMechanism::PhysicalSynthetic) + .with_verified(false), + ]), + ) +} diff --git a/crates/macos/src/actions/ax_helpers.rs b/crates/macos/src/actions/ax_helpers.rs index 8ad31ae..d20ca85 100644 --- a/crates/macos/src/actions/ax_helpers.rs +++ b/crates/macos/src/actions/ax_helpers.rs @@ -1,62 +1,34 @@ -use agent_desktop_core::error::{AdapterError, ErrorCode}; +use agent_desktop_core::{AdapterError, ErrorCode}; #[cfg(target_os = "macos")] mod imp { use super::*; + use crate::actions::ax_mutation; use crate::tree::AXElement; - use accessibility_sys::{ - AXUIElementCopyAttributeValue, AXUIElementIsAttributeSettable, AXUIElementPerformAction, - AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout, kAXErrorAPIDisabled, - kAXErrorCannotComplete, kAXErrorSuccess, kAXFocusedAttribute, kAXValueAttribute, - }; + use accessibility_sys::{kAXErrorSuccess, kAXFocusedAttribute, kAXValueAttribute}; use core_foundation::{ - base::{CFType, CFTypeRef, TCFType}, + base::{CFType, TCFType}, boolean::CFBoolean, number::CFNumber, string::CFString, }; - use std::os::raw::c_uchar; - pub(crate) fn try_ax_action(el: &AXElement, name: &str) -> bool { - let action = CFString::new(name); - let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - err == kAXErrorSuccess - } - - pub(crate) fn try_ax_action_retried(el: &AXElement, name: &str) -> bool { - try_ax_action_retried_or_err(el, name).unwrap_or(false) - } - - pub(crate) fn try_ax_action_retried_or_err( + pub(crate) fn try_ax_action_or_err( el: &AXElement, name: &str, + deadline: agent_desktop_core::Deadline, ) -> Result<bool, AdapterError> { let action = CFString::new(name); - let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - if err == kAXErrorSuccess { - return Ok(true); - } - if err == kAXErrorCannotComplete { - std::thread::sleep(std::time::Duration::from_millis(100)); - let retry = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - if retry == kAXErrorSuccess { - return Ok(true); - } - ax_error_result(name, retry)?; - return Ok(false); - } - ax_error_result(name, err)?; - Ok(false) - } - - pub(crate) fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool { - set_ax_bool_or_err(el, attr, value).unwrap_or(false) + run_mutation(el, name, "AXUIElementPerformAction", deadline, |deadline| { + crate::tree::ax_ipc::perform_action(el, action.as_concrete_TypeRef(), deadline) + }) } pub(crate) fn set_ax_bool_or_err( el: &AXElement, attr: &str, value: bool, + deadline: agent_desktop_core::Deadline, ) -> Result<bool, AdapterError> { let cf_attr = CFString::new(attr); let cf_val = if value { @@ -64,133 +36,97 @@ mod imp { } else { CFBoolean::false_value() }; - let err = unsafe { - AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) - }; - if err == kAXErrorSuccess { - return Ok(true); - } - ax_error_result(attr, err)?; - Ok(false) + run_mutation( + el, + attr, + "AXUIElementSetAttributeValue", + deadline, + |deadline| { + crate::tree::ax_ipc::set_attribute_value( + el, + cf_attr.as_concrete_TypeRef(), + cf_val.as_CFTypeRef(), + deadline, + ) + }, + ) } pub(crate) fn set_ax_string_or_err( el: &AXElement, attr: &str, value: &str, + deadline: agent_desktop_core::Deadline, ) -> Result<(), AdapterError> { let cf_attr = CFString::new(attr); let cf_val = CFString::new(value); - let err = unsafe { - AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) - }; - if err != kAXErrorSuccess { - ax_error_result(attr, err)?; + let delivered = run_mutation( + el, + attr, + "AXUIElementSetAttributeValue", + deadline, + |deadline| { + crate::tree::ax_ipc::set_attribute_value( + el, + cf_attr.as_concrete_TypeRef(), + cf_val.as_CFTypeRef(), + deadline, + ) + }, + )?; + if !delivered { return Err(AdapterError::new( ErrorCode::ActionFailed, - format!("AXSetAttributeValue({attr}) failed (err={err})"), + format!("AXSetAttributeValue({attr}) is unsupported"), ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) .with_suggestion("Attribute may be read-only. Try 'click' or 'type' instead.")); } Ok(()) } - pub(crate) fn is_attr_settable(el: &AXElement, attr: &str) -> bool { + pub(crate) fn is_attr_settable( + el: &AXElement, + attr: &str, + deadline: agent_desktop_core::Deadline, + ) -> Result<bool, AdapterError> { let cf_attr = CFString::new(attr); - let mut settable: c_uchar = 0; - let err = unsafe { - AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable) - }; - err == kAXErrorSuccess && settable != 0 + let (err, settable) = + crate::tree::ax_ipc::is_attribute_settable(el, cf_attr.as_concrete_TypeRef(), deadline); + ensure_read_finished(deadline)?; + classify_settable_read(attr, err, settable) } - pub(crate) fn list_ax_actions(el: &AXElement) -> Vec<String> { - crate::tree::capabilities::copy_action_names(el) - } - - pub(crate) fn has_ax_action(el: &AXElement, target: &str) -> bool { - list_ax_actions(el).iter().any(|a| a == target) - } - - pub(crate) fn try_action_from_list( + pub(crate) fn ax_focus_or_err( el: &AXElement, - actions: &[String], - targets: &[&str], - ) -> bool { - for target in targets { - if actions.iter().any(|a| a == target) && try_ax_action(el, target) { - return true; - } - } - false - } - - pub(crate) fn try_each_child( - el: &AXElement, - f: impl Fn(&AXElement) -> bool, - limit: usize, - ) -> bool { - let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - for child in children.iter().take(limit) { - if f(child) { - return true; - } - } - false - } - - pub(crate) fn try_each_ancestor( - el: &AXElement, - f: impl Fn(&AXElement) -> bool, - limit: usize, - ) -> bool { - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..limit { - let ancestor = match ¤t { - Some(a) => a, - None => return false, - }; - if f(ancestor) { - return true; - } - current = crate::tree::copy_element_attr(ancestor, "AXParent"); - } - false - } - - pub(crate) fn ensure_visible(el: &AXElement) { - let action = CFString::new("AXScrollToVisible"); - unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - } - - pub(crate) fn set_messaging_timeout(el: &AXElement, seconds: f32) { - let err = unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) }; - if err != kAXErrorSuccess { - tracing::warn!( - err, - "AXUIElementSetMessagingTimeout failed; AX calls may use the default timeout" - ); - } - } - - pub(crate) fn ax_focus_or_err(el: &AXElement) -> Result<bool, AdapterError> { - set_ax_bool_or_err(el, kAXFocusedAttribute, true) - } - - pub(crate) fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> { - set_ax_string_or_err(el, kAXValueAttribute, val) + deadline: agent_desktop_core::Deadline, + ) -> Result<bool, AdapterError> { + set_ax_bool_or_err(el, kAXFocusedAttribute, true, deadline) } /// Sets `AXValue` with a CoreFoundation type matching the element's /// current value: numeric controls (sliders, steppers, progress) hold a /// `CFNumber` and reject a `CFString`, so a typed write is required. Falls /// back to a string write when the current value is a string or absent. - pub(crate) fn set_ax_value_coerced(el: &AXElement, value: &str) -> Result<(), AdapterError> { + pub(crate) fn set_ax_value_coerced( + el: &AXElement, + value: &str, + deadline: agent_desktop_core::Deadline, + ) -> Result<(), AdapterError> { let cf_attr = CFString::new(kAXValueAttribute); - let mut current: CFTypeRef = std::ptr::null_mut(); - let read = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut current) - }; + let (read, current) = + crate::tree::ax_ipc::copy_attribute_value(el, cf_attr.as_concrete_TypeRef(), deadline); + ensure_read_finished(deadline)?; + if read != kAXErrorSuccess { + if !current.is_null() { + unsafe { core_foundation::base::CFRelease(current) }; + } + if read != accessibility_sys::kAXErrorAttributeUnsupported + && read != accessibility_sys::kAXErrorNoValue + { + return Err(read_failure(kAXValueAttribute, read)); + } + } let coerced: Option<CFType> = if read == kAXErrorSuccess && !current.is_null() { let cur = unsafe { CFType::wrap_under_create_rule(current) }; if cur.downcast::<CFNumber>().is_some() { @@ -207,26 +143,107 @@ mod imp { match coerced { Some(cf_value) => { - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - cf_value.as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - ax_error_result(kAXValueAttribute, err)?; + let delivered = run_mutation( + el, + kAXValueAttribute, + "AXUIElementSetAttributeValue", + deadline, + |deadline| { + crate::tree::ax_ipc::set_attribute_value( + el, + cf_attr.as_concrete_TypeRef(), + cf_value.as_CFTypeRef(), + deadline, + ) + }, + )?; + if !delivered { return Err(AdapterError::new( ErrorCode::ActionFailed, - format!("AXSetAttributeValue(AXValue) failed (err={err})"), + "AXSetAttributeValue(AXValue) is unsupported", ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) .with_suggestion( "Value may be read-only or out of range. Try 'click' to focus then arrow keys.", )); } Ok(()) } - None => set_ax_string_or_err(el, kAXValueAttribute, value), + None => set_ax_string_or_err(el, kAXValueAttribute, value, deadline), + } + } + + fn prepare( + element: &AXElement, + deadline: agent_desktop_core::Deadline, + ) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) + } + + fn run_mutation( + element: &AXElement, + operation: &str, + api: &str, + deadline: agent_desktop_core::Deadline, + mutate: impl FnOnce(agent_desktop_core::Deadline) -> Result<i32, AdapterError>, + ) -> Result<bool, AdapterError> { + let error = mutate(deadline)?; + let delivered = ax_mutation::classify_result(element, operation, api, error)?; + if deadline.is_expired() { + let disposition = if delivered { + agent_desktop_core::DeliverySemantics::delivered_unverified() + } else { + agent_desktop_core::DeliverySemantics::not_delivered() + }; + return Err(deadline.timeout_error().with_disposition(disposition)); + } + Ok(delivered) + } + + fn classify_settable_read( + attribute: &str, + error: i32, + settable: bool, + ) -> Result<bool, AdapterError> { + use accessibility_sys::{kAXErrorAttributeUnsupported, kAXErrorNoValue}; + if error == kAXErrorSuccess { + return Ok(settable); + } + if error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue { + return Ok(false); + } + Err(read_failure(attribute, error)) + } + + fn read_failure(attribute: &str, error: i32) -> AdapterError { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + let code = if error == kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else if error == kAXErrorCannotComplete { + ErrorCode::AppUnresponsive + } else { + ErrorCode::ActionFailed + }; + AdapterError::new(code, format!("Accessibility read failed for {attribute}")) + .with_details(serde_json::json!({ + "attribute": attribute, + "ax_error": error, + "retryable": error == kAXErrorCannotComplete, + })) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) + } + + fn ensure_read_finished(deadline: agent_desktop_core::Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline + .timeout_error() + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())) + } else { + Ok(()) } } @@ -247,67 +264,16 @@ mod imp { .with_suggestion("Pass a numeric value, e.g. set-value @e1 50")) } - pub(crate) fn ax_press(el: &AXElement) -> bool { - try_ax_action(el, "AXPress") - } - - pub(crate) fn element_role(el: &AXElement) -> Option<String> { + pub(crate) fn element_role( + el: &AXElement, + deadline: agent_desktop_core::Deadline, + ) -> Result<Option<String>, AdapterError> { use accessibility_sys::kAXRoleAttribute; - crate::tree::copy_string_attr(el, kAXRoleAttribute) - .map(|r| crate::tree::roles::ax_role_to_str(&r).to_string()) - } - - /// Soft-error gate for AX API return codes. - /// - /// Only `kAXErrorAPIDisabled` is promoted to a hard error (permission denied). - /// All other codes — including `kAXErrorInvalidUIElement` — intentionally - /// return `Ok(())` so that action-chain steps fall through to the next - /// strategy or physical fallback instead of aborting the whole chain. - /// Genuine staleness is caught earlier at resolve time (`STALE_REF`); - /// re-escalating other codes here requires a chain-level stale detector first. - fn ax_error_result(operation: &str, err: i32) -> Result<(), AdapterError> { - if err == kAXErrorAPIDisabled { - return Err(AdapterError::permission_denied() - .with_platform_detail(format!("{operation} failed with kAXErrorAPIDisabled"))); - } - Ok(()) - } - - #[cfg(test)] - mod tests { - use accessibility_sys::{ - kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, kAXErrorSuccess, - }; - use agent_desktop_core::error::ErrorCode; - - use super::ax_error_result; - - #[test] - fn success_code_is_ok() { - ax_error_result("op", kAXErrorSuccess).unwrap(); - } - - #[test] - fn api_disabled_yields_perm_denied() { - let err = ax_error_result("press", kAXErrorAPIDisabled).unwrap_err(); - assert_eq!(err.code, ErrorCode::PermDenied); - assert!( - err.platform_detail - .as_deref() - .unwrap() - .contains("kAXErrorAPIDisabled") - ); - } - - #[test] - fn invalid_ui_element_is_soft_ok_so_chains_fall_through() { - ax_error_result("press", kAXErrorInvalidUIElement).unwrap(); - } - - #[test] - fn unrecognised_ax_code_is_soft_ok() { - ax_error_result("op", kAXErrorCannotComplete).unwrap(); - } + prepare(el, deadline)?; + let role = crate::tree::attributes::copy_string_attr_result(el, kAXRoleAttribute, deadline) + .map_err(|error| read_failure(kAXRoleAttribute, error))?; + ensure_read_finished(deadline)?; + Ok(role.map(|role| crate::tree::roles::ax_role_to_str(&role).to_string())) } } @@ -316,25 +282,18 @@ mod imp { use super::*; use crate::tree::AXElement; - pub fn try_ax_action(_el: &AXElement, _name: &str) -> bool { - false - } - pub fn try_ax_action_retried(_el: &AXElement, _name: &str) -> bool { - false - } - pub fn try_ax_action_retried_or_err( + pub fn try_ax_action_or_err( _el: &AXElement, _name: &str, + _deadline: agent_desktop_core::Deadline, ) -> Result<bool, AdapterError> { Ok(false) } - pub fn set_ax_bool(_el: &AXElement, _attr: &str, _value: bool) -> bool { - false - } pub fn set_ax_bool_or_err( _el: &AXElement, _attr: &str, _value: bool, + _deadline: agent_desktop_core::Deadline, ) -> Result<bool, AdapterError> { Ok(false) } @@ -342,53 +301,39 @@ mod imp { _el: &AXElement, _attr: &str, _value: &str, + _deadline: agent_desktop_core::Deadline, ) -> Result<(), AdapterError> { Err(AdapterError::not_supported("set_ax_string_or_err")) } - pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool { - false - } - pub fn list_ax_actions(_el: &AXElement) -> Vec<String> { - Vec::new() - } - pub fn has_ax_action(_el: &AXElement, _target: &str) -> bool { - false - } - pub fn try_action_from_list(_el: &AXElement, _actions: &[String], _targets: &[&str]) -> bool { - false - } - pub fn try_each_child(_el: &AXElement, _f: impl Fn(&AXElement) -> bool, _limit: usize) -> bool { - false - } - pub fn try_each_ancestor( + pub fn is_attr_settable( _el: &AXElement, - _f: impl Fn(&AXElement) -> bool, - _limit: usize, - ) -> bool { - false - } - pub fn ensure_visible(_el: &AXElement) {} - pub fn set_messaging_timeout(_el: &AXElement, _seconds: f32) {} - pub fn ax_focus_or_err(_el: &AXElement) -> Result<bool, AdapterError> { + _attr: &str, + _deadline: agent_desktop_core::Deadline, + ) -> Result<bool, AdapterError> { Ok(false) } - pub fn ax_set_value(_el: &AXElement, _val: &str) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("ax_set_value")) + pub fn ax_focus_or_err( + _el: &AXElement, + _deadline: agent_desktop_core::Deadline, + ) -> Result<bool, AdapterError> { + Ok(false) } - pub fn set_ax_value_coerced(_el: &AXElement, _value: &str) -> Result<(), AdapterError> { + pub fn set_ax_value_coerced( + _el: &AXElement, + _value: &str, + _deadline: agent_desktop_core::Deadline, + ) -> Result<(), AdapterError> { Err(AdapterError::not_supported("set_ax_value_coerced")) } - pub fn ax_press(_el: &AXElement) -> bool { - false - } - pub fn element_role(_el: &AXElement) -> Option<String> { - None + pub fn element_role( + _el: &AXElement, + _deadline: agent_desktop_core::Deadline, + ) -> Result<Option<String>, AdapterError> { + Ok(None) } } pub(crate) use imp::{ - ax_focus_or_err, ax_press, ax_set_value, element_role, ensure_visible, has_ax_action, - is_attr_settable, list_ax_actions, set_ax_bool, set_ax_bool_or_err, set_ax_string_or_err, - set_ax_value_coerced, set_messaging_timeout, try_action_from_list, try_ax_action, - try_ax_action_retried, try_ax_action_retried_or_err, try_each_ancestor, try_each_child, + ax_focus_or_err, element_role, is_attr_settable, set_ax_bool_or_err, set_ax_string_or_err, + set_ax_value_coerced, try_ax_action_or_err, }; diff --git a/crates/macos/src/actions/ax_mutation.rs b/crates/macos/src/actions/ax_mutation.rs new file mode 100644 index 0000000..10384e8 --- /dev/null +++ b/crates/macos/src/actions/ax_mutation.rs @@ -0,0 +1,142 @@ +use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorActionUnsupported, kAXErrorAttributeUnsupported, + kAXErrorCannotComplete, kAXErrorIllegalArgument, kAXErrorInvalidUIElement, kAXErrorNoValue, + kAXErrorNotImplemented, kAXErrorSuccess, +}; +use agent_desktop_core::{AdapterError, DeliverySemantics, ErrorCode}; + +pub(crate) fn classify_result( + _element: &crate::tree::AXElement, + operation: &str, + api: &str, + error: i32, +) -> Result<bool, AdapterError> { + classify(operation, api, error) +} + +fn classify(operation: &str, api: &str, error: i32) -> Result<bool, AdapterError> { + if error == kAXErrorSuccess { + return Ok(true); + } + if error == kAXErrorAPIDisabled { + return Err(AdapterError::permission_denied() + .with_platform_detail(format!( + "{api}({operation}) failed with kAXErrorAPIDisabled" + )) + .with_disposition(DeliverySemantics::not_delivered())); + } + if error == kAXErrorActionUnsupported + || error == kAXErrorAttributeUnsupported + || error == kAXErrorNoValue + || error == kAXErrorNotImplemented + { + return Ok(false); + } + if error == kAXErrorInvalidUIElement { + return Err(AdapterError::new( + ErrorCode::StaleRef, + format!("{operation} targeted an invalid accessibility element"), + ) + .with_details(serde_json::json!({ + "ax_error": "kAXErrorInvalidUIElement", + "operation": operation, + })) + .with_disposition(DeliverySemantics::not_delivered()) + .with_suggestion("Refresh the snapshot and retry with the new element reference.")); + } + if error == kAXErrorIllegalArgument { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("{operation} rejected an invalid accessibility argument"), + ) + .with_platform_detail(format!("{api}({operation}) returned {error}")) + .with_details(serde_json::json!({ + "ax_error": error, + "operation": operation, + })) + .with_disposition(DeliverySemantics::not_delivered())); + } + let (code, label) = if error == kAXErrorCannotComplete { + (ErrorCode::AppUnresponsive, "kAXErrorCannotComplete") + } else { + (ErrorCode::ActionFailed, "unclassified AXError") + }; + Err(crate::actions::DeliveryTracker::uncertain( + AdapterError::new( + code, + format!("{operation} returned {label}; mutation outcome is uncertain"), + ) + .with_details(serde_json::json!({ + "ax_error": error, + "operation": operation, + })) + .with_platform_detail(format!("{api}({operation}) returned {error}")) + .with_suggestion( + "Inspect the target state with a fresh snapshot before deciding whether to retry.", + ), + )) +} + +#[cfg(test)] +mod tests { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorActionUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, kAXErrorSuccess, + }; + use agent_desktop_core::{DeliveryDisposition, ErrorCode, RetryDisposition}; + + use super::classify; + + #[test] + fn success_is_delivered() { + let result = classify("AXPress", "perform", kAXErrorSuccess); + assert!(result.unwrap()); + } + + #[test] + fn unsupported_action_remains_safe_non_delivery() { + let result = classify("AXPress", "perform", kAXErrorActionUnsupported); + assert!(!result.unwrap()); + } + + #[test] + fn invalid_element_is_stale_and_safe_to_retry_with_a_fresh_ref() { + let error = classify("AXPress", "perform", kAXErrorInvalidUIElement) + .expect_err("stale element must fail closed"); + + assert_eq!(error.code, ErrorCode::StaleRef); + assert_eq!( + error.disposition.delivery(), + DeliveryDisposition::NotDelivered + ); + assert_eq!(error.disposition.retry(), RetryDisposition::Safe); + } + + #[test] + fn api_disabled_remains_permission_denied_without_probe() { + let error = classify("AXPress", "perform", kAXErrorAPIDisabled).unwrap_err(); + assert_eq!(error.code, ErrorCode::PermDenied); + assert_eq!( + error.disposition.delivery(), + DeliveryDisposition::NotDelivered + ); + } + + #[test] + fn cannot_complete_reports_uncertainty_without_an_extra_read() { + let error = classify("AXValue", "set", kAXErrorCannotComplete).unwrap_err(); + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!( + error.disposition.delivery(), + DeliveryDisposition::DeliveryUncertain + ); + assert_eq!(error.disposition.retry(), RetryDisposition::Unsafe); + assert!( + error + .suggestion + .as_deref() + .unwrap() + .contains("Inspect the target state") + ); + } +} diff --git a/crates/macos/src/actions/chain.rs b/crates/macos/src/actions/chain.rs index ebe33f5..4c0df0f 100644 --- a/crates/macos/src/actions/chain.rs +++ b/crates/macos/src/actions/chain.rs @@ -1,7 +1,8 @@ -use agent_desktop_core::error::{AdapterError, ErrorCode}; -use agent_desktop_core::{action_step::ActionStep, interaction_policy::InteractionPolicy}; +use agent_desktop_core::step_mechanism::StepMechanism; +use agent_desktop_core::{ActionStep, interaction_policy::InteractionPolicy}; +use agent_desktop_core::{AdapterError, ErrorCode}; -use crate::actions::discovery::ElementCaps; +use crate::actions::chain_delivery::DeliveryOutcome; use crate::tree::AXElement; pub(crate) use super::chain_context::ChainContext; @@ -11,333 +12,139 @@ pub(crate) use super::chain_step::ChainStep; #[cfg(target_os = "macos")] mod imp { use super::*; - use crate::actions::{ax_helpers, chain_verify}; - use std::time::{Duration, Instant}; - - const DEFAULT_CHAIN_TIMEOUT: Duration = Duration::from_secs(10); - const MAX_CHAIN_TIMEOUT_MS: u64 = 300_000; + use crate::actions::chain_step_exec::execute_step; pub(crate) fn execute_chain( el: &AXElement, - caps: &ElementCaps, def: &ChainDef, ctx: &ChainContext, policy: InteractionPolicy, ) -> Result<Vec<ActionStep>, AdapterError> { - let deadline = ctx - .deadline - .unwrap_or_else(|| Instant::now() + chain_timeout()); - let ctx = ChainContext { - dynamic_value: ctx.dynamic_value, - deadline: Some(deadline), - }; let total = def.steps.len(); let mut steps = Vec::new(); - - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - ax_helpers::set_messaging_timeout(&crate::tree::element_for_pid(pid), 1.0); - } - ax_helpers::set_messaging_timeout(el, 1.0); - - if def.pre_scroll { - tracing::debug!("chain: pre-scroll AXScrollToVisible"); - ax_helpers::ensure_visible(el); - steps.push(ActionStep::attempted("AXScrollToVisible")); + if let Some(pid) = crate::system::app_ops::pid_from_element(el, ctx.deadline) { + crate::tree::attributes::set_messaging_timeout( + &crate::tree::element_for_pid(pid), + ctx.deadline, + )?; } + crate::tree::attributes::set_messaging_timeout(el, ctx.deadline)?; for (i, step) in def.steps.iter().enumerate() { - if Instant::now() > deadline { - tracing::debug!("chain: timeout after {i}/{total} steps, trying CGClick fallback"); - if let Some(cg) = def - .steps - .iter() - .find(|s| matches!(s, ChainStep::CGClick { .. })) - { - let label = step_label(cg); - if physical_click_permitted(policy) && execute_step(el, caps, cg, &ctx, policy)? - { - tracing::debug!("chain: CGClick fallback succeeded"); - steps.push(ActionStep::succeeded(label)); - return Ok(steps); - } - } - return Err( - AdapterError::timeout("Chain execution deadline exceeded").with_suggestion( - "Retry the command, refresh the snapshot, or increase AGENT_DESKTOP_CHAIN_TIMEOUT_MS for slow apps.", - ), - ); - } - if matches!(step, ChainStep::CGClick { .. }) && !physical_click_permitted(policy) { - return Err(AdapterError::policy_denied_for_policy( - "Physical click fallback is disabled by the current interaction policy", - policy, - )); + ctx.ensure_budget()?; + if !step_allowed(step, policy) { + continue; } let label = step_label(step); - if execute_step(el, caps, step, &ctx, policy)? { + let outcome = execute_step(el, step, ctx, policy)?; + if record_step_outcome( + &mut steps, + step, + outcome, + def.continue_after_unverified_delivery, + ) { tracing::debug!("chain: [{}/{}] {} -> success", i + 1, total, label); - steps.push(ActionStep::succeeded(label)); return Ok(steps); } tracing::debug!("chain: [{}/{}] {} -> skip", i + 1, total, label); - steps.push(ActionStep::skipped(label)); } tracing::debug!("chain: all {total} steps exhausted"); Err( AdapterError::new(ErrorCode::ActionFailed, "All chain steps exhausted") + .with_disposition(exhaustion_disposition(&steps)) .with_suggestion(def.suggestion), ) } + pub(crate) fn exhaustion_disposition( + steps: &[ActionStep], + ) -> agent_desktop_core::DeliverySemantics { + let delivered = steps.iter().any(|step| { + matches!( + step.outcome, + agent_desktop_core::ActionStepOutcome::Succeeded + ) + }); + if delivered { + agent_desktop_core::DeliverySemantics::delivered_unverified() + } else { + agent_desktop_core::DeliverySemantics::not_delivered() + } + } + + pub(crate) fn step_mechanism(step: &ChainStep) -> StepMechanism { + match step { + ChainStep::CGClick { .. } | ChainStep::FocusThenClearByKeyboard => { + StepMechanism::PhysicalSynthetic + } + _ => StepMechanism::SemanticApi, + } + } + + pub(crate) fn build_step(step: &ChainStep, outcome: DeliveryOutcome) -> ActionStep { + let label = step_label(step); + let mut built = match outcome { + DeliveryOutcome::NotDelivered => ActionStep::skipped(label), + DeliveryOutcome::SatisfiedNoDelivery => ActionStep::skipped(label).with_verified(true), + DeliveryOutcome::DeliveredUnverified | DeliveryOutcome::DeliveredVerified => { + ActionStep::succeeded(label) + } + }; + built = built.with_mechanism(step_mechanism(step)); + if outcome.was_delivered() { + built = built.with_verified(outcome.was_verified()); + } + built + } + + pub(crate) fn record_step_outcome( + steps: &mut Vec<ActionStep>, + step: &ChainStep, + outcome: DeliveryOutcome, + continue_after_unverified_delivery: bool, + ) -> bool { + steps.push(build_step(step, outcome)); + outcome.terminates_chain() + && !(continue_after_unverified_delivery + && outcome == DeliveryOutcome::DeliveredUnverified) + } + fn step_label(step: &ChainStep) -> &'static str { match step { ChainStep::Action(name) => name, ChainStep::SetBool { attr, .. } => attr, ChainStep::SetDynamic { attr } => attr, - ChainStep::FocusThenSetDynamic { attr } => attr, ChainStep::IncrementToDynamic => "IncrementToDynamic", ChainStep::FocusThenClearByKeyboard => "FocusThenClearByKeyboard", - ChainStep::ChildActions { .. } => "ChildActions", - ChainStep::AncestorActions { .. } => "AncestorActions", - ChainStep::Custom { label, .. } => label, ChainStep::CustomWithDeadline { label, .. } => label, ChainStep::CGClick { .. } => "CGClick", } } - fn execute_step( - el: &AXElement, - caps: &ElementCaps, - step: &ChainStep, - ctx: &ChainContext, - policy: InteractionPolicy, - ) -> Result<bool, AdapterError> { - match step { - ChainStep::Action(name) => ax_helpers::try_ax_action_retried_or_err(el, name), - - ChainStep::SetBool { attr, value } => { - let settable = match *attr { - "AXSelected" => caps.settable_selected, - "AXDisclosing" => caps.settable_disclosing, - "AXFocused" => caps.settable_focus, - _ => ax_helpers::is_attr_settable(el, attr), - }; - Ok(settable && set_bool_verified(el, attr, *value)?) - } - - ChainStep::SetDynamic { attr } => { - let value = match ctx.dynamic_value { - Some(v) => v, - None => return Ok(false), - }; - set_dynamic_verified(el, attr, value) - } - - ChainStep::FocusThenSetDynamic { attr } => { - if !policy.allow_focus_steal { - return Ok(false); - } - let value = match ctx.dynamic_value { - Some(v) => v, - None => return Ok(false), - }; - if !ax_helpers::ax_focus_or_err(el)? { - return Ok(false); - } - std::thread::sleep(Duration::from_millis(50)); - set_dynamic_verified(el, attr, value) - } - - ChainStep::IncrementToDynamic => match ctx.dynamic_value { - Some(value) => increment_to_value(el, value, ctx.deadline), - None => Ok(false), - }, - - ChainStep::FocusThenClearByKeyboard => { - if !policy.allow_focus_steal { - return Ok(false); - } - if !ax_helpers::ax_focus_or_err(el)? { - return Ok(false); - } - std::thread::sleep(Duration::from_millis(20)); - Ok(crate::input::keyboard::synthesize_key_for_element( - el, - &agent_desktop_core::action::KeyCombo { - key: "a".into(), - modifiers: vec![agent_desktop_core::action::Modifier::Cmd], - }, - ) - .and_then(|_| { - crate::input::keyboard::synthesize_key_for_element( - el, - &agent_desktop_core::action::KeyCombo { - key: "delete".into(), - modifiers: vec![], - }, - ) - }) - .is_ok()) - } - - ChainStep::ChildActions { actions, limit } => Ok(ax_helpers::try_each_child( - el, - |child| { - let child_actions = ax_helpers::list_ax_actions(child); - ax_helpers::try_action_from_list(child, &child_actions, actions) - }, - *limit, - )), - - ChainStep::AncestorActions { actions, limit } => Ok(ax_helpers::try_each_ancestor( - el, - |ancestor| { - let al = ax_helpers::list_ax_actions(ancestor); - ax_helpers::try_action_from_list(ancestor, &al, actions) - }, - *limit, - )), - - ChainStep::Custom { label: _, func } => func(el), - - ChainStep::CustomWithDeadline { label: _, func } => func(el, ctx.deadline), - - ChainStep::CGClick { button, count } => { - Ok( - crate::actions::dispatch::click_via_bounds(el, button.clone(), *count, policy) - .is_ok(), - ) - } - } - } - - fn chain_timeout() -> Duration { - std::env::var("AGENT_DESKTOP_CHAIN_TIMEOUT_MS") - .ok() - .and_then(|v| v.parse::<u64>().ok()) - .filter(|ms| *ms > 0) - .map(|ms| ms.min(MAX_CHAIN_TIMEOUT_MS)) - .map(Duration::from_millis) - .unwrap_or(DEFAULT_CHAIN_TIMEOUT) - } - - fn physical_click_permitted(policy: InteractionPolicy) -> bool { - policy.allow_focus_steal && policy.allow_cursor_move - } - - fn set_dynamic_verified(el: &AXElement, attr: &str, value: &str) -> Result<bool, AdapterError> { - if attr == "AXValue" { - ax_helpers::set_ax_value_coerced(el, value)?; - } else { - ax_helpers::set_ax_string_or_err(el, attr, value)?; - } - Ok(chain_verify::dynamic_write_had_effect( - attr, - ax_helpers::element_role(el).as_deref(), - value, - crate::tree::copy_value_typed(el).as_deref(), - )) - } - - /// Drives AXIncrement/AXDecrement until the control reaches `target`. - /// Steppers and some sliders expose no settable AXValue but step through - /// these actions. Stops on reaching the target or on no observable - /// progress (the action stopped moving the value). Deadline expiry is a - /// hard error: the control may sit at a half-applied value, and silently - /// reporting "step failed" would mask that mutation as ACTION_FAILED with - /// recovery guidance pointing the wrong way. - fn increment_to_value( - el: &AXElement, - target: &str, - deadline: Option<Instant>, - ) -> Result<bool, AdapterError> { - const MAX_INCREMENT_STEPS: usize = 1024; - - let target = match finite_target(target) { - Some(target) => target, - None => return Ok(false), - }; - let read = || crate::tree::copy_value_typed(el).and_then(|v| v.parse::<f64>().ok()); - let mut current = match read() { - Some(c) => c, - None => return Ok(false), - }; - let actions = ax_helpers::list_ax_actions(el); - if !actions.iter().any(|action| action == "AXIncrement") - && !actions.iter().any(|action| action == "AXDecrement") - { - return Ok(false); - } - let start = current; - for _ in 0..MAX_INCREMENT_STEPS { - if (current - target).abs() < 0.5 { - return Ok(true); - } - if deadline.is_some_and(|dl| Instant::now() > dl) { - return Err(chain_verify::increment_deadline_error( - start, current, target, - )); - } - let action = if current < target { - "AXIncrement" - } else { - "AXDecrement" - }; - if !ax_helpers::try_ax_action(el, action) { - break; - } - match read() { - Some(next) if (next - current).abs() >= f64::EPSILON => current = next, - _ => break, - } - } - if (current - target).abs() < 0.5 { - return Ok(true); - } - if (current - start).abs() >= f64::EPSILON { - return Err(chain_verify::increment_step_limit_error( - start, current, target, - )); - } - Ok(false) - } - - fn finite_target(target: &str) -> Option<f64> { - target.parse::<f64>().ok().filter(|value| value.is_finite()) - } - - fn set_bool_verified(el: &AXElement, attr: &str, value: bool) -> Result<bool, AdapterError> { - Ok(ax_helpers::set_ax_bool_or_err(el, attr, value)? - && chain_verify::bool_write_had_effect( - attr, - value, - crate::tree::copy_bool_attr(el, attr), - )) - } - - #[cfg(test)] - mod tests { - use super::finite_target; - - #[test] - fn finite_target_rejects_non_finite_numbers() { - assert_eq!(finite_target("42.5"), Some(42.5)); - assert_eq!(finite_target("NaN"), None); - assert_eq!(finite_target("inf"), None); - assert_eq!(finite_target("-inf"), None); - assert_eq!(finite_target("not-a-number"), None); - } + pub(crate) fn step_allowed(step: &ChainStep, policy: InteractionPolicy) -> bool { + !matches!( + step, + ChainStep::CGClick { .. } | ChainStep::FocusThenClearByKeyboard + ) || policy.is_headed() } } +#[cfg(all(test, target_os = "macos"))] +pub(crate) use imp::{ + build_step, exhaustion_disposition, record_step_outcome, step_allowed, step_mechanism, +}; + +#[cfg(test)] +#[path = "chain_tests.rs"] +mod tests; + #[cfg(not(target_os = "macos"))] mod imp { use super::*; pub fn execute_chain( _el: &AXElement, - _caps: &ElementCaps, def: &ChainDef, _ctx: &ChainContext, _policy: InteractionPolicy, diff --git a/crates/macos/src/actions/chain_context.rs b/crates/macos/src/actions/chain_context.rs index 733a41a..1c68079 100644 --- a/crates/macos/src/actions/chain_context.rs +++ b/crates/macos/src/actions/chain_context.rs @@ -1,4 +1,22 @@ pub(crate) struct ChainContext<'a> { pub(crate) dynamic_value: Option<&'a str>, - pub(crate) deadline: Option<std::time::Instant>, + pub(crate) verified_point: Option<&'a agent_desktop_core::Point>, + pub(crate) deadline: agent_desktop_core::Deadline, +} + +impl ChainContext<'_> { + pub(crate) fn remaining( + &self, + ) -> Result<std::time::Duration, agent_desktop_core::AdapterError> { + let remaining = self.deadline.remaining(); + if remaining.is_zero() { + Err(self.deadline.timeout_error()) + } else { + Ok(remaining) + } + } + + pub(crate) fn ensure_budget(&self) -> Result<(), agent_desktop_core::AdapterError> { + self.remaining().map(|_| ()) + } } diff --git a/crates/macos/src/actions/chain_def.rs b/crates/macos/src/actions/chain_def.rs index f1304fe..539a540 100644 --- a/crates/macos/src/actions/chain_def.rs +++ b/crates/macos/src/actions/chain_def.rs @@ -1,7 +1,7 @@ use super::chain_step::ChainStep; pub(crate) struct ChainDef { - pub(crate) pre_scroll: bool, pub(crate) steps: &'static [ChainStep], pub(crate) suggestion: &'static str, + pub(crate) continue_after_unverified_delivery: bool, } diff --git a/crates/macos/src/actions/chain_defs.rs b/crates/macos/src/actions/chain_defs.rs index c406cbb..d9396ef 100644 --- a/crates/macos/src/actions/chain_defs.rs +++ b/crates/macos/src/actions/chain_defs.rs @@ -1,204 +1,170 @@ -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::AdapterError; #[cfg(target_os = "macos")] mod imp { use super::*; use crate::actions::{ - ax_helpers, - chain::{ChainDef, ChainStep}, - chain_disclosure_steps, chain_menu_steps, chain_steps, + chain::ChainDef, chain::ChainStep, chain_disclosure_steps, chain_menu_steps, }; use crate::tree::AXElement; - use agent_desktop_core::{action::MouseButton, interaction_policy::InteractionPolicy}; + use agent_desktop_core::{ActionStep, Deadline, MouseButton, StepMechanism}; pub(crate) static CLICK_CHAIN: ChainDef = ChainDef { - pre_scroll: true, steps: &[ - ChainStep::Custom { - label: "verified_press", - func: chain_steps::do_verified_press, - }, - ChainStep::Action("AXConfirm"), - ChainStep::Action("AXOpen"), - ChainStep::Action("AXPick"), - ChainStep::Custom { - label: "show_alternate_ui", - func: chain_steps::try_show_alternate_ui, - }, - ChainStep::Custom { - label: "containing_item_select", - func: chain_steps::try_select_containing_item, - }, - ChainStep::SetBool { - attr: "AXSelected", - value: true, - }, - ChainStep::Custom { - label: "parent_row_select", - func: chain_steps::try_parent_row_select, - }, - ChainStep::Custom { - label: "value_relay", - func: chain_steps::try_value_relay, - }, - ChainStep::Custom { - label: "select_via_parent", - func: chain_steps::try_select_via_parent, - }, - ChainStep::ChildActions { - actions: &["AXPress", "AXConfirm", "AXOpen"], - limit: 3, - }, - ChainStep::Custom { - label: "custom_actions", - func: chain_steps::try_custom_actions, - }, - ChainStep::AncestorActions { - actions: &["AXPress", "AXConfirm"], - limit: 2, - }, ChainStep::CGClick { button: MouseButton::Left, count: 1, }, + ChainStep::Action("AXPress"), ], - suggestion: "Element may not be interactable. Try 'mouse-click --xy X,Y'.", + suggestion: "Target an element that advertises Click or use an explicit point click.", + continue_after_unverified_delivery: false, }; pub(crate) static RIGHT_CLICK_CHAIN: ChainDef = ChainDef { - pre_scroll: false, steps: &[ - ChainStep::Custom { - label: "show_menu", - func: chain_menu_steps::show_menu, - }, - ChainStep::Custom { - label: "select_then_show_menu", - func: chain_menu_steps::select_then_show_menu, - }, - ChainStep::Custom { - label: "selected_items_menu", - func: chain_menu_steps::select_then_selected_items_menu, - }, - ChainStep::Custom { - label: "child_show_menu", - func: chain_menu_steps::show_menu_on_children, - }, - ChainStep::Custom { - label: "ancestor_show_menu", - func: chain_menu_steps::show_menu_on_ancestors, - }, ChainStep::CGClick { button: MouseButton::Right, count: 1, }, + ChainStep::CustomWithDeadline { + label: "show_menu", + func: chain_menu_steps::show_menu, + }, + ChainStep::CustomWithDeadline { + label: "select_then_show_menu", + func: chain_menu_steps::select_then_show_menu, + }, + ChainStep::CustomWithDeadline { + label: "selected_items_menu", + func: chain_menu_steps::select_then_selected_items_menu, + }, + ChainStep::CustomWithDeadline { + label: "child_show_menu", + func: chain_menu_steps::show_menu_on_children, + }, + ChainStep::CustomWithDeadline { + label: "ancestor_show_menu", + func: chain_menu_steps::show_menu_on_ancestors, + }, ], suggestion: "Try 'mouse-click --button right --xy X,Y'.", + continue_after_unverified_delivery: false, }; - /// Every step is verified against the element's disclosed state: a bare - /// `AXExpand`/`AXSetAttributeValue` can return success without changing the - /// control (SwiftUI disclosures do this), so an unverified action must not - /// count as success. `expand_verified` tries the semantic action, then a - /// press, and only reports success when the state actually flipped. pub(crate) static EXPAND_CHAIN: ChainDef = ChainDef { - pre_scroll: false, steps: &[ChainStep::CustomWithDeadline { label: "expand_verified", func: chain_disclosure_steps::press_to_expand, }], - suggestion: "This control cannot be expanded via accessibility; try a physical 'click' on its disclosure triangle.", + suggestion: "Target a control with a readable expandable state.", + continue_after_unverified_delivery: false, }; pub(crate) static COLLAPSE_CHAIN: ChainDef = ChainDef { - pre_scroll: false, steps: &[ChainStep::CustomWithDeadline { label: "collapse_verified", func: chain_disclosure_steps::press_to_collapse, }], - suggestion: "This control cannot be collapsed via accessibility; try a physical 'click' on its disclosure triangle.", + suggestion: "Target a control with a readable expandable state.", + continue_after_unverified_delivery: false, }; - const VALUE_STEPS: &[ChainStep] = &[ - ChainStep::SetDynamic { attr: "AXValue" }, - ChainStep::FocusThenSetDynamic { attr: "AXValue" }, - ChainStep::IncrementToDynamic, - ]; - pub(crate) static SET_VALUE_CHAIN: ChainDef = ChainDef { - pre_scroll: false, - steps: VALUE_STEPS, - suggestion: "Try 'clear' then 'type', or check element is a text field.", + steps: &[ + ChainStep::SetDynamic { attr: "AXValue" }, + ChainStep::IncrementToDynamic, + ], + suggestion: "Target an element with a settable value or native increment/decrement actions.", + continue_after_unverified_delivery: true, }; pub(crate) static CLEAR_CHAIN: ChainDef = ChainDef { - pre_scroll: false, steps: &[ - ChainStep::SetDynamic { attr: "AXValue" }, - ChainStep::FocusThenSetDynamic { attr: "AXValue" }, ChainStep::FocusThenClearByKeyboard, + ChainStep::SetDynamic { attr: "AXValue" }, ], - suggestion: "Try 'press cmd+a' then 'press delete'.", + suggestion: "Target an editable control or allow the verified keyboard fallback.", + continue_after_unverified_delivery: true, + }; + + pub(crate) static SEMANTIC_CLICK_CHAIN: ChainDef = ChainDef { + steps: &[ChainStep::Action("AXPress")], + suggestion: "Target an element that advertises Click.", + continue_after_unverified_delivery: false, }; pub(crate) static FOCUS_CHAIN: ChainDef = ChainDef { - pre_scroll: false, - steps: &[ - ChainStep::SetBool { - attr: "AXFocused", - value: true, - }, - ChainStep::Action("AXRaise"), - ChainStep::Action("AXPress"), - ChainStep::SetBool { - attr: "AXSelected", - value: true, - }, - ChainStep::CGClick { - button: MouseButton::Left, - count: 1, - }, - ], - suggestion: "Try 'click' to focus the element instead.", + steps: &[ChainStep::SetBool { + attr: "AXFocused", + value: true, + }], + suggestion: "Target an element whose AXFocused attribute is settable.", + continue_after_unverified_delivery: false, }; pub(crate) static SCROLL_TO_CHAIN: ChainDef = ChainDef { - pre_scroll: false, - steps: &[ - ChainStep::Action("AXScrollToVisible"), - ChainStep::Custom { - label: "visible_in_scroll_context", - func: chain_steps::element_is_visible_in_scroll_context, - }, - ], - suggestion: "Element may not be in a scrollable container.", + steps: &[ChainStep::CustomWithDeadline { + label: "scroll_to_visible_verified", + func: crate::actions::scroll_into_view::scroll_into_view_outcome, + }], + suggestion: "Target an element that advertises AXScrollToVisible.", + continue_after_unverified_delivery: false, }; - /// Only treats AXOpen as a real double-click when the element actually - /// advertises it: `AXUIElementPerformAction` returns success for an - /// unsupported action on some controls, which would make double-click claim - /// success while doing nothing. Elements without AXOpen (e.g. a button that - /// only fires on a true mouse double-click) require the physical path, which - /// fails closed under the headless policy. pub(crate) fn double_click( - el: &AXElement, - policy: InteractionPolicy, - ) -> Result<(), AdapterError> { - if ax_helpers::has_ax_action(el, "AXOpen") && ax_helpers::try_ax_action(el, "AXOpen") { - return Ok(()); - } - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2, policy) + element: &AXElement, + request: &agent_desktop_core::ActionRequest, + deadline: Deadline, + ) -> Result<Vec<ActionStep>, AdapterError> { + physical_multi_click(element, 2, request, deadline) } - /// Triple-click has no AX semantic equivalent on macOS and is therefore - /// unconditionally physical: it always requires cursor-move + focus-steal - /// policy. Callers that gate on `allow_cursor_move` will receive a - /// `PolicyDenied` error from `click_via_bounds` rather than a silent no-op. pub(crate) fn triple_click( - el: &AXElement, - policy: InteractionPolicy, - ) -> Result<(), AdapterError> { - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3, policy) + element: &AXElement, + request: &agent_desktop_core::ActionRequest, + deadline: Deadline, + ) -> Result<Vec<ActionStep>, AdapterError> { + physical_multi_click(element, 3, request, deadline) + } + + fn physical_multi_click( + element: &AXElement, + count: u32, + request: &agent_desktop_core::ActionRequest, + deadline: Deadline, + ) -> Result<Vec<ActionStep>, AdapterError> { + crate::actions::physical_click::click_via_bounds( + element, + crate::actions::physical_click::PhysicalClick { + button: MouseButton::Left, + count, + verified_point: request.verified_point().cloned(), + }, + request.policy, + deadline, + )?; + Ok(vec![ + ActionStep::succeeded("CGClick") + .with_mechanism(StepMechanism::PhysicalSynthetic) + .with_verified(false), + ]) + } + + #[cfg(test)] + mod tests { + use super::{COLLAPSE_CHAIN, EXPAND_CHAIN}; + use crate::actions::chain_step::ChainStep; + + #[test] + fn disclosure_chains_use_verified_semantic_delivery() { + for chain in [&EXPAND_CHAIN, &COLLAPSE_CHAIN] { + assert!(matches!( + chain.steps.first(), + Some(ChainStep::CustomWithDeadline { .. }) + )); + } + } } } @@ -208,5 +174,5 @@ mod imp {} #[cfg(target_os = "macos")] pub(crate) use imp::{ CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, FOCUS_CHAIN, RIGHT_CLICK_CHAIN, - SCROLL_TO_CHAIN, SET_VALUE_CHAIN, double_click, triple_click, + SCROLL_TO_CHAIN, SEMANTIC_CLICK_CHAIN, SET_VALUE_CHAIN, double_click, triple_click, }; diff --git a/crates/macos/src/actions/chain_delivery.rs b/crates/macos/src/actions/chain_delivery.rs new file mode 100644 index 0000000..bfbc869 --- /dev/null +++ b/crates/macos/src/actions/chain_delivery.rs @@ -0,0 +1,53 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DeliveryOutcome { + NotDelivered, + SatisfiedNoDelivery, + DeliveredUnverified, + DeliveredVerified, +} + +impl DeliveryOutcome { + pub(crate) fn from_delivery(delivered: bool, verified: bool) -> Self { + match (delivered, verified) { + (false, _) => Self::NotDelivered, + (true, false) => Self::DeliveredUnverified, + (true, true) => Self::DeliveredVerified, + } + } + + pub(crate) fn was_delivered(self) -> bool { + matches!(self, Self::DeliveredUnverified | Self::DeliveredVerified) + } + + pub(crate) fn was_verified(self) -> bool { + matches!(self, Self::SatisfiedNoDelivery | Self::DeliveredVerified) + } + + pub(crate) fn terminates_chain(self) -> bool { + !matches!(self, Self::NotDelivered) + } +} + +#[cfg(test)] +mod tests { + use super::DeliveryOutcome; + + #[test] + fn delivery_and_verification_are_independent() { + assert_eq!( + DeliveryOutcome::from_delivery(false, true), + DeliveryOutcome::NotDelivered + ); + assert_eq!( + DeliveryOutcome::from_delivery(true, false), + DeliveryOutcome::DeliveredUnverified + ); + assert_eq!( + DeliveryOutcome::from_delivery(true, true), + DeliveryOutcome::DeliveredVerified + ); + assert!(DeliveryOutcome::SatisfiedNoDelivery.terminates_chain()); + assert!(!DeliveryOutcome::SatisfiedNoDelivery.was_delivered()); + assert!(DeliveryOutcome::SatisfiedNoDelivery.was_verified()); + } +} diff --git a/crates/macos/src/actions/chain_disclosure_steps.rs b/crates/macos/src/actions/chain_disclosure_steps.rs index abd63b8..d8347dd 100644 --- a/crates/macos/src/actions/chain_disclosure_steps.rs +++ b/crates/macos/src/actions/chain_disclosure_steps.rs @@ -1,117 +1,181 @@ #[cfg(target_os = "macos")] mod imp { - use crate::actions::ax_helpers; + use crate::actions::chain_delivery::DeliveryOutcome; use crate::tree::AXElement; - use agent_desktop_core::error::AdapterError; + use agent_desktop_core::{AdapterError, Deadline}; + use std::time::{Duration, Instant}; - /// Expands a disclosure that toggles via press (no settable `AXExpanded`). - /// Idempotent: a no-op when already expanded; otherwise presses and - /// confirms the disclosed state flipped. pub(crate) fn press_to_expand( - el: &AXElement, - chain_deadline: Option<std::time::Instant>, - ) -> Result<bool, AdapterError> { - press_toggle_disclosure(el, true, chain_deadline) + element: &AXElement, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + set_disclosure(element, true, deadline) } - /// Collapses a press-toggled disclosure, mirroring [`press_to_expand`]. pub(crate) fn press_to_collapse( - el: &AXElement, - chain_deadline: Option<std::time::Instant>, - ) -> Result<bool, AdapterError> { - press_toggle_disclosure(el, false, chain_deadline) + element: &AXElement, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + set_disclosure(element, false, deadline) } - /// Tries the semantic action / settable attribute, then a press. Each is - /// confirmed against the disclosed state; an action that succeeds at the AX - /// layer but does not move the control is not counted. A settle wait that - /// was truncated by the chain deadline is a hard TIMEOUT (mirroring the - /// increment path): the press may still land after the truncated wait, so - /// reporting a plain step failure would mask a possible mutation as - /// ACTION_FAILED. - fn press_toggle_disclosure( - el: &AXElement, - want_expanded: bool, - chain_deadline: Option<std::time::Instant>, - ) -> Result<bool, AdapterError> { - if disclosed_state(el) == Some(want_expanded) { - return Ok(true); + fn set_disclosure( + element: &AXElement, + expanded: bool, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + let (satisfied, allow_toggle) = + disclosure_plan(disclosed_state(element, deadline)?, expanded); + if satisfied { + return Ok(DeliveryOutcome::SatisfiedNoDelivery); } - let action = if want_expanded { - "AXExpand" - } else { - "AXCollapse" - }; - if ax_helpers::has_ax_action(el, action) { - let _ = ax_helpers::try_ax_action_retried_or_err(el, action)?; - if disclosure_settled(el, want_expanded, chain_deadline)? { - return Ok(true); + let action = if expanded { "AXExpand" } else { "AXCollapse" }; + prepare(element, deadline)?; + if crate::actions::ax_helpers::try_ax_action_or_err(element, action, deadline)? { + let outcome = verify_disclosure(element, expanded, deadline).map_err(after_delivery)?; + if let Some(delivered) = stop_if_delivered(outcome) { + return Ok(delivered); } } - if ax_helpers::is_attr_settable(el, "AXExpanded") { - let _ = ax_helpers::set_ax_bool_or_err(el, "AXExpanded", want_expanded)?; - if disclosure_settled(el, want_expanded, chain_deadline)? { - return Ok(true); + prepare(element, deadline)?; + if crate::actions::ax_helpers::is_attr_settable(element, "AXExpanded", deadline)? { + prepare(element, deadline)?; + if crate::actions::ax_helpers::set_ax_bool_or_err( + element, + "AXExpanded", + expanded, + deadline, + )? { + let outcome = + verify_disclosure(element, expanded, deadline).map_err(after_delivery)?; + if let Some(delivered) = stop_if_delivered(outcome) { + return Ok(delivered); + } } } - if ax_helpers::has_ax_action(el, "AXPress") - && ax_helpers::try_ax_action_retried_or_err(el, "AXPress")? - && disclosure_settled(el, want_expanded, chain_deadline)? - { - return Ok(true); + if allow_toggle && disclosed_state(element, deadline)? == Some(!expanded) { + prepare(element, deadline)?; + if crate::actions::ax_helpers::try_ax_action_or_err(element, "AXPress", deadline)? { + let outcome = + verify_disclosure(element, expanded, deadline).map_err(after_delivery)?; + if let Some(delivered) = stop_if_delivered(outcome) { + return Ok(delivered); + } + } } - Ok(false) + Ok(DeliveryOutcome::NotDelivered) } - /// Polls for the disclosed state instead of a fixed settle sleep: fast UIs - /// confirm on the first read, while animated disclosures get up to the - /// settle budget. The budget is capped to the chain's remaining deadline; - /// an exit forced by that cap (rather than the full budget elapsing) is - /// reported as `DeadlineExpired`, never as a plain failure. At least one - /// state read always happens, even with the deadline already past. - fn disclosure_settled( - el: &AXElement, - want_expanded: bool, - chain_deadline: Option<std::time::Instant>, - ) -> Result<bool, AdapterError> { - use std::time::{Duration, Instant}; + fn stop_if_delivered(outcome: DeliveryOutcome) -> Option<DeliveryOutcome> { + outcome.was_delivered().then_some(outcome) + } - const POLL_INTERVAL: Duration = Duration::from_millis(20); - const SETTLE_BUDGET: Duration = Duration::from_millis(200); + fn disclosure_plan(current: Option<bool>, desired: bool) -> (bool, bool) { + (current == Some(desired), current == Some(!desired)) + } - let budget_end = Instant::now() + SETTLE_BUDGET; - let deadline = chain_deadline.map_or(budget_end, |dl| dl.min(budget_end)); - let truncated = deadline < budget_end; + fn verify_disclosure( + element: &AXElement, + expanded: bool, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + let local_end = Instant::now() + Duration::from_millis(200); loop { - if disclosed_state(el) == Some(want_expanded) { - return Ok(true); + let current_state = disclosed_state(element, deadline)?; + match current_state { + Some(current) if current == expanded => { + return Ok(DeliveryOutcome::DeliveredVerified); + } + _ => {} } - let now = Instant::now(); - if now >= deadline { - return if truncated { - Err(crate::actions::chain_verify::disclosure_deadline_error( - want_expanded, - disclosed_state(el), - )) - } else { - Ok(false) - }; + if deadline.is_expired() { + return Err(deadline.timeout_error().with_details(serde_json::json!({ + "verification": "expanded_state_not_observed", + }))); } - std::thread::sleep(POLL_INTERVAL.min(deadline - now)); + if Instant::now() >= local_end { + return unobserved_delivery(current_state, expanded); + } + let pause = deadline.remaining_slice(Duration::from_millis(20))?; + std::thread::sleep(pause.min(Duration::from_millis(20))); } } - fn disclosed_state(el: &AXElement) -> Option<bool> { - crate::tree::copy_bool_attr(el, "AXExpanded") - .or_else(|| crate::tree::copy_bool_attr(el, "AXDisclosing")) - .or_else(|| value_as_bool(el)) + fn unobserved_delivery( + last_state: Option<bool>, + expanded: bool, + ) -> Result<DeliveryOutcome, AdapterError> { + if last_state == Some(!expanded) { + Ok(DeliveryOutcome::NotDelivered) + } else { + Ok(DeliveryOutcome::DeliveredUnverified) + } } - fn value_as_bool(el: &AXElement) -> Option<bool> { - match crate::tree::copy_value_typed(el).as_deref() { - Some("1" | "true" | "True") => Some(true), - Some("0" | "false" | "False") => Some(false), - _ => None, + fn disclosed_state( + element: &AXElement, + deadline: Deadline, + ) -> Result<Option<bool>, AdapterError> { + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + if let Some(value) = crate::tree::surface_read::boolean(element, "AXExpanded", instant)? { + return Ok(Some(value)); + } + crate::tree::surface_read::boolean(element, "AXDisclosing", instant) + } + + fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) + } + + fn after_delivery(error: AdapterError) -> AdapterError { + let mut delivery = crate::actions::DeliveryTracker::default(); + delivery.mark_delivered(); + delivery.annotate(error) + } + + #[cfg(test)] + mod tests { + use super::{DeliveryOutcome, disclosure_plan, stop_if_delivered, unobserved_delivery}; + + #[test] + fn disclosure_plan_never_blindly_toggles_unknown_state() { + assert_eq!(disclosure_plan(Some(true), true), (true, false)); + assert_eq!(disclosure_plan(Some(false), true), (false, true)); + assert_eq!(disclosure_plan(None, true), (false, false)); + assert_eq!(disclosure_plan(Some(false), false), (true, false)); + assert_eq!(disclosure_plan(Some(true), false), (false, true)); + assert_eq!(disclosure_plan(None, false), (false, false)); + } + + #[test] + fn unobserved_delivery_preserves_honest_unknown_state() { + assert_eq!( + unobserved_delivery(Some(false), true).expect("known opposite state"), + DeliveryOutcome::NotDelivered + ); + assert_eq!( + unobserved_delivery(None, true).expect("unreadable state after delivery"), + DeliveryOutcome::DeliveredUnverified + ); + assert_eq!( + stop_if_delivered( + unobserved_delivery(None, true).expect("unreadable state after delivery") + ), + Some(DeliveryOutcome::DeliveredUnverified) + ); + } + + #[test] + fn delivered_outcomes_halt_further_mutation_strategies() { + assert_eq!( + stop_if_delivered(DeliveryOutcome::DeliveredUnverified), + Some(DeliveryOutcome::DeliveredUnverified) + ); + assert_eq!( + stop_if_delivered(DeliveryOutcome::DeliveredVerified), + Some(DeliveryOutcome::DeliveredVerified) + ); + assert_eq!(stop_if_delivered(DeliveryOutcome::NotDelivered), None); } } } diff --git a/crates/macos/src/actions/chain_menu_steps.rs b/crates/macos/src/actions/chain_menu_steps.rs index d59f731..62f5a4a 100644 --- a/crates/macos/src/actions/chain_menu_steps.rs +++ b/crates/macos/src/actions/chain_menu_steps.rs @@ -1,234 +1,251 @@ -#[cfg(target_os = "macos")] -mod imp { - use crate::actions::ax_helpers; - use crate::tree::AXElement; - use agent_desktop_core::error::AdapterError; +use agent_desktop_core::{AdapterError, Deadline}; - pub(crate) fn show_menu(el: &AXElement) -> Result<bool, AdapterError> { - show_menu_on_element(el) - } +use crate::actions::chain_delivery::DeliveryOutcome; +use crate::tree::AXElement; - pub(crate) fn show_menu_on_ancestors(el: &AXElement) -> Result<bool, AdapterError> { - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..3 { - let Some(parent) = current else { - return Ok(false); - }; - if show_menu_on_element(&parent)? { - return Ok(true); - } - current = crate::tree::copy_element_attr(&parent, "AXParent"); +const MAX_MENU_SEARCH_NODES: usize = 2_048; + +pub(crate) fn show_menu( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + show_menu_on_element(element, deadline) +} + +pub(crate) fn show_menu_on_ancestors( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + let mut current = parent(element, deadline)?; + for _ in 0..3 { + let Some(ancestor) = current else { + return Ok(DeliveryOutcome::NotDelivered); + }; + let outcome = show_menu_on_element(&ancestor, deadline)?; + if outcome.terminates_chain() { + return Ok(outcome); } - Ok(false) + current = parent(&ancestor, deadline)?; } + Ok(DeliveryOutcome::NotDelivered) +} - pub(crate) fn show_menu_on_children(el: &AXElement) -> Result<bool, AdapterError> { - for child in crate::tree::copy_ax_array(el, "AXChildren") - .unwrap_or_default() - .iter() - .take(5) +pub(crate) fn show_menu_on_children( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + let instant = instant(deadline)?; + for child in crate::tree::surface_read::elements(element, "AXChildren", instant)? + .into_iter() + .take(5) + { + let outcome = show_menu_on_element(&child, deadline)?; + if outcome.terminates_chain() { + return Ok(outcome); + } + } + Ok(DeliveryOutcome::NotDelivered) +} + +pub(crate) fn select_then_show_menu( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + if !select_containing_item(element, deadline)? { + return Ok(DeliveryOutcome::NotDelivered); + } + show_menu_on_element(element, deadline) +} + +pub(crate) fn select_then_selected_items_menu( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + if !select_containing_item(element, deadline)? { + return Ok(DeliveryOutcome::NotDelivered); + } + let Some(window) = window_ancestor(element, deadline)? else { + return Ok(DeliveryOutcome::NotDelivered); + }; + let Some(menu_button) = selected_items_menu_button(&window, deadline)? else { + return Ok(DeliveryOutcome::NotDelivered); + }; + show_menu_or_press(&menu_button, deadline) +} + +fn show_menu_on_element( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + let Some(pid) = crate::system::app_ops::pid_from_element(element, deadline) else { + return Ok(DeliveryOutcome::NotDelivered); + }; + if menu_is_open(pid, deadline)? { + return Ok(DeliveryOutcome::NotDelivered); + } + prepare(element, deadline)?; + let delivered = + crate::actions::ax_helpers::try_ax_action_or_err(element, "AXShowMenu", deadline)?; + if !delivered { + return Ok(DeliveryOutcome::NotDelivered); + } + Ok(DeliveryOutcome::from_delivery( + true, + wait_for_new_menu(pid, deadline)?, + )) +} + +fn show_menu_or_press( + element: &AXElement, + deadline: Deadline, +) -> Result<DeliveryOutcome, AdapterError> { + let Some(pid) = crate::system::app_ops::pid_from_element(element, deadline) else { + return Ok(DeliveryOutcome::NotDelivered); + }; + if menu_is_open(pid, deadline)? { + return Ok(DeliveryOutcome::NotDelivered); + } + for action in ["AXShowMenu", "AXPress"] { + prepare(element, deadline)?; + if crate::actions::ax_helpers::try_ax_action_or_err(element, action, deadline)? { + return Ok(DeliveryOutcome::from_delivery( + true, + wait_for_new_menu(pid, deadline)?, + )); + } + } + Ok(DeliveryOutcome::NotDelivered) +} + +fn wait_for_new_menu(pid: i32, deadline: Deadline) -> Result<bool, AdapterError> { + let local_end = std::time::Instant::now() + std::time::Duration::from_millis(600); + loop { + if menu_is_open(pid, deadline)? { + return Ok(true); + } + if deadline.is_expired() || std::time::Instant::now() >= local_end { + return Ok(false); + } + let pause = deadline.remaining_slice(std::time::Duration::from_millis(25))?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(25))); + } +} + +fn menu_is_open(pid: i32, deadline: Deadline) -> Result<bool, AdapterError> { + crate::tree::surfaces::is_menu_open(pid, instant(deadline)?) +} + +fn select_containing_item(element: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + let mut current = Some(element.clone()); + for _ in 0..4 { + let Some(candidate) = current else { + return Ok(false); + }; + prepare(&candidate, deadline)?; + if crate::actions::ax_helpers::is_attr_settable(&candidate, "AXSelected", deadline)? { + prepare(&candidate, deadline)?; + if crate::actions::ax_helpers::set_ax_bool_or_err( + &candidate, + "AXSelected", + true, + deadline, + )? { + return selected(&candidate, deadline); + } + } + current = parent(&candidate, deadline)?; + } + Ok(false) +} + +fn selected(element: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + Ok( + crate::tree::surface_read::boolean(element, "AXSelected", instant(deadline)?)? + == Some(true), + ) +} + +fn window_ancestor( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + let mut current = parent(element, deadline)?; + for _ in 0..20 { + let Some(ancestor) = current else { + return Ok(None); + }; + if crate::tree::surface_read::string(&ancestor, "AXRole", instant(deadline)?)?.as_deref() + == Some("AXWindow") { - if show_menu_on_element(child)? { - return Ok(true); - } + return Ok(Some(ancestor)); } - Ok(false) + current = parent(&ancestor, deadline)?; } + Ok(None) +} - pub(crate) fn select_then_show_menu(el: &AXElement) -> Result<bool, AdapterError> { - if !select_containing_item(el)? { - return Ok(false); +fn selected_items_menu_button( + root: &AXElement, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + let mut stack = vec![(root.clone(), 0_u8)]; + let mut visited = 0_usize; + while let Some((candidate, depth)) = stack.pop() { + visited = visited.saturating_add(1); + if visited > MAX_MENU_SEARCH_NODES { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Selected-items menu search exceeded its accessibility-node budget", + ) + .with_details(serde_json::json!({ + "kind": "selected_items_menu_node_limit", + "limit": MAX_MENU_SEARCH_NODES, + "complete": false, + }))); } - std::thread::sleep(std::time::Duration::from_millis(50)); - show_menu_on_element(el) - } - - pub(crate) fn select_then_selected_items_menu(el: &AXElement) -> Result<bool, AdapterError> { - if !select_containing_item(el)? { - tracing::debug!("selected-items menu: could not select containing item"); - return Ok(false); + if is_selected_items_control(&candidate, deadline)? { + return Ok(Some(candidate)); } - let selected_name = crate::tree::resolve_element_name(el); - let Some(window) = window_ancestor(el) else { - tracing::debug!("selected-items menu: no window ancestor"); - return Ok(false); - }; - let Some(menu_button) = selected_items_menu_button(&window) else { - tracing::debug!("selected-items menu: no selected-items menu button"); - return Ok(false); - }; - show_menu_or_press_selected(&menu_button, selected_name.as_deref()) - } - - fn show_menu_on_element(el: &AXElement) -> Result<bool, AdapterError> { - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return Ok(false); - }; - let was_open = is_menu_open(pid); - if !ax_helpers::try_ax_action_retried_or_err(el, "AXShowMenu")? { - return Ok(false); + if depth >= 8 { + continue; } - Ok(wait_for_new_menu(pid, was_open)) + stack.extend( + crate::tree::surface_read::elements(&candidate, "AXChildren", instant(deadline)?)? + .into_iter() + .rev() + .map(|child| (child, depth.saturating_add(1))), + ); } + Ok(None) +} - fn show_menu_or_press_selected( - el: &AXElement, - selected_name: Option<&str>, - ) -> Result<bool, AdapterError> { - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return Ok(false); - }; - if ax_helpers::try_ax_action_retried_or_err(el, "AXShowMenu")? - && wait_for_selected_menu(pid, selected_name) +fn is_selected_items_control( + element: &AXElement, + deadline: Deadline, +) -> Result<bool, AdapterError> { + if crate::tree::surface_read::string(element, "AXRole", instant(deadline)?)?.as_deref() + != Some("AXMenuButton") + { + return Ok(false); + } + for attribute in ["AXHelp", "AXDescription", "AXTitle"] { + if crate::tree::surface_read::string(element, attribute, instant(deadline)?)? + .is_some_and(|value| value.to_ascii_lowercase().contains("selected item")) { return Ok(true); } - Ok(ax_helpers::try_ax_action_retried_or_err(el, "AXPress")? - && wait_for_selected_menu(pid, selected_name)) - } - - fn wait_for_new_menu(pid: i32, was_open: bool) -> bool { - if was_open { - return false; - } - crate::system::wait::wait_for_menu(pid, true, crate::system::wait::menu_timeout_ms()) - .is_ok() - } - - fn is_menu_open(pid: i32) -> bool { - crate::system::wait::wait_for_menu(pid, true, 0).is_ok() - } - - fn wait_for_selected_menu(pid: i32, selected_name: Option<&str>) -> bool { - let deadline = std::time::Instant::now() - + std::time::Duration::from_millis(crate::system::wait::menu_timeout_ms()); - loop { - if menu_matches_selection(pid, selected_name) { - return true; - } - if std::time::Instant::now() >= deadline { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - } - - fn menu_matches_selection(pid: i32, selected_name: Option<&str>) -> bool { - let Some(menu) = crate::tree::surfaces::menu_element_for_pid(pid) else { - return false; - }; - selected_name - .filter(|name| !name.is_empty()) - .is_none_or(|name| element_text_contains(&menu, name, 0)) - } - - fn element_text_contains(el: &AXElement, needle: &str, depth: usize) -> bool { - find_descendant_value(el, depth, &|candidate| { - element_own_text_contains(candidate, needle).then_some(()) - }) - .is_some() - } - - fn element_own_text_contains(el: &AXElement, needle: &str) -> bool { - ["AXTitle", "AXDescription", "AXValue", "AXHelp"] - .into_iter() - .filter_map(|attr| crate::tree::copy_string_attr(el, attr)) - .any(|value| value.contains(needle)) - } - - fn select_containing_item(el: &AXElement) -> Result<bool, AdapterError> { - Ok(ax_helpers::set_ax_bool_or_err(el, "AXSelected", true)? - || crate::actions::chain_steps::try_select_containing_item(el)?) - } - - fn window_ancestor(el: &AXElement) -> Option<AXElement> { - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..20 { - let ancestor = current?; - if crate::tree::copy_string_attr(&ancestor, "AXRole").as_deref() == Some("AXWindow") { - return Some(ancestor); - } - current = crate::tree::copy_element_attr(&ancestor, "AXParent"); - } - None - } - - fn selected_items_menu_button(root: &AXElement) -> Option<AXElement> { - find_descendant_value(root, 0, &|el| { - (crate::tree::copy_string_attr(el, "AXRole").as_deref() == Some("AXMenuButton") - && is_selected_items_control(el)) - .then(|| el.clone()) - }) - } - - fn is_selected_items_control(el: &AXElement) -> bool { - ["AXHelp", "AXDescription", "AXTitle"] - .into_iter() - .filter_map(|attr| crate::tree::copy_string_attr(el, attr)) - .any(|value| { - let matches = selected_items_text(&value); - if matches { - tracing::debug!( - text_chars = value.chars().count(), - "selected-items menu: matched control text" - ); - } - matches - }) - } - - fn selected_items_text(value: &str) -> bool { - let value = value.to_ascii_lowercase(); - value.contains("selected item") - } - - fn find_descendant_value<T>( - el: &AXElement, - depth: usize, - mapper: &impl Fn(&AXElement) -> Option<T>, - ) -> Option<T> { - if depth > 8 { - return None; - } - if let Some(value) = mapper(el) { - return Some(value); - } - for child in crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default() { - if let Some(found) = find_descendant_value(&child, depth + 1, mapper) { - return Some(found); - } - } - None } + Ok(false) } -#[cfg(not(target_os = "macos"))] -mod imp { - use crate::tree::AXElement; - use agent_desktop_core::error::AdapterError; - - pub(crate) fn show_menu(_el: &AXElement) -> Result<bool, AdapterError> { - Ok(false) - } - - pub(crate) fn show_menu_on_ancestors(_el: &AXElement) -> Result<bool, AdapterError> { - Ok(false) - } - - pub(crate) fn show_menu_on_children(_el: &AXElement) -> Result<bool, AdapterError> { - Ok(false) - } - - pub(crate) fn select_then_show_menu(_el: &AXElement) -> Result<bool, AdapterError> { - Ok(false) - } - - pub(crate) fn select_then_selected_items_menu(_el: &AXElement) -> Result<bool, AdapterError> { - Ok(false) - } +fn parent(element: &AXElement, deadline: Deadline) -> Result<Option<AXElement>, AdapterError> { + crate::tree::surface_read::element(element, "AXParent", instant(deadline)?) } -pub(crate) use imp::{ - select_then_selected_items_menu, select_then_show_menu, show_menu, show_menu_on_ancestors, - show_menu_on_children, -}; +fn instant(deadline: Deadline) -> Result<std::time::Instant, AdapterError> { + crate::tree::locator_deadline::from_operation(deadline) +} + +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} diff --git a/crates/macos/src/actions/chain_step.rs b/crates/macos/src/actions/chain_step.rs index 40e1739..abcc321 100644 --- a/crates/macos/src/actions/chain_step.rs +++ b/crates/macos/src/actions/chain_step.rs @@ -1,5 +1,6 @@ -use agent_desktop_core::{action::MouseButton, error::AdapterError}; +use agent_desktop_core::{AdapterError, MouseButton}; +use crate::actions::chain_delivery::DeliveryOutcome; use crate::tree::AXElement; pub(crate) enum ChainStep { @@ -11,31 +12,11 @@ pub(crate) enum ChainStep { SetDynamic { attr: &'static str, }, - FocusThenSetDynamic { - attr: &'static str, - }, - /// Converges a stepper/slider to the dynamic target value via repeated - /// AXIncrement/AXDecrement actions, for controls whose AXValue is not - /// directly settable. IncrementToDynamic, FocusThenClearByKeyboard, - ChildActions { - actions: &'static [&'static str], - limit: usize, - }, - AncestorActions { - actions: &'static [&'static str], - limit: usize, - }, - Custom { - label: &'static str, - func: fn(&AXElement) -> Result<bool, AdapterError>, - }, - /// Like `Custom`, for steps that poll for a settled state and must cap - /// that settle wait to the chain's remaining deadline budget. CustomWithDeadline { label: &'static str, - func: fn(&AXElement, Option<std::time::Instant>) -> Result<bool, AdapterError>, + func: fn(&AXElement, agent_desktop_core::Deadline) -> Result<DeliveryOutcome, AdapterError>, }, CGClick { button: MouseButton, diff --git a/crates/macos/src/actions/chain_step_exec.rs b/crates/macos/src/actions/chain_step_exec.rs new file mode 100644 index 0000000..7a1d4a0 --- /dev/null +++ b/crates/macos/src/actions/chain_step_exec.rs @@ -0,0 +1,121 @@ +#[cfg(target_os = "macos")] +mod imp { + use agent_desktop_core::AdapterError; + use agent_desktop_core::interaction_policy::InteractionPolicy; + + use crate::actions::ax_helpers; + use crate::actions::chain_context::ChainContext; + use crate::actions::chain_delivery::DeliveryOutcome; + use crate::actions::chain_step::ChainStep; + use crate::actions::chain_value_write::{ + increment_to_value, set_bool_verified, set_dynamic_verified, + }; + use crate::tree::AXElement; + + pub(crate) fn execute_step( + el: &AXElement, + step: &ChainStep, + ctx: &ChainContext, + policy: InteractionPolicy, + ) -> Result<DeliveryOutcome, AdapterError> { + ctx.ensure_budget()?; + match step { + ChainStep::Action(name) => { + prepare(el, ctx.deadline)?; + Ok(DeliveryOutcome::from_delivery( + ax_helpers::try_ax_action_or_err(el, name, ctx.deadline)?, + false, + )) + } + + ChainStep::SetBool { attr, value } => { + prepare(el, ctx.deadline)?; + let settable = ax_helpers::is_attr_settable(el, attr, ctx.deadline)?; + if settable { + set_bool_verified(el, attr, *value, ctx.deadline) + } else { + Ok(DeliveryOutcome::NotDelivered) + } + } + + ChainStep::SetDynamic { attr } => { + let value = match ctx.dynamic_value { + Some(v) => v, + None => return Ok(DeliveryOutcome::NotDelivered), + }; + prepare(el, ctx.deadline)?; + if !ax_helpers::is_attr_settable(el, attr, ctx.deadline)? { + return Ok(DeliveryOutcome::NotDelivered); + } + set_dynamic_verified(el, attr, value, ctx.deadline) + } + + ChainStep::IncrementToDynamic => match ctx.dynamic_value { + Some(value) => increment_to_value(el, value, ctx.deadline), + None => Ok(DeliveryOutcome::NotDelivered), + }, + + ChainStep::FocusThenClearByKeyboard => { + if !policy.is_headed() { + return Ok(DeliveryOutcome::NotDelivered); + } + crate::actions::physical_keyboard::press_sequence( + el, + &[ + agent_desktop_core::KeyCombo { + key: "a".into(), + modifiers: vec![agent_desktop_core::Modifier::Meta], + }, + agent_desktop_core::KeyCombo { + key: "delete".into(), + modifiers: vec![], + }, + ], + policy, + ctx.deadline, + )?; + Ok(DeliveryOutcome::DeliveredUnverified) + } + + ChainStep::CustomWithDeadline { label: _, func } => func(el, ctx.deadline), + + ChainStep::CGClick { button, count } => { + if !policy.is_headed() { + return Ok(DeliveryOutcome::NotDelivered); + } + physical_click(el, button.clone(), *count, ctx, policy)?; + Ok(DeliveryOutcome::DeliveredUnverified) + } + } + } + + fn physical_click( + element: &AXElement, + button: agent_desktop_core::MouseButton, + count: u32, + context: &ChainContext<'_>, + policy: InteractionPolicy, + ) -> Result<(), AdapterError> { + context.ensure_budget()?; + crate::actions::physical_click::click_via_bounds( + element, + crate::actions::physical_click::PhysicalClick { + button, + count, + verified_point: context.verified_point.cloned(), + }, + policy, + context.deadline, + ) + } + + fn prepare( + element: &AXElement, + deadline: agent_desktop_core::Deadline, + ) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) + } +} + +#[cfg(target_os = "macos")] +pub(crate) use imp::execute_step; diff --git a/crates/macos/src/actions/chain_steps.rs b/crates/macos/src/actions/chain_steps.rs deleted file mode 100644 index 7bdc4ae..0000000 --- a/crates/macos/src/actions/chain_steps.rs +++ /dev/null @@ -1,311 +0,0 @@ -#[cfg(target_os = "macos")] -mod imp { - use crate::actions::ax_helpers; - use crate::tree::AXElement; - use agent_desktop_core::error::AdapterError; - - pub(crate) fn do_verified_press(el: &AXElement) -> Result<bool, AdapterError> { - dispatch_verified_press(el, crate::actions::chain_web_steps::is_in_webarea(el)) - } - - fn dispatch_verified_press(el: &AXElement, in_web: bool) -> Result<bool, AdapterError> { - if !in_web { - return verified_press_native(el); - } - tracing::debug!("verified_press: web element detected"); - Ok(crate::actions::chain_web_steps::activate_web_element(el)) - } - - fn verified_press_native(el: &AXElement) -> Result<bool, AdapterError> { - use accessibility_sys::kAXRoleAttribute; - let parent = crate::tree::copy_element_attr(el, "AXParent"); - let in_container = parent.as_ref().is_some_and(|p| { - matches!( - crate::tree::copy_string_attr(p, kAXRoleAttribute).as_deref(), - Some("AXOutline" | "AXList" | "AXTable") - ) - }); - if !in_container { - return ax_helpers::try_ax_action_retried_or_err(el, "AXPress"); - } - tracing::debug!("verified_press: native element in container, using AXPress"); - let selected_before = crate::tree::copy_bool_attr(el, "AXSelected"); - if !ax_helpers::try_ax_action_retried_or_err(el, "AXPress")? { - return Ok(false); - } - if selected_before == Some(true) { - return Ok(true); - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let selected_after = crate::tree::copy_bool_attr(el, "AXSelected"); - if selected_after == Some(true) { - return Ok(true); - } - if crate::tree::copy_string_attr(el, kAXRoleAttribute).is_none() { - return Ok(true); - } - tracing::debug!("verified_press: AXPress ok but no state change"); - Ok(false) - } - - pub(crate) fn try_value_relay(el: &AXElement) -> Result<bool, AdapterError> { - if !ax_helpers::list_ax_actions(el).is_empty() { - return Ok(false); - } - let win = crate::tree::copy_element_attr(el, "AXWindow"); - let is_dialog = win.as_ref().is_some_and(|w| { - crate::tree::copy_string_attr(w, "AXSubrole").as_deref() == Some("AXDialog") - }); - if !is_dialog { - return Ok(false); - } - let label = std::cell::RefCell::new(None::<String>); - ax_helpers::try_each_child( - el, - |child| { - let d = crate::tree::copy_string_attr(child, "AXDescription").unwrap_or_default(); - if d.is_empty() { - return false; - } - *label.borrow_mut() = Some(d.split(',').next().unwrap_or(&d).trim().to_owned()); - true - }, - 5, - ); - let Some(label) = label.into_inner() else { - return Ok(false); - }; - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return Ok(false); - }; - let app = crate::tree::element_for_pid(pid); - let Some(owner) = crate::tree::copy_element_attr(&app, "AXFocusedUIElement") else { - return Ok(false); - }; - if !same_window(&owner, win.as_ref()) { - return Ok(false); - } - if !ax_helpers::is_attr_settable(&owner, "AXValue") { - return Ok(false); - } - let orig = crate::tree::copy_string_attr(&owner, "AXValue"); - ax_helpers::set_ax_string_or_err(&owner, "AXValue", &label)?; - std::thread::sleep(std::time::Duration::from_millis(150)); - if !ax_helpers::try_ax_action_retried_or_err(&owner, "AXConfirm")? { - if let Some(o) = &orig { - if let Err(e) = ax_helpers::set_ax_string_or_err(&owner, "AXValue", o) { - tracing::warn!(error = %e, "value_relay: rollback write failed; control may have corrupted value"); - } - } - return Ok(false); - } - std::thread::sleep(std::time::Duration::from_millis(150)); - let final_val = crate::tree::copy_string_attr(&owner, "AXValue"); - if final_val.as_deref() != Some(label.as_str()) { - tracing::debug!( - observed_chars = final_val.as_deref().map(|value| value.chars().count()), - expected_chars = label.chars().count(), - "value_relay: value reverted" - ); - } - Ok(final_val.as_deref() == Some(label.as_str())) - } - - fn same_window(owner: &AXElement, expected_window: Option<&AXElement>) -> bool { - let Some(expected_window) = expected_window else { - return false; - }; - let Some(owner_window) = crate::tree::copy_element_attr(owner, "AXWindow") else { - return false; - }; - crate::tree::same_element(&owner_window, expected_window) - } - - pub(crate) fn element_is_visible_in_scroll_context( - el: &AXElement, - ) -> Result<bool, AdapterError> { - use accessibility_sys::kAXRoleAttribute; - let Some(bounds) = crate::tree::read_bounds(el) else { - return Ok(false); - }; - if !rect_has_area(&bounds) { - return Ok(false); - } - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..8 { - let Some(parent) = ¤t else { - return Ok(true); - }; - if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref() - == Some("AXScrollArea") - { - let Some(pb) = crate::tree::read_bounds(parent) else { - return Ok(false); - }; - return Ok(rect_has_area(&pb) && center_is_inside(&bounds, &pb)); - } - current = crate::tree::copy_element_attr(parent, "AXParent"); - } - Ok(true) - } - - pub(crate) fn rect_has_area(rect: &agent_desktop_core::node::Rect) -> bool { - rect.width > 0.0 && rect.height > 0.0 - } - - pub(crate) fn center_is_inside( - inner: &agent_desktop_core::node::Rect, - outer: &agent_desktop_core::node::Rect, - ) -> bool { - let x = inner.x + inner.width / 2.0; - let y = inner.y + inner.height / 2.0; - x >= outer.x && x <= outer.x + outer.width && y >= outer.y && y <= outer.y + outer.height - } - - pub(crate) fn try_show_alternate_ui(el: &AXElement) -> Result<bool, AdapterError> { - if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") { - return Ok(false); - } - ax_helpers::try_ax_action_retried_or_err(el, "AXShowAlternateUI")?; - std::thread::sleep(std::time::Duration::from_millis(100)); - Ok(ax_helpers::try_each_child( - el, - |child| { - let ca = ax_helpers::list_ax_actions(child); - ax_helpers::try_action_from_list(child, &ca, &["AXPress"]) - }, - 5, - )) - } - - pub(crate) fn try_parent_row_select(el: &AXElement) -> Result<bool, AdapterError> { - use accessibility_sys::kAXRoleAttribute; - let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else { - return Ok(false); - }; - let role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute).unwrap_or_default(); - if !matches!(role.as_str(), "AXRow" | "AXOutlineRow") { - return Ok(false); - } - if !ax_helpers::is_attr_settable(&parent, "AXSelected") { - return Ok(false); - } - ax_helpers::set_ax_bool_or_err(&parent, "AXSelected", true) - } - - pub(crate) fn try_select_containing_item(el: &AXElement) -> Result<bool, AdapterError> { - let mut current = Some(el.clone()); - for _ in 0..4 { - let Some(candidate) = current else { - return Ok(false); - }; - if select_candidate(&candidate)? || select_candidate_in_container(&candidate) { - return Ok(true); - } - current = crate::tree::copy_element_attr(&candidate, "AXParent"); - } - Ok(false) - } - - fn select_candidate(candidate: &AXElement) -> Result<bool, AdapterError> { - Ok(ax_helpers::is_attr_settable(candidate, "AXSelected") - && ax_helpers::set_ax_bool_or_err(candidate, "AXSelected", true)? - && selected_state_settled(candidate)) - } - - fn selected_state_settled(candidate: &AXElement) -> bool { - std::thread::sleep(std::time::Duration::from_millis(50)); - crate::tree::copy_bool_attr(candidate, "AXSelected") == Some(true) - } - - fn select_candidate_in_container(candidate: &AXElement) -> bool { - for attr in ["AXSelectedChildren", "AXSelectedRows"] { - if set_container_selection(candidate, attr) - && container_selection_contains(candidate, attr) - { - return true; - } - } - false - } - - fn set_container_selection(candidate: &AXElement, attr: &str) -> bool { - let Some(container) = crate::tree::copy_element_attr(candidate, "AXParent") else { - return false; - }; - if !ax_helpers::is_attr_settable(&container, attr) { - return false; - } - set_single_element_selection(&container, candidate, attr) - } - - fn container_selection_contains(candidate: &AXElement, attr: &str) -> bool { - std::thread::sleep(std::time::Duration::from_millis(50)); - let Some(container) = crate::tree::copy_element_attr(candidate, "AXParent") else { - return false; - }; - crate::tree::copy_ax_array(&container, attr) - .unwrap_or_default() - .iter() - .any(|selected| crate::tree::same_element(selected, candidate)) - } - - pub(crate) fn try_select_via_parent(el: &AXElement) -> Result<bool, AdapterError> { - use accessibility_sys::kAXRoleAttribute; - let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else { - return Ok(false); - }; - let Some(role) = crate::tree::copy_string_attr(&parent, kAXRoleAttribute) else { - return Ok(false); - }; - if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") { - return Ok(false); - } - if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") { - return Ok(false); - } - Ok(set_single_element_selection(&parent, el, "AXSelectedRows")) - } - - fn set_single_element_selection( - container: &AXElement, - element: &AXElement, - attr: &str, - ) -> bool { - use accessibility_sys::{AXUIElementSetAttributeValue, kAXErrorSuccess}; - use core_foundation::{ - array::CFArray, - base::{CFRetain, CFType, CFTypeRef, TCFType}, - string::CFString, - }; - unsafe { CFRetain(element.0 as CFTypeRef) }; - let element_cf = unsafe { CFType::wrap_under_create_rule(element.0 as CFTypeRef) }; - let selected = CFArray::from_CFTypes(&[element_cf]); - let cf_attr = CFString::new(attr); - let err = unsafe { - AXUIElementSetAttributeValue( - container.0, - cf_attr.as_concrete_TypeRef(), - selected.as_CFTypeRef(), - ) - }; - err == kAXErrorSuccess - } - - pub(crate) fn try_custom_actions(el: &AXElement) -> Result<bool, AdapterError> { - let has = !crate::tree::copy_ax_array(el, "AXCustomActions") - .unwrap_or_default() - .is_empty(); - Ok(has && ax_helpers::try_ax_action_retried_or_err(el, "AXPerformCustomAction")?) - } -} - -#[cfg(target_os = "macos")] -pub(crate) use imp::{ - do_verified_press, element_is_visible_in_scroll_context, try_custom_actions, - try_parent_row_select, try_select_containing_item, try_select_via_parent, - try_show_alternate_ui, try_value_relay, -}; - -#[cfg(all(test, target_os = "macos"))] -pub(crate) use imp::{center_is_inside, rect_has_area}; diff --git a/crates/macos/src/actions/chain_steps_tests.rs b/crates/macos/src/actions/chain_steps_tests.rs deleted file mode 100644 index d66521a..0000000 --- a/crates/macos/src/actions/chain_steps_tests.rs +++ /dev/null @@ -1,47 +0,0 @@ -use agent_desktop_core::node::Rect; - -use super::chain_steps::{center_is_inside, rect_has_area}; - -#[test] -fn rect_area_requires_positive_dimensions() { - assert!(rect_has_area(&Rect { - x: 0.0, - y: 0.0, - width: 1.0, - height: 1.0, - })); - assert!(!rect_has_area(&Rect { - x: 0.0, - y: 0.0, - width: 0.0, - height: 1.0, - })); -} - -#[test] -fn center_visibility_uses_parent_bounds() { - let outer = Rect { - x: 10.0, - y: 10.0, - width: 100.0, - height: 100.0, - }; - assert!(center_is_inside( - &Rect { - x: 20.0, - y: 20.0, - width: 10.0, - height: 10.0, - }, - &outer - )); - assert!(!center_is_inside( - &Rect { - x: 200.0, - y: 20.0, - width: 10.0, - height: 10.0, - }, - &outer - )); -} diff --git a/crates/macos/src/actions/chain_tests.rs b/crates/macos/src/actions/chain_tests.rs new file mode 100644 index 0000000..705f66d --- /dev/null +++ b/crates/macos/src/actions/chain_tests.rs @@ -0,0 +1,243 @@ +use super::{ + ChainStep, build_step, exhaustion_disposition, record_step_outcome, step_allowed, + step_mechanism, +}; +use crate::actions::chain_delivery::DeliveryOutcome; +use agent_desktop_core::MouseButton; +use agent_desktop_core::step_mechanism::StepMechanism; + +#[test] +fn right_click_prefers_physical_input_before_semantic_fallbacks() { + let labels: Vec<&str> = crate::actions::chain_defs::RIGHT_CLICK_CHAIN + .steps + .iter() + .map(|step| match step { + ChainStep::CustomWithDeadline { label, .. } => *label, + ChainStep::CGClick { .. } => "CGClick", + ChainStep::Action(name) => name, + _ => "other", + }) + .collect(); + + assert_eq!( + labels, + [ + "CGClick", + "show_menu", + "select_then_show_menu", + "selected_items_menu", + "child_show_menu", + "ancestor_show_menu", + ] + ); +} + +#[test] +fn click_and_clear_prefer_physical_delivery_when_policy_allows_it() { + assert!(matches!( + crate::actions::chain_defs::CLICK_CHAIN.steps.first(), + Some(ChainStep::CGClick { + button: MouseButton::Left, + count: 1 + }) + )); + assert!(matches!( + crate::actions::chain_defs::CLEAR_CHAIN.steps.first(), + Some(ChainStep::FocusThenClearByKeyboard) + )); +} + +#[test] +fn step_mechanism_tags_physical_for_cgclick_and_keyboard_clear() { + assert_eq!( + step_mechanism(&ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }), + StepMechanism::PhysicalSynthetic + ); + assert_eq!( + step_mechanism(&ChainStep::FocusThenClearByKeyboard), + StepMechanism::PhysicalSynthetic + ); + assert_eq!( + step_mechanism(&ChainStep::Action("AXPress")), + StepMechanism::SemanticApi + ); +} + +#[test] +fn headless_chains_omit_physical_steps_before_reporting() { + let headless = agent_desktop_core::InteractionPolicy::headless(); + let headed = agent_desktop_core::InteractionPolicy::headed(); + let physical = ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }; + + assert!(!step_allowed(&physical, headless)); + assert!(step_allowed(&physical, headed)); + assert!(step_allowed(&ChainStep::Action("AXPress"), headless)); +} + +#[test] +fn build_step_tags_mechanism_and_verified_on_success() { + let step = ChainStep::SetBool { + attr: "AXSelected", + value: true, + }; + let built = build_step(&step, DeliveryOutcome::DeliveredVerified); + assert_eq!(built.mechanism(), Some(StepMechanism::SemanticApi)); + assert_eq!(built.verified(), Some(true)); +} + +#[test] +fn build_step_skipped_does_not_tag_verified() { + let step = ChainStep::SetBool { + attr: "AXSelected", + value: true, + }; + let built = build_step(&step, DeliveryOutcome::NotDelivered); + assert_eq!(built.mechanism(), Some(StepMechanism::SemanticApi)); + assert!(built.verified().is_none()); +} + +#[test] +fn satisfied_without_delivery_stops_fallback_and_is_skipped_verified() { + let step = ChainStep::Action("AlreadySatisfied"); + let mut steps = Vec::new(); + + assert!(record_step_outcome( + &mut steps, + &step, + DeliveryOutcome::SatisfiedNoDelivery, + false, + )); + assert!(matches!( + steps[0].outcome, + agent_desktop_core::ActionStepOutcome::Skipped + )); + assert_eq!(steps[0].verified(), Some(true)); +} + +#[test] +fn build_step_marks_delivered_unverified_explicitly() { + let built = build_step( + &ChainStep::Action("AXPress"), + DeliveryOutcome::DeliveredUnverified, + ); + + assert_eq!(built.mechanism(), Some(StepMechanism::SemanticApi)); + assert_eq!(built.verified(), Some(false)); +} + +#[test] +fn native_list_press_success_stops_click_chain_before_other_mutations() { + let rungs = [ + ( + ChainStep::Action("AXPress"), + DeliveryOutcome::DeliveredUnverified, + ), + ( + ChainStep::Action("AXConfirm"), + DeliveryOutcome::DeliveredVerified, + ), + ( + ChainStep::Action("AXOpen"), + DeliveryOutcome::DeliveredVerified, + ), + ( + ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }, + DeliveryOutcome::DeliveredUnverified, + ), + ]; + let mut calls = Vec::new(); + let mut steps = Vec::new(); + + for (step, outcome) in &rungs { + calls.push(match step { + ChainStep::Action(name) => *name, + ChainStep::CGClick { .. } => "CGClick", + _ => "other", + }); + if record_step_outcome(&mut steps, step, *outcome, false) { + break; + } + } + + assert_eq!(calls, ["AXPress"]); + assert_eq!(steps.len(), 1); + assert_eq!(steps[0].verified(), Some(false)); +} + +#[test] +fn idempotent_chain_continues_after_unverified_delivery() { + let step = ChainStep::SetDynamic { attr: "AXValue" }; + let mut steps = Vec::new(); + + assert!(!record_step_outcome( + &mut steps, + &step, + DeliveryOutcome::DeliveredUnverified, + true, + )); + assert_eq!(steps[0].verified(), Some(false)); +} + +#[test] +fn non_idempotent_chain_stops_after_unverified_delivery() { + let step = ChainStep::Action("AXPress"); + let mut steps = Vec::new(); + + assert!(record_step_outcome( + &mut steps, + &step, + DeliveryOutcome::DeliveredUnverified, + false, + )); +} + +#[test] +fn exhaustion_after_unverified_delivery_reports_delivered_unverified() { + let mut steps = Vec::new(); + assert!(!record_step_outcome( + &mut steps, + &ChainStep::FocusThenClearByKeyboard, + DeliveryOutcome::DeliveredUnverified, + true, + )); + assert!(!record_step_outcome( + &mut steps, + &ChainStep::SetDynamic { attr: "AXValue" }, + DeliveryOutcome::NotDelivered, + true, + )); + + assert_eq!( + exhaustion_disposition(&steps), + agent_desktop_core::DeliverySemantics::delivered_unverified() + ); +} + +#[test] +fn exhaustion_without_any_delivery_reports_not_delivered() { + let mut steps = Vec::new(); + assert!(!record_step_outcome( + &mut steps, + &ChainStep::Action("AXPress"), + DeliveryOutcome::NotDelivered, + false, + )); + + assert_eq!( + exhaustion_disposition(&steps), + agent_desktop_core::DeliverySemantics::not_delivered() + ); + assert_eq!( + exhaustion_disposition(&[]), + agent_desktop_core::DeliverySemantics::not_delivered() + ); +} diff --git a/crates/macos/src/actions/chain_value_write.rs b/crates/macos/src/actions/chain_value_write.rs new file mode 100644 index 0000000..60a3885 --- /dev/null +++ b/crates/macos/src/actions/chain_value_write.rs @@ -0,0 +1,153 @@ +#[cfg(target_os = "macos")] +mod imp { + use agent_desktop_core::{AdapterError, Deadline}; + + use crate::actions::{ax_helpers, chain_delivery::DeliveryOutcome, chain_verify}; + use crate::tree::AXElement; + + pub(crate) fn set_dynamic_verified( + element: &AXElement, + attribute: &str, + value: &str, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + prepare(element, deadline)?; + if attribute == "AXValue" { + ax_helpers::set_ax_value_coerced(element, value, deadline)?; + } else { + ax_helpers::set_ax_string_or_err(element, attribute, value, deadline)?; + } + let mut delivery = crate::actions::DeliveryTracker::default(); + delivery.mark_delivered(); + prepare(element, deadline).map_err(|error| delivery.annotate(error))?; + let role = ax_helpers::element_role(element, deadline) + .map_err(|error| delivery.annotate(error))?; + prepare(element, deadline).map_err(|error| delivery.annotate(error))?; + let observed = crate::tree::copy_value_typed(element, deadline); + Ok(DeliveryOutcome::from_delivery( + true, + chain_verify::dynamic_write_had_effect( + attribute, + role.as_deref(), + value, + observed.as_deref(), + ), + )) + } + + pub(crate) fn increment_to_value( + element: &AXElement, + target: &str, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + const MAX_INCREMENT_STEPS: usize = 1_024; + + let Some(target) = finite_target(target) else { + return Ok(DeliveryOutcome::NotDelivered); + }; + let Some(mut current) = read_number(element, deadline)? else { + return Ok(DeliveryOutcome::NotDelivered); + }; + let start = current; + let mut delivered = false; + let mut delivery = crate::actions::DeliveryTracker::default(); + for _ in 0..MAX_INCREMENT_STEPS { + if (current - target).abs() < 0.5 { + return Ok(if delivered { + DeliveryOutcome::DeliveredVerified + } else { + DeliveryOutcome::SatisfiedNoDelivery + }); + } + if deadline.is_expired() { + return Err(chain_verify::increment_deadline_error( + start, current, target, + )); + } + let action = if current < target { + "AXIncrement" + } else { + "AXDecrement" + }; + prepare(element, deadline).map_err(|error| delivery.annotate(error))?; + let delivered_step = match ax_helpers::try_ax_action_or_err(element, action, deadline) { + Ok(delivered) => delivered, + Err(error) if delivered => return Err(delivery.annotate(error)), + Err(error) => return Err(error), + }; + if !delivered_step { + break; + } + delivered = true; + delivery.mark_delivered(); + match read_number(element, deadline).map_err(|error| delivery.annotate(error))? { + Some(next) if (next - current).abs() >= f64::EPSILON => current = next, + _ => break, + } + } + if (current - target).abs() < 0.5 { + return Ok(DeliveryOutcome::DeliveredVerified); + } + if (current - start).abs() >= f64::EPSILON { + return Err(chain_verify::increment_step_limit_error( + start, current, target, + )); + } + Ok(DeliveryOutcome::from_delivery(delivered, false)) + } + + pub(crate) fn set_bool_verified( + element: &AXElement, + attribute: &str, + value: bool, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + prepare(element, deadline)?; + let delivered = ax_helpers::set_ax_bool_or_err(element, attribute, value, deadline)?; + if !delivered { + return Ok(DeliveryOutcome::NotDelivered); + } + let mut delivery = crate::actions::DeliveryTracker::default(); + delivery.mark_delivered(); + prepare(element, deadline).map_err(|error| delivery.annotate(error))?; + let observed = crate::tree::copy_bool_attr(element, attribute, deadline); + Ok(DeliveryOutcome::from_delivery( + delivered, + delivered && chain_verify::bool_write_had_effect(attribute, value, observed), + )) + } + + pub(crate) fn finite_target(target: &str) -> Option<f64> { + target.parse::<f64>().ok().filter(|value| value.is_finite()) + } + + fn read_number(element: &AXElement, deadline: Deadline) -> Result<Option<f64>, AdapterError> { + prepare(element, deadline)?; + Ok(crate::tree::copy_value_typed(element, deadline) + .and_then(|value| value.parse::<f64>().ok())) + } + + fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) + } + + #[cfg(test)] + pub(crate) fn verification_failure_after_write(error: AdapterError) -> AdapterError { + let mut delivery = crate::actions::DeliveryTracker::default(); + delivery.mark_delivered(); + delivery.annotate(error) + } +} + +#[cfg(target_os = "macos")] +pub(crate) use imp::{increment_to_value, set_bool_verified, set_dynamic_verified}; + +#[cfg(all(test, target_os = "macos"))] +use imp::finite_target; + +#[cfg(all(test, target_os = "macos"))] +use imp::verification_failure_after_write; + +#[cfg(test)] +#[path = "chain_value_write_tests.rs"] +mod tests; diff --git a/crates/macos/src/actions/chain_value_write_tests.rs b/crates/macos/src/actions/chain_value_write_tests.rs new file mode 100644 index 0000000..27bf430 --- /dev/null +++ b/crates/macos/src/actions/chain_value_write_tests.rs @@ -0,0 +1,22 @@ +use super::finite_target; + +#[test] +fn finite_target_rejects_non_finite_numbers() { + assert_eq!(finite_target("42.5"), Some(42.5)); + assert_eq!(finite_target("NaN"), None); + assert_eq!(finite_target("inf"), None); + assert_eq!(finite_target("-inf"), None); + assert_eq!(finite_target("not-a-number"), None); +} + +#[test] +fn verification_failure_after_first_write_is_unsafe_to_retry() { + let error = super::verification_failure_after_write(agent_desktop_core::AdapterError::timeout( + "verification fixture", + )); + + assert_eq!( + error.disposition, + agent_desktop_core::DeliverySemantics::delivered_unverified() + ); +} diff --git a/crates/macos/src/actions/chain_verify.rs b/crates/macos/src/actions/chain_verify.rs index 336d10d..c8d8069 100644 --- a/crates/macos/src/actions/chain_verify.rs +++ b/crates/macos/src/actions/chain_verify.rs @@ -1,12 +1,8 @@ -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::AdapterError; -/// Error for a chain deadline expiring mid-increment. Unlike a plain step -/// "skip", expiry can leave the control at a half-applied value, so the -/// error must be TIMEOUT (not ACTION_FAILED) and must carry the observed -/// state — the caller cannot read post-state on the error path. `kind` -/// discriminates this details schema from other TIMEOUT payloads. pub(crate) fn increment_deadline_error(start: f64, current: f64, target: f64) -> AdapterError { - AdapterError::timeout("Chain deadline expired while stepping the value toward the target") + partial_mutation_disposition( + AdapterError::timeout("Chain deadline expired while stepping the value toward the target") .with_suggestion( "Re-read the element value before retrying; increase the timeout or AGENT_DESKTOP_CHAIN_TIMEOUT_MS for slow controls.", ) @@ -16,12 +12,15 @@ pub(crate) fn increment_deadline_error(start: f64, current: f64, target: f64) -> "value_at_timeout": current, "target": target, "mutated": (current - start).abs() >= f64::EPSILON, - })) + })), + start, + current, + ) } pub(crate) fn increment_step_limit_error(start: f64, current: f64, target: f64) -> AdapterError { - AdapterError::new( - agent_desktop_core::error::ErrorCode::ActionFailed, + partial_mutation_disposition(AdapterError::new( + agent_desktop_core::ErrorCode::ActionFailed, "Chain step limit was reached while stepping the value toward the target", ) .with_suggestion( @@ -33,26 +32,16 @@ pub(crate) fn increment_step_limit_error(start: f64, current: f64, target: f64) "value_at_limit": current, "target": target, "mutated": (current - start).abs() >= f64::EPSILON, - })) + })), start, current) } -/// Error for the chain deadline truncating a disclosure settle wait. The -/// triggering action may still land after the truncated wait, so the -/// outcome is unknown — TIMEOUT with the observed state, mirroring -/// [`increment_deadline_error`], never a plain step failure. -pub(crate) fn disclosure_deadline_error( - want_expanded: bool, - observed: Option<bool>, -) -> AdapterError { - AdapterError::timeout("Chain deadline expired while waiting for the disclosure to settle") - .with_suggestion( - "Re-read the element's expanded state before retrying; increase the timeout or AGENT_DESKTOP_CHAIN_TIMEOUT_MS for slow apps.", - ) - .with_details(serde_json::json!({ - "kind": "chain_deadline", - "wanted_expanded": want_expanded, - "observed_expanded": observed, - })) +fn partial_mutation_disposition(error: AdapterError, start: f64, current: f64) -> AdapterError { + let disposition = if (current - start).abs() >= f64::EPSILON { + agent_desktop_core::DeliverySemantics::delivered_unverified() + } else { + agent_desktop_core::DeliverySemantics::not_delivered() + }; + error.with_disposition(disposition) } pub(crate) fn bool_write_had_effect(attr: &str, expected: bool, observed: Option<bool>) -> bool { @@ -74,9 +63,6 @@ pub(crate) fn dynamic_write_had_effect( observed == Some(expected) || numbers_match(expected, observed) } -/// Numeric controls report their value back in their own format (a slider -/// set to `50` reads back as `50.00`), so compare numerically when both -/// sides parse as numbers. fn numbers_match(expected: &str, observed: Option<&str>) -> bool { match ( expected.parse::<f64>(), @@ -101,7 +87,11 @@ mod tests { fn increment_deadline_error_is_timeout_and_reports_partial_mutation() { let err = increment_deadline_error(0.0, 37.0, 80.0); - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::Timeout); + assert_eq!(err.code, agent_desktop_core::ErrorCode::Timeout); + assert_eq!( + err.disposition, + agent_desktop_core::DeliverySemantics::delivered_unverified() + ); let details = err.details.expect("details must carry the observed state"); assert_eq!(details["value_before"], 0.0); assert_eq!(details["value_at_timeout"], 37.0); @@ -114,6 +104,10 @@ mod tests { fn increment_deadline_error_reports_unmutated_state() { let err = increment_deadline_error(5.0, 5.0, 9.0); + assert_eq!( + err.disposition, + agent_desktop_core::DeliverySemantics::not_delivered() + ); let details = err.details.unwrap(); assert_eq!(details["mutated"], false); assert_eq!(details["kind"], "chain_deadline"); @@ -123,32 +117,17 @@ mod tests { fn increment_step_limit_error_reports_partial_mutation() { let err = increment_step_limit_error(0.0, 1024.0, 5000.0); - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::ActionFailed); + assert_eq!(err.code, agent_desktop_core::ErrorCode::ActionFailed); + assert_eq!( + err.disposition, + agent_desktop_core::DeliverySemantics::delivered_unverified() + ); let details = err.details.unwrap(); assert_eq!(details["kind"], "chain_step_limit"); assert_eq!(details["value_at_limit"], 1024.0); assert_eq!(details["mutated"], true); } - #[test] - fn disclosure_deadline_error_is_timeout_with_observed_state() { - let err = super::disclosure_deadline_error(true, Some(false)); - - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::Timeout); - let details = err.details.expect("details must carry the observed state"); - assert_eq!(details["kind"], "chain_deadline"); - assert_eq!(details["wanted_expanded"], true); - assert_eq!(details["observed_expanded"], false); - assert!(err.suggestion.is_some()); - } - - #[test] - fn disclosure_deadline_error_reports_unreadable_state_as_null() { - let err = super::disclosure_deadline_error(false, None); - - assert!(err.details.unwrap()["observed_expanded"].is_null()); - } - #[test] fn ax_value_write_requires_readback_match() { assert!(!dynamic_write_had_effect( diff --git a/crates/macos/src/actions/chain_web_steps.rs b/crates/macos/src/actions/chain_web_steps.rs deleted file mode 100644 index 9775c6d..0000000 --- a/crates/macos/src/actions/chain_web_steps.rs +++ /dev/null @@ -1,118 +0,0 @@ -#[cfg(target_os = "macos")] -mod imp { - use crate::actions::ax_helpers; - use crate::tree::AXElement; - - struct PreActionState { - focused: Option<AXElement>, - value: Option<String>, - selected: Option<bool>, - } - - impl PreActionState { - fn capture(app: &AXElement, el: &AXElement) -> Self { - Self { - focused: crate::tree::copy_element_attr(app, "AXFocusedUIElement"), - value: crate::tree::copy_string_attr(el, "AXValue"), - selected: crate::tree::copy_bool_attr(el, "AXSelected"), - } - } - } - - pub(crate) fn is_in_webarea(el: &AXElement) -> bool { - use accessibility_sys::kAXRoleAttribute; - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..20 { - let Some(ref parent) = current else { - return false; - }; - if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref() - == Some("AXWebArea") - { - return true; - } - current = crate::tree::copy_element_attr(parent, "AXParent"); - } - false - } - - pub(crate) fn activate_web_element(el: &AXElement) -> bool { - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return false; - }; - - let app = crate::tree::element_for_pid(pid); - let before = PreActionState::capture(&app, el); - - if ax_helpers::try_ax_action_retried(el, "AXPress") { - std::thread::sleep(std::time::Duration::from_millis(100)); - if web_action_had_effect(&app, el, &before) { - tracing::debug!("activate_web: AXPress had real effect"); - return true; - } - tracing::debug!("activate_web: AXPress returned success but no DOM effect"); - } - - let actions = ax_helpers::list_ax_actions(el); - for action in &["AXConfirm", "AXOpen"] { - if actions.iter().any(|a| a == action) && ax_helpers::try_ax_action(el, action) { - std::thread::sleep(std::time::Duration::from_millis(100)); - if web_action_had_effect(&app, el, &before) { - tracing::debug!("activate_web: {action} had real effect"); - return true; - } - } - } - - if ax_helpers::try_each_child( - el, - |child| { - let child_actions = ax_helpers::list_ax_actions(child); - ax_helpers::try_action_from_list( - child, - &child_actions, - &["AXPress", "AXConfirm", "AXOpen"], - ) - }, - 5, - ) { - std::thread::sleep(std::time::Duration::from_millis(100)); - if web_action_had_effect(&app, el, &before) { - tracing::debug!("activate_web: child action had real effect"); - return true; - } - } - - tracing::debug!("activate_web: all AX methods had no effect"); - false - } - - fn web_action_had_effect(app: &AXElement, el: &AXElement, before: &PreActionState) -> bool { - use core_foundation::base::{CFEqual, CFTypeRef}; - - let value_after = crate::tree::copy_string_attr(el, "AXValue"); - if before.value != value_after { - return true; - } - - let selected_after = crate::tree::copy_bool_attr(el, "AXSelected"); - if before.selected != selected_after { - return true; - } - - let focused_after = crate::tree::copy_element_attr(app, "AXFocusedUIElement"); - match (&before.focused, &focused_after) { - (Some(before_f), Some(after_f)) => unsafe { - CFEqual(before_f.0 as CFTypeRef, after_f.0 as CFTypeRef) == 0 - }, - (None, Some(_)) => true, - _ => false, - } - } -} - -#[cfg(not(target_os = "macos"))] -mod imp {} - -#[cfg(target_os = "macos")] -pub(crate) use imp::{activate_web_element, is_in_webarea}; diff --git a/crates/macos/src/actions/delivery_tracker.rs b/crates/macos/src/actions/delivery_tracker.rs new file mode 100644 index 0000000..7112717 --- /dev/null +++ b/crates/macos/src/actions/delivery_tracker.rs @@ -0,0 +1,74 @@ +use agent_desktop_core::{AdapterError, DeliverySemantics}; + +#[derive(Clone, Copy, Default)] +pub(crate) struct DeliveryTracker { + delivered_units: usize, +} + +impl DeliveryTracker { + pub(crate) fn from_delivered_units(delivered_units: usize) -> Self { + Self { delivered_units } + } + + pub(crate) fn mark_delivered(&mut self) { + self.delivered_units = self.delivered_units.saturating_add(1); + } + + pub(crate) fn delivered_units(self) -> usize { + self.delivered_units + } + + pub(crate) fn annotate(self, error: AdapterError) -> AdapterError { + let disposition = match error.disposition { + DeliverySemantics::DeliveryUncertain + | DeliverySemantics::DeliveredUnverified + | DeliverySemantics::DeliveredVerified => error.disposition, + DeliverySemantics::Unknown | DeliverySemantics::NotDelivered => { + if self.delivered_units == 0 { + DeliverySemantics::not_delivered() + } else { + DeliverySemantics::delivered_unverified() + } + } + }; + error.with_disposition(disposition) + } + + pub(crate) fn uncertain(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::uncertain()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn delivery_tracker_changes_retry_semantics_after_first_post() { + let before = DeliveryTracker::default().annotate(AdapterError::internal("before")); + let mut tracker = DeliveryTracker::default(); + tracker.mark_delivered(); + let after = tracker.annotate(AdapterError::internal("after")); + + assert_eq!(before.disposition, DeliverySemantics::not_delivered()); + assert_eq!(after.disposition, DeliverySemantics::delivered_unverified()); + } + + #[test] + fn nested_partial_delivery_is_never_downgraded_by_an_outer_tracker() { + for disposition in [ + DeliverySemantics::uncertain(), + DeliverySemantics::delivered_unverified(), + DeliverySemantics::delivered_verified(), + ] { + let inner = AdapterError::internal("partial key pair").with_disposition(disposition); + let annotated = DeliveryTracker::default().annotate(inner); + + assert_eq!(annotated.disposition, disposition); + assert_eq!( + annotated.disposition.retry(), + agent_desktop_core::RetryDisposition::Unsafe + ); + } + } +} diff --git a/crates/macos/src/actions/discovery.rs b/crates/macos/src/actions/discovery.rs deleted file mode 100644 index 534a766..0000000 --- a/crates/macos/src/actions/discovery.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::tree::AXElement; - -pub(crate) struct ElementCaps { - pub settable_focus: bool, - pub settable_selected: bool, - pub settable_disclosing: bool, -} - -#[cfg(target_os = "macos")] -pub(crate) fn discover(el: &AXElement) -> ElementCaps { - use crate::actions::ax_helpers; - ElementCaps { - settable_focus: ax_helpers::is_attr_settable(el, "AXFocused"), - settable_selected: ax_helpers::is_attr_settable(el, "AXSelected"), - settable_disclosing: ax_helpers::is_attr_settable(el, "AXDisclosing"), - } -} - -#[cfg(not(target_os = "macos"))] -pub fn discover(_el: &AXElement) -> ElementCaps { - ElementCaps { - settable_focus: false, - settable_selected: false, - settable_disclosing: false, - } -} diff --git a/crates/macos/src/actions/dispatch.rs b/crates/macos/src/actions/dispatch.rs index 9c8e8e4..58d835b 100644 --- a/crates/macos/src/actions/dispatch.rs +++ b/crates/macos/src/actions/dispatch.rs @@ -1,230 +1,183 @@ use agent_desktop_core::{ - action::{Action, MouseButton, MouseEvent, MouseEventKind, Point}, - action_request::ActionRequest, - action_result::ActionResult, - element_state::ElementState, - error::{AdapterError, ErrorCode}, - interaction_policy::InteractionPolicy, + Action, ActionResult, ActionStep, AdapterError, Deadline, ErrorCode, StepMechanism, + action_request::ActionRequest, action_step_outcome::ActionStepOutcome, }; #[cfg(target_os = "macos")] mod imp { use super::*; use crate::actions::{ - ax_helpers, - chain::{ChainContext, execute_chain}, - chain_defs, discovery, toggle_state, + chain::{ChainContext, ChainDef, execute_chain}, + chain_defs, toggle_state, }; use crate::tree::AXElement; - pub(crate) fn click_via_bounds( - el: &AXElement, - button: MouseButton, - count: u32, - policy: InteractionPolicy, - ) -> Result<(), AdapterError> { - if !policy.allow_cursor_move || !policy.allow_focus_steal { - return Err(AdapterError::policy_denied_for_policy( - "Physical click fallback is disabled by the current interaction policy", - policy, - )); - } - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - crate::system::app_ops::focus_best_effort(pid); - } - if let Some(window) = crate::tree::copy_element_attr(el, "AXWindow") { - crate::system::window_ops::raise_window(&window); - } - let bounds = crate::tree::read_bounds(el).ok_or_else(|| { - AdapterError::new(ErrorCode::ActionFailed, "Element has no readable bounds") - .with_suggestion("AX action failed and CGEvent fallback unavailable") - })?; - if bounds.width <= 0.0 || bounds.height <= 0.0 { - return Err( - AdapterError::new(ErrorCode::ActionFailed, "Element has zero-size bounds") - .with_suggestion("Element may be hidden or off-screen. Try 'scroll-to' first."), - ); - } - let center = Point { - x: bounds.x + bounds.width / 2.0, - y: bounds.y + bounds.height / 2.0, - }; - tracing::debug!( - ?button, - count, - x = center.x, - y = center.y, - "AX action failed, falling back to CGEvent click" - ); - crate::input::mouse::synthesize_mouse(MouseEvent { - kind: MouseEventKind::Click { count }, - point: center, - button, - }) - } - pub(crate) fn perform_action( el: &AXElement, request: &ActionRequest, + deadline: Deadline, ) -> Result<ActionResult, AdapterError> { + let budget = ChainContext { + dynamic_value: None, + verified_point: request.verified_point(), + deadline, + }; + crate::tree::attributes::set_messaging_timeout(el, deadline)?; + budget.ensure_budget()?; let action = &request.action; let label = action.name(); let mut steps = Vec::new(); tracing::debug!("action: perform {label}"); match action { Action::Click => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::CLICK_CHAIN, - &ctx, - request.policy, + None, + request, + deadline, )?); } Action::DoubleClick => { - chain_defs::double_click(el, request.policy)?; + steps.extend(chain_defs::double_click(el, request, deadline)?); } Action::RightClick => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::RIGHT_CLICK_CHAIN, - &ctx, - request.policy, + None, + request, + deadline, )?); } Action::Toggle => { - toggle_state::toggle(el, request.policy)?; + steps.extend(toggle_state::toggle(el, request.policy, deadline)?); } Action::SetValue(val) => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: Some(val.as_str()), - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::SET_VALUE_CHAIN, - &ctx, - request.policy, + Some(val), + request, + deadline, )?); } Action::SetFocus => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::FOCUS_CHAIN, - &ctx, - request.policy, + None, + request, + deadline, )?); } Action::TypeText(text) => { - crate::actions::type_text::execute_type(el, text.as_str(), request.policy)?; + steps.push(crate::actions::type_text::execute_type( + el, + text.as_str(), + request.policy, + deadline, + )?); } Action::PressKey(combo) => { - crate::input::keyboard::synthesize_key(combo)?; + crate::actions::physical_keyboard::press(el, combo, request.policy, deadline)?; + steps.push( + ActionStep::succeeded("PressKey") + .with_mechanism(StepMechanism::PhysicalSynthetic) + .with_verified(false), + ); } Action::Expand => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::EXPAND_CHAIN, - &ctx, - request.policy, + None, + request, + deadline, )?); } Action::Collapse => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::COLLAPSE_CHAIN, - &ctx, - request.policy, + None, + request, + deadline, )?); } Action::Select(value) => { - crate::actions::extras::select_value(el, value.as_str())?; + let verified = crate::actions::extras::select_value(el, value.as_str(), deadline)?; + steps.push( + ActionStep::succeeded("Select") + .with_mechanism(StepMechanism::SemanticApi) + .with_verified(verified), + ); } Action::Scroll(direction, amount) => { - crate::actions::scroll::ax_scroll(el, direction, *amount, request.policy)?; + let (mechanism, verified) = crate::actions::scroll::ax_scroll( + el, + direction, + *amount, + request.policy, + deadline, + )?; + steps.push( + ActionStep::succeeded("Scroll") + .with_mechanism(mechanism) + .with_verified(verified), + ); } Action::Check => { - toggle_state::check_uncheck(el, true, request.policy)?; + steps.extend(toggle_state::check_uncheck( + el, + true, + request.policy, + deadline, + )?); } Action::Uncheck => { - toggle_state::check_uncheck(el, false, request.policy)?; + steps.extend(toggle_state::check_uncheck( + el, + false, + request.policy, + deadline, + )?); } Action::TripleClick => { - chain_defs::triple_click(el, request.policy)?; + steps.extend(chain_defs::triple_click(el, request, deadline)?); } Action::ScrollTo => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::SCROLL_TO_CHAIN, - &ctx, - request.policy, + None, + request, + deadline, )?); } Action::Clear => { - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: Some(""), - deadline: None, - }; - steps.extend(execute_chain( + steps.extend(run_chain( el, - &caps, &chain_defs::CLEAR_CHAIN, - &ctx, - request.policy, + Some(""), + request, + deadline, )?); } @@ -240,73 +193,42 @@ mod imp { } } - let mut result = ActionResult::new(label).with_steps(steps); - if let Some(state) = crate::actions::post_state::read_post_state(el, action) { - verify_post_state(action, &state)?; - result = result.with_state(state); - } - Ok(result) + let post_state = if delivery_occurred(&steps) && !deadline.is_expired() { + crate::actions::post_state::read_post_state(el, action, deadline) + .map_err(after_delivery)? + } else { + None + }; + ActionResult::from_execution(action, steps, post_state) } - fn verify_post_state(action: &Action, state: &ElementState) -> Result<(), AdapterError> { - if matches!(action, Action::Clear) - && state - .value - .as_deref() - .is_some_and(|value| !value.is_empty()) - { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - "Clear reported success but element value is still non-empty", - ) - .with_suggestion("Retry 'clear', or use 'press cmd+a' then 'press delete'.")); - } - Ok(()) + fn run_chain( + element: &AXElement, + definition: &ChainDef, + dynamic_value: Option<&str>, + request: &ActionRequest, + deadline: Deadline, + ) -> Result<Vec<ActionStep>, AdapterError> { + execute_chain( + element, + definition, + &ChainContext { + dynamic_value, + verified_point: request.verified_point(), + deadline, + }, + request.policy, + ) } - pub(crate) fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> { - if !ax_helpers::ax_press(el) { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("{context}: AXPress failed"), - ) - .with_suggestion("Element may not be pressable. Try 'click' instead.")); - } - Ok(()) + fn after_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(agent_desktop_core::DeliverySemantics::delivered_unverified()) } - #[cfg(test)] - mod tests { - use super::*; - use agent_desktop_core::element_state::ElementState; - - #[test] - fn clear_post_state_fails_when_value_remains() { - let err = verify_post_state( - &Action::Clear, - &ElementState { - role: "textfield".into(), - states: vec![], - value: Some("still here".into()), - }, - ) - .unwrap_err(); - - assert_eq!(err.code, ErrorCode::ActionFailed); - } - - #[test] - fn clear_post_state_accepts_empty_value() { - verify_post_state( - &Action::Clear, - &ElementState { - role: "textfield".into(), - states: vec![], - value: Some(String::new()), - }, - ) - .unwrap(); - } + fn delivery_occurred(steps: &[ActionStep]) -> bool { + steps + .iter() + .any(|step| matches!(step.outcome, ActionStepOutcome::Succeeded)) } } @@ -318,12 +240,10 @@ mod imp { pub fn perform_action( _el: &AXElement, _request: &ActionRequest, + _deadline: Deadline, ) -> Result<ActionResult, AdapterError> { Err(AdapterError::not_supported("perform_action")) } } pub(crate) use imp::perform_action; - -#[cfg(target_os = "macos")] -pub(crate) use imp::{ax_press_or_fail, click_via_bounds}; diff --git a/crates/macos/src/actions/extras.rs b/crates/macos/src/actions/extras.rs index 324e61a..054093f 100644 --- a/crates/macos/src/actions/extras.rs +++ b/crates/macos/src/actions/extras.rs @@ -1,242 +1,152 @@ #[cfg(target_os = "macos")] -use agent_desktop_core::error::{AdapterError, ErrorCode}; +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; #[cfg(target_os = "macos")] use crate::tree::AXElement; #[cfg(target_os = "macos")] -pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterError> { - use crate::actions::ax_helpers; +pub(crate) fn select_value( + element: &AXElement, + value: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + prepare(element, deadline)?; + let role = crate::actions::ax_helpers::element_role(element, deadline)?; + match select_role_family(role.as_deref()) { + Some("value_or_menu") => select_combobox(element, value, deadline), + Some("menu") => crate::actions::select_menu::select_from_menu(element, value, deadline), + Some("collection") => { + crate::actions::select_menu::select_collection_item(element, value, deadline) + } + _ => Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!( + "Select is not supported on role '{}'", + role.as_deref().unwrap_or("unknown") + ), + ) + .with_suggestion( + "Target a combobox, popup button, menu button, list, table, or outline; otherwise use click.", + )), + } +} - let role = ax_helpers::element_role(el); - match role.as_deref() { - Some("combobox") => { - if set_value_and_verify(el, value) { - return Ok(()); - } - let pid = crate::system::app_ops::pid_from_element(el); - open_menu(el, pid, "select (open combo box)")?; - if !wait_for_menu_item(el, pid, value) { - press_escape(el); - return Err(option_not_found(value)); - } - wait_for_value(el, value)?; - } - Some("popupbutton") | Some("menubutton") => { - let pid = crate::system::app_ops::pid_from_element(el); - open_menu(el, pid, "select (open popup)")?; - if !wait_for_menu_item(el, pid, value) { - press_escape(el); - return Err(option_not_found(value)); - } - if crate::tree::copy_value_typed(el).is_some() { - wait_for_value(el, value)?; - } - } - Some("list") | Some("table") | Some("outline") => { - if !select_child_by_name(el, value) { - return Err(AdapterError::new( - ErrorCode::ElementNotFound, - format!( - "No child matching the requested value ({} chars) found in list", - value.chars().count() - ), - ) - .with_suggestion("Use 'find --role' to discover available items.")); - } - } - _ => { - if ax_helpers::ax_set_value(el, value).is_err() { - return Err(AdapterError::new( - ErrorCode::ActionNotSupported, - format!( - "Select not supported on role '{}'", - role.as_deref().unwrap_or("unknown") - ), - ) - .with_suggestion("Use 'click' or 'set-value' instead.")); - } +#[cfg(target_os = "macos")] +fn select_combobox( + element: &AXElement, + value: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + prepare(element, deadline)?; + if crate::actions::ax_helpers::is_attr_settable(element, "AXValue", deadline)? { + match crate::actions::ax_helpers::set_ax_value_coerced(element, value, deadline) { + Ok(()) => match verify_value(element, value, deadline) { + Ok(verified) => return Ok(verified), + Err(error) if should_fallback_after_value_verification(&error) => { + return crate::actions::select_menu::select_from_menu(element, value, deadline) + .map_err(after_delivery); + } + Err(error) => return Err(after_delivery(error)), + }, + Err(error) + if error.disposition == agent_desktop_core::DeliverySemantics::not_delivered() => {} + Err(error) => return Err(error), } } - Ok(()) + crate::actions::select_menu::select_from_menu(element, value, deadline) +} + +fn should_fallback_after_value_verification(error: &AdapterError) -> bool { + error.code == ErrorCode::ActionFailed + && error + .details + .as_ref() + .and_then(|details| details.get("kind")) + .and_then(serde_json::Value::as_str) + == Some("selected_value_not_observed") +} + +fn select_role_family(role: Option<&str>) -> Option<&'static str> { + match role { + Some("combobox") => Some("value_or_menu"), + Some("popupbutton" | "menubutton") => Some("menu"), + Some("list" | "table" | "outline") => Some("collection"), + _ => None, + } } #[cfg(target_os = "macos")] -fn set_value_and_verify(el: &AXElement, value: &str) -> bool { - crate::actions::ax_helpers::ax_set_value(el, value).is_ok() && wait_for_value(el, value).is_ok() -} - -#[cfg(target_os = "macos")] -fn wait_for_value(el: &AXElement, value: &str) -> Result<(), AdapterError> { - let deadline = std::time::Instant::now() + std::time::Duration::from_millis(600); +fn verify_value( + element: &AXElement, + expected: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + let local_end = std::time::Instant::now() + std::time::Duration::from_millis(600); loop { - if crate::tree::copy_value_typed(el) - .as_deref() - .is_some_and(|current| current.eq_ignore_ascii_case(value)) - { - return Ok(()); + prepare(element, deadline)?; + if crate::tree::copy_value_typed(element, deadline).as_deref() == Some(expected) { + return Ok(true); } - if std::time::Instant::now() >= deadline { + if deadline.is_expired() { + return Err(deadline.timeout_error().with_details(serde_json::json!({ + "verification": "selected_value_not_observed", + }))); + } + if std::time::Instant::now() >= local_end { return Err(AdapterError::new( ErrorCode::ActionFailed, - format!( - "Selection did not change to the requested value ({} chars)", - value.chars().count() - ), + "Selection write completed but the exact requested value was not observed", ) - .with_suggestion("Refresh the snapshot and inspect available values.")); + .with_details(serde_json::json!({ + "kind": "selected_value_not_observed", + }))); } - std::thread::sleep(std::time::Duration::from_millis(25)); + let pause = deadline.remaining_slice(std::time::Duration::from_millis(25))?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(25))); } } #[cfg(target_os = "macos")] -fn option_not_found(value: &str) -> AdapterError { - AdapterError::new( - ErrorCode::ElementNotFound, - format!( - "No menu item matching the requested value ({} chars) found", - value.chars().count() - ), - ) - .with_suggestion("Use 'click' to open the menu, then 'snapshot' to see available options.") +fn after_delivery(error: AdapterError) -> AdapterError { + let mut delivery = crate::actions::DeliveryTracker::default(); + delivery.mark_delivered(); + delivery.annotate(error) } #[cfg(target_os = "macos")] -fn find_and_press_open_menu_item(pid: i32, target_value: &str) -> bool { - crate::tree::surfaces::menu_element_for_pid(pid) - .as_ref() - .is_some_and(|menu| find_and_press_menu_item(menu, target_value)) +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) } -#[cfg(target_os = "macos")] -fn open_menu(el: &AXElement, pid: Option<i32>, context: &str) -> Result<(), AdapterError> { - let was_open = pid.is_some_and(is_menu_open); - if was_open { - return Err(menu_already_open()); - } - if crate::actions::ax_helpers::try_ax_action_retried(el, "AXShowMenu") - && menu_opened_after_action(was_open, wait_for_open_menu(pid)) - { - return Ok(()); - } - crate::actions::dispatch::ax_press_or_fail(el, context)?; - if pid.is_none() || menu_opened_after_action(was_open, wait_for_open_menu(pid)) { - return Ok(()); - } - Err(AdapterError::timeout(format!( - "No context menu opened within {}ms", - crate::system::wait::menu_timeout_ms() - ))) -} - -#[cfg(target_os = "macos")] -fn wait_for_open_menu(pid: Option<i32>) -> bool { - pid.is_some_and(|p| { - crate::system::wait::wait_for_menu(p, true, crate::system::wait::menu_timeout_ms()).is_ok() - }) -} - -#[cfg(target_os = "macos")] -fn is_menu_open(pid: i32) -> bool { - crate::system::wait::wait_for_menu(pid, true, 60).is_ok() -} - -#[cfg(target_os = "macos")] -fn wait_for_menu_item(el: &AXElement, pid: Option<i32>, target_value: &str) -> bool { - let deadline = std::time::Instant::now() - + std::time::Duration::from_millis(crate::system::wait::menu_timeout_ms()); - loop { - if find_and_press_menu_item(el, target_value) - || pid.is_some_and(|p| find_and_press_open_menu_item(p, target_value)) - { - return true; - } - if std::time::Instant::now() >= deadline { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(25)); - } -} - -#[cfg(target_os = "macos")] -fn menu_already_open() -> AdapterError { - AdapterError::new( - ErrorCode::ActionFailed, - "Refusing to select from a menu while another menu is already open", - ) - .with_suggestion("Dismiss the open menu and retry the select command.") -} - -#[cfg(target_os = "macos")] -fn menu_opened_after_action(was_open: bool, is_open: bool) -> bool { - !was_open && is_open -} - -#[cfg(target_os = "macos")] -fn find_and_press_menu_item(el: &AXElement, target_value: &str) -> bool { - use accessibility_sys::{AXUIElementPerformAction, kAXPressAction}; - use core_foundation::{base::TCFType, string::CFString}; - - let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - for child in &children { - let title = crate::tree::copy_string_attr(child, "AXTitle"); - if let Some(t) = &title { - if t.eq_ignore_ascii_case(target_value) { - let action = CFString::new(kAXPressAction); - unsafe { AXUIElementPerformAction(child.0, action.as_concrete_TypeRef()) }; - return true; - } - } - if find_and_press_menu_item(child, target_value) { - return true; - } - } - false -} - -#[cfg(target_os = "macos")] -fn press_escape(el: &AXElement) { - use accessibility_sys::AXUIElementPerformAction; - use core_foundation::{base::TCFType, string::CFString}; - - let cancel = CFString::new("AXCancel"); - unsafe { AXUIElementPerformAction(el.0, cancel.as_concrete_TypeRef()) }; -} - -#[cfg(target_os = "macos")] -fn select_child_by_name(el: &AXElement, name: &str) -> bool { - use accessibility_sys::{AXUIElementPerformAction, kAXPressAction}; - use core_foundation::{base::TCFType, string::CFString}; - - let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - for child in &children { - let child_name = crate::tree::copy_string_attr(child, "AXTitle") - .or_else(|| crate::tree::copy_string_attr(child, "AXDescription")); - if let Some(n) = &child_name { - if n.eq_ignore_ascii_case(name) { - let action = CFString::new(kAXPressAction); - unsafe { AXUIElementPerformAction(child.0, action.as_concrete_TypeRef()) }; - return true; - } - } - } - false -} - -#[cfg(all(test, target_os = "macos"))] +#[cfg(test)] mod tests { - use super::menu_opened_after_action; + use super::{select_role_family, should_fallback_after_value_verification}; + use agent_desktop_core::{AdapterError, ErrorCode}; #[test] - fn menu_guard_rejects_preexisting_menu() { - assert!(!menu_opened_after_action(true, true)); - assert!(!menu_opened_after_action(true, false)); + fn native_select_roles_keep_their_specialized_paths() { + assert_eq!(select_role_family(Some("combobox")), Some("value_or_menu")); + assert_eq!(select_role_family(Some("popupbutton")), Some("menu")); + assert_eq!(select_role_family(Some("menubutton")), Some("menu")); + assert_eq!(select_role_family(Some("list")), Some("collection")); + assert_eq!(select_role_family(Some("table")), Some("collection")); + assert_eq!(select_role_family(Some("outline")), Some("collection")); + assert_eq!(select_role_family(Some("button")), None); } #[test] - fn menu_guard_requires_closed_to_open_transition() { - assert!(menu_opened_after_action(false, true)); - assert!(!menu_opened_after_action(false, false)); + fn completed_combobox_write_with_verify_miss_uses_menu_fallback() { + let error = AdapterError::new( + ErrorCode::ActionFailed, + "Selection write completed but the exact requested value was not observed", + ) + .with_details(serde_json::json!({ + "kind": "selected_value_not_observed", + })); + + assert!(should_fallback_after_value_verification(&error)); + assert!(!should_fallback_after_value_verification( + &AdapterError::timeout("verification timed out") + )); } } diff --git a/crates/macos/src/actions/mod.rs b/crates/macos/src/actions/mod.rs index 036c00c..4618e2c 100644 --- a/crates/macos/src/actions/mod.rs +++ b/crates/macos/src/actions/mod.rs @@ -1,23 +1,31 @@ +mod adapter; pub(crate) mod ax_helpers; +#[cfg(target_os = "macos")] +pub(crate) mod ax_mutation; pub(crate) mod chain; mod chain_context; mod chain_def; pub(crate) mod chain_defs; +mod chain_delivery; pub(crate) mod chain_disclosure_steps; pub(crate) mod chain_menu_steps; mod chain_step; -pub(crate) mod chain_steps; +pub(crate) mod chain_step_exec; +pub(crate) mod chain_value_write; pub(crate) mod chain_verify; -pub(crate) mod chain_web_steps; -pub(crate) mod discovery; +pub(crate) mod delivery_tracker; pub(crate) mod dispatch; pub(crate) mod extras; +mod physical_click; +mod physical_keyboard; +mod physical_target; pub(crate) mod post_state; pub(crate) mod scroll; +pub(crate) mod scroll_into_view; +mod scroll_read; +pub(crate) mod select_menu; pub(crate) mod toggle_state; pub(crate) mod type_text; -#[cfg(test)] -mod chain_steps_tests; - +pub(crate) use delivery_tracker::DeliveryTracker; pub(crate) use dispatch::perform_action; diff --git a/crates/macos/src/actions/physical_click.rs b/crates/macos/src/actions/physical_click.rs new file mode 100644 index 0000000..56d97e1 --- /dev/null +++ b/crates/macos/src/actions/physical_click.rs @@ -0,0 +1,84 @@ +use agent_desktop_core::{ + AdapterError, Deadline, ErrorCode, InteractionPolicy, MouseButton, MouseEvent, MouseEventKind, + Point, +}; + +use crate::tree::AXElement; + +pub(crate) struct PhysicalClick { + pub(crate) button: MouseButton, + pub(crate) count: u32, + pub(crate) verified_point: Option<Point>, +} + +pub(crate) fn click_via_bounds( + element: &AXElement, + click: PhysicalClick, + policy: InteractionPolicy, + deadline: Deadline, +) -> Result<(), AdapterError> { + if !policy.allow_cursor_move || !policy.allow_focus_steal { + return Err(AdapterError::policy_denied_for_policy( + "Physical click fallback is disabled by the current interaction policy", + policy, + )); + } + let prepared = + crate::actions::physical_target::PreparedPhysicalTarget::prepare(element, deadline)?; + let read_deadline = crate::tree::locator_deadline::from_operation(deadline)?; + let bounds = crate::tree::element_bounds::read_bounds_with_deadline(element, read_deadline)? + .ok_or_else(|| { + AdapterError::new(ErrorCode::ActionFailed, "Element has no readable bounds") + .with_suggestion("AX action failed and CGEvent fallback unavailable") + })?; + if bounds.width <= 0.0 || bounds.height <= 0.0 { + return Err( + AdapterError::new(ErrorCode::ActionFailed, "Element has zero-size bounds") + .with_suggestion("Element may be hidden or off-screen. Try 'scroll-to' first."), + ); + } + let point = delivery_point(bounds, click.verified_point.as_ref())?; + crate::input::mouse::validate_point(&point)?; + tracing::debug!( + button = ?click.button, + count = click.count, + x = point.x, + y = point.y, + "AX action failed, falling back to CGEvent click" + ); + let mut verify_target = || prepared.verify_pointer(element, &point, deadline); + crate::input::mouse::synthesize_mouse_after( + MouseEvent { + kind: MouseEventKind::Click { count: click.count }, + point: point.clone(), + button: click.button, + modifiers: Vec::new(), + }, + deadline, + &mut verify_target, + ) +} + +fn delivery_point( + bounds: agent_desktop_core::Rect, + verified: Option<&Point>, +) -> Result<Point, AdapterError> { + let point = verified.cloned().unwrap_or(Point { + x: bounds.x + bounds.width / 2.0, + y: bounds.y + bounds.height / 2.0, + }); + if point.x < bounds.x + || point.x > bounds.x + bounds.width + || point.y < bounds.y + || point.y > bounds.y + bounds.height + { + return Err(AdapterError::stale_ref( + "Actionability-verified input point is outside the target's live bounds", + )); + } + Ok(point) +} + +#[cfg(test)] +#[path = "physical_click_tests.rs"] +mod tests; diff --git a/crates/macos/src/actions/physical_click_tests.rs b/crates/macos/src/actions/physical_click_tests.rs new file mode 100644 index 0000000..6e05793 --- /dev/null +++ b/crates/macos/src/actions/physical_click_tests.rs @@ -0,0 +1,35 @@ +use super::delivery_point; +use agent_desktop_core::{Point, Rect}; + +fn bounds() -> Rect { + Rect { + x: 10.0, + y: 20.0, + width: 100.0, + height: 40.0, + } +} + +#[test] +fn physical_delivery_uses_the_point_proven_by_preflight() { + let verified = Point { x: 35.0, y: 30.0 }; + + assert_eq!(delivery_point(bounds(), Some(&verified)).unwrap(), verified); +} + +#[test] +fn physical_delivery_centers_only_without_a_verified_point() { + assert_eq!( + delivery_point(bounds(), None).unwrap(), + Point { x: 60.0, y: 40.0 } + ); +} + +#[test] +fn physical_delivery_rejects_a_verified_point_outside_live_bounds() { + let stale = Point { x: 9.0, y: 30.0 }; + + let error = delivery_point(bounds(), Some(&stale)).unwrap_err(); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::StaleRef); +} diff --git a/crates/macos/src/actions/physical_keyboard.rs b/crates/macos/src/actions/physical_keyboard.rs new file mode 100644 index 0000000..64bc7e1 --- /dev/null +++ b/crates/macos/src/actions/physical_keyboard.rs @@ -0,0 +1,209 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, InteractionPolicy, KeyCombo}; +use std::time::{Duration, Instant}; + +use crate::tree::AXElement; + +pub(crate) fn press( + element: &AXElement, + combo: &KeyCombo, + policy: InteractionPolicy, + deadline: Deadline, +) -> Result<(), AdapterError> { + press_sequence(element, std::slice::from_ref(combo), policy, deadline) +} + +pub(crate) fn press_sequence( + element: &AXElement, + combos: &[KeyCombo], + policy: InteractionPolicy, + deadline: Deadline, +) -> Result<(), AdapterError> { + let mut delivery = crate::actions::DeliveryTracker::default(); + let identity = + prepare_target(element, policy, deadline).map_err(|error| delivery.annotate(error))?; + let pid = identity.pid(); + for combo in combos { + verify_delivery_target(element, identity, deadline) + .map_err(|error| delivery.annotate(error))?; + crate::input::keyboard::synthesize_key(combo, Some(pid), deadline) + .map_err(|error| delivery.annotate(error))?; + delivery.mark_delivered(); + } + Ok(()) +} + +pub(crate) fn type_text( + element: &AXElement, + text: &str, + policy: InteractionPolicy, + deadline: Deadline, +) -> Result<(), AdapterError> { + crate::input::keyboard::preflight_text(text, deadline)?; + let identity = prepare_target(element, policy, deadline)?; + let pid = identity.pid(); + crate::input::keyboard::synthesize_text(text, pid, deadline, |deadline| { + verify_delivery_target(element, identity, deadline) + }) +} + +fn prepare_target( + element: &AXElement, + policy: InteractionPolicy, + deadline: Deadline, +) -> Result<crate::system::process_identity::ProcessIdentity, AdapterError> { + if !policy.allow_focus_steal { + return Err(AdapterError::policy_denied_for_policy( + "Physical keyboard fallback requires focus permission", + policy, + )); + } + let pid = crate::system::app_ops::pid_from_element(element, deadline) + .filter(|pid| *pid > 0) + .ok_or_else(|| { + AdapterError::new( + ErrorCode::StaleRef, + "Keyboard target no longer has a valid owning process", + ) + })?; + let identity = + crate::system::process_identity::ProcessIdentity::capture(pid)?.ok_or_else(|| { + AdapterError::new( + ErrorCode::StaleRef, + "Keyboard target process exited before physical input preparation", + ) + })?; + if let Some(window) = target_window(element, deadline)? { + crate::system::focus::verify_window_main(&window, deadline)?; + } + crate::system::focus::verify_app_focused(pid, deadline)?; + prepare(element, deadline)?; + if !crate::actions::ax_helpers::ax_focus_or_err(element, deadline)? { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "Target element could not be focused for keyboard input", + ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())); + } + wait_for_focused_element(element, pid, deadline)?; + crate::system::focus::verify_app_focused(pid, deadline)?; + verify_delivery_target(element, identity, deadline)?; + Ok(identity) +} + +fn verify_delivery_target( + expected: &AXElement, + identity: crate::system::process_identity::ProcessIdentity, + deadline: Deadline, +) -> Result<(), AdapterError> { + if !identity.still_matches()? { + return Err(AdapterError::new( + ErrorCode::StaleRef, + "Keyboard target process instance changed before physical input delivery", + ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())); + } + verify_focused_element(expected, identity.pid(), deadline) +} + +fn target_window( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_element_attr_result(element, "AXWindow", deadline); + ensure_budget(deadline)?; + result.map_err(|error| read_error("AXWindow", error)) +} + +fn wait_for_focused_element( + expected: &AXElement, + pid: i32, + deadline: Deadline, +) -> Result<(), AdapterError> { + let local_deadline = Instant::now() + Duration::from_millis(500); + loop { + if focused_element_matches(expected, pid, deadline)? { + return Ok(()); + } + ensure_budget(deadline)?; + if Instant::now() >= local_deadline { + return Err(AdapterError::timeout( + "Target element did not become focused before keyboard delivery", + ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())); + } + let pause = deadline.remaining_slice(Duration::from_millis(5))?; + std::thread::sleep(pause.min(Duration::from_millis(5))); + } +} + +fn verify_focused_element( + expected: &AXElement, + pid: i32, + deadline: Deadline, +) -> Result<(), AdapterError> { + crate::system::focus::verify_app_focused(pid, deadline)?; + if focused_element_matches(expected, pid, deadline)? { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::ActionFailed, + "Target element lost focus before PID-targeted keyboard delivery", + ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())) +} + +fn focused_element_matches( + expected: &AXElement, + pid: i32, + deadline: Deadline, +) -> Result<bool, AdapterError> { + use core_foundation::base::{CFEqual, CFTypeRef}; + + let app = crate::tree::element_for_pid(pid); + prepare(&app, deadline)?; + let result = + crate::tree::attributes::copy_element_attr_result(&app, "AXFocusedUIElement", deadline); + ensure_budget(deadline)?; + let focused = result.map_err(|error| read_error("AXFocusedUIElement", error))?; + Ok(focused.is_some_and(|focused| unsafe { + CFEqual(focused.0 as CFTypeRef, expected.0 as CFTypeRef) != 0 + })) +} + +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn read_error(attribute: &str, error: i32) -> AdapterError { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + + let code = if error == kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }; + AdapterError::new( + code, + format!("Could not read {attribute} for keyboard targeting"), + ) + .with_details(serde_json::json!({ + "attribute": attribute, + "ax_error": error, + })) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) +} diff --git a/crates/macos/src/actions/physical_target.rs b/crates/macos/src/actions/physical_target.rs new file mode 100644 index 0000000..09abf60 --- /dev/null +++ b/crates/macos/src/actions/physical_target.rs @@ -0,0 +1,93 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, Point}; + +use crate::tree::AXElement; + +pub(crate) struct PreparedPhysicalTarget { + identity: crate::system::process_identity::ProcessIdentity, + window: AXElement, +} + +impl PreparedPhysicalTarget { + pub(crate) fn prepare(element: &AXElement, deadline: Deadline) -> Result<Self, AdapterError> { + let pid = crate::system::app_ops::pid_from_element(element, deadline).ok_or_else(|| { + AdapterError::new( + ErrorCode::StaleRef, + "Physical input target no longer has an owning application", + ) + })?; + let identity = + crate::system::process_identity::ProcessIdentity::capture(pid)?.ok_or_else(|| { + AdapterError::new( + ErrorCode::StaleRef, + "Physical input target process exited before input preparation", + ) + })?; + let window = target_window(element, deadline)?; + crate::system::focus::verify_app_focused(pid, deadline)?; + crate::system::focus::verify_window_main(&window, deadline)?; + Ok(Self { identity, window }) + } + + pub(crate) fn verify_pointer( + &self, + element: &AXElement, + point: &Point, + deadline: Deadline, + ) -> Result<(), AdapterError> { + crate::system::focus::verify_app_focused(self.identity.pid(), deadline)?; + crate::system::focus::verify_window_main(&self.window, deadline)?; + match crate::tree::hit_test::hit_test_ax_element(element, point.clone(), deadline)? { + agent_desktop_core::hit_test::HitTestResult::ReachesTarget => {} + agent_desktop_core::hit_test::HitTestResult::InterceptedBy { role, name, .. } => { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "Physical input point is intercepted by another accessibility element", + ) + .with_details(serde_json::json!({ + "physical_delivery_started": false, + "occluder_role": role, + "occluder_name": name, + }))); + } + agent_desktop_core::hit_test::HitTestResult::Unknown => { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "Physical input target could not be proven at the final input point", + ) + .with_details(serde_json::json!({ "physical_delivery_started": false }))); + } + } + if !self.identity.still_matches()? { + return Err(AdapterError::new( + ErrorCode::StaleRef, + "Physical input target process instance changed at input delivery", + ) + .with_details(serde_json::json!({ "physical_delivery_started": false }))); + } + Ok(()) + } +} + +fn target_window(element: &AXElement, deadline: Deadline) -> Result<AXElement, AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline)?; + let result = crate::tree::attributes::copy_element_attr_result(element, "AXWindow", deadline); + if deadline.is_expired() { + return Err(deadline.timeout_error()); + } + match result { + Ok(Some(window)) => Ok(window), + Ok(None) => Err(AdapterError::new( + ErrorCode::ActionFailed, + "Physical input target has no verified owning window", + ) + .with_details(serde_json::json!({ "physical_delivery_started": false }))), + Err(error) => Err(AdapterError::new( + ErrorCode::ActionFailed, + "Could not verify the physical input target window", + ) + .with_details(serde_json::json!({ + "ax_error": error, + "physical_delivery_started": false, + }))), + } +} diff --git a/crates/macos/src/actions/post_state.rs b/crates/macos/src/actions/post_state.rs index 4ef3781..261d324 100644 --- a/crates/macos/src/actions/post_state.rs +++ b/crates/macos/src/actions/post_state.rs @@ -1,20 +1,27 @@ -use agent_desktop_core::{action::Action, adapter::LiveElement, element_state::ElementState}; +use agent_desktop_core::{ + Action, AdapterError, Deadline, ElementState, ErrorCode, EvidenceRequirements, LiveElement, + LiveIdentity, LocatorField, ObservationBudget, Rect, +}; +use std::time::{Duration, Instant}; + +use crate::tree::AXElement; -#[cfg(target_os = "macos")] pub(crate) fn read_post_state( - el: &crate::tree::AXElement, + element: &AXElement, action: &Action, -) -> Option<ElementState> { - let delay_ms = match action { - Action::Click | Action::TypeText(_) => 50, + deadline: Deadline, +) -> Result<Option<ElementState>, AdapterError> { + let delay = match action { + Action::TypeText(_) => Duration::from_millis(50), Action::Toggle | Action::Check | Action::Uncheck | Action::SetValue(_) | Action::Clear | Action::Expand - | Action::Collapse => 0, - Action::DoubleClick + | Action::Collapse => Duration::ZERO, + Action::Click + | Action::DoubleClick | Action::RightClick | Action::TripleClick | Action::SetFocus @@ -25,78 +32,254 @@ pub(crate) fn read_post_state( | Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover - | Action::Drag(_) => return None, + | Action::Drag(_) => return Ok(None), }; - if delay_ms > 0 { - std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + if !delay.is_zero() && !pause_if_budget_allows(deadline, delay) { + return Ok(None); } - Some(read_element_state(el)) + read_element_state(element, deadline).map(Some) } -pub(crate) fn read_element_state(el: &crate::tree::AXElement) -> ElementState { - let attrs = crate::tree::element::fetch_node_attrs(el); - let role = normalized_role(attrs.role.as_deref()); - element_state_from_attrs(attrs, role) +pub(crate) fn read_element_state( + element: &AXElement, + deadline: Deadline, +) -> Result<ElementState, AdapterError> { + let read = read_live_observation(element, deadline)?; + let attrs = read.attrs; + let role = known_role(&read.evidence.role)?; + element_state_from_attrs( + element, + attrs, + role, + owning_window_bounds(element, deadline)?, + ) } -pub(crate) fn read_live_element(el: &crate::tree::AXElement) -> LiveElement { - let attrs = crate::tree::element::fetch_node_attrs(el); - let role = normalized_role(attrs.role.as_deref()); +pub(crate) fn read_live_element( + element: &AXElement, + deadline: Deadline, +) -> Result<LiveElement, AdapterError> { + let read = read_live_observation(element, deadline)?; + let role = known_role(&read.evidence.role)?; + let identity = LiveIdentity { + name: read.evidence.name, + description: read.evidence.description, + identifiers: read.evidence.identifiers, + }; + let available_actions = known_actions(read.evidence.ref_evidence.available_actions)?; + let attrs = read.attrs; let bounds = attrs.bounds; - let has_scrollbars = attrs.has_scrollbars; - let state = element_state_from_attrs(attrs, role.clone()); - LiveElement { - state: Some(state), + let window_bounds = owning_window_bounds(element, deadline)?; + let state = element_state_from_attrs(element, attrs, role, window_bounds)?; + Ok(LiveElement { + identity, + state, + states_complete: true, bounds, - available_actions: Some(crate::tree::action_list::platform_available_actions( - el, - &role, - has_scrollbars, - )), + available_actions, + }) +} + +pub(crate) fn read_live_actions( + element: &AXElement, + deadline: Deadline, +) -> Result<Vec<String>, AdapterError> { + Ok(read_live_element(element, deadline)?.available_actions) +} + +fn read_live_observation( + element: &AXElement, + deadline: Deadline, +) -> Result<crate::tree::query::node_read::NodeRead, AdapterError> { + let mut usage = new_usage(); + if !usage.claim_node() { + return Err(incomplete_live_evidence()); + } + let mut stats = agent_desktop_core::LocatorStats::default(); + let child_plan = + crate::tree::query::child_read_plan::ChildReadPlan::load(usage.child_capacity()); + let read = crate::tree::query::node_read::read_node( + element, + crate::tree::query::node_read_context::NodeReadContext { + tree: &crate::tree::TreeBuildContext::empty(false), + stats: &mut stats, + usage: &mut usage, + requirements: EvidenceRequirements::snapshot(), + deadline: deadline_instant(deadline)?, + child_plan, + }, + )?; + usage.note_child_demand(read.child_read.total_count, &mut stats); + usage.claim_edges(read.child_read.elements.len()); + if read.invalid_element { + return Err(AdapterError::stale_ref( + "Element became invalid while reading live state", + )); + } + if deadline.is_expired() { + return Err(deadline.timeout_error()); + } + if !essential_live_evidence_complete(&read.evidence) { + return Err(incomplete_live_evidence().with_details(serde_json::json!({ + "kind": "live_element_evidence", + "complete": false, + "query_stats": stats, + }))); + } + Ok(read) +} + +fn essential_live_evidence_complete(evidence: &agent_desktop_core::LocatorEvidence) -> bool { + !evidence.role.is_unknown() + && !evidence.value.is_unknown() + && !evidence.states.is_unknown() + && !evidence.ref_evidence.bounds.is_unknown() + && !evidence.ref_evidence.available_actions.is_unknown() +} + +fn new_usage() -> crate::tree::observation_usage::ObservationUsage { + crate::tree::observation_usage::ObservationUsage::new(ObservationBudget::default()) +} + +fn owning_window_bounds( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<Rect>, AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline)?; + let window = first_owning_container(|attribute| { + crate::tree::attributes::copy_element_attr_result(element, attribute, deadline) + }) + .map_err(|(attribute, error)| read_error(attribute, error))?; + if deadline.is_expired() { + return Err(deadline.timeout_error()); + } + let Some(window) = window else { + return Ok(None); + }; + crate::tree::element_bounds::read_bounds_with_deadline(&window, deadline_instant(deadline)?) +} + +fn first_owning_container( + mut read: impl FnMut(&'static str) -> Result<Option<AXElement>, i32>, +) -> Result<Option<AXElement>, (&'static str, i32)> { + for attribute in ["AXWindow", "AXTopLevelUIElement"] { + match read(attribute) { + Ok(Some(element)) => return Ok(Some(element)), + Ok(None) => {} + Err(error) => return Err((attribute, error)), + } + } + Ok(None) +} + +fn known_role(role: &LocatorField<String>) -> Result<String, AdapterError> { + role.known().cloned().ok_or_else(incomplete_live_evidence) +} + +fn known_actions(actions: LocatorField<Vec<String>>) -> Result<Vec<String>, AdapterError> { + match actions { + LocatorField::Known(actions) => Ok(actions), + LocatorField::Absent => Ok(Vec::new()), + LocatorField::Unknown => Err(incomplete_live_evidence()), } } -pub(crate) fn read_live_actions(el: &crate::tree::AXElement) -> Vec<String> { - let attrs = crate::tree::element::fetch_node_attrs(el); - let role = normalized_role(attrs.role.as_deref()); - crate::tree::action_list::platform_available_actions(el, &role, attrs.has_scrollbars) +fn incomplete_live_evidence() -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Live element evidence was incomplete", + ) + .with_details(serde_json::json!({ + "kind": "live_element_evidence", + "complete": false, + "retryable": true, + })) } -fn element_state_from_attrs(attrs: crate::tree::NodeAttrs, role: String) -> ElementState { - let value = attrs.value; - let focused = attrs.states.focused.unwrap_or(false); - let expanded = attrs - .states - .expanded - .or(attrs.states.disclosing) - .unwrap_or(false); - let mut states = Vec::new(); - if focused { - states.push("focused".into()); - } - if !attrs.states.enabled { - states.push("disabled".into()); - } - if expanded { - states.push("expanded".into()); - } - if crate::tree::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) { - states.push("checked".into()); - } - ElementState { +fn element_state_from_attrs( + element: &AXElement, + attrs: crate::tree::NodeAttrs, + role: String, + window_bounds: Option<Rect>, +) -> Result<ElementState, AdapterError> { + let is_secure = attrs.role.as_deref() == Some("AXSecureTextField") + || attrs.subrole.as_deref() == Some("AXSecureTextField"); + let context = crate::tree::state_reader::StateReaderContext { + focused: None, + window_bounds, + is_secure_text: is_secure, + }; + let states = crate::tree::state_reader::states_from_element(element, &attrs, &role, &context); + let enabled = Some(attrs.states.enabled); + let hidden = hidden_state(attrs.states.semantic.hidden); + let offscreen = offscreen(attrs.bounds, window_bounds); + Ok(ElementState { role, states, - value, - } + value: (!is_secure).then_some(attrs.value).flatten(), + enabled, + hidden, + offscreen, + }) } -fn normalized_role(ax_role: Option<&str>) -> String { +fn hidden_state(reported: Option<bool>) -> Option<bool> { + reported +} + +fn offscreen(bounds: Option<Rect>, window: Option<Rect>) -> Option<bool> { + let (bounds, window) = bounds.zip(window)?; + Some( + bounds.x + bounds.width <= window.x + || bounds.x >= window.x + window.width + || bounds.y + bounds.height <= window.y + || bounds.y >= window.y + window.height, + ) +} + +#[cfg(test)] +fn normalized_role(ax_role: Option<&str>, ax_subrole: Option<&str>) -> String { ax_role - .map(crate::tree::roles::ax_role_to_str) + .map(|role| crate::tree::roles::ax_role_and_subrole_to_str(role, ax_subrole)) .unwrap_or("unknown") .to_string() } -fn value_is_checked(value: Option<&str>) -> bool { - matches!(value, Some("1" | "true")) +fn pause_if_budget_allows(deadline: Deadline, delay: Duration) -> bool { + let remaining = deadline.remaining(); + if remaining <= delay { + return false; + } + std::thread::sleep(delay); + !deadline.is_expired() } + +fn deadline_instant(deadline: Deadline) -> Result<Instant, AdapterError> { + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(deadline.timeout_error()); + } + Instant::now() + .checked_add(remaining) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Deadline is out of range")) +} + +fn read_error(attribute: &str, error: i32) -> AdapterError { + AdapterError::new( + if error == accessibility_sys::kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == accessibility_sys::kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == accessibility_sys::kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }, + format!("Could not read {attribute} for live state"), + ) + .with_details(serde_json::json!({ "attribute": attribute, "ax_error": error })) +} + +#[cfg(test)] +#[path = "post_state_tests.rs"] +mod tests; diff --git a/crates/macos/src/actions/post_state_tests.rs b/crates/macos/src/actions/post_state_tests.rs new file mode 100644 index 0000000..4001104 --- /dev/null +++ b/crates/macos/src/actions/post_state_tests.rs @@ -0,0 +1,172 @@ +use super::*; +use crate::tree::{node_attr_states::NodeAttrStates, node_attrs::NodeAttrs}; + +fn attrs_with_bounds(bounds: Rect) -> NodeAttrs { + NodeAttrs { + role: Some("AXButton".into()), + subrole: None, + value: None, + name_evidence: agent_desktop_core::NameEvidence { + native_title: Some("Target".into()), + ..agent_desktop_core::NameEvidence::default() + }, + states: NodeAttrStates::default(), + bounds: Some(bounds), + has_scrollbars: false, + } +} + +#[test] +fn element_state_from_attrs_includes_offscreen_when_window_bounds_supplied() { + let el = crate::tree::AXElement(std::ptr::null_mut()); + let attrs = attrs_with_bounds(Rect { + x: 1000.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + let window = Rect { + x: 0.0, + y: 0.0, + width: 50.0, + height: 50.0, + }; + + let state = element_state_from_attrs(&el, attrs, "button".into(), Some(window)).unwrap(); + + assert!( + state + .states + .contains(&agent_desktop_core::state::OFFSCREEN.to_string()) + ); +} + +#[test] +fn element_state_from_attrs_omits_offscreen_without_window_bounds() { + let el = crate::tree::AXElement(std::ptr::null_mut()); + let attrs = attrs_with_bounds(Rect { + x: 1000.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + + let state = element_state_from_attrs(&el, attrs, "button".into(), None).unwrap(); + + assert!( + !state + .states + .contains(&agent_desktop_core::state::OFFSCREEN.to_string()) + ); +} + +#[test] +fn post_delay_is_skipped_when_it_would_exhaust_the_budget() { + let deadline = Deadline::after(1).unwrap(); + + assert!(!pause_if_budget_allows( + deadline, + std::time::Duration::from_millis(50) + )); +} + +#[test] +fn click_does_not_post_read_a_target_that_navigation_may_detach() { + let element = crate::tree::AXElement(std::ptr::null_mut()); + let state = read_post_state(&element, &Action::Click, Deadline::after(1).unwrap()).unwrap(); + + assert!(state.is_none()); +} + +#[test] +fn post_state_uses_the_same_subrole_mapping_as_snapshot_observation() { + assert_eq!( + normalized_role(Some("AXRow"), Some("AXOutlineRow")), + "treeitem" + ); +} + +#[test] +fn secure_subrole_never_exposes_its_value() { + let el = crate::tree::AXElement(std::ptr::null_mut()); + let mut attrs = attrs_with_bounds(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + attrs.role = Some("AXTextField".into()); + attrs.subrole = Some("AXSecureTextField".into()); + attrs.value = Some("secret".into()); + + let state = element_state_from_attrs(&el, attrs, "textfield".into(), None).unwrap(); + + assert_eq!(state.value, None); +} + +#[test] +fn element_visibility_preserves_live_hidden_evidence_for_every_role() { + assert_eq!(hidden_state(None), None); + assert_eq!(hidden_state(Some(false)), Some(false)); + assert_eq!(hidden_state(Some(true)), Some(true)); +} + +#[test] +fn top_level_container_is_used_only_when_window_is_authoritatively_absent() { + let mut attributes = Vec::new(); + let container = first_owning_container(|attribute| { + attributes.push(attribute); + Ok((attribute == "AXTopLevelUIElement") + .then(|| crate::tree::AXElement(std::ptr::null_mut()))) + }) + .unwrap(); + + assert!(container.is_some()); + assert_eq!(attributes, ["AXWindow", "AXTopLevelUIElement"]); +} + +#[test] +fn incomplete_window_read_never_falls_through_to_a_weaker_container() { + for error in [ + accessibility_sys::kAXErrorCannotComplete, + accessibility_sys::kAXErrorInvalidUIElement, + ] { + let calls = std::cell::Cell::new(0); + let result = first_owning_container(|_| { + calls.set(calls.get() + 1); + Err(error) + }); + + let failure = match result { + Err(failure) => failure, + Ok(_) => panic!("incomplete AXWindow read must fail"), + }; + assert_eq!(failure, ("AXWindow", error)); + assert_eq!(calls.get(), 1); + } +} + +#[test] +fn optional_identity_gaps_do_not_poison_complete_actionability_evidence() { + let mut evidence = agent_desktop_core::LocatorEvidence { + role: LocatorField::Known("scrollarea".into()), + name: LocatorField::Unknown, + description: LocatorField::Unknown, + value: LocatorField::Absent, + identifiers: agent_desktop_core::IdentifierEvidence::unknown(), + states: LocatorField::Known(Vec::new()), + ref_evidence: agent_desktop_core::LocatorRefEvidence { + bounds: LocatorField::Known(Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }), + available_actions: LocatorField::Known(vec!["Scroll".into()]), + }, + }; + + assert!(essential_live_evidence_complete(&evidence)); + evidence.states = LocatorField::Unknown; + assert!(!essential_live_evidence_complete(&evidence)); +} diff --git a/crates/macos/src/actions/scroll.rs b/crates/macos/src/actions/scroll.rs index 3ad9062..6a923e3 100644 --- a/crates/macos/src/actions/scroll.rs +++ b/crates/macos/src/actions/scroll.rs @@ -1,146 +1,42 @@ -#[cfg(target_os = "macos")] use agent_desktop_core::{ - error::{AdapterError, ErrorCode}, - interaction_policy::InteractionPolicy, + AdapterError, Deadline, Direction, ErrorCode, InteractionPolicy, StepMechanism, }; -#[cfg(target_os = "macos")] use crate::tree::AXElement; -#[cfg(target_os = "macos")] +const MAX_SCROLL_AMOUNT: u32 = 1_000; + pub(crate) fn ax_scroll( - el: &AXElement, - direction: &agent_desktop_core::action::Direction, + element: &AXElement, + direction: &Direction, amount: u32, policy: InteractionPolicy, -) -> Result<(), AdapterError> { - use accessibility_sys::{ - AXUIElementPerformAction, AXUIElementSetAttributeValue, kAXErrorSuccess, - }; - use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; - - let scroll_area = find_scroll_area(el); - let target = scroll_area.as_ref().unwrap_or(el); - - let scroll_visible = CFString::new("AXScrollToVisible"); - let scroll_visible_err = - unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) }; - if scroll_visible_err != kAXErrorSuccess { - tracing::debug!( - ax_error = scroll_visible_err, - "AXScrollToVisible returned non-success; continuing to next scroll strategy" - ); + deadline: Deadline, +) -> Result<(StepMechanism, bool), AdapterError> { + validate_amount(amount)?; + let scroll_area = find_scroll_area(element, deadline)?; + let target = scroll_area.as_ref().unwrap_or(element); + if policy.is_headed() { + physical_wheel(target, direction, amount, deadline)?; + return Ok((StepMechanism::PhysicalSynthetic, false)); } + accept_optional_visibility_result(try_action(element, "AXScrollToVisible", deadline))?; + let (bar_attribute, increment_action) = scroll_bar_action(direction); - let (bar_attr, inc_action) = match direction { - Direction::Down => ("AXVerticalScrollBar", "AXIncrement"), - Direction::Up => ("AXVerticalScrollBar", "AXDecrement"), - Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"), - Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"), - }; - - if let Some(bar) = get_scroll_bar(target, bar_attr) { - let ax_action = CFString::new(inc_action); - let mut ok = true; - for _ in 0..amount { - if unsafe { AXUIElementPerformAction(bar.0, ax_action.as_concrete_TypeRef()) } - != kAXErrorSuccess - { - ok = false; - break; - } + if let Some(bar) = crate::actions::scroll_read::element(target, bar_attribute, deadline)? { + if perform_repeated_action(&bar, increment_action, amount, deadline)? { + return Ok((StepMechanism::SemanticApi, false)); } - if ok { - return Ok(()); + if try_value_shift(&bar, direction, amount, deadline)? { + return Ok((StepMechanism::SemanticApi, true)); + } + if try_sub_elements(&bar, direction, amount, deadline)? { + return Ok((StepMechanism::SemanticApi, false)); } } - - let page_action = match direction { - Direction::Down => "AXScrollDownByPage", - Direction::Up => "AXScrollUpByPage", - Direction::Right => "AXScrollRightByPage", - Direction::Left => "AXScrollLeftByPage", - }; - if crate::actions::ax_helpers::has_ax_action(target, page_action) { - let ax = CFString::new(page_action); - let mut completed = 0; - for _ in 0..amount { - if unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) } - != kAXErrorSuccess - { - break; - } - completed += 1; - } - if completed == amount { - return Ok(()); - } + if perform_repeated_action(target, page_action(direction), amount, deadline)? { + return Ok((StepMechanism::SemanticApi, false)); } - - if let Some(bar) = get_scroll_bar(target, bar_attr) { - if try_scroll_bar_value_shift(&bar, direction, amount) { - return Ok(()); - } - if try_scroll_bar_sub_elements(&bar, direction) { - return Ok(()); - } - } - - if policy.allow_focus_steal && try_focus_child_in_direction(target, direction) { - return Ok(()); - } - if policy.allow_focus_steal && try_select_row_in_direction(target, direction) { - return Ok(()); - } - - if policy.allow_focus_steal { - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - let keycode: u16 = match direction { - Direction::Down => 121, - Direction::Up => 116, - Direction::Right => 124, - Direction::Left => 123, - }; - crate::system::app_ops::focus_best_effort(pid); - let cf_focused = CFString::new("AXFocused"); - let focus_err = unsafe { - AXUIElementSetAttributeValue( - target.0, - cf_focused.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if focus_err == kAXErrorSuccess { - std::thread::sleep(std::time::Duration::from_millis(50)); - crate::input::keyboard::synthesize_keycode(keycode, amount)?; - return Ok(()); - } - } - } - - if policy.allow_focus_steal && policy.allow_cursor_move { - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - crate::system::app_ops::focus_best_effort(pid); - if let Some(b) = crate::tree::read_bounds(target) { - let (dy, dx) = scroll_wheel_delta(direction, amount); - return crate::input::mouse::synthesize_scroll_at( - b.x + b.width / 2.0, - b.y + b.height / 2.0, - dy, - dx, - ); - } - } - } - - if policy.allow_focus_steal && !policy.allow_cursor_move { - return Err(AdapterError::policy_denied_for_policy( - "Cursor-moving scroll fallback is disabled by the current interaction policy", - policy, - )); - } - Err(AdapterError::new( ErrorCode::ActionNotSupported, "No scroll mechanism found on element", @@ -148,192 +44,281 @@ pub(crate) fn ax_scroll( .with_suggestion("Element may not be scrollable, or try the parent container.")) } -#[cfg(target_os = "macos")] -fn scroll_wheel_delta( - direction: &agent_desktop_core::action::Direction, - amount: u32, -) -> (i32, i32) { - use agent_desktop_core::action::Direction; - match direction { - Direction::Down => (-(amount as i32) * 5, 0), - Direction::Up => (amount as i32 * 5, 0), - Direction::Right => (0, amount as i32 * 5), - Direction::Left => (0, -(amount as i32) * 5), - } -} - -#[cfg(target_os = "macos")] -fn try_scroll_bar_value_shift( - bar: &AXElement, - direction: &agent_desktop_core::action::Direction, - amount: u32, -) -> bool { - use accessibility_sys::{AXUIElementSetAttributeValue, kAXErrorSuccess}; - use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; - - if !crate::actions::ax_helpers::is_attr_settable(bar, "AXValue") { - return false; - } - let current = read_scroll_bar_value(bar).unwrap_or(0.0); - let delta = 0.1 * amount as f64; - let new_val = match direction { - Direction::Down | Direction::Right => (current + delta).min(1.0), - Direction::Up | Direction::Left => (current - delta).max(0.0), - }; - let cf_num = CFNumber::from(new_val as f32); - let cf_attr = CFString::new("AXValue"); - let err = unsafe { - AXUIElementSetAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), cf_num.as_CFTypeRef()) - }; - err == kAXErrorSuccess -} - -#[cfg(target_os = "macos")] -fn read_scroll_bar_value(bar: &AXElement) -> Option<f64> { - use accessibility_sys::{AXUIElementCopyAttributeValue, kAXErrorSuccess}; - use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; - - let cf_attr = CFString::new("AXValue"); - let mut value: core_foundation::base::CFTypeRef = std::ptr::null_mut(); - let err = - unsafe { AXUIElementCopyAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), &mut value) }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; - cf.downcast::<CFNumber>().and_then(|n| n.to_f64()) -} - -#[cfg(target_os = "macos")] -fn try_scroll_bar_sub_elements( - bar: &AXElement, - direction: &agent_desktop_core::action::Direction, -) -> bool { - use accessibility_sys::{AXUIElementPerformAction, kAXErrorSuccess}; - use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, string::CFString}; - - let children = crate::tree::copy_ax_array(bar, "AXChildren").unwrap_or_default(); - let target_subroles = match direction { - Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"], - Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"], - }; - let press = CFString::new("AXPress"); - for child in &children { - let sr = crate::tree::copy_string_attr(child, "AXSubrole").unwrap_or_default(); - if target_subroles.iter().any(|t| *t == sr) - && unsafe { AXUIElementPerformAction(child.0, press.as_concrete_TypeRef()) } - == kAXErrorSuccess +fn accept_optional_visibility_result( + result: Result<bool, AdapterError>, +) -> Result<(), AdapterError> { + match result { + Ok(_) => Ok(()), + Err(error) + if matches!( + error.code, + ErrorCode::ActionFailed + | ErrorCode::ActionNotSupported + | ErrorCode::AppUnresponsive + ) => { - return true; + Ok(()) } + Err(error) => Err(error), } - false } -#[cfg(target_os = "macos")] -fn try_focus_child_in_direction( - scroll_area: &AXElement, - _direction: &agent_desktop_core::action::Direction, -) -> bool { - use accessibility_sys::{AXUIElementSetAttributeValue, kAXErrorSuccess}; - use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; - - let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); - let child = match children.first() { - Some(c) => c, - None => return false, - }; - let grandchildren = crate::tree::copy_ax_array(child, "AXChildren").unwrap_or_default(); - let target = match grandchildren.last() { - Some(t) => t, - None => return false, - }; - let cf_attr = CFString::new("AXFocused"); - let err = unsafe { - AXUIElementSetAttributeValue( - target.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - err == kAXErrorSuccess +fn validate_amount(amount: u32) -> Result<(), AdapterError> { + if amount == 0 || amount > MAX_SCROLL_AMOUNT { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Scroll amount must be between 1 and 1000", + )); + } + Ok(()) } -#[cfg(target_os = "macos")] -fn try_select_row_in_direction( - scroll_area: &AXElement, - _direction: &agent_desktop_core::action::Direction, -) -> bool { - use accessibility_sys::{AXUIElementSetAttributeValue, kAXErrorSuccess, kAXRoleAttribute}; - use core_foundation::{ - array::CFArray, - base::{CFRetain, CFType, CFTypeRef, TCFType}, - string::CFString, - }; +fn scroll_bar_action(direction: &Direction) -> (&'static str, &'static str) { + match direction { + Direction::Down => ("AXVerticalScrollBar", "AXIncrement"), + Direction::Up => ("AXVerticalScrollBar", "AXDecrement"), + Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"), + Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"), + } +} - let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); - for child in &children { - let role = crate::tree::copy_string_attr(child, kAXRoleAttribute); - if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) { - continue; - } - if !crate::actions::ax_helpers::is_attr_settable(child, "AXSelectedRows") { - continue; - } - let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default(); - if let Some(last) = rows.last() { - unsafe { CFRetain(last.0 as CFTypeRef) }; - let el_as_cftype = unsafe { CFType::wrap_under_create_rule(last.0 as CFTypeRef) }; - let arr = CFArray::from_CFTypes(&[el_as_cftype]); - let cf_attr = CFString::new("AXSelectedRows"); - let err = unsafe { - AXUIElementSetAttributeValue( - child.0, - cf_attr.as_concrete_TypeRef(), - arr.as_CFTypeRef(), - ) - }; - if err == kAXErrorSuccess { - return true; +fn page_action(direction: &Direction) -> &'static str { + match direction { + Direction::Down => "AXScrollDownByPage", + Direction::Up => "AXScrollUpByPage", + Direction::Right => "AXScrollRightByPage", + Direction::Left => "AXScrollLeftByPage", + } +} + +fn perform_repeated_action( + element: &AXElement, + action: &'static str, + amount: u32, + deadline: Deadline, +) -> Result<bool, AdapterError> { + repeat_action(action, amount, || try_action(element, action, deadline)) +} + +fn repeat_action( + action: &'static str, + amount: u32, + mut attempt: impl FnMut() -> Result<bool, AdapterError>, +) -> Result<bool, AdapterError> { + for completed in 0..amount { + match attempt() { + Ok(true) => {} + Ok(false) if completed == 0 => return Ok(false), + Ok(false) => return Err(partial_scroll_error(action, completed, amount, None)), + Err(error) if completed == 0 => return Err(error), + Err(error) => { + return Err(partial_scroll_error(action, completed, amount, Some(error))); } } } - false + Ok(true) } -#[cfg(target_os = "macos")] -fn find_scroll_area(el: &AXElement) -> Option<AXElement> { - use accessibility_sys::kAXRoleAttribute; +fn try_action(element: &AXElement, action: &str, deadline: Deadline) -> Result<bool, AdapterError> { + prepare(element, deadline)?; + let result = crate::actions::ax_helpers::try_ax_action_or_err(element, action, deadline); + ensure_budget(deadline, result.as_ref().is_ok_and(|delivered| *delivered))?; + result +} - let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?; - if role == "AXScrollArea" { - return Some(el.clone()); +fn partial_scroll_error( + action: &str, + completed: u32, + requested: u32, + source: Option<AdapterError>, +) -> AdapterError { + let mut details = serde_json::json!({ + "action_may_have_completed": true, + "completed_steps": completed, + "requested_steps": requested, + }); + if let Some(source) = source.as_ref() { + details["source_code"] = source.code.as_str().into(); + details["source_message"] = source.message.as_str().into(); } - let mut current = crate::tree::copy_element_attr(el, "AXParent")?; - for _ in 0..5 { - let r = crate::tree::copy_string_attr(¤t, kAXRoleAttribute)?; - if r == "AXScrollArea" { - return Some(current); + AdapterError::new( + source.map_or(ErrorCode::ActionFailed, |error| error.code), + format!("{action} stopped after {completed} of {requested} requested scroll steps"), + ) + .with_details(details) + .with_disposition(agent_desktop_core::DeliverySemantics::delivered_unverified()) + .with_suggestion("Inspect the current scroll position before deciding whether to retry.") +} + +fn try_value_shift( + bar: &AXElement, + direction: &Direction, + amount: u32, + deadline: Deadline, +) -> Result<bool, AdapterError> { + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + + let Some(current) = crate::actions::scroll_read::number(bar, "AXValue", deadline)? else { + return Ok(false); + }; + let next = shifted_value(current, direction, amount); + if (next - current).abs() <= f64::EPSILON { + return Ok(false); + } + prepare(bar, deadline)?; + let value = CFNumber::from(next as f32); + let attribute = CFString::new("AXValue"); + let error = crate::tree::ax_ipc::set_attribute_value( + bar, + attribute.as_concrete_TypeRef(), + value.as_CFTypeRef(), + deadline, + )?; + let result = crate::actions::ax_mutation::classify_result( + bar, + "AXValue", + "AXUIElementSetAttributeValue", + error, + ); + ensure_budget(deadline, result.as_ref().is_ok_and(|delivered| *delivered))?; + if !result? { + return Ok(false); + } + let observed = crate::actions::scroll_read::number(bar, "AXValue", deadline) + .map_err(value_write_unverified)? + .ok_or_else(|| value_write_unverified(missing_value_after_write()))?; + if value_shift_verified(current, observed, direction) { + return Ok(true); + } + Err(value_write_unverified(AdapterError::new( + ErrorCode::ActionFailed, + "Scrollbar AXValue write completed without the requested movement", + ))) +} + +fn shifted_value(current: f64, direction: &Direction, amount: u32) -> f64 { + let delta = 0.1 * f64::from(amount); + match direction { + Direction::Down | Direction::Right => (current + delta).min(1.0), + Direction::Up | Direction::Left => (current - delta).max(0.0), + } +} + +fn value_shift_verified(current: f64, observed: f64, direction: &Direction) -> bool { + match direction { + Direction::Down | Direction::Right => observed > current + f64::EPSILON, + Direction::Up | Direction::Left => observed < current - f64::EPSILON, + } +} + +fn missing_value_after_write() -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + "Scrollbar AXValue disappeared after a successful write", + ) +} + +fn value_write_unverified(error: AdapterError) -> AdapterError { + error + .with_disposition(agent_desktop_core::DeliverySemantics::delivered_unverified()) + .with_suggestion("Inspect the current scroll position before deciding whether to retry.") +} + +fn try_sub_elements( + bar: &AXElement, + direction: &Direction, + amount: u32, + deadline: Deadline, +) -> Result<bool, AdapterError> { + let children = crate::actions::scroll_read::children(bar, deadline)?; + let subroles = match direction { + Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"], + Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"], + }; + for child in children { + if crate::actions::scroll_read::string(&child, "AXSubrole", deadline)? + .is_some_and(|subrole| subroles.contains(&subrole.as_str())) + && perform_repeated_action(&child, "AXPress", amount, deadline)? + { + return Ok(true); } - current = crate::tree::copy_element_attr(¤t, "AXParent")?; } - None + Ok(false) } -#[cfg(target_os = "macos")] -fn get_scroll_bar(scroll_area: &AXElement, bar_attr: &str) -> Option<AXElement> { - crate::tree::copy_element_attr(scroll_area, bar_attr) +fn find_scroll_area( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + let mut current = Some(element.clone()); + for _ in 0..=5 { + let Some(candidate) = current else { + return Ok(None); + }; + if crate::actions::scroll_read::string(&candidate, "AXRole", deadline)?.as_deref() + == Some("AXScrollArea") + { + return Ok(Some(candidate)); + } + current = crate::actions::scroll_read::element(&candidate, "AXParent", deadline)?; + } + Ok(None) +} + +fn physical_wheel( + target: &AXElement, + direction: &Direction, + amount: u32, + deadline: Deadline, +) -> Result<(), AdapterError> { + let prepared = + crate::actions::physical_target::PreparedPhysicalTarget::prepare(target, deadline)?; + let bounds = + crate::tree::hit_test::visible_bounds_ax_element(target, deadline)?.ok_or_else(|| { + AdapterError::new( + ErrorCode::ActionFailed, + "Scroll target has no visible delivery area", + ) + })?; + let (vertical, horizontal) = wheel_delta(direction, amount); + let point = agent_desktop_core::Point { + x: bounds.x + bounds.width / 2.0, + y: bounds.y + bounds.height / 2.0, + }; + prepared.verify_pointer(target, &point, deadline)?; + crate::input::mouse_scroll::synthesize_scroll_at(point, (vertical, horizontal), &[], deadline) +} + +fn wheel_delta(direction: &Direction, amount: u32) -> (i32, i32) { + let units = amount.min(i32::MAX as u32) as i32; + match direction { + Direction::Down => (-units, 0), + Direction::Up => (units, 0), + Direction::Right => (0, units), + Direction::Left => (0, -units), + } } #[cfg(test)] -mod tests { - use agent_desktop_core::action::Direction; - - #[test] - fn horizontal_wheel_delta_matches_direction() { - assert_eq!(super::scroll_wheel_delta(&Direction::Right, 2), (0, 10)); - assert_eq!(super::scroll_wheel_delta(&Direction::Left, 2), (0, -10)); - } +fn scroll_wheel_delta(direction: &Direction, amount: u32) -> (i32, i32) { + wheel_delta(direction, amount) } + +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + +fn ensure_budget(deadline: Deadline, delivery_started: bool) -> Result<(), AdapterError> { + if !deadline.is_expired() { + return Ok(()); + } + let mut delivery = crate::actions::DeliveryTracker::default(); + if delivery_started { + delivery.mark_delivered(); + } + Err(delivery.annotate(deadline.timeout_error())) +} + +#[cfg(test)] +#[path = "scroll_tests.rs"] +mod tests; diff --git a/crates/macos/src/actions/scroll_into_view.rs b/crates/macos/src/actions/scroll_into_view.rs new file mode 100644 index 0000000..15f73cb --- /dev/null +++ b/crates/macos/src/actions/scroll_into_view.rs @@ -0,0 +1,218 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::actions::chain_delivery::DeliveryOutcome; + use crate::tree::AXElement; + use agent_desktop_core::{ + AdapterError, Deadline, DeliverySemantics, Direction, ErrorCode, InteractionPolicy, Rect, + }; + use std::time::{Duration, Instant}; + + const MAX_ANCESTOR_SCROLLS: usize = 10; + + pub fn scroll_into_view_impl( + element: &AXElement, + deadline: Deadline, + ) -> Result<(), AdapterError> { + scroll_into_view_outcome(element, deadline).map(|_| ()) + } + + pub(crate) fn scroll_into_view_outcome( + element: &AXElement, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + match scroll_to_verified(element, deadline)? { + outcome @ (DeliveryOutcome::SatisfiedNoDelivery + | DeliveryOutcome::DeliveredVerified) => Ok(outcome), + DeliveryOutcome::NotDelivered => scroll_ancestor_until_visible(element, deadline), + DeliveryOutcome::DeliveredUnverified => Err(AdapterError::new( + ErrorCode::ActionFailed, + "AXScrollToVisible completed without verified visibility", + ) + .with_disposition(DeliverySemantics::delivered_unverified())), + } + } + + fn scroll_ancestor_until_visible( + element: &AXElement, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + for attempt in 0..MAX_ANCESTOR_SCROLLS { + let Some(direction) = direction_to_window(element, deadline)? else { + return Ok(if attempt == 0 { + DeliveryOutcome::SatisfiedNoDelivery + } else { + DeliveryOutcome::DeliveredVerified + }); + }; + crate::actions::scroll::ax_scroll( + element, + &direction, + 1, + InteractionPolicy::headless(), + deadline, + )?; + } + Err(AdapterError::new( + ErrorCode::ActionFailed, + "Semantic ancestor scrolling did not bring the target into view", + ) + .with_disposition(DeliverySemantics::delivered_unverified())) + } + + fn direction_to_window( + element: &AXElement, + deadline: Deadline, + ) -> Result<Option<Direction>, AdapterError> { + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + let bounds = crate::tree::element_bounds::read_bounds_with_deadline(element, instant)? + .ok_or_else(|| AdapterError::new(ErrorCode::ActionFailed, "Target has no bounds"))?; + let window = crate::tree::surface_read::element(element, "AXWindow", instant)? + .ok_or_else(|| AdapterError::new(ErrorCode::ActionFailed, "Target has no window"))?; + let window_bounds = crate::tree::element_bounds::read_bounds_with_deadline( + &window, instant, + )? + .ok_or_else(|| AdapterError::new(ErrorCode::ActionFailed, "Target window has no bounds"))?; + Ok(direction_for_visibility(bounds, window_bounds)) + } + + pub(crate) fn direction_for_visibility(target: Rect, viewport: Rect) -> Option<Direction> { + if target.y < viewport.y { + Some(Direction::Up) + } else if target.y + target.height > viewport.y + viewport.height { + Some(Direction::Down) + } else if target.x < viewport.x { + Some(Direction::Left) + } else if target.x + target.width > viewport.x + viewport.width { + Some(Direction::Right) + } else { + None + } + } + + pub(crate) fn scroll_to_verified( + element: &AXElement, + deadline: Deadline, + ) -> Result<DeliveryOutcome, AdapterError> { + prepare(element, deadline)?; + let before = element_bounds(element, deadline)?; + if !crate::actions::ax_helpers::try_ax_action_or_err( + element, + "AXScrollToVisible", + deadline, + )? { + return Ok(DeliveryOutcome::NotDelivered); + } + let local_end = Instant::now() + Duration::from_millis(800); + loop { + if visible_in_window(element, deadline).map_err(after_delivery)? { + return Ok(DeliveryOutcome::DeliveredVerified); + } + if deadline.is_expired() { + return Err(after_delivery(deadline.timeout_error().with_details( + serde_json::json!({ + "verification": "scroll_visibility_not_observed", + }), + ))); + } + if Instant::now() >= local_end { + if !scroll_effect_observed(before, element_bounds(element, deadline)?) { + return Ok(DeliveryOutcome::NotDelivered); + } + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "AXScrollToVisible completed but target visibility was not verified", + ) + .with_disposition(DeliverySemantics::delivered_unverified())); + } + let pause = deadline + .remaining_slice(Duration::from_millis(20)) + .map_err(after_delivery)?; + std::thread::sleep(pause.min(Duration::from_millis(20))); + } + } + + fn element_bounds( + element: &AXElement, + deadline: Deadline, + ) -> Result<Option<Rect>, AdapterError> { + crate::tree::element_bounds::read_bounds_with_deadline( + element, + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + + pub(crate) fn scroll_effect_observed(before: Option<Rect>, after: Option<Rect>) -> bool { + match (before, after) { + (Some(before), Some(after)) => before.bounds_hash() != after.bounds_hash(), + (None, None) => false, + _ => true, + } + } + + fn visible_in_window(element: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + let Some(bounds) = + crate::tree::element_bounds::read_bounds_with_deadline(element, instant)? + else { + return Ok(false); + }; + let Some(window) = crate::tree::surface_read::element(element, "AXWindow", instant)? else { + return Ok(false); + }; + let Some(window_bounds) = + crate::tree::element_bounds::read_bounds_with_deadline(&window, instant)? + else { + return Ok(false); + }; + Ok(rect_has_area(bounds) && intersects(bounds, window_bounds)) + } + + pub(crate) fn rect_has_area(rect: Rect) -> bool { + rect.x.is_finite() + && rect.y.is_finite() + && rect.width.is_finite() + && rect.height.is_finite() + && rect.width > 0.0 + && rect.height > 0.0 + } + + pub(crate) fn intersects(left: Rect, right: Rect) -> bool { + left.x < right.x + right.width + && left.x + left.width > right.x + && left.y < right.y + right.height + && left.y + left.height > right.y + } + + fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) + } + + fn after_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use crate::tree::AXElement; + use agent_desktop_core::{AdapterError, Deadline}; + + pub fn scroll_into_view_impl( + _element: &AXElement, + _deadline: Deadline, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("scroll_into_view")) + } +} + +pub(crate) use imp::scroll_into_view_impl; + +#[cfg(target_os = "macos")] +pub(crate) use imp::scroll_into_view_outcome; + +#[cfg(all(test, target_os = "macos"))] +pub(crate) use imp::{direction_for_visibility, intersects, rect_has_area, scroll_effect_observed}; + +#[cfg(all(test, target_os = "macos"))] +#[path = "scroll_into_view_tests.rs"] +mod tests; diff --git a/crates/macos/src/actions/scroll_into_view_tests.rs b/crates/macos/src/actions/scroll_into_view_tests.rs new file mode 100644 index 0000000..30cc470 --- /dev/null +++ b/crates/macos/src/actions/scroll_into_view_tests.rs @@ -0,0 +1,58 @@ +use agent_desktop_core::{Direction, Rect}; + +use super::{direction_for_visibility, intersects, rect_has_area, scroll_effect_observed}; + +fn rect(x: f64, y: f64, width: f64, height: f64) -> Rect { + Rect { + x, + y, + width, + height, + } +} + +#[test] +fn area_requires_finite_positive_dimensions_and_coordinates() { + assert!(rect_has_area(rect(0.0, 0.0, 10.0, 10.0))); + assert!(!rect_has_area(rect(0.0, 0.0, 0.0, 10.0))); + assert!(!rect_has_area(rect(0.0, 0.0, -1.0, 10.0))); + assert!(!rect_has_area(rect(f64::NAN, 0.0, 10.0, 10.0))); + assert!(!rect_has_area(rect(0.0, 0.0, f64::INFINITY, 10.0))); +} + +#[test] +fn intersection_requires_positive_overlapping_area() { + let window = rect(0.0, 0.0, 100.0, 100.0); + + assert!(intersects(rect(10.0, 10.0, 20.0, 20.0), window)); + assert!(intersects(rect(90.0, 90.0, 20.0, 20.0), window)); + assert!(intersects(rect(80.0, 10.0, 20.0, 20.0), window)); + assert!(!intersects(rect(101.0, 10.0, 20.0, 20.0), window)); + assert!(!intersects(rect(100.0, 10.0, 20.0, 20.0), window)); +} + +#[test] +fn offscreen_direction_uses_global_viewport_edges() { + let viewport = rect(1496.0, 87.0, 1496.0, 937.0); + + assert!(matches!( + direction_for_visibility(rect(2030.0, 1026.0, 73.0, 24.0), viewport), + Some(Direction::Down) + )); + assert!(matches!( + direction_for_visibility(rect(2030.0, 80.0, 73.0, 24.0), viewport), + Some(Direction::Up) + )); + assert!(direction_for_visibility(rect(1500.0, 100.0, 73.0, 24.0), viewport).is_none()); +} + +#[test] +fn acknowledged_scroll_without_geometry_change_is_not_delivery() { + let before = rect(2622.0, 1063.0, 177.0, 24.0); + + assert!(!scroll_effect_observed(Some(before), Some(before))); + assert!(scroll_effect_observed( + Some(before), + Some(rect(2622.0, 900.0, 177.0, 24.0)) + )); +} diff --git a/crates/macos/src/actions/scroll_read.rs b/crates/macos/src/actions/scroll_read.rs new file mode 100644 index 0000000..c32d7d0 --- /dev/null +++ b/crates/macos/src/actions/scroll_read.rs @@ -0,0 +1,114 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; + +use crate::tree::AXElement; + +const MAX_SCROLL_CHILDREN: usize = 128; + +pub(crate) fn element( + source: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + prepare(source, deadline)?; + let result = crate::tree::attributes::copy_element_attr_result(source, attribute, deadline); + ensure_budget(deadline)?; + result.map_err(|error| read_error(attribute, error)) +} + +pub(crate) fn children( + element: &AXElement, + deadline: Deadline, +) -> Result<Vec<AXElement>, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_ax_array_prefix_result( + element, + "AXChildren", + MAX_SCROLL_CHILDREN, + deadline, + ); + ensure_budget(deadline)?; + let children = result + .map(|children| children.unwrap_or_default()) + .map_err(|error| read_error("AXChildren", error))?; + if children.len() == MAX_SCROLL_CHILDREN { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Scroll sub-element lookup reached its bounded child limit", + ) + .with_details(serde_json::json!({ "complete": false }))); + } + Ok(children) +} + +pub(crate) fn string( + element: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Option<String>, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_string_attr_result(element, attribute, deadline); + ensure_budget(deadline)?; + result.map_err(|error| read_error(attribute, error)) +} + +pub(crate) fn number( + element: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Option<f64>, AdapterError> { + use accessibility_sys::kAXErrorSuccess; + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + + prepare(element, deadline)?; + let attribute = CFString::new(attribute); + let (error, value) = crate::tree::ax_ipc::copy_attribute_value( + element, + attribute.as_concrete_TypeRef(), + deadline, + ); + ensure_budget(deadline)?; + if error != kAXErrorSuccess { + return if error == accessibility_sys::kAXErrorNoValue + || error == accessibility_sys::kAXErrorAttributeUnsupported + { + Ok(None) + } else { + Err(read_error("AXValue", error)) + }; + } + if value.is_null() { + return Ok(None); + } + let value = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; + Ok(value + .downcast::<CFNumber>() + .and_then(|number| number.to_f64())) +} + +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn read_error(attribute: &str, error: i32) -> AdapterError { + AdapterError::new( + if error == accessibility_sys::kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == accessibility_sys::kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == accessibility_sys::kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }, + format!("Accessibility read failed for {attribute} during scroll"), + ) + .with_details(serde_json::json!({ "attribute": attribute, "ax_error": error })) +} diff --git a/crates/macos/src/actions/scroll_tests.rs b/crates/macos/src/actions/scroll_tests.rs new file mode 100644 index 0000000..51d23a5 --- /dev/null +++ b/crates/macos/src/actions/scroll_tests.rs @@ -0,0 +1,125 @@ +use std::cell::Cell; + +use agent_desktop_core::{AdapterError, Direction, ErrorCode}; + +use super::repeat_action; + +#[test] +fn optional_visibility_pre_step_ignores_recoverable_ax_failures() { + for code in [ + ErrorCode::ActionFailed, + ErrorCode::ActionNotSupported, + ErrorCode::AppUnresponsive, + ] { + let result = + super::accept_optional_visibility_result(Err(AdapterError::new(code.clone(), "ax"))); + assert!( + result.is_ok(), + "{code:?} should not abort the real scroll path" + ); + } +} + +#[test] +fn optional_visibility_pre_step_preserves_terminal_failures() { + for code in [ + ErrorCode::PermDenied, + ErrorCode::StaleRef, + ErrorCode::Timeout, + ErrorCode::Internal, + ] { + let error = + super::accept_optional_visibility_result(Err(AdapterError::new(code.clone(), "ax"))) + .expect_err("terminal pre-step failure must remain visible"); + assert_eq!(error.code, code); + } +} + +#[test] +fn horizontal_wheel_delta_matches_direction() { + assert_eq!(super::scroll_wheel_delta(&Direction::Right, 2), (0, 2)); + assert_eq!(super::scroll_wheel_delta(&Direction::Left, 2), (0, -2)); +} + +#[test] +fn wheel_delta_saturates_without_integer_wraparound() { + assert_eq!( + super::scroll_wheel_delta(&Direction::Down, u32::MAX), + (-i32::MAX, 0) + ); +} + +#[test] +fn semantic_scroll_actions_and_value_fallback_preserve_direction() { + assert_eq!( + super::scroll_bar_action(&Direction::Down), + ("AXVerticalScrollBar", "AXIncrement") + ); + assert_eq!( + super::scroll_bar_action(&Direction::Up), + ("AXVerticalScrollBar", "AXDecrement") + ); + assert_eq!(super::page_action(&Direction::Down), "AXScrollDownByPage"); + assert_eq!(super::page_action(&Direction::Up), "AXScrollUpByPage"); + assert!((super::shifted_value(0.0, &Direction::Down, 1) - 0.1).abs() < f64::EPSILON); + assert!(super::shifted_value(0.1, &Direction::Up, 1).abs() < f64::EPSILON); + assert!(super::value_shift_verified(0.2, 0.3, &Direction::Down)); + assert!(super::value_shift_verified(0.3, 0.2, &Direction::Up)); + assert!(super::value_shift_verified(0.2, 0.3, &Direction::Right)); + assert!(super::value_shift_verified(0.3, 0.2, &Direction::Left)); + assert!(!super::value_shift_verified(0.2, 0.2, &Direction::Down)); + assert!(!super::value_shift_verified(0.2, 0.1, &Direction::Down)); +} + +#[test] +fn repeated_scroll_stops_after_partial_definite_failure() { + let calls = Cell::new(0); + let error = repeat_action("AXIncrement", 3, || { + calls.set(calls.get() + 1); + Ok(calls.get() == 1) + }) + .unwrap_err(); + + assert_eq!(calls.get(), 2); + assert_eq!(error.code, ErrorCode::ActionFailed); + assert_eq!( + error.details.as_ref().unwrap()["action_may_have_completed"], + true + ); + assert_eq!(error.details.as_ref().unwrap()["completed_steps"], 1); + assert_eq!(error.details.as_ref().unwrap()["requested_steps"], 3); +} + +#[test] +fn repeated_scroll_allows_fallback_only_before_any_delivery() { + let calls = Cell::new(0); + let delivered = repeat_action("AXIncrement", 3, || { + calls.set(calls.get() + 1); + Ok(false) + }) + .unwrap(); + + assert!(!delivered); + assert_eq!(calls.get(), 1); +} + +#[test] +fn repeated_scroll_propagates_uncertain_mutation_without_another_attempt() { + let calls = Cell::new(0); + let error = repeat_action("AXIncrement", 3, || { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(AdapterError::new(ErrorCode::AppUnresponsive, "uncertain") + .with_details(serde_json::json!({ "action_may_have_completed": true }))); + } + Ok(true) + }) + .unwrap_err(); + + assert_eq!(calls.get(), 2); + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!( + error.details.as_ref().unwrap()["action_may_have_completed"], + true + ); +} diff --git a/crates/macos/src/actions/select_menu.rs b/crates/macos/src/actions/select_menu.rs new file mode 100644 index 0000000..914bfd6 --- /dev/null +++ b/crates/macos/src/actions/select_menu.rs @@ -0,0 +1,358 @@ +use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics, ErrorCode}; + +use crate::tree::AXElement; + +const MAX_SELECT_NODES: usize = 2_048; +const MAX_SELECT_DEPTH: u8 = 8; + +pub(crate) fn select_from_menu( + element: &AXElement, + value: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + let pid = require_pid(element, deadline)?; + let identity = require_identity(pid)?; + if menu_root(pid, deadline)?.is_some() { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "Select refused to reuse a menu that was already open", + ) + .with_suggestion("Dismiss the open menu and retry select.")); + } + let exposes_value = crate::tree::copy_value_typed(element, deadline).is_some(); + let delivered = open_menu(element, deadline)?; + if !delivered { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "The target did not support AXShowMenu or AXPress", + ) + .with_suggestion("Use click to open the control, then snapshot the menu.")); + } + let result = select_open_menu_item(pid, identity, value, deadline); + match result { + Ok(_) if exposes_value => { + verify_selected_value(element, value, deadline).map_err(after_menu_delivery) + } + Ok(verified) => Ok(verified), + Err(error) => { + let _ = crate::actions::ax_helpers::try_ax_action_or_err(element, "AXCancel", deadline); + Err(error.with_disposition(DeliverySemantics::delivered_unverified())) + } + } +} + +fn verify_selected_value( + element: &AXElement, + expected: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + let local_end = std::time::Instant::now() + std::time::Duration::from_millis(600); + loop { + prepare(element, deadline)?; + if crate::tree::copy_value_typed(element, deadline) + .as_deref() + .is_some_and(|value| value.eq_ignore_ascii_case(expected)) + { + return Ok(true); + } + if deadline.is_expired() { + return Err(deadline.timeout_error().with_details(serde_json::json!({ + "kind": "select_value_not_observed", + "complete": false, + }))); + } + if std::time::Instant::now() >= local_end { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "The menu item was activated but the control value did not change", + )); + } + let pause = deadline.remaining_slice(std::time::Duration::from_millis(25))?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(25))); + } +} + +fn after_menu_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) +} + +pub(crate) fn select_collection_item( + element: &AXElement, + value: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + let Some(candidate) = find_named_descendant(element, value, deadline)? else { + return Err(AdapterError::new( + ErrorCode::ElementNotFound, + format!( + "No collection item matched the requested value ({} chars)", + value.chars().count() + ), + ) + .with_suggestion("Use find to inspect the collection's available items.")); + }; + select_collection_candidate(&candidate, deadline) +} + +fn open_menu(element: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + prepare(element, deadline)?; + if crate::actions::ax_helpers::try_ax_action_or_err(element, "AXShowMenu", deadline)? { + return Ok(true); + } + prepare(element, deadline)?; + crate::actions::ax_helpers::try_ax_action_or_err(element, "AXPress", deadline) +} + +fn select_open_menu_item( + pid: i32, + identity: crate::system::process_identity::ProcessIdentity, + value: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + loop { + if !identity.still_matches()? { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "The target process changed while selecting a menu item", + )); + } + if let Some(menu) = menu_root(pid, deadline)? + && let Some(candidate) = find_named_descendant(&menu, value, deadline)? + { + let verified = activate_menu_item(&candidate, deadline)?; + return if verified { + Ok(true) + } else { + wait_for_menu_to_close(pid, deadline) + }; + } + if deadline.is_expired() { + return Err(deadline.timeout_error().with_details(serde_json::json!({ + "kind": "select_menu_item_not_found", + "requested_chars": value.chars().count(), + "complete": false, + }))); + } + let pause = deadline.remaining_slice(std::time::Duration::from_millis(25))?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(25))); + } +} + +fn find_named_descendant( + root: &AXElement, + value: &str, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + let mut stack = vec![(root.clone(), 0_u8)]; + let mut visited = 0_usize; + while let Some((candidate, depth)) = stack.pop() { + visited = visited.saturating_add(1); + if visited > MAX_SELECT_NODES { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Select search exceeded its accessibility-node budget", + ) + .with_details(serde_json::json!({ + "kind": "select_node_limit", + "limit": MAX_SELECT_NODES, + "complete": false, + }))); + } + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + if candidate_matches(&candidate, value, instant)? { + return Ok(Some(candidate)); + } + if depth >= MAX_SELECT_DEPTH { + continue; + } + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + stack.extend( + crate::tree::surface_read::elements(&candidate, "AXChildren", instant)? + .into_iter() + .rev() + .map(|child| (child, depth.saturating_add(1))), + ); + } + Ok(None) +} + +fn candidate_matches( + candidate: &AXElement, + value: &str, + deadline: std::time::Instant, +) -> Result<bool, AdapterError> { + for attribute in ["AXTitle", "AXDescription"] { + if crate::tree::surface_read::string(candidate, attribute, deadline)? + .as_deref() + .is_some_and(|text| text.eq_ignore_ascii_case(value)) + { + return Ok(true); + } + } + Ok(false) +} + +fn activate_menu_item(candidate: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + deliver_candidate(false, || Ok(None), || press_candidate(candidate, deadline)) +} + +fn select_collection_candidate( + candidate: &AXElement, + deadline: Deadline, +) -> Result<bool, AdapterError> { + deliver_candidate( + true, + || select_candidate_attribute(candidate, deadline), + || press_candidate(candidate, deadline), + ) +} + +fn deliver_candidate( + allow_selected_attribute: bool, + mut select_attribute: impl FnMut() -> Result<Option<bool>, AdapterError>, + mut press: impl FnMut() -> Result<bool, AdapterError>, +) -> Result<bool, AdapterError> { + let selected_error = if allow_selected_attribute { + match select_attribute() { + Ok(Some(verified)) => return Ok(verified), + Ok(None) => None, + Err(error) => Some(error), + } + } else { + None + }; + if press()? { + return Ok(false); + } + if let Some(error) = selected_error { + return Err(error); + } + Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "The matching selection item did not support AXSelected or AXPress", + ) + .with_disposition(DeliverySemantics::not_delivered())) +} + +fn select_candidate_attribute( + candidate: &AXElement, + deadline: Deadline, +) -> Result<Option<bool>, AdapterError> { + prepare(candidate, deadline)?; + if !crate::actions::ax_helpers::is_attr_settable(candidate, "AXSelected", deadline)? { + return Ok(None); + } + prepare(candidate, deadline)?; + if !crate::actions::ax_helpers::set_ax_bool_or_err(candidate, "AXSelected", true, deadline)? { + return Ok(None); + } + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + Ok(Some( + crate::tree::surface_read::boolean(candidate, "AXSelected", instant)? == Some(true), + )) +} + +fn press_candidate(candidate: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + prepare(candidate, deadline)?; + crate::actions::ax_helpers::try_ax_action_or_err(candidate, "AXPress", deadline) +} + +fn wait_for_menu_to_close(pid: i32, deadline: Deadline) -> Result<bool, AdapterError> { + let local_end = std::time::Instant::now() + std::time::Duration::from_millis(600); + loop { + if menu_root(pid, deadline)?.is_none() { + return Ok(true); + } + if deadline.is_expired() || std::time::Instant::now() >= local_end { + return Ok(false); + } + let pause = deadline.remaining_slice(std::time::Duration::from_millis(25))?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(25))); + } +} + +fn menu_root(pid: i32, deadline: Deadline) -> Result<Option<AXElement>, AdapterError> { + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + crate::tree::surfaces::menu_element_for_pid(pid, instant) +} + +fn require_pid(element: &AXElement, deadline: Deadline) -> Result<i32, AdapterError> { + crate::system::app_ops::pid_from_element(element, deadline).ok_or_else(|| { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Could not determine the target process for menu selection", + ) + }) +} + +fn require_identity( + pid: i32, +) -> Result<crate::system::process_identity::ProcessIdentity, AdapterError> { + crate::system::process_identity::ProcessIdentity::capture(pid)?.ok_or_else(|| { + AdapterError::new( + ErrorCode::AppUnresponsive, + "The target process exited before menu selection", + ) + }) +} + +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + #[test] + fn select_traversal_limits_are_bounded() { + assert_eq!(MAX_SELECT_NODES, 2_048); + assert_eq!(MAX_SELECT_DEPTH, 8); + } + + #[test] + fn menu_items_are_pressed_without_setting_selected_first() { + let selected_calls = Cell::new(0); + let press_calls = Cell::new(0); + + let verified = deliver_candidate( + false, + || { + selected_calls.set(selected_calls.get() + 1); + Ok(Some(true)) + }, + || { + press_calls.set(press_calls.get() + 1); + Ok(true) + }, + ) + .expect("menu candidate delivery"); + + assert!(!verified); + assert_eq!(selected_calls.get(), 0); + assert_eq!(press_calls.get(), 1); + } + + #[test] + fn collection_selection_falls_back_to_press_when_selected_write_fails() { + let press_calls = Cell::new(0); + let verified = deliver_candidate( + true, + || { + Err(AdapterError::new( + ErrorCode::ActionFailed, + "AXSelected is unavailable", + )) + }, + || { + press_calls.set(press_calls.get() + 1); + Ok(true) + }, + ) + .expect("press fallback"); + + assert!(!verified); + assert_eq!(press_calls.get(), 1); + } +} diff --git a/crates/macos/src/actions/toggle_state.rs b/crates/macos/src/actions/toggle_state.rs index 07d2d61..caa1b0c 100644 --- a/crates/macos/src/actions/toggle_state.rs +++ b/crates/macos/src/actions/toggle_state.rs @@ -1,24 +1,26 @@ use agent_desktop_core::{ - error::{AdapterError, ErrorCode}, - interaction_policy::InteractionPolicy, + ActionStep, AdapterError, Deadline, DeliverySemantics, ErrorCode, InteractionPolicy, + StepMechanism, }; use crate::{ actions::{ ax_helpers, chain::{ChainContext, execute_chain}, - chain_defs, discovery, + chain_defs, }, tree::AXElement, }; -const DEFAULT_TOGGLE_TIMEOUT_MS: u64 = 600; -const MAX_TOGGLE_TIMEOUT_MS: u64 = 10_000; -const DEFAULT_TOGGLE_STABLE_MS: u64 = 200; -const MAX_TOGGLE_STABLE_MS: u64 = 2_000; +const TOGGLE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(600); +const TOGGLE_STABLE: std::time::Duration = std::time::Duration::from_millis(200); -pub(crate) fn toggle(el: &AXElement, policy: InteractionPolicy) -> Result<(), AdapterError> { - let role = ax_helpers::element_role(el); +pub(crate) fn toggle( + el: &AXElement, + policy: InteractionPolicy, + deadline: Deadline, +) -> Result<Vec<ActionStep>, AdapterError> { + let role = read_role(el, deadline)?; if !role .as_deref() .is_some_and(crate::tree::roles::is_toggleable_role) @@ -34,25 +36,30 @@ pub(crate) fn toggle(el: &AXElement, policy: InteractionPolicy) -> Result<(), Ad "Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.", )); } - let before = crate::tree::copy_value_typed(el); - let caps = discovery::discover(el); + let before = read_value(el, deadline)?; let ctx = ChainContext { dynamic_value: None, - deadline: None, + verified_point: None, + deadline, }; - execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx, policy)?; - if let Some(before) = before { - wait_for_value_change(el, &before)?; - } - Ok(()) + let mut steps = execute_chain(el, &chain_defs::SEMANTIC_CLICK_CHAIN, &ctx, policy)?; + let verified = if let Some(before) = before { + wait_for_value_change(el, &before, deadline).map_err(after_delivery)?; + true + } else { + false + }; + mark_last_verified(&mut steps, verified); + Ok(steps) } pub(crate) fn check_uncheck( el: &AXElement, want_checked: bool, policy: InteractionPolicy, -) -> Result<(), AdapterError> { - let role = ax_helpers::element_role(el); + deadline: Deadline, +) -> Result<Vec<ActionStep>, AdapterError> { + let role = read_role(el, deadline)?; if !role .as_deref() .is_some_and(crate::tree::roles::is_toggleable_role) @@ -66,26 +73,50 @@ pub(crate) fn check_uncheck( ) .with_suggestion("Only works on checkboxes, switches, and radio buttons.")); } - if checked_state(el) == Some(want_checked) { - return Ok(()); + if checked_state(el, deadline)? == Some(want_checked) { + return Ok(vec![already_in_state_step()]); } - if ax_helpers::is_attr_settable(el, "AXValue") - && ax_helpers::set_ax_bool(el, "AXValue", want_checked) - && wait_for_checked_state(el, want_checked).is_ok() - { - return Ok(()); + prepare(el, deadline)?; + if ax_helpers::is_attr_settable(el, "AXValue", deadline)? && { + prepare(el, deadline)?; + ax_helpers::set_ax_bool_or_err(el, "AXValue", want_checked, deadline)? + } { + wait_for_checked_state(el, want_checked, deadline).map_err(after_delivery)?; + return Ok(vec![ + ActionStep::succeeded("AXValue") + .with_mechanism(StepMechanism::SemanticApi) + .with_verified(true), + ]); } - let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, - deadline: None, + verified_point: None, + deadline, }; - execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx, policy)?; - wait_for_checked_state(el, want_checked) + let mut steps = execute_chain(el, &chain_defs::SEMANTIC_CLICK_CHAIN, &ctx, policy)?; + wait_for_checked_state(el, want_checked, deadline).map_err(after_delivery)?; + mark_last_verified(&mut steps, true); + Ok(steps) } -fn checked_state(el: &AXElement) -> Option<bool> { - crate::tree::copy_value_typed(el).and_then(|value| parse_checked_value(&value)) +fn already_in_state_step() -> ActionStep { + ActionStep::skipped("AlreadyInState").with_verified(true) +} + +fn after_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) +} + +fn mark_last_verified(steps: &mut [ActionStep], verified: bool) { + if let Some(step) = steps.last_mut() { + if verified || step.verified.is_none() { + step.verified = Some(verified); + } + } +} + +fn checked_state(el: &AXElement, deadline: Deadline) -> Result<Option<bool>, AdapterError> { + Ok(read_value(el, deadline)?.and_then(|value| parse_checked_value(&value))) } fn parse_checked_value(value: &str) -> Option<bool> { @@ -97,10 +128,14 @@ fn parse_checked_value(value: &str) -> Option<bool> { } } -fn wait_for_checked_state(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> { - let deadline = std::time::Instant::now() + toggle_timeout(); +fn wait_for_checked_state( + el: &AXElement, + want_checked: bool, + action_deadline: Deadline, +) -> Result<(), AdapterError> { + let deadline = verification_deadline(action_deadline)?; loop { - if checked_state(el) == Some(want_checked) { + if checked_state(el, action_deadline)? == Some(want_checked) { return Ok(()); } if std::time::Instant::now() >= deadline { @@ -108,22 +143,30 @@ fn wait_for_checked_state(el: &AXElement, want_checked: bool) -> Result<(), Adap ErrorCode::ActionFailed, "check/uncheck did not reach the requested state", ) - .with_suggestion("Retry after refreshing the snapshot.")); + .with_details(serde_json::json!({ + "verification": "requested_checked_state_not_observed" + })) + .with_suggestion( + "Refresh the snapshot and inspect the checked state before deciding whether to retry.", + )); } - std::thread::sleep(std::time::Duration::from_millis(25)); + sleep_poll(deadline, action_deadline)?; } } -fn wait_for_value_change(el: &AXElement, before: &str) -> Result<(), AdapterError> { - let deadline = std::time::Instant::now() + toggle_timeout(); - let stable_for = toggle_stable_duration(); +fn wait_for_value_change( + el: &AXElement, + before: &str, + action_deadline: Deadline, +) -> Result<(), AdapterError> { + let deadline = verification_deadline(action_deadline)?; let mut candidate: Option<(String, std::time::Instant)> = None; loop { - if let Some(changed) = crate::tree::copy_value_typed(el) { + if let Some(changed) = read_value(el, action_deadline)? { if changed != before { match &mut candidate { Some((candidate_value, since)) if candidate_value == &changed => { - if since.elapsed() >= stable_for { + if since.elapsed() >= TOGGLE_STABLE { return Ok(()); } } @@ -140,41 +183,83 @@ fn wait_for_value_change(el: &AXElement, before: &str) -> Result<(), AdapterErro ErrorCode::ActionFailed, "toggle did not change the element value", ) - .with_suggestion("Use 'click' for controls that do not expose stable toggle state.")); + .with_details(serde_json::json!({ + "verification": "stable_value_change_not_observed" + })) + .with_suggestion( + "Refresh the snapshot and inspect the value before deciding whether to retry or use 'click'.", + )); } - std::thread::sleep(std::time::Duration::from_millis(25)); + sleep_poll(deadline, action_deadline)?; } } -fn toggle_timeout() -> std::time::Duration { - env_duration_ms( - "AGENT_DESKTOP_TOGGLE_TIMEOUT_MS", - DEFAULT_TOGGLE_TIMEOUT_MS, - MAX_TOGGLE_TIMEOUT_MS, - ) +fn verification_deadline(action_deadline: Deadline) -> Result<std::time::Instant, AdapterError> { + let local = std::time::Instant::now() + TOGGLE_TIMEOUT; + let remaining = action_deadline.remaining(); + if remaining.is_zero() { + Err(action_deadline.timeout_error()) + } else { + Ok(std::time::Instant::now() + .checked_add(remaining) + .map_or(local, |deadline| deadline.min(local))) + } } -fn toggle_stable_duration() -> std::time::Duration { - env_duration_ms( - "AGENT_DESKTOP_TOGGLE_STABLE_MS", - DEFAULT_TOGGLE_STABLE_MS, - MAX_TOGGLE_STABLE_MS, - ) +fn sleep_poll(deadline: std::time::Instant, action_deadline: Deadline) -> Result<(), AdapterError> { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if !remaining.is_zero() { + std::thread::sleep(remaining.min(std::time::Duration::from_millis(25))); + } + if action_deadline.is_expired() { + Err(action_deadline + .timeout_error() + .with_details(serde_json::json!({ + "verification": "action_deadline_elapsed", + }))) + } else { + Ok(()) + } } -fn env_duration_ms(name: &str, default_ms: u64, max_ms: u64) -> std::time::Duration { - let ms = std::env::var(name) - .ok() - .and_then(|raw| raw.parse::<u64>().ok()) - .filter(|ms| *ms > 0) - .map(|ms| ms.min(max_ms)) - .unwrap_or(default_ms); - std::time::Duration::from_millis(ms) +fn read_value(el: &AXElement, deadline: Deadline) -> Result<Option<String>, AdapterError> { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + + crate::tree::attributes::set_messaging_timeout(el, deadline)?; + let result = crate::tree::attributes::copy_value_typed_result(el, deadline); + if deadline.is_expired() { + return Err(deadline.timeout_error()); + } + result.map_err(|error| { + let code = if error == kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }; + AdapterError::new(code, "Could not verify the live toggle value") + .with_details(serde_json::json!({ "ax_error": error })) + }) +} + +fn read_role(el: &AXElement, deadline: Deadline) -> Result<Option<String>, AdapterError> { + ax_helpers::element_role(el, deadline) +} + +fn prepare(el: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(el, deadline) } #[cfg(test)] mod tests { - use super::parse_checked_value; + use agent_desktop_core::action_step::ActionStep; + + use super::{already_in_state_step, mark_last_verified, parse_checked_value}; #[test] fn parses_checked_values_from_common_ax_strings() { @@ -192,4 +277,29 @@ mod tests { assert_eq!(parse_checked_value(value), None); } } + + #[test] + fn absent_toggle_state_does_not_erase_existing_verification() { + let mut steps = vec![ActionStep::succeeded("verified_press").with_verified(true)]; + mark_last_verified(&mut steps, false); + assert_eq!(steps[0].verified(), Some(true)); + } + + #[test] + fn toggle_state_verification_upgrades_an_unverified_step() { + let mut steps = vec![ActionStep::succeeded("AXPress").with_verified(false)]; + mark_last_verified(&mut steps, true); + assert_eq!(steps[0].verified(), Some(true)); + } + + #[test] + fn already_in_state_is_verified_without_claiming_delivery() { + let step = already_in_state_step(); + assert!(matches!( + step.outcome, + agent_desktop_core::action_step_outcome::ActionStepOutcome::Skipped + )); + assert!(step.mechanism().is_none()); + assert_eq!(step.verified(), Some(true)); + } } diff --git a/crates/macos/src/actions/type_text.rs b/crates/macos/src/actions/type_text.rs index 4e7f47b..7205a15 100644 --- a/crates/macos/src/actions/type_text.rs +++ b/crates/macos/src/actions/type_text.rs @@ -1,187 +1,108 @@ -#[cfg(target_os = "macos")] use agent_desktop_core::{ - error::{AdapterError, ErrorCode}, - interaction_policy::InteractionPolicy, + ActionStep, AdapterError, Deadline, DeliverySemantics, ErrorCode, InteractionPolicy, + StepMechanism, }; -#[cfg(target_os = "macos")] use crate::tree::AXElement; -#[cfg(target_os = "macos")] pub(crate) fn execute_type( - el: &AXElement, + element: &AXElement, text: &str, policy: InteractionPolicy, -) -> Result<(), AdapterError> { - match type_via_ax_value(el, text) { - Ok(()) => return Ok(()), - Err(err) if !policy.allow_focus_steal => return Err(err), - Err(_) => {} - } - - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - let _ = crate::system::app_ops::ensure_app_focused(pid); - } - crate::actions::ax_helpers::ax_focus_or_err(el)?; - std::thread::sleep(std::time::Duration::from_millis(50)); - if !text.is_ascii() { - return type_via_clipboard_paste(el, text); - } - crate::input::keyboard::synthesize_text(text) -} - -/// Restores the user's clipboard on every scope exit — success, error, or -/// panic — so a failure or early return mid-paste cannot leave the pasted text -/// behind. (A SIGKILL between set and restore is unpreventable in any process.) -#[cfg(target_os = "macos")] -struct ClipboardRestore { - previous: crate::input::clipboard::ClipboardSnapshot, -} - -#[cfg(target_os = "macos")] -impl Drop for ClipboardRestore { - fn drop(&mut self) { - if let Err(e) = self.previous.restore() { - tracing::warn!(error = %e, "clipboard restore failed after type_text; prior clipboard may not be restored"); - } - } -} - -#[cfg(target_os = "macos")] -fn type_via_clipboard_paste(el: &AXElement, text: &str) -> Result<(), AdapterError> { - let before = readable_value(el); - let previous = crate::input::clipboard::ClipboardSnapshot::capture()?; - let _restore = ClipboardRestore { previous }; - crate::input::clipboard::set(text)?; - std::thread::sleep(std::time::Duration::from_millis(50)); - let combo = agent_desktop_core::action::KeyCombo { - key: "v".into(), - modifiers: vec![agent_desktop_core::action::Modifier::Cmd], - }; - crate::input::keyboard::synthesize_key(&combo)?; - std::thread::sleep(std::time::Duration::from_millis(100)); - verify_paste_effect(before.as_deref(), readable_value(el).as_deref()) -} - -#[cfg(target_os = "macos")] -fn type_via_ax_value(el: &AXElement, text: &str) -> Result<(), AdapterError> { - if !is_text_target(el) || !crate::actions::ax_helpers::is_attr_settable(el, "AXValue") { - return Err(AdapterError::policy_denied( - "Headless typing requires a settable text value; use set-value or an explicit focus command", + deadline: Deadline, +) -> Result<ActionStep, AdapterError> { + if !is_text_target(element, deadline)? { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "Type requires a text field, secure text field, or combo box", )); } + if policy.is_headed() { + crate::actions::physical_keyboard::type_text(element, text, policy, deadline)?; + return Ok(ActionStep::succeeded("PIDTargetedUnicodeText") + .with_mechanism(StepMechanism::PhysicalSynthetic) + .with_verified(false)); + } + insert_selected_text(element, text, deadline)?; + Ok(ActionStep::succeeded("AXSelectedText") + .with_mechanism(StepMechanism::SemanticApi) + .with_verified(false)) +} - let current = if is_secure_text_field(el) { - String::new() - } else { - crate::tree::copy_value_typed(el).unwrap_or_default() - }; - let next = typed_value(¤t, text); - crate::actions::ax_helpers::ax_set_value(el, &next)?; - if is_secure_text_field(el) { - return Ok(()); +fn insert_selected_text( + element: &AXElement, + text: &str, + deadline: Deadline, +) -> Result<(), AdapterError> { + prepare(element, deadline)?; + write_selected_text(text, |attribute, value| { + crate::actions::ax_helpers::set_ax_string_or_err(element, attribute, value, deadline) + })?; + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_details(serde_json::json!({ "operation": "AXSelectedText" })) + .with_disposition(DeliverySemantics::delivered_unverified())); } - let after = crate::tree::copy_value_typed(el).unwrap_or_default(); - if after == next { - return Ok(()); + Ok(()) +} + +fn write_selected_text( + text: &str, + write: impl FnOnce(&str, &str) -> Result<(), AdapterError>, +) -> Result<(), AdapterError> { + write("AXSelectedText", text) +} + +fn is_text_target(element: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_string_attr_result(element, "AXRole", deadline); + if deadline.is_expired() { + return Err(deadline.timeout_error()); } - Err(AdapterError::new( - ErrorCode::ActionFailed, - "AX value write reported success but the element value did not change", - ) - .with_suggestion( - "Use explicit keyboard input for web-backed fields that ignore AXValue writes", + let role = result.map_err(|error| { + AdapterError::new( + ErrorCode::ActionFailed, + "Could not read keyboard target role", + ) + .with_details(serde_json::json!({ "ax_error": error })) + })?; + Ok(matches!( + role.as_deref(), + Some("AXTextField" | "AXTextArea" | "AXSecureTextField" | "AXComboBox") )) } +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + #[cfg(not(target_os = "macos"))] pub(crate) fn execute_type( - _el: &crate::tree::AXElement, + _element: &crate::tree::AXElement, _text: &str, - _policy: agent_desktop_core::interaction_policy::InteractionPolicy, -) -> Result<(), agent_desktop_core::error::AdapterError> { - Err(agent_desktop_core::error::AdapterError::new( - agent_desktop_core::error::ErrorCode::PlatformNotSupported, - "type_text is not supported on this platform", - )) -} - -#[cfg(target_os = "macos")] -fn is_text_target(el: &AXElement) -> bool { - matches!( - crate::actions::ax_helpers::element_role(el).as_deref(), - Some("textfield" | "combobox") - ) -} - -#[cfg(target_os = "macos")] -fn is_secure_text_field(el: &AXElement) -> bool { - crate::tree::copy_string_attr(el, "AXRole").as_deref() == Some("AXSecureTextField") -} - -#[cfg(target_os = "macos")] -fn readable_value(el: &AXElement) -> Option<String> { - if is_secure_text_field(el) { - None - } else { - crate::tree::copy_value_typed(el) - } -} - -/// Checks whether a clipboard paste changed the element's visible value. -/// -/// When `before` was readable but `after` is `None`, returns `Ok(())` rather -/// than an error: the post-paste read reuses the same element handle, which -/// can go stale after a successful paste in re-rendering UIs (Electron/web — -/// documented targets). Failing here would cause false-failure → agent retry -/// → double-paste corruption. The unverifiable case is therefore treated as -/// success; revisit only by re-resolving the element before failing. -#[cfg(target_os = "macos")] -fn verify_paste_effect(before: Option<&str>, after: Option<&str>) -> Result<(), AdapterError> { - if before.is_none() { - return Ok(()); - } - if after.is_none() { - tracing::warn!("paste could not be verified: post-paste field value is unreadable"); - return Ok(()); - } - if before != after { - return Ok(()); - } - Err(AdapterError::new( - ErrorCode::ActionFailed, - "Clipboard paste completed but the target value did not change", - ) - .with_suggestion( - "Use set-value for fields that expose AXValue, or retry with physical keyboard input.", - )) -} - -#[cfg(target_os = "macos")] -fn typed_value(current: &str, text: &str) -> String { - let mut value = String::with_capacity(current.len() + text.len()); - value.push_str(current); - value.push_str(text); - value + _policy: agent_desktop_core::InteractionPolicy, + _deadline: Deadline, +) -> Result<agent_desktop_core::ActionStep, AdapterError> { + Err(AdapterError::not_supported("type_text")) } #[cfg(test)] mod tests { - #[test] - fn typed_value_appends_without_losing_existing_text() { - assert_eq!(super::typed_value("abc", "123"), "abc123"); - } + use std::cell::RefCell; #[test] - fn paste_verification_rejects_readable_no_change() { - let err = super::verify_paste_effect(Some("before"), Some("before")).unwrap_err(); - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::ActionFailed); - } + fn type_writes_the_current_selection_instead_of_the_whole_value() { + let observed = RefCell::new(None); + super::write_selected_text("inserted", |attribute, value| { + observed.replace(Some((attribute.to_owned(), value.to_owned()))); + Ok(()) + }) + .unwrap(); - #[test] - fn paste_verification_accepts_unreadable_or_changed_values() { - assert!(super::verify_paste_effect(None, Some("after")).is_ok()); - assert!(super::verify_paste_effect(Some("before"), None).is_ok()); - assert!(super::verify_paste_effect(Some("before"), Some("after")).is_ok()); + assert_eq!( + observed.into_inner(), + Some(("AXSelectedText".into(), "inserted".into())) + ); } } diff --git a/crates/macos/src/adapter.rs b/crates/macos/src/adapter.rs index 5e9623d..670df24 100644 --- a/crates/macos/src/adapter.rs +++ b/crates/macos/src/adapter.rs @@ -1,20 +1,3 @@ -use agent_desktop_core::{ - PermissionReport, - action::{DragParams, MouseEvent, WindowOp}, - action_request::ActionRequest, - action_result::ActionResult, - adapter::{ - ImageBuffer, LiveElement, NativeHandle, PlatformAdapter, ScreenshotTarget, SnapshotSurface, - TreeOptions, WindowFilter, - }, - element_state::ElementState, - error::AdapterError, - node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo}, - notification::{NotificationFilter, NotificationIdentity, NotificationInfo}, - refs::RefEntry, -}; -use rustc_hash::FxHashSet; - pub struct MacOSAdapter; impl MacOSAdapter { @@ -29,317 +12,48 @@ impl Default for MacOSAdapter { } } -impl PlatformAdapter for MacOSAdapter { - fn permission_report(&self) -> PermissionReport { - crate::system::permissions::report() - } - - fn request_permissions(&self) -> PermissionReport { - crate::system::permissions::request_report() - } - - fn unknown_accessibility_means_unsupported(&self) -> bool { - false - } - - fn get_tree( - &self, - win: &WindowInfo, - opts: &TreeOptions, - ) -> Result<AccessibilityNode, AdapterError> { - let el = match opts.surface { - SnapshotSurface::Window => crate::tree::window_element_for(win.pid, &win.title), - SnapshotSurface::Focused => crate::tree::surfaces::focused_surface_for_pid(win.pid) - .ok_or_else(|| AdapterError::internal("No focused surface found"))?, - SnapshotSurface::Menu => crate::tree::surfaces::menu_element_for_pid(win.pid) - .ok_or_else(|| AdapterError::element_not_found("No open context menu"))?, - SnapshotSurface::Menubar => crate::tree::surfaces::menubar_for_pid(win.pid) - .ok_or_else(|| AdapterError::element_not_found("No menu bar found"))?, - SnapshotSurface::Sheet => crate::tree::surfaces::sheet_for_pid(win.pid) - .ok_or_else(|| AdapterError::element_not_found("No open sheet"))?, - SnapshotSurface::Popover => crate::tree::surfaces::popover_for_pid(win.pid) - .ok_or_else(|| AdapterError::element_not_found("No visible popover"))?, - SnapshotSurface::Alert => crate::tree::surfaces::alert_for_pid(win.pid) - .ok_or_else(|| AdapterError::element_not_found("No open alert or dialog"))?, - _ => return Err(AdapterError::not_supported("snapshot surface")), - }; - let mut visited = FxHashSet::default(); - let context = crate::tree::TreeBuildContext::for_pid(win.pid, opts.include_bounds); - crate::tree::build_subtree( - &el, - 0, - 0, - opts.max_depth, - &mut visited, - opts.skeleton, - &context, - ) - .ok_or_else(|| AdapterError::internal("Empty AX tree for surface")) - } - - fn execute_action( - &self, - handle: &NativeHandle, - request: ActionRequest, - ) -> Result<ActionResult, AdapterError> { - execute_action_impl(handle, request) - } - - fn resolve_element_strict(&self, entry: &RefEntry) -> Result<NativeHandle, AdapterError> { - crate::tree::resolve::resolve_element_impl(entry) - } - - fn resolve_element_strict_with_timeout( - &self, - entry: &RefEntry, - timeout: std::time::Duration, - ) -> Result<NativeHandle, AdapterError> { - crate::tree::resolve::resolve_element_with_timeout(entry, timeout) - } - - fn release_handle(&self, handle: &NativeHandle) -> Result<(), AdapterError> { - let raw = handle.as_raw(); - if raw.is_null() { - return Ok(()); - } - unsafe { - core_foundation::base::CFRelease(raw as core_foundation::base::CFTypeRef); - } - Ok(()) - } - - fn list_windows(&self, filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { - crate::system::window_list::list_windows_impl(filter) - } - - fn list_apps(&self) -> Result<Vec<AppInfo>, AdapterError> { - crate::system::app_list::list_apps_impl() - } - - fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError> { - crate::system::app_ops::focus_window_impl(win) - } - - fn focus_app(&self, pid: i32) -> Result<(), AdapterError> { - crate::system::app_ops::ensure_app_focused(pid) - } - - fn launch_app(&self, id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError> { - crate::system::app_ops::launch_app_impl(id, timeout_ms) - } - - fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError> { - crate::system::app_ops::close_app_impl(id, force) - } - - fn is_protected_process(&self, identifier: &str) -> bool { - crate::system::app_ops::is_protected_process(identifier) - } - - fn is_blocked_combo(&self, combo: &agent_desktop_core::action::KeyCombo) -> bool { - crate::input::blocked_combo::is_blocked(combo) - } - - fn screenshot(&self, target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> { - match target { - ScreenshotTarget::Window(pid) => crate::system::screenshot::capture_app(pid), - ScreenshotTarget::Screen(idx) => crate::system::screenshot::capture_screen(idx), - ScreenshotTarget::FullScreen => crate::system::screenshot::capture_screen(0), - } - } - - fn get_clipboard(&self) -> Result<String, AdapterError> { - crate::input::clipboard::get() - } - - fn set_clipboard(&self, text: &str) -> Result<(), AdapterError> { - crate::input::clipboard::set(text) - } - - fn press_key_for_app( - &self, - app_name: &str, - combo: &agent_desktop_core::action::KeyCombo, - ) -> Result<ActionResult, AdapterError> { - crate::system::key_dispatch::press_for_app_impl(app_name, combo) - } - - fn wait_for_menu(&self, pid: i32, open: bool, timeout_ms: u64) -> Result<(), AdapterError> { - crate::system::wait::wait_for_menu(pid, open, timeout_ms) - } - - fn list_surfaces(&self, pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError> { - Ok(crate::tree::surfaces::list_surfaces_for_pid(pid)) - } - - fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError> { - let filter = WindowFilter { - focused_only: true, - app: None, - }; - let windows = self.list_windows(&filter)?; - Ok(windows.into_iter().next()) - } - - fn get_live_value(&self, handle: &NativeHandle) -> Result<Option<String>, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(with_borrowed_ax_element( - handle, - crate::tree::copy_value_typed, - )) - } - #[cfg(not(target_os = "macos"))] - Err(AdapterError::not_supported("get_live_value")) - } - - fn get_live_state(&self, handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(Some(with_borrowed_ax_element( - handle, - crate::actions::post_state::read_element_state, - ))) - } - #[cfg(not(target_os = "macos"))] - Err(AdapterError::not_supported("get_live_state")) - } - - fn get_live_actions(&self, handle: &NativeHandle) -> Result<Option<Vec<String>>, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(Some(with_borrowed_ax_element( - handle, - crate::actions::post_state::read_live_actions, - ))) - } - #[cfg(not(target_os = "macos"))] - Err(AdapterError::not_supported("get_live_actions")) - } - - fn get_live_element(&self, handle: &NativeHandle) -> Result<LiveElement, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(with_borrowed_ax_element( - handle, - crate::actions::post_state::read_live_element, - )) - } - #[cfg(not(target_os = "macos"))] - Err(AdapterError::not_supported("get_live_element")) - } - - fn get_element_bounds(&self, handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(with_borrowed_ax_element(handle, crate::tree::read_bounds)) - } - #[cfg(not(target_os = "macos"))] - { - let _ = handle; - Err(AdapterError::not_supported("get_element_bounds")) - } - } - - fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> { - crate::system::window_ops::execute(win, op) - } - - fn mouse_event(&self, event: MouseEvent) -> Result<(), AdapterError> { - crate::input::mouse::synthesize_mouse(event) - } - - fn key_event( - &self, - combo: &agent_desktop_core::action::KeyCombo, - down: bool, - ) -> Result<(), AdapterError> { - crate::input::keyboard::synthesize_key_state(combo, down) - } - - fn drag(&self, params: DragParams) -> Result<(), AdapterError> { - crate::input::mouse::synthesize_drag(params) - } - - fn clear_clipboard(&self) -> Result<(), AdapterError> { - crate::input::clipboard::clear() - } - - fn list_notifications( - &self, - filter: &NotificationFilter, - ) -> Result<Vec<NotificationInfo>, AdapterError> { - crate::notifications::list::list_notifications(filter) - } - - fn dismiss_notification( - &self, - index: usize, - app_filter: Option<&str>, - ) -> Result<NotificationInfo, AdapterError> { - crate::notifications::actions::dismiss_notification(index, app_filter) - } - - fn dismiss_all_notifications( - &self, - app_filter: Option<&str>, - ) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> { - crate::notifications::actions::dismiss_all(app_filter) - } - - fn notification_action( - &self, - index: usize, - identity: Option<&NotificationIdentity>, - action_name: &str, - ) -> Result<ActionResult, AdapterError> { - crate::notifications::actions::notification_action(index, identity, action_name) - } - - fn get_subtree( - &self, - handle: &NativeHandle, - opts: &TreeOptions, - ) -> Result<AccessibilityNode, AdapterError> { - with_borrowed_ax_element(handle, |el| { - let mut ancestors = FxHashSet::default(); - let context = crate::tree::TreeBuildContext::empty(opts.include_bounds); - crate::tree::build_subtree( - el, - 0, - 0, - opts.max_depth, - &mut ancestors, - opts.skeleton, - &context, +pub(crate) fn ax_element( + handle: &agent_desktop_core::NativeHandle, +) -> Result<&crate::tree::AXElement, agent_desktop_core::AdapterError> { + handle + .downcast_ref::<crate::tree::AXElement>() + .ok_or_else(|| { + agent_desktop_core::AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Native handle does not contain a macOS accessibility element", ) - .ok_or_else(|| { - AdapterError::new( - agent_desktop_core::error::ErrorCode::ElementNotFound, - "Element no longer exists in accessibility tree", - ) - .with_suggestion("Run 'snapshot' to refresh refs, then retry.") - }) + .with_details(serde_json::json!({ + "kind": "invalid_native_handle", + "platform": "macos", + "empty": handle.is_null() + })) }) +} + +#[cfg(test)] +mod tests { + use super::ax_element; + use agent_desktop_core::{ErrorCode, NativeHandle}; + + #[test] + fn empty_handle_is_rejected_without_a_pointer_cast() { + let empty = NativeHandle::null(); + let Err(error) = ax_element(&empty) else { + panic!("empty handle must be rejected"); + }; + + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert_eq!(error.details.unwrap()["kind"], "invalid_native_handle"); + } + + #[test] + fn wrong_platform_payload_is_rejected_without_a_pointer_cast() { + let wrong = NativeHandle::new(String::from("uia-token")); + let Err(error) = ax_element(&wrong) else { + panic!("wrong payload type must be rejected"); + }; + + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert_eq!(error.details.unwrap()["platform"], "macos"); } } - -fn execute_action_impl( - handle: &NativeHandle, - request: ActionRequest, -) -> Result<ActionResult, AdapterError> { - with_borrowed_ax_element(handle, |el| crate::actions::perform_action(el, &request)) -} - -#[cfg(target_os = "macos")] -fn with_borrowed_ax_element<T>( - handle: &NativeHandle, - f: impl FnOnce(&crate::tree::AXElement) -> T, -) -> T { - use std::mem::ManuallyDrop; - - let el = ManuallyDrop::new(crate::tree::AXElement( - handle.as_raw() as accessibility_sys::AXUIElementRef - )); - f(&el) -} diff --git a/crates/macos/src/bin/agent-desktop-macos-helper.rs b/crates/macos/src/bin/agent-desktop-macos-helper.rs new file mode 100644 index 0000000..de5666f --- /dev/null +++ b/crates/macos/src/bin/agent-desktop-macos-helper.rs @@ -0,0 +1,3 @@ +fn main() { + std::process::exit(agent_desktop_macos::clipboard_helper_from_env()); +} diff --git a/crates/macos/src/input/adapter.rs b/crates/macos/src/input/adapter.rs new file mode 100644 index 0000000..86c06db --- /dev/null +++ b/crates/macos/src/input/adapter.rs @@ -0,0 +1,45 @@ +use agent_desktop_core::{ + AdapterError, ClipboardContent, ClipboardFormat, Deadline, DragParams, InputOps, + InteractionLease, KeyCombo, MouseEvent, +}; + +use crate::adapter::MacOSAdapter; + +impl InputOps for MacOSAdapter { + fn mouse_event(&self, event: MouseEvent, lease: &InteractionLease) -> Result<(), AdapterError> { + crate::input::mouse::synthesize_mouse(event, lease.deadline()) + } + + fn key_event( + &self, + combo: &KeyCombo, + down: bool, + _lease: &InteractionLease, + ) -> Result<(), AdapterError> { + crate::input::keyboard::reject_standalone_key_state(combo, down) + } + + fn drag(&self, params: DragParams, lease: &InteractionLease) -> Result<(), AdapterError> { + crate::input::mouse::synthesize_drag(params, lease.deadline()) + } + + fn clear_clipboard(&self, lease: &InteractionLease) -> Result<(), AdapterError> { + crate::input::clipboard::clear(lease.deadline()) + } + + fn get_clipboard_content( + &self, + format: ClipboardFormat, + deadline: Deadline, + ) -> Result<Option<ClipboardContent>, AdapterError> { + crate::input::clipboard::get_content(format, deadline) + } + + fn set_clipboard_content( + &self, + content: &ClipboardContent, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + crate::input::clipboard::set_content(content, lease.deadline()) + } +} diff --git a/crates/macos/src/input/blocked_combo.rs b/crates/macos/src/input/blocked_combo.rs index 6105c72..5f10b3f 100644 --- a/crates/macos/src/input/blocked_combo.rs +++ b/crates/macos/src/input/blocked_combo.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::action::{KeyCombo, Modifier}; +use agent_desktop_core::{KeyCombo, Modifier}; const BLOCKED: &[&str] = &[ "cmd+q", @@ -25,7 +25,7 @@ fn combo_to_string(combo: &KeyCombo) -> String { fn modifier_name(modifier: &Modifier) -> &'static str { match modifier { - Modifier::Cmd => "cmd", + Modifier::Meta => "cmd", Modifier::Ctrl => "ctrl", Modifier::Alt => "alt", Modifier::Shift => "shift", diff --git a/crates/macos/src/input/blocked_combo_tests.rs b/crates/macos/src/input/blocked_combo_tests.rs index b119442..13ecde9 100644 --- a/crates/macos/src/input/blocked_combo_tests.rs +++ b/crates/macos/src/input/blocked_combo_tests.rs @@ -1,5 +1,5 @@ use super::is_blocked; -use agent_desktop_core::action::{KeyCombo, Modifier}; +use agent_desktop_core::{KeyCombo, Modifier}; fn combo(modifiers: Vec<Modifier>, key: &str) -> KeyCombo { KeyCombo { @@ -10,18 +10,21 @@ fn combo(modifiers: Vec<Modifier>, key: &str) -> KeyCombo { #[test] fn dangerous_shortcuts_are_blocked() { - assert!(is_blocked(&combo(vec![Modifier::Cmd], "q"))); + assert!(is_blocked(&combo(vec![Modifier::Meta], "q"))); assert!(is_blocked(&combo( - vec![Modifier::Cmd, Modifier::Shift], + vec![Modifier::Meta, Modifier::Shift], "q" ))); assert!(is_blocked(&combo( - vec![Modifier::Cmd, Modifier::Alt], + vec![Modifier::Meta, Modifier::Alt], "esc" ))); - assert!(is_blocked(&combo(vec![Modifier::Ctrl, Modifier::Cmd], "q"))); assert!(is_blocked(&combo( - vec![Modifier::Cmd, Modifier::Shift], + vec![Modifier::Ctrl, Modifier::Meta], + "q" + ))); + assert!(is_blocked(&combo( + vec![Modifier::Meta, Modifier::Shift], "delete" ))); } @@ -29,7 +32,7 @@ fn dangerous_shortcuts_are_blocked() { #[test] fn modifier_order_does_not_matter() { assert!( - is_blocked(&combo(vec![Modifier::Cmd, Modifier::Ctrl], "q")), + is_blocked(&combo(vec![Modifier::Meta, Modifier::Ctrl], "q")), "cmd+ctrl+q must match the blocked ctrl+cmd+q regardless of order" ); } @@ -37,22 +40,22 @@ fn modifier_order_does_not_matter() { #[test] fn key_aliases_are_blocked() { assert!( - is_blocked(&combo(vec![Modifier::Cmd, Modifier::Alt], "escape")), + is_blocked(&combo(vec![Modifier::Meta, Modifier::Alt], "escape")), "escape is the same physical key as esc" ); assert!( - is_blocked(&combo(vec![Modifier::Cmd, Modifier::Shift], "backspace")), + is_blocked(&combo(vec![Modifier::Meta, Modifier::Shift], "backspace")), "backspace is the same physical key as delete" ); } #[test] fn benign_combos_are_not_blocked() { - assert!(!is_blocked(&combo(vec![Modifier::Cmd], "c"))); - assert!(!is_blocked(&combo(vec![Modifier::Cmd], "v"))); - assert!(!is_blocked(&combo(vec![Modifier::Cmd], "w"))); + assert!(!is_blocked(&combo(vec![Modifier::Meta], "c"))); + assert!(!is_blocked(&combo(vec![Modifier::Meta], "v"))); + assert!(!is_blocked(&combo(vec![Modifier::Meta], "w"))); assert!(!is_blocked(&combo( - vec![Modifier::Cmd, Modifier::Shift], + vec![Modifier::Meta, Modifier::Shift], "r" ))); assert!(!is_blocked(&combo(vec![Modifier::Ctrl], "s"))); diff --git a/crates/macos/src/input/clipboard.rs b/crates/macos/src/input/clipboard.rs index f2acd5b..02c5530 100644 --- a/crates/macos/src/input/clipboard.rs +++ b/crates/macos/src/input/clipboard.rs @@ -1,272 +1,319 @@ -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::{AdapterError, ErrorCode}; + +#[cfg(target_os = "macos")] +#[path = "clipboard_runtime.rs"] +mod clipboard_runtime; + +#[cfg(target_os = "macos")] +#[path = "clipboard_file_urls.rs"] +mod clipboard_file_urls; + +#[cfg(target_os = "macos")] +#[path = "clipboard_rich.rs"] +mod clipboard_rich; + +#[cfg(target_os = "macos")] +#[path = "clipboard_image_io.rs"] +mod clipboard_image_io; + +#[cfg(target_os = "macos")] +#[path = "clipboard_transaction.rs"] +mod clipboard_transaction; + +#[cfg(target_os = "macos")] +#[path = "clipboard_helper_client.rs"] +mod clipboard_helper_client; + +#[cfg(target_os = "macos")] +#[path = "clipboard_helper_protocol.rs"] +mod clipboard_helper_protocol; + +#[cfg(target_os = "macos")] +#[path = "clipboard_helper_entry.rs"] +mod clipboard_helper_entry; + +#[cfg(target_os = "macos")] +#[path = "clipboard_helper_dl.rs"] +mod clipboard_helper_dl; + +#[cfg(target_os = "macos")] +#[path = "clipboard_helper_identity.rs"] +mod clipboard_helper_identity; + +#[cfg(target_os = "macos")] +#[path = "clipboard_helper_process.rs"] +mod clipboard_helper_process; #[cfg(target_os = "macos")] mod imp { + use super::clipboard_runtime::{ + AutoreleasePool, Pasteboard as Id, change_count, ensure_read_access, pasteboard, + }; use super::*; + use agent_desktop_core::{ + ClipboardContent, ClipboardFormat, Deadline, ImageBuffer, ImageFormat, + }; use core_foundation::base::TCFType; use std::ffi::c_void; - type Id = *mut c_void; - type Class = *mut c_void; + #[cfg(all(test, feature = "interactive-tests"))] + mod interactive_tests { + include!("clipboard_tests.rs"); + } + type Sel = *mut c_void; + const MAX_CLIPBOARD_TEXT_UTF16: usize = 1_000_000; + unsafe extern "C" { - fn objc_getClass(name: *const core::ffi::c_char) -> Class; fn sel_registerName(name: *const core::ffi::c_char) -> Sel; fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; static NSPasteboardTypeString: Id; } - pub(crate) struct ClipboardSnapshot { - items: Id, - } - - impl ClipboardSnapshot { - pub(crate) fn capture() -> Result<Self, AdapterError> { - unsafe { - let pb = pasteboard()?; - Ok(Self { - items: deep_copy_pasteboard_items(pb), - }) - } - } - - pub(crate) fn restore(&self) -> Result<(), AdapterError> { - unsafe { - let pb = pasteboard()?; - clear_pasteboard(pb); - if !self.items.is_null() && !write_objects(pb, self.items) { - tracing::warn!( - "clipboard restore failed after clearContents; original clipboard content is lost" - ); - return Err(AdapterError::internal("NSPasteboard writeObjects: failed")); - } - Ok(()) - } - } - } - - impl Drop for ClipboardSnapshot { - fn drop(&mut self) { - unsafe { release_object(self.items) }; - } - } - - fn pasteboard() -> Result<Id, AdapterError> { - unsafe { - let cls = objc_getClass(c"NSPasteboard".as_ptr()); - if cls.is_null() { - return Err(AdapterError::internal("NSPasteboard class not found")); - } - let sel = sel_registerName(c"generalPasteboard".as_ptr()); - let send: unsafe extern "C" fn(Class, Sel) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - let pb = send(cls, sel); - if pb.is_null() { - return Err(AdapterError::internal("generalPasteboard returned null")); - } - Ok(pb) - } - } - - pub fn get() -> Result<String, AdapterError> { - tracing::debug!("clipboard: get"); - unsafe { - let pb = pasteboard()?; - let Some(result) = read_string(pb) else { - tracing::debug!("clipboard: get -> empty"); - return Ok(String::new()); - }; - tracing::debug!("clipboard: get -> {} chars", result.len()); - Ok(result) - } - } - - pub fn set(text: &str) -> Result<(), AdapterError> { - tracing::debug!("clipboard: set {} chars", text.len()); - unsafe { - let pb = pasteboard()?; - let previous = ClipboardSnapshot::capture()?; - clear_pasteboard(pb); - if !write_string(pb, text) { - let _ = previous.restore(); - return Err(AdapterError::internal( - "NSPasteboard setString:forType: failed", - )); - } - Ok(()) - } - } - - pub fn clear() -> Result<(), AdapterError> { + pub(crate) fn clear_direct(deadline: Deadline) -> Result<(), AdapterError> { tracing::debug!("clipboard: clear"); + let _pool = AutoreleasePool::new()?; + let pb = pasteboard()?; + super::clipboard_transaction::clear_verified(pb, deadline) + } + + pub(crate) fn get_content_direct( + format: ClipboardFormat, + deadline: Deadline, + ) -> Result<Option<ClipboardContent>, AdapterError> { + tracing::debug!("clipboard: get_content format={format:?}"); + let _pool = AutoreleasePool::new()?; + let pb = pasteboard()?; + ensure_read_access(pb)?; + for attempt in 0..2 { + ensure_read_budget(deadline)?; + let before = unsafe { change_count(pb) }; + let content = unsafe { get_content_from(pb, format, deadline) }?; + ensure_read_budget(deadline)?; + if before == unsafe { change_count(pb) } { + return Ok(content); + } + if attempt == 1 { + return Err(concurrent_change_error("read")); + } + } + Err(AdapterError::internal( + "Clipboard stable-read loop exited unexpectedly", + )) + } + + unsafe fn get_content_from( + pb: Id, + format: ClipboardFormat, + deadline: Deadline, + ) -> Result<Option<ClipboardContent>, AdapterError> { unsafe { - let pb = pasteboard()?; - clear_pasteboard(pb); - Ok(()) + match format { + ClipboardFormat::Text => Ok(read_string(pb)?.map(ClipboardContent::Text)), + ClipboardFormat::Image => { + Ok(super::clipboard_rich::read_image(pb, deadline)?.map(build_image_content)) + } + ClipboardFormat::FileUrls => { + let urls = super::clipboard_rich::read_file_urls(pb, deadline)?; + Ok((!urls.is_empty()).then_some(ClipboardContent::FileUrls(urls))) + } + ClipboardFormat::Auto => auto_content(pb, deadline), + } } } - unsafe fn read_string(pb: Id) -> Option<String> { + unsafe fn auto_content( + pb: Id, + deadline: Deadline, + ) -> Result<Option<ClipboardContent>, AdapterError> { + unsafe { + let urls = super::clipboard_rich::read_file_urls(pb, deadline)?; + if !urls.is_empty() { + return Ok(Some(ClipboardContent::FileUrls(urls))); + } + if let Some(image) = super::clipboard_rich::read_image(pb, deadline)? { + return Ok(Some(build_image_content(image))); + } + Ok(read_string(pb)?.map(ClipboardContent::Text)) + } + } + + fn build_image_content(image: (Vec<u8>, (u32, u32))) -> ClipboardContent { + let (bytes, (width, height)) = image; + ClipboardContent::Image(ImageBuffer { + data: bytes, + format: ImageFormat::Png, + width, + height, + scale_factor: 1.0, + }) + } + + pub(crate) fn set_content_direct( + content: &ClipboardContent, + deadline: Deadline, + ) -> Result<(), AdapterError> { + validate_content(content)?; + let _pool = AutoreleasePool::new()?; + ensure_read_budget(deadline)?; + let pb = pasteboard()?; + set_content_on(pb, content, deadline) + } + + fn set_content_on( + pb: Id, + content: &ClipboardContent, + deadline: Deadline, + ) -> Result<(), AdapterError> { + match content { + ClipboardContent::Text(text) => replace_on( + pb, + "text", + deadline, + |pb, deadline| unsafe { write_string(pb, text, deadline) }, + |pb, deadline| unsafe { + ensure_read_budget(deadline)?; + let matches = read_string(pb)?.as_deref() == Some(text); + ensure_read_budget(deadline)?; + Ok(matches) + }, + ), + ClipboardContent::Image(image) => { + let (prepared, dimensions) = super::clipboard_rich::prepare_image(&image.data)?; + if !matches!(&image.format, ImageFormat::Png) + || dimensions != (image.width, image.height) + || !image.scale_factor.is_finite() + || image.scale_factor <= 0.0 + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Clipboard image metadata does not match its PNG payload", + )); + } + replace_on( + pb, + "image", + deadline, + |pb, deadline| { + super::clipboard_rich::write_image(pb, prepared.as_ref(), deadline) + }, + |pb, deadline| { + Ok(super::clipboard_rich::read_image(pb, deadline)? + .as_ref() + .is_some_and(|(bytes, _)| bytes.as_slice() == prepared.as_ref())) + }, + ) + } + ClipboardContent::FileUrls(paths) => { + let prepared = super::clipboard_rich::prepare_file_urls(paths)?; + replace_on( + pb, + "file URLs", + deadline, + |pb, deadline| super::clipboard_rich::write_file_urls(pb, &prepared, deadline), + |pb, deadline| { + Ok( + super::clipboard_rich::read_file_urls(pb, deadline)? + == prepared.paths(), + ) + }, + ) + } + } + } + + fn replace_on( + pb: Id, + kind: &str, + deadline: Deadline, + write: impl FnOnce(Id, Deadline) -> Result<bool, AdapterError>, + verify: impl FnOnce(Id, Deadline) -> Result<bool, AdapterError>, + ) -> Result<(), AdapterError> { + super::clipboard_transaction::replace_on(pb, kind, deadline, write, verify) + } + + unsafe fn read_string(pb: Id) -> Result<Option<String>, AdapterError> { unsafe { let sel = sel_registerName(c"stringForType:".as_ptr()); let send: unsafe extern "C" fn(Id, Sel, Id) -> Id = std::mem::transmute(objc_msgSend as *const c_void); let ns_string = send(pb, sel, NSPasteboardTypeString); if ns_string.is_null() { - return None; + return Ok(None); } - let cf_str = core_foundation::string::CFString::wrap_under_get_rule( - ns_string as core_foundation_sys::string::CFStringRef, - ); - Some(cf_str.to_string()) + let string_ref = ns_string as core_foundation_sys::string::CFStringRef; + let length = core_foundation_sys::string::CFStringGetLength(string_ref); + if length < 0 || length as usize > MAX_CLIPBOARD_TEXT_UTF16 { + return Err(clipboard_resource_limit_error( + "clipboard text", + length.max(0) as usize, + )); + } + let cf_str = core_foundation::string::CFString::wrap_under_get_rule(string_ref); + Ok(Some(cf_str.to_string())) } } - unsafe fn deep_copy_pasteboard_items(pb: Id) -> Id { - unsafe { - let alloc_sel = sel_registerName(c"alloc".as_ptr()); - let init_sel = sel_registerName(c"init".as_ptr()); - let send: unsafe extern "C" fn(Id, Sel) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - - let ma_cls = objc_getClass(c"NSMutableArray".as_ptr()); - if ma_cls.is_null() { - return std::ptr::null_mut(); - } - let ma_alloc = send(ma_cls as Id, alloc_sel); - if ma_alloc.is_null() { - return std::ptr::null_mut(); - } - let mutable_array = send(ma_alloc, init_sel); - if mutable_array.is_null() { - return std::ptr::null_mut(); - } - - let items_sel = sel_registerName(c"pasteboardItems".as_ptr()); - let pb_items = send(pb, items_sel); - if pb_items.is_null() { - release_object(mutable_array); - return std::ptr::null_mut(); - } - - let count_sel = sel_registerName(c"count".as_ptr()); - let send_usize: unsafe extern "C" fn(Id, Sel) -> usize = - std::mem::transmute(objc_msgSend as *const c_void); - let item_count = send_usize(pb_items, count_sel); - - let idx_sel = sel_registerName(c"objectAtIndex:".as_ptr()); - let send_at_idx: unsafe extern "C" fn(Id, Sel, usize) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - - let types_sel = sel_registerName(c"types".as_ptr()); - let data_sel = sel_registerName(c"dataForType:".as_ptr()); - let send_with_id: unsafe extern "C" fn(Id, Sel, Id) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - - let set_data_sel = sel_registerName(c"setData:forType:".as_ptr()); - let send_set_data: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool = - std::mem::transmute(objc_msgSend as *const c_void); - - let add_sel = sel_registerName(c"addObject:".as_ptr()); - let send_add: unsafe extern "C" fn(Id, Sel, Id) = - std::mem::transmute(objc_msgSend as *const c_void); - - let pbi_cls = objc_getClass(c"NSPasteboardItem".as_ptr()); - if pbi_cls.is_null() { - release_object(mutable_array); - return std::ptr::null_mut(); - } - - let mut added = false; - for i in 0..item_count { - let orig_item = send_at_idx(pb_items, idx_sel, i); - if orig_item.is_null() { - continue; + fn validate_content(content: &ClipboardContent) -> Result<(), AdapterError> { + match content { + ClipboardContent::Text(text) => { + let utf16_units = text.encode_utf16().count(); + if utf16_units > MAX_CLIPBOARD_TEXT_UTF16 { + Err(input_resource_limit_error("clipboard text", utf16_units)) + } else { + Ok(()) } - - let types = send(orig_item, types_sel); - if types.is_null() { - continue; - } - - let type_count = send_usize(types, count_sel); - if type_count == 0 { - continue; - } - - let fresh_alloc = send(pbi_cls as Id, alloc_sel); - if fresh_alloc.is_null() { - continue; - } - let fresh_item = send(fresh_alloc, init_sel); - if fresh_item.is_null() { - continue; - } - - for j in 0..type_count { - let type_str = send_at_idx(types, idx_sel, j); - if type_str.is_null() { - continue; - } - let data = send_with_id(orig_item, data_sel, type_str); - if data.is_null() { - continue; - } - let _ = send_set_data(fresh_item, set_data_sel, data, type_str); - } - - send_add(mutable_array, add_sel, fresh_item); - release_object(fresh_item); - added = true; } - - if !added { - release_object(mutable_array); - return std::ptr::null_mut(); - } - mutable_array + ClipboardContent::Image(_) | ClipboardContent::FileUrls(_) => Ok(()), } } - unsafe fn release_object(object: Id) { - if object.is_null() { - return; - } - unsafe { - let sel = sel_registerName(c"release".as_ptr()); - let send: unsafe extern "C" fn(Id, Sel) = - std::mem::transmute(objc_msgSend as *const c_void); - send(object, sel); - } + fn concurrent_change_error(phase: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Clipboard changed concurrently during a stable operation", + ) + .with_details(serde_json::json!({ + "phase": phase, + "concurrent_change": true, + "retryable": true, + })) } - unsafe fn clear_pasteboard(pb: Id) { - unsafe { - let clear_sel = sel_registerName(c"clearContents".as_ptr()); - let send_void: unsafe extern "C" fn(Id, Sel) = - std::mem::transmute(objc_msgSend as *const c_void); - send_void(pb, clear_sel); - } + fn input_resource_limit_error(kind: &str, observed: usize) -> AdapterError { + AdapterError::new( + ErrorCode::InvalidArgs, + format!("{kind} exceeds the supported resource budget"), + ) + .with_details(serde_json::json!({ "kind": kind, "observed": observed })) } - unsafe fn write_string(pb: Id, text: &str) -> bool { + fn clipboard_resource_limit_error(kind: &str, observed: usize) -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + format!("{kind} exceeds the supported resource budget"), + ) + .with_details(serde_json::json!({ "kind": kind, "observed": observed })) + } + + unsafe fn write_string(pb: Id, text: &str, deadline: Deadline) -> Result<bool, AdapterError> { unsafe { + ensure_read_budget(deadline)?; let cf_text = core_foundation::string::CFString::new(text); let ns_text = cf_text.as_concrete_TypeRef() as Id; let set_sel = sel_registerName(c"setString:forType:".as_ptr()); let send_two: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool = std::mem::transmute(objc_msgSend as *const c_void); - send_two(pb, set_sel, ns_text, NSPasteboardTypeString) + Ok(send_two(pb, set_sel, ns_text, NSPasteboardTypeString)) } } - unsafe fn write_objects(pb: Id, objects: Id) -> bool { - unsafe { - let sel = sel_registerName(c"writeObjects:".as_ptr()); - let send: unsafe extern "C" fn(Id, Sel, Id) -> bool = - std::mem::transmute(objc_msgSend as *const c_void); - send(pb, sel, objects) + fn ensure_read_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) } } } @@ -275,30 +322,51 @@ mod imp { mod imp { use super::*; - pub fn get() -> Result<String, AdapterError> { - Err(AdapterError::not_supported("clipboard_get")) - } - - pub fn set(_text: &str) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("clipboard_set")) - } - - pub fn clear() -> Result<(), AdapterError> { + pub fn clear(_deadline: agent_desktop_core::Deadline) -> Result<(), AdapterError> { Err(AdapterError::not_supported("clipboard_clear")) } - pub(crate) struct ClipboardSnapshot; + pub fn get_content( + _format: agent_desktop_core::ClipboardFormat, + _deadline: agent_desktop_core::Deadline, + ) -> Result<Option<agent_desktop_core::ClipboardContent>, AdapterError> { + Err(AdapterError::not_supported("get_clipboard_content")) + } - impl ClipboardSnapshot { - pub(crate) fn capture() -> Result<Self, AdapterError> { - Err(AdapterError::not_supported("clipboard_snapshot")) - } - - pub(crate) fn restore(&self) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("clipboard_snapshot")) - } + pub fn set_content( + _content: &agent_desktop_core::ClipboardContent, + _deadline: agent_desktop_core::Deadline, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("set_clipboard_content")) } } -pub(crate) use imp::ClipboardSnapshot; -pub use imp::{clear, get, set}; +#[cfg(target_os = "macos")] +pub(crate) fn clear(deadline: agent_desktop_core::Deadline) -> Result<(), AdapterError> { + clipboard_helper_client::clear(deadline) +} + +#[cfg(target_os = "macos")] +pub(crate) fn get_content( + format: agent_desktop_core::ClipboardFormat, + deadline: agent_desktop_core::Deadline, +) -> Result<Option<agent_desktop_core::ClipboardContent>, AdapterError> { + clipboard_helper_client::read(format, deadline) +} + +#[cfg(target_os = "macos")] +pub(crate) fn set_content( + content: &agent_desktop_core::ClipboardContent, + deadline: agent_desktop_core::Deadline, +) -> Result<(), AdapterError> { + clipboard_helper_client::write(content, deadline) +} + +#[cfg(target_os = "macos")] +pub use clipboard_helper_entry::entry_from_env as helper_entry_from_env; + +#[cfg(target_os = "macos")] +pub(crate) use imp::{clear_direct, get_content_direct, set_content_direct}; + +#[cfg(not(target_os = "macos"))] +pub(crate) use imp::{clear, get_content, set_content}; diff --git a/crates/macos/src/input/clipboard_file_urls.rs b/crates/macos/src/input/clipboard_file_urls.rs new file mode 100644 index 0000000..c9f81b0 --- /dev/null +++ b/crates/macos/src/input/clipboard_file_urls.rs @@ -0,0 +1,234 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; +use core_foundation::base::TCFType; +use core_foundation::string::CFString; +use core_foundation::url::CFURL; +use core_foundation_sys::base::kCFAllocatorDefault; +use core_foundation_sys::string::CFStringGetLength; +use core_foundation_sys::url::CFURLCreateWithString; +use std::ffi::c_void; + +use crate::input::owned_object::OwnedObject; + +type Id = *mut c_void; +type Class = *mut c_void; +type Sel = *mut c_void; + +const MAX_FILE_URLS: usize = 1_024; +const MAX_FILE_URL_UTF16: usize = 16_384; +const MAX_FILE_URL_TOTAL_UTF16: usize = 1_000_000; + +unsafe extern "C" { + fn objc_getClass(name: *const core::ffi::c_char) -> Class; + fn sel_registerName(name: *const core::ffi::c_char) -> Sel; + fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; + static NSPasteboardTypeFileURL: Id; +} + +pub(crate) struct PreparedFileUrls { + urls: Vec<CFURL>, + paths: Vec<String>, +} + +impl PreparedFileUrls { + pub(crate) fn paths(&self) -> &[String] { + &self.paths + } +} + +pub(crate) fn prepare_file_urls(paths: &[String]) -> Result<PreparedFileUrls, AdapterError> { + if paths.is_empty() || paths.len() > MAX_FILE_URLS { + return Err(invalid("File URL list must contain 1 to 1024 paths")); + } + let mut urls = Vec::with_capacity(paths.len()); + let mut normalized_paths = Vec::with_capacity(paths.len()); + let mut total_units = 0_usize; + for path in paths { + let units = path.encode_utf16().count(); + if path.is_empty() + || path.contains('\0') + || !std::path::Path::new(path).is_absolute() + || units > MAX_FILE_URL_UTF16 + { + return Err(invalid("Every file path must be a bounded absolute path")); + } + let url = CFURL::from_path(path, false) + .ok_or_else(|| invalid("Every file path must be representable as a file URL"))?; + let normalized = url + .to_path() + .and_then(|value| value.to_str().map(str::to_owned)) + .filter(|value| !value.contains('\0') && std::path::Path::new(value).is_absolute()) + .ok_or_else(|| invalid("Every file URL must resolve to a local UTF-8 path"))?; + total_units = total_units + .checked_add(normalized.encode_utf16().count()) + .ok_or_else(|| invalid("File URL text budget overflowed"))?; + if total_units > MAX_FILE_URL_TOTAL_UTF16 { + return Err(invalid("File URLs exceed the total text budget")); + } + urls.push(url); + normalized_paths.push(normalized); + } + Ok(PreparedFileUrls { + urls, + paths: normalized_paths, + }) +} + +pub(crate) fn read_file_urls(pb: Id, deadline: Deadline) -> Result<Vec<String>, AdapterError> { + ensure_budget(deadline)?; + let result = unsafe { read_file_urls_inner(pb, deadline) }?; + ensure_budget(deadline)?; + Ok(result) +} + +unsafe fn read_file_urls_inner(pb: Id, deadline: Deadline) -> Result<Vec<String>, AdapterError> { + unsafe { + let send: unsafe extern "C" fn(Id, Sel) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let types = send(pb, sel_registerName(c"types".as_ptr())); + if types.is_null() { + return Ok(Vec::new()); + } + let send_contains: unsafe extern "C" fn(Id, Sel, Id) -> bool = + std::mem::transmute(objc_msgSend as *const c_void); + if !send_contains( + types, + sel_registerName(c"containsObject:".as_ptr()), + NSPasteboardTypeFileURL, + ) { + return Ok(Vec::new()); + } + let items = send(pb, sel_registerName(c"pasteboardItems".as_ptr())); + if items.is_null() { + return Err(data_error("Clipboard file URL items are unavailable")); + } + let send_usize: unsafe extern "C" fn(Id, Sel) -> usize = + std::mem::transmute(objc_msgSend as *const c_void); + let count = send_usize(items, sel_registerName(c"count".as_ptr())); + if count > MAX_FILE_URLS { + return Err(data_error("Clipboard contains too many file URLs")); + } + let send_at: unsafe extern "C" fn(Id, Sel, usize) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let send_string: unsafe extern "C" fn(Id, Sel, Id) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let mut urls = Vec::new(); + let mut total_units = 0_usize; + for index in 0..count { + ensure_budget(deadline)?; + let item = send_at(items, sel_registerName(c"objectAtIndex:".as_ptr()), index); + if item.is_null() { + return Err(data_error("Pasteboard item disappeared during read")); + } + let value = send_string( + item, + sel_registerName(c"stringForType:".as_ptr()), + NSPasteboardTypeFileURL, + ); + if value.is_null() { + continue; + } + let string_ref = value as core_foundation_sys::string::CFStringRef; + let units = CFStringGetLength(string_ref); + if units < 0 || units as usize > MAX_FILE_URL_UTF16 { + return Err(data_error("File URL exceeds its text budget")); + } + total_units = total_units + .checked_add(units as usize) + .ok_or_else(|| data_error("File URL text budget overflowed"))?; + if total_units > MAX_FILE_URL_TOTAL_UTF16 { + return Err(data_error("File URLs exceed the total text budget")); + } + let url = CFString::wrap_under_get_rule(string_ref).to_string(); + let path = file_url_to_path(&url) + .ok_or_else(|| data_error("Clipboard file URL is not a local UTF-8 path"))?; + urls.push(path); + } + Ok(urls) + } +} + +pub(crate) fn file_url_to_path(url_string: &str) -> Option<String> { + if !url_string.starts_with("file:///") || url_string.contains('\0') { + return None; + } + let value = CFString::new(url_string); + let url_ref = unsafe { + CFURLCreateWithString( + kCFAllocatorDefault, + value.as_concrete_TypeRef(), + std::ptr::null(), + ) + }; + if url_ref.is_null() { + return None; + } + let url: CFURL = unsafe { TCFType::wrap_under_create_rule(url_ref) }; + let path = url.to_path()?; + let path = path.to_str()?; + (!path.contains('\0') && std::path::Path::new(path).is_absolute()).then(|| path.to_owned()) +} + +pub(crate) fn write_file_urls( + pb: Id, + prepared: &PreparedFileUrls, + deadline: Deadline, +) -> Result<bool, AdapterError> { + ensure_budget(deadline)?; + unsafe { write_file_urls_inner(pb, prepared, deadline) } +} + +unsafe fn write_file_urls_inner( + pb: Id, + prepared: &PreparedFileUrls, + deadline: Deadline, +) -> Result<bool, AdapterError> { + unsafe { + let class = objc_getClass(c"NSMutableArray".as_ptr()); + if class.is_null() { + return Ok(false); + } + let send: unsafe extern "C" fn(Id, Sel) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let array = OwnedObject::from_id( + send( + send(class as Id, sel_registerName(c"alloc".as_ptr())), + sel_registerName(c"init".as_ptr()), + ), + "NSMutableArray initialization", + )?; + let send_add: unsafe extern "C" fn(Id, Sel, Id) = + std::mem::transmute(objc_msgSend as *const c_void); + for url in &prepared.urls { + ensure_budget(deadline)?; + send_add( + array.as_id(), + sel_registerName(c"addObject:".as_ptr()), + url.as_concrete_TypeRef() as Id, + ); + } + ensure_budget(deadline)?; + let send_write: unsafe extern "C" fn(Id, Sel, Id) -> bool = + std::mem::transmute(objc_msgSend as *const c_void); + Ok(send_write( + pb, + sel_registerName(c"writeObjects:".as_ptr()), + array.as_id(), + )) + } +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn invalid(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::InvalidArgs, message) +} + +fn data_error(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::ActionFailed, message) +} diff --git a/crates/macos/src/input/clipboard_helper_client.rs b/crates/macos/src/input/clipboard_helper_client.rs new file mode 100644 index 0000000..803f91a --- /dev/null +++ b/crates/macos/src/input/clipboard_helper_client.rs @@ -0,0 +1,220 @@ +use agent_desktop_core::{ + AdapterError, ClipboardContent, ClipboardFormat, Deadline, ErrorCode, ImageBuffer, ImageFormat, + MAX_PNG_INPUT_BYTES, +}; +use serde_json::Value; +use std::process::Command; + +use super::clipboard_helper_protocol as protocol; + +struct HelperResponse { + metadata: Value, + payload: Vec<u8>, +} + +pub(crate) fn clear(deadline: Deadline) -> Result<(), AdapterError> { + run("clear", &[], &[], deadline).map(|_| ()) +} + +pub(crate) fn read( + format: ClipboardFormat, + deadline: Deadline, +) -> Result<Option<ClipboardContent>, AdapterError> { + let operation = format!("read:{}", format.as_str()); + let response = run(&operation, &[], &[], deadline)?; + match response + .metadata + .get("kind") + .and_then(Value::as_str) + .unwrap_or("invalid") + { + "none" if response.payload.is_empty() => Ok(None), + "text" => String::from_utf8(response.payload) + .map(ClipboardContent::Text) + .map(Some) + .map_err(|_| protocol::protocol_error()), + "file_urls" => serde_json::from_slice::<Vec<String>>(&response.payload) + .map(ClipboardContent::FileUrls) + .map(Some) + .map_err(|_| protocol::protocol_error()), + "image" => decode_image(response).map(Some), + _ => Err(protocol::protocol_error()), + } +} + +pub(crate) fn write(content: &ClipboardContent, deadline: Deadline) -> Result<(), AdapterError> { + match content { + ClipboardContent::Text(text) => run("write:text", &[], text.as_bytes(), deadline), + ClipboardContent::FileUrls(paths) => { + let payload = serde_json::to_vec(paths) + .map_err(|error| AdapterError::internal(format!("Encode file URLs: {error}")))?; + run("write:file_urls", &[], &payload, deadline) + } + ClipboardContent::Image(image) => { + let args = [ + image.width.to_string(), + image.height.to_string(), + image.scale_factor.to_string(), + ]; + run("write:image", &args, &image.data, deadline) + } + } + .map(|_| ()) +} + +fn decode_image(response: HelperResponse) -> Result<ClipboardContent, AdapterError> { + let width = response + .metadata + .get("width") + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(protocol::protocol_error)?; + let height = response + .metadata + .get("height") + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(protocol::protocol_error)?; + Ok(ClipboardContent::Image(ImageBuffer { + data: response.payload, + format: ImageFormat::Png, + width, + height, + scale_factor: 1.0, + })) +} + +fn run( + operation: &str, + args: &[String], + input: &[u8], + deadline: Deadline, +) -> Result<HelperResponse, AdapterError> { + if input.len() > MAX_PNG_INPUT_BYTES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "macOS clipboard helper input exceeds the protocol limit", + )); + } + let token = random_token()?; + let identity = super::clipboard_helper_identity::HelperIdentity::discover()?; + let mut command = Command::new(&identity.path); + command + .arg(operation) + .args(args) + .env_clear() + .env("AGENT_DESKTOP_MACOS_HELPER_MODE", "clipboard") + .env( + "AGENT_DESKTOP_MACOS_HELPER_PROTOCOL", + protocol::PROTOCOL_VERSION, + ) + .env("AGENT_DESKTOP_MACOS_HELPER_BUILD", protocol::BUILD_IDENTITY) + .env("AGENT_DESKTOP_MACOS_HELPER_TOKEN", &token) + .env("AGENT_DESKTOP_MACOS_HELPER_OUTPUT_FD", protocol::OUTPUT_FD) + .env( + "AGENT_DESKTOP_MACOS_HELPER_DEADLINE_MS", + deadline.remaining_ms().to_string(), + ); + let output = + super::clipboard_helper_process::run(&mut command, input, deadline, Some(&identity)) + .map_err(|error| classify_mutation_failure(operation, error, false))?; + let response = parse_output(&output, &token, operation) + .map_err(|error| classify_mutation_failure(operation, error, true))?; + if protocol::is_mutating(operation) + && response.metadata.get("delivery").and_then(Value::as_str) != Some("committed_verified") + { + return Err(classify_mutation_failure( + operation, + protocol::protocol_error(), + true, + )); + } + Ok(response) +} + +fn parse_output( + output: &[u8], + token: &str, + operation: &str, +) -> Result<HelperResponse, AdapterError> { + let newline = output + .iter() + .position(|byte| *byte == b'\n') + .filter(|index| *index <= protocol::MAX_HEADER_BYTES) + .ok_or_else(protocol::protocol_error)?; + let header: Value = + serde_json::from_slice(&output[..newline]).map_err(|_| protocol::protocol_error())?; + let payload = output + .get(newline + 1..) + .ok_or_else(protocol::protocol_error)?; + let metadata = protocol::validate_header(&header, token, operation, payload.len())?.clone(); + Ok(HelperResponse { + metadata, + payload: payload.to_vec(), + }) +} + +fn random_token() -> Result<String, AdapterError> { + let mut bytes = [0_u8; 32]; + if unsafe { getentropy(bytes.as_mut_ptr().cast(), bytes.len()) } != 0 { + return Err(AdapterError::internal(format!( + "Generate clipboard helper token: {}", + std::io::Error::last_os_error() + ))); + } + Ok(bytes.iter().map(|byte| format!("{byte:02x}")).collect()) +} + +fn classify_mutation_failure( + operation: &str, + error: AdapterError, + known_dispatched: bool, +) -> AdapterError { + let dispatched = known_dispatched + || error + .details + .as_ref() + .and_then(|details| details.get("helper_dispatched")) + .and_then(Value::as_bool) + == Some(true); + if protocol::is_mutating(operation) + && dispatched + && error.disposition == agent_desktop_core::DeliverySemantics::unknown() + { + error.with_disposition(agent_desktop_core::DeliverySemantics::uncertain()) + } else { + error + } +} + +unsafe extern "C" { + fn getentropy(buffer: *mut std::ffi::c_void, length: usize) -> i32; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mutation_classifier_never_marks_post_dispatch_failures_retry_safe() { + let timeout = super::super::clipboard_helper_process::mark_dispatched( + AdapterError::timeout("helper timed out"), + ); + let classified = classify_mutation_failure("write:text", timeout, false); + + assert_eq!( + classified.disposition, + agent_desktop_core::DeliverySemantics::uncertain() + ); + assert_eq!( + classify_mutation_failure( + "write:text", + AdapterError::new(ErrorCode::InvalidArgs, "preflight") + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered(),), + true, + ) + .disposition, + agent_desktop_core::DeliverySemantics::not_delivered() + ); + } +} diff --git a/crates/macos/src/input/clipboard_helper_dl.rs b/crates/macos/src/input/clipboard_helper_dl.rs new file mode 100644 index 0000000..38d9c5d --- /dev/null +++ b/crates/macos/src/input/clipboard_helper_dl.rs @@ -0,0 +1,28 @@ +use std::ffi::{c_char, c_int, c_void}; + +#[repr(C)] +struct DlInfo { + filename: *const c_char, + base: *mut c_void, + symbol_name: *const c_char, + symbol_address: *mut c_void, +} + +pub(crate) fn containing_image() -> Option<std::path::PathBuf> { + let mut info = DlInfo { + filename: std::ptr::null(), + base: std::ptr::null_mut(), + symbol_name: std::ptr::null(), + symbol_address: std::ptr::null_mut(), + }; + let address = containing_image as *const () as *const c_void; + if unsafe { dladdr(address, &mut info) } == 0 || info.filename.is_null() { + return None; + } + let bytes = unsafe { std::ffi::CStr::from_ptr(info.filename) }; + Some(std::path::PathBuf::from(bytes.to_string_lossy().as_ref())) +} + +unsafe extern "C" { + fn dladdr(address: *const c_void, info: *mut DlInfo) -> c_int; +} diff --git a/crates/macos/src/input/clipboard_helper_entry.rs b/crates/macos/src/input/clipboard_helper_entry.rs new file mode 100644 index 0000000..b680d28 --- /dev/null +++ b/crates/macos/src/input/clipboard_helper_entry.rs @@ -0,0 +1,230 @@ +use agent_desktop_core::{ + AdapterError, ClipboardContent, ClipboardFormat, Deadline, ErrorCode, ImageBuffer, ImageFormat, + MAX_PNG_INPUT_BYTES, +}; +use serde_json::{Value, json}; +use std::io::{Read, Write}; +use std::os::fd::FromRawFd; + +use super::clipboard_helper_protocol as protocol; + +pub fn entry_from_env() -> i32 { + let Some(context) = protocol_context() else { + return 2; + }; + let mut output = unsafe { std::fs::File::from_raw_fd(1) }; + let result = execute(&context.operation, &context.args, context.deadline); + let (header, payload) = match result { + Ok((metadata, payload)) => ( + protocol::response_header( + &context.token, + &context.operation, + Ok((metadata, payload.len())), + ), + payload, + ), + Err(error) => { + let error = classify_operation_error(&context.operation, error); + ( + protocol::response_header(&context.token, &context.operation, Err(&error)), + Vec::new(), + ) + } + }; + if write_response(&mut output, &header, &payload).is_err() { + return 3; + } + if header.get("ok").and_then(Value::as_bool) == Some(true) { + 0 + } else { + 1 + } +} + +fn classify_operation_error(operation: &str, error: AdapterError) -> AdapterError { + if protocol::is_mutating(operation) + && error.disposition == agent_desktop_core::DeliverySemantics::unknown() + { + error.with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) + } else { + error + } +} + +struct ProtocolContext { + token: String, + operation: String, + args: Vec<String>, + deadline: Deadline, +} + +fn protocol_context() -> Option<ProtocolContext> { + let exact = [ + ("AGENT_DESKTOP_MACOS_HELPER_MODE", "clipboard"), + ( + "AGENT_DESKTOP_MACOS_HELPER_PROTOCOL", + protocol::PROTOCOL_VERSION, + ), + ("AGENT_DESKTOP_MACOS_HELPER_BUILD", protocol::BUILD_IDENTITY), + ("AGENT_DESKTOP_MACOS_HELPER_OUTPUT_FD", protocol::OUTPUT_FD), + ]; + if exact + .iter() + .any(|(key, value)| std::env::var(key).as_deref() != Ok(*value)) + { + return None; + } + let token = std::env::var("AGENT_DESKTOP_MACOS_HELPER_TOKEN").ok()?; + if token.len() != 64 || !token.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let timeout_ms = std::env::var("AGENT_DESKTOP_MACOS_HELPER_DEADLINE_MS") + .ok()? + .parse::<u64>() + .ok()? + .clamp(1, 60_000); + let mut args = std::env::args().skip(1); + let operation = args.next()?; + Some(ProtocolContext { + token, + operation, + args: args.collect(), + deadline: Deadline::after(timeout_ms).ok()?, + }) +} + +fn execute( + operation: &str, + args: &[String], + deadline: Deadline, +) -> Result<(Value, Vec<u8>), AdapterError> { + match operation { + "clear" if args.is_empty() => { + super::clear_direct(deadline)?; + Ok(( + json!({ "kind": "none", "delivery": "committed_verified" }), + Vec::new(), + )) + } + "read:auto" | "read:text" | "read:image" | "read:file_urls" if args.is_empty() => { + let format = match operation { + "read:auto" => ClipboardFormat::Auto, + "read:text" => ClipboardFormat::Text, + "read:image" => ClipboardFormat::Image, + _ => ClipboardFormat::FileUrls, + }; + encode_content(super::get_content_direct(format, deadline)?) + } + "write:text" if args.is_empty() => { + let input = read_input(MAX_PNG_INPUT_BYTES)?; + let text = String::from_utf8(input).map_err(|_| { + AdapterError::new(ErrorCode::InvalidArgs, "Clipboard text must be valid UTF-8") + })?; + super::set_content_direct(&ClipboardContent::Text(text), deadline)?; + Ok(( + json!({ "kind": "none", "delivery": "committed_verified" }), + Vec::new(), + )) + } + "write:file_urls" if args.is_empty() => { + let input = read_input(MAX_PNG_INPUT_BYTES)?; + let paths = serde_json::from_slice::<Vec<String>>(&input).map_err(|_| { + AdapterError::new(ErrorCode::InvalidArgs, "Invalid clipboard file URL request") + })?; + super::set_content_direct(&ClipboardContent::FileUrls(paths), deadline)?; + Ok(( + json!({ "kind": "none", "delivery": "committed_verified" }), + Vec::new(), + )) + } + "write:image" if args.len() == 3 => { + let width = parse_arg::<u32>(&args[0], "width")?; + let height = parse_arg::<u32>(&args[1], "height")?; + let scale_factor = parse_arg::<f64>(&args[2], "scale factor")?; + let image = ClipboardContent::Image(ImageBuffer { + data: read_input(MAX_PNG_INPUT_BYTES)?, + format: ImageFormat::Png, + width, + height, + scale_factor, + }); + super::set_content_direct(&image, deadline)?; + Ok(( + json!({ "kind": "none", "delivery": "committed_verified" }), + Vec::new(), + )) + } + "validate:png" if args.is_empty() => { + let input = read_input(MAX_PNG_INPUT_BYTES)?; + if !super::clipboard_image_io::is_complete_png(&input) { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Clipboard PNG failed platform validation", + )); + } + Ok((json!({ "kind": "none" }), Vec::new())) + } + _ => Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Unsupported macOS clipboard helper operation", + )), + } +} + +fn encode_content(content: Option<ClipboardContent>) -> Result<(Value, Vec<u8>), AdapterError> { + match content { + None => Ok((json!({ "kind": "none" }), Vec::new())), + Some(ClipboardContent::Text(text)) => Ok((json!({ "kind": "text" }), text.into_bytes())), + Some(ClipboardContent::FileUrls(paths)) => Ok(( + json!({ "kind": "file_urls" }), + serde_json::to_vec(&paths) + .map_err(|error| AdapterError::internal(format!("Encode file URLs: {error}")))?, + )), + Some(ClipboardContent::Image(image)) => Ok(( + json!({ + "kind": "image", + "width": image.width, + "height": image.height, + }), + image.data, + )), + } +} + +fn read_input(limit: usize) -> Result<Vec<u8>, AdapterError> { + let mut input = Vec::new(); + std::io::stdin() + .take((limit + 1) as u64) + .read_to_end(&mut input) + .map_err(|error| AdapterError::internal(format!("Read helper request: {error}")))?; + if input.len() > limit { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Clipboard helper request exceeds the protocol limit", + )); + } + Ok(input) +} + +fn parse_arg<T: std::str::FromStr>(value: &str, name: &str) -> Result<T, AdapterError> { + value + .parse() + .map_err(|_| AdapterError::new(ErrorCode::InvalidArgs, format!("Invalid image {name}"))) +} + +fn write_response( + output: &mut std::fs::File, + header: &Value, + payload: &[u8], +) -> std::io::Result<()> { + let encoded = serde_json::to_vec(header).map_err(std::io::Error::other)?; + if encoded.len() > protocol::MAX_HEADER_BYTES || payload.len() > protocol::MAX_RESPONSE_BYTES { + return Err(std::io::Error::other( + "clipboard helper response exceeds limit", + )); + } + output.write_all(&encoded)?; + output.write_all(b"\n")?; + output.write_all(payload)?; + output.flush() +} diff --git a/crates/macos/src/input/clipboard_helper_identity.rs b/crates/macos/src/input/clipboard_helper_identity.rs new file mode 100644 index 0000000..1943e1c --- /dev/null +++ b/crates/macos/src/input/clipboard_helper_identity.rs @@ -0,0 +1,118 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::os::unix::fs::MetadataExt; + +use super::clipboard_helper_protocol as protocol; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HelperIdentity { + pub(crate) path: std::path::PathBuf, + device: u64, + inode: u64, + owner: u32, + mode: u32, + links: u64, +} + +impl HelperIdentity { + pub(crate) fn discover() -> Result<Self, AdapterError> { + if let Some(path) = std::env::var_os("AGENT_DESKTOP_MACOS_HELPER_PATH") { + let path = std::path::PathBuf::from(path); + if !path.is_absolute() { + return Err(invalid("Explicit helper path must be absolute")); + } + return Self::read(path); + } + let mut candidates = Vec::new(); + if let Ok(executable) = std::env::current_exe() + && let Some(parent) = executable.parent() + { + candidates.push(parent.join(protocol::HELPER_BASENAME)); + } + if let Some(image) = super::clipboard_helper_dl::containing_image() + && let Some(parent) = image.parent() + { + let candidate = parent.join(protocol::HELPER_BASENAME); + if !candidates.contains(&candidate) { + candidates.push(candidate); + } + } + for candidate in candidates { + match Self::read(candidate) { + Ok(identity) => return Ok(identity), + Err(error) if missing(&error) => {} + Err(error) => return Err(error), + } + } + Err(invalid("No colocated packaged helper was found")) + } + + pub(crate) fn revalidate(&self) -> Result<(), AdapterError> { + let current = Self::read(self.path.clone())?; + if current == *self { + Ok(()) + } else { + Err(invalid("Helper filesystem identity changed during launch")) + } + } + + fn read(path: std::path::PathBuf) -> Result<Self, AdapterError> { + if path.file_name().and_then(|name| name.to_str()) != Some(protocol::HELPER_BASENAME) { + return Err(invalid("Helper path has the wrong packaged basename")); + } + let metadata = std::fs::symlink_metadata(&path).map_err(|error| { + invalid_with_kind( + format!("{}: {error}", path.display()), + if error.kind() == std::io::ErrorKind::NotFound { + "clipboard_helper_not_found" + } else { + "clipboard_helper_invalid" + }, + ) + })?; + let mode = metadata.mode(); + if !metadata.file_type().is_file() + || metadata.uid() != unsafe { libc::geteuid() } + || mode & 0o022 != 0 + || mode & 0o111 == 0 + || metadata.nlink() != 1 + { + return Err(invalid( + "Helper must be an owner-matched, executable, single-link regular file with no group/world writes", + )); + } + Ok(Self { + path, + device: metadata.dev(), + inode: metadata.ino(), + owner: metadata.uid(), + mode, + links: metadata.nlink(), + }) + } +} + +fn invalid(detail: impl Into<String>) -> AdapterError { + invalid_with_kind(detail, "clipboard_helper_invalid") +} + +fn invalid_with_kind(detail: impl Into<String>, kind: &str) -> AdapterError { + AdapterError::new( + ErrorCode::ActionNotSupported, + "The packaged macOS clipboard helper is missing or invalid", + ) + .with_platform_detail(detail) + .with_details(serde_json::json!({ + "kind": kind, + "helper": protocol::HELPER_BASENAME, + })) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) +} + +fn missing(error: &AdapterError) -> bool { + error + .details + .as_ref() + .and_then(|details| details.get("kind")) + .and_then(serde_json::Value::as_str) + == Some("clipboard_helper_not_found") +} diff --git a/crates/macos/src/input/clipboard_helper_process.rs b/crates/macos/src/input/clipboard_helper_process.rs new file mode 100644 index 0000000..779dc35 --- /dev/null +++ b/crates/macos/src/input/clipboard_helper_process.rs @@ -0,0 +1,244 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; +use serde_json::{Value, json}; +use std::io::{Read, Write}; +use std::os::unix::process::CommandExt; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use super::clipboard_helper_identity::HelperIdentity; +use super::clipboard_helper_protocol as protocol; + +const CLEANUP_RESERVE: Duration = Duration::from_millis(100); +const POLL_INTERVAL: Duration = Duration::from_millis(5); + +pub(crate) fn run( + command: &mut Command, + input: &[u8], + deadline: Deadline, + identity: Option<&HelperIdentity>, +) -> Result<Vec<u8>, AdapterError> { + let absolute = Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::timeout("Clipboard helper deadline overflowed"))?; + let work_deadline = absolute + .checked_sub(CLEANUP_RESERVE) + .filter(|limit| *limit > Instant::now()) + .ok_or_else(|| deadline.timeout_error())?; + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .process_group(0); + let mut child = command.spawn().map_err(spawn_error)?; + let process_group = i32::try_from(child.id()).map_err(|_| { + let _ = child.kill(); + let _ = child.wait(); + AdapterError::internal("Clipboard helper PID exceeds the macOS pid_t range") + })?; + let Some(stdin) = child.stdin.take() else { + return cleanup_bare( + child, + process_group, + absolute, + AdapterError::internal("Clipboard helper stdin is unavailable"), + ); + }; + let Some(stdout) = child.stdout.take() else { + drop(stdin); + return cleanup_bare( + child, + process_group, + absolute, + AdapterError::internal("Clipboard helper output FD is unavailable"), + ); + }; + let writer = spawn_writer(stdin, input.to_vec()); + let reader = spawn_reader(stdout); + if let Some(identity) = identity + && let Err(error) = identity.revalidate() + { + return cleanup(child, process_group, writer, reader, absolute, error); + } + let status = match wait_status(&mut child, work_deadline) { + Ok(status) => status, + Err(error) => return cleanup(child, process_group, writer, reader, absolute, error), + }; + let write_result = receive(&writer, work_deadline, "writing request"); + let read_result = receive(&reader, work_deadline, "reading response"); + match (write_result, read_result) { + (Ok(()), Ok(output)) => { + join_thread(writer.1)?; + join_thread(reader.1)?; + if !status.success() && output.is_empty() { + return Err(mark_dispatched(AdapterError::new( + ErrorCode::AppUnresponsive, + format!("macOS clipboard helper exited with {status}"), + ))); + } + Ok(output) + } + (write, read) => cleanup( + child, + process_group, + writer, + reader, + absolute, + write + .err() + .or_else(|| read.err()) + .unwrap_or_else(|| AdapterError::internal("Clipboard helper I/O failed")), + ), + } +} + +fn spawn_error(error: std::io::Error) -> AdapterError { + AdapterError::new( + ErrorCode::ActionNotSupported, + "The packaged macOS clipboard helper could not be started", + ) + .with_platform_detail(error.to_string()) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) +} + +fn wait_status(child: &mut Child, deadline: Instant) -> Result<ExitStatus, AdapterError> { + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(POLL_INTERVAL), + Ok(None) => return Err(AdapterError::timeout("macOS clipboard helper timed out")), + Err(error) => { + return Err(AdapterError::internal(format!( + "Inspect macOS clipboard helper: {error}" + ))); + } + } + } +} + +type IoThread<T> = ( + mpsc::Receiver<std::io::Result<T>>, + std::thread::JoinHandle<()>, +); + +fn spawn_writer(mut stdin: impl Write + Send + 'static, input: Vec<u8>) -> IoThread<()> { + spawn_io(move || stdin.write_all(&input)) +} + +fn spawn_reader(mut output: impl Read + Send + 'static) -> IoThread<Vec<u8>> { + spawn_io(move || { + let mut bytes = Vec::new(); + output + .by_ref() + .take((protocol::MAX_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut bytes)?; + if bytes.len() > protocol::MAX_RESPONSE_BYTES { + return Err(std::io::Error::other( + "clipboard helper response exceeds limit", + )); + } + Ok(bytes) + }) +} + +fn spawn_io<T: Send + 'static>( + operation: impl FnOnce() -> std::io::Result<T> + Send + 'static, +) -> IoThread<T> { + let (sender, receiver) = mpsc::sync_channel(1); + let thread = std::thread::spawn(move || { + let _ = sender.send(operation()); + }); + (receiver, thread) +} + +fn receive<T>(thread: &IoThread<T>, deadline: Instant, phase: &str) -> Result<T, AdapterError> { + thread + .0 + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .map_err(|_| AdapterError::timeout(format!("Timed out {phase} for clipboard helper")))? + .map_err(|error| AdapterError::internal(format!("Clipboard helper {phase}: {error}"))) +} + +fn cleanup( + mut child: Child, + process_group: i32, + writer: IoThread<()>, + reader: IoThread<Vec<u8>>, + deadline: Instant, + error: AdapterError, +) -> Result<Vec<u8>, AdapterError> { + kill_and_reap(&mut child, process_group, deadline); + finish_thread(writer, deadline); + finish_thread(reader, deadline); + Err(mark_dispatched(error)) +} + +fn cleanup_bare( + mut child: Child, + process_group: i32, + deadline: Instant, + error: AdapterError, +) -> Result<Vec<u8>, AdapterError> { + kill_and_reap(&mut child, process_group, deadline); + Err(mark_dispatched(error)) +} + +fn kill_and_reap(child: &mut Child, process_group: i32, deadline: Instant) { + unsafe { + libc::kill(-process_group, libc::SIGKILL); + } + while Instant::now() < deadline { + if child.try_wait().ok().flatten().is_some() { + return; + } + std::thread::sleep(POLL_INTERVAL); + } +} + +fn finish_thread<T>(thread: IoThread<T>, deadline: Instant) { + if thread + .0 + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .is_ok() + { + let _ = thread.1.join(); + } +} + +fn join_thread(thread: std::thread::JoinHandle<()>) -> Result<(), AdapterError> { + thread.join().map_err(|_| { + mark_dispatched(AdapterError::internal( + "Clipboard helper I/O thread panicked", + )) + }) +} + +pub(crate) fn mark_dispatched(mut error: AdapterError) -> AdapterError { + let mut details = error.details.take().unwrap_or_else(|| json!({})); + if let Some(object) = details.as_object_mut() { + object.insert("helper_dispatched".into(), Value::Bool(true)); + } + error.with_details(details) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hard_deadline_kills_the_helper_process_group() { + let mut command = Command::new("/bin/sh"); + command.args(["-c", "sleep 5"]); + let error = run(&mut command, &[], Deadline::after(150).unwrap(), None).unwrap_err(); + + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(error.details.unwrap()["helper_dispatched"], true); + } + + #[test] + fn post_dispatch_thread_failure_is_never_unmarked() { + let error = join_thread(std::thread::spawn(|| panic!("fault"))).unwrap_err(); + + assert_eq!(error.details.unwrap()["helper_dispatched"], true); + } +} diff --git a/crates/macos/src/input/clipboard_helper_protocol.rs b/crates/macos/src/input/clipboard_helper_protocol.rs new file mode 100644 index 0000000..ec64d25 --- /dev/null +++ b/crates/macos/src/input/clipboard_helper_protocol.rs @@ -0,0 +1,136 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use serde_json::{Value, json}; + +pub(crate) const HELPER_BASENAME: &str = "agent-desktop-macos-helper"; +pub(crate) const PROTOCOL_VERSION: &str = "1"; +pub(crate) const BUILD_IDENTITY: &str = env!("AGENT_DESKTOP_MACOS_HELPER_BUILD_ID"); +pub(crate) const OUTPUT_FD: &str = "1"; +pub(crate) const MAX_HEADER_BYTES: usize = 16 * 1024; +pub(crate) const MAX_RESPONSE_BYTES: usize = + agent_desktop_core::MAX_PNG_INPUT_BYTES + MAX_HEADER_BYTES; + +pub(crate) fn is_mutating(operation: &str) -> bool { + operation == "clear" || operation.starts_with("write:") +} + +pub(crate) fn response_header( + token: &str, + operation: &str, + result: Result<(Value, usize), &AdapterError>, +) -> Value { + match result { + Ok((metadata, payload_len)) => json!({ + "protocol": PROTOCOL_VERSION, + "build": BUILD_IDENTITY, + "token": token, + "operation": operation, + "ok": true, + "payload_len": payload_len, + "metadata": metadata, + }), + Err(error) => json!({ + "protocol": PROTOCOL_VERSION, + "build": BUILD_IDENTITY, + "token": token, + "operation": operation, + "ok": false, + "payload_len": 0, + "error_code": error.code.as_str(), + "message": error.message, + "disposition": error.disposition, + }), + } +} + +pub(crate) fn validate_header<'a>( + header: &'a Value, + token: &str, + operation: &str, + payload_len: usize, +) -> Result<&'a Value, AdapterError> { + let object = header.as_object().ok_or_else(protocol_error)?; + let expected = [ + ("protocol", PROTOCOL_VERSION), + ("build", BUILD_IDENTITY), + ("token", token), + ("operation", operation), + ]; + if expected + .iter() + .any(|(key, value)| object.get(*key).and_then(Value::as_str) != Some(*value)) + { + return Err(protocol_error()); + } + let declared = object + .get("payload_len") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .ok_or_else(protocol_error)?; + if declared != payload_len || declared > MAX_RESPONSE_BYTES { + return Err(protocol_error()); + } + if object.get("ok").and_then(Value::as_bool) == Some(true) { + return object.get("metadata").ok_or_else(protocol_error); + } + let code = object + .get("error_code") + .and_then(Value::as_str) + .map(error_code) + .unwrap_or(ErrorCode::ActionFailed); + let message = object + .get("message") + .and_then(Value::as_str) + .unwrap_or("macOS clipboard helper failed"); + let disposition = object + .get("disposition") + .cloned() + .and_then(|value| serde_json::from_value(value).ok()) + .unwrap_or_else(agent_desktop_core::DeliverySemantics::unknown); + Err(AdapterError::new(code, message).with_disposition(disposition)) +} + +pub(crate) fn error_code(code: &str) -> ErrorCode { + match code { + "PERM_DENIED" => ErrorCode::PermDenied, + "INVALID_ARGS" => ErrorCode::InvalidArgs, + "TIMEOUT" => ErrorCode::Timeout, + "APP_UNRESPONSIVE" => ErrorCode::AppUnresponsive, + "ACTION_NOT_SUPPORTED" => ErrorCode::ActionNotSupported, + _ => ErrorCode::ActionFailed, + } +} + +pub(crate) fn protocol_error() -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "macOS clipboard helper returned an invalid authenticated response", + ) + .with_details(json!({ + "kind": "clipboard_helper_protocol", + "complete": false, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn protocol_rejects_wrong_token_version_and_build() { + let valid = response_header("token", "read:text", Ok((json!({}), 0))); + assert!(validate_header(&valid, "token", "read:text", 0).is_ok()); + for field in ["token", "protocol", "build"] { + let mut invalid = valid.clone(); + invalid[field] = Value::String("wrong".into()); + assert!(validate_header(&invalid, "token", "read:text", 0).is_err()); + } + } + + #[test] + fn protocol_rejects_trailing_or_missing_payload() { + let header = response_header("token", "read:image", Ok((json!({}), 7))); + + assert!(validate_header(&header, "token", "read:image", 6).is_err()); + assert!(validate_header(&header, "token", "read:image", 8).is_err()); + } +} diff --git a/crates/macos/src/input/clipboard_image_io.rs b/crates/macos/src/input/clipboard_image_io.rs new file mode 100644 index 0000000..7b2f1df --- /dev/null +++ b/crates/macos/src/input/clipboard_image_io.rs @@ -0,0 +1,187 @@ +use core_foundation::base::{CFType, TCFType, kCFAllocatorDefault, kCFAllocatorNull}; +use core_foundation::boolean::CFBoolean; +use core_foundation::dictionary::CFDictionary; +use core_foundation::number::CFNumber; +use core_foundation::string::CFString; +use core_foundation_sys::base::{CFEqual, CFTypeRef}; +use core_foundation_sys::data::{CFDataCreateWithBytesNoCopy, CFDataRef}; +use core_foundation_sys::dictionary::CFDictionaryRef; +use core_foundation_sys::string::CFStringRef; +use libc::{c_uint, c_ulong}; +use std::ffi::c_void; + +type ImageSource = *const c_void; + +const MAX_PNG_CHUNKS: usize = 65_536; + +#[link(name = "ImageIO", kind = "framework")] +unsafe extern "C" { + fn CGImageSourceCreateWithData(data: CFDataRef, options: CFDictionaryRef) -> ImageSource; + fn CGImageSourceGetCount(source: ImageSource) -> usize; + fn CGImageSourceGetStatus(source: ImageSource) -> i32; + fn CGImageSourceGetStatusAtIndex(source: ImageSource, index: usize) -> i32; + fn CGImageSourceGetType(source: ImageSource) -> CFStringRef; + fn CGImageSourceCreateThumbnailAtIndex( + source: ImageSource, + index: usize, + options: CFDictionaryRef, + ) -> CFTypeRef; + static kCGImageSourceCreateThumbnailFromImageAlways: CFStringRef; + static kCGImageSourceThumbnailMaxPixelSize: CFStringRef; + static kCGImageSourceShouldCacheImmediately: CFStringRef; +} + +#[link(name = "z")] +unsafe extern "C" { + fn crc32(crc: c_ulong, buffer: *const u8, length: c_uint) -> c_ulong; +} + +pub(crate) fn is_complete_png(bytes: &[u8]) -> bool { + if !has_complete_chunk_stream(bytes) { + return false; + } + let Ok(length) = isize::try_from(bytes.len()) else { + return false; + }; + unsafe { + let data = CFDataCreateWithBytesNoCopy( + kCFAllocatorDefault, + bytes.as_ptr(), + length, + kCFAllocatorNull, + ); + if data.is_null() { + return false; + } + let data = CFType::wrap_under_create_rule(data as CFTypeRef); + let source = + CGImageSourceCreateWithData(data.as_CFTypeRef() as CFDataRef, std::ptr::null()); + if source.is_null() { + return false; + } + let source = CFType::wrap_under_create_rule(source as CFTypeRef); + let source_ref = source.as_CFTypeRef() as ImageSource; + let source_type = CGImageSourceGetType(source_ref); + let png_type = CFString::new("public.png"); + let structurally_complete = !source_type.is_null() + && CFEqual(source_type as CFTypeRef, png_type.as_CFTypeRef()) != 0 + && CGImageSourceGetCount(source_ref) == 1 + && CGImageSourceGetStatus(source_ref) == 0 + && CGImageSourceGetStatusAtIndex(source_ref, 0) == 0; + structurally_complete && decodes_thumbnail(source_ref) + } +} + +fn has_complete_chunk_stream(bytes: &[u8]) -> bool { + let mut cursor = 8_usize; + let mut seen_header = false; + let mut seen_image_data = false; + let mut image_data_ended = false; + let mut chunks = 0_usize; + while cursor < bytes.len() { + chunks += 1; + if chunks > MAX_PNG_CHUNKS { + return false; + } + let Some(length) = read_u32(bytes, cursor).and_then(|value| usize::try_from(value).ok()) + else { + return false; + }; + let Some(kind_start) = cursor.checked_add(4) else { + return false; + }; + let Some(data_start) = kind_start.checked_add(4) else { + return false; + }; + let Some(data_end) = data_start.checked_add(length) else { + return false; + }; + let Some(chunk_end) = data_end.checked_add(4) else { + return false; + }; + let Some(kind): Option<&[u8; 4]> = bytes + .get(kind_start..data_start) + .and_then(|value| value.try_into().ok()) + else { + return false; + }; + let Some(data) = bytes.get(data_start..data_end) else { + return false; + }; + let Some(stored_crc) = read_u32(bytes, data_end) else { + return false; + }; + if chunk_end > bytes.len() + || !kind.iter().all(u8::is_ascii_alphabetic) + || !kind[2].is_ascii_uppercase() + || chunk_crc(kind, data) != stored_crc + { + return false; + } + match kind { + b"IHDR" => { + if seen_header || cursor != 8 || length != 13 { + return false; + } + seen_header = true; + } + b"IDAT" => { + if !seen_header || image_data_ended { + return false; + } + seen_image_data = true; + } + b"IEND" => { + return seen_header && seen_image_data && length == 0 && chunk_end == bytes.len(); + } + _ => { + image_data_ended |= seen_image_data; + } + } + cursor = chunk_end; + } + false +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option<u32> { + Some(u32::from_be_bytes( + bytes.get(offset..offset.checked_add(4)?)?.try_into().ok()?, + )) +} + +fn chunk_crc(kind: &[u8; 4], data: &[u8]) -> u32 { + unsafe { + let crc = crc32(0, kind.as_ptr(), kind.len() as c_uint); + crc32(crc, data.as_ptr(), data.len() as c_uint) as u32 + } +} + +unsafe fn decodes_thumbnail(source: ImageSource) -> bool { + unsafe { + let keys = [ + CFString::wrap_under_get_rule(kCGImageSourceCreateThumbnailFromImageAlways).as_CFType(), + CFString::wrap_under_get_rule(kCGImageSourceThumbnailMaxPixelSize).as_CFType(), + CFString::wrap_under_get_rule(kCGImageSourceShouldCacheImmediately).as_CFType(), + ]; + let values = [ + CFBoolean::true_value().as_CFType(), + CFNumber::from(1_i32).as_CFType(), + CFBoolean::true_value().as_CFType(), + ]; + let options = CFDictionary::from_CFType_pairs(&[ + (keys[0].clone(), values[0].clone()), + (keys[1].clone(), values[1].clone()), + (keys[2].clone(), values[2].clone()), + ]); + let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options.as_concrete_TypeRef()); + if image.is_null() { + return false; + } + let _image = CFType::wrap_under_create_rule(image); + true + } +} + +#[cfg(test)] +#[path = "clipboard_image_io_tests.rs"] +mod tests; diff --git a/crates/macos/src/input/clipboard_image_io_tests.rs b/crates/macos/src/input/clipboard_image_io_tests.rs new file mode 100644 index 0000000..0be1e4f --- /dev/null +++ b/crates/macos/src/input/clipboard_image_io_tests.rs @@ -0,0 +1,29 @@ +use super::*; + +#[test] +fn image_io_accepts_complete_png() { + assert!(is_complete_png(&one_pixel_png())); +} + +#[test] +fn image_io_rejects_truncated_png() { + let png = one_pixel_png(); + + assert!(!is_complete_png(&png[..png.len() - 12])); +} + +#[test] +fn image_io_rejects_corrupted_png() { + let mut png = one_pixel_png(); + png[45] ^= 0xff; + + assert!(!is_complete_png(&png)); +} + +fn one_pixel_png() -> [u8; 68] { + [ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 4, + 0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 100, 248, 15, 0, 1, 5, + 1, 1, 39, 24, 227, 102, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ] +} diff --git a/crates/macos/src/input/clipboard_rich.rs b/crates/macos/src/input/clipboard_rich.rs new file mode 100644 index 0000000..f0908e6 --- /dev/null +++ b/crates/macos/src/input/clipboard_rich.rs @@ -0,0 +1,209 @@ +use agent_desktop_core::{ + AdapterError, Deadline, ErrorCode, MAX_PNG_INPUT_BYTES, parse_png_dimensions, +}; +use std::borrow::Cow; +use std::ffi::c_void; + +pub(crate) use super::clipboard_file_urls::{prepare_file_urls, read_file_urls, write_file_urls}; + +type Id = *mut c_void; +type Sel = *mut c_void; +type ImageDimensions = (u32, u32); +type PreparedImage<'a> = (Cow<'a, [u8]>, ImageDimensions); +type OwnedImage = (Vec<u8>, ImageDimensions); + +const MAX_IMAGE_PIXELS: u64 = 64 * 1024 * 1024; +const PNG_HEADER_BYTES: usize = 24; +const PNG_SIGNATURE: &[u8; 8] = b"\x89PNG\r\n\x1a\n"; + +unsafe extern "C" { + fn sel_registerName(name: *const core::ffi::c_char) -> Sel; + fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; + static NSPasteboardTypePNG: Id; +} + +pub(crate) fn prepare_image(bytes: &[u8]) -> Result<PreparedImage<'_>, AdapterError> { + validate_byte_count(bytes.len(), true)?; + let header_dimensions = validate_png_header(bytes, true)?; + let dimensions = parse_png_dimensions(bytes) + .ok_or_else(|| invalid_image("Clipboard images must be complete, valid PNG payloads"))?; + if dimensions != header_dimensions || !super::clipboard_image_io::is_complete_png(bytes) { + return Err(invalid_image("Clipboard PNG failed platform validation")); + } + Ok((Cow::Borrowed(bytes), dimensions)) +} + +pub(crate) fn read_image(pb: Id, deadline: Deadline) -> Result<Option<OwnedImage>, AdapterError> { + ensure_budget(deadline)?; + let png = unsafe { read_data(pb, NSPasteboardTypePNG, deadline) }?; + let result = normalize_image_data(png)?; + ensure_budget(deadline)?; + Ok(result) +} + +fn normalize_image_data(png: Option<Vec<u8>>) -> Result<Option<OwnedImage>, AdapterError> { + let Some(bytes) = png else { + return Ok(None); + }; + let header_dimensions = validate_png_header(&bytes, false)?; + let dimensions = parse_png_dimensions(&bytes) + .ok_or_else(|| clipboard_data_error("Clipboard PNG payload failed complete validation"))?; + if dimensions != header_dimensions || !super::clipboard_image_io::is_complete_png(&bytes) { + return Err(clipboard_data_error( + "Clipboard PNG failed platform validation", + )); + } + Ok(Some((bytes, dimensions))) +} + +pub(crate) fn write_image(pb: Id, png: &[u8], deadline: Deadline) -> Result<bool, AdapterError> { + ensure_budget(deadline)?; + if png.len() > MAX_PNG_INPUT_BYTES { + return Ok(false); + } + unsafe { Ok(write_data(pb, png, NSPasteboardTypePNG)) } +} + +unsafe fn read_data( + pb: Id, + pasteboard_type: Id, + deadline: Deadline, +) -> Result<Option<Vec<u8>>, AdapterError> { + unsafe { + let send: unsafe extern "C" fn(Id, Sel, Id) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let data = send( + pb, + sel_registerName(c"dataForType:".as_ptr()), + pasteboard_type, + ); + if data.is_null() { + return Ok(None); + } + copy_nsdata(data, deadline) + } +} + +unsafe fn copy_nsdata(data: Id, deadline: Deadline) -> Result<Option<Vec<u8>>, AdapterError> { + unsafe { + ensure_budget(deadline)?; + let send_usize: unsafe extern "C" fn(Id, Sel) -> usize = + std::mem::transmute(objc_msgSend as *const c_void); + let len = send_usize(data, sel_registerName(c"length".as_ptr())); + validate_byte_count(len, false)?; + if len == 0 { + return Ok(Some(Vec::new())); + } + let send_ptr: unsafe extern "C" fn(Id, Sel) -> *const u8 = + std::mem::transmute(objc_msgSend as *const c_void); + let ptr = send_ptr(data, sel_registerName(c"bytes".as_ptr())); + if ptr.is_null() { + return Err(clipboard_data_error("NSData returned null bytes")); + } + let bytes = std::slice::from_raw_parts(ptr, len); + validate_png_header(bytes, false)?; + let copy = bytes.to_vec(); + ensure_budget(deadline)?; + Ok(Some(copy)) + } +} + +unsafe fn write_data(pb: Id, bytes: &[u8], pasteboard_type: Id) -> bool { + unsafe extern "C" { + fn objc_getClass(name: *const core::ffi::c_char) -> *mut c_void; + } + unsafe { + let class = objc_getClass(c"NSData".as_ptr()); + if class.is_null() { + return false; + } + let send_data: unsafe extern "C" fn(Id, Sel, *const u8, usize) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let data = send_data( + class as Id, + sel_registerName(c"dataWithBytes:length:".as_ptr()), + bytes.as_ptr(), + bytes.len(), + ); + if data.is_null() { + return false; + } + let send_set: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool = + std::mem::transmute(objc_msgSend as *const c_void); + send_set( + pb, + sel_registerName(c"setData:forType:".as_ptr()), + data, + pasteboard_type, + ) + } +} + +fn validate_png_header(data: &[u8], argument: bool) -> Result<(u32, u32), AdapterError> { + let dimensions = data + .get(..PNG_HEADER_BYTES) + .filter(|header| header.get(..8) == Some(PNG_SIGNATURE.as_slice())) + .filter(|header| header.get(8..12) == Some(13_u32.to_be_bytes().as_slice())) + .filter(|header| header.get(12..16) == Some(b"IHDR")) + .and_then(|header| { + Some(( + u32::from_be_bytes(header.get(16..20)?.try_into().ok()?), + u32::from_be_bytes(header.get(20..24)?.try_into().ok()?), + )) + }); + let Some(dimensions) = dimensions else { + return Err(if argument { + invalid_image("Clipboard image is missing a valid PNG header") + } else { + clipboard_data_error("Clipboard image is missing a valid PNG header") + }); + }; + validate_dimensions(dimensions, argument)?; + Ok(dimensions) +} + +fn validate_byte_count(bytes: usize, argument: bool) -> Result<(), AdapterError> { + if bytes <= MAX_PNG_INPUT_BYTES { + return Ok(()); + } + Err(if argument { + invalid_image("Image exceeds the 64 MiB encoded-data budget") + } else { + clipboard_data_error("Clipboard image exceeds the 64 MiB encoded-data budget") + }) +} + +fn validate_dimensions(dimensions: (u32, u32), argument: bool) -> Result<(), AdapterError> { + let (width, height) = dimensions; + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .ok_or_else(|| clipboard_data_error("Image pixel count overflowed"))?; + if width > 0 && height > 0 && pixels <= MAX_IMAGE_PIXELS { + return Ok(()); + } + Err(if argument { + invalid_image("Image exceeds the decoded-image budget") + } else { + clipboard_data_error("Clipboard image exceeds the decoded-image budget") + }) +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn invalid_image(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::InvalidArgs, message) +} + +fn clipboard_data_error(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::ActionFailed, message) +} + +#[cfg(test)] +#[path = "clipboard_rich_tests.rs"] +mod tests; diff --git a/crates/macos/src/input/clipboard_rich_tests.rs b/crates/macos/src/input/clipboard_rich_tests.rs new file mode 100644 index 0000000..bd3d564 --- /dev/null +++ b/crates/macos/src/input/clipboard_rich_tests.rs @@ -0,0 +1,88 @@ +use super::super::clipboard_file_urls::file_url_to_path; +use super::*; + +#[test] +fn complete_png_reports_verified_dimensions() { + let (_, dimensions) = prepare_image(&one_pixel_png()).unwrap(); + + assert_eq!(dimensions, (1, 1)); +} + +#[test] +fn undersized_png_is_rejected_without_panicking() { + assert!(prepare_image(&[1, 2, 3]).is_err()); +} + +#[test] +fn file_url_to_path_round_trips_a_plain_path() { + let path = file_url_to_path("file:///tmp/agent-desktop-test.txt") + .expect("well-formed file:// URL must decode to a path"); + assert_eq!(path, "/tmp/agent-desktop-test.txt"); +} + +#[test] +fn file_url_to_path_decodes_percent_escaped_spaces() { + let path = file_url_to_path("file:///tmp/agent%20desktop/note.txt") + .expect("percent-encoded file:// URL must decode"); + assert_eq!(path, "/tmp/agent desktop/note.txt"); +} + +#[test] +fn file_url_to_path_rejects_non_file_scheme() { + assert!(file_url_to_path("https://example.com/a.txt").is_none()); +} + +#[test] +fn non_png_image_payload_is_rejected_before_native_decode() { + let error = prepare_image(b"II*\0unbounded-tiff").expect_err("TIFF is unsupported"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +#[test] +fn valid_png_is_passed_through_without_decode_or_reencode() { + let png = one_pixel_png(); + let (prepared, dimensions) = prepare_image(&png).unwrap(); + + assert!(matches!(prepared, std::borrow::Cow::Borrowed(_))); + assert_eq!(prepared.as_ref(), png); + assert_eq!(dimensions, (1, 1)); +} + +#[test] +fn file_url_validation_is_all_or_nothing() { + let paths = vec!["/tmp/good".to_string(), String::new()]; + + assert!(prepare_file_urls(&paths).is_err()); +} + +#[test] +fn file_urls_reject_remote_hosts_and_relative_paths() { + assert!(file_url_to_path("file://server/share/note.txt").is_none()); + assert!(prepare_file_urls(&["relative/note.txt".into()]).is_err()); +} + +#[test] +fn file_urls_reject_embedded_nul() { + assert!(file_url_to_path("file:///tmp/a%00b").is_none()); + assert!(prepare_file_urls(&["/tmp/a\0b".into()]).is_err()); +} + +#[test] +fn oversized_dimensions_are_rejected_from_the_header() { + let mut png = one_pixel_png(); + png[16..20].copy_from_slice(&100_000_u32.to_be_bytes()); + png[20..24].copy_from_slice(&100_000_u32.to_be_bytes()); + + let error = prepare_image(&png).expect_err("pixel bomb metadata must be rejected"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +fn one_pixel_png() -> [u8; 68] { + [ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 4, + 0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 100, 248, 15, 0, 1, 5, + 1, 1, 39, 24, 227, 102, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ] +} diff --git a/crates/macos/src/input/clipboard_runtime.rs b/crates/macos/src/input/clipboard_runtime.rs new file mode 100644 index 0000000..b73a418 --- /dev/null +++ b/crates/macos/src/input/clipboard_runtime.rs @@ -0,0 +1,128 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::ffi::c_void; + +pub(crate) type Pasteboard = *mut c_void; +type Class = *mut c_void; +type Sel = *mut c_void; + +const PASTEBOARD_ACCESS_ALWAYS_DENY: isize = 3; + +unsafe extern "C" { + fn objc_getClass(name: *const core::ffi::c_char) -> Class; + fn sel_registerName(name: *const core::ffi::c_char) -> Sel; + fn objc_msgSend(receiver: Pasteboard, sel: Sel, ...) -> Pasteboard; +} + +pub(crate) struct AutoreleasePool { + pool: Pasteboard, +} + +impl AutoreleasePool { + pub(crate) fn new() -> Result<Self, AdapterError> { + crate::system::cocoa_runtime::ensure_cocoa_multithreaded()?; + unsafe { + let class = objc_getClass(c"NSAutoreleasePool".as_ptr()); + if class.is_null() { + return Err(unavailable("NSAutoreleasePool class was not found")); + } + let send: unsafe extern "C" fn(Pasteboard, Sel) -> Pasteboard = + std::mem::transmute(objc_msgSend as *const c_void); + let pool = send( + send(class as Pasteboard, sel_registerName(c"alloc".as_ptr())), + sel_registerName(c"init".as_ptr()), + ); + if pool.is_null() { + return Err(unavailable("NSAutoreleasePool initialization failed")); + } + Ok(Self { pool }) + } + } +} + +impl Drop for AutoreleasePool { + fn drop(&mut self) { + unsafe { + let send: unsafe extern "C" fn(Pasteboard, Sel) = + std::mem::transmute(objc_msgSend as *const c_void); + send(self.pool, sel_registerName(c"drain".as_ptr())); + } + } +} + +pub(crate) fn pasteboard() -> Result<Pasteboard, AdapterError> { + unsafe { + let class = objc_getClass(c"NSPasteboard".as_ptr()); + if class.is_null() { + return Err(unavailable("NSPasteboard class was not found")); + } + let send: unsafe extern "C" fn(Pasteboard, Sel) -> Pasteboard = + std::mem::transmute(objc_msgSend as *const c_void); + let pasteboard = send( + class as Pasteboard, + sel_registerName(c"generalPasteboard".as_ptr()), + ); + if pasteboard.is_null() { + return Err(unavailable("NSPasteboard generalPasteboard returned null")); + } + Ok(pasteboard) + } +} + +pub(crate) unsafe fn clear_contents(pasteboard: Pasteboard) -> isize { + unsafe { + let send: unsafe extern "C" fn(Pasteboard, Sel) -> isize = + std::mem::transmute(objc_msgSend as *const c_void); + send(pasteboard, sel_registerName(c"clearContents".as_ptr())) + } +} + +pub(crate) unsafe fn change_count(pasteboard: Pasteboard) -> isize { + unsafe { + let send: unsafe extern "C" fn(Pasteboard, Sel) -> isize = + std::mem::transmute(objc_msgSend as *const c_void); + send(pasteboard, sel_registerName(c"changeCount".as_ptr())) + } +} + +pub(crate) fn ensure_read_access(pasteboard: Pasteboard) -> Result<(), AdapterError> { + unsafe { + let access_selector = sel_registerName(c"accessBehavior".as_ptr()); + let responds: unsafe extern "C" fn(Pasteboard, Sel, Sel) -> bool = + std::mem::transmute(objc_msgSend as *const c_void); + if !responds( + pasteboard, + sel_registerName(c"respondsToSelector:".as_ptr()), + access_selector, + ) { + return Ok(()); + } + let send: unsafe extern "C" fn(Pasteboard, Sel) -> isize = + std::mem::transmute(objc_msgSend as *const c_void); + if !read_access_denied(send(pasteboard, access_selector)) { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::PermDenied, + "System clipboard read access is denied", + ) + .with_suggestion( + "Allow pasteboard access for the app that launches agent-desktop in System Settings, then retry.", + )) + } +} + +fn read_access_denied(behavior: isize) -> bool { + behavior == PASTEBOARD_ACCESS_ALWAYS_DENY +} + +fn unavailable(detail: impl Into<String>) -> AdapterError { + AdapterError::new(ErrorCode::ActionFailed, "System clipboard is unavailable") + .with_platform_detail(detail) + .with_suggestion( + "Retry from an interactive macOS login session after the pasteboard service is available.", + ) +} + +#[cfg(test)] +#[path = "clipboard_runtime_tests.rs"] +mod tests; diff --git a/crates/macos/src/input/clipboard_runtime_tests.rs b/crates/macos/src/input/clipboard_runtime_tests.rs new file mode 100644 index 0000000..3164a9a --- /dev/null +++ b/crates/macos/src/input/clipboard_runtime_tests.rs @@ -0,0 +1,9 @@ +use super::*; + +#[test] +fn only_always_deny_is_rejected_without_prompting() { + assert!(!read_access_denied(0)); + assert!(!read_access_denied(1)); + assert!(!read_access_denied(2)); + assert!(read_access_denied(3)); +} diff --git a/crates/macos/src/input/clipboard_tests.rs b/crates/macos/src/input/clipboard_tests.rs new file mode 100644 index 0000000..66a37d6 --- /dev/null +++ b/crates/macos/src/input/clipboard_tests.rs @@ -0,0 +1,122 @@ +use super::*; +use crate::input::interactive_test::{is_worker, run_bounded}; +use agent_desktop_core::{ClipboardContent, ClipboardFormat, Deadline, ImageBuffer, ImageFormat}; +use core_foundation::{base::TCFType, string::CFString}; +use std::ffi::c_void; +use std::time::Duration; + +type Class = *mut c_void; + +unsafe extern "C" { + fn objc_getClass(name: *const core::ffi::c_char) -> Class; +} + +#[test] +fn native_clipboard_contract_is_bounded() { + if is_worker("clipboard") { + let _pool = AutoreleasePool::new().expect("autorelease pool is available"); + let pb = unique_pasteboard().expect("isolated pasteboard is available"); + let result = exercise_clipboard(pb); + unsafe { release_globally(pb) }; + result.expect("isolated clipboard contract succeeds"); + } else { + run_bounded( + "native_clipboard_contract_is_bounded", + "clipboard", + Duration::from_secs(15), + ); + } +} + +fn exercise_clipboard(pb: Id) -> Result<(), AdapterError> { + let deadline = Deadline::after(5_000)?; + replace_on( + pb, + "text", + deadline, + |pb, deadline| unsafe { write_string(pb, "original clipboard value", deadline) }, + |pb, deadline| unsafe { + ensure_read_budget(deadline)?; + Ok(read_string(pb)?.as_deref() == Some("original clipboard value")) + }, + )?; + + set_content_on( + pb, + &ClipboardContent::Text(String::from("replacement")), + deadline, + )?; + assert_eq!( + unsafe { get_content_from(pb, ClipboardFormat::Text, deadline) }?, + Some(ClipboardContent::Text(String::from("replacement"))) + ); + + let image = ClipboardContent::Image(ImageBuffer { + data: one_pixel_png().to_vec(), + format: ImageFormat::Png, + width: 1, + height: 1, + scale_factor: 1.0, + }); + set_content_on(pb, &image, deadline)?; + let image_result = unsafe { get_content_from(pb, ClipboardFormat::Image, deadline) }?; + assert!(matches!( + image_result, + Some(ClipboardContent::Image(ImageBuffer { + width: 1, + height: 1, + .. + })) + )); + Ok(()) +} + +fn unique_pasteboard() -> Result<Id, AdapterError> { + unsafe { + let class = objc_getClass(c"NSPasteboard".as_ptr()); + if class.is_null() { + return Err(pasteboard_unavailable("NSPasteboard class was not found")); + } + let name = CFString::new(&format!( + "com.norolabs.agent-desktop.tests.{}", + std::process::id() + )); + let send: unsafe extern "C" fn(Class, Sel, Id) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let pb = send( + class, + sel_registerName(c"pasteboardWithName:".as_ptr()), + name.as_concrete_TypeRef() as Id, + ); + if pb.is_null() { + return Err(pasteboard_unavailable( + "NSPasteboard pasteboardWithName returned null", + )); + } + Ok(pb) + } +} + +fn pasteboard_unavailable(detail: &str) -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + "Isolated test pasteboard is unavailable", + ) + .with_platform_detail(detail) +} + +unsafe fn release_globally(pb: Id) { + unsafe { + let send: unsafe extern "C" fn(Id, Sel) = + std::mem::transmute(objc_msgSend as *const c_void); + send(pb, sel_registerName(c"releaseGlobally".as_ptr())); + } +} + +fn one_pixel_png() -> [u8; 68] { + [ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 4, + 0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 100, 248, 15, 0, 1, 5, + 1, 1, 39, 24, 227, 102, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, + ] +} diff --git a/crates/macos/src/input/clipboard_transaction.rs b/crates/macos/src/input/clipboard_transaction.rs new file mode 100644 index 0000000..48815ca --- /dev/null +++ b/crates/macos/src/input/clipboard_transaction.rs @@ -0,0 +1,197 @@ +use super::clipboard_runtime::{Pasteboard, change_count, clear_contents}; +use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics, ErrorCode}; +use std::time::Duration; + +pub(super) fn replace_on( + pasteboard: Pasteboard, + kind: &str, + deadline: Deadline, + write: impl FnOnce(Pasteboard, Deadline) -> Result<bool, AdapterError>, + verify: impl FnOnce(Pasteboard, Deadline) -> Result<bool, AdapterError>, +) -> Result<(), AdapterError> { + let mutation_deadline = reserve_cleanup_budget(deadline)?; + ensure_not_started(mutation_deadline)?; + let ownership = unsafe { clear_contents(pasteboard) }; + if ownership < 0 { + return Err(delivery_uncertain( + kind, + "clearContents returned an invalid ownership token", + )); + } + if !owns(ownership, unsafe { change_count(pasteboard) }) { + return Err(delivered_unverified( + kind, + "clipboard ownership changed immediately after clearContents", + )); + } + if let Err(error) = ensure_not_started(mutation_deadline) { + return Err(post_clear_failure( + pasteboard, ownership, kind, deadline, error, + )); + } + match write(pasteboard, mutation_deadline) { + Ok(true) => {} + Ok(false) => { + return Err(post_clear_failure( + pasteboard, + ownership, + kind, + deadline, + AdapterError::new( + ErrorCode::ActionFailed, + format!("Clipboard {kind} write failed"), + ), + )); + } + Err(error) => { + return Err(post_clear_failure( + pasteboard, ownership, kind, deadline, error, + )); + } + } + if !owns(ownership, unsafe { change_count(pasteboard) }) { + return Err(delivered_unverified( + kind, + "clipboard ownership changed before verification", + )); + } + let intended = verify(pasteboard, mutation_deadline).map_err(|error| { + delivered_unverified(kind, &format!("content verification failed: {error}")) + })?; + if !owns(ownership, unsafe { change_count(pasteboard) }) { + return Err(delivered_unverified( + kind, + "clipboard ownership changed during verification", + )); + } + if !intended { + return Err(post_clear_failure( + pasteboard, + ownership, + kind, + deadline, + AdapterError::new( + ErrorCode::ActionFailed, + format!("Clipboard {kind} did not retain the intended content"), + ), + )); + } + ensure_verified_before_return(deadline) +} + +pub(super) fn clear_verified( + pasteboard: Pasteboard, + deadline: Deadline, +) -> Result<(), AdapterError> { + ensure_not_started(deadline)?; + let ownership = unsafe { clear_contents(pasteboard) }; + if ownership < 0 || !owns(ownership, unsafe { change_count(pasteboard) }) { + return Err(delivery_uncertain( + "clear", + "clipboard ownership could not be verified", + )); + } + ensure_verified_before_return(deadline) +} + +fn post_clear_failure( + pasteboard: Pasteboard, + ownership: isize, + kind: &str, + deadline: Deadline, + cause: AdapterError, +) -> AdapterError { + match clear_if_owned(pasteboard, ownership, deadline) { + Ok(cleaned) => AdapterError::new( + cause.code.clone(), + format!("Clipboard {kind} replacement failed after clearing prior content"), + ) + .with_platform_detail(cause.to_string()) + .with_details(serde_json::json!({ + "cleanup_verified": cleaned, + "concurrent_change": !cleaned, + })) + .with_disposition(DeliverySemantics::delivered_unverified()), + Err(cleanup_error) => { + delivery_uncertain(kind, &format!("{cause}; cleanup failed: {cleanup_error}")) + } + } +} + +fn clear_if_owned( + pasteboard: Pasteboard, + ownership: isize, + deadline: Deadline, +) -> Result<bool, AdapterError> { + if !owns(ownership, unsafe { change_count(pasteboard) }) { + return Ok(false); + } + ensure_not_started(deadline)?; + let cleanup_ownership = unsafe { clear_contents(pasteboard) }; + if cleanup_ownership < 0 || !owns(cleanup_ownership, unsafe { change_count(pasteboard) }) { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Clipboard cleanup could not be verified", + )); + } + Ok(true) +} + +fn reserve_cleanup_budget(deadline: Deadline) -> Result<Deadline, AdapterError> { + let remaining = deadline.remaining(); + if remaining <= Duration::from_millis(2) { + return Err(deadline + .timeout_error() + .with_disposition(DeliverySemantics::not_delivered())); + } + let reserve = (remaining / 2).min(Duration::from_millis(250)); + Deadline::from_duration(remaining.saturating_sub(reserve)) +} + +fn owns(ownership: isize, current: isize) -> bool { + ownership >= 0 && ownership == current +} + +fn ensure_not_started(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline + .timeout_error() + .with_disposition(DeliverySemantics::not_delivered())) + } else { + Ok(()) + } +} + +fn ensure_verified_before_return(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline + .timeout_error() + .with_disposition(DeliverySemantics::delivered_verified())) + } else { + Ok(()) + } +} + +fn delivered_unverified(kind: &str, reason: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("Clipboard {kind} delivery could not be verified"), + ) + .with_platform_detail(reason) + .with_suggestion("Inspect the clipboard state before deciding whether to repeat the write") + .with_disposition(DeliverySemantics::delivered_unverified()) +} + +fn delivery_uncertain(kind: &str, reason: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("Clipboard {kind} delivery is uncertain"), + ) + .with_platform_detail(reason) + .with_suggestion("Inspect the clipboard state before deciding whether to repeat the write") + .with_disposition(DeliverySemantics::uncertain()) +} + +#[cfg(test)] +#[path = "clipboard_transaction_tests.rs"] +mod tests; diff --git a/crates/macos/src/input/clipboard_transaction_tests.rs b/crates/macos/src/input/clipboard_transaction_tests.rs new file mode 100644 index 0000000..38a050f --- /dev/null +++ b/crates/macos/src/input/clipboard_transaction_tests.rs @@ -0,0 +1,41 @@ +use super::*; +use agent_desktop_core::{DeliveryDisposition, RetryDisposition}; + +#[test] +fn ownership_requires_the_clear_contents_change_count() { + assert!(owns(42, 42)); + assert!(!owns(42, 43)); + assert!(!owns(-1, -1)); +} + +#[test] +fn ownership_loss_after_clear_is_delivered_and_unsafe_to_retry() { + let error = delivered_unverified("text", "ownership changed after clearContents"); + + assert_eq!( + error.disposition.delivery(), + DeliveryDisposition::DeliveredUnverified + ); + assert_eq!(error.disposition.retry(), RetryDisposition::Unsafe); +} + +#[test] +fn verified_delivery_after_deadline_is_not_retryable() { + let deadline = Deadline::after(0).expect("zero-duration deadline is valid"); + let error = ensure_verified_before_return(deadline).expect_err("deadline must expire"); + + assert_eq!( + error.disposition.delivery(), + DeliveryDisposition::DeliveredVerified + ); + assert_eq!(error.disposition.retry(), RetryDisposition::Unsafe); +} + +#[test] +fn cleanup_is_reserved_before_mutation() { + let deadline = Deadline::after(1_000).expect("deadline is valid"); + let mutation = reserve_cleanup_budget(deadline).expect("cleanup budget can be reserved"); + + assert!(mutation.remaining() < deadline.remaining()); + assert!(mutation.remaining() >= std::time::Duration::from_millis(700)); +} diff --git a/crates/macos/src/input/interactive_test.rs b/crates/macos/src/input/interactive_test.rs new file mode 100644 index 0000000..1749152 --- /dev/null +++ b/crates/macos/src/input/interactive_test.rs @@ -0,0 +1,27 @@ +use std::{process::Command, time::Duration}; + +const WORKER_ENV: &str = "AGENT_DESKTOP_INTERACTIVE_TEST_WORKER"; + +pub(crate) fn is_worker(name: &str) -> bool { + std::env::var(WORKER_ENV).is_ok_and(|value| value == name) +} + +pub(crate) fn run_bounded(test_filter: &str, worker: &str, timeout: Duration) { + let executable = std::env::current_exe().expect("current test executable is available"); + let mut command = Command::new(executable); + command + .arg(test_filter) + .arg("--nocapture") + .env(WORKER_ENV, worker); + let output = + crate::system::process::run_with_timeout(&mut command, "interactive test worker", timeout) + .unwrap_or_else(|error| { + panic!("interactive test worker did not finish safely: {error}") + }); + assert!( + output.status.success(), + "interactive test worker failed: {}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/macos/src/input/keyboard.rs b/crates/macos/src/input/keyboard.rs index fdaad99..5947d7a 100644 --- a/crates/macos/src/input/keyboard.rs +++ b/crates/macos/src/input/keyboard.rs @@ -1,343 +1,113 @@ -use agent_desktop_core::{ - action::KeyCombo, - error::{AdapterError, ErrorCode}, -}; +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, KeyCombo}; #[cfg(target_os = "macos")] -mod imp { - use super::*; - use accessibility_sys::{ - AXUIElementCreateSystemWide, AXUIElementPostKeyboardEvent, kAXErrorCannotComplete, - kAXErrorSuccess, - }; - use std::time::Duration; +pub(crate) fn synthesize_key( + combo: &KeyCombo, + target_pid: Option<i32>, + deadline: Deadline, +) -> Result<(), AdapterError> { + tracing::debug!( + key = combo.key, + modifiers = ?combo.modifiers, + target_pid, + "keyboard: synthesize atomic key press" + ); + let key_code = crate::input::keyboard_map::key_name_to_code(&combo.key)?; + crate::input::keyboard_event::post_key( + key_code, + crate::input::mouse::event_flags(&combo.modifiers), + target_pid, + deadline, + (0, 1), + ) +} - pub fn synthesize_key(combo: &KeyCombo) -> Result<(), AdapterError> { - tracing::debug!( - "keyboard: synthesize_key {}{}", - if combo.modifiers.is_empty() { - String::new() - } else { - format!( - "{}+", - combo - .modifiers - .iter() - .map(|m| format!("{m:?}")) - .collect::<Vec<_>>() - .join("+") - ) - }, - combo.key - ); - let key_code = key_name_to_code(&combo.key)?; +#[cfg(target_os = "macos")] +pub(crate) fn reject_standalone_key_state( + _combo: &KeyCombo, + _down: bool, +) -> Result<(), AdapterError> { + Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "Standalone key-down/key-up is unavailable in stateless mode", + ) + .with_details(serde_json::json!({ + "raw_input_emitted": false, + "requires_daemon_owned_transaction": true, + })) + .with_suggestion( + "Use the atomic 'press' command; spanning key holds require a daemon-owned session that can release keys after disconnect", + )) +} - let sys_wide = unsafe { AXUIElementCreateSystemWide() }; - if sys_wide.is_null() { - return Err(AdapterError::internal( - "Failed to create system-wide AX element", - )); - } +#[cfg(target_os = "macos")] +pub(crate) fn synthesize_text( + text: &str, + target_pid: i32, + deadline: Deadline, + verify_target: impl FnMut(Deadline) -> Result<(), AdapterError>, +) -> Result<(), AdapterError> { + tracing::debug!( + characters = text.chars().count(), + target_pid, + "keyboard: synthesize Unicode text" + ); + crate::input::keyboard_event::post_text(text, target_pid, deadline, verify_target) +} - let mut pressed_mods = Vec::new(); - for m in &combo.modifiers { - let mod_code = modifier_keycode(m); - let err = unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, mod_code, true) }; - if err != kAXErrorSuccess { - release_modifiers(sys_wide, &pressed_mods); - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - return Err(AdapterError::internal(format!( - "AXUIElementPostKeyboardEvent modifier-down failed (err={err})" - ))); - } - pressed_mods.push(m.clone()); - } - - let err_down = post_event(sys_wide, key_code, true); - let err_up = post_event(sys_wide, key_code, false); - - if !combo.modifiers.is_empty() { - release_modifiers(sys_wide, &combo.modifiers); - } - - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - - if err_down != kAXErrorSuccess { - return Err(AdapterError::internal(format!( - "AXUIElementPostKeyboardEvent key-down failed (err={err_down})" - ))); - } - if err_up != kAXErrorSuccess { - release_key_system_wide(key_code); - return Err(AdapterError::internal(format!( - "AXUIElementPostKeyboardEvent key-up failed (err={err_up})" - ))); - } - Ok(()) - } - - pub fn synthesize_key_state(combo: &KeyCombo, down: bool) -> Result<(), AdapterError> { - tracing::debug!( - "keyboard: synthesize_key_state key={} down={down}", - combo.key - ); - let key_code = key_name_to_code(&combo.key)?; - let sys_wide = unsafe { AXUIElementCreateSystemWide() }; - if sys_wide.is_null() { - return Err(AdapterError::internal( - "Failed to create system-wide AX element", - )); - } - - let result = (|| { - if down { - let mut pressed_mods = Vec::new(); - for m in &combo.modifiers { - if let Err(err) = post_checked(sys_wide, modifier_keycode(m), true, 0, 1) { - release_modifiers(sys_wide, &pressed_mods); - return Err(err); - } - pressed_mods.push(m.clone()); - } - if let Err(err) = post_checked(sys_wide, key_code, true, 0, 1) { - release_modifiers(sys_wide, &pressed_mods); - return Err(err); - } - Ok(()) - } else { - let key_result = post_checked(sys_wide, key_code, false, 0, 1); - if key_result.is_err() { - release_key_system_wide(key_code); - } - let mut first_mod_err: Option<AdapterError> = None; - for m in combo.modifiers.iter().rev() { - if let Err(err) = post_checked(sys_wide, modifier_keycode(m), false, 0, 1) { - if first_mod_err.is_none() { - first_mod_err = Some(err); - } - } - } - key_result.and(first_mod_err.map_or(Ok(()), Err)) - } - })(); - - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - result - } - - pub fn synthesize_text(text: &str) -> Result<(), AdapterError> { - tracing::debug!("keyboard: synthesize_text {} chars", text.chars().count()); - let sys_wide = unsafe { AXUIElementCreateSystemWide() }; - if sys_wide.is_null() { - return Err(AdapterError::internal( - "Failed to create system-wide AX element", - )); - } - - let total = text.chars().count(); - let mut delivered = 0usize; - let result = (|| { - for ch in text.chars() { - if ch == '\n' { - post_char_key(sys_wide, 36, delivered, total)?; - } else if let Some(code) = char_to_keycode(ch) { - let needs_shift = ch.is_ascii_uppercase() || is_shifted_char(ch); - if needs_shift { - post_checked(sys_wide, 56, true, delivered, total)?; - } - let char_result = post_char_key(sys_wide, code, delivered, total); - if needs_shift { - let shift_up = post_checked(sys_wide, 56, false, delivered, total); - char_result.and(shift_up)?; - } else { - char_result?; - } - } else { - return Err(AdapterError::new( - ErrorCode::ActionNotSupported, - format!("Cannot synthesize character '{ch}' with keyboard fallback"), - ) - .with_suggestion("Use set-value for non-ASCII text when supported.")); - } - delivered += 1; - std::thread::sleep(Duration::from_millis(4)); - } - Ok(()) - })(); - - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - result - } - - pub fn synthesize_keycode(key_code: u16, repeats: u32) -> Result<(), AdapterError> { - let sys_wide = unsafe { AXUIElementCreateSystemWide() }; - if sys_wide.is_null() { - return Err(AdapterError::internal( - "Failed to create system-wide AX element", - )); - } - let result = (|| { - for i in 0..repeats { - post_char_key(sys_wide, key_code, i as usize, repeats as usize)?; - std::thread::sleep(Duration::from_millis(10)); - } - Ok(()) - })(); - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - result - } - - pub fn synthesize_key_for_element( - el: &crate::tree::AXElement, - combo: &KeyCombo, - ) -> Result<(), AdapterError> { - let key_code = key_name_to_code(&combo.key)?; - let mut pressed = Vec::new(); - for m in &combo.modifiers { - if let Err(err) = post_checked(el.0, modifier_keycode(m), true, 0, 1) { - release_modifiers(el.0, &pressed); - return Err(err); - } - pressed.push(m.clone()); - } - let key_result = post_char_key(el.0, key_code, 0, 1); - let mut release_result = Ok(()); - for m in pressed.iter().rev() { - if let Err(err) = post_checked(el.0, modifier_keycode(m), false, 0, 1) { - if release_result.is_ok() { - release_result = Err(err); - } - } - } - if key_result.is_err() { - release_key_system_wide(key_code); - } - if key_result.is_err() || release_result.is_err() { - release_modifiers_system_wide(&pressed); - } - key_result.and(release_result) - } - - fn release_modifiers( - el: accessibility_sys::AXUIElementRef, - modifiers: &[agent_desktop_core::action::Modifier], - ) { - for m in modifiers.iter().rev() { - let code = modifier_keycode(m); - unsafe { AXUIElementPostKeyboardEvent(el, 0, code, false) }; - } - } - - fn release_modifiers_system_wide(modifiers: &[agent_desktop_core::action::Modifier]) { - let sys_wide = unsafe { AXUIElementCreateSystemWide() }; - if sys_wide.is_null() { - return; - } - release_modifiers(sys_wide, modifiers); - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - } - - fn release_key_system_wide(key_code: u16) { - let sys_wide = unsafe { AXUIElementCreateSystemWide() }; - if sys_wide.is_null() { - return; - } - unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, key_code, false) }; - unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - } - - fn post_char_key( - el: accessibility_sys::AXUIElementRef, - code: u16, - delivered: usize, - total: usize, - ) -> Result<(), AdapterError> { - post_checked(el, code, true, delivered, total)?; - post_checked(el, code, false, delivered, total) - } - - fn post_checked( - el: accessibility_sys::AXUIElementRef, - code: u16, - down: bool, - delivered: usize, - total: usize, - ) -> Result<(), AdapterError> { - let err = post_event(el, code, down); - if err == kAXErrorSuccess { - return Ok(()); - } - if err == kAXErrorCannotComplete { - std::thread::sleep(Duration::from_millis(10)); - let retry = post_event(el, code, down); - if retry == kAXErrorSuccess { - return Ok(()); - } - return Err(post_error(retry, delivered, total)); - } - Err(post_error(err, delivered, total)) - } - - fn post_event(el: accessibility_sys::AXUIElementRef, code: u16, down: bool) -> i32 { - unsafe { AXUIElementPostKeyboardEvent(el, 0, code, down) } - } - - fn post_error(err: i32, delivered: usize, total: usize) -> AdapterError { - AdapterError::new( - ErrorCode::ActionFailed, - format!( - "Keyboard synthesis failed after {delivered}/{total} characters delivered (err={err})" - ), - ) - .with_suggestion("Retry after refreshing the target snapshot and confirming focus.") - } - - fn modifier_keycode(m: &agent_desktop_core::action::Modifier) -> u16 { - crate::input::keyboard_map::modifier_keycode(m) - } - - fn is_shifted_char(ch: char) -> bool { - crate::input::keyboard_map::is_shifted_char(ch) - } - - fn char_to_keycode(ch: char) -> Option<u16> { - crate::input::keyboard_map::char_to_keycode(ch) - } - - fn key_name_to_code(key: &str) -> Result<u16, AdapterError> { - crate::input::keyboard_map::key_name_to_code(key) - } +#[cfg(target_os = "macos")] +pub(crate) fn preflight_text(text: &str, deadline: Deadline) -> Result<(), AdapterError> { + crate::input::keyboard_event::preflight_text(text, deadline) } #[cfg(not(target_os = "macos"))] -mod imp { - use super::*; - - pub fn synthesize_key(_combo: &KeyCombo) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("synthesize_key")) - } - - pub fn synthesize_key_state(_combo: &KeyCombo, _down: bool) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("synthesize_key_state")) - } - - pub fn synthesize_text(_text: &str) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("synthesize_text")) - } - - pub fn synthesize_keycode(_key_code: u16, _repeats: u32) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("synthesize_keycode")) - } - - pub fn synthesize_key_for_element( - _el: &crate::tree::AXElement, - _combo: &KeyCombo, - ) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("synthesize_key_for_element")) - } +pub(crate) fn synthesize_key( + _combo: &KeyCombo, + _target_pid: Option<i32>, + _deadline: Deadline, +) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("synthesize_key")) } -pub use imp::{ - synthesize_key, synthesize_key_for_element, synthesize_key_state, synthesize_keycode, - synthesize_text, -}; +#[cfg(not(target_os = "macos"))] +pub(crate) fn reject_standalone_key_state( + _combo: &KeyCombo, + _down: bool, +) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("key_state")) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn synthesize_text( + _text: &str, + _target_pid: i32, + _deadline: Deadline, + _verify_target: impl FnMut(Deadline) -> Result<(), AdapterError>, +) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("synthesize_text")) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn preflight_text(_text: &str, _deadline: Deadline) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("synthesize_text")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn standalone_key_state_is_rejected_without_emission() { + let error = reject_standalone_key_state( + &KeyCombo { + key: "shift".into(), + modifiers: Vec::new(), + }, + true, + ) + .expect_err("stateless holds must fail closed"); + + assert_eq!(error.code, ErrorCode::ActionNotSupported); + assert_eq!(error.details.unwrap()["raw_input_emitted"], false); + } +} diff --git a/crates/macos/src/input/keyboard_event.rs b/crates/macos/src/input/keyboard_event.rs new file mode 100644 index 0000000..2059146 --- /dev/null +++ b/crates/macos/src/input/keyboard_event.rs @@ -0,0 +1,251 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; +use core_graphics::event::{CGEvent, CGEventFlags, CGEventTapLocation}; +use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; +use std::time::Duration; + +const KEY_SETTLE: Duration = Duration::from_millis(4); +const TEXT_CHUNK_BUDGET: Duration = Duration::from_millis(400); +const MAX_TEXT_UTF16: usize = 1_000_000; +const TEXT_CHUNK_UTF16: usize = 32; + +pub(crate) fn post_key( + key_code: u16, + flags: CGEventFlags, + target_pid: Option<i32>, + deadline: Deadline, + progress: (usize, usize), +) -> Result<(), AdapterError> { + let mut delivery = crate::actions::DeliveryTracker::from_delivered_units(progress.0); + ensure_budget(deadline, progress.1, delivery)?; + let source = event_source().map_err(|error| delivery.annotate(error))?; + let down = create_key_event(&source, key_code, true, flags) + .map_err(|error| delivery.annotate(error))?; + let up = create_key_event(&source, key_code, false, flags) + .map_err(|error| delivery.annotate(error))?; + post_pair((down, up), target_pid, deadline, &mut delivery, progress.1) +} + +pub(crate) fn post_text( + text: &str, + target_pid: i32, + deadline: Deadline, + mut verify_target: impl FnMut(Deadline) -> Result<(), AdapterError>, +) -> Result<(), AdapterError> { + preflight_text(text, deadline)?; + let chunks = text_chunks(text)?; + let total = chunks.len(); + let mut delivery = crate::actions::DeliveryTracker::default(); + for chunk in chunks { + ensure_budget(deadline, total, delivery)?; + verify_target(deadline).map_err(|error| { + delivery.annotate(error.with_details(serde_json::json!({ + "delivered_chunks": delivery.delivered_units(), + "total_chunks": total, + }))) + })?; + let source = event_source().map_err(|error| delivery.annotate(error))?; + let down = create_key_event(&source, 0, true, CGEventFlags::empty()) + .map_err(|error| delivery.annotate(error))?; + down.set_string_from_utf16_unchecked(&chunk); + let up = create_key_event(&source, 0, false, CGEventFlags::empty()) + .map_err(|error| delivery.annotate(error))?; + post_pair((down, up), Some(target_pid), deadline, &mut delivery, total)?; + } + Ok(()) +} + +pub(crate) fn preflight_text(text: &str, deadline: Deadline) -> Result<(), AdapterError> { + let chunks = planned_chunk_count(text)?; + let multiplier = u32::try_from(chunks) + .map_err(|_| AdapterError::new(ErrorCode::InvalidArgs, "Text payload is too large"))?; + let required = TEXT_CHUNK_BUDGET + .checked_mul(multiplier) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Text payload is too large"))?; + let remaining = deadline.remaining(); + if remaining < required { + return Err(crate::actions::DeliveryTracker::default().annotate( + AdapterError::timeout("Text cannot be delivered safely within the remaining deadline") + .with_details(serde_json::json!({ + "delivered_chunks": 0, + "total_chunks": chunks, + "required_ms": required.as_millis(), + "remaining_ms": remaining.as_millis(), + })), + )); + } + Ok(()) +} + +fn post_pair( + events: (CGEvent, CGEvent), + target_pid: Option<i32>, + deadline: Deadline, + delivery: &mut crate::actions::DeliveryTracker, + total: usize, +) -> Result<(), AdapterError> { + let (down, up) = events; + let mut release = KeyReleaseGuard { + event: Some(up), + target_pid, + }; + post(&down, target_pid); + delivery.mark_delivered(); + sleep_bounded(deadline, KEY_SETTLE, *delivery, total)?; + let up = release + .event + .take() + .ok_or_else(|| AdapterError::internal("Keyboard release guard lost its key-up event")) + .map_err(|error| delivery.annotate(error))?; + post(&up, target_pid); + ensure_budget(deadline, total, *delivery) +} + +struct KeyReleaseGuard { + event: Option<CGEvent>, + target_pid: Option<i32>, +} + +impl Drop for KeyReleaseGuard { + fn drop(&mut self) { + if let Some(event) = self.event.take() { + post(&event, self.target_pid); + } + } +} + +fn create_key_event( + source: &CGEventSource, + key_code: u16, + down: bool, + flags: CGEventFlags, +) -> Result<CGEvent, AdapterError> { + let event = CGEvent::new_keyboard_event(source.clone(), key_code, down) + .map_err(|()| AdapterError::internal("CGEvent::new_keyboard_event failed"))?; + event.set_flags(flags); + Ok(event) +} + +fn event_source() -> Result<CGEventSource, AdapterError> { + CGEventSource::new(CGEventSourceStateID::HIDSystemState) + .map_err(|()| AdapterError::internal("Failed to create keyboard CGEventSource")) +} + +fn post(event: &CGEvent, target_pid: Option<i32>) { + if let Some(pid) = target_pid { + event.post_to_pid(pid); + } else { + event.post(CGEventTapLocation::HID); + } +} + +fn sleep_bounded( + deadline: Deadline, + duration: Duration, + delivery: crate::actions::DeliveryTracker, + total: usize, +) -> Result<(), AdapterError> { + let pause = deadline + .remaining_slice(duration) + .map_err(|error| delivery.annotate(error))?; + std::thread::sleep(pause); + if pause < duration { + ensure_budget(deadline, total, delivery) + } else { + Ok(()) + } +} + +fn ensure_budget( + deadline: Deadline, + total: usize, + delivery: crate::actions::DeliveryTracker, +) -> Result<(), AdapterError> { + if !deadline.is_expired() { + return Ok(()); + } + Err( + delivery.annotate(deadline.timeout_error().with_details(serde_json::json!({ + "delivered_chunks": delivery.delivered_units(), + "total_chunks": total, + }))), + ) +} + +fn text_chunks(text: &str) -> Result<Vec<Vec<u16>>, AdapterError> { + let _ = planned_chunk_count(text)?; + let mut chunks = Vec::new(); + let mut current = Vec::with_capacity(TEXT_CHUNK_UTF16); + for character in text.chars() { + let mut encoded = [0_u16; 2]; + let encoded = character.encode_utf16(&mut encoded); + if !current.is_empty() && current.len() + encoded.len() > TEXT_CHUNK_UTF16 { + chunks.push(std::mem::take(&mut current)); + } + current.extend_from_slice(encoded); + } + if !current.is_empty() { + chunks.push(current); + } + Ok(chunks) +} + +fn planned_chunk_count(text: &str) -> Result<usize, AdapterError> { + let mut units = 0_usize; + let mut chunks = 0_usize; + let mut current = 0_usize; + for character in text.chars() { + let encoded = character.len_utf16(); + units = units.checked_add(encoded).ok_or_else(text_too_large)?; + if units > MAX_TEXT_UTF16 { + return Err(text_too_large()); + } + if current > 0 && current + encoded > TEXT_CHUNK_UTF16 { + chunks += 1; + current = 0; + } + current += encoded; + } + Ok(chunks + usize::from(current > 0)) +} + +fn text_too_large() -> AdapterError { + AdapterError::new( + ErrorCode::InvalidArgs, + "Text input must not exceed 1000000 UTF-16 code units", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_chunks_preserve_unicode_without_splitting_surrogates() { + let text = format!("{}😀tail", "a".repeat(TEXT_CHUNK_UTF16 - 1)); + let chunks = text_chunks(&text).unwrap(); + let roundtrip = String::from_utf16(&chunks.concat()).unwrap(); + + assert_eq!(roundtrip, text); + assert!(chunks.iter().all(|chunk| chunk.len() <= TEXT_CHUNK_UTF16)); + } + + #[test] + fn text_budget_rejects_unbounded_payloads() { + let text = "x".repeat(MAX_TEXT_UTF16 + 1); + let error = text_chunks(&text).expect_err("oversized text must fail before CGEvent"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + } + + #[test] + fn impossible_delivery_deadline_fails_before_input() { + let deadline = Deadline::after(1).unwrap(); + let error = preflight_text("payload", deadline).expect_err("deadline must reject plan"); + + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!( + error.disposition, + agent_desktop_core::DeliverySemantics::not_delivered() + ); + } +} diff --git a/crates/macos/src/input/keyboard_map.rs b/crates/macos/src/input/keyboard_map.rs index a5a7eeb..d6a3da8 100644 --- a/crates/macos/src/input/keyboard_map.rs +++ b/crates/macos/src/input/keyboard_map.rs @@ -1,98 +1,4 @@ -use agent_desktop_core::{ - action::Modifier, - error::{AdapterError, ErrorCode}, -}; - -pub(crate) fn modifier_keycode(m: &Modifier) -> u16 { - match m { - Modifier::Cmd => 55, - Modifier::Shift => 56, - Modifier::Alt => 58, - Modifier::Ctrl => 59, - } -} - -pub(crate) fn is_shifted_char(ch: char) -> bool { - matches!( - ch, - '!' | '@' - | '#' - | '$' - | '%' - | '^' - | '&' - | '*' - | '(' - | ')' - | '_' - | '+' - | '{' - | '}' - | '|' - | ':' - | '"' - | '<' - | '>' - | '?' - | '~' - ) -} - -pub(crate) fn char_to_keycode(ch: char) -> Option<u16> { - let lower = ch.to_ascii_lowercase(); - Some(match lower { - 'a' => 0, - 'b' => 11, - 'c' => 8, - 'd' => 2, - 'e' => 14, - 'f' => 3, - 'g' => 5, - 'h' => 4, - 'i' => 34, - 'j' => 38, - 'k' => 40, - 'l' => 37, - 'm' => 46, - 'n' => 45, - 'o' => 31, - 'p' => 35, - 'q' => 12, - 'r' => 15, - 's' => 1, - 't' => 17, - 'u' => 32, - 'v' => 9, - 'w' => 13, - 'x' => 7, - 'y' => 16, - 'z' => 6, - '0' | ')' => 29, - '1' | '!' => 18, - '2' | '@' => 19, - '3' | '#' => 20, - '4' | '$' => 21, - '5' | '%' => 23, - '6' | '^' => 22, - '7' | '&' => 26, - '8' | '*' => 28, - '9' | '(' => 25, - ' ' => 49, - '-' | '_' => 27, - '=' | '+' => 24, - '[' | '{' => 33, - ']' | '}' => 30, - '\\' | '|' => 42, - ';' | ':' => 41, - '\'' | '"' => 39, - ',' | '<' => 43, - '.' | '>' => 47, - '/' | '?' => 44, - '`' | '~' => 50, - '\t' => 48, - _ => return None, - }) -} +use agent_desktop_core::{AdapterError, ErrorCode}; pub(crate) fn key_name_to_code(key: &str) -> Result<u16, AdapterError> { let code = match key { @@ -175,44 +81,9 @@ pub(crate) fn key_name_to_code(key: &str) -> Result<u16, AdapterError> { #[cfg(test)] mod tests { - use agent_desktop_core::{action::Modifier, error::ErrorCode}; + use agent_desktop_core::ErrorCode; - use super::{char_to_keycode, is_shifted_char, key_name_to_code, modifier_keycode}; - - #[test] - fn modifier_keycodes_match_macos_virtual_key_codes() { - assert_eq!(modifier_keycode(&Modifier::Cmd), 55); - assert_eq!(modifier_keycode(&Modifier::Shift), 56); - assert_eq!(modifier_keycode(&Modifier::Alt), 58); - assert_eq!(modifier_keycode(&Modifier::Ctrl), 59); - } - - #[test] - fn shifted_chars_are_detected() { - assert!(is_shifted_char('!')); - assert!(is_shifted_char('@')); - assert!(is_shifted_char('~')); - assert!(is_shifted_char('_')); - assert!(!is_shifted_char('a')); - assert!(!is_shifted_char('1')); - assert!(!is_shifted_char('A')); - assert!(!is_shifted_char(' ')); - } - - #[test] - fn char_to_keycode_lowercases_before_lookup() { - assert_eq!(char_to_keycode('a'), Some(0)); - assert_eq!(char_to_keycode('A'), Some(0)); - assert_eq!(char_to_keycode('z'), Some(6)); - assert_eq!(char_to_keycode(' '), Some(49)); - assert_eq!(char_to_keycode('\t'), Some(48)); - } - - #[test] - fn char_to_keycode_returns_none_for_unmapped_chars() { - assert_eq!(char_to_keycode('€'), None); - assert_eq!(char_to_keycode('\n'), None); - } + use super::key_name_to_code; #[test] fn named_key_aliases_resolve_to_same_code() { diff --git a/crates/macos/src/input/mod.rs b/crates/macos/src/input/mod.rs index b8291ff..d5656e3 100644 --- a/crates/macos/src/input/mod.rs +++ b/crates/macos/src/input/mod.rs @@ -1,5 +1,17 @@ +mod adapter; pub(crate) mod blocked_combo; -pub mod clipboard; -pub mod keyboard; +pub(crate) mod clipboard; +#[cfg(all(test, target_os = "macos", feature = "interactive-tests"))] +mod interactive_test; +pub(crate) mod keyboard; +#[cfg(target_os = "macos")] +mod keyboard_event; pub(crate) mod keyboard_map; -pub mod mouse; +pub(crate) mod mouse; +#[cfg(target_os = "macos")] +mod mouse_drag; +#[cfg(target_os = "macos")] +mod mouse_drag_state; +pub(crate) mod mouse_move; +pub(crate) mod mouse_scroll; +mod owned_object; diff --git a/crates/macos/src/input/mouse.rs b/crates/macos/src/input/mouse.rs index a941033..7efe5f9 100644 --- a/crates/macos/src/input/mouse.rs +++ b/crates/macos/src/input/mouse.rs @@ -1,254 +1,241 @@ +#[cfg(not(target_os = "macos"))] +use agent_desktop_core::DragParams; use agent_desktop_core::{ - action::{DragParams, MouseButton, MouseEvent, MouseEventKind}, - error::AdapterError, + AdapterError, Deadline, ErrorCode, Modifier, MouseButton, MouseEvent, MouseEventKind, Point, }; #[cfg(target_os = "macos")] mod imp { use super::*; use core_graphics::event::{ - CGEvent, CGEventTapLocation, CGEventType, CGMouseButton, EventField, ScrollEventUnit, + CGEvent, CGEventFlags, CGEventTapLocation, CGEventType, CGMouseButton, EventField, }; use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; use core_graphics::geometry::CGPoint; - pub fn synthesize_mouse(event: MouseEvent) -> Result<(), AdapterError> { - tracing::debug!( - "mouse: {:?} {:?} at ({:.0}, {:.0})", - event.kind, - event.button, - event.point.x, - event.point.y - ); - let point = CGPoint::new(event.point.x, event.point.y); - let cg_button = to_cg_button(&event.button); - match event.kind { - MouseEventKind::Move => post_event(CGEventType::MouseMoved, point, cg_button), - MouseEventKind::Down => post_event(down_type(&event.button), point, cg_button), - MouseEventKind::Up => post_event(up_type(&event.button), point, cg_button), - MouseEventKind::Click { count } => { - synthesize_click(point, cg_button, &event.button, count) - } - } - } - - pub fn synthesize_drag(params: DragParams) -> Result<(), AdapterError> { - drag_sequence(params).map_err(|err| { - if err.suggestion.is_some() { - return err; - } - err.with_suggestion( - "The drag was aborted: the button was released back at the origin (best-effort) and no drop was committed at the destination. The cursor ends at the origin. Re-check the source state before retrying.", - ) + pub(crate) fn event_flags(modifiers: &[Modifier]) -> CGEventFlags { + modifiers.iter().fold(CGEventFlags::empty(), |flags, m| { + flags + | match m { + Modifier::Meta => CGEventFlags::CGEventFlagCommand, + Modifier::Shift => CGEventFlags::CGEventFlagShift, + Modifier::Alt => CGEventFlags::CGEventFlagAlternate, + Modifier::Ctrl => CGEventFlags::CGEventFlagControl, + } }) } - fn drag_sequence(params: DragParams) -> Result<(), AdapterError> { + pub fn synthesize_mouse(event: MouseEvent, deadline: Deadline) -> Result<(), AdapterError> { + synthesize_mouse_after(event, deadline, &mut || Ok(())) + } + + pub(crate) fn synthesize_mouse_after( + event: MouseEvent, + deadline: Deadline, + verify_target: &mut dyn FnMut() -> Result<(), AdapterError>, + ) -> Result<(), AdapterError> { tracing::debug!( - "mouse: drag ({:.0},{:.0}) -> ({:.0},{:.0}) duration={}ms", - params.from.x, - params.from.y, - params.to.x, - params.to.y, - params.duration_ms.unwrap_or(300) + "mouse: {:?} {:?} at ({:.0}, {:.0}) modifiers={:?}", + event.kind, + event.button, + event.point.x, + event.point.y, + event.modifiers ); - use std::thread::sleep; - use std::time::Duration; - - const PICKUP_DELAY_MS: u64 = 200; - const DEFAULT_DROP_DELAY_MS: u64 = 500; - const DWELL_TICK_MS: u64 = 16; - - let from = CGPoint::new(params.from.x, params.from.y); - let to = CGPoint::new(params.to.x, params.to.y); - let duration_ms = params.duration_ms.unwrap_or(300); - let steps = (duration_ms / DWELL_TICK_MS).max(4) as usize; - let step_delay = Duration::from_millis(duration_ms / steps as u64); - let source = event_source()?; - - post_event_with_source( - &source, - CGEventType::LeftMouseDown, - from, - CGMouseButton::Left, - )?; - let mut release = MouseUpGuard { - source: &source, - origin: from, - armed: true, - }; - sleep(Duration::from_millis(PICKUP_DELAY_MS)); - - for i in 1..=steps { - let t = i as f64 / steps as f64; - let x = params.from.x + (params.to.x - params.from.x) * t; - let y = params.from.y + (params.to.y - params.from.y) * t; - post_event_with_source( - &source, - CGEventType::LeftMouseDragged, - CGPoint::new(x, y), - CGMouseButton::Left, - )?; - sleep(step_delay); - } - - dwell_over_destination( - &source, - to, - params.drop_delay_ms.unwrap_or(DEFAULT_DROP_DELAY_MS), - DWELL_TICK_MS, - )?; - release.release_at(to) - } - - /// Releases the left mouse button exactly once. Every fallible step between - /// `LeftMouseDown` and the final `LeftMouseUp` would otherwise leave the - /// button logically held down system-wide on error. The happy path calls - /// `release_at(to)`, which disarms only after the up event actually posts. - /// On any early return, `Drop` cancels the gesture by dragging back to the - /// origin and releasing there — never at the unreached destination, where - /// CGEvent's embedded coordinates would silently commit the aborted drag - /// as a completed drop. The cancel is best-effort twice over: the - /// corrective posts themselves can fail (typically the same systemic - /// CGEventSource failure that aborted the drag, leaving the button held - /// and the cursor wherever the last successful event put it), and a - /// drop target under the origin still sees a self-drop, which most - /// targets treat as a no-op. - struct MouseUpGuard<'a> { - source: &'a CGEventSource, - origin: CGPoint, - armed: bool, - } - - impl MouseUpGuard<'_> { - fn release_at(&mut self, point: CGPoint) -> Result<(), AdapterError> { - post_event_with_source( - self.source, - CGEventType::LeftMouseUp, - point, - CGMouseButton::Left, - )?; - self.armed = false; - Ok(()) - } - } - - impl Drop for MouseUpGuard<'_> { - fn drop(&mut self) { - if self.armed { - let _ = post_event_with_source( - self.source, - CGEventType::LeftMouseDragged, - self.origin, - CGMouseButton::Left, - ); - let _ = post_event_with_source( - self.source, - CGEventType::LeftMouseUp, - self.origin, - CGMouseButton::Left, - ); + validate_point(&event.point)?; + ensure_budget(deadline, crate::actions::DeliveryTracker::default())?; + let point = CGPoint::new(event.point.x, event.point.y); + let cg_button = to_cg_button(&event.button); + let flags = event_flags(&event.modifiers); + match event.kind { + MouseEventKind::Move => { + let mut delivery = crate::actions::DeliveryTracker::default(); + crate::input::mouse_move::post_move_events( + point, + cg_button, + flags, + deadline, + &mut delivery, + ) + } + MouseEventKind::Down | MouseEventKind::Up => Err(standalone_state_error()), + MouseEventKind::Click { count } => { + agent_desktop_core::validate_mouse_click_count(count)?; + synthesize_click( + ClickSpec { + point, + cg_button, + button: &event.button, + count, + flags, + }, + deadline, + verify_target, + ) + } + MouseEventKind::Wheel { delta_x, delta_y } => { + crate::input::mouse_scroll::synthesize_scroll_at( + event.point, + (wheel_lines_to_i32(delta_y)?, wheel_lines_to_i32(delta_x)?), + &event.modifiers, + deadline, + ) } } } - /// Holds the dragged item over the destination while the drop target - /// activates. Posting `LeftMouseDragged` on each tick (instead of a dead - /// sleep) keeps the destination engaged so the release registers as a - /// drop rather than a bare drag — macOS targets can drop the highlight if - /// no movement arrives. A zero delay still posts one settling event. - fn dwell_over_destination( - source: &CGEventSource, - to: CGPoint, - drop_delay_ms: u64, - tick_ms: u64, - ) -> Result<(), AdapterError> { - use std::thread::sleep; - use std::time::Duration; + pub(crate) fn standalone_state_error() -> AdapterError { + AdapterError::new( + ErrorCode::ActionNotSupported, + "Standalone mouse-down/mouse-up is unavailable in stateless mode", + ) + .with_details(serde_json::json!({ + "raw_input_emitted": false, + "requires_daemon_owned_transaction": true, + })) + .with_suggestion( + "Use atomic 'mouse-click' or 'drag'; spanning holds require a daemon-owned session that can release buttons after disconnect", + ) + } - let ticks = drop_delay_ms.div_ceil(tick_ms).max(1); - for _ in 0..ticks { - post_event_with_source( - source, - CGEventType::LeftMouseDragged, - to, - CGMouseButton::Left, - )?; - sleep(Duration::from_millis(tick_ms)); + pub(crate) fn validate_point(point: &Point) -> Result<(), AdapterError> { + const MAX_COORDINATE: f64 = 1_000_000.0; + if !point.x.is_finite() + || !point.y.is_finite() + || point.x.abs() > MAX_COORDINATE + || point.y.abs() > MAX_COORDINATE + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Mouse coordinates must be finite and within -1000000..=1000000", + )); } Ok(()) } + pub(crate) fn wheel_lines_to_i32(delta: f64) -> Result<i32, AdapterError> { + if !delta.is_finite() || delta < f64::from(i32::MIN) || delta > f64::from(i32::MAX) { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Wheel line delta must be a finite 32-bit value", + )); + } + let rounded = delta.round(); + if rounded == 0.0 && delta != 0.0 { + return Ok(if delta.is_sign_positive() { 1 } else { -1 }); + } + Ok(rounded as i32) + } + + struct ClickSpec<'a> { + point: CGPoint, + cg_button: CGMouseButton, + button: &'a MouseButton, + count: u32, + flags: CGEventFlags, + } + fn synthesize_click( - point: CGPoint, - cg_button: CGMouseButton, - button: &MouseButton, - count: u32, + spec: ClickSpec, + deadline: Deadline, + verify_target: &mut dyn FnMut() -> Result<(), AdapterError>, ) -> Result<(), AdapterError> { - let down_ty = down_type(button); - let up_ty = up_type(button); - for i in 1..=count { - let down = create_event(down_ty, point, cg_button)?; - let up = create_event(up_ty, point, cg_button)?; + let down_ty = down_type(spec.button); + let up_ty = up_type(spec.button); + let mut delivery = crate::actions::DeliveryTracker::default(); + crate::input::mouse_move::post_move_events( + spec.point, + spec.cg_button, + spec.flags, + deadline, + &mut delivery, + )?; + for i in 1..=spec.count { + ensure_budget(deadline, delivery)?; + let down = create_event(down_ty, spec.point, spec.cg_button, spec.flags) + .map_err(|error| delivery.annotate(error))?; + let up = create_event(up_ty, spec.point, spec.cg_button, spec.flags) + .map_err(|error| delivery.annotate(error))?; set_click_count(&down, i as i64); set_click_count(&up, i as i64); + ensure_budget(deadline, delivery)?; + verify_target().map_err(|error| delivery.annotate(error))?; down.post(CGEventTapLocation::HID); - std::thread::sleep(std::time::Duration::from_millis(10)); + delivery.mark_delivered(); + let mut release = ClickReleaseGuard { event: Some(up) }; + sleep_bounded(deadline, std::time::Duration::from_millis(10), delivery)?; + let up = release + .event + .take() + .ok_or_else(|| { + AdapterError::internal("Mouse click release guard lost its pending event") + }) + .map_err(|error| delivery.annotate(error))?; up.post(CGEventTapLocation::HID); - if i < count { - std::thread::sleep(std::time::Duration::from_millis(30)); + ensure_budget(deadline, delivery)?; + if i < spec.count { + sleep_bounded(deadline, std::time::Duration::from_millis(30), delivery)?; } } Ok(()) } + struct ClickReleaseGuard { + event: Option<CGEvent>, + } + + impl Drop for ClickReleaseGuard { + fn drop(&mut self) { + if let Some(event) = self.event.take() { + event.post(CGEventTapLocation::HID); + } + } + } + fn set_click_count(event: &CGEvent, count: i64) { event.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, count); } - fn create_event( + pub(crate) fn create_event( event_type: CGEventType, point: CGPoint, button: CGMouseButton, + flags: CGEventFlags, ) -> Result<CGEvent, AdapterError> { let source = event_source()?; - create_event_with_source(&source, event_type, point, button) + create_event_with_source(&source, event_type, point, button, flags) } - fn create_event_with_source( + pub(crate) fn create_event_with_source( source: &CGEventSource, event_type: CGEventType, point: CGPoint, button: CGMouseButton, + flags: CGEventFlags, ) -> Result<CGEvent, AdapterError> { - CGEvent::new_mouse_event(source.clone(), event_type, point, button) - .map_err(|()| AdapterError::internal("CGEvent::new_mouse_event failed")) + let event = CGEvent::new_mouse_event(source.clone(), event_type, point, button) + .map_err(|()| AdapterError::internal("CGEvent::new_mouse_event failed"))?; + event.set_flags(flags); + Ok(event) } - fn event_source() -> Result<CGEventSource, AdapterError> { + pub(crate) fn event_source() -> Result<CGEventSource, AdapterError> { CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|()| AdapterError::internal("Failed to create CGEventSource")) } - fn post_event( - event_type: CGEventType, - point: CGPoint, - button: CGMouseButton, - ) -> Result<(), AdapterError> { - let ev = create_event(event_type, point, button)?; - ev.post(CGEventTapLocation::HID); - Ok(()) - } - - fn post_event_with_source( + pub(crate) fn post_event_with_source( source: &CGEventSource, - event_type: CGEventType, - point: CGPoint, - button: CGMouseButton, + event: (CGEventType, CGPoint, CGMouseButton, CGEventFlags), + deadline: Deadline, + delivery: &mut crate::actions::DeliveryTracker, ) -> Result<(), AdapterError> { - let ev = create_event_with_source(source, event_type, point, button)?; + ensure_budget(deadline, *delivery)?; + let ev = create_event_with_source(source, event.0, event.1, event.2, event.3) + .map_err(|error| delivery.annotate(error))?; ev.post(CGEventTapLocation::HID); - Ok(()) + delivery.mark_delivered(); + ensure_budget(deadline, *delivery) } fn to_cg_button(button: &MouseButton) -> CGMouseButton { @@ -275,34 +262,34 @@ mod imp { } } - pub fn synthesize_scroll_at(x: f64, y: f64, dy: i32, dx: i32) -> Result<(), AdapterError> { - tracing::debug!("mouse: scroll at ({x:.0},{y:.0}) dy={dy} dx={dx}"); - use core_graphics::geometry::CGPoint; + pub(crate) fn sleep_bounded( + deadline: Deadline, + duration: std::time::Duration, + delivery: crate::actions::DeliveryTracker, + ) -> Result<(), AdapterError> { + let pause = deadline + .remaining_slice(duration) + .map_err(|error| delivery.annotate(error))?; + if pause < duration { + std::thread::sleep(pause); + return ensure_budget(deadline, delivery); + } + std::thread::sleep(duration); + ensure_budget(deadline, delivery) + } - unsafe extern "C" { - fn CGEventCreateScrollWheelEvent( - source: *const std::ffi::c_void, - units: u32, - wheel_count: u32, - wheel1: i32, - wheel2: i32, - ) -> *mut std::ffi::c_void; - fn CGEventSetLocation(event: *mut std::ffi::c_void, point: CGPoint); - fn CGEventPost(tap: u32, event: *mut std::ffi::c_void); + pub(crate) fn ensure_budget( + deadline: Deadline, + delivery: crate::actions::DeliveryTracker, + ) -> Result<(), AdapterError> { + if !deadline.is_expired() { + return Ok(()); } - - let event = unsafe { - CGEventCreateScrollWheelEvent(std::ptr::null(), ScrollEventUnit::LINE, 2, dy, dx) - }; - if event.is_null() { - return Err(AdapterError::internal("scroll event creation failed")); - } - unsafe { - CGEventSetLocation(event, CGPoint::new(x, y)); - CGEventPost(0, event); - core_foundation::base::CFRelease(event as _); - } - Ok(()) + Err( + delivery.annotate(deadline.timeout_error().with_details(serde_json::json!({ + "delivered_events": delivery.delivered_units(), + }))), + ) } } @@ -310,17 +297,43 @@ mod imp { mod imp { use super::*; - pub fn synthesize_mouse(_event: MouseEvent) -> Result<(), AdapterError> { + pub fn synthesize_mouse(_event: MouseEvent, _deadline: Deadline) -> Result<(), AdapterError> { Err(AdapterError::not_supported("mouse_event")) } - pub fn synthesize_drag(_params: DragParams) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("drag")) + pub(crate) fn synthesize_mouse_after( + _event: MouseEvent, + _deadline: Deadline, + _verify_target: &mut dyn FnMut() -> Result<(), AdapterError>, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("mouse_event")) } - pub fn synthesize_scroll_at(_x: f64, _y: f64, _dy: i32, _dx: i32) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("scroll")) + pub fn synthesize_drag(_params: DragParams, _deadline: Deadline) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("drag")) } } -pub use imp::{synthesize_drag, synthesize_mouse, synthesize_scroll_at}; +pub(crate) use imp::{synthesize_mouse, synthesize_mouse_after}; + +#[cfg(target_os = "macos")] +pub(crate) use crate::input::mouse_drag::synthesize_drag; + +#[cfg(not(target_os = "macos"))] +pub(crate) use imp::synthesize_drag; + +#[cfg(target_os = "macos")] +pub(crate) use imp::{ + create_event_with_source, ensure_budget, event_flags, event_source, post_event_with_source, + sleep_bounded, validate_point, +}; + +#[cfg(all(test, target_os = "macos", feature = "interactive-tests"))] +pub(crate) use imp::{create_event, standalone_state_error, wheel_lines_to_i32}; + +#[cfg(all(test, target_os = "macos", not(feature = "interactive-tests")))] +pub(crate) use imp::{standalone_state_error, wheel_lines_to_i32}; + +#[cfg(all(test, target_os = "macos"))] +#[path = "mouse_tests.rs"] +mod tests; diff --git a/crates/macos/src/input/mouse_drag.rs b/crates/macos/src/input/mouse_drag.rs new file mode 100644 index 0000000..8c7d293 --- /dev/null +++ b/crates/macos/src/input/mouse_drag.rs @@ -0,0 +1,296 @@ +use agent_desktop_core::{AdapterError, Deadline, DragParams, ErrorCode}; +use core_graphics::event::{CGEvent, CGEventFlags, CGEventTapLocation, CGEventType, CGMouseButton}; +use core_graphics::event_source::CGEventSource; +use core_graphics::geometry::CGPoint; +use std::time::Duration; + +const DEFAULT_DURATION_MS: u64 = 300; +const PICKUP_DELAY_MS: u64 = 200; +const DEFAULT_DROP_DELAY_MS: u64 = 500; +const DWELL_TICK_MS: u64 = 16; +const MAX_STEPS: u64 = 4_096; +const MAX_DRAG_MS: u64 = 60_000; +const MAX_DROP_DELAY_MS: u64 = 30_000; + +pub(crate) fn synthesize_drag(params: DragParams, deadline: Deadline) -> Result<(), AdapterError> { + validate_drag(¶ms)?; + preflight_drag(¶ms, deadline)?; + drag_sequence(params, deadline) +} + +fn drag_sequence(params: DragParams, deadline: Deadline) -> Result<(), AdapterError> { + tracing::debug!( + "mouse: drag ({:.0},{:.0}) -> ({:.0},{:.0}) duration={}ms", + params.from.x, + params.from.y, + params.to.x, + params.to.y, + params.duration_ms.unwrap_or(DEFAULT_DURATION_MS) + ); + let from = CGPoint::new(params.from.x, params.from.y); + let to = CGPoint::new(params.to.x, params.to.y); + let duration_ms = params.duration_ms.unwrap_or(DEFAULT_DURATION_MS); + let steps = duration_ms.div_ceil(DWELL_TICK_MS).clamp(1, MAX_STEPS); + let step_delay = Duration::from_secs_f64(duration_ms as f64 / steps as f64 / 1_000.0); + let pre_delivery = crate::actions::DeliveryTracker::default(); + let source = + crate::input::mouse::event_source().map_err(|error| pre_delivery.annotate(error))?; + let down = crate::input::mouse::create_event_with_source( + &source, + CGEventType::LeftMouseDown, + from, + CGMouseButton::Left, + CGEventFlags::empty(), + ) + .map_err(|error| pre_delivery.annotate(error))?; + let mut release = DragReleaseGuard::prepare(&source, from, to) + .map_err(|error| pre_delivery.annotate(error))?; + crate::input::mouse::ensure_budget(deadline, pre_delivery)?; + release.arm(); + down.post(CGEventTapLocation::HID); + release.mark_down_posted(); + + let outcome = (|| { + crate::input::mouse::ensure_budget(deadline, release.delivery())?; + crate::input::mouse::sleep_bounded( + deadline, + Duration::from_millis(PICKUP_DELAY_MS), + release.delivery(), + )?; + + for index in 1..=steps { + let progress = index as f64 / steps as f64; + let point = CGPoint::new( + params.from.x + (params.to.x - params.from.x) * progress, + params.from.y + (params.to.y - params.from.y) * progress, + ); + crate::input::mouse::post_event_with_source( + &source, + ( + CGEventType::LeftMouseDragged, + point, + CGMouseButton::Left, + CGEventFlags::empty(), + ), + deadline, + release.delivery_mut(), + )?; + crate::input::mouse::sleep_bounded(deadline, step_delay, release.delivery())?; + } + + dwell_over_destination( + &source, + to, + params.drop_delay_ms.unwrap_or(DEFAULT_DROP_DELAY_MS), + deadline, + release.delivery_mut(), + )?; + release.release_at_destination(deadline) + })(); + outcome.map_err(|error| release.enrich_error(error)) +} + +struct DragReleaseGuard { + abort_drag: CGEvent, + abort_up: CGEvent, + destination_up: Option<CGEvent>, + delivery: crate::input::mouse_drag_state::DragDeliveryState, +} + +impl DragReleaseGuard { + fn prepare( + source: &CGEventSource, + origin: CGPoint, + destination: CGPoint, + ) -> Result<Self, AdapterError> { + let flags = CGEventFlags::empty(); + Ok(Self { + abort_drag: crate::input::mouse::create_event_with_source( + source, + CGEventType::LeftMouseDragged, + origin, + CGMouseButton::Left, + flags, + )?, + abort_up: crate::input::mouse::create_event_with_source( + source, + CGEventType::LeftMouseUp, + origin, + CGMouseButton::Left, + flags, + )?, + destination_up: Some(crate::input::mouse::create_event_with_source( + source, + CGEventType::LeftMouseUp, + destination, + CGMouseButton::Left, + flags, + )?), + delivery: crate::input::mouse_drag_state::DragDeliveryState::default(), + }) + } + + fn arm(&mut self) { + self.delivery.arm(); + } + + fn mark_down_posted(&mut self) { + self.delivery.mark_down_posted(); + } + + fn delivery(&self) -> crate::actions::DeliveryTracker { + self.delivery.delivery() + } + + fn delivery_mut(&mut self) -> &mut crate::actions::DeliveryTracker { + self.delivery.delivery_mut() + } + + fn release_at_destination(&mut self, deadline: Deadline) -> Result<(), AdapterError> { + crate::input::mouse::ensure_budget(deadline, self.delivery())?; + let event = self.destination_up.take().ok_or_else(|| { + AdapterError::internal("Drag release guard lost its destination event") + })?; + event.post(CGEventTapLocation::HID); + self.delivery.disarm(); + crate::input::mouse::ensure_budget(deadline, self.delivery()) + } + + fn enrich_error(&self, error: AdapterError) -> AdapterError { + self.delivery.enrich_error(error) + } +} + +impl Drop for DragReleaseGuard { + fn drop(&mut self) { + if self.delivery.should_release() { + self.abort_drag.post(CGEventTapLocation::HID); + self.abort_up.post(CGEventTapLocation::HID); + } + } +} + +fn dwell_over_destination( + source: &CGEventSource, + destination: CGPoint, + delay_ms: u64, + deadline: Deadline, + delivery: &mut crate::actions::DeliveryTracker, +) -> Result<(), AdapterError> { + if delay_ms == 0 { + return Ok(()); + } + let mut remaining_ms = delay_ms; + while remaining_ms > 0 { + crate::input::mouse::post_event_with_source( + source, + ( + CGEventType::LeftMouseDragged, + destination, + CGMouseButton::Left, + CGEventFlags::empty(), + ), + deadline, + delivery, + )?; + let tick_ms = remaining_ms.min(DWELL_TICK_MS); + crate::input::mouse::sleep_bounded(deadline, Duration::from_millis(tick_ms), *delivery)?; + remaining_ms -= tick_ms; + } + Ok(()) +} + +fn preflight_drag(params: &DragParams, deadline: Deadline) -> Result<(), AdapterError> { + let duration_ms = params.duration_ms.unwrap_or(DEFAULT_DURATION_MS); + let drop_delay_ms = params.drop_delay_ms.unwrap_or(DEFAULT_DROP_DELAY_MS); + let required_ms = PICKUP_DELAY_MS + .checked_add(duration_ms) + .and_then(|total| total.checked_add(drop_delay_ms)) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Drag timing is too large"))?; + let remaining = deadline.remaining(); + let required = Duration::from_millis(required_ms); + if remaining < required { + return Err(crate::actions::DeliveryTracker::default().annotate( + AdapterError::timeout("Drag cannot complete within the remaining deadline") + .with_details(serde_json::json!({ + "physical_delivery_started": false, + "required_ms": required_ms, + "remaining_ms": remaining.as_millis(), + })), + )); + } + Ok(()) +} + +fn validate_drag(params: &DragParams) -> Result<(), AdapterError> { + crate::input::mouse::validate_point(¶ms.from)?; + crate::input::mouse::validate_point(¶ms.to)?; + if params.duration_ms.is_some_and(|value| value > MAX_DRAG_MS) { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Drag duration must not exceed 60000ms", + )); + } + if params + .drop_delay_ms + .is_some_and(|value| value > MAX_DROP_DELAY_MS) + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Drag drop delay must not exceed 30000ms", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use agent_desktop_core::Point; + + #[test] + fn drag_limits_reject_unbounded_work() { + let base = DragParams { + from: Point { x: 0.0, y: 0.0 }, + to: Point { x: 10.0, y: 10.0 }, + duration_ms: Some(MAX_DRAG_MS + 1), + drop_delay_ms: None, + }; + assert!(validate_drag(&base).is_err()); + let excessive_dwell = DragParams { + duration_ms: None, + drop_delay_ms: Some(MAX_DROP_DELAY_MS + 1), + ..base + }; + assert!(validate_drag(&excessive_dwell).is_err()); + } + + #[test] + fn zero_drop_delay_has_no_forced_dwell_ticks() { + assert_eq!(0_u64.div_ceil(DWELL_TICK_MS), 0); + } + + #[test] + fn impossible_drag_deadline_fails_before_mouse_down() { + let params = DragParams { + from: Point { x: 0.0, y: 0.0 }, + to: Point { x: 10.0, y: 10.0 }, + duration_ms: Some(1), + drop_delay_ms: Some(0), + }; + let error = preflight_drag(¶ms, Deadline::after(1).unwrap()).unwrap_err(); + + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(error.details.unwrap()["physical_delivery_started"], false); + } + + #[test] + fn sub_tick_drag_uses_one_nonzero_duration_step() { + let duration_ms = 1_u64; + let steps = duration_ms.div_ceil(DWELL_TICK_MS).clamp(1, MAX_STEPS); + let step_delay = Duration::from_secs_f64(duration_ms as f64 / steps as f64 / 1_000.0); + + assert_eq!(steps, 1); + assert_eq!(step_delay, Duration::from_millis(1)); + } +} diff --git a/crates/macos/src/input/mouse_drag_state.rs b/crates/macos/src/input/mouse_drag_state.rs new file mode 100644 index 0000000..000f165 --- /dev/null +++ b/crates/macos/src/input/mouse_drag_state.rs @@ -0,0 +1,95 @@ +use agent_desktop_core::AdapterError; + +#[derive(Default)] +pub(crate) struct DragDeliveryState { + armed: bool, + delivery: crate::actions::DeliveryTracker, +} + +impl DragDeliveryState { + pub(crate) fn arm(&mut self) { + self.armed = true; + } + + pub(crate) fn mark_down_posted(&mut self) { + self.delivery.mark_delivered(); + } + + pub(crate) fn disarm(&mut self) { + self.armed = false; + } + + pub(crate) fn should_release(&self) -> bool { + self.armed + } + + pub(crate) fn delivery(&self) -> crate::actions::DeliveryTracker { + self.delivery + } + + pub(crate) fn delivery_mut(&mut self) -> &mut crate::actions::DeliveryTracker { + &mut self.delivery + } + + pub(crate) fn enrich_error(&self, mut error: AdapterError) -> AdapterError { + error = self.delivery.annotate(error); + if self.delivery.delivered_units() == 0 { + return error; + } + let mut details = error + .details + .take() + .unwrap_or_else(|| serde_json::json!({})); + if let Some(details) = details.as_object_mut() { + details.insert( + "delivered_events".into(), + self.delivery.delivered_units().into(), + ); + if self.armed { + details.insert("emergency_release_posted".into(), true.into()); + details.insert("emergency_release_acknowledged".into(), false.into()); + } + } + error + .with_details(details) + .with_suggestion( + "Inspect the source and destination before retrying; the emergency release was posted without an OS acknowledgement", + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pre_delivery_error_does_not_claim_mouse_down_or_release() { + let state = DragDeliveryState::default(); + let error = state.enrich_error(AdapterError::internal("pre-post failure")); + + assert_eq!( + error.disposition, + agent_desktop_core::DeliverySemantics::not_delivered() + ); + assert!(!state.should_release()); + } + + #[test] + fn deadline_after_down_requires_emergency_release_and_no_retry() { + let mut state = DragDeliveryState::default(); + state.arm(); + state.mark_down_posted(); + for _ in 0..3 { + state.delivery_mut().mark_delivered(); + } + let error = state.enrich_error(AdapterError::timeout("deadline")); + assert_eq!( + error.disposition, + agent_desktop_core::DeliverySemantics::delivered_unverified() + ); + let details = error.details.unwrap(); + + assert_eq!(details["delivered_events"], 4); + assert_eq!(details["emergency_release_posted"], true); + } +} diff --git a/crates/macos/src/input/mouse_move.rs b/crates/macos/src/input/mouse_move.rs new file mode 100644 index 0000000..8bbc2bc --- /dev/null +++ b/crates/macos/src/input/mouse_move.rs @@ -0,0 +1,141 @@ +#[cfg(target_os = "macos")] +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, Modifier, Point}; +#[cfg(target_os = "macos")] +use core_graphics::{ + event::{CGEvent, CGEventFlags, CGEventTapLocation, CGEventType, CGMouseButton}, + geometry::CGPoint, +}; + +#[cfg(target_os = "macos")] +pub(crate) fn post_move_events( + point: CGPoint, + button: CGMouseButton, + flags: CGEventFlags, + deadline: Deadline, + delivery: &mut crate::actions::DeliveryTracker, +) -> Result<(), AdapterError> { + let source = crate::input::mouse::event_source().map_err(|error| delivery.annotate(error))?; + for position in [approach_point(point), point] { + crate::input::mouse::ensure_budget(deadline, *delivery)?; + let event = crate::input::mouse::create_event_with_source( + &source, + CGEventType::MouseMoved, + position, + button, + flags, + ) + .map_err(|error| delivery.annotate(error))?; + event.post(CGEventTapLocation::HID); + delivery.mark_delivered(); + crate::input::mouse::sleep_bounded( + deadline, + std::time::Duration::from_millis(10), + *delivery, + )?; + } + verify_position(source, point, deadline, delivery) +} + +#[cfg(target_os = "macos")] +fn verify_position( + source: core_graphics::event_source::CGEventSource, + requested: CGPoint, + deadline: Deadline, + delivery: &mut crate::actions::DeliveryTracker, +) -> Result<(), AdapterError> { + let verification_end = std::time::Instant::now() + std::time::Duration::from_millis(100); + loop { + let observed = CGEvent::new(source.clone()) + .map_err(|()| AdapterError::internal("Failed to read the current pointer position")) + .map_err(|error| delivery.annotate(error))? + .location(); + if pointer_position_matches(observed, requested) { + return Ok(()); + } + if std::time::Instant::now() >= verification_end { + return Err(pointer_position_error(observed, requested, delivery)); + } + crate::input::mouse::sleep_bounded( + deadline, + std::time::Duration::from_millis(5), + *delivery, + )?; + } +} + +#[cfg(target_os = "macos")] +fn pointer_position_error( + observed: CGPoint, + requested: CGPoint, + delivery: &crate::actions::DeliveryTracker, +) -> AdapterError { + delivery.annotate( + AdapterError::new( + ErrorCode::ActionFailed, + "Physical pointer did not reach the requested position", + ) + .with_details(serde_json::json!({ + "requested": { "x": requested.x, "y": requested.y }, + "observed": { "x": observed.x, "y": observed.y }, + })), + ) +} + +#[cfg(target_os = "macos")] +pub(crate) fn preposition_pointer( + point: &Point, + modifiers: &[Modifier], + deadline: Deadline, + delivery: &mut crate::actions::DeliveryTracker, +) -> Result<(), AdapterError> { + post_move_events( + CGPoint::new(point.x, point.y), + CGMouseButton::Left, + crate::input::mouse::event_flags(modifiers), + deadline, + delivery, + ) +} + +#[cfg(target_os = "macos")] +fn approach_point(point: CGPoint) -> CGPoint { + CGPoint::new( + if point.x > -999_999.0 { + point.x - 1.0 + } else { + point.x + 1.0 + }, + point.y, + ) +} + +#[cfg(target_os = "macos")] +fn pointer_position_matches(observed: CGPoint, requested: CGPoint) -> bool { + (observed.x - requested.x).abs() <= 0.5 && (observed.y - requested.y).abs() <= 0.5 +} + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::{approach_point, pointer_position_matches}; + use core_graphics::geometry::CGPoint; + + #[test] + fn approach_moves_one_point_before_the_exact_destination() { + let approach = approach_point(CGPoint::new(2065.0, 636.0)); + assert_eq!(approach.x, 2064.0); + assert_eq!(approach.y, 636.0); + } + + #[test] + fn verification_allows_subpixel_rounding_only() { + let requested = CGPoint::new(2065.0, 636.0); + assert!(pointer_position_matches( + CGPoint::new(2065.4, 635.6), + requested + )); + assert!(!pointer_position_matches( + CGPoint::new(2065.6, 636.0), + requested + )); + } +} diff --git a/crates/macos/src/input/mouse_scroll.rs b/crates/macos/src/input/mouse_scroll.rs new file mode 100644 index 0000000..8c79560 --- /dev/null +++ b/crates/macos/src/input/mouse_scroll.rs @@ -0,0 +1,105 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, Modifier, Point}; + +const MAX_LINES_PER_EVENT: i32 = 10; +const MAX_TOTAL_LINES: i32 = 1_000; + +#[cfg(target_os = "macos")] +pub(crate) fn synthesize_scroll_at( + point: Point, + delta: (i32, i32), + modifiers: &[Modifier], + deadline: Deadline, +) -> Result<(), AdapterError> { + use core_graphics::event::{CGEvent, CGEventTapLocation, ScrollEventUnit}; + use core_graphics::geometry::CGPoint; + + crate::input::mouse::validate_point(&point)?; + let chunks = scroll_chunks(delta)?; + let mut delivery = crate::actions::DeliveryTracker::default(); + crate::input::mouse::ensure_budget(deadline, delivery)?; + crate::input::mouse_move::preposition_pointer(&point, modifiers, deadline, &mut delivery)?; + let source = crate::input::mouse::event_source().map_err(|error| delivery.annotate(error))?; + let flags = crate::input::mouse::event_flags(modifiers); + for (index, (dy, dx)) in chunks.iter().copied().enumerate() { + crate::input::mouse::ensure_budget(deadline, delivery)?; + tracing::debug!(x = point.x, y = point.y, dy, dx, "mouse: scroll chunk"); + let event = CGEvent::new_scroll_event(source.clone(), ScrollEventUnit::LINE, 2, dy, dx, 0) + .map_err(|()| AdapterError::internal("CGEvent::new_scroll_event failed")) + .map_err(|error| delivery.annotate(error))?; + event.set_location(CGPoint::new(point.x, point.y)); + event.set_flags(flags); + event.post(CGEventTapLocation::HID); + delivery.mark_delivered(); + if index + 1 < chunks.len() { + crate::input::mouse::sleep_bounded( + deadline, + std::time::Duration::from_millis(5), + delivery, + )?; + } + } + crate::input::mouse::ensure_budget(deadline, delivery) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn synthesize_scroll_at( + _point: Point, + _delta: (i32, i32), + _modifiers: &[Modifier], + _deadline: Deadline, +) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("scroll")) +} + +fn scroll_chunks(delta: (i32, i32)) -> Result<Vec<(i32, i32)>, AdapterError> { + let (mut dy, mut dx) = delta; + if dy == 0 && dx == 0 { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Wheel delta must be non-zero", + )); + } + if dy.unsigned_abs() > MAX_TOTAL_LINES as u32 || dx.unsigned_abs() > MAX_TOTAL_LINES as u32 { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Wheel delta must be within -1000..=1000 lines per axis", + )); + } + let mut chunks = Vec::new(); + while dy != 0 || dx != 0 { + let next_y = dy.clamp(-MAX_LINES_PER_EVENT, MAX_LINES_PER_EVENT); + let next_x = dx.clamp(-MAX_LINES_PER_EVENT, MAX_LINES_PER_EVENT); + chunks.push((next_y, next_x)); + dy -= next_y; + dx -= next_x; + } + Ok(chunks) +} + +#[cfg(test)] +mod tests { + use super::{MAX_LINES_PER_EVENT, scroll_chunks}; + + #[test] + fn large_wheel_delta_is_split_into_apple_sized_signed_chunks() { + assert_eq!( + scroll_chunks((-25, 12)).expect("bounded wheel delta"), + vec![(-10, 10), (-10, 2), (-5, 0)] + ); + } + + #[test] + fn every_wheel_chunk_stays_within_the_native_event_range() { + let chunks = scroll_chunks((1_000, -1_000)).expect("maximum wheel delta"); + assert_eq!(chunks.len(), 100); + assert!(chunks.iter().all(|(dy, dx)| { + dy.abs() <= MAX_LINES_PER_EVENT && dx.abs() <= MAX_LINES_PER_EVENT + })); + } + + #[test] + fn zero_and_unbounded_wheel_deltas_are_rejected() { + assert!(scroll_chunks((0, 0)).is_err()); + assert!(scroll_chunks((1_001, 0)).is_err()); + } +} diff --git a/crates/macos/src/input/mouse_tests.rs b/crates/macos/src/input/mouse_tests.rs new file mode 100644 index 0000000..28b10e6 --- /dev/null +++ b/crates/macos/src/input/mouse_tests.rs @@ -0,0 +1,113 @@ +use super::{Modifier, event_flags, standalone_state_error, wheel_lines_to_i32}; +use core_graphics::event::CGEventFlags; + +#[test] +fn standalone_mouse_state_is_rejected_without_emission() { + let error = standalone_state_error(); + + assert_eq!( + error.code, + agent_desktop_core::ErrorCode::ActionNotSupported + ); + assert_eq!(error.details.unwrap()["raw_input_emitted"], false); +} + +#[test] +fn event_flags_maps_cmd_to_command_bit() { + assert_eq!( + event_flags(&[Modifier::Meta]), + CGEventFlags::CGEventFlagCommand + ); +} + +#[test] +fn event_flags_maps_shift_to_shift_bit() { + assert_eq!( + event_flags(&[Modifier::Shift]), + CGEventFlags::CGEventFlagShift + ); +} + +#[test] +fn event_flags_maps_alt_to_alternate_bit() { + assert_eq!( + event_flags(&[Modifier::Alt]), + CGEventFlags::CGEventFlagAlternate + ); +} + +#[test] +fn event_flags_maps_ctrl_to_control_bit() { + assert_eq!( + event_flags(&[Modifier::Ctrl]), + CGEventFlags::CGEventFlagControl + ); +} + +#[test] +fn event_flags_combines_multiple_modifiers_via_bitwise_or() { + let combined = event_flags(&[Modifier::Meta, Modifier::Shift]); + assert_eq!( + combined, + CGEventFlags::CGEventFlagCommand | CGEventFlags::CGEventFlagShift + ); + assert!(combined.contains(CGEventFlags::CGEventFlagCommand)); + assert!(combined.contains(CGEventFlags::CGEventFlagShift)); + assert!(!combined.contains(CGEventFlags::CGEventFlagAlternate)); +} + +#[test] +fn event_flags_empty_slice_yields_no_flags() { + assert_eq!(event_flags(&[]), CGEventFlags::empty()); +} + +#[test] +fn wheel_line_conversion_preserves_direction_and_small_nonzero_input() { + assert_eq!(wheel_lines_to_i32(-3.0).unwrap(), -3); + assert_eq!(wheel_lines_to_i32(2.6).unwrap(), 3); + assert_eq!(wheel_lines_to_i32(0.1).unwrap(), 1); + assert_eq!(wheel_lines_to_i32(-0.1).unwrap(), -1); +} + +#[test] +fn wheel_line_conversion_rejects_non_finite_input() { + assert!(wheel_lines_to_i32(f64::NAN).is_err()); + assert!(wheel_lines_to_i32(f64::INFINITY).is_err()); +} + +#[cfg(feature = "interactive-tests")] +#[test] +fn native_cg_event_contract_is_bounded() { + use super::create_event; + use crate::input::interactive_test::{is_worker, run_bounded}; + use core_graphics::event::{CGEventType, CGMouseButton}; + use core_graphics::geometry::CGPoint; + use std::time::Duration; + + if is_worker("mouse") { + let flags = event_flags(&[Modifier::Meta, Modifier::Ctrl]); + let event = create_event( + CGEventType::LeftMouseDown, + CGPoint::new(0.0, 0.0), + CGMouseButton::Left, + flags, + ) + .expect("CGEvent construction succeeds"); + assert_eq!(event.get_flags(), flags); + + let plain = create_event( + CGEventType::LeftMouseUp, + CGPoint::new(0.0, 0.0), + CGMouseButton::Left, + event_flags(&[]), + ) + .expect("second CGEvent construction succeeds"); + assert_eq!(plain.get_flags(), CGEventFlags::empty()); + } else { + run_bounded( + "native_cg_event_contract_is_bounded", + "mouse", + Duration::from_secs(5), + ); + } +} diff --git a/crates/macos/src/input/owned_object.rs b/crates/macos/src/input/owned_object.rs new file mode 100644 index 0000000..b2af297 --- /dev/null +++ b/crates/macos/src/input/owned_object.rs @@ -0,0 +1,38 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::{ffi::c_void, ptr::NonNull}; + +type Id = *mut c_void; +type Sel = *mut c_void; + +unsafe extern "C" { + fn sel_registerName(name: *const core::ffi::c_char) -> Sel; + fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; +} + +pub(crate) struct OwnedObject(NonNull<c_void>); + +impl OwnedObject { + pub(crate) fn from_id(id: Id, operation: &str) -> Result<Self, AdapterError> { + NonNull::new(id).map(Self).ok_or_else(|| { + AdapterError::new(ErrorCode::ActionFailed, "System clipboard is unavailable") + .with_platform_detail(format!("{operation} returned an unavailable object")) + .with_suggestion( + "Retry from an interactive macOS login session after the pasteboard service is available.", + ) + }) + } + + pub(crate) fn as_id(&self) -> Id { + self.0.as_ptr() + } +} + +impl Drop for OwnedObject { + fn drop(&mut self) { + unsafe { + let send: unsafe extern "C" fn(Id, Sel) = + std::mem::transmute(objc_msgSend as *const c_void); + send(self.as_id(), sel_registerName(c"release".as_ptr())); + } + } +} diff --git a/crates/macos/src/lib.rs b/crates/macos/src/lib.rs index 811f0c9..fd2e857 100644 --- a/crates/macos/src/lib.rs +++ b/crates/macos/src/lib.rs @@ -1,3 +1,5 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + mod actions; mod adapter; mod cf_type; @@ -7,3 +9,6 @@ mod system; mod tree; pub use adapter::MacOSAdapter; +pub use input::clipboard::helper_entry_from_env as clipboard_helper_from_env; +pub use system::cocoa_runtime::ensure_cocoa_multithreaded; +pub use system::permission_helper::entry_from_env as permission_prompt_helper_from_env; diff --git a/crates/macos/src/notifications/actions.rs b/crates/macos/src/notifications/actions.rs index bbdf088..302047b 100644 --- a/crates/macos/src/notifications/actions.rs +++ b/crates/macos/src/notifications/actions.rs @@ -1,25 +1,35 @@ use agent_desktop_core::{ - action_result::ActionResult, - error::{AdapterError, ErrorCode}, - notification::{NotificationFilter, NotificationIdentity, NotificationInfo}, + ActionResult, AdapterError, Deadline, ErrorCode, InteractionPolicy, NotificationFilter, + NotificationIdentity, NotificationInfo, }; use super::nc_session::{NcSession, close_session}; +#[cfg(test)] +#[path = "actions_tests.rs"] +mod tests; + pub fn dismiss_notification( index: usize, app_filter: Option<&str>, + identity: Option<&NotificationIdentity>, + policy: InteractionPolicy, + deadline: Deadline, ) -> Result<NotificationInfo, AdapterError> { - let session = NcSession::open()?; - let result = dismiss_impl(index, app_filter); + require_foreground_policy(policy)?; + let session = NcSession::open(policy, deadline)?; + let result = dismiss_impl(index, app_filter, identity, policy, session.pid(), deadline); close_session(session, result) } pub fn dismiss_all( app_filter: Option<&str>, + policy: InteractionPolicy, + deadline: Deadline, ) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> { - let session = NcSession::open()?; - let result = dismiss_all_impl(app_filter); + require_foreground_policy(policy)?; + let session = NcSession::open(policy, deadline)?; + let result = dismiss_all_impl(app_filter, policy, session.pid(), deadline); close_session(session, result) } @@ -27,41 +37,68 @@ pub fn notification_action( index: usize, identity: Option<&NotificationIdentity>, action_name: &str, + policy: InteractionPolicy, + deadline: Deadline, ) -> Result<ActionResult, AdapterError> { - let session = NcSession::open()?; - let result = action_impl(index, identity, action_name); + require_foreground_policy(policy)?; + let session = NcSession::open(policy, deadline)?; + let result = action_impl(index, identity, action_name, session.pid(), deadline); close_session(session, result) } #[cfg(target_os = "macos")] -fn dismiss_impl(index: usize, app_filter: Option<&str>) -> Result<NotificationInfo, AdapterError> { +fn dismiss_impl( + index: usize, + app_filter: Option<&str>, + identity: Option<&NotificationIdentity>, + policy: InteractionPolicy, + pid: i32, + deadline: Deadline, +) -> Result<NotificationInfo, AdapterError> { let filter = build_filter(app_filter); - let entries = super::list::list_entries(&filter)?; + let entries = super::list::list_entries(&filter, pid, deadline)?; let entry = entries - .into_iter() + .iter() .find(|e| e.info.index == index) .ok_or_else(|| AdapterError::notification_not_found(index))?; + verify_identity(index, identity, &entry.info)?; - let info = entry.info; - dismiss_entry(&entry.element)?; + let info = entry.info.clone(); + let matching_before = super::dismiss_verify::matching_count(&entries, entry); + dismiss_entry(entry, policy, &filter, matching_before, pid, deadline)?; Ok(info) } #[cfg(target_os = "macos")] fn dismiss_all_impl( app_filter: Option<&str>, + policy: InteractionPolicy, + pid: i32, + deadline: Deadline, ) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> { let filter = build_filter(app_filter); - let entries = super::list::list_entries(&filter)?; + let entries = super::list::list_entries(&filter, pid, deadline)?; let mut dismissed = Vec::new(); let mut failures = Vec::new(); - for entry in entries.into_iter().rev() { - match dismiss_entry(&entry.element) { - Ok(()) => dismissed.push(entry.info), - Err(e) => failures.push(format!("#{}: {}", entry.info.index, e)), + for original in entries.iter().rev() { + let current_entries = super::list::list_entries(&filter, pid, deadline)?; + let Some(current) = current_entries + .iter() + .find(|entry| super::dismiss_verify::matches(original, entry)) + else { + failures.push(format!( + "#{}: notification disappeared before dismissal was attempted", + original.info.index + )); + continue; + }; + let matching_before = super::dismiss_verify::matching_count(¤t_entries, current); + match dismiss_entry(current, policy, &filter, matching_before, pid, deadline) { + Ok(()) => dismissed.push(original.info.clone()), + Err(e) => failures.push(format!("#{}: {}", original.info.index, e)), } } @@ -69,56 +106,96 @@ fn dismiss_all_impl( } #[cfg(target_os = "macos")] -fn dismiss_entry(element: &crate::tree::AXElement) -> Result<(), AdapterError> { - use crate::actions::ax_helpers::{list_ax_actions, try_action_from_list}; - use crate::tree::copy_ax_array; - use accessibility_sys::kAXChildrenAttribute; +fn dismiss_entry( + entry: &super::list::NotificationEntry, + policy: InteractionPolicy, + filter: &NotificationFilter, + matching_before: usize, + pid: i32, + deadline: Deadline, +) -> Result<(), AdapterError> { + for action in ["AXDismiss", "AXRemoveFromParent"] { + let outcome = + crate::actions::ax_helpers::try_ax_action_or_err(&entry.element, action, deadline); + if strategy_verified(outcome, entry, filter, matching_before, pid, deadline)? { + return Ok(()); + } + } - let actions = list_ax_actions(element); - if try_action_from_list(element, &actions, &["AXDismiss", "AXRemoveFromParent"]) { + let children = strategy_read( + crate::notifications::read::children(&entry.element, deadline), + deadline, + )? + .unwrap_or_default(); + let pressed = try_dismiss_button(&children, deadline)?; + if strategy_verified(Ok(pressed), entry, filter, matching_before, pid, deadline)? { return Ok(()); } - let children = copy_ax_array(element, kAXChildrenAttribute).unwrap_or_default(); - if try_dismiss_button(&children) { - return Ok(()); - } - - if !crate::system::permissions::report().accessibility_granted() { + if !crate::system::permissions::report(deadline)?.accessibility_granted() { return Err(AdapterError::permission_denied()); } - hover_over(element)?; + if !policy.allow_cursor_move { + return Err(AdapterError::policy_denied_for_policy( + "Notification dismissal requires revealing its close control with the pointer", + policy, + )); + } + if let Err(error) = hover_over(&entry.element, deadline) { + strategy_succeeded(Err(error), deadline)?; + return Err(all_dismiss_strategies_failed()); + } std::thread::sleep(std::time::Duration::from_millis(300)); - let children = copy_ax_array(element, kAXChildrenAttribute).unwrap_or_default(); - if try_dismiss_button(&children) { + let children = strategy_read( + crate::notifications::read::children(&entry.element, deadline), + deadline, + )? + .unwrap_or_default(); + let pressed = try_dismiss_button(&children, deadline)?; + if strategy_verified(Ok(pressed), entry, filter, matching_before, pid, deadline)? { return Ok(()); } - Err(AdapterError::new( - ErrorCode::ActionFailed, - "All dismiss strategies failed (AXDismiss, AXRemoveFromParent, close button, hover+close)", - )) + Err(all_dismiss_strategies_failed()) } #[cfg(target_os = "macos")] -fn try_dismiss_button(children: &[crate::tree::AXElement]) -> bool { - use crate::actions::ax_helpers::try_ax_action; - use crate::tree::copy_string_attr; - use accessibility_sys::kAXRoleAttribute; - - let close_btn = children.iter().find(|c| { - if copy_string_attr(c, kAXRoleAttribute).as_deref() != Some("AXButton") { - return false; +fn try_dismiss_button( + children: &[crate::tree::AXElement], + deadline: Deadline, +) -> Result<bool, AdapterError> { + for child in children { + let role = strategy_read( + crate::notifications::read::string(child, "AXRole", deadline), + deadline, + )?; + let Some(role) = role else { + continue; + }; + if role.as_deref() != Some("AXButton") { + continue; } - let name = copy_string_attr(c, "AXTitle") - .or_else(|| copy_string_attr(c, "AXDescription")) - .unwrap_or_default() - .to_lowercase(); - name.contains("close") || name.contains("clear") || name.contains("dismiss") - }); - close_btn.is_some_and(|btn| try_ax_action(btn, "AXPress")) + let name = strategy_read( + crate::notifications::read::title_or_description(child, deadline), + deadline, + )? + .flatten() + .unwrap_or_default() + .to_lowercase(); + let is_dismiss = + name.contains("close") || name.contains("clear") || name.contains("dismiss"); + if is_dismiss + && strategy_succeeded( + crate::actions::ax_helpers::try_ax_action_or_err(child, "AXPress", deadline), + deadline, + )? + { + return Ok(true); + } + } + Ok(false) } /// Presses a named action button on the notification at `index`. @@ -129,46 +206,35 @@ fn action_impl( index: usize, identity: Option<&NotificationIdentity>, action_name: &str, + pid: i32, + deadline: Deadline, ) -> Result<ActionResult, AdapterError> { - use crate::actions::ax_helpers::try_ax_action; - use crate::tree::{copy_ax_array, copy_string_attr}; - use accessibility_sys::{kAXChildrenAttribute, kAXRoleAttribute}; - let filter = NotificationFilter::default(); - let entries = super::list::list_entries(&filter)?; + let entries = super::list::list_entries(&filter, pid, deadline)?; let entry = entries .into_iter() .find(|e| e.info.index == index) .ok_or_else(|| AdapterError::notification_not_found(index))?; - if let Some(id) = identity { - if !id.is_empty() && !id.matches(&entry.info) { - return Err(AdapterError::new( - ErrorCode::NotificationNotFound, - format!( - "Notification at index {index} does not match the expected identity (app={:?}, title={:?}); NC likely reordered", - id.expected_app, id.expected_title - ), - ) - .with_suggestion( - "Run list-notifications again and retry with the freshly-observed index", - )); + verify_identity(index, identity, &entry.info)?; + + let children = crate::notifications::read::children(&entry.element, deadline)?; + let action_lower = action_name.to_lowercase(); + let mut action_btn = None; + for child in &children { + let role = crate::notifications::read::string(child, "AXRole", deadline)?; + let identifier = crate::notifications::read::string(child, "AXIdentifier", deadline)?; + let name = + crate::notifications::read::title_or_description(child, deadline)?.unwrap_or_default(); + if role.as_deref() == Some("AXButton") + && crate::notifications::scan::is_notification_action(identifier.as_deref()) + && name.to_lowercase() == action_lower + { + action_btn = Some(child); + break; } } - - let children = copy_ax_array(&entry.element, kAXChildrenAttribute).unwrap_or_default(); - let action_lower = action_name.to_lowercase(); - let action_btn = children.iter().find(|c| { - if copy_string_attr(c, kAXRoleAttribute).as_deref() != Some("AXButton") { - return false; - } - let name = copy_string_attr(c, "AXTitle") - .or_else(|| copy_string_attr(c, "AXDescription")) - .unwrap_or_default(); - name.to_lowercase() == action_lower - }); - let btn = action_btn.ok_or_else(|| { AdapterError::new( ErrorCode::ActionFailed, @@ -179,7 +245,7 @@ fn action_impl( ) })?; - if !try_ax_action(btn, "AXPress") { + if !crate::actions::ax_helpers::try_ax_action_or_err(btn, "AXPress", deadline)? { return Err(AdapterError::new( ErrorCode::ActionFailed, format!( @@ -189,25 +255,54 @@ fn action_impl( )); } - Ok(ActionResult::new(action_name)) + Ok(ActionResult::delivered_unverified(action_name)) } #[cfg(target_os = "macos")] -fn hover_over(el: &crate::tree::AXElement) -> Result<(), AdapterError> { - use crate::tree::read_bounds; - use agent_desktop_core::action::{MouseButton, MouseEvent, MouseEventKind, Point}; +fn hover_over(el: &crate::tree::AXElement, deadline: Deadline) -> Result<(), AdapterError> { + use agent_desktop_core::{MouseButton, MouseEvent, MouseEventKind, Point}; - let bounds = read_bounds(el) + let bounds = crate::notifications::read::bounds(el, deadline)? .ok_or_else(|| AdapterError::internal("Cannot read notification bounds for hover"))?; - crate::input::mouse::synthesize_mouse(MouseEvent { - kind: MouseEventKind::Move, - point: Point { - x: bounds.x + bounds.width / 2.0, - y: bounds.y + bounds.height / 2.0, + crate::input::mouse::synthesize_mouse( + MouseEvent { + kind: MouseEventKind::Move, + point: Point { + x: bounds.x + bounds.width / 2.0, + y: bounds.y + bounds.height / 2.0, + }, + button: MouseButton::Left, + modifiers: Vec::new(), }, - button: MouseButton::Left, - }) + deadline, + ) +} + +fn require_foreground_policy(policy: InteractionPolicy) -> Result<(), AdapterError> { + if policy.allow_focus_steal { + Ok(()) + } else { + Err(AdapterError::policy_denied_for_policy( + "Notification Center interaction requires foreground access", + policy, + )) + } +} + +fn verify_identity( + index: usize, + identity: Option<&NotificationIdentity>, + info: &NotificationInfo, +) -> Result<(), AdapterError> { + if identity.is_none_or(|value| value.is_empty() || value.matches(info)) { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::NotificationNotFound, + format!("Notification at index {index} no longer matches its expected identity"), + ) + .with_suggestion("Run list-notifications again and retry with the freshly-observed index")) } fn build_filter(app_filter: Option<&str>) -> NotificationFilter { @@ -217,10 +312,66 @@ fn build_filter(app_filter: Option<&str>) -> NotificationFilter { } } +fn strategy_succeeded( + outcome: Result<bool, AdapterError>, + deadline: Deadline, +) -> Result<bool, AdapterError> { + match outcome { + Ok(succeeded) => Ok(succeeded), + Err(error) => { + crate::notifications::read::tolerate_ax_strategy_error(error, deadline)?; + Ok(false) + } + } +} + +fn strategy_read<T>( + outcome: Result<T, AdapterError>, + deadline: Deadline, +) -> Result<Option<T>, AdapterError> { + match outcome { + Ok(value) => Ok(Some(value)), + Err(error) => { + crate::notifications::read::tolerate_ax_strategy_error(error, deadline)?; + Ok(None) + } + } +} + +fn strategy_verified( + outcome: Result<bool, AdapterError>, + entry: &super::list::NotificationEntry, + filter: &NotificationFilter, + matching_before: usize, + pid: i32, + deadline: Deadline, +) -> Result<bool, AdapterError> { + super::dismiss_verify::verified_after_strategy(outcome, deadline, || { + match super::dismiss_verify::disappeared(entry, filter, matching_before, pid, deadline) { + Ok(disappeared) => Ok(disappeared), + Err(error) => { + crate::notifications::read::tolerate_ax_strategy_error(error, deadline)?; + Ok(false) + } + } + }) +} + +fn all_dismiss_strategies_failed() -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + "All dismiss strategies failed (AXDismiss, AXRemoveFromParent, close button, hover+close)", + ) +} + #[cfg(not(target_os = "macos"))] fn dismiss_impl( _index: usize, _app_filter: Option<&str>, + _identity: Option<&NotificationIdentity>, + _policy: InteractionPolicy, + _pid: i32, + _deadline: Deadline, ) -> Result<NotificationInfo, AdapterError> { Err(AdapterError::not_supported("dismiss_notification")) } @@ -228,6 +379,9 @@ fn dismiss_impl( #[cfg(not(target_os = "macos"))] fn dismiss_all_impl( _app_filter: Option<&str>, + _policy: InteractionPolicy, + _pid: i32, + _deadline: Deadline, ) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> { Err(AdapterError::not_supported("dismiss_all_notifications")) } @@ -237,6 +391,8 @@ fn action_impl( _index: usize, _identity: Option<&NotificationIdentity>, _action_name: &str, + _pid: i32, + _deadline: Deadline, ) -> Result<ActionResult, AdapterError> { Err(AdapterError::not_supported("notification_action")) } diff --git a/crates/macos/src/notifications/actions_tests.rs b/crates/macos/src/notifications/actions_tests.rs new file mode 100644 index 0000000..7f7726d --- /dev/null +++ b/crates/macos/src/notifications/actions_tests.rs @@ -0,0 +1,56 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; + +#[test] +fn hard_ax_error_falls_through_to_the_next_dismiss_strategy() { + let deadline = Deadline::standard().expect("deadline"); + let outcomes = [ + Err(AdapterError::new( + ErrorCode::ActionFailed, + "AXDismiss failed", + )), + Ok(true), + ]; + let mut attempts = 0; + let mut dismissed = false; + + for outcome in outcomes { + attempts += 1; + if super::strategy_succeeded(outcome, deadline).expect("strategy result") { + dismissed = true; + break; + } + } + + assert!(dismissed); + assert_eq!(attempts, 2); +} + +#[test] +fn permission_denial_is_terminal_for_dismiss_strategies() { + let result = super::strategy_succeeded( + Err(AdapterError::permission_denied()), + Deadline::standard().expect("deadline"), + ); + + assert_eq!( + result.expect_err("permission denial must abort").code, + ErrorCode::PermDenied + ); +} + +#[test] +fn expired_deadline_is_terminal_for_dismiss_strategies() { + let deadline = Deadline::after(0).expect("deadline"); + let result = super::strategy_succeeded( + Err(AdapterError::new( + ErrorCode::ActionFailed, + "AXDismiss failed", + )), + deadline, + ); + + assert_eq!( + result.expect_err("expired deadline must abort").code, + ErrorCode::Timeout + ); +} diff --git a/crates/macos/src/notifications/dismiss_verify.rs b/crates/macos/src/notifications/dismiss_verify.rs new file mode 100644 index 0000000..2844b01 --- /dev/null +++ b/crates/macos/src/notifications/dismiss_verify.rs @@ -0,0 +1,76 @@ +use std::time::Duration; + +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, NotificationFilter}; + +use super::list::NotificationEntry; + +const REOBSERVATION_INTERVAL: Duration = Duration::from_millis(25); +const STRATEGY_SETTLE_TIME: Duration = Duration::from_millis(250); + +#[cfg(test)] +#[path = "dismiss_verify_tests.rs"] +mod tests; + +pub(super) fn disappeared( + original: &NotificationEntry, + filter: &NotificationFilter, + matching_before: usize, + pid: i32, + deadline: Deadline, +) -> Result<bool, AdapterError> { + let settle_deadline = deadline.capped(STRATEGY_SETTLE_TIME); + let result = wait_with( + || { + let current = super::list::list_entries(filter, pid, settle_deadline)?; + Ok(matching_count(¤t, original) >= matching_before) + }, + settle_deadline, + ); + match result { + Err(error) if error.code == ErrorCode::Timeout && !deadline.is_expired() => Ok(false), + other => other, + } +} + +pub(super) fn verified_after_strategy( + outcome: Result<bool, AdapterError>, + deadline: Deadline, + verify: impl FnOnce() -> Result<bool, AdapterError>, +) -> Result<bool, AdapterError> { + if let Err(error) = outcome { + super::read::tolerate_ax_strategy_error(error, deadline)?; + } + verify() +} + +fn wait_with( + mut is_present: impl FnMut() -> Result<bool, AdapterError>, + deadline: Deadline, +) -> Result<bool, AdapterError> { + loop { + if !is_present()? { + return Ok(true); + } + std::thread::sleep(deadline.remaining_slice(REOBSERVATION_INTERVAL)?); + } +} + +pub(super) fn matching_count(entries: &[NotificationEntry], original: &NotificationEntry) -> usize { + entries + .iter() + .filter(|current| matches(original, current)) + .count() +} + +pub(super) fn matches(original: &NotificationEntry, current: &NotificationEntry) -> bool { + same_info(&original.info, ¤t.info) +} + +fn same_info( + original: &agent_desktop_core::NotificationInfo, + current: &agent_desktop_core::NotificationInfo, +) -> bool { + original.app_name == current.app_name + && original.title == current.title + && original.body == current.body +} diff --git a/crates/macos/src/notifications/dismiss_verify_tests.rs b/crates/macos/src/notifications/dismiss_verify_tests.rs new file mode 100644 index 0000000..fdcb825 --- /dev/null +++ b/crates/macos/src/notifications/dismiss_verify_tests.rs @@ -0,0 +1,52 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, NotificationInfo}; + +#[test] +fn acknowledged_strategy_is_not_success_while_the_row_remains() { + let result = super::wait_with(|| Ok(true), Deadline::after(0).expect("deadline")); + + assert_eq!( + result.expect_err("present row must not verify").code, + ErrorCode::Timeout + ); +} + +#[test] +fn acknowledged_strategy_succeeds_only_after_reobservation_finds_no_row() { + let mut observations = [true, false].into_iter(); + let result = super::wait_with( + || -> Result<bool, AdapterError> { Ok(observations.next().unwrap_or(false)) }, + Deadline::standard().expect("deadline"), + ); + + assert!(result.expect("verification succeeds")); +} + +#[test] +fn tolerated_native_error_succeeds_when_reobservation_proves_removal() { + let result = super::verified_after_strategy( + Err(AdapterError::new( + ErrorCode::ActionFailed, + "target disappeared during AXDismiss", + )), + Deadline::standard().expect("deadline"), + || Ok(true), + ); + + assert!(result.expect("verified removal must dominate native acknowledgement")); +} + +#[test] +fn fallback_identity_includes_the_body() { + let original = NotificationInfo { + index: 1, + app_name: "Calendar".to_owned(), + title: "Reminder".to_owned(), + body: Some("First".to_owned()), + actions: Vec::new(), + }; + let mut other = original.clone(); + other.index = 2; + other.body = Some("Second".to_owned()); + + assert!(!super::same_info(&original, &other)); +} diff --git a/crates/macos/src/notifications/list.rs b/crates/macos/src/notifications/list.rs index b1d2b8b..c9a0bbd 100644 --- a/crates/macos/src/notifications/list.rs +++ b/crates/macos/src/notifications/list.rs @@ -1,21 +1,26 @@ use agent_desktop_core::{ - error::AdapterError, - notification::{NotificationFilter, NotificationInfo}, + AdapterError, Deadline, InteractionPolicy, NotificationFilter, NotificationInfo, }; use super::nc_session::{NcSession, close_session}; pub fn list_notifications( filter: &NotificationFilter, + policy: InteractionPolicy, + deadline: Deadline, ) -> Result<Vec<NotificationInfo>, AdapterError> { - let session = NcSession::open()?; - let result = list_from_nc(filter); + let session = NcSession::open(policy, deadline)?; + let result = list_from_nc(filter, session.pid(), deadline); close_session(session, result) } #[cfg(target_os = "macos")] -fn list_from_nc(filter: &NotificationFilter) -> Result<Vec<NotificationInfo>, AdapterError> { - let entries = list_entries(filter)?; +fn list_from_nc( + filter: &NotificationFilter, + pid: i32, + deadline: Deadline, +) -> Result<Vec<NotificationInfo>, AdapterError> { + let entries = list_entries(filter, pid, deadline)?; Ok(entries.into_iter().map(|e| e.info).collect()) } @@ -28,178 +33,29 @@ pub(super) struct NotificationEntry { #[cfg(target_os = "macos")] pub(super) fn list_entries( filter: &NotificationFilter, + pid: i32, + deadline: Deadline, ) -> Result<Vec<NotificationEntry>, AdapterError> { - use crate::tree::{copy_ax_array, element_for_pid}; - use accessibility_sys::kAXChildrenAttribute; - - let pid = super::nc_session::nc_pid() - .ok_or_else(|| AdapterError::internal("Notification Center process not found"))?; + use crate::tree::element_for_pid; let app = element_for_pid(pid); - let windows = copy_ax_array(&app, "AXWindows").unwrap_or_default(); + let windows = crate::notifications::read::children_for_attribute(&app, "AXWindows", deadline)?; if windows.is_empty() { return Ok(vec![]); } - let app_filter = filter.app.as_deref().map(|s| s.to_lowercase()); - let text_filter = filter.text.as_deref().map(|s| s.to_lowercase()); - let limit = filter.limit.unwrap_or(usize::MAX); - - let mut entries = Vec::new(); - let mut index: usize = 1; - + let mut scan = super::scan::NotificationScan::new(filter, deadline); for window in &windows { - let top_children = copy_ax_array(window, kAXChildrenAttribute).unwrap_or_default(); - collect_notifications( - &top_children, - &app_filter, - &text_filter, - limit, - &mut index, - &mut entries, - 0, - ); - if entries.len() >= limit { + let top_children = crate::notifications::read::children(window, deadline)?; + scan.collect(&top_children, 0)?; + if scan.is_full() { break; } } - - Ok(entries) + Ok(scan.finish()) } -#[cfg(target_os = "macos")] -fn collect_notifications( - elements: &[crate::tree::AXElement], - app_filter: &Option<String>, - text_filter: &Option<String>, - limit: usize, - index: &mut usize, - out: &mut Vec<NotificationEntry>, - depth: u8, -) { - use crate::tree::{copy_ax_array, copy_string_attr}; - use accessibility_sys::{kAXChildrenAttribute, kAXRoleAttribute}; - - if depth > 10 || out.len() >= limit { - return; - } - - for el in elements { - if out.len() >= limit { - return; - } - - let role = copy_string_attr(el, kAXRoleAttribute); - let children = copy_ax_array(el, kAXChildrenAttribute).unwrap_or_default(); - - if is_notification_group(role.as_deref(), &children) { - if let Some(info) = extract_notification(el, &children, *index) { - if matches_filters(&info, app_filter, text_filter) { - out.push(NotificationEntry { - info, - element: el.clone(), - }); - } - *index += 1; - continue; - } - } - - collect_notifications( - &children, - app_filter, - text_filter, - limit, - index, - out, - depth + 1, - ); - } -} - -#[cfg(target_os = "macos")] -fn is_notification_group(role: Option<&str>, children: &[crate::tree::AXElement]) -> bool { - use crate::tree::copy_string_attr; - use accessibility_sys::kAXRoleAttribute; - - if role != Some("AXGroup") { - return false; - } - children.iter().any(|c| { - matches!( - copy_string_attr(c, kAXRoleAttribute).as_deref(), - Some("AXStaticText") | Some("AXButton") - ) - }) -} - -#[cfg(target_os = "macos")] -fn extract_notification( - _group: &crate::tree::AXElement, - children: &[crate::tree::AXElement], - index: usize, -) -> Option<NotificationInfo> { - use crate::tree::copy_string_attr; - use accessibility_sys::{kAXRoleAttribute, kAXValueAttribute}; - - let mut texts: Vec<String> = Vec::new(); - let mut actions: Vec<String> = Vec::new(); - - for child in children { - let role = copy_string_attr(child, kAXRoleAttribute); - match role.as_deref() { - Some("AXStaticText") => { - if let Some(val) = copy_string_attr(child, kAXValueAttribute) { - if !val.is_empty() { - texts.push(val); - } - } - } - Some("AXButton") => { - let name = copy_string_attr(child, "AXTitle") - .or_else(|| copy_string_attr(child, "AXDescription")); - if let Some(n) = name { - if !n.is_empty() && n != "Close" && n != "clear" { - actions.push(n); - } - } - } - _ => {} - } - } - - if texts.is_empty() { - return None; - } - - let app_name = if texts.len() >= 2 { - texts[0].clone() - } else { - String::from("Unknown") - }; - - let title = if texts.len() >= 2 { - texts[1].clone() - } else { - texts[0].clone() - }; - - let body = if texts.len() >= 3 { - Some(texts[2..].join(" ")) - } else { - None - }; - - Some(NotificationInfo { - index, - app_name, - title, - body, - actions, - }) -} - -fn matches_filters( +pub(super) fn matches_filters( info: &NotificationInfo, app_filter: &Option<String>, text_filter: &Option<String>, @@ -225,7 +81,11 @@ fn matches_filters( } #[cfg(not(target_os = "macos"))] -fn list_from_nc(_filter: &NotificationFilter) -> Result<Vec<NotificationInfo>, AdapterError> { +fn list_from_nc( + _filter: &NotificationFilter, + _pid: i32, + _deadline: Deadline, +) -> Result<Vec<NotificationInfo>, AdapterError> { Err(AdapterError::not_supported("list_notifications")) } diff --git a/crates/macos/src/notifications/mod.rs b/crates/macos/src/notifications/mod.rs index f0b4ebb..59e3cc1 100644 --- a/crates/macos/src/notifications/mod.rs +++ b/crates/macos/src/notifications/mod.rs @@ -1,3 +1,6 @@ -pub mod actions; -pub mod list; -pub mod nc_session; +pub(crate) mod actions; +mod dismiss_verify; +pub(crate) mod list; +pub(crate) mod nc_session; +mod read; +mod scan; diff --git a/crates/macos/src/notifications/nc_session.rs b/crates/macos/src/notifications/nc_session.rs index 5eec7cd..f3c52a4 100644 --- a/crates/macos/src/notifications/nc_session.rs +++ b/crates/macos/src/notifications/nc_session.rs @@ -1,248 +1,368 @@ -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::{AdapterError, Deadline, InteractionPolicy, KeyCombo, ProcessIdentity}; + +const CLEANUP_TIMEOUT_MS: u64 = 2_000; pub(crate) fn close_session<T>( session: NcSession, result: Result<T, AdapterError>, ) -> Result<T, AdapterError> { - let close_result = session.close(); - match (result, close_result) { + merge_session_result(result, session.close()) +} + +fn merge_session_result<T>( + result: Result<T, AdapterError>, + cleanup: Result<(), AdapterError>, +) -> Result<T, AdapterError> { + match (result, cleanup) { (Ok(value), Ok(())) => Ok(value), - (Ok(value), Err(close_err)) => { - tracing::warn!(error = %close_err, "notification center close failed after a successful operation"); - Ok(value) + (Ok(_), Err(close_err)) => Err(close_err), + (Err(err), Ok(())) => Err(err), + (Err(err), Err(close_err)) => { + tracing::warn!(error = %close_err, "notification center cleanup also failed after the operation failed"); + Err(err) } - (Err(err), _) => Err(err), } } pub(crate) struct NcSession { - was_already_open: bool, - previous_app: Option<String>, - closed: bool, + pid: i32, + close_pending: bool, + previous_app: Option<ProcessIdentity>, + cleanup_on_drop: bool, +} + +struct NcSessionOps<Open, WaitUntilReady, Close, Reactivate> +where + Open: FnMut(Deadline) -> Result<(), AdapterError>, + WaitUntilReady: FnMut(Deadline) -> Result<i32, AdapterError>, + Close: FnMut(Deadline) -> Result<(), AdapterError>, + Reactivate: FnMut(&ProcessIdentity, Deadline) -> Result<(), AdapterError>, +{ + open: Open, + wait_until_ready: WaitUntilReady, + close: Close, + reactivate: Reactivate, } impl NcSession { - pub(crate) fn open() -> Result<Self, AdapterError> { - let previous_app = frontmost_app(); - let was_already_open = is_nc_open(); - if !was_already_open { - open_nc()?; - wait_for_nc_ready()?; + pub(crate) fn open( + policy: InteractionPolicy, + deadline: Deadline, + ) -> Result<Self, AdapterError> { + if let Some(pid) = nc_pid(deadline)? + && is_nc_open(pid, deadline) + { + return Ok(Self { + pid, + close_pending: false, + previous_app: None, + cleanup_on_drop: true, + }); } - Ok(Self { - was_already_open, + if !policy.is_headed() { + return Err(closed_center_policy_error(policy)); + } + let previous_app = frontmost_app(deadline)?; + crate::system::permissions::require_automation_permission()?; + Self::open_with( previous_app, - closed: false, - }) + deadline, + NcSessionOps { + open: open_nc, + wait_until_ready: wait_for_nc_ready, + close: close_nc, + reactivate: reactivate_app, + }, + ) + } + + fn open_with<Open, WaitUntilReady, Close, Reactivate>( + previous_app: Option<ProcessIdentity>, + deadline: Deadline, + mut ops: NcSessionOps<Open, WaitUntilReady, Close, Reactivate>, + ) -> Result<Self, AdapterError> + where + Open: FnMut(Deadline) -> Result<(), AdapterError>, + WaitUntilReady: FnMut(Deadline) -> Result<i32, AdapterError>, + Close: FnMut(Deadline) -> Result<(), AdapterError>, + Reactivate: FnMut(&ProcessIdentity, Deadline) -> Result<(), AdapterError>, + { + let mut session = Self { + pid: 0, + close_pending: true, + previous_app, + cleanup_on_drop: true, + }; + let result = (ops.open)(deadline).and_then(|()| (ops.wait_until_ready)(deadline)); + match result { + Ok(pid) => { + session.pid = pid; + Ok(session) + } + Err(error) => { + let cleanup = session.cleanup_with(ops.close, ops.reactivate); + merge_session_result(Err(error), cleanup) + } + } + } + + pub(crate) fn pid(&self) -> i32 { + self.pid } pub(crate) fn close(mut self) -> Result<(), AdapterError> { - let close_result = if self.was_already_open { - Ok(()) - } else { - close_nc() - }; - if let Some(ref app) = self.previous_app { - reactivate_app(app); - } - self.closed = true; - close_result + self.close_with(close_nc, reactivate_app) } + + fn close_with( + &mut self, + mut close: impl FnMut(Deadline) -> Result<(), AdapterError>, + mut reactivate: impl FnMut(&ProcessIdentity, Deadline) -> Result<(), AdapterError>, + ) -> Result<(), AdapterError> { + let first = self.cleanup_with(&mut close, &mut reactivate); + let result = if first.is_err() && self.has_pending_cleanup() { + self.cleanup_with(close, reactivate) + } else { + first + }; + self.cleanup_on_drop = false; + result + } + + fn cleanup_with( + &mut self, + mut close: impl FnMut(Deadline) -> Result<(), AdapterError>, + mut reactivate: impl FnMut(&ProcessIdentity, Deadline) -> Result<(), AdapterError>, + ) -> Result<(), AdapterError> { + let close_result = if self.close_pending { + close(Deadline::detached_after(CLEANUP_TIMEOUT_MS)?) + .inspect(|()| self.close_pending = false) + } else { + Ok(()) + }; + let restore_result = if let Some(app) = self.previous_app.as_ref() { + reactivate(app, Deadline::detached_after(CLEANUP_TIMEOUT_MS)?) + .inspect(|()| self.previous_app = None) + } else { + Ok(()) + }; + merge_cleanup_results(close_result, restore_result) + } + + fn has_pending_cleanup(&self) -> bool { + self.close_pending || self.previous_app.is_some() + } +} + +fn closed_center_policy_error(policy: InteractionPolicy) -> AdapterError { + AdapterError::policy_denied_for_policy( + "Notification Center is closed and observation cannot open it in headless mode", + policy, + ) + .with_suggestion( + "Open Notification Center yourself or pass --headed to allow opening and restoring desktop focus.", + ) } impl Drop for NcSession { fn drop(&mut self) { - if self.closed { + if !self.cleanup_on_drop { return; } - if !self.was_already_open { - if let Err(e) = close_nc() { - tracing::warn!("Failed to close NC in Drop: {e}"); - } - } - if let Some(ref app) = self.previous_app { - reactivate_app(app); + if let Err(error) = self.cleanup_with(close_nc, reactivate_app) { + tracing::warn!(%error, "Notification Center cleanup retry failed in Drop"); } } } #[cfg(target_os = "macos")] -fn frontmost_app() -> Option<String> { - let mut command = std::process::Command::new("/usr/bin/osascript"); - command.args([ - "-e", - "tell application \"System Events\" to get name of first application process whose frontmost is true", - ]); - let output = crate::system::process::run_with_timeout( - &mut command, - "frontmost-app osascript", +fn frontmost_app(deadline: Deadline) -> Result<Option<ProcessIdentity>, AdapterError> { + let snapshot = crate::system::workspace_apps::window_owner_snapshot_until(operation_deadline( + deadline, std::time::Duration::from_secs(2), - ) - .ok()?; - if output.status.success() { - let name = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if name.is_empty() { None } else { Some(name) } - } else { - None - } + )?)?; + let Some(owner) = snapshot.frontmost() else { + return Ok(None); + }; + let Some(instance) = crate::system::process_identity::token_for_pid(owner.pid)? else { + return Ok(None); + }; + Ok(Some(ProcessIdentity::new( + crate::system::process_identity::from_pid_t(owner.pid)?, + instance, + ))) } #[cfg(not(target_os = "macos"))] -fn frontmost_app() -> Option<String> { - None +fn frontmost_app(_deadline: Deadline) -> Result<Option<ProcessIdentity>, AdapterError> { + Ok(None) } #[cfg(target_os = "macos")] -fn reactivate_app(name: &str) { - let script = format!("tell application {} to activate", applescript_string(name)); - let mut command = std::process::Command::new("/usr/bin/osascript"); - command.arg("-e").arg(script); - if let Err(e) = crate::system::process::run_with_timeout( - &mut command, - "reactivate-app osascript", - std::time::Duration::from_secs(1), - ) { - tracing::warn!("reactivate_app osascript failed for app {:?}: {e}", name); +fn reactivate_app(app: &ProcessIdentity, deadline: Deadline) -> Result<(), AdapterError> { + let pid = crate::system::process_identity::to_pid_t(app.pid)?; + if !crate::system::process_identity::matches_instance(pid, &app.instance)? { + return Ok(()); } -} - -#[cfg(target_os = "macos")] -fn applescript_string(value: &str) -> String { - let mut escaped = String::with_capacity(value.len() + 2); - escaped.push('"'); - for ch in value.chars() { - if ch.is_control() { - continue; - } - if matches!(ch, '\\' | '"') { - escaped.push('\\'); - } - escaped.push(ch); - } - escaped.push('"'); - escaped + crate::system::focus::ensure_app_focused(pid, deadline) } #[cfg(not(target_os = "macos"))] -fn reactivate_app(_name: &str) {} +fn reactivate_app(_app: &ProcessIdentity, _deadline: Deadline) -> Result<(), AdapterError> { + Ok(()) +} + +fn merge_cleanup_results( + close: Result<(), AdapterError>, + restore: Result<(), AdapterError>, +) -> Result<(), AdapterError> { + match (close, restore) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(error), Err(restore_error)) => { + tracing::warn!(error = %restore_error, "focus restoration also failed during Notification Center cleanup"); + Err(error) + } + } +} #[cfg(target_os = "macos")] -pub(super) fn nc_pid() -> Option<i32> { +fn nc_pid(deadline: Deadline) -> Result<Option<i32>, AdapterError> { let mut command = std::process::Command::new("/usr/bin/pgrep"); command.arg("-x").arg("NotificationCenter"); - let output = crate::system::process::run_with_timeout( + let output = crate::system::process::run_with_deadline( &mut command, "pgrep NotificationCenter", - std::time::Duration::from_secs(1), - ) - .ok()?; + operation_deadline(deadline, std::time::Duration::from_secs(1))?, + ); + nc_pid_from_output(output) +} - String::from_utf8_lossy(&output.stdout) +#[cfg(target_os = "macos")] +fn nc_pid_from_output( + output: Result<std::process::Output, AdapterError>, +) -> Result<Option<i32>, AdapterError> { + let output = output?; + if !output.status.success() { + return Ok(None); + } + Ok(String::from_utf8_lossy(&output.stdout) .trim() .lines() .next() - .and_then(|line| line.trim().parse::<i32>().ok()) + .and_then(|line| line.trim().parse::<i32>().ok())) } #[cfg(target_os = "macos")] -fn is_nc_open() -> bool { - use crate::tree::{copy_ax_array, element_for_pid}; +fn is_nc_open(pid: i32, deadline: Deadline) -> bool { + use crate::tree::element_for_pid; - let pid = match nc_pid() { - Some(p) => p, - None => return false, - }; let app = element_for_pid(pid); - let windows = copy_ax_array(&app, "AXWindows").unwrap_or_default(); + let windows = crate::notifications::read::children_for_attribute(&app, "AXWindows", deadline) + .unwrap_or_default(); !windows.is_empty() } #[cfg(not(target_os = "macos"))] -fn is_nc_open() -> bool { +fn nc_pid(_deadline: Deadline) -> Result<Option<i32>, AdapterError> { + Ok(None) +} + +#[cfg(not(target_os = "macos"))] +fn is_nc_open(_pid: i32, _deadline: Deadline) -> bool { false } #[cfg(target_os = "macos")] -fn open_nc() -> Result<(), AdapterError> { +fn open_nc(deadline: Deadline) -> Result<(), AdapterError> { let script = r#"tell application "System Events" to tell its application process "ControlCenter" click (first menu bar item of menu bar 1 whose description is "Clock") end tell"#; let mut command = std::process::Command::new("/usr/bin/osascript"); command.arg("-e").arg(script); - crate::system::process::run_with_timeout( + let output = crate::system::process::run_with_deadline( &mut command, "osascript open-nc", - std::time::Duration::from_secs(2), + operation_deadline(deadline, std::time::Duration::from_secs(2))?, )?; + if !output.status.success() { + return Err(crate::system::permissions::map_automation_command_failure( + output.status, + &output.stderr, + )); + } std::thread::sleep(std::time::Duration::from_millis(500)); Ok(()) } #[cfg(not(target_os = "macos"))] -fn open_nc() -> Result<(), AdapterError> { +fn open_nc(_deadline: Deadline) -> Result<(), AdapterError> { Err(AdapterError::not_supported("open_nc")) } #[cfg(target_os = "macos")] -fn close_nc() -> Result<(), AdapterError> { +fn close_nc(deadline: Deadline) -> Result<(), AdapterError> { use crate::input::keyboard; - use agent_desktop_core::action::KeyCombo; + let Some(pid) = nc_pid(deadline)? else { + return Ok(()); + }; + if !is_nc_open(pid, deadline) { + return Ok(()); + } let combo = KeyCombo { key: "escape".into(), modifiers: vec![], }; - keyboard::synthesize_key(&combo)?; + keyboard::synthesize_key(&combo, None, deadline)?; std::thread::sleep(std::time::Duration::from_millis(300)); Ok(()) } #[cfg(all(test, target_os = "macos"))] -mod tests { - use super::applescript_string; - - #[test] - fn applescript_string_escapes_quotes_and_backslashes() { - assert_eq!( - applescript_string(r#"Bad \ "Name""#), - r#""Bad \\ \"Name\"""# - ); - } - - #[test] - fn applescript_string_strips_control_chars() { - assert_eq!(applescript_string("a\nb"), r#""ab""#); - assert_eq!(applescript_string("a\tb"), r#""ab""#); - assert_eq!(applescript_string("a\\\nb"), r#""a\\b""#); - assert_eq!(applescript_string("a\"b\nc"), r#""a\"bc""#); - } -} +#[path = "nc_session_tests.rs"] +mod tests; #[cfg(not(target_os = "macos"))] -fn close_nc() -> Result<(), AdapterError> { +fn close_nc(_deadline: Deadline) -> Result<(), AdapterError> { Err(AdapterError::not_supported("close_nc")) } #[cfg(target_os = "macos")] -fn wait_for_nc_ready() -> Result<(), AdapterError> { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); +fn wait_for_nc_ready(deadline: Deadline) -> Result<i32, AdapterError> { let poll = std::time::Duration::from_millis(50); loop { - if is_nc_open() { - return Ok(()); + if let Some(pid) = nc_pid(deadline)? { + if is_nc_open(pid, deadline) { + return Ok(pid); + } } - if std::time::Instant::now() > deadline { + if deadline.is_expired() { return Err(AdapterError::timeout( - "Notification Center did not open within 2 seconds", + "Notification Center did not open within the operation deadline", )); } - std::thread::sleep(poll); + std::thread::sleep(poll.min(deadline.remaining())); } } #[cfg(not(target_os = "macos"))] -fn wait_for_nc_ready() -> Result<(), AdapterError> { +fn wait_for_nc_ready(_deadline: Deadline) -> Result<i32, AdapterError> { Err(AdapterError::not_supported("wait_for_nc_ready")) } + +fn operation_deadline( + deadline: Deadline, + maximum: std::time::Duration, +) -> Result<std::time::Instant, AdapterError> { + let remaining = deadline.remaining(); + if remaining.is_zero() { + Err(deadline.timeout_error()) + } else { + std::time::Instant::now() + .checked_add(remaining.min(maximum)) + .ok_or_else(|| AdapterError::timeout("Notification subprocess deadline overflowed")) + } +} diff --git a/crates/macos/src/notifications/nc_session_tests.rs b/crates/macos/src/notifications/nc_session_tests.rs new file mode 100644 index 0000000..7cd804f --- /dev/null +++ b/crates/macos/src/notifications/nc_session_tests.rs @@ -0,0 +1,210 @@ +use super::{ + NcSession, NcSessionOps, closed_center_policy_error, merge_session_result, nc_pid_from_output, +}; +use agent_desktop_core::{AdapterError, ErrorCode, ProcessIdentity}; + +#[test] +fn nc_pid_preserves_probe_errors() { + let error = nc_pid_from_output(Err(AdapterError::timeout("pid probe timed out"))) + .expect_err("timeout must not become process-not-found"); + + assert_eq!(error.code, ErrorCode::Timeout); +} + +#[test] +fn closed_notification_center_is_policy_denied_headlessly() { + let error = closed_center_policy_error(agent_desktop_core::InteractionPolicy::headless()); + + assert_eq!(error.code, ErrorCode::PolicyDenied); + assert!(error.message.contains("headless")); + assert!( + error + .suggestion + .as_deref() + .is_some_and(|value| value.contains("--headed")) + ); +} + +#[test] +fn cleanup_retries_only_the_failed_step_with_a_fresh_budget() { + let mut session = NcSession { + pid: 7, + close_pending: true, + previous_app: Some(ProcessIdentity::new(9_u32, "instance")), + cleanup_on_drop: false, + }; + let mut close_attempts = 0; + let mut restore_attempts = 0; + + let error = session + .cleanup_with( + |deadline| { + assert!(!deadline.is_expired()); + close_attempts += 1; + Err(AdapterError::timeout("close failed")) + }, + |_, deadline| { + assert!(!deadline.is_expired()); + restore_attempts += 1; + Ok(()) + }, + ) + .expect_err("close failure must be reported"); + + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(close_attempts, 1); + assert_eq!(restore_attempts, 1); + assert!(session.close_pending); + assert!(session.previous_app.is_none()); + + session + .cleanup_with( + |deadline| { + assert!(!deadline.is_expired()); + close_attempts += 1; + Ok(()) + }, + |_, _| panic!("successful restoration must not repeat"), + ) + .unwrap(); + assert_eq!(close_attempts, 2); + assert!(!session.close_pending); +} + +#[test] +fn operation_failure_wins_when_cleanup_also_fails() { + let operation = AdapterError::new(ErrorCode::ElementNotFound, "operation failed"); + let cleanup = AdapterError::timeout("cleanup failed"); + + let error = merge_session_result::<()>(Err(operation), Err(cleanup)).unwrap_err(); + + assert_eq!(error.code, ErrorCode::ElementNotFound); +} + +#[test] +fn cleanup_failure_replaces_an_apparent_operation_success() { + let cleanup = AdapterError::timeout("cleanup failed"); + + let error = merge_session_result(Ok("value"), Err(cleanup)).unwrap_err(); + + assert_eq!(error.code, ErrorCode::Timeout); +} + +#[test] +fn explicit_close_returns_success_when_bounded_retry_recovers() { + let mut session = NcSession { + pid: 7, + close_pending: true, + previous_app: None, + cleanup_on_drop: true, + }; + let attempts = std::cell::Cell::new(0); + + let result = session.close_with( + |_| { + attempts.set(attempts.get() + 1); + if attempts.get() == 1 { + Err(AdapterError::timeout("transient close failure")) + } else { + Ok(()) + } + }, + |_, _| Ok(()), + ); + + assert!(result.is_ok()); + assert_eq!(attempts.get(), 2); + assert!(!session.cleanup_on_drop); +} + +#[test] +fn explicit_close_returns_final_error_without_arming_drop_retry() { + let mut session = NcSession { + pid: 7, + close_pending: true, + previous_app: None, + cleanup_on_drop: true, + }; + let attempts = std::cell::Cell::new(0); + + let error = session + .close_with( + |_| { + attempts.set(attempts.get() + 1); + Err(AdapterError::timeout(format!( + "close attempt {} failed", + attempts.get() + ))) + }, + |_, _| Ok(()), + ) + .unwrap_err(); + + assert_eq!(attempts.get(), 2); + assert!(error.message.contains("attempt 2")); + assert!(!session.cleanup_on_drop); +} + +#[test] +fn partial_open_failure_closes_center_and_restores_previous_app() { + let previous = ProcessIdentity::new(9_u32, "instance"); + let mut close_attempts = 0; + let mut restore_attempts = 0; + + let result = NcSession::open_with( + Some(previous.clone()), + agent_desktop_core::Deadline::after(0).unwrap(), + NcSessionOps { + open: |_| Ok(()), + wait_until_ready: |_| Err(AdapterError::timeout("readiness failed")), + close: |deadline| { + assert!(!deadline.is_expired()); + close_attempts += 1; + Ok(()) + }, + reactivate: |app, deadline| { + assert_eq!(app, &previous); + assert!(!deadline.is_expired()); + restore_attempts += 1; + Ok(()) + }, + }, + ); + let error = match result { + Ok(_) => panic!("readiness failure must be preserved"), + Err(error) => error, + }; + + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(close_attempts, 1); + assert_eq!(restore_attempts, 1); +} + +#[test] +fn close_delay_does_not_consume_focus_restoration_budget() { + let mut session = NcSession { + pid: 7, + close_pending: true, + previous_app: Some(ProcessIdentity::new(9_u32, "instance")), + cleanup_on_drop: false, + }; + let mut restore_attempted = false; + + session + .cleanup_with( + |_| { + std::thread::sleep(std::time::Duration::from_millis(200)); + Err(AdapterError::timeout("close timed out")) + }, + |_, deadline| { + restore_attempted = true; + assert!(deadline.remaining() > std::time::Duration::from_millis(1_900)); + Ok(()) + }, + ) + .expect_err("close failure must remain visible"); + + assert!(restore_attempted); + assert!(session.previous_app.is_none()); + session.close_pending = false; +} diff --git a/crates/macos/src/notifications/read.rs b/crates/macos/src/notifications/read.rs new file mode 100644 index 0000000..d6a2da8 --- /dev/null +++ b/crates/macos/src/notifications/read.rs @@ -0,0 +1,120 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, Rect}; + +use crate::tree::AXElement; + +const MAX_NOTIFICATION_CHILDREN: usize = 256; + +#[cfg(test)] +#[path = "read_tests.rs"] +mod tests; + +pub(super) fn children( + element: &AXElement, + deadline: Deadline, +) -> Result<Vec<AXElement>, AdapterError> { + children_for_attribute(element, "AXChildren", deadline) +} + +pub(super) fn children_for_attribute( + element: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Vec<AXElement>, AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline)?; + let result = crate::tree::attributes::copy_ax_array_prefix_result( + element, + attribute, + MAX_NOTIFICATION_CHILDREN, + deadline, + ); + ensure_budget(deadline)?; + let children = result + .map(|value| value.unwrap_or_default()) + .map_err(|error| read_error(attribute, error))?; + if children.len() == MAX_NOTIFICATION_CHILDREN { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Notification traversal reached its bounded child limit", + ) + .with_details(serde_json::json!({ "complete": false }))); + } + Ok(children) +} + +pub(super) fn string( + element: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Option<String>, AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline)?; + let result = crate::tree::attributes::copy_string_attr_result(element, attribute, deadline); + ensure_budget(deadline)?; + result.map_err(|error| read_error(attribute, error)) +} + +pub(super) fn title_or_description( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<String>, AdapterError> { + title_or_description_with(|attribute| string(element, attribute, deadline)) +} + +fn title_or_description_with( + mut read: impl FnMut(&str) -> Result<Option<String>, AdapterError>, +) -> Result<Option<String>, AdapterError> { + match read("AXTitle")? { + Some(title) => Ok(Some(title)), + None => read("AXDescription"), + } +} + +pub(super) fn tolerate_ax_strategy_error( + error: AdapterError, + deadline: Deadline, +) -> Result<(), AdapterError> { + if deadline.is_expired() { + return Err(deadline.timeout_error()); + } + match error.code { + ErrorCode::ActionFailed + | ErrorCode::ActionNotSupported + | ErrorCode::StaleRef + | ErrorCode::AppUnresponsive => Ok(()), + _ => Err(error), + } +} + +pub(super) fn bounds( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<Rect>, AdapterError> { + let read_deadline = crate::tree::locator_deadline::from_operation(deadline)?; + crate::tree::element_bounds::read_bounds_with_deadline(element, read_deadline) +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn read_error(attribute: &str, error: i32) -> AdapterError { + let code = if error == accessibility_sys::kAXErrorCannotComplete { + ErrorCode::AppUnresponsive + } else if error == accessibility_sys::kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == accessibility_sys::kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }; + AdapterError::new(code, format!("Accessibility read failed for {attribute}")) + .with_details(serde_json::json!({ + "attribute": attribute, + "ax_error": error, + "retryable": error == accessibility_sys::kAXErrorCannotComplete, + })) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) +} diff --git a/crates/macos/src/notifications/read_tests.rs b/crates/macos/src/notifications/read_tests.rs new file mode 100644 index 0000000..9dd96d2 --- /dev/null +++ b/crates/macos/src/notifications/read_tests.rs @@ -0,0 +1,17 @@ +use agent_desktop_core::AdapterError; + +#[test] +fn title_fallback_is_lazy_when_title_is_present() { + let mut attributes = Vec::new(); + let value = super::title_or_description_with(|attribute: &str| { + attributes.push(attribute.to_owned()); + match attribute { + "AXTitle" => Ok(Some("Close".to_owned())), + _ => Err(AdapterError::internal("description must not be read")), + } + }) + .expect("title read"); + + assert_eq!(value.as_deref(), Some("Close")); + assert_eq!(attributes, ["AXTitle"]); +} diff --git a/crates/macos/src/notifications/scan.rs b/crates/macos/src/notifications/scan.rs new file mode 100644 index 0000000..ea785cf --- /dev/null +++ b/crates/macos/src/notifications/scan.rs @@ -0,0 +1,181 @@ +use agent_desktop_core::{AdapterError, Deadline, NotificationFilter, NotificationInfo}; + +use super::list::NotificationEntry; + +#[cfg(test)] +#[path = "scan_tests.rs"] +mod tests; + +pub(super) struct NotificationScan { + app_filter: Option<String>, + text_filter: Option<String>, + limit: usize, + index: usize, + entries: Vec<NotificationEntry>, + deadline: Deadline, +} + +impl NotificationScan { + pub(super) fn new(filter: &NotificationFilter, deadline: Deadline) -> Self { + Self { + app_filter: filter.app.as_deref().map(str::to_lowercase), + text_filter: filter.text.as_deref().map(str::to_lowercase), + limit: filter.limit.unwrap_or(usize::MAX), + index: 1, + entries: Vec::new(), + deadline, + } + } + + pub(super) fn collect( + &mut self, + elements: &[crate::tree::AXElement], + depth: u8, + ) -> Result<(), AdapterError> { + if depth > 10 || self.entries.len() >= self.limit { + return Ok(()); + } + for element in elements { + if self.entries.len() >= self.limit { + return Ok(()); + } + let role = match super::read::string(element, "AXRole", self.deadline) { + Ok(role) => role, + Err(error) => { + tolerate_element_error(error, self.deadline)?; + continue; + } + }; + let children = match super::read::children(element, self.deadline) { + Ok(children) => children, + Err(error) => { + tolerate_element_error(error, self.deadline)?; + continue; + } + }; + if matches!(role.as_deref(), Some("AXGroup" | "AXButton")) && !children.is_empty() { + let extracted = + match extract_notification(element, &children, self.index, self.deadline) { + Ok(info) => info, + Err(error) => { + tolerate_element_error(error, self.deadline)?; + None + } + }; + if let Some(info) = extracted { + if super::list::matches_filters(&info, &self.app_filter, &self.text_filter) { + self.entries.push(NotificationEntry { + info, + element: element.clone(), + }); + } + self.index += 1; + continue; + } + } + self.collect(&children, depth.saturating_add(1))?; + } + Ok(()) + } + + pub(super) fn is_full(&self) -> bool { + self.entries.len() >= self.limit + } + + pub(super) fn finish(self) -> Vec<NotificationEntry> { + self.entries + } +} + +fn extract_notification( + element: &crate::tree::AXElement, + children: &[crate::tree::AXElement], + index: usize, + deadline: Deadline, +) -> Result<Option<NotificationInfo>, AdapterError> { + let stacking_id = super::read::string(element, "AXStackingIdentifier", deadline)?; + let subrole = super::read::string(element, "AXSubrole", deadline)?; + let is_notification = stacking_id + .as_deref() + .is_some_and(|value| !value.is_empty()) + || matches!( + subrole.as_deref(), + Some("AXNotificationCenterAlert" | "AXNotificationCenterBanner") + ); + if !is_notification { + return Ok(None); + } + let description = super::read::string(element, "AXDescription", deadline)?; + let mut text_values = Vec::new(); + let mut actions = Vec::new(); + for child in children { + match super::read::string(child, "AXRole", deadline)?.as_deref() { + Some("AXStaticText") => { + if let Some(value) = super::read::string(child, "AXValue", deadline)? + && !value.is_empty() + { + text_values.push(value); + } + } + Some("AXButton") => { + let identifier = super::read::string(child, "AXIdentifier", deadline)?; + if is_notification_action(identifier.as_deref()) + && let Some(name) = super::read::title_or_description(child, deadline)? + && !name.is_empty() + { + actions.push(name); + } + } + _ => {} + } + } + let Some((app_name, title, body)) = description + .as_deref() + .and_then(|value| parse_row_description(value, &text_values)) + else { + return Ok(None); + }; + Ok(Some(NotificationInfo { + index, + app_name, + title, + body, + actions, + })) +} + +fn parse_row_description( + description: &str, + text_values: &[String], +) -> Option<(String, String, Option<String>)> { + let (field, start) = text_values + .iter() + .filter_map(|value| { + description + .find(&format!(", {value}")) + .map(|start| (value, start)) + }) + .min_by_key(|(_, start)| *start)?; + let app_name = description[..start].trim(); + if app_name.is_empty() { + return None; + } + let remainder = description[start + field.len() + 2..].trim(); + if remainder.is_empty() { + return Some(( + app_name.to_owned(), + app_name.to_owned(), + Some(field.clone()), + )); + } + let body = remainder.strip_prefix(',')?.trim(); + (!body.is_empty()).then(|| (app_name.to_owned(), field.clone(), Some(body.to_owned()))) +} + +pub(super) fn is_notification_action(identifier: Option<&str>) -> bool { + identifier.is_some_and(|value| value.eq_ignore_ascii_case("action_button")) +} + +fn tolerate_element_error(error: AdapterError, deadline: Deadline) -> Result<(), AdapterError> { + super::read::tolerate_ax_strategy_error(error, deadline) +} diff --git a/crates/macos/src/notifications/scan_tests.rs b/crates/macos/src/notifications/scan_tests.rs new file mode 100644 index 0000000..f7785b7 --- /dev/null +++ b/crates/macos/src/notifications/scan_tests.rs @@ -0,0 +1,76 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; + +#[test] +fn transient_element_error_keeps_the_partial_scan() { + let mut collected = vec!["first"]; + let error = AdapterError::new(ErrorCode::AppUnresponsive, "element read failed"); + + super::tolerate_element_error(error, Deadline::standard().expect("deadline")) + .expect("skip element"); + collected.push("third"); + + assert_eq!(collected, ["first", "third"]); +} + +#[test] +fn permission_error_aborts_the_scan() { + let result = super::tolerate_element_error( + AdapterError::permission_denied(), + Deadline::standard().expect("deadline"), + ); + + assert_eq!( + result.expect_err("permission denial must abort").code, + ErrorCode::PermDenied + ); +} + +#[test] +fn row_description_maps_named_notification_fields() { + let texts = [ + "Probe title".to_owned(), + "Probe body".to_owned(), + "now".to_owned(), + ]; + let parsed = super::parse_row_description("Script Editor, Probe title, Probe body", &texts) + .expect("description"); + + assert_eq!(parsed.0, "Script Editor"); + assert_eq!(parsed.1, "Probe title"); + assert_eq!(parsed.2.as_deref(), Some("Probe body")); +} + +#[test] +fn two_part_description_keeps_the_application_as_the_title() { + let texts = ["Probe body".to_owned(), "now".to_owned()]; + let parsed = + super::parse_row_description("Script Editor, Probe body", &texts).expect("description"); + + assert_eq!(parsed.0, "Script Editor"); + assert_eq!(parsed.1, "Script Editor"); + assert_eq!(parsed.2.as_deref(), Some("Probe body")); +} + +#[test] +fn description_correlation_preserves_commas_in_title_and_body() { + let texts = [ + "Hello, world".to_owned(), + "Body, with, commas".to_owned(), + "2m ago".to_owned(), + ]; + let parsed = + super::parse_row_description("Script Editor, Hello, world, Body, with, commas", &texts) + .expect("description"); + + assert_eq!(parsed.0, "Script Editor"); + assert_eq!(parsed.1, "Hello, world"); + assert_eq!(parsed.2.as_deref(), Some("Body, with, commas")); +} + +#[test] +fn only_identified_notification_action_buttons_are_reported() { + assert!(super::is_notification_action(Some("action_button"))); + assert!(!super::is_notification_action(None)); + assert!(!super::is_notification_action(Some("calendar-event"))); + assert!(!super::is_notification_action(Some("close_button"))); +} diff --git a/crates/macos/src/system/adapter.rs b/crates/macos/src/system/adapter.rs new file mode 100644 index 0000000..f3156f5 --- /dev/null +++ b/crates/macos/src/system/adapter.rs @@ -0,0 +1,244 @@ +use agent_desktop_core::{ + ActionResult, AdapterError, Deadline, DismissAllNotificationsRequest, + DismissNotificationRequest, ImageBuffer, InteractionLease, NotificationActionRequest, + NotificationFilter, NotificationInfo, ObservationOps, PermissionReport, ProcessIdentity, + ScreenshotTarget, SignalBaseline, SignalFilter, SnapshotSurface, SystemOps, WindowFilter, + WindowInfo, WindowOp, +}; + +use crate::adapter::MacOSAdapter; + +impl SystemOps for MacOSAdapter { + fn acquire_interaction_lease( + &self, + deadline: Deadline, + ) -> Result<InteractionLease, AdapterError> { + let Some(raw) = std::env::var_os(agent_desktop_core::INTERACTION_LEASE_FD_ENV) else { + return agent_desktop_core::acquire_unix_interaction_lease(deadline); + }; + let raw = raw.into_string().map_err(|_| { + AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Inherited interaction lease FD must be valid UTF-8", + ) + })?; + let fd = raw.parse::<std::os::fd::RawFd>().map_err(|_| { + AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Inherited interaction lease FD must be a nonnegative integer", + ) + })?; + if fd < 0 { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::InvalidArgs, + "Inherited interaction lease FD must be a nonnegative integer", + )); + } + agent_desktop_core::adopt_inherited_unix_interaction_lease(fd, deadline) + } + + fn permission_report(&self, deadline: Deadline) -> Result<PermissionReport, AdapterError> { + crate::system::permissions::report(deadline) + } + + fn request_permissions( + &self, + lease: &InteractionLease, + ) -> Result<PermissionReport, AdapterError> { + crate::system::permissions::request_report(lease.deadline()) + } + + fn unknown_accessibility_means_unsupported(&self) -> bool { + false + } + + fn activate_renderer_accessibility( + &self, + process: ProcessIdentity, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + crate::system::renderer_activation::activate(process, lease.deadline()) + } + + fn focus_window(&self, win: &WindowInfo, lease: &InteractionLease) -> Result<(), AdapterError> { + crate::system::focus::focus_window_impl(win, lease.deadline()) + } + + fn launch_app( + &self, + id: &str, + options: &agent_desktop_core::launch_options::LaunchOptions, + lease: &InteractionLease, + ) -> Result<WindowInfo, AdapterError> { + crate::system::launch::launch_app_impl(id, options, lease.deadline()) + } + + fn process_state( + &self, + process: ProcessIdentity, + deadline: Deadline, + ) -> Result<agent_desktop_core::process_state::ProcessState, AdapterError> { + crate::system::process_state::process_state_impl(process, deadline) + } + + fn supported_surfaces(&self) -> Vec<SnapshotSurface> { + crate::system::signals::supported_surfaces_impl() + } + + fn capture_signal_baseline( + &self, + filter: &SignalFilter, + deadline: Deadline, + ) -> Result<SignalBaseline, AdapterError> { + crate::system::signals::capture_signal_baseline_impl( + filter, + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + + fn close_app( + &self, + app: &agent_desktop_core::AppInfo, + force: bool, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + crate::system::app_ops::close_app_impl(app, force, lease.deadline()) + } + + fn is_protected_process(&self, identifier: &str) -> bool { + crate::system::app_ops::is_protected_process(identifier) + } + + fn is_blocked_combo(&self, combo: &agent_desktop_core::KeyCombo) -> bool { + crate::input::blocked_combo::is_blocked(combo) + } + + fn list_displays( + &self, + deadline: Deadline, + ) -> Result<Vec<agent_desktop_core::DisplayInfo>, AdapterError> { + crate::system::display::list_displays_impl(deadline) + } + + fn screenshot( + &self, + target: ScreenshotTarget, + deadline: Deadline, + ) -> Result<ImageBuffer, AdapterError> { + match target { + ScreenshotTarget::Screen(idx) => { + crate::system::screenshot::capture_screen(idx, deadline) + } + ScreenshotTarget::Display { index, expected } => { + crate::system::screenshot::capture_display(index, &expected, deadline) + } + ScreenshotTarget::ExactWindow(window) => { + crate::system::screenshot::capture_window(&window, deadline) + } + ScreenshotTarget::FullScreen => crate::system::screenshot::capture_screen(0, deadline), + } + } + + fn focused_window(&self, deadline: Deadline) -> Result<Option<WindowInfo>, AdapterError> { + let filter = WindowFilter { + focused_only: true, + app: None, + }; + let windows = self.list_windows(&filter, deadline)?; + Ok(windows.into_iter().next()) + } + + fn press_key_for_app( + &self, + process: ProcessIdentity, + combo: &agent_desktop_core::KeyCombo, + policy: agent_desktop_core::InteractionPolicy, + lease: &InteractionLease, + ) -> Result<ActionResult, AdapterError> { + crate::system::key_dispatch::press_for_app_impl(process, combo, policy, lease.deadline()) + } + + fn wait_for_menu( + &self, + process: ProcessIdentity, + open: bool, + deadline: Deadline, + ) -> Result<(), AdapterError> { + crate::system::wait::wait_for_menu(process, open, deadline) + } + + fn resolve_window_strict( + &self, + win: &WindowInfo, + deadline: Deadline, + ) -> Result<WindowInfo, AdapterError> { + crate::system::window_resolve::resolve_window_strict( + win, + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + + fn window_op( + &self, + win: &WindowInfo, + op: WindowOp, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + crate::system::window_ops::execute(win, op, lease.deadline()) + } + + fn list_notifications( + &self, + filter: &NotificationFilter, + policy: agent_desktop_core::InteractionPolicy, + deadline: Deadline, + lease: Option<&InteractionLease>, + ) -> Result<Vec<NotificationInfo>, AdapterError> { + if policy.allow_focus_steal && lease.is_none() { + return Err(AdapterError::internal( + "Headed notification observation requires an interaction lease", + )); + } + crate::notifications::list::list_notifications(filter, policy, deadline) + } + + fn dismiss_notification( + &self, + request: DismissNotificationRequest<'_>, + _lease: &InteractionLease, + ) -> Result<NotificationInfo, AdapterError> { + crate::notifications::actions::dismiss_notification( + request.index, + request.app_filter, + Some(request.identity), + request.policy, + _lease.deadline(), + ) + } + + fn dismiss_all_notifications( + &self, + request: DismissAllNotificationsRequest<'_>, + _lease: &InteractionLease, + ) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> { + crate::notifications::actions::dismiss_all( + request.app_filter, + request.policy, + _lease.deadline(), + ) + } + + fn notification_action( + &self, + request: NotificationActionRequest<'_>, + _lease: &InteractionLease, + ) -> Result<ActionResult, AdapterError> { + crate::notifications::actions::notification_action( + request.index, + Some(request.identity), + request.action_name, + request.policy, + _lease.deadline(), + ) + } +} diff --git a/crates/macos/src/system/app_inventory.rs b/crates/macos/src/system/app_inventory.rs index 2b5e3a8..b7e7512 100644 --- a/crates/macos/src/system/app_inventory.rs +++ b/crates/macos/src/system/app_inventory.rs @@ -1,110 +1,241 @@ -use agent_desktop_core::{ - adapter::WindowFilter, - node::{AppInfo, WindowInfo}, -}; +use agent_desktop_core::{AdapterError, AppInfo, ErrorCode, ProcessId, WindowFilter, WindowInfo}; +use std::time::Instant; use crate::system::{process_apps, window_inventory, workspace_apps}; -pub(crate) fn list_apps() -> Vec<AppInfo> { - let visible = window_inventory::visible_apps(); - let workspace = workspace_apps::list_apps(); - let process = process_apps::list_apps(); - tracing::debug!( - workspace_count = workspace.len(), - visible_count = visible.len(), - process_count = process.len(), - "system: app inventory sources" - ); - let mut apps = list_apps_from_sources(workspace, visible, process); - if apps.is_empty() { - let fallback = window_inventory::visible_apps(); - tracing::debug!( - fallback_count = fallback.len(), - "system: app inventory visible-window fallback" - ); - merge_apps(&mut apps, fallback); - } - sort_apps(&mut apps); - apps +pub(crate) fn list_apps_complete_until(deadline: Instant) -> Result<Vec<AppInfo>, AdapterError> { + stabilize_apps_until(deadline, || capture_complete_apps(deadline)) } -pub(crate) fn list_windows(filter: &WindowFilter) -> Vec<WindowInfo> { - window_inventory::list_windows(filter, |app_name, visible_apps| { - app_for_name_from_sources( - app_name, - workspace_apps::list_apps(), - visible_apps, - process_apps::list_apps, - ) +pub(crate) fn list_apps_scoped_until( + name: &str, + bundle_id: Option<&str>, + deadline: Instant, +) -> Result<Vec<AppInfo>, AdapterError> { + stabilize_apps_until(deadline, || capture_scoped_apps(name, bundle_id, deadline)) +} + +fn capture_complete_apps(deadline: Instant) -> Result<Vec<AppInfo>, AdapterError> { + ensure_before_deadline(deadline)?; + let apps = complete_apps_from_sources( + workspace_apps::list_apps_until(deadline), + process_apps::list_apps_until(deadline), + )?; + ensure_before_deadline(deadline)?; + validate_app_instances(apps) +} + +fn capture_scoped_apps( + name: &str, + bundle_id: Option<&str>, + deadline: Instant, +) -> Result<Vec<AppInfo>, AdapterError> { + ensure_before_deadline(deadline)?; + let process = if bundle_id.is_none() { + process_apps::list_apps_scoped_until(name, deadline) + } else { + Ok(Vec::new()) + }; + let apps = complete_apps_from_sources( + workspace_apps::list_apps_scoped_until(name, bundle_id, deadline), + process, + )?; + ensure_before_deadline(deadline)?; + validate_app_instances(apps) +} + +fn stabilize_apps_until( + deadline: Instant, + mut capture: impl FnMut() -> Result<Vec<AppInfo>, AdapterError>, +) -> Result<Vec<AppInfo>, AdapterError> { + let mut previous: Option<Vec<AppInfo>> = None; + let mut attempts = 0_u64; + let mut churn_events = 0_u64; + let mut last_failure: Option<AdapterError> = None; + loop { + if Instant::now() >= deadline { + return Err(unstable_apps_error( + attempts, + churn_events, + last_failure.as_ref(), + )); + } + attempts += 1; + match capture() { + Ok(current) => { + if previous + .as_ref() + .is_some_and(|prior| app_signature(prior) == app_signature(¤t)) + { + return Ok(current); + } + churn_events += u64::from(previous.is_some()); + previous = Some(current); + last_failure = None; + } + Err(error) if retryable_inventory_error(&error) => { + churn_events += 1; + previous = None; + last_failure = Some(error); + } + Err(error) => return Err(error), + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if !remaining.is_zero() { + std::thread::sleep(remaining.min(std::time::Duration::from_millis(5))); + } + } +} + +fn app_signature(apps: &[AppInfo]) -> Vec<(ProcessId, &str, Option<&str>, Option<&str>)> { + let mut signature = apps + .iter() + .map(|app| { + ( + app.pid, + app.name.as_str(), + app.bundle_id.as_deref(), + app.process_instance.as_deref(), + ) + }) + .collect::<Vec<_>>(); + signature.sort_unstable(); + signature +} + +fn retryable_inventory_error(error: &AdapterError) -> bool { + error.code == ErrorCode::AppUnresponsive && error.is_explicitly_retryable() +} + +fn unstable_apps_error( + attempts: u64, + churn_events: u64, + last_failure: Option<&AdapterError>, +) -> AdapterError { + AdapterError::timeout("macOS application inventory did not stabilize before the deadline") + .with_suggestion("Retry after application launches and exits settle") + .with_details(serde_json::json!({ + "kind": "application_inventory_unstable", + "attempts": attempts, + "churn_events": churn_events, + "last_failure": last_failure.map(|error| &error.message), + "retryable": true, + })) +} + +pub(crate) fn list_windows_until( + filter: &WindowFilter, + deadline: Instant, +) -> Result<Vec<WindowInfo>, AdapterError> { + window_inventory::list_windows_until(filter, deadline) +} + +fn complete_apps_from_sources( + workspace: Result<Vec<AppInfo>, AdapterError>, + process: Result<Vec<AppInfo>, AdapterError>, +) -> Result<Vec<AppInfo>, AdapterError> { + let (workspace, process) = match (workspace, process) { + (Ok(workspace), Ok(process)) => (workspace, process), + (workspace, process) => return Err(required_sources_failed(&workspace, &process)), + }; + let mut apps = Vec::new(); + merge_apps(&mut apps, workspace)?; + merge_apps(&mut apps, process)?; + sort_apps(&mut apps); + Ok(apps) +} + +fn required_sources_failed( + workspace: &Result<Vec<AppInfo>, AdapterError>, + process: &Result<Vec<AppInfo>, AdapterError>, +) -> AdapterError { + let failures = [ + source_failure("ns_workspace", workspace), + source_failure("ps", process), + ] + .into_iter() + .filter(|failure| !failure["code"].is_null()) + .collect::<Vec<_>>(); + AdapterError::new( + ErrorCode::AppUnresponsive, + "A required macOS app inventory source failed", + ) + .with_suggestion("Retry the event observation with a fresh complete inventory") + .with_details(serde_json::json!({ + "kind": "inventory_sources", + "retryable": true, + "complete": false, + "failures": failures, + })) +} + +fn source_failure(source: &str, result: &Result<Vec<AppInfo>, AdapterError>) -> serde_json::Value { + let error = result.as_ref().err(); + serde_json::json!({ + "source": source, + "code": error.map(|error| error.code.as_str()), + "message": error.map(|error| error.message.as_str()), }) } -pub(crate) fn pid_for_app_name(app_name: &str) -> Option<i32> { - app_for_name(app_name).map(|app| app.pid) +fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { + if Instant::now() >= deadline { + return Err(AdapterError::timeout("macOS app inventory timed out")); + } + Ok(()) } -pub(crate) fn pids_for_app_name(app_name: &str) -> Vec<i32> { - matching_pids(&list_apps(), app_name) -} - -pub(crate) fn app_for_name(app_name: &str) -> Option<AppInfo> { - app_for_name_from_sources( - app_name, - workspace_apps::list_apps(), - &window_inventory::visible_apps(), - process_apps::list_apps, - ) -} - -fn app_for_name_from_sources( - app_name: &str, - workspace: Vec<AppInfo>, - visible: &[AppInfo], - process: impl FnOnce() -> Vec<AppInfo>, -) -> Option<AppInfo> { - let primary = merge_primary_sources(workspace, visible.to_vec()); - find_app_with_process_fallback(&primary, process, app_name) -} - -fn list_apps_from_sources( - workspace: Vec<AppInfo>, - visible: Vec<AppInfo>, - process: Vec<AppInfo>, -) -> Vec<AppInfo> { - let mut apps = merge_primary_sources(workspace, visible); - merge_apps(&mut apps, process); - apps -} - -fn merge_primary_sources(workspace: Vec<AppInfo>, visible: Vec<AppInfo>) -> Vec<AppInfo> { - let mut apps = workspace; - merge_apps(&mut apps, visible); - apps -} - -fn find_app_with_process_fallback( - primary: &[AppInfo], - process: impl FnOnce() -> Vec<AppInfo>, - app_name: &str, -) -> Option<AppInfo> { - find_app_in_apps(primary, app_name).or_else(|| find_app_in_apps(&process(), app_name)) -} - -fn merge_apps(apps: &mut Vec<AppInfo>, incoming: Vec<AppInfo>) { - let mut seen_pids = apps - .iter() - .map(|app| app.pid) - .collect::<rustc_hash::FxHashSet<_>>(); - +fn merge_apps(apps: &mut Vec<AppInfo>, incoming: Vec<AppInfo>) -> Result<(), AdapterError> { for app in incoming { - if seen_pids.insert(app.pid) { - apps.push(app); - } else if let Some(existing) = apps.iter_mut().find(|existing| existing.pid == app.pid) { + let incoming_instance = app.process_instance.as_deref().ok_or_else(|| { + incomplete_identity_error(app.pid, "source omitted the process instance token") + })?; + if let Some(existing) = apps.iter_mut().find(|existing| existing.pid == app.pid) { + if existing.process_instance.as_deref() != Some(incoming_instance) { + return Err(incomplete_identity_error( + app.pid, + "process instance changed between inventory sources", + )); + } if existing.bundle_id.is_none() { existing.bundle_id = app.bundle_id; } + } else { + apps.push(app); } } + Ok(()) +} + +fn validate_app_instances(apps: Vec<AppInfo>) -> Result<Vec<AppInfo>, AdapterError> { + for app in &apps { + let captured = app.process_instance.as_deref().ok_or_else(|| { + incomplete_identity_error(app.pid, "inventory omitted the process instance token") + })?; + let pid = crate::system::process_identity::to_pid_t(app.pid)?; + if crate::system::process_identity::token_for_pid(pid)?.as_deref() != Some(captured) { + return Err(incomplete_identity_error( + app.pid, + "process instance changed before inventory completion", + )); + } + } + Ok(apps) +} + +fn incomplete_identity_error(pid: ProcessId, phase: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Application inventory changed while exact identity was being assembled", + ) + .with_details(serde_json::json!({ + "kind": "inventory_identity_race", + "pid": pid, + "phase": phase, + "complete": false, + "retryable": true, + })) + .with_suggestion("Retry after the application process list stabilizes") } fn sort_apps(apps: &mut [AppInfo]) { @@ -116,13 +247,8 @@ fn sort_apps(apps: &mut [AppInfo]) { }); } -fn find_app_in_apps(apps: &[AppInfo], app_name: &str) -> Option<AppInfo> { - apps.iter() - .find(|app| app.name.eq_ignore_ascii_case(app_name)) - .cloned() -} - -fn matching_pids(apps: &[AppInfo], app_name: &str) -> Vec<i32> { +#[cfg(test)] +fn matching_pids(apps: &[AppInfo], app_name: &str) -> Vec<ProcessId> { let mut pids = apps .iter() .filter(|app| app.name.eq_ignore_ascii_case(app_name)) diff --git a/crates/macos/src/system/app_inventory_tests.rs b/crates/macos/src/system/app_inventory_tests.rs index e49afe6..9ee08bd 100644 --- a/crates/macos/src/system/app_inventory_tests.rs +++ b/crates/macos/src/system/app_inventory_tests.rs @@ -1,18 +1,20 @@ use super::*; -fn app(name: &str, pid: i32) -> AppInfo { +fn app(name: &str, pid: u32) -> AppInfo { AppInfo { name: name.to_string(), - pid, + pid: agent_desktop_core::ProcessId::new(pid), bundle_id: None, + process_instance: Some(format!("instance-{pid}")), } } -fn app_with_bundle(name: &str, pid: i32, bundle_id: &str) -> AppInfo { +fn app_with_bundle(name: &str, pid: u32, bundle_id: &str) -> AppInfo { AppInfo { name: name.to_string(), - pid, + pid: agent_desktop_core::ProcessId::new(pid), bundle_id: Some(bundle_id.to_string()), + process_instance: Some(format!("instance-{pid}")), } } @@ -20,7 +22,7 @@ fn app_with_bundle(name: &str, pid: i32, bundle_id: &str) -> AppInfo { fn merge_apps_does_not_duplicate_same_pid_with_different_name() { let mut apps = vec![app("Preview", 42)]; - merge_apps(&mut apps, vec![app("Preview Helper", 42)]); + merge_apps(&mut apps, vec![app("Preview Helper", 42)]).unwrap(); assert_eq!(apps.len(), 1); assert_eq!(apps[0].name, "Preview"); @@ -33,7 +35,8 @@ fn merge_apps_adds_bundle_id_for_existing_pid() { merge_apps( &mut apps, vec![app_with_bundle("Preview Helper", 42, "com.apple.Preview")], - ); + ) + .unwrap(); assert_eq!(apps.len(), 1); assert_eq!(apps[0].bundle_id.as_deref(), Some("com.apple.Preview")); @@ -43,30 +46,12 @@ fn merge_apps_adds_bundle_id_for_existing_pid() { fn merge_apps_keeps_distinct_pids_with_same_name() { let mut apps = vec![app("Terminal", 10)]; - merge_apps(&mut apps, vec![app("Terminal", 11)]); + merge_apps(&mut apps, vec![app("Terminal", 11)]).unwrap(); assert_eq!(apps.len(), 2); assert_eq!(apps[1].pid, 11); } -#[test] -fn find_app_in_apps_prefers_exact_case_insensitive_match() { - let apps = vec![app("Finder Helper", 10), app("Finder", 11)]; - - assert_eq!( - find_app_in_apps(&apps, "finder").map(|app| app.pid), - Some(11) - ); -} - -#[test] -fn find_app_in_apps_rejects_contains_match() { - let apps = vec![app("Mail Helper", 10), app("Docker Desktop", 11)]; - - assert!(find_app_in_apps(&apps, "Mail").is_none()); - assert!(find_app_in_apps(&apps, "Docker").is_none()); -} - #[test] fn matching_pids_returns_all_exact_name_instances() { let apps = vec![ @@ -80,65 +65,34 @@ fn matching_pids_returns_all_exact_name_instances() { } #[test] -fn find_app_with_process_fallback_uses_process_entries_after_primary_miss() { - let primary = vec![app("Finder", 10)]; +fn merge_apps_rejects_pid_reuse_between_sources() { + let mut apps = vec![app("Old App", 42)]; + let mut replacement = app("Replacement App", 42); + replacement.process_instance = Some("replacement-generation".into()); - assert_eq!( - find_app_with_process_fallback(&primary, || vec![app("Mail", 11)], "Mail") - .map(|app| app.pid), - Some(11) - ); + let error = merge_apps(&mut apps, vec![replacement]).expect_err("PID reuse must fail closed"); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.details.unwrap()["kind"], "inventory_identity_race"); + assert_eq!(apps[0].name, "Old App"); } #[test] -fn find_app_with_process_fallback_prefers_primary_entries() { - let primary = vec![app("Mail", 10)]; - let mut process_called = false; +fn signal_inventory_rejects_even_one_missing_complete_source() { + let error = complete_apps_from_sources(Ok(vec![app("Finder", 10)]), Err(source_error("ps"))) + .unwrap_err(); - assert_eq!( - find_app_with_process_fallback( - &primary, - || { - process_called = true; - vec![app("Mail", 11)] - }, - "Mail" - ) - .map(|app| app.pid), - Some(10) - ); - assert!(!process_called); + assert_eq!(error.code, ErrorCode::AppUnresponsive); + let details = error.details.unwrap(); + assert_eq!(details["complete"], false); + assert_eq!(details["failures"].as_array().unwrap().len(), 1); } #[test] -fn find_app_with_process_fallback_does_not_cross_match_helpers() { - let primary = Vec::new(); +fn signal_inventory_accepts_complete_successful_empty_sources() { + let apps = complete_apps_from_sources(Ok(Vec::new()), Ok(Vec::new())).unwrap(); - assert!( - find_app_with_process_fallback(&primary, || vec![app("Mail Helper", 11)], "Mail").is_none() - ); -} - -#[test] -fn list_apps_from_sources_includes_process_apps_when_primary_has_entries() { - let apps = list_apps_from_sources(vec![app("Finder", 10)], Vec::new(), vec![app("Mail", 11)]); - - assert_eq!( - apps.iter().map(|app| app.name.as_str()).collect::<Vec<_>>(), - vec!["Finder", "Mail"] - ); -} - -#[test] -fn app_for_name_from_sources_uses_visible_entries_without_process_lookup() { - let mut process_called = false; - let app = app_for_name_from_sources("Finder", Vec::new(), &[app("Finder", 10)], || { - process_called = true; - Vec::new() - }); - - assert_eq!(app.map(|app| app.pid), Some(10)); - assert!(!process_called); + assert!(apps.is_empty()); } #[test] @@ -152,3 +106,10 @@ fn sort_apps_orders_by_name_then_pid() { vec![1, 2, 3] ); } + +fn source_error(source: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("{source} inventory failed"), + ) +} diff --git a/crates/macos/src/system/app_list.rs b/crates/macos/src/system/app_list.rs deleted file mode 100644 index f155f37..0000000 --- a/crates/macos/src/system/app_list.rs +++ /dev/null @@ -1,20 +0,0 @@ -use agent_desktop_core::{error::AdapterError, node::AppInfo}; - -pub fn list_apps_impl() -> Result<Vec<AppInfo>, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(crate::system::app_inventory::list_apps()) - } - #[cfg(not(target_os = "macos"))] - Err(AdapterError::not_supported("list_apps")) -} - -#[cfg(target_os = "macos")] -pub(crate) fn pid_for_app_name(app_name: &str) -> Option<i32> { - crate::system::app_inventory::pid_for_app_name(app_name) -} - -#[cfg(target_os = "macos")] -pub(crate) fn pids_for_app_name(app_name: &str) -> Vec<i32> { - crate::system::app_inventory::pids_for_app_name(app_name) -} diff --git a/crates/macos/src/system/app_ops.rs b/crates/macos/src/system/app_ops.rs index 5c2c162..5dc258a 100644 --- a/crates/macos/src/system/app_ops.rs +++ b/crates/macos/src/system/app_ops.rs @@ -1,304 +1,180 @@ -use agent_desktop_core::{ - adapter::WindowFilter, - error::{AdapterError, ErrorCode}, - node::WindowInfo, -}; -use std::time::Duration; +use agent_desktop_core::{AdapterError, AppInfo, Deadline, ErrorCode, ProcessIdentity}; #[cfg(target_os = "macos")] -pub fn pid_from_element(el: &crate::tree::AXElement) -> Option<i32> { - let mut pid: i32 = 0; - let err = unsafe { accessibility_sys::AXUIElementGetPid(el.0, &mut pid) }; - if err == accessibility_sys::kAXErrorSuccess { - Some(pid) - } else { - None - } +pub(crate) fn pid_from_element( + element: &crate::tree::AXElement, + deadline: Deadline, +) -> Option<i32> { + crate::tree::ax_ipc::pid(element, deadline).ok() } -/// Ensures the app is frontmost: a no-op when it already is, otherwise a -/// best-effort raise confirmed by polling. `Ok` therefore means "frontmost -/// ensured", not "a raise happened" — callers surfacing `focused:true` get -/// exactly that ensured semantics. -#[cfg(target_os = "macos")] -pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> { - tracing::debug!("system: ensure_app_focused pid={pid}"); - use accessibility_sys::{AXUIElementSetAttributeValue, kAXErrorSuccess}; - use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; - - let app_el = crate::tree::element_for_pid(pid); - if crate::tree::copy_bool_attr(&app_el, "AXFrontmost") == Some(true) { - return Ok(()); - } - let frontmost_attr = CFString::new("AXFrontmost"); - let err = unsafe { - AXUIElementSetAttributeValue( - app_el.0, - frontmost_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return Err(AdapterError::internal(format!( - "Failed to focus app pid={pid}" - ))); - } - wait_until_frontmost(&app_el); - Ok(()) -} - -/// Focuses the app identified by `pid` on a best-effort basis; if the focus -/// attempt fails the warning is emitted and the caller proceeds unchanged. -#[cfg(target_os = "macos")] -pub(crate) fn focus_best_effort(pid: i32) { - if let Err(e) = ensure_app_focused(pid) { - tracing::warn!(error = %e, "failed to focus app before physical input"); - } -} - -/// Polls `AXFrontmost` until the app actually reports frontmost instead of a -/// fixed settle sleep, so an already-frontmost app costs one read and a slow -/// activation gets the full window. Best-effort: timing out just means the -/// caller proceeds as before the poll existed. -#[cfg(target_os = "macos")] -fn wait_until_frontmost(app_el: &crate::tree::AXElement) { - use std::time::{Duration, Instant}; - - const POLL_INTERVAL: Duration = Duration::from_millis(5); - const FRONTMOST_DEADLINE: Duration = Duration::from_millis(50); - - let deadline = Instant::now() + FRONTMOST_DEADLINE; - loop { - if crate::tree::copy_bool_attr(app_el, "AXFrontmost") == Some(true) { - return; - } - if Instant::now() >= deadline { - return; - } - std::thread::sleep(POLL_INTERVAL); - } -} - -#[cfg(target_os = "macos")] -pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> { - tracing::debug!( - "system: focus_window app={:?} title={:?}", - win.app, - win.title - ); - ensure_app_focused(win.pid)?; - let main_win = crate::tree::window_element_for(win.pid, &win.title); - crate::system::window_ops::raise_window(&main_win); - Ok(()) -} - -#[cfg(not(target_os = "macos"))] -pub fn focus_window_impl(_win: &WindowInfo) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("focus_window")) -} - -#[cfg(target_os = "macos")] -pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError> { - tracing::debug!("system: launch app={id:?} timeout={timeout_ms}ms"); - use crate::system::window_list::list_windows_impl; - use std::process::Command; - use std::time::{Duration, Instant}; - - const OPEN_TIMEOUT: Duration = Duration::from_secs(5); - - if id.contains("..") || id.starts_with('/') { - return Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - format!("Invalid app identifier: '{id}'"), - ) - .with_suggestion("Use an app name like 'Safari' or bundle ID like 'com.apple.Safari'.")); - } - - let filter = WindowFilter { - focused_only: false, - app: Some(id.to_string()), - }; - if let Ok(wins) = list_windows_impl(&filter) { - if let Some(win) = wins.into_iter().next() { - return Ok(win); - } - } - - let mut command = Command::new("/usr/bin/open"); - command.args(open_app_args(id)); - crate::system::process::run_with_timeout(&mut command, "open", OPEN_TIMEOUT)?; - - let start = Instant::now(); - let timeout = Duration::from_millis(timeout_ms); - let mut poll_interval = Duration::from_millis(100); - let max_interval = Duration::from_millis(500); - - loop { - std::thread::sleep(poll_interval); - let filter = WindowFilter { - focused_only: false, - app: Some(id.to_string()), - }; - if let Ok(wins) = list_windows_impl(&filter) { - if let Some(win) = wins.into_iter().next() { - return Ok(win); - } - } - if start.elapsed() > timeout { - break; - } - poll_interval = (poll_interval * 3 / 2).min(max_interval); - } - - Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::AppNotFound, - format!("App '{id}' launched but no window appeared within {timeout_ms} ms"), - ) - .with_suggestion("The app may take longer to start, or it may not create a visible window")) -} - -#[cfg(target_os = "macos")] -fn open_app_args(id: &str) -> [&str; 3] { - ["-g", "-a", id] -} - -#[cfg(not(target_os = "macos"))] -pub fn launch_app_impl(_id: &str, _timeout_ms: u64) -> Result<WindowInfo, AdapterError> { - Err(AdapterError::not_supported("launch_app")) -} - -/// Processes whose termination would break the macOS session: the window -/// server, login session, launchd, the Dock, and Finder. Matched as an -/// exact lowercase name or an exact dot-separated bundle-id component, so -/// display names (`Dock`) and bundle ids (`com.apple.dock`) both resolve -/// while lookalikes (`Docker`, `FinderSync`) stay closable — a substring -/// match would permanently block them. Windows and Linux adapters define -/// their own equivalents (`csrss.exe`/`winlogon.exe`, `gnome-shell`/`Xorg`). const PROTECTED_PROCESSES: &[&str] = &["loginwindow", "windowserver", "dock", "launchd", "finder"]; -pub fn is_protected_process(identifier: &str) -> bool { +pub(crate) fn is_protected_process(identifier: &str) -> bool { let lower = identifier.to_lowercase(); PROTECTED_PROCESSES .iter() - .any(|p| lower == *p || lower.split('.').any(|component| component == *p)) + .any(|protected| lower == *protected || lower.split('.').any(|part| part == *protected)) } fn ensure_not_protected(id: &str) -> Result<(), AdapterError> { if is_protected_process(id) { return Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, + ErrorCode::InvalidArgs, format!("'{id}' is a protected system process and cannot be closed"), ) .with_suggestion( - "Target a regular application; session-critical processes (loginwindow, WindowServer, Dock, Finder, launchd) are never closed.", + "Target a regular application; session-critical processes are never closed.", )); } Ok(()) } -#[cfg(test)] -#[path = "app_ops_tests.rs"] -mod tests; - -/// Closes an app after the protected-process guard. The guard runs here — -/// inside the adapter — so every consumer (CLI, FFI, future MCP) refuses -/// session-critical processes identically; the CLI command's own preflight -/// is an earlier check against the same predicate, not the enforcement -/// point. The error mirrors the CLI contract exactly (code and message). #[cfg(target_os = "macos")] -pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> { - ensure_not_protected(id)?; - tracing::debug!("system: close app={id:?} force={force}"); - use std::process::Command; - - const QUIT_TIMEOUT: Duration = Duration::from_secs(3); - - if force { - let pids = crate::system::app_list::pids_for_app_name(id); - if pids.is_empty() { - return Err(AdapterError::new( - ErrorCode::AppNotFound, - format!("App '{id}' was not running or could not be matched for force close"), - ) - .with_suggestion("Use 'list-apps' to verify the running app name before retrying.")); - } - crate::system::force_close::terminate_app(id, &pids, QUIT_TIMEOUT)?; - } else { - let pid = crate::system::key_dispatch::find_pid_by_name(id)?; - let app_ax = crate::tree::element_for_pid(pid); - let closed = try_quit_via_menu_bar(&app_ax); - if !closed { - if id - .chars() - .any(|c| !c.is_alphanumeric() && !matches!(c, ' ' | '-' | '.' | '_')) - { - return Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - format!("Invalid app name '{id}'"), - ) - .with_suggestion("App name should only contain letters, numbers, spaces, hyphens, dots, or underscores.")); - } - let script = format!( - r#"tell application "System Events" - set theProc to first process whose name is "{id}" - tell theProc to quit -end tell"# - ); - let mut command = Command::new("/usr/bin/osascript"); - command.arg("-e").arg(script); - let output = - crate::system::process::run_with_timeout(&mut command, "osascript", QUIT_TIMEOUT)?; - if !output.status.success() { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("Failed to request graceful quit for app '{id}'"), - ) - .with_platform_detail(String::from_utf8_lossy(&output.stderr).trim().to_string()) - .with_suggestion( - "Use 'list-apps' to verify the app name, or retry with --force.", - )); - } - } +pub(crate) fn close_app_impl( + app: &AppInfo, + force: bool, + deadline: Deadline, +) -> Result<(), AdapterError> { + ensure_not_protected(&app.name)?; + if let Some(bundle_id) = &app.bundle_id { + ensure_not_protected(bundle_id)?; } - Ok(()) + let instance = app.process_instance.as_deref().ok_or_else(|| { + AdapterError::new( + ErrorCode::InvalidArgs, + "Exact close requires a process instance token", + ) + })?; + let identity = crate::system::process_identity::require_core(&ProcessIdentity { + pid: app.pid, + instance: instance.to_owned(), + })?; + terminate_running_application(&app.name, identity, force, deadline) } #[cfg(target_os = "macos")] -fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool { - use accessibility_sys::{AXUIElementPerformAction, kAXErrorSuccess}; - use core_foundation::{base::TCFType, string::CFString}; - - let Some(menu_bar) = crate::tree::copy_element_attr(app_el, "AXMenuBar") else { - return false; - }; - let Some(bar_items) = crate::tree::copy_ax_array(&menu_bar, "AXChildren") else { - return false; - }; - for bar_item in bar_items.iter().skip(1) { - let Some(menus) = crate::tree::copy_ax_array(bar_item, "AXChildren") else { - continue; - }; - for menu in &menus { - let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") else { - continue; - }; - for item in &items { - let Some(t) = crate::tree::copy_string_attr(item, "AXTitle") else { - continue; - }; - if t.starts_with("Quit") { - let press = CFString::new("AXPress"); - let err = - unsafe { AXUIElementPerformAction(item.0, press.as_concrete_TypeRef()) }; - return err == kAXErrorSuccess; - } - } - } +fn terminate_running_application( + id: &str, + identity: crate::system::process_identity::ProcessIdentity, + force: bool, + deadline: Deadline, +) -> Result<(), AdapterError> { + if deadline.is_expired() { + return Err(before_termination_request(deadline.timeout_error())); } - false + crate::system::cocoa_runtime::ensure_cocoa_multithreaded() + .map_err(before_termination_request)?; + if !identity + .still_matches() + .map_err(before_termination_request)? + { + return Ok(()); + } + let outcome = crate::system::appkit_bridge::terminate( + identity.pid(), + identity.launch_time_seconds(), + force, + )?; + match outcome { + crate::system::appkit_bridge::TerminationOutcome::Missing => { + return if identity + .still_matches() + .map_err(before_termination_request)? + { + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "NSRunningApplication could not resolve the verified process instance", + ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())) + } else { + Ok(()) + }; + } + crate::system::appkit_bridge::TerminationOutcome::Rejected => { + if identity + .still_matches() + .map_err(before_termination_request)? + { + return Err(termination_request_not_accepted(id, identity.pid(), force)); + } + return Ok(()); + } + crate::system::appkit_bridge::TerminationOutcome::IdentityMismatch => { + return Err(AdapterError::new( + ErrorCode::StaleRef, + "NSRunningApplication changed identity before termination delivery", + ) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())); + } + crate::system::appkit_bridge::TerminationOutcome::Accepted => {} + } + wait_for_exit(id, identity, force, deadline) +} + +#[cfg(target_os = "macos")] +fn termination_request_not_accepted(id: &str, pid: i32, force: bool) -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + format!("The native termination API did not accept the request for '{id}'"), + ) + .with_details(serde_json::json!({ + "pid": pid, + "force": force, + })) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) +} + +#[cfg(target_os = "macos")] +fn wait_for_exit( + id: &str, + identity: crate::system::process_identity::ProcessIdentity, + force: bool, + deadline: Deadline, +) -> Result<(), AdapterError> { + loop { + if !identity + .still_matches() + .map_err(after_termination_request)? + { + return Ok(()); + } + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_details(serde_json::json!({ + "app": id, + "pid": identity.pid(), + "force": force, + })) + .with_disposition(agent_desktop_core::DeliverySemantics::delivered_unverified())); + } + let pause = deadline + .remaining_slice(std::time::Duration::from_millis(25)) + .map_err(after_termination_request)?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(25))); + } +} + +#[cfg(target_os = "macos")] +fn after_termination_request(error: AdapterError) -> AdapterError { + error.with_disposition(agent_desktop_core::DeliverySemantics::delivered_unverified()) +} + +#[cfg(target_os = "macos")] +fn before_termination_request(error: AdapterError) -> AdapterError { + error.with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) } #[cfg(not(target_os = "macos"))] -pub fn close_app_impl(_id: &str, _force: bool) -> Result<(), AdapterError> { +pub(crate) fn close_app_impl( + _app: &AppInfo, + _force: bool, + _deadline: Deadline, +) -> Result<(), AdapterError> { Err(AdapterError::not_supported("close_app")) } + +#[cfg(test)] +#[path = "app_ops_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/app_ops_tests.rs b/crates/macos/src/system/app_ops_tests.rs index b42a484..9d1f886 100644 --- a/crates/macos/src/system/app_ops_tests.rs +++ b/crates/macos/src/system/app_ops_tests.rs @@ -1,10 +1,5 @@ use super::*; -#[test] -fn open_app_args_preserve_current_focus() { - assert_eq!(open_app_args("Mail"), ["-g", "-a", "Mail"]); -} - #[test] fn protected_processes_match_display_and_bundle_forms() { assert!(is_protected_process("Finder")); @@ -35,8 +30,23 @@ fn lookalike_names_containing_protected_substrings_stay_closable() { fn adapter_guard_refuses_protected_processes_with_the_cli_contract() { let err = ensure_not_protected("loginwindow").unwrap_err(); - assert_eq!(err.code, agent_desktop_core::error::ErrorCode::InvalidArgs); + assert_eq!(err.code, agent_desktop_core::ErrorCode::InvalidArgs); assert!(err.message.contains("protected")); assert!(err.suggestion.is_some()); assert!(ensure_not_protected("TextEdit").is_ok()); } + +#[test] +fn native_termination_rejection_does_not_blame_the_target_application() { + let error = termination_request_not_accepted("Fixture", 42, false); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::ActionFailed); + assert!( + error + .message + .contains("native termination API did not accept") + ); + assert!(!error.message.contains("App 'Fixture' rejected")); + assert_eq!(error.details.as_ref().unwrap()["pid"], 42); + assert_eq!(error.details.as_ref().unwrap()["force"], false); +} diff --git a/crates/macos/src/system/appkit_bridge.m b/crates/macos/src/system/appkit_bridge.m new file mode 100644 index 0000000..5b532a2 --- /dev/null +++ b/crates/macos/src/system/appkit_bridge.m @@ -0,0 +1,214 @@ +#import <AppKit/AppKit.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <math.h> + +typedef struct { + uint8_t status; + uint8_t deliveryStarted; +} AgentDesktopTerminateResult; + +typedef struct { + uint8_t status; + uint8_t *bytes; + size_t length; +} AgentDesktopBytesResult; + +AgentDesktopTerminateResult agent_desktop_terminate_application( + int32_t pid, + double expectedLaunchTime, + uint8_t force +) { + AgentDesktopTerminateResult result = { .status = 4, .deliveryStarted = 0 }; + @try { + @autoreleasepool { + NSRunningApplication *app = + [NSRunningApplication runningApplicationWithProcessIdentifier:pid]; + if (app == nil) { + result.status = 1; + return result; + } + NSDate *launchDate = app.launchDate; + if (launchDate == nil || + fabs(launchDate.timeIntervalSince1970 - expectedLaunchTime) > 5.0) { + result.status = 5; + return result; + } + result.deliveryStarted = 1; + BOOL accepted = force != 0 ? [app forceTerminate] : [app terminate]; + result.status = accepted ? 0 : 2; + return result; + } + } @catch (NSException *exception) { + (void)exception; + result.status = 3; + return result; + } +} + +uint8_t agent_desktop_ensure_cocoa_multithreaded(void) { + @try { + @autoreleasepool { + if ([NSThread isMultiThreaded]) { + return 0; + } + NSThread *thread = [[NSThread alloc] initWithBlock:^{}]; + if (thread == nil) { + return 1; + } + [thread start]; + uint32_t remaining = 1000; + while (!thread.isFinished && remaining > 0) { + usleep(1000); + remaining -= 1; + } + if (!thread.isFinished) { + return 2; + } + return [NSThread isMultiThreaded] ? 0 : 3; + } + } @catch (NSException *exception) { + (void)exception; + return 4; + } +} + +AgentDesktopBytesResult agent_desktop_copy_workspace_snapshot_json(void) { + AgentDesktopBytesResult result = { .status = 5, .bytes = NULL, .length = 0 }; + @try { + @autoreleasepool { + NSWorkspace *workspace = [NSWorkspace sharedWorkspace]; + NSArray<NSRunningApplication *> *running = workspace.runningApplications; + if (running == nil || running.count > 8192) { + result.status = 1; + return result; + } + int32_t frontmostPID = 0; + id frontmostLaunchTime = [NSNull null]; + NSRunningApplication *frontmost = workspace.frontmostApplication; + if (frontmost != nil) { + if (![frontmost isKindOfClass:[NSRunningApplication class]]) { + result.status = 2; + return result; + } + frontmostPID = frontmost.processIdentifier; + NSDate *frontmostLaunchDate = frontmost.launchDate; + if (frontmostPID <= 0) { + result.status = 2; + return result; + } + if (frontmostLaunchDate != nil) { + double launchTime = frontmostLaunchDate.timeIntervalSince1970; + if (!isfinite(launchTime) || launchTime <= 0.0) { + result.status = 2; + return result; + } + frontmostLaunchTime = @(launchTime); + } + } + NSMutableArray<NSDictionary *> *records = + [NSMutableArray arrayWithCapacity:running.count]; + NSMutableSet<NSNumber *> *seen = [NSMutableSet setWithCapacity:running.count]; + for (NSRunningApplication *app in running) { + if (![app isKindOfClass:[NSRunningApplication class]]) { + result.status = 2; + return result; + } + NSApplicationActivationPolicy policy = app.activationPolicy; + NSString *policyName = nil; + switch (policy) { + case NSApplicationActivationPolicyRegular: + policyName = @"regular"; + break; + case NSApplicationActivationPolicyAccessory: + policyName = @"accessory"; + break; + case NSApplicationActivationPolicyProhibited: + continue; + default: + result.status = 2; + return result; + } + if (policyName == nil) { + continue; + } + int32_t pid = app.processIdentifier; + NSString *name = app.localizedName; + if (pid <= 0 || name == nil || name.length == 0 || + [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding] > 16384) { + result.status = 2; + return result; + } + NSNumber *pidNumber = @(pid); + if ([seen containsObject:pidNumber]) { + result.status = 2; + return result; + } + [seen addObject:pidNumber]; + NSDate *launchDate = app.launchDate; + id launchTime = [NSNull null]; + if (launchDate != nil) { + double seconds = launchDate.timeIntervalSince1970; + if (!isfinite(seconds) || seconds <= 0.0) { + result.status = 2; + return result; + } + launchTime = @(seconds); + } + NSMutableDictionary *record = [@{ + @"name": name, + @"pid": pidNumber, + @"launch_time": launchTime, + @"activation_policy": policyName, + } mutableCopy]; + NSString *bundle = app.bundleIdentifier; + if (bundle != nil) { + if ([bundle lengthOfBytesUsingEncoding:NSUTF8StringEncoding] > 16384) { + result.status = 2; + return result; + } + record[@"bundle_id"] = bundle; + } + [records addObject:record]; + } + NSDictionary *snapshot = @{ + @"applications": records, + @"frontmost_pid": @(frontmostPID), + @"frontmost_launch_time": frontmostLaunchTime, + }; + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:snapshot options:0 error:&error]; + if (data == nil || error != nil || data.length > 1048576) { + result.status = 3; + return result; + } + if (data.length > 0) { + result.bytes = malloc(data.length); + if (result.bytes == NULL) { + result.status = 4; + return result; + } + memcpy(result.bytes, data.bytes, data.length); + } + result.length = data.length; + result.status = 0; + return result; + } + } @catch (NSException *exception) { + (void)exception; + if (result.bytes != NULL) { + free(result.bytes); + result.bytes = NULL; + result.length = 0; + } + result.status = 5; + return result; + } +} + +void agent_desktop_free_bridge_bytes(uint8_t *bytes) { + free(bytes); +} diff --git a/crates/macos/src/system/appkit_bridge.rs b/crates/macos/src/system/appkit_bridge.rs new file mode 100644 index 0000000..b8e9817 --- /dev/null +++ b/crates/macos/src/system/appkit_bridge.rs @@ -0,0 +1,179 @@ +use agent_desktop_core::{AdapterError, DeliverySemantics, ErrorCode}; + +const MAX_BRIDGE_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TerminationOutcome { + Accepted, + Missing, + Rejected, + IdentityMismatch, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +struct TerminateResult { + status: u8, + delivery_started: u8, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +struct BytesResult { + status: u8, + bytes: *mut u8, + length: usize, +} + +#[cfg(target_os = "macos")] +pub(crate) fn terminate( + pid: i32, + expected_launch_time: f64, + force: bool, +) -> Result<TerminationOutcome, AdapterError> { + let result = + unsafe { agent_desktop_terminate_application(pid, expected_launch_time, u8::from(force)) }; + match result.status { + 0 => Ok(TerminationOutcome::Accepted), + 1 => Ok(TerminationOutcome::Missing), + 2 => Ok(TerminationOutcome::Rejected), + 5 => Ok(TerminationOutcome::IdentityMismatch), + 3 => Err(bridge_error( + "termination", + result.status, + result.delivery_started != 0, + )), + status => Err(bridge_error("termination", status, false)), + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn ensure_cocoa_multithreaded() -> Result<(), String> { + let status = unsafe { agent_desktop_ensure_cocoa_multithreaded() }; + match status { + 0 => Ok(()), + 1 => Err("NSThread initialization returned null".into()), + 2 => Err("NSThread initialization did not finish within one second".into()), + 3 => Err("Foundation did not enter multithreaded mode".into()), + 4 => Err("Foundation raised an exception during multithreaded initialization".into()), + _ => Err("Foundation returned an invalid initialization status".into()), + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn workspace_snapshot_json() -> Result<Vec<u8>, AdapterError> { + let result = unsafe { agent_desktop_copy_workspace_snapshot_json() }; + let bytes = BridgeBytes(result.bytes); + if result.status != 0 { + return Err(bridge_error("workspace_snapshot", result.status, false)); + } + if result.length > MAX_BRIDGE_BYTES || (result.length > 0 && bytes.0.is_null()) { + return Err(bridge_error("workspace_snapshot", u8::MAX, false)); + } + if result.length == 0 { + return Ok(Vec::new()); + } + Ok(unsafe { std::slice::from_raw_parts(bytes.0, result.length) }.to_vec()) +} + +#[cfg(target_os = "macos")] +struct BridgeBytes(*mut u8); + +#[cfg(target_os = "macos")] +impl Drop for BridgeBytes { + fn drop(&mut self) { + unsafe { agent_desktop_free_bridge_bytes(self.0) }; + } +} + +#[cfg(target_os = "macos")] +fn bridge_error(operation: &str, status: u8, delivery_started: bool) -> AdapterError { + let disposition = if delivery_started { + DeliverySemantics::uncertain() + } else { + DeliverySemantics::not_delivered() + }; + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("The macOS AppKit bridge failed during {operation}"), + ) + .with_suggestion("Retry after macOS finishes updating application state") + .with_details(serde_json::json!({ + "kind": "appkit_bridge", + "operation": operation, + "status": status, + "retryable": !delivery_started, + })) + .with_disposition(disposition) +} + +#[cfg(target_os = "macos")] +unsafe extern "C" { + fn agent_desktop_terminate_application( + pid: i32, + expected_launch_time: f64, + force: u8, + ) -> TerminateResult; + fn agent_desktop_ensure_cocoa_multithreaded() -> u8; + fn agent_desktop_copy_workspace_snapshot_json() -> BytesResult; + fn agent_desktop_free_bridge_bytes(bytes: *mut u8); +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn terminate( + _pid: i32, + _expected_launch_time: f64, + _force: bool, +) -> Result<TerminationOutcome, AdapterError> { + Err(AdapterError::not_supported("terminate application")) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn ensure_cocoa_multithreaded() -> Result<(), String> { + Err("AppKit is unavailable".into()) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn workspace_snapshot_json() -> Result<Vec<u8>, AdapterError> { + Err(AdapterError::not_supported( + "workspace application snapshot", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn termination_outcomes_are_closed_and_distinct() { + assert_ne!(TerminationOutcome::Accepted, TerminationOutcome::Rejected); + assert_ne!(TerminationOutcome::Missing, TerminationOutcome::Rejected); + assert_ne!( + TerminationOutcome::IdentityMismatch, + TerminationOutcome::Rejected + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn workspace_snapshot_bridge_failures_are_structured() { + let error = bridge_error("workspace_snapshot", 2, false); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.disposition, DeliverySemantics::not_delivered()); + let details = error.details.unwrap(); + assert_eq!(details["kind"], "appkit_bridge"); + assert_eq!(details["operation"], "workspace_snapshot"); + assert_eq!(details["status"], 2); + assert_eq!(details["retryable"], true); + } + + #[cfg(target_os = "macos")] + #[test] + fn post_selector_exception_is_delivery_uncertain() { + let error = bridge_error("termination", 3, true); + + assert_eq!(error.disposition, DeliverySemantics::uncertain()); + assert_eq!(error.details.unwrap()["retryable"], false); + } +} diff --git a/crates/macos/src/system/cg_window.rs b/crates/macos/src/system/cg_window.rs index 1827236..fbf2dbb 100644 --- a/crates/macos/src/system/cg_window.rs +++ b/crates/macos/src/system/cg_window.rs @@ -1,6 +1,8 @@ +use agent_desktop_core::{AdapterError, ErrorCode, Rect, WindowInfo}; use core_foundation::{base::CFType, dictionary::CFDictionary, string::CFString}; +use std::time::Instant; -type WindowDictionary = CFDictionary<CFString, CFType>; +pub(super) type WindowDictionary = CFDictionary<CFString, CFType>; #[derive(Clone, Debug, PartialEq)] pub(crate) struct WindowRecord { @@ -8,55 +10,291 @@ pub(crate) struct WindowRecord { pub(crate) pid: i32, pub(crate) title: Option<String>, pub(crate) window_number: i64, - pub(crate) area: f64, + pub(crate) bounds: Rect, + pub(crate) visible: bool, + pub(crate) process_instance: Option<String>, } -pub(crate) fn visible_window_records() -> Vec<WindowRecord> { - window_dictionaries() - .into_iter() - .filter_map(|dict| { - if int_field(&dict, "kCGWindowLayer")? != 0 { - return None; - } +impl WindowRecord { + pub(crate) fn display_title(&self) -> &str { + self.title.as_deref().unwrap_or(&self.app_name) + } - let pid = int_field(&dict, "kCGWindowOwnerPID")? as i32; - if pid <= 0 { - return None; - } - - let app_name = string_field(&dict, "kCGWindowOwnerName")?; - if app_name.is_empty() { - return None; - } - - Some(WindowRecord { - app_name, - pid, - title: string_field(&dict, "kCGWindowName").filter(|title| !title.is_empty()), - window_number: int_field(&dict, "kCGWindowNumber").unwrap_or(0), - area: area_field(&dict, "kCGWindowBounds").unwrap_or(0.0), - }) + pub(crate) fn into_window_info( + self, + is_focused: bool, + minimized: Option<bool>, + ) -> Result<WindowInfo, AdapterError> { + let title = self.title.unwrap_or_else(|| self.app_name.clone()); + Ok(WindowInfo { + id: format!("w-{}", self.window_number), + title, + app: self.app_name, + pid: crate::system::process_identity::from_pid_t(self.pid)?, + process_instance: self.process_instance, + bounds: Some(self.bounds), + state: agent_desktop_core::WindowState { + is_focused, + minimized, + visible: Some(self.visible), + }, }) - .collect() + } } -fn window_dictionaries() -> Vec<WindowDictionary> { +#[derive(Clone, Copy)] +pub(crate) enum WindowRecordScope<'a> { + App(&'a str), + Pid(i32), + Pids(&'a rustc_hash::FxHashSet<i32>), + Window(i64), +} + +impl WindowRecordScope<'_> { + fn matches(self, app_name: &str, pid: i32, window_number: i64) -> bool { + match self { + Self::App(expected) => app_name.eq_ignore_ascii_case(expected), + Self::Pid(expected) => pid == expected, + Self::Pids(expected) => expected.contains(&pid), + Self::Window(expected) => window_number == expected, + } + } +} + +pub(crate) fn window_records_until( + deadline: Instant, + scope: WindowRecordScope<'_>, +) -> Result<Vec<WindowRecord>, AdapterError> { + stabilize_records_until(deadline, || capture_once(deadline, scope)) +} + +fn capture_once( + deadline: Instant, + scope: WindowRecordScope<'_>, +) -> Result<Vec<WindowRecord>, AdapterError> { + ensure_before_deadline(deadline)?; + let mut records = records_from_dictionaries(window_dictionaries()?, scope)?; + capture_process_instances(&mut records, deadline)?; + ensure_before_deadline(deadline)?; + Ok(records) +} + +fn stabilize_records_until( + deadline: Instant, + mut capture: impl FnMut() -> Result<Vec<WindowRecord>, AdapterError>, +) -> Result<Vec<WindowRecord>, AdapterError> { + let mut previous: Option<Vec<WindowRecord>> = None; + let mut attempts = 0_u64; + let mut churn_events = 0_u64; + let mut last_failure: Option<AdapterError> = None; + loop { + if Instant::now() >= deadline { + return Err(unstable_inventory_error( + attempts, + churn_events, + last_failure.as_ref(), + )); + } + attempts += 1; + match capture() { + Ok(current) => { + if previous.as_ref().is_some_and(|prior| { + inventory_signature(prior) == inventory_signature(¤t) + }) { + return Ok(current); + } + churn_events += u64::from(previous.is_some()); + previous = Some(current); + last_failure = None; + } + Err(error) if retryable_inventory_error(&error) => { + churn_events += 1; + previous = None; + last_failure = Some(error); + } + Err(error) => return Err(error), + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if !remaining.is_zero() { + std::thread::sleep(remaining.min(std::time::Duration::from_millis(5))); + } + } +} + +pub(super) fn records_from_dictionaries( + dictionaries: Vec<WindowDictionary>, + scope: WindowRecordScope<'_>, +) -> Result<Vec<WindowRecord>, AdapterError> { + let mut records = Vec::new(); + for dictionary in dictionaries { + let layer = required_int_field(&dictionary, "kCGWindowLayer")?; + if layer != 0 { + continue; + } + let pid = i32::try_from(required_int_field(&dictionary, "kCGWindowOwnerPID")?) + .map_err(|_| invalid_field_error("kCGWindowOwnerPID"))?; + if pid <= 0 { + continue; + } + let app_name = required_string_field(&dictionary, "kCGWindowOwnerName")?; + if app_name.is_empty() { + continue; + } + let window_number = required_int_field(&dictionary, "kCGWindowNumber")?; + if !scope.matches(&app_name, pid, window_number) { + continue; + } + let bounds = rect_field(&dictionary, "kCGWindowBounds") + .ok_or_else(|| missing_field_error("kCGWindowBounds"))?; + records.push(WindowRecord { + app_name, + pid, + title: string_field(&dictionary, "kCGWindowName").filter(|title| !title.is_empty()), + window_number, + bounds, + visible: bool_field(&dictionary, "kCGWindowIsOnscreen")?, + process_instance: None, + }); + } + Ok(records) +} + +fn window_dictionaries() -> Result<Vec<WindowDictionary>, AdapterError> { use crate::cf_type::borrowed_cf_dictionary; use core_graphics::display::CGDisplay; - use core_graphics::window::{ - kCGWindowListExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, - }; + let options = window_list_options(); + let array = CGDisplay::window_list_info(options, None).ok_or_else(|| { + inventory_error("CGWindowListCopyWindowInfo returned null instead of a window inventory") + })?; - let options = kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements; - let Some(array) = CGDisplay::window_list_info(options, None) else { - return Vec::new(); - }; + let values = array.get_all_values(); + let mut dictionaries = Vec::with_capacity(values.len()); + for raw in values { + let dictionary = borrowed_cf_dictionary(raw as core_foundation::base::CFTypeRef) + .ok_or_else(|| inventory_error("CoreGraphics returned a non-dictionary window"))?; + dictionaries.push(dictionary); + } + Ok(dictionaries) +} - array - .get_all_values() - .into_iter() - .filter_map(|raw| borrowed_cf_dictionary(raw as core_foundation::base::CFTypeRef)) - .collect() +fn window_list_options() -> core_graphics::window::CGWindowListOption { + core_graphics::window::kCGWindowListOptionAll + | core_graphics::window::kCGWindowListExcludeDesktopElements +} + +pub(super) fn capture_process_instances( + records: &mut [WindowRecord], + deadline: Instant, +) -> Result<(), AdapterError> { + capture_process_instances_with(records, deadline, |pid| { + crate::system::process_identity::token_for_pid(pid) + }) +} + +fn capture_process_instances_with( + records: &mut [WindowRecord], + deadline: Instant, + mut resolve: impl FnMut(i32) -> Result<Option<String>, AdapterError>, +) -> Result<(), AdapterError> { + let mut instances = std::collections::HashMap::<i32, String>::new(); + for record in records { + ensure_before_deadline(deadline)?; + let instance = match instances.entry(record.pid) { + std::collections::hash_map::Entry::Occupied(entry) => entry.get().clone(), + std::collections::hash_map::Entry::Vacant(entry) => { + let instance = resolve(record.pid)? + .ok_or_else(|| inventory_error("Window owner exited during inventory"))?; + entry.insert(instance).clone() + } + }; + record.process_instance = Some(instance); + } + Ok(()) +} + +type InventorySignatureEntry<'a> = (i32, i64, &'a str, Option<&'a str>, [u64; 4], bool, &'a str); + +fn inventory_signature(records: &[WindowRecord]) -> Vec<InventorySignatureEntry<'_>> { + let mut signature = records + .iter() + .map(|record| { + ( + record.pid, + record.window_number, + record.app_name.as_str(), + record.title.as_deref(), + [ + record.bounds.x.to_bits(), + record.bounds.y.to_bits(), + record.bounds.width.to_bits(), + record.bounds.height.to_bits(), + ], + record.visible, + record.process_instance.as_deref().unwrap_or(""), + ) + }) + .collect::<Vec<_>>(); + signature.sort_unstable(); + signature +} + +pub(super) fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { + if Instant::now() >= deadline { + return Err(AdapterError::timeout( + "CoreGraphics window inventory timed out", + )); + } + Ok(()) +} + +pub(super) fn inventory_error(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::AppUnresponsive, message) + .with_suggestion("Retry after WindowServer finishes updating the window inventory") + .with_details(serde_json::json!({ + "kind": "inventory_source", + "source": "core_graphics_windows", + "retryable": true, + })) +} + +fn retryable_inventory_error(error: &AdapterError) -> bool { + error.code == ErrorCode::AppUnresponsive && error.is_explicitly_retryable() +} + +fn unstable_inventory_error( + attempts: u64, + churn_events: u64, + last_failure: Option<&AdapterError>, +) -> AdapterError { + AdapterError::timeout("CoreGraphics window inventory did not stabilize before the deadline") + .with_suggestion("Retry after active window animations and application launches settle") + .with_details(serde_json::json!({ + "kind": "core_graphics_window_inventory_unstable", + "attempts": attempts, + "churn_events": churn_events, + "last_failure": last_failure.map(|error| &error.message), + "retryable": true, + })) +} + +fn missing_field_error(field: &str) -> AdapterError { + inventory_error(&format!( + "CoreGraphics window inventory omitted required field {field}" + )) +} + +fn invalid_field_error(field: &str) -> AdapterError { + inventory_error(&format!( + "CoreGraphics window inventory returned invalid field {field}" + )) +} + +fn required_int_field(dict: &WindowDictionary, key: &str) -> Result<i64, AdapterError> { + int_field(dict, key).ok_or_else(|| missing_field_error(key)) +} + +fn required_string_field(dict: &WindowDictionary, key: &str) -> Result<String, AdapterError> { + string_field(dict, key).ok_or_else(|| missing_field_error(key)) } fn int_field(dict: &WindowDictionary, key: &str) -> Option<i64> { @@ -79,16 +317,32 @@ fn string_field(dict: &WindowDictionary, key: &str) -> Option<String> { .map(|value| value.to_string()) } -fn area_field(dict: &WindowDictionary, key: &str) -> Option<f64> { +fn bool_field(dict: &WindowDictionary, key: &str) -> Result<bool, AdapterError> { + use core_foundation::boolean::CFBoolean; + + let cf_key = CFString::new(key); + let Some(value) = dict.find(&cf_key) else { + return Ok(false); + }; + value + .downcast::<CFBoolean>() + .map(bool::from) + .ok_or_else(|| invalid_field_error(key)) +} + +fn rect_field(dict: &WindowDictionary, key: &str) -> Option<Rect> { use crate::cf_type::borrowed_cf_dictionary; use core_foundation::base::TCFType; let bounds = dict .find(CFString::new(key)) .and_then(|value| borrowed_cf_dictionary(value.as_concrete_TypeRef()))?; - let width = int_or_float_field(&bounds, "Width").unwrap_or(0.0); - let height = int_or_float_field(&bounds, "Height").unwrap_or(0.0); - Some(width * height) + Some(Rect { + x: int_or_float_field(&bounds, "X")?, + y: int_or_float_field(&bounds, "Y")?, + width: int_or_float_field(&bounds, "Width")?, + height: int_or_float_field(&bounds, "Height")?, + }) } fn int_or_float_field(dict: &WindowDictionary, key: &str) -> Option<f64> { @@ -100,3 +354,7 @@ fn int_or_float_field(dict: &WindowDictionary, key: &str) -> Option<f64> { .and_then(|value| borrowed_cf_number(value.as_concrete_TypeRef())) .and_then(|number| number.to_f64()) } + +#[cfg(test)] +#[path = "cg_window_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/cg_window_exact.rs b/crates/macos/src/system/cg_window_exact.rs new file mode 100644 index 0000000..dce76cf --- /dev/null +++ b/crates/macos/src/system/cg_window_exact.rs @@ -0,0 +1,76 @@ +use agent_desktop_core::AdapterError; +use core_foundation::array::CFArray; +use std::time::Instant; + +use super::cg_window::{WindowDictionary, WindowRecord, WindowRecordScope}; + +pub(crate) fn exact_window_record_until( + window_number: i64, + deadline: Instant, +) -> Result<Option<WindowRecord>, AdapterError> { + exact_window_record_until_with(window_number, deadline, || { + let dictionaries = exact_window_dictionaries(window_number)?; + let mut records = super::cg_window::records_from_dictionaries( + dictionaries, + WindowRecordScope::Window(window_number), + )?; + super::cg_window::capture_process_instances(&mut records, deadline)?; + Ok(records) + }) +} + +fn exact_window_record_until_with( + window_number: i64, + deadline: Instant, + capture: impl FnOnce() -> Result<Vec<WindowRecord>, AdapterError>, +) -> Result<Option<WindowRecord>, AdapterError> { + super::cg_window::ensure_before_deadline(deadline)?; + let mut records = capture()?; + super::cg_window::ensure_before_deadline(deadline)?; + if records.len() > 1 { + return Err(exact_window_source_error( + "CoreGraphics returned multiple records for one exact window ID", + window_number, + )); + } + let record = records.pop(); + if record + .as_ref() + .is_some_and(|record| record.window_number != window_number) + { + return Err(exact_window_source_error( + "CoreGraphics returned a different record for one exact window ID", + window_number, + )); + } + Ok(record) +} + +fn exact_window_dictionaries(window_number: i64) -> Result<Vec<WindowDictionary>, AdapterError> { + let window_id = u32::try_from(window_number).map_err(|_| { + exact_window_source_error("Window ID is outside the CoreGraphics range", window_number) + })?; + let requested = CFArray::from_copyable(&[window_id]); + let descriptions = + core_graphics::window::create_description_from_array(requested).ok_or_else(|| { + exact_window_source_error( + "CGWindowListCreateDescriptionFromArray returned null", + window_number, + ) + })?; + Ok(descriptions.iter().map(|record| record.clone()).collect()) +} + +fn exact_window_source_error(message: &str, window_number: i64) -> AdapterError { + super::cg_window::inventory_error(message).with_details(serde_json::json!({ + "kind": "exact_window_inventory_source", + "source": "core_graphics_exact_window", + "window_id": format!("w-{window_number}"), + "complete": false, + "retryable": true, + })) +} + +#[cfg(test)] +#[path = "cg_window_exact_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/cg_window_exact_tests.rs b/crates/macos/src/system/cg_window_exact_tests.rs new file mode 100644 index 0000000..ed488c3 --- /dev/null +++ b/crates/macos/src/system/cg_window_exact_tests.rs @@ -0,0 +1,88 @@ +use super::*; +use agent_desktop_core::{ErrorCode, Rect}; +use std::cell::Cell; + +#[test] +fn exact_window_capture_does_not_wait_for_unrelated_inventory_stability() { + let captures = Cell::new(0_u32); + let global_inventory_calls = Cell::new(0_u32); + let unrelated_inventory_generation = Cell::new(0_u32); + let result = exact_window_record_until_with( + 7, + Instant::now() + std::time::Duration::from_secs(1), + || { + captures.set(captures.get() + 1); + unrelated_inventory_generation.set(unrelated_inventory_generation.get() + 1); + Ok(vec![record(10, 7, "instance-10")]) + }, + ) + .unwrap() + .unwrap(); + + assert_eq!(captures.get(), 1); + assert_eq!(global_inventory_calls.get(), 0); + assert_eq!(unrelated_inventory_generation.get(), 1); + assert_eq!(result.window_number, 7); +} + +#[test] +fn exact_window_capture_accepts_absence_without_broadening() { + let result = exact_window_record_until_with( + 7, + Instant::now() + std::time::Duration::from_secs(1), + || Ok(Vec::new()), + ) + .unwrap(); + + assert!(result.is_none()); +} + +#[test] +fn exact_window_capture_rejects_multiple_records() { + let error = exact_window_record_until_with( + 7, + Instant::now() + std::time::Duration::from_secs(1), + || { + Ok(vec![ + record(10, 7, "instance-10"), + record(10, 7, "instance-10"), + ]) + }, + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!( + error.details.unwrap()["kind"], + "exact_window_inventory_source" + ); +} + +#[test] +fn exact_window_capture_rejects_a_different_window_id() { + let error = exact_window_record_until_with( + 7, + Instant::now() + std::time::Duration::from_secs(1), + || Ok(vec![record(10, 8, "instance-10")]), + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); +} + +fn record(pid: i32, window_number: i64, process_instance: &str) -> WindowRecord { + WindowRecord { + app_name: "Fixture".into(), + pid, + title: Some("Window".into()), + window_number, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + visible: true, + process_instance: Some(process_instance.into()), + } +} diff --git a/crates/macos/src/system/cg_window_tests.rs b/crates/macos/src/system/cg_window_tests.rs new file mode 100644 index 0000000..1a305c6 --- /dev/null +++ b/crates/macos/src/system/cg_window_tests.rs @@ -0,0 +1,224 @@ +use super::*; + +#[test] +fn expired_cg_window_deadline_is_rejected_before_native_reads() { + let error = window_records_until(Instant::now(), WindowRecordScope::Pid(1)).unwrap_err(); + + assert_eq!(error.code.as_str(), "TIMEOUT"); +} + +#[test] +fn malformed_cg_window_data_is_a_retryable_source_failure() { + let error = missing_field_error("kCGWindowNumber"); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + let details = error.details.unwrap(); + assert_eq!(details["source"], "core_graphics_windows"); + assert_eq!(details["retryable"], true); +} + +#[test] +fn inventory_includes_offscreen_and_minimized_windows() { + let options = window_list_options(); + + assert_eq!( + options & core_graphics::window::kCGWindowListOptionOnScreenOnly, + 0 + ); +} + +#[test] +fn changing_inventory_retries_until_two_consecutive_captures_match() { + let mut attempt = 0; + let stable = records_fixture(true); + let result = + stabilize_records_until(Instant::now() + std::time::Duration::from_secs(1), || { + attempt += 1; + Ok(if attempt == 1 { + records_fixture(false) + } else { + stable.clone() + }) + }) + .unwrap(); + + assert_eq!(attempt, 3); + assert_eq!(result, stable); +} + +#[test] +fn persistent_window_churn_times_out_with_exact_attempt_metrics() { + let mut visible = false; + let mut attempts = 0_u64; + let error = stabilize_records_until( + Instant::now() + std::time::Duration::from_millis(20), + || { + attempts += 1; + visible = !visible; + Ok(records_fixture(visible)) + }, + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::Timeout); + let details = error.details.unwrap(); + assert!(attempts >= 1); + assert_eq!(details["attempts"].as_u64(), Some(attempts)); + assert_eq!( + details["churn_events"].as_u64(), + Some(attempts.saturating_sub(1)) + ); +} + +#[test] +fn scoped_capture_never_probes_an_unrelated_inaccessible_owner() { + let mut records = vec![record("Target", 10, 7, true), record("Other", 418, 8, true)]; + retain_scope(&mut records, WindowRecordScope::App("Target")); + let mut probed = Vec::new(); + + capture_process_instances_with( + &mut records, + Instant::now() + std::time::Duration::from_secs(1), + |pid| { + probed.push(pid); + if pid == 418 { + Err(AdapterError::permission_denied()) + } else { + Ok(Some(format!("instance-{pid}"))) + } + }, + ) + .unwrap(); + + assert_eq!(probed, [10]); + assert_eq!(records[0].process_instance.as_deref(), Some("instance-10")); +} + +#[test] +fn scoped_capture_propagates_the_selected_owners_denial() { + let mut records = vec![record("Target", 418, 8, true)]; + let error = capture_process_instances_with( + &mut records, + Instant::now() + std::time::Duration::from_secs(1), + |_| Err(AdapterError::permission_denied()), + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::PermDenied); +} + +#[test] +fn app_scope_keeps_all_matching_processes_and_ignores_unrelated_churn() { + let mut first = vec![record("Target", 10, 7, true), record("Other", 20, 8, false)]; + let mut second = vec![record("Target", 10, 7, true), record("Other", 20, 8, true)]; + retain_scope(&mut first, WindowRecordScope::App("Target")); + retain_scope(&mut second, WindowRecordScope::App("Target")); + first.push(record("Target", 11, 9, true)); + second.push(record("Target", 11, 9, true)); + + assert_eq!(first.len(), 2); + assert_eq!(inventory_signature(&first), inventory_signature(&second)); +} + +#[test] +fn pid_and_window_scopes_exclude_unrelated_owners_before_identity_reads() { + for scope in [WindowRecordScope::Pid(10), WindowRecordScope::Window(7)] { + let mut records = vec![record("Target", 10, 7, true), record("Other", 20, 8, true)]; + retain_scope(&mut records, scope); + let mut probed = Vec::new(); + capture_process_instances_with( + &mut records, + Instant::now() + std::time::Duration::from_secs(1), + |pid| { + probed.push(pid); + Ok(Some(format!("instance-{pid}"))) + }, + ) + .unwrap(); + + assert_eq!(probed, [10]); + } +} + +#[test] +fn pid_set_scope_excludes_ineligible_owners_before_identity_reads() { + let eligible = rustc_hash::FxHashSet::from_iter([10, 11]); + let mut records = vec![ + record("Target", 10, 7, true), + record("Accessory", 11, 8, true), + record("Protected", 418, 9, true), + ]; + retain_scope(&mut records, WindowRecordScope::Pids(&eligible)); + let mut probed = Vec::new(); + + capture_process_instances_with( + &mut records, + Instant::now() + std::time::Duration::from_secs(1), + |pid| { + probed.push(pid); + if pid == 418 { + Err(AdapterError::permission_denied()) + } else { + Ok(Some(format!("instance-{pid}"))) + } + }, + ) + .unwrap(); + + assert_eq!(probed, [10, 11]); + assert!(records.iter().all(|record| record.pid != 418)); +} + +#[test] +fn target_churn_remains_visible_after_scoping() { + let mut first = vec![record("Target", 10, 7, false)]; + let mut second = vec![record("Target", 10, 7, true)]; + retain_scope(&mut first, WindowRecordScope::App("Target")); + retain_scope(&mut second, WindowRecordScope::App("Target")); + + assert_ne!(inventory_signature(&first), inventory_signature(&second)); +} + +#[test] +fn unfiltered_capture_remains_fail_closed() { + let mut records = vec![record("Target", 10, 7, true), record("Other", 418, 8, true)]; + let error = capture_process_instances_with( + &mut records, + Instant::now() + std::time::Duration::from_secs(1), + |pid| { + if pid == 418 { + Err(AdapterError::permission_denied()) + } else { + Ok(Some(format!("instance-{pid}"))) + } + }, + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::PermDenied); +} + +fn records_fixture(visible: bool) -> Vec<WindowRecord> { + vec![record("Fixture", 1, 7, visible)] +} + +fn retain_scope(records: &mut Vec<WindowRecord>, scope: WindowRecordScope<'_>) { + records.retain(|record| scope.matches(&record.app_name, record.pid, record.window_number)); +} + +fn record(app_name: &str, pid: i32, window_number: i64, visible: bool) -> WindowRecord { + WindowRecord { + app_name: app_name.into(), + pid, + title: Some("Window".into()), + window_number, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + visible, + process_instance: Some(format!("instance-{pid}")), + } +} diff --git a/crates/macos/src/system/cocoa_runtime.rs b/crates/macos/src/system/cocoa_runtime.rs new file mode 100644 index 0000000..4791e2b --- /dev/null +++ b/crates/macos/src/system/cocoa_runtime.rs @@ -0,0 +1,28 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::sync::OnceLock; + +pub fn ensure_cocoa_multithreaded() -> Result<(), AdapterError> { + static INITIALIZED: OnceLock<Result<(), String>> = OnceLock::new(); + INITIALIZED + .get_or_init(crate::system::appkit_bridge::ensure_cocoa_multithreaded) + .clone() + .map_err(|message| { + AdapterError::new(ErrorCode::AppUnresponsive, message).with_details(serde_json::json!({ + "kind": "cocoa_runtime_initialization", + "retryable": false, + })) + }) +} + +#[cfg(test)] +mod tests { + #[test] + fn initializes_once_from_a_worker_thread() { + let first = std::thread::spawn(super::ensure_cocoa_multithreaded) + .join() + .expect("worker thread did not panic"); + + first.expect("Cocoa initialization failed"); + super::ensure_cocoa_multithreaded().expect("repeated initialization failed"); + } +} diff --git a/crates/macos/src/system/display.rs b/crates/macos/src/system/display.rs new file mode 100644 index 0000000..602076e --- /dev/null +++ b/crates/macos/src/system/display.rs @@ -0,0 +1,234 @@ +use agent_desktop_core::{AdapterError, Deadline, DisplayInfo, ErrorCode, Rect}; + +#[cfg(target_os = "macos")] +mod imp { + use super::*; + use core_graphics::display::CGDisplay; + + pub fn list_displays_impl(deadline: Deadline) -> Result<Vec<DisplayInfo>, AdapterError> { + let mut displays = raw_display_inventory(deadline)?; + order_public_displays(&mut displays); + Ok(displays + .into_iter() + .map(|(_, _, display)| display) + .collect()) + } + + pub fn display_at(index: usize, deadline: Deadline) -> Result<DisplayInfo, AdapterError> { + let displays = list_displays_impl(deadline)?; + displays + .into_iter() + .nth(index) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "display index out of range")) + } + + pub fn capture_selection( + expected: &DisplayInfo, + deadline: Deadline, + ) -> Result<(usize, DisplayInfo), AdapterError> { + capture_selection_in(raw_display_inventory(deadline)?, &expected.id) + .ok_or_else(|| missing_display_error(&expected.id)) + } + + pub fn display_at_capture_index( + raw_index: usize, + deadline: Deadline, + ) -> Result<DisplayInfo, AdapterError> { + display_at_capture_index_in(raw_display_inventory(deadline)?, raw_index) + .ok_or_else(|| missing_display_error(&raw_index.to_string())) + } + + pub fn scale_for_bounds(bounds: Option<Rect>, deadline: Deadline) -> Result<f64, AdapterError> { + Ok(display_for_bounds(bounds, deadline)?.scale) + } + + pub fn display_for_bounds( + bounds: Option<Rect>, + deadline: Deadline, + ) -> Result<DisplayInfo, AdapterError> { + let displays = list_displays_impl(deadline)?; + select_display(&displays, bounds).cloned() + } + + #[cfg(test)] + pub(super) fn scale_for_bounds_in( + displays: &[DisplayInfo], + bounds: Option<Rect>, + ) -> Result<f64, AdapterError> { + Ok(select_display(displays, bounds)?.scale) + } + + fn select_display( + displays: &[DisplayInfo], + bounds: Option<Rect>, + ) -> Result<&DisplayInfo, AdapterError> { + let selected = bounds.and_then(|bounds| { + displays + .iter() + .max_by(|left, right| { + intersection_area(bounds, left.bounds) + .total_cmp(&intersection_area(bounds, right.bounds)) + }) + .filter(|display| intersection_area(bounds, display.bounds) > 0.0) + }); + selected + .or_else(|| displays.iter().find(|display| display.is_primary)) + .or_else(|| displays.first()) + .ok_or_else(|| display_inventory_error("CoreGraphics returned no active displays")) + } + + pub(super) fn intersection_area(left: Rect, right: Rect) -> f64 { + let width = (left.x + left.width).min(right.x + right.width) - left.x.max(right.x); + let height = (left.y + left.height).min(right.y + right.height) - left.y.max(right.y); + width.max(0.0) * height.max(0.0) + } + + fn display_info(display: &CGDisplay, main_id: u32) -> Result<DisplayInfo, AdapterError> { + let bounds = display.bounds(); + let mode = display.display_mode().ok_or_else(|| { + display_inventory_error("CoreGraphics returned no current display mode") + })?; + let scale = scale_from_mode(mode.width() as f64, mode.pixel_width() as f64)?; + Ok(DisplayInfo { + id: display.id.to_string(), + bounds: Rect { + x: bounds.origin.x, + y: bounds.origin.y, + width: bounds.size.width, + height: bounds.size.height, + }, + is_primary: display.id == main_id, + scale, + }) + } + + fn raw_display_inventory( + deadline: Deadline, + ) -> Result<Vec<(u32, usize, DisplayInfo)>, AdapterError> { + ensure_budget(deadline)?; + let display_ids = CGDisplay::active_displays() + .map_err(|_| AdapterError::internal("Failed to enumerate active displays"))?; + ensure_budget(deadline)?; + let main_id = CGDisplay::main().id; + display_ids + .into_iter() + .enumerate() + .map(|(raw_index, display_id)| { + ensure_budget(deadline)?; + Ok(( + display_id, + raw_index, + display_info(&CGDisplay::new(display_id), main_id)?, + )) + }) + .collect() + } + + pub(super) fn order_public_displays(displays: &mut [(u32, usize, DisplayInfo)]) { + displays.sort_by_key(|(id, _, display)| (!display.is_primary, *id)); + } + + pub(super) fn capture_selection_in( + displays: Vec<(u32, usize, DisplayInfo)>, + expected_id: &str, + ) -> Option<(usize, DisplayInfo)> { + displays + .into_iter() + .find(|(_, _, display)| display.id == expected_id) + .map(|(_, raw_index, display)| (raw_index, display)) + } + + pub(super) fn display_at_capture_index_in( + displays: Vec<(u32, usize, DisplayInfo)>, + raw_index: usize, + ) -> Option<DisplayInfo> { + displays + .into_iter() + .find(|(_, index, _)| *index == raw_index) + .map(|(_, _, display)| display) + } + + pub(super) fn scale_from_mode(point_width: f64, pixel_width: f64) -> Result<f64, AdapterError> { + if !point_width.is_finite() + || !pixel_width.is_finite() + || point_width <= 0.0 + || pixel_width <= 0.0 + { + return Err(display_inventory_error( + "CoreGraphics returned invalid display mode dimensions", + )); + } + Ok(pixel_width / point_width) + } + + fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } + } + + fn display_inventory_error(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::AppUnresponsive, message) + .with_suggestion("Retry after WindowServer finishes updating the display inventory") + } + + fn missing_display_error(id: &str) -> AdapterError { + AdapterError::new( + ErrorCode::InvalidArgs, + format!("Display '{id}' is no longer active"), + ) + .with_suggestion("Run 'list-displays' to refresh display indexes, then retry.") + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use super::*; + + pub fn list_displays_impl(_deadline: Deadline) -> Result<Vec<DisplayInfo>, AdapterError> { + Err(AdapterError::not_supported("list_displays")) + } + + pub fn display_at(_index: usize, _deadline: Deadline) -> Result<DisplayInfo, AdapterError> { + Err(AdapterError::not_supported("list_displays")) + } + + pub fn capture_selection( + _expected: &DisplayInfo, + _deadline: Deadline, + ) -> Result<(usize, DisplayInfo), AdapterError> { + Err(AdapterError::not_supported("list_displays")) + } + + pub fn display_at_capture_index( + _raw_index: usize, + _deadline: Deadline, + ) -> Result<DisplayInfo, AdapterError> { + Err(AdapterError::not_supported("list_displays")) + } + + pub fn scale_for_bounds( + _bounds: Option<Rect>, + _deadline: Deadline, + ) -> Result<f64, AdapterError> { + Ok(1.0) + } + + pub fn display_for_bounds( + _bounds: Option<Rect>, + _deadline: Deadline, + ) -> Result<DisplayInfo, AdapterError> { + Err(AdapterError::not_supported("list_displays")) + } +} + +pub(crate) use imp::{ + capture_selection, display_at, display_at_capture_index, display_for_bounds, + list_displays_impl, scale_for_bounds, +}; + +#[cfg(all(test, target_os = "macos"))] +#[path = "display_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/display_tests.rs b/crates/macos/src/system/display_tests.rs new file mode 100644 index 0000000..abf869c --- /dev/null +++ b/crates/macos/src/system/display_tests.rs @@ -0,0 +1,125 @@ +use super::imp::{ + capture_selection_in, display_at_capture_index_in, intersection_area, order_public_displays, + scale_for_bounds_in, scale_from_mode, +}; +use agent_desktop_core::{DisplayInfo, ErrorCode, Rect}; + +#[test] +fn intersection_area_selects_the_display_containing_most_of_a_window() { + let window = rect(90.0, 0.0, 40.0, 50.0); + let left = rect(0.0, 0.0, 100.0, 100.0); + let right = rect(100.0, 0.0, 100.0, 100.0); + + assert_eq!(intersection_area(window, left), 500.0); + assert_eq!(intersection_area(window, right), 1_500.0); +} + +#[test] +fn window_capture_scale_comes_from_the_display_with_largest_overlap() { + let displays = vec![ + display("main", true, 2.0, rect(0.0, 0.0, 100.0, 100.0)), + display("external", false, 1.0, rect(100.0, 0.0, 100.0, 100.0)), + ]; + let window = rect(90.0, 0.0, 40.0, 50.0); + + assert_eq!( + scale_for_bounds_in(&displays, Some(window)).expect("window display"), + 1.0 + ); + assert_eq!( + scale_for_bounds_in(&displays, None).expect("primary display"), + 2.0 + ); +} + +#[test] +fn missing_display_inventory_is_not_silently_scaled() { + let error = scale_for_bounds_in(&[], None).expect_err("missing displays"); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); +} + +#[test] +fn mode_scale_is_orientation_independent() { + assert_eq!(scale_from_mode(1440.0, 2880.0).unwrap(), 2.0); + assert_eq!(scale_from_mode(900.0, 1800.0).unwrap(), 2.0); +} + +#[test] +fn public_order_is_primary_then_numeric_display_id() { + let mut displays = inventory(); + + order_public_displays(&mut displays); + + let ordered = displays + .into_iter() + .map(|(id, raw_index, _)| (id, raw_index)) + .collect::<Vec<_>>(); + assert_eq!(ordered, vec![(100, 0), (20, 2), (300, 1)]); +} + +#[test] +fn stable_display_id_maps_to_its_raw_capture_index() { + let (raw_index, selected) = capture_selection_in(inventory(), "20").expect("display selection"); + + assert_eq!(raw_index, 2); + assert_eq!(selected.id, "20"); +} + +#[test] +fn missing_stable_display_id_has_no_capture_selection() { + assert!(capture_selection_in(inventory(), "404").is_none()); +} + +#[test] +fn raw_slot_reorder_exposes_identity_change() { + let reordered = vec![ + inventory_display(100, 0, true), + inventory_display(20, 1, false), + inventory_display(300, 2, false), + ]; + + let after = display_at_capture_index_in(reordered, 2).expect("raw slot"); + + assert_eq!(after.id, "300"); + assert_ne!(after.id, "20"); +} + +fn inventory() -> Vec<(u32, usize, DisplayInfo)> { + vec![ + inventory_display(100, 0, true), + inventory_display(300, 1, false), + inventory_display(20, 2, false), + ] +} + +fn inventory_display(id: u32, raw_index: usize, is_primary: bool) -> (u32, usize, DisplayInfo) { + ( + id, + raw_index, + display( + &id.to_string(), + is_primary, + 1.0, + rect(0.0, 0.0, 100.0, 100.0), + ), + ) +} + +fn display(id: &str, is_primary: bool, scale: f64, bounds: Rect) -> DisplayInfo { + DisplayInfo { + id: id.into(), + bounds, + is_primary, + scale, + } +} + +fn rect(x: f64, y: f64, width: f64, height: f64) -> Rect { + Rect { + x, + y, + width, + height, + } +} diff --git a/crates/macos/src/system/display_work_area.rs b/crates/macos/src/system/display_work_area.rs new file mode 100644 index 0000000..6944034 --- /dev/null +++ b/crates/macos/src/system/display_work_area.rs @@ -0,0 +1,87 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, Rect}; + +#[repr(C)] +struct ScreenRect { + x: f64, + y: f64, + width: f64, + height: f64, +} + +pub(crate) fn for_window(bounds: Option<Rect>, deadline: Deadline) -> Result<Rect, AdapterError> { + let display = super::display::display_for_bounds(bounds, deadline)?; + let display_id = display.id.parse::<u32>().map_err(|_| { + AdapterError::new( + ErrorCode::AppUnresponsive, + "CoreGraphics returned an invalid display identifier", + ) + })?; + ensure_budget(deadline)?; + crate::system::cocoa_runtime::ensure_cocoa_multithreaded()?; + let mut visible = ScreenRect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }; + let found = unsafe { agent_desktop_visible_frame(display_id, &mut visible) }; + ensure_budget(deadline)?; + let rect = Rect { + x: visible.x, + y: visible.y, + width: visible.width, + height: visible.height, + }; + if !found || !valid(rect) { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "AppKit returned no valid visible work area for the target display", + ) + .with_details(serde_json::json!({ + "display_id": display.id, + "complete": false, + }))); + } + Ok(rect) +} + +fn valid(rect: Rect) -> bool { + [rect.x, rect.y, rect.width, rect.height] + .into_iter() + .all(f64::is_finite) + && rect.width > 0.0 + && rect.height > 0.0 +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +unsafe extern "C" { + fn agent_desktop_visible_frame(display_id: u32, output: *mut ScreenRect) -> bool; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn visible_work_area_rejects_non_finite_or_empty_geometry() { + assert!(!valid(Rect { + x: 0.0, + y: 0.0, + width: f64::NAN, + height: 100.0, + })); + assert!(!valid(Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 100.0, + })); + } +} diff --git a/crates/macos/src/system/focus.rs b/crates/macos/src/system/focus.rs new file mode 100644 index 0000000..53a44eb --- /dev/null +++ b/crates/macos/src/system/focus.rs @@ -0,0 +1,226 @@ +use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics, ErrorCode, WindowInfo}; + +#[cfg(target_os = "macos")] +use std::time::{Duration, Instant}; + +#[cfg(target_os = "macos")] +const VERIFY_LIMIT: Duration = Duration::from_millis(500); + +#[cfg(target_os = "macos")] +pub(crate) fn ensure_app_focused(pid: i32, deadline: Deadline) -> Result<(), AdapterError> { + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; + + tracing::debug!("system: ensure app focused pid={pid}"); + let app = crate::tree::element_for_pid(pid); + if read_frontmost(&app, deadline)? { + return Ok(()); + } + prepare(&app, deadline)?; + let attribute = CFString::new("AXFrontmost"); + let error = crate::tree::ax_ipc::set_attribute_value( + &app, + attribute.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + deadline, + )?; + finish_mutation(&app, error, "focus application", deadline)?; + wait_until( + &app, + "AXFrontmost", + deadline, + "application did not become frontmost", + ) + .map_err(after_delivery) +} + +#[cfg(target_os = "macos")] +pub(crate) fn verify_app_focused(pid: i32, deadline: Deadline) -> Result<(), AdapterError> { + let app = crate::tree::element_for_pid(pid); + if read_frontmost(&app, deadline)? { + Ok(()) + } else { + Err(AdapterError::new( + ErrorCode::ActionFailed, + "Target application lost focus before physical input delivery", + ) + .with_details(serde_json::json!({ + "pid": pid, + "physical_delivery_started": false, + })) + .with_suggestion("Retry after ensuring the target application remains frontmost")) + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn focus_window_impl( + window: &WindowInfo, + deadline: Deadline, +) -> Result<(), AdapterError> { + tracing::debug!( + "system: focus window app={:?} title={:?}", + window.app, + window.title + ); + let pid = crate::system::process_identity::to_pid_t(window.pid)?; + ensure_app_focused(pid, deadline)?; + let element = crate::system::window_resolve::window_element_for_info(window, deadline)?; + crate::system::window_ops::raise_window(&element, deadline)?; + verify_app_focused(pid, deadline).map_err(after_delivery) +} + +#[cfg(target_os = "macos")] +pub(crate) fn wait_until_main( + window: &crate::tree::AXElement, + deadline: Deadline, +) -> Result<(), AdapterError> { + wait_until(window, "AXMain", deadline, "window did not become main") +} + +#[cfg(target_os = "macos")] +pub(crate) fn verify_window_main( + window: &crate::tree::AXElement, + deadline: Deadline, +) -> Result<(), AdapterError> { + if read_boolean(window, "AXMain", deadline)? { + Ok(()) + } else { + Err(AdapterError::new( + ErrorCode::ActionFailed, + "Target window lost main-window status before physical input delivery", + ) + .with_details(serde_json::json!({ "physical_delivery_started": false })) + .with_suggestion("Retry after bringing the target window to the front")) + } +} + +#[cfg(target_os = "macos")] +fn read_frontmost(app: &crate::tree::AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + read_boolean(app, "AXFrontmost", deadline) +} + +#[cfg(target_os = "macos")] +fn read_boolean( + element: &crate::tree::AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<bool, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_bool_attr_result(element, attribute, deadline); + ensure_remaining(deadline)?; + match result { + Ok(Some(value)) => Ok(value), + Ok(None) => Ok(false), + Err(error) => Err(map_ax_read_error(error, "verify focus")), + } +} + +#[cfg(target_os = "macos")] +fn wait_until( + element: &crate::tree::AXElement, + attribute: &str, + deadline: Deadline, + timeout_message: &str, +) -> Result<(), AdapterError> { + let local_deadline = Instant::now() + VERIFY_LIMIT; + loop { + if read_boolean(element, attribute, deadline)? { + return Ok(()); + } + ensure_remaining(deadline)?; + if Instant::now() >= local_deadline { + return Err( + AdapterError::timeout(timeout_message).with_details(serde_json::json!({ + "kind": "focus_verification", + "physical_delivery_started": false, + })), + ); + } + let pause = deadline.remaining_slice(Duration::from_millis(5))?; + std::thread::sleep(pause.min(Duration::from_millis(5))); + } +} + +#[cfg(target_os = "macos")] +fn prepare(element: &crate::tree::AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + +#[cfg(target_os = "macos")] +fn ensure_remaining(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn finish_mutation( + element: &crate::tree::AXElement, + error: i32, + operation: &str, + deadline: Deadline, +) -> Result<(), AdapterError> { + let delivered = crate::actions::ax_mutation::classify_result( + element, + operation, + "AXUIElement mutation", + error, + )?; + if !delivered { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!("Accessibility does not support {operation}"), + ) + .with_disposition(DeliverySemantics::not_delivered())); + } + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_details(serde_json::json!({ + "kind": "mutation_completed_after_deadline", + "operation": operation, + })) + .with_disposition(DeliverySemantics::delivered_unverified())); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn map_ax_read_error(error: i32, operation: &str) -> AdapterError { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + + let code = if error == kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }; + AdapterError::new(code, format!("Could not {operation} through accessibility")) + .with_details(serde_json::json!({ "ax_error": error })) + .with_suggestion("Refresh the target and retry") + .with_disposition(DeliverySemantics::not_delivered()) +} + +#[cfg(target_os = "macos")] +fn after_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn ensure_app_focused(_pid: i32, _deadline: Deadline) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("focus_app")) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn focus_window_impl( + _window: &WindowInfo, + _deadline: Deadline, +) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("focus_window")) +} diff --git a/crates/macos/src/system/force_close.rs b/crates/macos/src/system/force_close.rs deleted file mode 100644 index d34f982..0000000 --- a/crates/macos/src/system/force_close.rs +++ /dev/null @@ -1,247 +0,0 @@ -use agent_desktop_core::error::AdapterError; -use std::time::{Duration, Instant}; - -const KILL_CONFIRM_FLOOR: Duration = Duration::from_millis(500); - -pub(crate) fn terminate_app(id: &str, pids: &[i32], timeout: Duration) -> Result<(), AdapterError> { - let start = Instant::now(); - let mut failures = signal_failures(pids, Signal::Term); - let remaining = remaining_pids_after_wait(pids, timeout.saturating_sub(start.elapsed())); - if remaining.is_empty() { - return Ok(()); - } - - failures.extend(signal_failures(&remaining, Signal::Kill)); - let still_running = remaining_pids_after_wait(&remaining, kill_confirm_budget(timeout, start)); - if still_running.is_empty() { - return Ok(()); - } - - let mut err = AdapterError::timeout(format!( - "App '{id}' still has running pid(s) after force close: {}", - format_pids(&still_running) - )) - .with_suggestion("Retry after checking for save dialogs or helper processes with 'list-apps'."); - if !failures.is_empty() { - err = err.with_platform_detail(failures.join("; ")); - } - Err(err) -} - -fn kill_confirm_budget(timeout: Duration, start: Instant) -> Duration { - if timeout.is_zero() { - return Duration::ZERO; - } - timeout - .saturating_sub(start.elapsed()) - .max(KILL_CONFIRM_FLOOR) -} - -fn signal_failures(pids: &[i32], signal: Signal) -> Vec<String> { - collect_signal_failures(pids, signal, signal_result) -} - -fn collect_signal_failures<F>(pids: &[i32], signal: Signal, mut signal_fn: F) -> Vec<String> -where - F: FnMut(i32, Signal) -> Result<bool, String>, -{ - let mut failures = Vec::new(); - for &pid in pids { - if let Err(detail) = signal_fn(pid, signal).map(|_| ()) { - failures.push(format!("pid {pid} {}: {detail}", signal.verb())); - } - } - failures -} - -fn remaining_pids_after_wait(pids: &[i32], timeout: Duration) -> Vec<i32> { - let start = Instant::now(); - while start.elapsed() < timeout { - let remaining = running_pids(pids); - if remaining.is_empty() { - return remaining; - } - std::thread::sleep(Duration::from_millis(50)); - } - running_pids(pids) -} - -fn running_pids(pids: &[i32]) -> Vec<i32> { - pids.iter() - .copied() - .filter(|pid| process_is_running(*pid)) - .collect() -} - -fn process_is_running(pid: i32) -> bool { - if let Some(running) = child_process_is_running(pid) { - return running; - } - signal_result(pid, Signal::None).unwrap_or(true) -} - -fn child_process_is_running(pid: i32) -> Option<bool> { - const POSIX_ECHILD: i32 = 10; - const WNOHANG: i32 = 1; - - unsafe extern "C" { - fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; - } - let mut status = 0; - match unsafe { waitpid(pid, &mut status, WNOHANG) } { - child if child == pid => Some(false), - 0 => Some(true), - _ if std::io::Error::last_os_error().raw_os_error() == Some(POSIX_ECHILD) => None, - _ => None, - } -} - -fn signal_result(pid: i32, signal: Signal) -> Result<bool, String> { - const POSIX_ESRCH: i32 = 3; - - unsafe extern "C" { - fn kill(pid: i32, sig: i32) -> i32; - } - if unsafe { kill(pid, signal.number()) } == 0 { - return Ok(true); - } - let err = std::io::Error::last_os_error(); - if err.raw_os_error() == Some(POSIX_ESRCH) { - return Ok(false); - } - Err(err.to_string()) -} - -fn format_pids(pids: &[i32]) -> String { - pids.iter() - .map(i32::to_string) - .collect::<Vec<_>>() - .join(", ") -} - -#[derive(Clone, Copy)] -enum Signal { - None, - Term, - Kill, -} - -impl Signal { - fn number(self) -> i32 { - match self { - Self::None => 0, - Self::Term => 15, - Self::Kill => 9, - } - } - - fn verb(self) -> &'static str { - match self { - Self::None => "inspect", - Self::Term => "terminate", - Self::Kill => "kill", - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::process::{Child, Command}; - - struct ChildGuard(Child); - - impl ChildGuard { - fn spawn_term_ignoring() -> Self { - let child = Command::new("/bin/sh") - .args(["-c", "trap '' TERM; while :; do sleep 1; done"]) - .spawn() - .unwrap(); - Self(child) - } - - fn pid(&self) -> i32 { - self.0.id() as i32 - } - - fn has_exited(&mut self) -> bool { - !process_is_running(self.pid()) - } - } - - impl Drop for ChildGuard { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } - } - - #[test] - fn missing_pids_are_not_reported_as_running() { - assert!(remaining_pids_after_wait(&[999_999], Duration::from_millis(1)).is_empty()); - } - - #[test] - fn current_pid_remains_running_after_short_wait() { - let pid = std::process::id() as i32; - - assert_eq!( - remaining_pids_after_wait(&[999_999, pid], Duration::from_millis(1)), - vec![pid] - ); - } - - #[test] - fn missing_pids_are_accepted_during_signal_race() { - assert!(signal_failures(&[999_999], Signal::Term).is_empty()); - assert!(signal_failures(&[999_999], Signal::Kill).is_empty()); - } - - #[test] - fn signal_collection_attempts_every_pid_after_failure() { - let mut attempted = Vec::new(); - - let failures = collect_signal_failures(&[11, 22, 33], Signal::Term, |pid, _signal| { - attempted.push(pid); - if pid == 11 { - Err("operation not permitted".to_owned()) - } else { - Ok(true) - } - }); - - assert_eq!(attempted, vec![11, 22, 33]); - assert_eq!(failures.len(), 1); - assert!(failures[0].contains("pid 11 terminate")); - } - - #[test] - fn terminate_app_escalates_to_kill_for_all_remaining_pids() { - let mut first = ChildGuard::spawn_term_ignoring(); - let mut second = ChildGuard::spawn_term_ignoring(); - let pids = [first.pid(), second.pid()]; - - terminate_app("term-ignoring-test", &pids, Duration::from_millis(100)).unwrap(); - - assert!(first.has_exited()); - assert!(second.has_exited()); - } - - #[test] - fn kill_confirm_budget_keeps_a_small_floor_after_term_timeout() { - let start = Instant::now() - Duration::from_secs(1); - - assert_eq!( - kill_confirm_budget(Duration::from_millis(10), start), - KILL_CONFIRM_FLOOR - ); - } - - #[test] - fn kill_confirm_budget_preserves_explicit_zero_timeout() { - assert_eq!( - kill_confirm_budget(Duration::ZERO, Instant::now()), - Duration::ZERO - ); - } -} diff --git a/crates/macos/src/system/key_dispatch.rs b/crates/macos/src/system/key_dispatch.rs index 6854de7..54f0100 100644 --- a/crates/macos/src/system/key_dispatch.rs +++ b/crates/macos/src/system/key_dispatch.rs @@ -1,254 +1,389 @@ -use agent_desktop_core::{action::KeyCombo, action_result::ActionResult, error::AdapterError}; +use agent_desktop_core::{ + ActionResult, AdapterError, Deadline, DeliverySemantics, ErrorCode, KeyCombo, Modifier, + ProcessIdentity, +}; +use std::collections::VecDeque; +use std::time::{Duration, Instant}; -#[cfg(target_os = "macos")] -use agent_desktop_core::{action::Modifier, adapter::WindowFilter}; +use crate::tree::AXElement; -#[cfg(target_os = "macos")] -pub fn press_for_app_impl(app_name: &str, combo: &KeyCombo) -> Result<ActionResult, AdapterError> { - tracing::debug!("system: press_for_app app={app_name:?} key={:?}", combo.key); - let pid = find_pid_by_name(app_name)?; - let app_el = crate::tree::element_for_pid(pid); - if app_el.0.is_null() { - return Err(AdapterError::internal("Failed to create AX app element")); - } +const MAX_MENU_NODES: usize = 2_048; +const MAX_MENU_CHILDREN: usize = 128; +const MAX_MENU_DEPTH: u8 = 8; +const AX_MENU_MODIFIER_SHIFT: u32 = 1 << 0; +const AX_MENU_MODIFIER_OPTION: u32 = 1 << 1; +const AX_MENU_MODIFIER_CONTROL: u32 = 1 << 2; +const AX_MENU_MODIFIER_NO_COMMAND: u32 = 1 << 3; - if let Err(err) = crate::system::app_ops::ensure_app_focused(pid) { - tracing::debug!("press_for_app: focus before key dispatch failed: {err}"); - } +pub(crate) fn press_for_app_impl( + process: ProcessIdentity, + combo: &KeyCombo, + policy: agent_desktop_core::InteractionPolicy, + deadline: Deadline, +) -> Result<ActionResult, AdapterError> { + tracing::debug!( + pid = process.pid.get(), + key = combo.key, + "system: press key for app" + ); + let identity = crate::system::process_identity::require_core(&process)?; + let pid = identity.pid(); + let app = crate::tree::element_for_pid(pid); + prepare(&app, deadline)?; if !combo.modifiers.is_empty() { - if let Some(result) = try_menu_bar_shortcut(&app_el, combo) { - return result; + match try_menu_bar_shortcut(&app, combo, deadline) { + Ok(true) => return post_delivery_process_result(&process), + Ok(false) => {} + Err(error) if is_bounded_menu_incomplete(&error) => { + tracing::debug!(details = ?error.details, "bounded menu lookup fell back to exact PID-targeted key delivery"); + } + Err(error) => return Err(error), } } - - let simple_result = try_simple_key_action(app_el.0, combo); - if let Some(result) = simple_result { - return result; + if try_simple_key_action(&app, combo, deadline)? { + return post_delivery_process_result(&process); } - ax_post_keyboard_event(app_el.0, combo)?; - Ok(ActionResult::new("press_key".to_string())) + if policy.allow_focus_steal { + crate::system::process_identity::require_core(&process)?; + crate::system::focus::verify_app_focused(pid, deadline)?; + } + crate::system::process_identity::require_core(&process)?; + require_focused_element(&app, deadline)?; + crate::system::process_identity::require_core(&process)?; + crate::input::keyboard::synthesize_key(combo, Some(pid), deadline)?; + post_delivery_process_result(&process) +} + +fn post_delivery_process_result(process: &ProcessIdentity) -> Result<ActionResult, AdapterError> { + crate::system::process_identity::require_core(process) + .map_err(|error| error.with_disposition(DeliverySemantics::delivered_unverified()))?; + Ok(ActionResult::delivered_unverified("press_key")) } -#[cfg(target_os = "macos")] fn try_simple_key_action( - app_el: accessibility_sys::AXUIElementRef, + app: &AXElement, combo: &KeyCombo, -) -> Option<Result<ActionResult, AdapterError>> { - use accessibility_sys::{AXUIElementPerformAction, kAXErrorSuccess}; - use core_foundation::{base::TCFType, string::CFString}; - + deadline: Deadline, +) -> Result<bool, AdapterError> { if !combo.modifiers.is_empty() { - return None; + return Ok(false); } - - let focused = get_focused_element(app_el)?; - let action_name = match combo.key.as_str() { + let action = match combo.key.as_str() { "return" | "enter" => "AXConfirm", "escape" | "esc" => "AXCancel", "space" => "AXPress", - _ => return None, + _ => return Ok(false), }; - - let ax_action = CFString::new(action_name); - let err = unsafe { AXUIElementPerformAction(focused.0, ax_action.as_concrete_TypeRef()) }; - if err == kAXErrorSuccess { - Some(Ok(ActionResult::new("press_key".to_string()))) - } else { - None - } + let Some(focused) = focused_element(app, deadline)? else { + return Ok(false); + }; + prepare(&focused, deadline)?; + crate::actions::ax_helpers::try_ax_action_or_err(&focused, action, deadline) } -#[cfg(target_os = "macos")] -fn get_focused_element( - app_el: accessibility_sys::AXUIElementRef, -) -> Option<crate::tree::AXElement> { - use accessibility_sys::{AXUIElementCopyAttributeValue, kAXErrorSuccess}; - use core_foundation::{base::TCFType, string::CFString}; - - let attr = CFString::new("AXFocusedUIElement"); - let mut value: core_foundation_sys::base::CFTypeRef = std::ptr::null_mut(); - let err = - unsafe { AXUIElementCopyAttributeValue(app_el, attr.as_concrete_TypeRef(), &mut value) }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - crate::tree::ax_value::created_ax_element(value) -} - -#[cfg(target_os = "macos")] fn try_menu_bar_shortcut( - app_el: &crate::tree::AXElement, + app: &AXElement, combo: &KeyCombo, -) -> Option<Result<ActionResult, AdapterError>> { - use accessibility_sys::{AXUIElementPerformAction, kAXErrorSuccess}; - use core_foundation::{base::TCFType, string::CFString}; - - let menu_bar = crate::tree::copy_element_attr(app_el, "AXMenuBar")?; - let menu_bar_items = crate::tree::copy_ax_array(&menu_bar, "AXChildren")?; - - let target_char = if combo.key.len() == 1 { - combo.key.to_uppercase() - } else { - return None; + deadline: Deadline, +) -> Result<bool, AdapterError> { + let Some(target_char) = single_uppercase_character(&combo.key) else { + return Ok(false); }; + let Some(menu_bar) = element_attribute(app, "AXMenuBar", deadline)? else { + return Ok(false); + }; + let target_modifiers = combo_to_ax_modifiers(combo); + let mut queue = VecDeque::from([(menu_bar, 0_u8)]); + let mut visited = 0_usize; - let target_mods = combo_to_ax_modifiers(combo); - - for bar_item in &menu_bar_items { - if let Some(menu) = crate::tree::copy_ax_array(bar_item, "AXChildren") { - for menu_group in &menu { - if let Some(items) = crate::tree::copy_ax_array(menu_group, "AXChildren") { - for item in &items { - let cmd_char = crate::tree::copy_string_attr(item, "AXMenuItemCmdChar"); - let cmd_mods = read_menu_item_modifiers(item); - - if let Some(ch) = &cmd_char { - if ch.to_uppercase() == target_char && cmd_mods == target_mods { - let press = CFString::new("AXPress"); - let err = unsafe { - AXUIElementPerformAction(item.0, press.as_concrete_TypeRef()) - }; - if err == kAXErrorSuccess { - return Some(Ok(ActionResult::new("press_key".to_string()))); - } - } - } - } - } + while let Some((element, depth)) = queue.pop_front() { + ensure_budget(deadline)?; + visited += 1; + if visited > MAX_MENU_NODES { + return Err(incomplete_menu_error("node_limit")); + } + if read_string(&element, "AXMenuItemCmdChar", deadline)? + .is_some_and(|value| value.to_uppercase() == target_char) + && read_menu_item_modifiers(&element, deadline)? == Some(target_modifiers) + { + prepare(&element, deadline)?; + return crate::actions::ax_helpers::try_ax_action_or_err(&element, "AXPress", deadline); + } + if depth >= MAX_MENU_DEPTH { + if has_children(&element, deadline)? { + return Err(incomplete_menu_error("depth_limit")); } + continue; } + let children = children(&element, deadline)?; + queue.extend( + children + .into_iter() + .map(|child| (child, depth.saturating_add(1))), + ); } - None + Ok(false) } -#[cfg(target_os = "macos")] -fn read_menu_item_modifiers(el: &crate::tree::AXElement) -> u32 { - use accessibility_sys::{AXUIElementCopyAttributeValue, kAXErrorSuccess}; - use core_foundation::{base::TCFType, string::CFString}; - - let attr = CFString::new("AXMenuItemCmdModifiers"); - let mut value: core_foundation_sys::base::CFTypeRef = std::ptr::null_mut(); - let err = - unsafe { AXUIElementCopyAttributeValue(el.0, attr.as_concrete_TypeRef(), &mut value) }; - if err != kAXErrorSuccess || value.is_null() { - return 0; - } - let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; - crate::cf_type::borrowed_cf_number(cf.as_concrete_TypeRef()) - .and_then(|number| number.to_i64()) - .map(|v| v as u32) - .unwrap_or(0) -} - -#[cfg(target_os = "macos")] -fn combo_to_ax_modifiers(combo: &KeyCombo) -> u32 { - let mut mods: u32 = 0; - for m in &combo.modifiers { - match m { - Modifier::Shift => mods |= 1 << 0, - Modifier::Alt => mods |= 1 << 1, - Modifier::Ctrl => mods |= 1 << 2, - Modifier::Cmd => {} +fn require_focused_element(app: &AXElement, deadline: Deadline) -> Result<AXElement, AdapterError> { + let local_deadline = local_deadline(deadline, Duration::from_millis(500))?; + loop { + if let Some(focused) = focused_element(app, deadline)? { + return Ok(focused); } + ensure_budget(deadline)?; + if Instant::now() >= local_deadline { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "Application has no verified focused element for keyboard delivery", + ) + .with_details(serde_json::json!({ "physical_delivery_started": false }))); + } + let pause = deadline.remaining_slice(Duration::from_millis(5))?; + std::thread::sleep(pause.min(Duration::from_millis(5))); } - mods } -#[cfg(target_os = "macos")] -fn ax_post_keyboard_event( - app_el: accessibility_sys::AXUIElementRef, - combo: &KeyCombo, +fn focused_element(app: &AXElement, deadline: Deadline) -> Result<Option<AXElement>, AdapterError> { + element_attribute(app, "AXFocusedUIElement", deadline) +} + +fn element_attribute( + element: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Option<AXElement>, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_element_attr_result(element, attribute, deadline); + ensure_budget(deadline)?; + result.map_err(|error| read_error(attribute, error)) +} + +fn children(element: &AXElement, deadline: Deadline) -> Result<Vec<AXElement>, AdapterError> { + let read = crate::tree::query::child_read::read_attribute_children( + element, + "AXChildren", + MAX_MENU_CHILDREN, + instant(deadline)?, + ); + ensure_budget(deadline)?; + validate_menu_children(read) +} + +fn has_children(element: &AXElement, deadline: Deadline) -> Result<bool, AdapterError> { + let read = crate::tree::query::child_read::read_attribute_children( + element, + "AXChildren", + 0, + instant(deadline)?, + ); + ensure_budget(deadline)?; + validate_child_status(&read)?; + Ok(read.total_count > 0) +} + +fn validate_menu_children( + read: crate::tree::query::child_read::ChildRead, +) -> Result<Vec<AXElement>, AdapterError> { + validate_child_status(&read)?; + if !read.complete || read.truncated() { + return Err( + incomplete_menu_error("child_limit").with_details(serde_json::json!({ + "kind": "child_limit", + "complete": false, + "total_count": read.total_count, + "loaded_count": read.elements.len(), + })), + ); + } + Ok(read.elements) +} + +fn validate_child_status( + read: &crate::tree::query::child_read::ChildRead, ) -> Result<(), AdapterError> { - use accessibility_sys::AXUIElementPostKeyboardEvent; - - let key_code = key_to_keycode(&combo.key).ok_or_else(|| { - AdapterError::new( - agent_desktop_core::error::ErrorCode::ActionNotSupported, - format!( - "No AX equivalent for key combo '{}'. This combo has no menu-bar action.", - format_combo(combo) - ), - ) - .with_suggestion("This key combo cannot be executed via accessibility APIs alone.") - })?; - - let err = unsafe { AXUIElementPostKeyboardEvent(app_el, 0 as _, key_code, true) }; - if err != accessibility_sys::kAXErrorSuccess { - return Err(AdapterError::internal(format!( - "AXUIElementPostKeyboardEvent key-down failed (err={err})" - ))); + if read.status.api_disabled { + return Err(read_error( + "AXChildren", + accessibility_sys::kAXErrorAPIDisabled, + )); } - - let err = unsafe { AXUIElementPostKeyboardEvent(app_el, 0 as _, key_code, false) }; - if err != accessibility_sys::kAXErrorSuccess { - return Err(AdapterError::internal(format!( - "AXUIElementPostKeyboardEvent key-up failed (err={err})" - ))); + if read.status.invalid_element { + return Err(read_error( + "AXChildren", + accessibility_sys::kAXErrorInvalidUIElement, + )); + } + if read.status.health.cannot_complete > 0 || read.status.health.deadline_exhausted > 0 { + return Err(read_error( + "AXChildren", + accessibility_sys::kAXErrorCannotComplete, + )); } - Ok(()) } -#[cfg(target_os = "macos")] -fn format_combo(combo: &KeyCombo) -> String { - let mods: Vec<&str> = combo - .modifiers - .iter() - .map(|m| match m { - Modifier::Cmd => "cmd", - Modifier::Ctrl => "ctrl", - Modifier::Alt => "alt", - Modifier::Shift => "shift", - }) - .collect(); - if mods.is_empty() { - combo.key.clone() +fn read_string( + element: &AXElement, + attribute: &str, + deadline: Deadline, +) -> Result<Option<String>, AdapterError> { + prepare(element, deadline)?; + let result = crate::tree::attributes::copy_string_attr_result(element, attribute, deadline); + ensure_budget(deadline)?; + result.map_err(|error| read_error(attribute, error)) +} + +fn read_menu_item_modifiers( + element: &AXElement, + deadline: Deadline, +) -> Result<Option<u32>, AdapterError> { + use accessibility_sys::kAXErrorSuccess; + use core_foundation::{base::TCFType, string::CFString}; + + prepare(element, deadline)?; + let attribute = CFString::new("AXMenuItemCmdModifiers"); + let (error, value) = crate::tree::ax_ipc::copy_attribute_value( + element, + attribute.as_concrete_TypeRef(), + deadline, + ); + ensure_budget(deadline)?; + if error != kAXErrorSuccess { + return if error == accessibility_sys::kAXErrorNoValue + || error == accessibility_sys::kAXErrorAttributeUnsupported + { + Ok(None) + } else { + Err(read_error("AXMenuItemCmdModifiers", error)) + }; + } + if value.is_null() { + return Err(malformed_menu_modifiers()); + } + let value = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; + let number = crate::cf_type::borrowed_cf_number(value.as_concrete_TypeRef()) + .and_then(|number| number.to_i64()) + .ok_or_else(malformed_menu_modifiers)?; + normalize_menu_modifiers(number).map(Some) +} + +fn single_uppercase_character(key: &str) -> Option<String> { + (key.chars().count() == 1).then(|| key.to_uppercase()) +} + +fn combo_to_ax_modifiers(combo: &KeyCombo) -> u32 { + let mut encoded = if combo.modifiers.contains(&Modifier::Meta) { + 0 } else { - format!("{}+{}", mods.join("+"), combo.key) - } -} - -#[cfg(target_os = "macos")] -fn key_to_keycode(key: &str) -> Option<u16> { - match key { - "cmd" | "command" | "shift" | "alt" | "option" | "ctrl" | "control" => None, - other => crate::input::keyboard_map::key_name_to_code(other).ok(), - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn find_pid_by_name(app_name: &str) -> Result<i32, AdapterError> { - let filter = WindowFilter { - focused_only: false, - app: Some(app_name.to_string()), + AX_MENU_MODIFIER_NO_COMMAND }; - let windows = crate::system::window_list::list_windows_impl(&filter)?; - windows - .first() - .map(|w| w.pid) - .or_else(|| crate::system::app_list::pid_for_app_name(app_name)) - .ok_or_else(|| { - AdapterError::new( - agent_desktop_core::error::ErrorCode::AppNotFound, - format!("App '{app_name}' not found"), - ) - .with_suggestion( - "Verify the app is running. Use 'list-apps' to see running applications.", - ) - }) + encoded |= combo.modifiers.iter().fold(0, |modifiers, modifier| { + modifiers + | match modifier { + Modifier::Shift => AX_MENU_MODIFIER_SHIFT, + Modifier::Alt => AX_MENU_MODIFIER_OPTION, + Modifier::Ctrl => AX_MENU_MODIFIER_CONTROL, + Modifier::Meta => 0, + } + }); + encoded +} + +fn normalize_menu_modifiers(number: i64) -> Result<u32, AdapterError> { + let value = u32::try_from(number).map_err(|_| malformed_menu_modifiers())?; + let supported = AX_MENU_MODIFIER_SHIFT + | AX_MENU_MODIFIER_OPTION + | AX_MENU_MODIFIER_CONTROL + | AX_MENU_MODIFIER_NO_COMMAND; + if value & !supported != 0 { + return Err(malformed_menu_modifiers()); + } + Ok(value) +} + +fn malformed_menu_modifiers() -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "AXMenuItemCmdModifiers had an invalid value", + ) + .with_details(serde_json::json!({ + "kind": "menu_modifier_value_invalid", + "complete": false, + })) +} + +fn instant(deadline: Deadline) -> Result<Instant, AdapterError> { + Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Deadline is out of range")) +} + +fn is_bounded_menu_incomplete(error: &AdapterError) -> bool { + error.code == ErrorCode::AppUnresponsive + && error + .details + .as_ref() + .and_then(|details| details.get("kind")) + .and_then(serde_json::Value::as_str) + .is_some_and(|kind| matches!(kind, "node_limit" | "child_limit" | "depth_limit")) +} + +fn prepare(element: &AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(element, deadline) +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn local_deadline(deadline: Deadline, maximum: Duration) -> Result<Instant, AdapterError> { + let remaining = deadline.remaining_slice(maximum)?; + Instant::now() + .checked_add(remaining) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Deadline is out of range")) +} + +fn read_error(attribute: &str, error: i32) -> AdapterError { + AdapterError::new( + if error == accessibility_sys::kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == accessibility_sys::kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == accessibility_sys::kAXErrorInvalidUIElement { + ErrorCode::StaleRef + } else { + ErrorCode::ActionFailed + }, + format!("Accessibility read failed for {attribute} during key dispatch"), + ) + .with_details(serde_json::json!({ "attribute": attribute, "ax_error": error })) +} + +fn incomplete_menu_error(kind: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Menu shortcut lookup exceeded its bounded accessibility search", + ) + .with_details(serde_json::json!({ "kind": kind, "complete": false })) } #[cfg(not(target_os = "macos"))] -pub fn press_for_app_impl( - _app_name: &str, +pub(crate) fn press_for_app_impl( + _process: ProcessIdentity, _combo: &KeyCombo, + _policy: agent_desktop_core::InteractionPolicy, + _deadline: Deadline, ) -> Result<ActionResult, AdapterError> { Err(AdapterError::not_supported("press_for_app")) } -#[cfg(not(target_os = "macos"))] -pub(crate) fn find_pid_by_name(_app_name: &str) -> Result<i32, AdapterError> { - Err(AdapterError::not_supported("find_pid_by_name")) -} +#[cfg(test)] +#[path = "key_dispatch_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/key_dispatch_tests.rs b/crates/macos/src/system/key_dispatch_tests.rs new file mode 100644 index 0000000..22003a7 --- /dev/null +++ b/crates/macos/src/system/key_dispatch_tests.rs @@ -0,0 +1,52 @@ +use super::*; + +#[test] +fn key_dispatch_rejects_non_unique_display_names() { + let error = match [10, 11].as_slice() { + [pid] => Ok(*pid), + pids => Err(AdapterError::ambiguous_target("duplicate app name") + .with_details(serde_json::json!({ "candidate_pids": pids }))), + } + .expect_err("duplicate names must be ambiguous"); + + assert_eq!(error.code, ErrorCode::AmbiguousTarget); +} + +#[test] +fn menu_shortcut_requires_one_character() { + assert_eq!(single_uppercase_character("a").as_deref(), Some("A")); + assert!(single_uppercase_character("enter").is_none()); +} + +fn combo(modifiers: Vec<Modifier>) -> KeyCombo { + KeyCombo { + key: "a".into(), + modifiers, + } +} + +#[test] +fn menu_modifier_encoding_distinguishes_command_from_no_command() { + assert_eq!( + combo_to_ax_modifiers(&combo(vec![Modifier::Meta, Modifier::Alt])), + AX_MENU_MODIFIER_OPTION + ); + assert_eq!( + combo_to_ax_modifiers(&combo(vec![Modifier::Alt])), + AX_MENU_MODIFIER_OPTION | AX_MENU_MODIFIER_NO_COMMAND + ); + assert_ne!( + combo_to_ax_modifiers(&combo(vec![Modifier::Meta, Modifier::Alt])), + combo_to_ax_modifiers(&combo(vec![Modifier::Alt])) + ); +} + +#[test] +fn malformed_menu_modifier_values_are_not_command_shortcuts() { + assert!(normalize_menu_modifiers(-1).is_err()); + assert!(normalize_menu_modifiers(1 << 8).is_err()); + assert_eq!( + normalize_menu_modifiers(AX_MENU_MODIFIER_NO_COMMAND as i64).unwrap(), + AX_MENU_MODIFIER_NO_COMMAND + ); +} diff --git a/crates/macos/src/system/launch.rs b/crates/macos/src/system/launch.rs new file mode 100644 index 0000000..4a42568 --- /dev/null +++ b/crates/macos/src/system/launch.rs @@ -0,0 +1,314 @@ +use agent_desktop_core::{ + AdapterError, AppInfo, Deadline, DeliverySemantics, ErrorCode, WindowInfo, + launch_options::LaunchOptions, +}; +use std::time::{Duration, Instant}; + +const MAX_ARGUMENT_COUNT: usize = 256; +const MAX_ENVIRONMENT_COUNT: usize = 256; +const MAX_LAUNCH_TEXT_BYTES: usize = 1024 * 1024; + +enum LaunchTarget { + Existing { pid: i32, process_instance: String }, + New { baseline_instances: Vec<String> }, +} + +#[cfg(target_os = "macos")] +pub(crate) fn launch_app_impl( + id: &str, + options: &LaunchOptions, + parent_deadline: Deadline, +) -> Result<WindowInfo, AdapterError> { + validate_app_identifier(id).map_err(before_launch)?; + validate_launch_options(options).map_err(before_launch)?; + let deadline = if options.timeout_ms == 0 { + parent_deadline + } else { + parent_deadline.capped(Duration::from_millis(options.timeout_ms)) + }; + ensure_launch_budget(deadline, id).map_err(before_launch)?; + let timeout_ms = options.timeout_ms; + let initial = matching_apps(id, deadline).map_err(before_launch)?; + if options.attach_if_running && initial.len() == 1 { + let app = &initial[0]; + let instance = required_instance(app).map_err(before_launch)?; + let pid = crate::system::process_identity::to_pid_t(app.pid).map_err(before_launch)?; + if let Some(window) = exact_window(pid, &instance, deadline).map_err(before_launch)? { + return Ok(window); + } + } + let target = launch_target(options, initial).map_err(before_launch)?; + + let launched = crate::system::launch_workspace::open(id, options, deadline)?; + validate_launched_target(&target, &launched).map_err(after_launch)?; + let mut poll_interval = Duration::from_millis(50); + loop { + if let Some(window) = + exact_window(launched.0, &launched.1, deadline).map_err(after_launch)? + { + return Ok(window); + } + if !should_poll_after_first_observation(options.timeout_ms) { + return Err(launch_no_window_error(id, timeout_ms, &launched)); + } + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(launch_no_window_error(id, timeout_ms, &launched)); + } + std::thread::sleep(poll_interval.min(remaining)); + poll_interval = (poll_interval * 3 / 2).min(Duration::from_millis(250)); + } +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn launch_app_impl( + _id: &str, + _options: &LaunchOptions, + _deadline: Deadline, +) -> Result<WindowInfo, AdapterError> { + Err(AdapterError::not_supported("launch_app")) +} + +#[cfg(target_os = "macos")] +fn launch_target( + options: &LaunchOptions, + mut initial: Vec<AppInfo>, +) -> Result<LaunchTarget, AdapterError> { + if options.attach_if_running { + if initial.len() > 1 { + return Err(ambiguous_apps(&initial)); + } + if let Some(app) = initial.pop() { + let process_instance = required_instance(&app)?; + return Ok(LaunchTarget::Existing { + pid: crate::system::process_identity::to_pid_t(app.pid)?, + process_instance, + }); + } + } + let baseline_instances = initial + .iter() + .map(required_instance) + .collect::<Result<Vec<_>, _>>()?; + Ok(LaunchTarget::New { baseline_instances }) +} + +#[cfg(target_os = "macos")] +fn validate_launched_target( + target: &LaunchTarget, + launched: &(i32, String), +) -> Result<(), AdapterError> { + match target { + LaunchTarget::Existing { + pid, + process_instance, + } => { + if launched.0 == *pid && launched.1 == *process_instance { + Ok(()) + } else { + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "NSWorkspace returned a different application while attaching", + ) + .with_details(serde_json::json!({ + "expected_pid": pid, + "returned_pid": launched.0, + "complete": false, + }))) + } + } + LaunchTarget::New { baseline_instances } => { + if baseline_instances.contains(&launched.1) { + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "NSWorkspace reused an existing application during an exact fresh launch", + ) + .with_details(serde_json::json!({ + "returned_pid": launched.0, + "complete": false, + }))) + } else { + Ok(()) + } + } + } +} + +#[cfg(target_os = "macos")] +fn matching_apps(id: &str, deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError> { + ensure_launch_budget(deadline, id)?; + Ok( + crate::system::workspace_apps::list_apps_until(deadline_instant(deadline)?)? + .into_iter() + .filter(|app| { + app.name.eq_ignore_ascii_case(id) + || app + .bundle_id + .as_deref() + .is_some_and(|bundle_id| bundle_id.eq_ignore_ascii_case(id)) + }) + .collect(), + ) +} + +#[cfg(target_os = "macos")] +fn required_instance(app: &AppInfo) -> Result<String, AdapterError> { + app.process_instance.clone().ok_or_else(|| { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Running application has no exact process instance token", + ) + .with_details(serde_json::json!({ "pid": app.pid, "complete": false })) + }) +} + +#[cfg(target_os = "macos")] +fn ambiguous_apps(apps: &[AppInfo]) -> AdapterError { + AdapterError::ambiguous_target("More than one application instance matches the launch target") + .with_details(serde_json::json!({ + "candidate_pids": apps.iter().map(|app| app.pid).collect::<Vec<_>>(), + })) +} + +#[cfg(target_os = "macos")] +fn exact_window( + pid: i32, + process_instance: &str, + deadline: Deadline, +) -> Result<Option<WindowInfo>, AdapterError> { + let window = match crate::system::window_inventory::exact_window_for_pid_until( + pid, + deadline_instant(deadline)?, + ) { + Ok(window) => window, + Err(error) if error.code == ErrorCode::WindowNotFound => return Ok(None), + Err(error) => return Err(error), + }; + if window.process_instance.as_deref() != Some(process_instance) { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Application process instance changed while waiting for its window", + ) + .with_details(serde_json::json!({ "pid": pid, "complete": false }))); + } + Ok(Some(window)) +} + +#[cfg(target_os = "macos")] +fn ensure_launch_budget(deadline: Deadline, id: &str) -> Result<(), AdapterError> { + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_details(serde_json::json!({ "app_name": id }))); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn before_launch(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::not_delivered()) +} + +#[cfg(target_os = "macos")] +fn should_poll_after_first_observation(timeout_ms: u64) -> bool { + timeout_ms > 0 +} + +#[cfg(target_os = "macos")] +fn validate_app_identifier(id: &str) -> Result<(), AdapterError> { + let safe_bundle_id = !looks_like_bundle_id(id) + || id.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }); + if id.is_empty() + || id.contains("..") + || id.contains('/') + || id.chars().any(char::is_control) + || !safe_bundle_id + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Invalid app identifier: use a bare app name or bundle identifier", + ) + .with_details(serde_json::json!({ "app_name": id })) + .with_suggestion("Use an app name like 'Safari' or bundle ID like 'com.apple.Safari'.")); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn validate_launch_options(options: &LaunchOptions) -> Result<(), AdapterError> { + if options.cwd.is_some() { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "macOS Launch Services does not support an exact launch working directory", + ) + .with_suggestion("Remove --cwd, or start the app through a dedicated launcher script")); + } + if options.args.len() > MAX_ARGUMENT_COUNT || options.env.len() > MAX_ENVIRONMENT_COUNT { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Launch argument or environment entry count exceeds the supported limit", + )); + } + let text_bytes = options + .args + .iter() + .map(String::len) + .chain( + options + .env + .iter() + .map(|(key, value)| key.len() + value.len()), + ) + .try_fold(0_usize, usize::checked_add) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Launch options are too large"))?; + if text_bytes > MAX_LAUNCH_TEXT_BYTES { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Launch argument and environment data exceeds one MiB", + )); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(crate) fn looks_like_bundle_id(id: &str) -> bool { + id.contains('.') && !id.ends_with(".app") && !id.contains(' ') +} + +#[cfg(target_os = "macos")] +fn launch_no_window_error(id: &str, timeout_ms: u64, launched: &(i32, String)) -> AdapterError { + AdapterError::new( + ErrorCode::WindowNotFound, + format!( + "Application started, but no exact accessible window appeared within {timeout_ms} ms" + ), + ) + .with_details(serde_json::json!({ + "app_name": id, + "pid": launched.0, + "process_instance": launched.1, + "retry_safe": false, + })) + .with_disposition(DeliverySemantics::delivered_unverified()) + .with_suggestion( + "Inspect list-apps and list-windows for the returned process; do not repeat the launch blindly.", + ) +} + +#[cfg(target_os = "macos")] +fn after_launch(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) +} + +#[cfg(target_os = "macos")] +fn deadline_instant(deadline: Deadline) -> Result<Instant, AdapterError> { + Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Launch deadline is out of range")) +} + +#[cfg(test)] +#[path = "launch_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/launch_bridge.m b/crates/macos/src/system/launch_bridge.m new file mode 100644 index 0000000..90595b4 --- /dev/null +++ b/crates/macos/src/system/launch_bridge.m @@ -0,0 +1,215 @@ +#import <AppKit/AppKit.h> +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> + +typedef struct { + void *application; + int32_t pid; + double launchTime; + uint8_t terminated; + uint8_t deliveryStarted; + uint8_t errorKind; + const char *error; + size_t errorLength; +} AgentDesktopLaunchResult; + +typedef void (*AgentDesktopLaunchCompletion)(void *, const AgentDesktopLaunchResult *); +typedef void (*AgentDesktopLaunchRelease)(void *); + +typedef struct { + void *context; + AgentDesktopLaunchRelease releaseContext; + bool released; +} AgentDesktopLaunchReleaseBox; + +static void releaseBox(AgentDesktopLaunchReleaseBox *box) { + if (box == NULL) { + return; + } + if (!box->released) { + box->released = true; + box->releaseContext(box->context); + } + free(box); +} + +@interface AgentDesktopLaunchContext : NSObject { +@public + AgentDesktopLaunchReleaseBox *_releaseBox; + AgentDesktopLaunchCompletion _completion; +} +@end + +@implementation AgentDesktopLaunchContext +- (void)dealloc { + @try { + releaseBox(_releaseBox); + _releaseBox = NULL; + } @catch (NSException *exception) { + (void)exception; + } +} +@end + +static void completeError( + AgentDesktopLaunchContext *holder, + NSString *message, + uint8_t kind, + uint8_t deliveryStarted +) { + const char *bytes = message.UTF8String; + AgentDesktopLaunchResult result = { + .application = NULL, + .pid = 0, + .launchTime = 0.0, + .terminated = true, + .deliveryStarted = deliveryStarted, + .errorKind = kind, + .error = bytes, + .errorLength = bytes == NULL ? 0 : strlen(bytes), + }; + holder->_completion(holder->_releaseBox->context, &result); +} + +static BOOL isStringArray(id value) { + if (![value isKindOfClass:[NSArray class]]) { + return NO; + } + for (id item in (NSArray *)value) { + if (![item isKindOfClass:[NSString class]]) { + return NO; + } + } + return YES; +} + +static BOOL isStringDictionary(id value) { + if (![value isKindOfClass:[NSDictionary class]]) { + return NO; + } + for (id key in (NSDictionary *)value) { + if (![key isKindOfClass:[NSString class]] || + ![value[key] isKindOfClass:[NSString class]]) { + return NO; + } + } + return YES; +} + +bool agent_desktop_open_application( + const uint8_t *requestBytes, + size_t requestLength, + void *context, + AgentDesktopLaunchCompletion completion, + AgentDesktopLaunchRelease releaseContext +) { + AgentDesktopLaunchReleaseBox *box = calloc(1, sizeof(*box)); + if (box == NULL) { + releaseContext(context); + return false; + } + box->context = context; + box->releaseContext = releaseContext; + AgentDesktopLaunchContext *holder = nil; + @try { + holder = [AgentDesktopLaunchContext new]; + if (holder == nil) { + releaseBox(box); + return false; + } + holder->_releaseBox = box; + holder->_completion = completion; + box = NULL; + NSData *data = [NSData dataWithBytes:requestBytes length:requestLength]; + NSError *jsonError = nil; + id decoded = [NSJSONSerialization JSONObjectWithData:data options:0 error:&jsonError]; + if (![decoded isKindOfClass:[NSDictionary class]]) { + completeError(holder, jsonError.localizedDescription ?: @"Invalid launch request", 2, 0); + return true; + } + NSDictionary *request = (NSDictionary *)decoded; + NSString *identifier = request[@"identifier"]; + NSArray *arguments = request[@"arguments"]; + NSDictionary *environment = request[@"environment"]; + if (![identifier isKindOfClass:[NSString class]] || + !isStringArray(arguments) || !isStringDictionary(environment)) { + completeError(holder, @"Invalid launch request fields", 2, 0); + return true; + } + NSWorkspace *workspace = [NSWorkspace sharedWorkspace]; + NSURL *url = nil; + if ([request[@"bundle_id"] boolValue]) { + url = [workspace URLForApplicationWithBundleIdentifier:identifier]; + } else { + NSString *path = [workspace fullPathForApplication:identifier]; + if (path != nil) { + url = [NSURL fileURLWithPath:path]; + } + } + if (url == nil) { + completeError(holder, @"Application bundle was not found", 1, 0); + return true; + } + NSWorkspaceOpenConfiguration *configuration = + [NSWorkspaceOpenConfiguration configuration]; + configuration.activates = [request[@"activates"] boolValue]; + configuration.promptsUserIfNeeded = [request[@"prompts"] boolValue]; + configuration.allowsRunningApplicationSubstitution = + [request[@"substitution"] boolValue]; + configuration.createsNewApplicationInstance = [request[@"new_instance"] boolValue]; + configuration.arguments = arguments; + configuration.environment = environment; + [workspace openApplicationAtURL:url + configuration:configuration + completionHandler:^(NSRunningApplication *app, NSError *error) { + @try { + if (error != nil || app == nil) { + completeError( + holder, + error.localizedDescription ?: @"Launch Services returned no application", + 4, + 1 + ); + return; + } + NSDate *launchDate = app.launchDate; + AgentDesktopLaunchResult result = { + .application = (__bridge void *)app, + .pid = app.processIdentifier, + .launchTime = launchDate == nil ? 0.0 : launchDate.timeIntervalSince1970, + .terminated = app.isTerminated, + .deliveryStarted = 1, + .errorKind = 0, + .error = NULL, + .errorLength = 0, + }; + holder->_completion(holder->_releaseBox->context, &result); + } @catch (NSException *exception) { + completeError(holder, exception.reason ?: @"Launch callback exception", 3, 1); + } + }]; + } @catch (NSException *exception) { + if (holder != nil && holder->_releaseBox != NULL) { + @try { + completeError(holder, exception.reason ?: @"Launch bridge exception", 3, 0); + } @catch (NSException *completionException) { + (void)completionException; + } + return true; + } + releaseBox(box); + return false; + } + return true; +} + +uint8_t agent_desktop_running_application_is_live(void *application, int32_t expectedPID) { + @try { + NSRunningApplication *app = (__bridge NSRunningApplication *)application; + return app != nil && !app.isTerminated && app.processIdentifier == expectedPID; + } @catch (NSException *exception) { + (void)exception; + return 0; + } +} diff --git a/crates/macos/src/system/launch_callback_result.rs b/crates/macos/src/system/launch_callback_result.rs new file mode 100644 index 0000000..7fb2439 --- /dev/null +++ b/crates/macos/src/system/launch_callback_result.rs @@ -0,0 +1,13 @@ +use std::ffi::{c_char, c_void}; + +#[repr(C)] +pub(crate) struct LaunchCallbackResult { + pub(crate) application: *mut c_void, + pub(crate) pid: i32, + pub(crate) launch_time: f64, + pub(crate) terminated: u8, + pub(crate) delivery_started: u8, + pub(crate) error_kind: u8, + pub(crate) error: *const c_char, + pub(crate) error_len: usize, +} diff --git a/crates/macos/src/system/launch_completion.rs b/crates/macos/src/system/launch_completion.rs new file mode 100644 index 0000000..5837486 --- /dev/null +++ b/crates/macos/src/system/launch_completion.rs @@ -0,0 +1,272 @@ +use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics, ErrorCode}; +use std::ffi::c_void; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +const MAX_ERROR_BYTES: usize = 64 * 1024; +const MAX_OUTSTANDING_LAUNCHES: usize = 16; +static OUTSTANDING_LAUNCHES: AtomicUsize = AtomicUsize::new(0); + +type LaunchResult = Result<(i32, String), AdapterError>; + +struct LaunchCompletion { + result: Mutex<Option<LaunchResult>>, + changed: Condvar, +} + +impl LaunchCompletion { + fn new() -> Self { + Self { + result: Mutex::new(None), + changed: Condvar::new(), + } + } + + fn wait(&self, deadline: Deadline) -> LaunchResult { + let mut result = self.result.lock().map_err(|_| { + AdapterError::internal("Launch completion lock was poisoned") + .with_disposition(DeliverySemantics::uncertain()) + })?; + loop { + if let Some(completed) = result.take() { + return completed; + } + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(deadline + .timeout_error() + .with_details(serde_json::json!({ + "kind": "ns_workspace_launch_completion", + "callback_may_arrive_late": true, + })) + .with_disposition(DeliverySemantics::uncertain())); + } + let waited = self.changed.wait_timeout(result, remaining).map_err(|_| { + AdapterError::internal("Launch completion wait was poisoned") + .with_disposition(DeliverySemantics::uncertain()) + })?; + result = waited.0; + } + } + + fn complete(&self, result: LaunchResult) { + if let Ok(mut slot) = self.result.lock() + && slot.is_none() + { + *slot = Some(result); + self.changed.notify_all(); + } + } +} + +pub(crate) unsafe fn open_and_wait(request: &[u8], deadline: Deadline) -> LaunchResult { + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_disposition(DeliverySemantics::not_delivered())); + } + reserve_launch()?; + let state = Arc::new(LaunchCompletion::new()); + let context = Arc::into_raw(Arc::clone(&state)) as *mut c_void; + let accepted = unsafe { + agent_desktop_open_application( + request.as_ptr(), + request.len(), + context, + launch_completed, + release_context, + ) + }; + if !accepted { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "NSWorkspace could not allocate a launch completion context", + ) + .with_disposition(DeliverySemantics::not_delivered())); + } + state.wait(deadline) +} + +unsafe extern "C" fn launch_completed( + context: *mut c_void, + result: *const crate::system::launch_callback_result::LaunchCallbackResult, +) { + let _ = std::panic::catch_unwind(|| unsafe { + let state = &*(context as *const LaunchCompletion); + let result = result + .as_ref() + .ok_or_else(|| AdapterError::internal("NSWorkspace returned a null launch result")) + .and_then(validate_result); + state.complete(result); + }); +} + +unsafe extern "C" fn release_context(context: *mut c_void) { + let _ = std::panic::catch_unwind(|| unsafe { + drop(Arc::from_raw(context as *const LaunchCompletion)); + release_launch(); + }); +} + +fn validate_result( + result: &crate::system::launch_callback_result::LaunchCallbackResult, +) -> LaunchResult { + if result.pid <= 0 { + let code = if result.error_kind == 1 { + ErrorCode::AppNotFound + } else { + ErrorCode::ActionFailed + }; + return Err(callback_error( + result, + AdapterError::new( + code, + "NSWorkspace failed to launch the requested application", + ) + .with_platform_detail(error_detail(result.error, result.error_len)) + .with_suggestion("Verify the app name or bundle identifier and retry"), + )); + } + if result.terminated != 0 { + return Err(callback_error( + result, + AdapterError::new( + ErrorCode::AppUnresponsive, + "Launched application terminated before identity verification", + ) + .with_details(serde_json::json!({ "pid": result.pid, "complete": false })), + )); + } + let identity = crate::system::process_identity::ProcessIdentity::capture(result.pid) + .map_err(|error| callback_error(result, error))? + .ok_or_else(|| { + callback_error(result, { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Launched application exited before identity verification", + ) + }) + })?; + if !identity.matches_launch_time(result.launch_time) { + return Err(callback_error( + result, + AdapterError::new( + ErrorCode::AppUnresponsive, + "NSWorkspace and libproc returned different launch identities", + ) + .with_details(serde_json::json!({ + "pid": result.pid, + "launch_time": result.launch_time, + "complete": false, + })), + )); + } + if unsafe { agent_desktop_running_application_is_live(result.application, result.pid) } == 0 { + return Err(callback_error( + result, + AdapterError::new( + ErrorCode::AppUnresponsive, + "Launched application changed after libproc identity verification", + ) + .with_details(serde_json::json!({ "pid": result.pid, "complete": false })), + )); + } + Ok((result.pid, identity.token())) +} + +fn callback_error( + result: &crate::system::launch_callback_result::LaunchCallbackResult, + error: AdapterError, +) -> AdapterError { + if result.delivery_started == 0 { + error.with_disposition(DeliverySemantics::not_delivered()) + } else if result.pid <= 0 { + error.with_disposition(DeliverySemantics::uncertain()) + } else { + error.with_disposition(DeliverySemantics::delivered_unverified()) + } +} + +fn reserve_launch() -> Result<(), AdapterError> { + OUTSTANDING_LAUNCHES + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < MAX_OUTSTANDING_LAUNCHES).then_some(current + 1) + }) + .map(|_| ()) + .map_err(|_| { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Too many macOS launch completions are still outstanding", + ) + .with_details(serde_json::json!({ + "kind": "launch_completion_backpressure", + "limit": MAX_OUTSTANDING_LAUNCHES, + "retryable": true, + })) + .with_suggestion("Wait for earlier launch requests to settle before trying again") + .with_disposition(DeliverySemantics::not_delivered()) + }) +} + +fn release_launch() { + let _ = OUTSTANDING_LAUNCHES.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + current.checked_sub(1) + }); +} + +fn error_detail(error: *const std::ffi::c_char, error_len: usize) -> String { + if error.is_null() || error_len == 0 { + return "No NSWorkspace diagnostic".into(); + } + let retained = error_len.min(MAX_ERROR_BYTES); + let bytes = unsafe { std::slice::from_raw_parts(error.cast::<u8>(), retained) }; + String::from_utf8_lossy(bytes).into_owned() +} + +unsafe extern "C" { + fn agent_desktop_open_application( + request: *const u8, + request_len: usize, + context: *mut c_void, + completion: unsafe extern "C" fn( + *mut c_void, + *const crate::system::launch_callback_result::LaunchCallbackResult, + ), + release: unsafe extern "C" fn(*mut c_void), + ) -> bool; + fn agent_desktop_running_application_is_live(application: *mut c_void, expected_pid: i32) + -> u8; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn late_completion_state_remains_safe_after_timeout() { + let state = LaunchCompletion::new(); + let timeout = state.wait(Deadline::after(1).unwrap()).unwrap_err(); + + assert_eq!(timeout.code, ErrorCode::Timeout); + state.complete(Ok((7, "generation".into()))); + assert_eq!(state.wait(Deadline::after(50).unwrap()).unwrap().0, 7); + } + + #[test] + fn outstanding_launch_quota_applies_backpressure() { + let mut reserved = 0; + while reserve_launch().is_ok() { + reserved += 1; + } + + assert!(reserved <= MAX_OUTSTANDING_LAUNCHES); + let error = reserve_launch().unwrap_err(); + assert_eq!( + error.details.unwrap()["kind"], + "launch_completion_backpressure" + ); + for _ in 0..reserved { + release_launch(); + } + } +} diff --git a/crates/macos/src/system/launch_tests.rs b/crates/macos/src/system/launch_tests.rs new file mode 100644 index 0000000..19189fb --- /dev/null +++ b/crates/macos/src/system/launch_tests.rs @@ -0,0 +1,102 @@ +use super::*; + +#[test] +fn no_attach_requests_a_fresh_application_instance() { + let options = LaunchOptions { + attach_if_running: false, + ..Default::default() + }; + + assert!(crate::system::launch_workspace::creates_new_instance( + &options + )); +} + +#[test] +fn default_launch_allows_attaching_to_a_running_instance() { + let options = LaunchOptions::default(); + + assert!(!crate::system::launch_workspace::creates_new_instance( + &options + )); +} + +#[test] +fn validates_safe_names_and_bundle_identifiers() { + assert!(validate_app_identifier("Safari").is_ok()); + assert!(validate_app_identifier("com.apple.Safari").is_ok()); +} + +#[test] +fn rejects_paths_and_unsafe_bundle_identifiers() { + for identifier in [ + "../Evil", + "/abs/path", + "Foo/Bar", + "bad\0name", + "bad\nname", + "com.apple.$evil", + ] { + let error = validate_app_identifier(identifier).expect_err("unsafe identifier"); + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert!(!error.message.contains(identifier)); + assert_eq!(error.details.expect("details")["app_name"], identifier); + } +} + +#[test] +fn zero_wait_still_launches_but_never_polls_after_first_observation() { + assert!(!should_poll_after_first_observation(0)); + assert!(should_poll_after_first_observation(1)); +} + +#[test] +fn exact_native_launch_rejects_a_working_directory() { + let options = LaunchOptions { + cwd: Some(std::path::PathBuf::from("/tmp")), + ..Default::default() + }; + + let error = validate_launch_options(&options).expect_err("cwd is unsupported"); + + assert_eq!(error.code, ErrorCode::ActionNotSupported); +} + +#[test] +fn launch_options_enforce_bounded_entry_counts() { + let options = LaunchOptions { + args: (0..=MAX_ARGUMENT_COUNT) + .map(|index| index.to_string()) + .collect(), + ..Default::default() + }; + + let error = validate_launch_options(&options).expect_err("too many args"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +#[test] +fn launch_options_enforce_a_bounded_text_budget() { + let options = LaunchOptions { + args: vec!["x".repeat(MAX_LAUNCH_TEXT_BYTES + 1)], + ..Default::default() + }; + + let error = validate_launch_options(&options).expect_err("payload too large"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); +} + +#[test] +fn launch_no_window_error_keeps_identifier_in_details_only() { + let marker = "MARKER_APP_ID_9f31c4"; + let error = launch_no_window_error(marker, 5000, &(77, "generation".into())); + + assert!(!error.message.contains(marker)); + assert!(error.message.contains("5000")); + let details = error.details.expect("details"); + assert_eq!(details["app_name"], marker); + assert_eq!(details["pid"], 77); + assert_eq!(details["retry_safe"], false); +} diff --git a/crates/macos/src/system/launch_workspace.rs b/crates/macos/src/system/launch_workspace.rs new file mode 100644 index 0000000..8e84635 --- /dev/null +++ b/crates/macos/src/system/launch_workspace.rs @@ -0,0 +1,35 @@ +use agent_desktop_core::{AdapterError, Deadline, launch_options::LaunchOptions}; + +pub(crate) fn open( + id: &str, + options: &LaunchOptions, + deadline: Deadline, +) -> Result<(i32, String), AdapterError> { + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered())); + } + crate::system::cocoa_runtime::ensure_cocoa_multithreaded().map_err(|error| { + error.with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) + })?; + let request = serde_json::to_vec(&serde_json::json!({ + "identifier": id, + "bundle_id": super::launch::looks_like_bundle_id(id), + "arguments": options.args, + "environment": options.env, + "activates": false, + "prompts": false, + "substitution": false, + "new_instance": creates_new_instance(options), + })) + .map_err(|error| { + AdapterError::internal(format!("Encode launch request: {error}")) + .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) + })?; + unsafe { crate::system::launch_completion::open_and_wait(&request, deadline) } +} + +pub(crate) fn creates_new_instance(options: &LaunchOptions) -> bool { + !options.attach_if_running +} diff --git a/crates/macos/src/system/mod.rs b/crates/macos/src/system/mod.rs index a0fcb0d..0a0026b 100644 --- a/crates/macos/src/system/mod.rs +++ b/crates/macos/src/system/mod.rs @@ -1,15 +1,38 @@ +mod adapter; pub(crate) mod app_inventory; -pub mod app_list; -pub mod app_ops; +pub(crate) mod app_ops; +pub(crate) mod appkit_bridge; pub(crate) mod cg_window; -pub(crate) mod force_close; -pub mod key_dispatch; -pub mod permissions; +pub(crate) mod cg_window_exact; +pub(crate) mod cocoa_runtime; +pub(crate) mod display; +pub(crate) mod display_work_area; +pub(crate) mod focus; +pub(crate) mod key_dispatch; +pub(crate) mod launch; +pub(crate) mod launch_callback_result; +pub(crate) mod launch_completion; +pub(crate) mod launch_workspace; +pub(crate) mod permission_helper; +pub(crate) mod permission_operation; +pub(crate) mod permissions; pub(crate) mod process; pub(crate) mod process_apps; -pub mod screenshot; -pub mod wait; +pub(crate) mod process_identity; +pub(crate) mod process_state; +pub(crate) mod renderer_activation; +pub(crate) mod screenshot; +pub(crate) mod signals; +pub(crate) mod wait; +pub(crate) mod window_ax_state; +pub(crate) mod window_bridge; pub(crate) mod window_inventory; -pub mod window_list; -pub mod window_ops; +pub(crate) mod window_inventory_global; +pub(crate) mod window_ops; +pub(crate) mod window_postcondition; +pub(crate) mod window_resolve; pub(crate) mod workspace_apps; + +#[cfg(test)] +#[path = "screen_bridge_contract_tests.rs"] +mod screen_bridge_contract_tests; diff --git a/crates/macos/src/system/permission_helper.rs b/crates/macos/src/system/permission_helper.rs new file mode 100644 index 0000000..b99cb8e --- /dev/null +++ b/crates/macos/src/system/permission_helper.rs @@ -0,0 +1,260 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; +use serde_json::{Value, json}; +use std::process::Command; + +use super::permission_operation::PermissionOperation; + +const MARKER: &str = "AGENT_DESKTOP_PERMISSION_HELPER"; +const OPERATION: &str = "AGENT_DESKTOP_PERMISSION_OPERATION"; +const TOKEN: &str = "AGENT_DESKTOP_PERMISSION_TOKEN"; +const PARENT_PID: &str = "AGENT_DESKTOP_PERMISSION_PARENT_PID"; +const PARENT_INSTANCE: &str = "AGENT_DESKTOP_PERMISSION_PARENT_INSTANCE"; +const EXECUTABLE: &str = "AGENT_DESKTOP_PERMISSION_EXECUTABLE"; +const PROTOCOL_VERSION: &str = "v1"; +const TOKEN_BYTES: usize = 32; +const MAX_OUTPUT_BYTES: usize = 4 * 1024; + +type HelperRequest = (PermissionOperation, String, i32, String, String); + +pub fn entry_from_env() -> Option<(u8, String)> { + let get = |name: &str| std::env::var(name).ok(); + if !helper_environment_present(&get) { + return None; + } + Some(match execute_child(&get) { + Ok(response) => (0, response.to_string()), + Err(message) => (2, helper_error(&message).to_string()), + }) +} + +pub(crate) fn request( + operation: PermissionOperation, + deadline: Deadline, +) -> Result<bool, AdapterError> { + ensure_budget(deadline)?; + let executable = canonical_current_executable()?; + let executable_text = executable.to_str().ok_or_else(|| { + AdapterError::internal("Permission helper executable path is not valid UTF-8") + })?; + let parent_pid = i32::try_from(std::process::id()) + .map_err(|_| AdapterError::internal("Permission helper parent PID is out of range"))?; + let parent_instance = super::process_identity::token_for_pid(parent_pid)? + .ok_or_else(|| AdapterError::internal("Permission helper parent identity disappeared"))?; + let correlation_token = random_token()?; + let mut command = Command::new(&executable); + command + .env(MARKER, PROTOCOL_VERSION) + .env(OPERATION, operation.as_str()) + .env(TOKEN, &correlation_token) + .env(PARENT_PID, parent_pid.to_string()) + .env(PARENT_INSTANCE, parent_instance) + .env(EXECUTABLE, executable_text); + let output = super::process::run_with_deadline( + &mut command, + "macOS permission prompt helper", + deadline_instant(deadline)?, + )?; + if !output.status.success() { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "macOS permission prompt helper rejected its invocation", + ) + .with_platform_detail(bounded_text(&output.stderr)) + .with_details(json!({ + "kind": "permission_prompt_helper", + "exit_code": output.status.code(), + "complete": false, + }))); + } + parse_response(&output.stdout, operation, &correlation_token) +} + +fn execute_child(get: &impl Fn(&str) -> Option<String>) -> Result<Value, String> { + let request = parse_request(get)?; + validate_request( + &request, + unsafe { libc::getppid() }, + canonical_current_executable_text()?, + super::process_identity::matches_instance, + )?; + let granted = match request.0 { + PermissionOperation::Accessibility => { + super::permissions::prompt_accessibility(); + super::permissions::preflight_accessibility() + } + PermissionOperation::ScreenRecording => { + super::permissions::prompt_screen_recording(); + super::permissions::preflight_screen_recording() + } + PermissionOperation::Automation => super::permissions::prompt_automation(), + }; + Ok(json!({ + "version": 1, + "token": request.1, + "operation": request.0.as_str(), + "granted": granted, + })) +} + +fn helper_environment_present(get: &impl Fn(&str) -> Option<String>) -> bool { + [ + MARKER, + OPERATION, + TOKEN, + PARENT_PID, + PARENT_INSTANCE, + EXECUTABLE, + ] + .into_iter() + .any(|name| get(name).is_some()) +} + +fn parse_request(get: &impl Fn(&str) -> Option<String>) -> Result<HelperRequest, String> { + if get(MARKER).as_deref() != Some(PROTOCOL_VERSION) { + return Err("invalid permission helper protocol marker".into()); + } + let operation = get(OPERATION) + .as_deref() + .and_then(PermissionOperation::parse) + .ok_or_else(|| "invalid permission helper operation".to_string())?; + let token = get(TOKEN) + .filter(|value| valid_token(value)) + .ok_or_else(|| "invalid permission helper correlation token".to_string())?; + let parent_pid = get(PARENT_PID) + .and_then(|value| value.parse::<i32>().ok()) + .filter(|pid| *pid > 0) + .ok_or_else(|| "invalid permission helper parent PID".to_string())?; + let parent_instance = get(PARENT_INSTANCE) + .filter(|value| !value.is_empty() && value.len() <= 128) + .ok_or_else(|| "invalid permission helper parent identity".to_string())?; + let executable = get(EXECUTABLE) + .filter(|value| !value.is_empty() && value.len() <= 16 * 1024) + .ok_or_else(|| "invalid permission helper executable".to_string())?; + Ok((operation, token, parent_pid, parent_instance, executable)) +} + +fn validate_request( + request: &HelperRequest, + actual_parent: i32, + actual_executable: String, + matches_parent: impl FnOnce(i32, &str) -> Result<bool, AdapterError>, +) -> Result<(), String> { + if request.2 != actual_parent { + return Err("permission helper is detached from its requesting parent".into()); + } + if request.4 != actual_executable { + return Err("permission helper executable identity mismatch".into()); + } + match matches_parent(request.2, &request.3) { + Ok(true) => Ok(()), + Ok(false) => Err("permission helper parent process instance changed".into()), + Err(error) => Err(format!( + "permission helper parent validation failed: {error}" + )), + } +} + +fn parse_response( + bytes: &[u8], + expected_operation: PermissionOperation, + expected_token: &str, +) -> Result<bool, AdapterError> { + if bytes.len() > MAX_OUTPUT_BYTES { + return Err(response_error("response exceeded the protocol size limit")); + } + let response: Value = serde_json::from_slice(bytes) + .map_err(|_| response_error("response was not exactly one JSON value"))?; + let object = response + .as_object() + .filter(|object| object.len() == 4) + .ok_or_else(|| response_error("response had an invalid field set"))?; + if object.get("version").and_then(Value::as_u64) != Some(1) + || object.get("token").and_then(Value::as_str) != Some(expected_token) + || object.get("operation").and_then(Value::as_str) != Some(expected_operation.as_str()) + { + return Err(response_error( + "response identity did not match the request", + )); + } + object + .get("granted") + .and_then(Value::as_bool) + .ok_or_else(|| response_error("response omitted the diagnostic grant state")) +} + +fn random_token() -> Result<String, AdapterError> { + let mut bytes = [0_u8; TOKEN_BYTES]; + let result = unsafe { getentropy(bytes.as_mut_ptr().cast(), bytes.len()) }; + if result != 0 { + return Err(AdapterError::internal(format!( + "Could not create permission helper correlation token: {}", + std::io::Error::last_os_error() + ))); + } + Ok(bytes.iter().map(|byte| format!("{byte:02x}")).collect()) +} + +fn valid_token(value: &str) -> bool { + value.len() == TOKEN_BYTES * 2 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn canonical_current_executable() -> Result<std::path::PathBuf, AdapterError> { + std::env::current_exe() + .and_then(std::fs::canonicalize) + .map_err(|error| AdapterError::internal(format!("Resolve current executable: {error}"))) +} + +fn canonical_current_executable_text() -> Result<String, String> { + canonical_current_executable() + .map_err(|error| error.to_string())? + .into_os_string() + .into_string() + .map_err(|_| "permission helper executable path is not valid UTF-8".into()) +} + +fn deadline_instant(deadline: Deadline) -> Result<std::time::Instant, AdapterError> { + std::time::Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::internal("Permission helper deadline is out of range")) +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +fn response_error(reason: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "macOS permission prompt helper returned an invalid response", + ) + .with_platform_detail(reason) + .with_details(json!({ "kind": "permission_prompt_helper", "complete": false })) +} + +fn helper_error(message: &str) -> Value { + json!({ + "version": 1, + "ok": false, + "error": "invalid_helper_invocation", + "message": message, + }) +} + +fn bounded_text(bytes: &[u8]) -> String { + String::from_utf8_lossy(&bytes[..bytes.len().min(MAX_OUTPUT_BYTES)]).into_owned() +} + +unsafe extern "C" { + fn getentropy(buffer: *mut std::ffi::c_void, size: usize) -> i32; +} + +#[cfg(test)] +#[path = "permission_helper_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/permission_helper_tests.rs b/crates/macos/src/system/permission_helper_tests.rs new file mode 100644 index 0000000..52225f6 --- /dev/null +++ b/crates/macos/src/system/permission_helper_tests.rs @@ -0,0 +1,121 @@ +use super::*; +use std::collections::HashMap; + +fn valid_request() -> HelperRequest { + ( + PermissionOperation::Accessibility, + "ab".repeat(TOKEN_BYTES), + 42, + "macos-proc-v1:1:2".into(), + "/tmp/agent-desktop".into(), + ) +} + +#[test] +fn partial_or_malformed_helper_environment_never_falls_through() { + let values = HashMap::from([(OPERATION.to_string(), "accessibility".to_string())]); + let get = |name: &str| values.get(name).cloned(); + + assert!(helper_environment_present(&get)); + assert!(parse_request(&get).is_err()); +} + +#[test] +fn parser_accepts_only_the_closed_protocol_and_correlation_token_shape() { + let values = HashMap::from([ + (MARKER.to_string(), PROTOCOL_VERSION.to_string()), + (OPERATION.to_string(), "accessibility".to_string()), + (TOKEN.to_string(), "ab".repeat(TOKEN_BYTES)), + (PARENT_PID.to_string(), "42".to_string()), + (PARENT_INSTANCE.to_string(), "macos-proc-v1:1:2".to_string()), + (EXECUTABLE.to_string(), "/tmp/agent-desktop".to_string()), + ]); + + assert!(parse_request(&|name| values.get(name).cloned()).is_ok()); + let mut invalid = values; + invalid.insert(OPERATION.to_string(), "shell".to_string()); + assert!(parse_request(&|name| invalid.get(name).cloned()).is_err()); +} + +#[test] +fn detached_replayed_or_substituted_helper_invocations_are_rejected() { + let request = valid_request(); + assert!(validate_request(&request, 41, request.4.clone(), |_, _| Ok(true)).is_err()); + assert!(validate_request(&request, 42, "/tmp/other".into(), |_, _| Ok(true)).is_err()); + assert!(validate_request(&request, 42, request.4.clone(), |_, _| Ok(false)).is_err()); +} + +#[test] +fn response_requires_exact_cardinality_token_and_operation() { + let token = "cd".repeat(TOKEN_BYTES); + let valid = json!({ + "version": 1, + "token": token, + "operation": "accessibility", + "granted": true, + }); + assert!( + parse_response( + valid.to_string().as_bytes(), + PermissionOperation::Accessibility, + &token, + ) + .unwrap() + ); + + let wrong_token = json!({ + "version": 1, + "token": "ef".repeat(TOKEN_BYTES), + "operation": "accessibility", + "granted": true, + }); + assert!( + parse_response( + wrong_token.to_string().as_bytes(), + PermissionOperation::Accessibility, + &token, + ) + .is_err() + ); + + let two_values = format!("{}{}", valid, valid); + assert!( + parse_response( + two_values.as_bytes(), + PermissionOperation::Accessibility, + &token, + ) + .is_err() + ); + assert!( + parse_response( + &vec![b'x'; MAX_OUTPUT_BYTES + 1], + PermissionOperation::Accessibility, + &token, + ) + .is_err() + ); +} + +#[test] +fn correlation_tokens_use_full_os_random_shape() { + let token = random_token().unwrap(); + + assert!(valid_token(&token)); + assert_eq!(token.len(), TOKEN_BYTES * 2); +} + +#[cfg(unix)] +#[test] +fn helper_subprocess_timeout_kills_descendants_without_blocking() { + let started = std::time::Instant::now(); + let error = super::super::process::run_with_timeout( + Command::new("/bin/sh").args(["-c", "sleep 5 & wait"]), + "permission helper timeout fixture", + std::time::Duration::from_millis(200), + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::Timeout); + assert!(started.elapsed() < std::time::Duration::from_secs(1)); +} diff --git a/crates/macos/src/system/permission_operation.rs b/crates/macos/src/system/permission_operation.rs new file mode 100644 index 0000000..09b912b --- /dev/null +++ b/crates/macos/src/system/permission_operation.rs @@ -0,0 +1,47 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PermissionOperation { + Accessibility, + ScreenRecording, + Automation, +} + +impl PermissionOperation { + pub(crate) fn parse(value: &str) -> Option<Self> { + match value { + "accessibility" => Some(Self::Accessibility), + "screen_recording" => Some(Self::ScreenRecording), + "automation" => Some(Self::Automation), + _ => None, + } + } + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Accessibility => "accessibility", + Self::ScreenRecording => "screen_recording", + Self::Automation => "automation", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn helper_operation_protocol_accepts_only_permission_requests() { + assert_eq!( + PermissionOperation::parse("accessibility"), + Some(PermissionOperation::Accessibility) + ); + assert_eq!( + PermissionOperation::parse("screen_recording"), + Some(PermissionOperation::ScreenRecording) + ); + assert_eq!( + PermissionOperation::parse("automation"), + Some(PermissionOperation::Automation) + ); + assert_eq!(PermissionOperation::parse("shell"), None); + } +} diff --git a/crates/macos/src/system/permissions.rs b/crates/macos/src/system/permissions.rs index 18536dc..86e676f 100644 --- a/crates/macos/src/system/permissions.rs +++ b/crates/macos/src/system/permissions.rs @@ -1,7 +1,13 @@ -use agent_desktop_core::{PermissionReport, PermissionState}; +use agent_desktop_core::{AdapterError, Deadline, PermissionReport, PermissionState}; const ACCESSIBILITY_SUGGESTION: &str = "Open System Settings > Privacy & Security > Accessibility and add the app that launches agent-desktop, such as Terminal, iTerm, or Codex. If macOS lists the built binary separately, add that binary too."; const SCREEN_RECORDING_SUGGESTION: &str = "Open System Settings > Privacy & Security > Screen Recording and add the app that launches agent-desktop, such as Terminal, iTerm, or Codex. If macOS lists the built binary separately, add that binary too."; +pub(crate) const AUTOMATION_SUGGESTION: &str = "Open System Settings > Privacy & Security > Automation and allow the app that launches agent-desktop, such as Terminal, iTerm, or Codex, to control System Events. If macOS lists the built binary separately, add that binary too."; + +const NO_ERR: i32 = 0; +const PROC_NOT_FOUND: i32 = -600; +const ERR_AE_EVENT_NOT_PERMITTED: i32 = -1743; +const ERR_AE_EVENT_WOULD_REQUIRE_USER_CONSENT: i32 = -1744; #[cfg(target_os = "macos")] mod imp { @@ -38,6 +44,67 @@ mod imp { pub(super) fn request_screen_recording() -> bool { unsafe { CGRequestScreenCaptureAccess() } } + + const TYPE_APPLICATION_BUNDLE_ID: u32 = 0x6275_6E64; + const TYPE_WILD_CARD: u32 = 0x2A2A_2A2A; + const SYSTEM_EVENTS_BUNDLE_ID: &[u8] = b"com.apple.systemevents"; + + #[repr(C)] + struct AEAddressDesc { + descriptor_type: u32, + data_handle: *mut std::ffi::c_void, + } + + #[link(name = "ApplicationServices", kind = "framework")] + unsafe extern "C" { + fn AECreateDesc( + type_code: u32, + data_ptr: *const std::ffi::c_void, + data_size: isize, + result: *mut AEAddressDesc, + ) -> i32; + fn AEDisposeDesc(the_aedesc: *mut AEAddressDesc) -> i32; + fn AEDeterminePermissionToAutomateTarget( + target: *const AEAddressDesc, + event_class: u32, + event_id: u32, + ask_user_if_needed: u8, + ) -> i32; + } + + fn determine_automation_permission(ask_user_if_needed: u8) -> i32 { + unsafe { + let mut target = AEAddressDesc { + descriptor_type: 0, + data_handle: std::ptr::null_mut(), + }; + let create_status = AECreateDesc( + TYPE_APPLICATION_BUNDLE_ID, + SYSTEM_EVENTS_BUNDLE_ID.as_ptr().cast(), + SYSTEM_EVENTS_BUNDLE_ID.len() as isize, + &mut target, + ); + if create_status != 0 { + return create_status; + } + let status = AEDeterminePermissionToAutomateTarget( + &target, + TYPE_WILD_CARD, + TYPE_WILD_CARD, + ask_user_if_needed, + ); + let _ = AEDisposeDesc(&mut target); + status + } + } + + pub(super) fn probe_automation_permission() -> i32 { + determine_automation_permission(0) + } + + pub(super) fn request_automation_permission() -> bool { + determine_automation_permission(1) == 0 + } } #[cfg(not(target_os = "macos"))] @@ -54,25 +121,74 @@ mod imp { pub fn request_screen_recording() -> bool { false } + pub fn probe_automation_permission() -> i32 { + -600 + } + pub fn request_automation_permission() -> bool { + false + } } -pub fn report() -> PermissionReport { - PermissionReport { +pub(crate) fn report(deadline: Deadline) -> Result<PermissionReport, AdapterError> { + ensure_budget(deadline)?; + let report = PermissionReport { accessibility: accessibility_report_state(), screen_recording: screen_recording_report_state(), - automation: PermissionState::NotRequired, - } + automation: automation_report_state(), + }; + ensure_budget(deadline)?; + Ok(report) } -pub fn request_report() -> PermissionReport { - PermissionReport { - accessibility: permission_state(imp::request_trust(), ACCESSIBILITY_SUGGESTION), - screen_recording: permission_state( - imp::request_screen_recording(), - SCREEN_RECORDING_SUGGESTION, - ), - automation: PermissionState::NotRequired, - } +pub(crate) fn request_report(deadline: Deadline) -> Result<PermissionReport, AdapterError> { + request_report_with(deadline, crate::system::permission_helper::request, report) +} + +fn request_report_with( + deadline: Deadline, + mut request: impl FnMut( + crate::system::permission_operation::PermissionOperation, + Deadline, + ) -> Result<bool, AdapterError>, + report: impl FnOnce(Deadline) -> Result<PermissionReport, AdapterError>, +) -> Result<PermissionReport, AdapterError> { + ensure_budget(deadline)?; + let _ = request( + crate::system::permission_operation::PermissionOperation::Accessibility, + deadline, + )?; + ensure_budget(deadline)?; + let _ = request( + crate::system::permission_operation::PermissionOperation::ScreenRecording, + deadline, + )?; + ensure_budget(deadline)?; + let _ = request( + crate::system::permission_operation::PermissionOperation::Automation, + deadline, + )?; + ensure_budget(deadline)?; + report(deadline) +} + +pub(crate) fn prompt_accessibility() -> bool { + imp::request_trust() +} + +pub(crate) fn prompt_screen_recording() -> bool { + imp::request_screen_recording() +} + +pub(crate) fn prompt_automation() -> bool { + imp::request_automation_permission() +} + +pub(crate) fn preflight_accessibility() -> bool { + imp::is_trusted() +} + +pub(crate) fn preflight_screen_recording() -> bool { + imp::screen_recording_granted() } fn permission_state(granted: bool, suggestion: &'static str) -> PermissionState { @@ -92,3 +208,98 @@ fn accessibility_report_state() -> PermissionState { fn screen_recording_report_state() -> PermissionState { permission_state(imp::screen_recording_granted(), SCREEN_RECORDING_SUGGESTION) } + +fn automation_report_state() -> PermissionState { + map_automation_probe(imp::probe_automation_permission()) +} + +pub(crate) fn require_automation_permission() -> Result<(), AdapterError> { + let status = imp::probe_automation_permission(); + match status { + NO_ERR => Ok(()), + ERR_AE_EVENT_NOT_PERMITTED | ERR_AE_EVENT_WOULD_REQUIRE_USER_CONSENT => { + Err(automation_permission_error(status)) + } + _ => Err(AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Could not verify Automation permission without prompting", + ) + .with_details(serde_json::json!({ + "kind": "automation_permission_probe", + "os_status": status, + "target": "System Events", + "prompted": false, + })) + .with_suggestion("Ensure System Events is running, then retry the command")), + } +} + +pub(crate) fn map_automation_probe(status: i32) -> PermissionState { + match status { + NO_ERR => PermissionState::Granted, + ERR_AE_EVENT_NOT_PERMITTED => PermissionState::Denied { + suggestion: AUTOMATION_SUGGESTION.into(), + }, + PROC_NOT_FOUND | ERR_AE_EVENT_WOULD_REQUIRE_USER_CONSENT => PermissionState::Unknown, + _ => PermissionState::Unknown, + } +} + +pub(crate) fn map_automation_command_failure( + status: std::process::ExitStatus, + stderr: &[u8], +) -> AdapterError { + let detail = bounded_platform_text(stderr); + if automation_denial_text(&detail) { + return automation_permission_error(ERR_AE_EVENT_NOT_PERMITTED) + .with_platform_detail(detail); + } + AdapterError::new( + agent_desktop_core::ErrorCode::ActionFailed, + "System Events did not complete the requested Automation operation", + ) + .with_platform_detail(detail) + .with_details(serde_json::json!({ + "kind": "automation_command", + "exit_code": status.code(), + "target": "System Events", + })) +} + +fn automation_permission_error(status: i32) -> AdapterError { + AdapterError::new( + agent_desktop_core::ErrorCode::PermDenied, + "Automation permission for System Events is required", + ) + .with_suggestion(AUTOMATION_SUGGESTION) + .with_details(serde_json::json!({ + "kind": "automation_permission", + "os_status": status, + "target": "System Events", + "prompted": false, + })) +} + +fn automation_denial_text(value: &str) -> bool { + let lower = value.to_ascii_lowercase(); + lower.contains("-1743") + || lower.contains("not authorized to send apple events") + || lower.contains("not permitted to send apple events") +} + +fn bounded_platform_text(bytes: &[u8]) -> String { + const MAX_BYTES: usize = 4 * 1024; + String::from_utf8_lossy(&bytes[..bytes.len().min(MAX_BYTES)]).into_owned() +} + +fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } +} + +#[cfg(test)] +#[path = "permissions_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/permissions_tests.rs b/crates/macos/src/system/permissions_tests.rs new file mode 100644 index 0000000..06acc16 --- /dev/null +++ b/crates/macos/src/system/permissions_tests.rs @@ -0,0 +1,72 @@ +use super::*; + +#[test] +fn expired_permission_deadline_fails_without_native_calls() { + let error = report(Deadline::after(0).unwrap()).unwrap_err(); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::Timeout); +} + +#[test] +fn automation_probe_maps_without_a_prompting_state() { + assert_eq!(map_automation_probe(0), PermissionState::Granted); + assert!(matches!( + map_automation_probe(-1743), + PermissionState::Denied { .. } + )); + assert_eq!(map_automation_probe(-1744), PermissionState::Unknown); + assert_eq!(map_automation_probe(-600), PermissionState::Unknown); +} + +#[test] +fn post_helper_preflight_is_the_authoritative_permission_state() { + let mut requested = Vec::new(); + let report = request_report_with( + Deadline::after(1_000).unwrap(), + |operation, _| { + requested.push(operation); + Ok(false) + }, + |_| { + Ok(PermissionReport { + accessibility: PermissionState::Granted, + screen_recording: PermissionState::Granted, + automation: PermissionState::Unknown, + }) + }, + ) + .unwrap(); + + assert_eq!(requested.len(), 3); + assert_eq!(report.accessibility, PermissionState::Granted); + assert_eq!(report.screen_recording, PermissionState::Granted); + assert_eq!(report.automation, PermissionState::Unknown); +} + +#[test] +fn authorization_denial_is_structured_and_nonprompting() { + let error = automation_permission_error(-1744); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::PermDenied); + assert_eq!(error.details.unwrap()["prompted"], false); + assert!( + error + .suggestion + .as_deref() + .is_some_and(|value| value.contains("Automation")) + ); +} + +#[cfg(unix)] +#[test] +fn osascript_denial_output_maps_to_permission_denied() { + use std::os::unix::process::ExitStatusExt; + + let error = map_automation_command_failure( + std::process::ExitStatus::from_raw(1 << 8), + b"execution error: Not authorized to send Apple events to System Events. (-1743)", + ); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::PermDenied); + assert_eq!(error.details.unwrap()["os_status"], -1743); +} diff --git a/crates/macos/src/system/process.rs b/crates/macos/src/system/process.rs index e98a293..8c6a71d 100644 --- a/crates/macos/src/system/process.rs +++ b/crates/macos/src/system/process.rs @@ -1,115 +1,329 @@ -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::{AdapterError, ErrorCode}; use std::io::Read; -use std::process::{Command, Output, Stdio}; -use std::thread; +use std::process::{Child, Command, ExitStatus, Output, Stdio}; +use std::sync::mpsc; +use std::thread::JoinHandle; use std::time::{Duration, Instant}; +const MAX_CAPTURED_STREAM_BYTES: usize = 8 * 1024 * 1024; +const OUTPUT_CONTEXT_BYTES: usize = 4 * 1024; +const MAX_CLEANUP_RESERVE: Duration = Duration::from_millis(250); +const TERM_GRACE: Duration = Duration::from_millis(25); +const POSIX_EPERM: i32 = 1; +const POSIX_ESRCH: i32 = 3; + +struct DrainResult { + bytes: Vec<u8>, + tail: Vec<u8>, + exceeded_limit: bool, +} + +type DrainHandle = (mpsc::Receiver<std::io::Result<DrainResult>>, JoinHandle<()>); + +#[cfg(test)] pub(crate) fn run_with_timeout( command: &mut Command, label: &str, timeout: Duration, ) -> Result<Output, AdapterError> { + let deadline = Instant::now().checked_add(timeout).ok_or_else(|| { + AdapterError::timeout(format!("{label} timeout exceeds the supported range")) + })?; + run_with_deadline(command, label, deadline) +} + +pub(crate) fn run_with_deadline( + command: &mut Command, + label: &str, + deadline: Instant, +) -> Result<Output, AdapterError> { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(AdapterError::timeout(format!( + "{label} has no subprocess cleanup budget" + ))); + } + let cleanup_reserve = (remaining / 4).min(MAX_CLEANUP_RESERVE); + let work_deadline = deadline.checked_sub(cleanup_reserve).ok_or_else(|| { + AdapterError::timeout(format!("{label} has no subprocess cleanup budget")) + })?; + configure_process_group(command); command.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = command .spawn() - .map_err(|e| AdapterError::internal(format!("{label}: {e}")))?; + .map_err(|error| AdapterError::internal(format!("{label}: {error}")))?; + let process_group = i32::try_from(child.id()).map_err(|_| { + let _ = child.kill(); + let _ = child.wait(); + AdapterError::internal(format!("{label}: child PID exceeds the macOS pid_t range")) + })?; + let mut stdout = child.stdout.take().map(spawn_drain); + let mut stderr = child.stderr.take().map(spawn_drain); - let stdout_handle = child.stdout.take().map(spawn_drain); - let stderr_handle = child.stderr.take().map(spawn_drain); - let started = Instant::now(); - - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break status, - Ok(None) if started.elapsed() >= timeout => { - let _ = child.kill(); - let _ = child.wait(); - join_drain(stdout_handle); - join_drain(stderr_handle); - return Err(AdapterError::timeout(format!("{label} timed out")) - .with_platform_detail(format!("timeout after {timeout:?}"))); - } - Ok(None) => thread::sleep(Duration::from_millis(20)), - Err(e) => { - let _ = child.kill(); - let _ = child.wait(); - join_drain(stdout_handle); - join_drain(stderr_handle); - return Err(AdapterError::internal(format!("{label} status: {e}"))); - } + let stderr_result = match receive_drain(&mut stderr, label, "stderr", work_deadline) { + Ok(result) => result, + Err(error) => { + return cleanup_after_error( + &mut child, + process_group, + &mut stdout, + &mut stderr, + label, + deadline, + error, + ); } }; - - let stdout = join_drain(stdout_handle); - let stderr = join_drain(stderr_handle); + let stdout_result = match receive_drain(&mut stdout, label, "stdout", work_deadline) { + Ok(result) => result, + Err(error) => { + return cleanup_after_error( + &mut child, + process_group, + &mut stdout, + &mut stderr, + label, + deadline, + error, + ); + } + }; + let status = match wait_for_status(&mut child, work_deadline) { + Ok(status) => status, + Err(error) => { + return cleanup_after_error( + &mut child, + process_group, + &mut stdout, + &mut stderr, + label, + deadline, + error, + ); + } + }; + if stdout_result.exceeded_limit || stderr_result.exceeded_limit { + return Err(output_limit_error(label, &stdout_result, &stderr_result)); + } Ok(Output { status, - stdout, - stderr, + stdout: stdout_result.bytes, + stderr: stderr_result.bytes, }) } -fn spawn_drain<R>(mut reader: R) -> thread::JoinHandle<Vec<u8>> +fn configure_process_group(command: &mut Command) { + use std::os::unix::process::CommandExt; + command.process_group(0); +} + +fn wait_for_status(child: &mut Child, deadline: Instant) -> Result<ExitStatus, AdapterError> { + loop { + if Instant::now() >= deadline { + return Err(AdapterError::timeout( + "Subprocess exceeded its work deadline", + )); + } + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => std::thread::sleep(poll_interval(deadline)), + Err(error) => { + return Err(AdapterError::internal(format!( + "Could not inspect subprocess status: {error}" + ))); + } + } + } +} + +fn cleanup_after_error( + child: &mut Child, + process_group: i32, + stdout: &mut Option<DrainHandle>, + stderr: &mut Option<DrainHandle>, + label: &str, + deadline: Instant, + mut original: AdapterError, +) -> Result<Output, AdapterError> { + let cleanup_failures = terminate_process_group(child, process_group, deadline); + let _ = receive_drain(stderr, label, "stderr", deadline); + let _ = receive_drain(stdout, label, "stdout", deadline); + if !cleanup_failures.is_empty() { + original = original.with_platform_detail(cleanup_failures.join("; ")); + } + Err(original) +} + +fn terminate_process_group( + child: &mut Child, + process_group: i32, + deadline: Instant, +) -> Vec<String> { + terminate_process_group_with(child, process_group, deadline, signal_group) +} + +fn terminate_process_group_with( + child: &mut Child, + process_group: i32, + deadline: Instant, + mut signal: impl FnMut(i32, i32) -> std::io::Result<bool>, +) -> Vec<String> { + let mut failures = Vec::new(); + if let Err(error) = signal(process_group, 15) { + failures.push(format!("SIGTERM process group {process_group}: {error}")); + } + let grace_deadline = Instant::now() + .checked_add(TERM_GRACE) + .map_or(deadline, |grace| grace.min(deadline)); + std::thread::sleep(grace_deadline.saturating_duration_since(Instant::now())); + let kill_error = signal(process_group, 9).err(); + let reaped = poll_reap(child, deadline); + if let Some(error) = kill_error { + let group_is_gone = reaped + && error.raw_os_error() == Some(POSIX_EPERM) + && matches!(signal(process_group, 0), Ok(false)); + if !group_is_gone { + failures.push(format!("SIGKILL process group {process_group}: {error}")); + } + } + if !reaped { + failures.push("subprocess could not be reaped before cleanup deadline".into()); + } + failures +} + +fn signal_group(process_group: i32, signal: i32) -> std::io::Result<bool> { + unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; + } + if unsafe { kill(-process_group, signal) } == 0 { + return Ok(true); + } + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(POSIX_ESRCH) { + Ok(false) + } else { + Err(error) + } +} + +fn poll_reap(child: &mut Child, deadline: Instant) -> bool { + loop { + match child.try_wait() { + Ok(Some(_)) => return true, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(poll_interval(deadline)); + } + Ok(None) | Err(_) => return false, + } + } +} + +fn poll_interval(deadline: Instant) -> Duration { + deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(20)) +} + +fn spawn_drain<R>(mut reader: R) -> DrainHandle where R: Read + Send + 'static, { - thread::spawn(move || { - let mut buf = Vec::new(); - let _ = reader.read_to_end(&mut buf); - buf - }) + let (sender, receiver) = mpsc::sync_channel(1); + let thread = std::thread::spawn(move || { + let mut bytes = Vec::new(); + let mut tail = Vec::new(); + let mut exceeded_limit = false; + let mut chunk = [0_u8; 8192]; + let result = loop { + match reader.read(&mut chunk) { + Ok(0) => { + break Ok(DrainResult { + bytes, + tail, + exceeded_limit, + }); + } + Ok(count) => { + let remaining = MAX_CAPTURED_STREAM_BYTES.saturating_sub(bytes.len()); + let retained = remaining.min(count); + bytes.extend_from_slice(&chunk[..retained]); + append_tail(&mut tail, &chunk[..count]); + exceeded_limit |= retained < count; + } + Err(error) => break Err(error), + } + }; + let _ = sender.send(result); + }); + (receiver, thread) } -fn join_drain(handle: Option<thread::JoinHandle<Vec<u8>>>) -> Vec<u8> { - handle.and_then(|h| h.join().ok()).unwrap_or_default() +fn receive_drain( + handle: &mut Option<DrainHandle>, + label: &str, + stream: &str, + deadline: Instant, +) -> Result<DrainResult, AdapterError> { + let (receiver, _) = handle + .as_ref() + .ok_or_else(|| AdapterError::internal(format!("{label}: missing {stream}")))?; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(AdapterError::timeout(format!( + "{label}: timed out draining {stream}" + ))); + } + let drained = receiver + .recv_timeout(remaining) + .map_err(|error| match error { + mpsc::RecvTimeoutError::Timeout => { + AdapterError::timeout(format!("{label}: timed out draining {stream}")) + } + mpsc::RecvTimeoutError::Disconnected => { + AdapterError::internal(format!("{label}: {stream} reader stopped unexpectedly")) + } + })?; + let (_, thread) = handle + .take() + .ok_or_else(|| AdapterError::internal(format!("{label}: lost {stream} drain thread")))?; + thread + .join() + .map_err(|_| AdapterError::internal(format!("{label}: {stream} drain thread panicked")))?; + drained.map_err(|error| AdapterError::internal(format!("{label}: read {stream}: {error}"))) +} + +fn append_tail(tail: &mut Vec<u8>, chunk: &[u8]) { + if chunk.len() >= OUTPUT_CONTEXT_BYTES { + tail.clear(); + tail.extend_from_slice(&chunk[chunk.len() - OUTPUT_CONTEXT_BYTES..]); + return; + } + let overflow = tail + .len() + .saturating_add(chunk.len()) + .saturating_sub(OUTPUT_CONTEXT_BYTES); + if overflow > 0 { + tail.drain(..overflow); + } + tail.extend_from_slice(chunk); +} + +fn output_limit_error(label: &str, stdout: &DrainResult, stderr: &DrainResult) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("{label}: subprocess output exceeded the capture limit"), + ) + .with_details(serde_json::json!({ + "kind": "subprocess_output_limit", + "limit_bytes": MAX_CAPTURED_STREAM_BYTES, + "stderr_exceeded": stderr.exceeded_limit, + "stderr_tail": String::from_utf8_lossy(&stderr.tail), + "stdout_exceeded": stdout.exceeded_limit, + "stdout_tail": String::from_utf8_lossy(&stdout.tail), + })) } #[cfg(all(test, unix))] -mod tests { - use super::*; - - #[test] - fn run_with_timeout_returns_output_for_successful_process() { - let mut command = Command::new("/bin/echo"); - command.arg("ok"); - - let output = run_with_timeout(&mut command, "echo", Duration::from_secs(1)).unwrap(); - - assert!(output.status.success()); - assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); - } - - #[test] - fn run_with_timeout_kills_slow_process() { - let mut command = Command::new("/bin/sleep"); - command.arg("1"); - - let err = run_with_timeout(&mut command, "sleep", Duration::from_millis(10)).unwrap_err(); - - assert_eq!(err.code.as_str(), "TIMEOUT"); - } - - #[test] - fn run_with_timeout_drains_large_stdout_without_deadlock() { - let mut command = Command::new("/bin/sh"); - command.args(["-c", "yes ABCDEFGHIJ | head -c 200000"]); - - let output = run_with_timeout(&mut command, "yes-head", Duration::from_secs(5)).unwrap(); - - assert!(output.status.success()); - assert!( - output.stdout.len() >= 200_000, - "expected >=200000 bytes of drained stdout, got {}", - output.stdout.len() - ); - } - - #[test] - fn run_with_timeout_returns_internal_for_missing_binary() { - let mut command = Command::new("/nonexistent/binary-zzz"); - - let err = run_with_timeout(&mut command, "missing", Duration::from_secs(1)).unwrap_err(); - - assert_eq!(err.code.as_str(), "INTERNAL"); - } -} +#[path = "process_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/process_apps.rs b/crates/macos/src/system/process_apps.rs index ebb603e..3f6f2a9 100644 --- a/crates/macos/src/system/process_apps.rs +++ b/crates/macos/src/system/process_apps.rs @@ -1,51 +1,135 @@ -use agent_desktop_core::node::AppInfo; -use std::time::Duration; +use agent_desktop_core::{AdapterError, AppInfo, ErrorCode}; +use std::{process::Output, time::Instant}; -const PS_TIMEOUT: Duration = Duration::from_secs(2); +pub(crate) fn list_apps_until(deadline: Instant) -> Result<Vec<AppInfo>, AdapterError> { + list_apps_with_filter(None, deadline) +} -pub(crate) fn list_apps() -> Vec<AppInfo> { +pub(crate) fn list_apps_scoped_until( + name: &str, + deadline: Instant, +) -> Result<Vec<AppInfo>, AdapterError> { + list_apps_with_filter(Some(name), deadline) +} + +fn list_apps_with_filter( + name: Option<&str>, + deadline: Instant, +) -> Result<Vec<AppInfo>, AdapterError> { + if Instant::now() >= deadline { + return Err(AdapterError::timeout("ps app inventory timed out")); + } let mut command = std::process::Command::new("/bin/ps"); command.args(["-axo", "pid=,comm="]); - let output = match crate::system::process::run_with_timeout(&mut command, "ps", PS_TIMEOUT) { - Ok(output) if output.status.success() => output, - Ok(output) => { - tracing::debug!(status = ?output.status, "system: ps app inventory failed"); - return Vec::new(); - } - Err(error) => { - tracing::debug!(message = %error.message, "system: ps app inventory failed"); - return Vec::new(); - } - }; - let text = String::from_utf8_lossy(&output.stdout); + let output = crate::system::process::run_with_deadline(&mut command, "ps", deadline)?; + let mut apps = apps_from_output(output)?; + apps.retain(|app| name.is_none_or(|name| app.name.eq_ignore_ascii_case(name))); + enrich_process_instances(apps, crate::system::process_identity::token_for_pid) +} + +fn enrich_process_instances( + apps: Vec<AppInfo>, + mut resolve: impl FnMut(i32) -> Result<Option<String>, AdapterError>, +) -> Result<Vec<AppInfo>, AdapterError> { + let mut enriched = Vec::with_capacity(apps.len()); + for mut app in apps { + let pid = crate::system::process_identity::to_pid_t(app.pid)?; + app.process_instance = match resolve(pid) { + Ok(Some(instance)) => Some(instance), + Ok(None) => continue, + Err(error) if is_cross_uid_identity_error(&error) => continue, + Err(error) => return Err(error), + }; + enriched.push(app); + } + Ok(enriched) +} + +fn is_cross_uid_identity_error(error: &AdapterError) -> bool { + error.code == ErrorCode::PermDenied + && error + .details + .as_ref() + .and_then(|details| details.get("kind")) + .and_then(serde_json::Value::as_str) + == Some("process_identity_permission") +} + +fn apps_from_output(output: Output) -> Result<Vec<AppInfo>, AdapterError> { + if !output.status.success() { + let detail = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(inventory_error( + format!("ps app inventory exited with {}", output.status), + detail, + )); + } + let text = String::from_utf8(output.stdout).map_err(|error| { + inventory_error( + "ps app inventory returned non-UTF-8 output".to_string(), + error.to_string(), + ) + })?; + parse_apps(&text) +} + +fn parse_apps(text: &str) -> Result<Vec<AppInfo>, AdapterError> { let mut seen_pids = rustc_hash::FxHashSet::default(); let mut apps = Vec::new(); for line in text.lines() { let line = line.trim_start(); + if line.is_empty() { + continue; + } let mut fields = line.splitn(2, char::is_whitespace); - let Some(pid_text) = fields.next() else { - continue; - }; - let Some(command) = fields.next().map(str::trim) else { - continue; - }; - let Ok(pid) = pid_text.parse::<i32>() else { - continue; - }; + let pid_text = fields.next().ok_or_else(malformed_output)?; + let command = fields + .next() + .map(str::trim) + .filter(|command| !command.is_empty()) + .ok_or_else(malformed_output)?; + let pid = pid_text.parse::<i32>().map_err(|_| malformed_output())?; + if pid <= 0 { + return Err(malformed_output()); + } let Some(name) = app_name_from_command(command) else { + if command.contains(".app/Contents/MacOS") + && !command.contains("/Contents/Frameworks/") + && !command.contains("/Contents/PlugIns/") + { + return Err(malformed_output()); + } continue; }; if seen_pids.insert(pid) { apps.push(AppInfo { name, - pid, + pid: crate::system::process_identity::from_pid_t(pid)?, bundle_id: None, + process_instance: None, }); } } - apps + Ok(apps) +} + +fn malformed_output() -> AdapterError { + inventory_error( + "ps app inventory returned malformed output".to_string(), + "expected one positive pid and executable path per line".to_string(), + ) +} + +fn inventory_error(message: String, detail: String) -> AdapterError { + AdapterError::new(ErrorCode::AppUnresponsive, message) + .with_suggestion("Retry after macOS finishes updating the process inventory") + .with_platform_detail(detail) + .with_details(serde_json::json!({ + "kind": "inventory_source", + "source": "ps", + "retryable": true, + })) } fn app_name_from_command(command: &str) -> Option<String> { diff --git a/crates/macos/src/system/process_apps_tests.rs b/crates/macos/src/system/process_apps_tests.rs index 4ca7f38..2a2728b 100644 --- a/crates/macos/src/system/process_apps_tests.rs +++ b/crates/macos/src/system/process_apps_tests.rs @@ -1,4 +1,5 @@ use super::*; +use std::os::unix::process::ExitStatusExt; #[test] fn app_name_from_command_extracts_app_bundle_name() { @@ -51,3 +52,111 @@ fn app_name_from_command_rejects_empty_app_name() { None ); } + +#[test] +fn successful_empty_ps_output_is_a_real_empty_inventory() { + let output = Output { + status: std::process::ExitStatus::from_raw(0), + stdout: Vec::new(), + stderr: Vec::new(), + }; + + let apps = apps_from_output(output).unwrap(); + + assert!(apps.is_empty()); +} + +#[test] +fn failed_ps_output_is_not_an_empty_inventory() { + let output = Output { + status: std::process::ExitStatus::from_raw(256), + stdout: Vec::new(), + stderr: b"resource temporarily unavailable".to_vec(), + }; + + let error = apps_from_output(output).unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.details.unwrap()["source"], "ps"); +} + +#[test] +fn expired_ps_deadline_is_rejected_before_starting_a_process() { + let error = list_apps_until(Instant::now()).unwrap_err(); + + assert_eq!(error.code.as_str(), "TIMEOUT"); +} + +#[test] +fn malformed_ps_line_is_not_silently_dropped() { + let error = parse_apps("not-a-pid /Applications/Mail.app/Contents/MacOS/Mail").unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); +} + +#[test] +fn valid_non_app_processes_still_produce_successful_empty_inventory() { + let apps = parse_apps("1 /sbin/launchd\n2 /usr/libexec/logd").unwrap(); + + assert!(apps.is_empty()); +} + +#[test] +fn inaccessible_cross_uid_process_is_skipped() { + let apps = parse_apps( + "42 /Applications/Visible.app/Contents/MacOS/Visible\n43 /Applications/Private.app/Contents/MacOS/Private", + ) + .unwrap(); + let enriched = enrich_process_instances(apps, |pid| { + if pid == 43 { + Err( + AdapterError::new(ErrorCode::PermDenied, "permission denied").with_details( + serde_json::json!({ + "kind": "process_identity_permission", + "retryable": false, + }), + ), + ) + } else { + Ok(Some(format!("instance-{pid}"))) + } + }) + .unwrap(); + + assert_eq!(enriched.len(), 1); + assert_eq!(enriched[0].name, "Visible"); + assert_eq!(enriched[0].process_instance.as_deref(), Some("instance-42")); +} + +#[test] +fn process_that_exits_during_identity_enrichment_is_skipped() { + let apps = parse_apps( + "42 /Applications/Visible.app/Contents/MacOS/Visible\n43 /Applications/Exited.app/Contents/MacOS/Exited", + ) + .unwrap(); + let enriched = enrich_process_instances(apps, |pid| { + if pid == 43 { + Ok(None) + } else { + Ok(Some(format!("instance-{pid}"))) + } + }) + .unwrap(); + + assert_eq!(enriched.len(), 1); + assert_eq!(enriched[0].name, "Visible"); +} + +#[test] +fn process_identity_failure_other_than_cross_uid_permission_is_not_hidden() { + let apps = parse_apps("42 /Applications/Broken.app/Contents/MacOS/Broken").unwrap(); + let error = enrich_process_instances(apps, |_| { + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "identity unavailable", + )) + }) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); +} diff --git a/crates/macos/src/system/process_identity.rs b/crates/macos/src/system/process_identity.rs new file mode 100644 index 0000000..2cb9e5c --- /dev/null +++ b/crates/macos/src/system/process_identity.rs @@ -0,0 +1,310 @@ +use agent_desktop_core::{AdapterError, ErrorCode, ProcessId}; +#[cfg(target_os = "macos")] +use std::mem::{MaybeUninit, size_of}; + +const TOKEN_PREFIX: &str = "macos-proc-v1"; +const MAX_LAUNCH_TIME_DELTA_SECONDS: f64 = 5.0; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ProcessIdentity { + pid: i32, + start_seconds: u64, + start_microseconds: u64, +} + +impl ProcessIdentity { + #[cfg(target_os = "macos")] + pub(crate) fn capture(pid: i32) -> Result<Option<Self>, AdapterError> { + if pid <= 0 { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Process identity requires a positive PID", + )); + } + let mut info = MaybeUninit::<libc::proc_bsdinfo>::zeroed(); + let expected = i32::try_from(size_of::<libc::proc_bsdinfo>()) + .map_err(|_| AdapterError::internal("proc_bsdinfo size exceeds the libproc ABI"))?; + let returned = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + expected, + ) + }; + if returned <= 0 { + return classify_missing_or_inaccessible(pid, std::io::Error::last_os_error()); + } + if returned != expected { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + format!("libproc returned an incomplete process identity for pid {pid}"), + ) + .with_details(serde_json::json!({ + "pid": pid, + "returned_bytes": returned, + "expected_bytes": expected, + }))); + } + let info = unsafe { info.assume_init() }; + let expected_pid = u32::try_from(pid).map_err(|_| { + AdapterError::internal("validated macOS pid_t could not convert to u32") + })?; + if info.pbi_pid != expected_pid { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "libproc returned a mismatched process identity", + )); + } + Ok(Some(Self { + pid, + start_seconds: info.pbi_start_tvsec, + start_microseconds: info.pbi_start_tvusec, + })) + } + + #[cfg(not(target_os = "macos"))] + pub(crate) fn capture(pid: i32) -> Result<Option<Self>, AdapterError> { + let _ = pid; + Err(AdapterError::not_supported("macos_process_identity")) + } + + pub(crate) fn pid(self) -> i32 { + self.pid + } + + pub(crate) fn launch_time_seconds(self) -> f64 { + self.start_seconds as f64 + self.start_microseconds as f64 / 1_000_000.0 + } + + pub(crate) fn token(self) -> String { + format!( + "{TOKEN_PREFIX}:{}:{}", + self.start_seconds, self.start_microseconds + ) + } + + pub(crate) fn matches_launch_time(self, launch_time: f64) -> bool { + if !launch_time.is_finite() || launch_time <= 0.0 { + return false; + } + (self.launch_time_seconds() - launch_time).abs() <= MAX_LAUNCH_TIME_DELTA_SECONDS + } + + pub(crate) fn still_matches(self) -> Result<bool, AdapterError> { + Ok(Self::capture(self.pid)?.is_some_and(|current| current == self)) + } +} + +pub(crate) fn token_for_pid(pid: i32) -> Result<Option<String>, AdapterError> { + Ok(ProcessIdentity::capture(pid)?.map(ProcessIdentity::token)) +} + +pub(crate) fn matches_instance(pid: i32, token: &str) -> Result<bool, AdapterError> { + let expected = parse_token(pid, token)?; + expected.still_matches() +} + +pub(crate) fn instance_matches_launch_time( + pid: i32, + token: &str, + launch_time: f64, +) -> Result<bool, AdapterError> { + Ok(parse_token(pid, token)?.matches_launch_time(launch_time)) +} + +pub(crate) fn require_core( + process: &agent_desktop_core::ProcessIdentity, +) -> Result<ProcessIdentity, AdapterError> { + let expected = parse_token(to_pid_t(process.pid)?, &process.instance)?; + if expected.still_matches()? { + Ok(expected) + } else { + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Target process instance is no longer running", + ) + .with_details(serde_json::json!({ + "pid": process.pid, + "process_instance": process.instance, + "complete": false, + }))) + } +} + +pub(crate) fn to_pid_t(pid: ProcessId) -> Result<i32, AdapterError> { + i32::try_from(pid).map_err(|_| { + AdapterError::new( + ErrorCode::InvalidArgs, + format!("Process id {pid} exceeds the macOS pid_t range"), + ) + .with_details(serde_json::json!({ + "pid": pid, + "max_pid": i32::MAX, + "complete": false, + "retryable": false, + })) + }) +} + +pub(crate) fn from_pid_t(pid: i32) -> Result<ProcessId, AdapterError> { + ProcessId::try_from(pid).map_err(|_| { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("macOS returned invalid process id {pid}"), + ) + .with_details(serde_json::json!({ + "pid": pid, + "complete": false, + "retryable": false, + })) + }) +} + +fn parse_token(pid: i32, token: &str) -> Result<ProcessIdentity, AdapterError> { + let mut parts = token.split(':'); + let prefix = parts.next(); + let seconds = parts.next().and_then(|value| value.parse::<u64>().ok()); + let microseconds = parts.next().and_then(|value| value.parse::<u64>().ok()); + if prefix != Some(TOKEN_PREFIX) + || seconds.is_none() + || microseconds.is_none() + || parts.next().is_some() + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Malformed macOS process instance token", + )); + } + Ok(ProcessIdentity { + pid, + start_seconds: seconds.unwrap_or_default(), + start_microseconds: microseconds.unwrap_or_default(), + }) +} + +#[cfg(target_os = "macos")] +fn classify_missing_or_inaccessible( + pid: i32, + process_error: std::io::Error, +) -> Result<Option<ProcessIdentity>, AdapterError> { + match process_error.raw_os_error() { + Some(libc::ESRCH) => return Ok(None), + Some(libc::EPERM) => return Err(process_identity_permission_error(pid, process_error)), + _ => {} + } + let probe = unsafe { libc::kill(pid, 0) }; + if probe == 0 { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + format!("libproc could not read the live process identity for pid {pid}"), + ) + .with_platform_detail(process_error.to_string())); + } + let error = std::io::Error::last_os_error(); + match error.raw_os_error() { + Some(libc::ESRCH) => Ok(None), + Some(libc::EPERM) => Err(process_identity_permission_error(pid, error)), + _ => Err(AdapterError::new( + ErrorCode::AppUnresponsive, + format!("Could not determine whether pid {pid} is still live"), + ) + .with_platform_detail(error.to_string())), + } +} + +#[cfg(target_os = "macos")] +fn process_identity_permission_error(pid: i32, error: std::io::Error) -> AdapterError { + AdapterError::new( + ErrorCode::PermDenied, + format!("Permission denied reading process identity for pid {pid}"), + ) + .with_platform_detail(error.to_string()) + .with_details(serde_json::json!({ + "kind": "process_identity_permission", + "source": "libproc", + "operation": "PROC_PIDTBSDINFO", + "pid": pid, + "complete": false, + "retryable": false, + })) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::*; + + #[test] + fn current_process_token_roundtrips_and_matches() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let token = token_for_pid(pid).unwrap().expect("current process token"); + + assert!(matches_instance(pid, &token).unwrap()); + } + + #[test] + fn malformed_token_is_not_treated_as_a_process_match() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let error = matches_instance(pid, "broken").expect_err("malformed token must fail closed"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + } + + #[test] + fn core_pid_conversion_is_checked_at_the_macos_boundary() { + let max_pid_t = ProcessId::new(u32::try_from(i32::MAX).unwrap()); + assert_eq!(to_pid_t(max_pid_t).unwrap(), i32::MAX); + + let overflow = to_pid_t(ProcessId::new(u32::MAX)).unwrap_err(); + assert_eq!(overflow.code, ErrorCode::InvalidArgs); + + let invalid_native = from_pid_t(-1).unwrap_err(); + assert_eq!(invalid_native.code, ErrorCode::AppUnresponsive); + } + + #[test] + fn missing_process_has_no_identity() { + assert!(ProcessIdentity::capture(999_999).unwrap().is_none()); + } + + #[test] + fn launch_time_match_rejects_a_reused_pid_generation() { + let identity = ProcessIdentity { + pid: 42, + start_seconds: 1_700_000_000, + start_microseconds: 250_000, + }; + + assert!(identity.matches_launch_time(1_700_000_000.25)); + assert!(!identity.matches_launch_time(1_700_000_006.0)); + assert!(!identity.matches_launch_time(0.0)); + } + + #[test] + fn instance_launch_time_match_reuses_the_process_token_contract() { + let identity = ProcessIdentity { + pid: 42, + start_seconds: 1_700_000_000, + start_microseconds: 250_000, + }; + let token = identity.token(); + + assert!(instance_matches_launch_time(42, &token, 1_700_000_000.25).unwrap()); + assert!(!instance_matches_launch_time(42, &token, 1_700_000_006.0).unwrap()); + } + + #[test] + fn libproc_permission_error_is_structured_for_owner_diagnostics() { + let error = + classify_missing_or_inaccessible(418, std::io::Error::from_raw_os_error(libc::EPERM)) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::PermDenied); + let details = error.details.unwrap(); + assert_eq!(details["kind"], "process_identity_permission"); + assert_eq!(details["source"], "libproc"); + assert_eq!(details["pid"], 418); + assert_eq!(details["retryable"], false); + } +} diff --git a/crates/macos/src/system/process_state.rs b/crates/macos/src/system/process_state.rs new file mode 100644 index 0000000..15f88f3 --- /dev/null +++ b/crates/macos/src/system/process_state.rs @@ -0,0 +1,115 @@ +use agent_desktop_core::AdapterError; +use agent_desktop_core::process_state::ProcessState; + +/// Result of one AX responsiveness read, decoupled from the raw AXError so +/// `classify` (below) is testable without a live accessibility tree. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AxProbeResult { + Responsive, + CannotComplete, +} + +/// Pure classification: alive/dead + a probe closure in, `ProcessState` out. +/// Kept free of any platform API so the retry threshold (one transient +/// `CannotComplete` must not classify `Unresponsive`; two consecutive must) +/// is unit-testable on every host, not just macOS with a live AX tree. +pub(crate) fn classify( + pid_alive: bool, + mut probe: impl FnMut() -> Result<AxProbeResult, AdapterError>, +) -> Result<ProcessState, AdapterError> { + if !pid_alive { + return Ok(ProcessState::Exited { code: None }); + } + Ok(match probe()? { + AxProbeResult::Responsive => ProcessState::Running, + AxProbeResult::CannotComplete => match probe()? { + AxProbeResult::Responsive => ProcessState::Running, + AxProbeResult::CannotComplete => ProcessState::Unresponsive, + }, + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn process_state_impl( + process: agent_desktop_core::ProcessIdentity, + deadline: agent_desktop_core::Deadline, +) -> Result<ProcessState, AdapterError> { + use crate::tree::element_for_pid; + + let pid = crate::system::process_identity::to_pid_t(process.pid)?; + if !crate::system::process_identity::matches_instance(pid, &process.instance)? { + return Ok(ProcessState::Exited { code: None }); + } + let app = element_for_pid(pid); + let state = classify(true, || { + prepare_probe(&app, deadline)?; + Ok(ax_probe(&app, deadline)) + })?; + if crate::system::process_identity::matches_instance(pid, &process.instance)? { + Ok(state) + } else { + Ok(ProcessState::Exited { code: None }) + } +} + +#[cfg(target_os = "macos")] +fn prepare_probe( + app: &crate::tree::AXElement, + deadline: agent_desktop_core::Deadline, +) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(app, deadline) +} + +/// `kill(pid, 0)`-style liveness check: signal 0 sends no actual signal, the +/// kernel only validates the target exists and is reachable. Mirrors the +/// convention already used by `system::force_close::signal_result`. +#[cfg(all(test, target_os = "macos"))] +fn pid_is_alive(pid: i32) -> bool { + const POSIX_ESRCH: i32 = 3; + + if pid <= 0 { + return false; + } + unsafe extern "C" { + fn kill(pid: i32, sig: i32) -> i32; + } + if unsafe { kill(pid, 0) } == 0 { + return true; + } + std::io::Error::last_os_error().raw_os_error() != Some(POSIX_ESRCH) +} + +#[cfg(target_os = "macos")] +fn ax_probe(app: &crate::tree::AXElement, deadline: agent_desktop_core::Deadline) -> AxProbeResult { + use accessibility_sys::{kAXErrorCannotComplete, kAXErrorSuccess, kAXRoleAttribute}; + use core_foundation::{ + base::{CFType, TCFType}, + string::CFString, + }; + + if app.0.is_null() { + return AxProbeResult::Responsive; + } + let cf_attr = CFString::new(kAXRoleAttribute); + let (err, value) = + crate::tree::ax_ipc::copy_attribute_value(app, cf_attr.as_concrete_TypeRef(), deadline); + if err == kAXErrorCannotComplete { + return AxProbeResult::CannotComplete; + } + if err == kAXErrorSuccess && !value.is_null() { + unsafe { CFType::wrap_under_create_rule(value) }; + } + AxProbeResult::Responsive +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn process_state_impl( + _process: agent_desktop_core::ProcessIdentity, + _deadline: agent_desktop_core::Deadline, +) -> Result<ProcessState, AdapterError> { + Err(AdapterError::not_supported("process_state")) +} + +#[cfg(test)] +#[path = "process_state_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/process_state_tests.rs b/crates/macos/src/system/process_state_tests.rs new file mode 100644 index 0000000..628822d --- /dev/null +++ b/crates/macos/src/system/process_state_tests.rs @@ -0,0 +1,112 @@ +use super::*; +use agent_desktop_core::process_state::ProcessState; +use std::cell::Cell; + +#[test] +fn dead_pid_classifies_exited_without_consulting_probe() { + let probe_calls = Cell::new(0); + let state = classify(false, || { + probe_calls.set(probe_calls.get() + 1); + Ok(AxProbeResult::Responsive) + }) + .unwrap(); + assert_eq!(state, ProcessState::Exited { code: None }); + assert_eq!( + probe_calls.get(), + 0, + "a dead pid must short-circuit before probing AX at all" + ); +} + +#[test] +fn single_transient_cannot_complete_does_not_classify_unresponsive() { + let calls = Cell::new(0); + let state = classify(true, || { + let n = calls.get() + 1; + calls.set(n); + if n == 1 { + Ok(AxProbeResult::CannotComplete) + } else { + Ok(AxProbeResult::Responsive) + } + }) + .unwrap(); + assert_eq!( + state, + ProcessState::Running, + "one transient AX blip on a healthy-but-busy app must not hard-fail as Unresponsive" + ); + assert_eq!(calls.get(), 2, "a transient failure retries exactly once"); +} + +#[test] +fn two_consecutive_cannot_complete_classifies_unresponsive() { + let calls = Cell::new(0); + let state = classify(true, || { + calls.set(calls.get() + 1); + Ok(AxProbeResult::CannotComplete) + }) + .unwrap(); + assert_eq!(state, ProcessState::Unresponsive); + assert_eq!( + calls.get(), + 2, + "classification requires exactly a second consecutive CannotComplete, not more" + ); +} + +#[test] +fn immediately_responsive_pid_classifies_running_with_one_probe() { + let calls = Cell::new(0); + let state = classify(true, || { + calls.set(calls.get() + 1); + Ok(AxProbeResult::Responsive) + }) + .unwrap(); + assert_eq!(state, ProcessState::Running); + assert_eq!(calls.get(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn exited_child_process_is_classified_exited_with_no_code() { + let mut child = std::process::Command::new("/bin/echo") + .arg("hi") + .spawn() + .expect("spawn /bin/echo"); + let pid = i32::try_from(child.id()).expect("child pid fits macOS pid_t"); + let instance = crate::system::process_identity::token_for_pid(pid) + .expect("capture child process identity") + .expect("child remains live before wait"); + child.wait().expect("wait for exit"); + for _ in 0..50 { + if !pid_is_alive(pid) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + + let state = process_state_impl( + agent_desktop_core::ProcessIdentity { + pid: agent_desktop_core::ProcessId::try_from(pid).unwrap(), + instance, + }, + agent_desktop_core::Deadline::after(1_000).unwrap(), + ) + .expect("process_state_impl should not error"); + assert_eq!(state, ProcessState::Exited { code: None }); +} + +#[cfg(target_os = "macos")] +#[test] +fn nonpositive_pid_is_never_alive() { + assert!(!pid_is_alive(0)); + assert!(!pid_is_alive(-1)); +} + +#[cfg(target_os = "macos")] +#[test] +fn currently_running_pid_is_alive() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + assert!(pid_is_alive(pid)); +} diff --git a/crates/macos/src/system/process_tests.rs b/crates/macos/src/system/process_tests.rs new file mode 100644 index 0000000..dc1b733 --- /dev/null +++ b/crates/macos/src/system/process_tests.rs @@ -0,0 +1,130 @@ +use super::*; + +const PROCESS_TIMEOUT: Duration = Duration::from_secs(1); +const PROCESS_TEST_LIMIT: Duration = Duration::from_secs(2); + +fn process_exists(process: i32) -> bool { + unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; + } + unsafe { kill(process, 0) == 0 } +} + +#[test] +fn successful_process_returns_output() { + let mut command = Command::new("/bin/echo"); + command.arg("ok"); + let output = run_with_timeout(&mut command, "echo", Duration::from_secs(1)).unwrap(); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +fn slow_process_is_killed_within_the_absolute_deadline() { + let mut command = Command::new("/bin/sleep"); + command.arg("5"); + let started = Instant::now(); + let error = run_with_timeout(&mut command, "sleep", PROCESS_TIMEOUT) + .expect_err("slow process must time out"); + + assert_eq!(error.code, ErrorCode::Timeout); + assert!( + error.platform_detail.is_none(), + "subprocess cleanup failed: {:?}", + error.platform_detail + ); + assert!(started.elapsed() < PROCESS_TEST_LIMIT); +} + +#[test] +fn descendant_holding_stdout_is_killed_with_its_process_group() { + let mut command = Command::new("/bin/sh"); + command.args(["-c", "trap '' TERM; sleep 60 >&1 & exit 0"]); + let started = Instant::now(); + let error = run_with_timeout(&mut command, "pipe-holder", PROCESS_TIMEOUT) + .expect_err("inherited pipe must not outlive the deadline"); + + assert_eq!(error.code, ErrorCode::Timeout); + assert!( + error.platform_detail.is_none(), + "subprocess cleanup failed: {:?}", + error.platform_detail + ); + assert!(started.elapsed() < PROCESS_TEST_LIMIT); +} + +#[test] +fn injected_group_kill_failure_is_reported_without_blocking_wait() { + let mut command = Command::new("/bin/sleep"); + command.arg("60"); + configure_process_group(&mut command); + let mut child = command.spawn().unwrap(); + let process_group = i32::try_from(child.id()).expect("child pid fits macOS pid_t"); + let deadline = Instant::now() + Duration::from_millis(30); + let failures = terminate_process_group_with(&mut child, process_group, deadline, |_, _| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected failure", + )) + }); + let _ = child.kill(); + + assert!( + failures + .iter() + .any(|failure| failure.contains("injected failure")) + ); +} + +#[test] +fn zombie_only_group_does_not_turn_sigkill_eperm_into_cleanup_failure() { + let mut command = Command::new("/usr/bin/true"); + configure_process_group(&mut command); + let mut child = command.spawn().unwrap(); + let process_group = i32::try_from(child.id()).expect("child pid fits macOS pid_t"); + std::thread::sleep(Duration::from_millis(25)); + let mut signals = Vec::new(); + let deadline = Instant::now() + Duration::from_millis(100); + let failures = + terminate_process_group_with(&mut child, process_group, deadline, |group, signal| { + signals.push(signal); + match signal { + 15 => Ok(true), + 9 => { + assert!(process_exists(group)); + Err(std::io::Error::from_raw_os_error(POSIX_EPERM)) + } + 0 => Ok(false), + _ => unreachable!(), + } + }); + + assert_eq!(signals, [15, 9, 0]); + assert!(failures.is_empty()); + assert!(child.try_wait().unwrap().is_some()); +} + +#[test] +fn oversized_output_is_capped_with_stderr_context() { + let mut command = Command::new("/bin/sh"); + command.args(["-c", "echo diagnostic-marker >&2; yes X | head -c 9000000"]); + let error = run_with_timeout(&mut command, "oversized", Duration::from_secs(5)) + .expect_err("oversized output must fail"); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert!( + error.details.unwrap()["stderr_tail"] + .as_str() + .is_some_and(|tail| tail.contains("diagnostic-marker")) + ); +} + +#[test] +fn output_context_keeps_latest_bytes() { + let mut tail = vec![b'a'; OUTPUT_CONTEXT_BYTES]; + append_tail(&mut tail, b"final diagnostic"); + + assert_eq!(tail.len(), OUTPUT_CONTEXT_BYTES); + assert!(tail.ends_with(b"final diagnostic")); +} diff --git a/crates/macos/src/system/renderer_activation.rs b/crates/macos/src/system/renderer_activation.rs new file mode 100644 index 0000000..ddf2491 --- /dev/null +++ b/crates/macos/src/system/renderer_activation.rs @@ -0,0 +1,61 @@ +use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics, ErrorCode, ProcessIdentity}; + +pub(crate) fn activate(process: ProcessIdentity, deadline: Deadline) -> Result<(), AdapterError> { + let identity = + crate::system::process_identity::require_core(&process).map_err(not_delivered)?; + let application = crate::tree::element_for_pid(identity.pid()); + let current = crate::tree::surface_read::boolean( + &application, + "AXManualAccessibility", + instant(deadline).map_err(not_delivered)?, + ) + .map_err(not_delivered)?; + if current == Some(true) { + return Ok(()); + } + set_manual_accessibility(&application, deadline) +} + +fn set_manual_accessibility( + application: &crate::tree::AXElement, + deadline: Deadline, +) -> Result<(), AdapterError> { + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; + + let attribute = CFString::new("AXManualAccessibility"); + let error = crate::tree::ax_ipc::set_attribute_value( + application, + attribute.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + deadline, + )?; + let delivered = crate::actions::ax_mutation::classify_result( + application, + "AXManualAccessibility", + "AXUIElementSetAttributeValue", + error, + )?; + if !delivered { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "Renderer does not support AXManualAccessibility activation", + ) + .with_disposition(DeliverySemantics::not_delivered())); + } + if deadline.is_expired() { + return Err(deadline + .timeout_error() + .with_disposition(DeliverySemantics::delivered_unverified())); + } + Ok(()) +} + +fn instant(deadline: Deadline) -> Result<std::time::Instant, AdapterError> { + std::time::Instant::now() + .checked_add(deadline.remaining_slice(std::time::Duration::from_millis(250))?) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "Deadline is out of range")) +} + +fn not_delivered(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::not_delivered()) +} diff --git a/crates/macos/src/system/screen_bridge.m b/crates/macos/src/system/screen_bridge.m new file mode 100644 index 0000000..3da4195 --- /dev/null +++ b/crates/macos/src/system/screen_bridge.m @@ -0,0 +1,35 @@ +#import <AppKit/AppKit.h> + +typedef struct { + double x; + double y; + double width; + double height; +} AgentDesktopScreenRect; + +bool agent_desktop_visible_frame(uint32_t displayID, AgentDesktopScreenRect *output) { + if (output == NULL) { + return false; + } + @try { + @autoreleasepool { + for (NSScreen *screen in NSScreen.screens) { + NSNumber *number = screen.deviceDescription[@"NSScreenNumber"]; + if (number == nil || number.unsignedIntValue != displayID) { + continue; + } + NSRect visible = screen.visibleFrame; + CGRect mainBounds = CGDisplayBounds(CGMainDisplayID()); + output->x = visible.origin.x; + output->y = mainBounds.size.height - NSMaxY(visible); + output->width = visible.size.width; + output->height = visible.size.height; + return true; + } + return false; + } + } @catch (NSException *exception) { + (void)exception; + return false; + } +} diff --git a/crates/macos/src/system/screen_bridge_contract_tests.rs b/crates/macos/src/system/screen_bridge_contract_tests.rs new file mode 100644 index 0000000..f37054b --- /dev/null +++ b/crates/macos/src/system/screen_bridge_contract_tests.rs @@ -0,0 +1,8 @@ +#[test] +fn objective_c_screen_boundary_contains_exceptions_and_temporary_objects() { + let source = include_str!("screen_bridge.m"); + + assert!(source.contains("@try")); + assert!(source.contains("@autoreleasepool")); + assert!(source.contains("@catch (NSException *exception)")); +} diff --git a/crates/macos/src/system/screenshot.rs b/crates/macos/src/system/screenshot.rs index 7f0ec6f..8d35eed 100644 --- a/crates/macos/src/system/screenshot.rs +++ b/crates/macos/src/system/screenshot.rs @@ -1,51 +1,119 @@ use agent_desktop_core::{ - adapter::{ImageBuffer, ImageFormat}, - error::{AdapterError, ErrorCode}, + AdapterError, Deadline, DisplayInfo, ErrorCode, ImageBuffer, ImageFormat, WindowInfo, + parse_png_dimensions, }; #[cfg(target_os = "macos")] mod imp { use super::*; - use std::os::unix::fs::DirBuilderExt; + use std::ffi::OsString; + use std::io::Read; + use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::atomic::{AtomicU64, Ordering}; - use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use std::time::Instant; const SCREENCAPTURE: &str = "/usr/sbin/screencapture"; - #[cfg(not(test))] - const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(5); - #[cfg(test)] - const SCREENSHOT_TIMEOUT: Duration = Duration::from_millis(20); + const MAX_PNG_BYTES: u64 = 64 * 1024 * 1024; + const MAX_IMAGE_PIXELS: u64 = 100_000_000; static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(0); - fn capture(window_id: Option<u32>) -> Result<ImageBuffer, AdapterError> { + fn capture( + scale_factor: f64, + deadline: Deadline, + arguments: impl FnOnce(&Path) -> Result<Vec<OsString>, AdapterError>, + ) -> Result<ImageBuffer, AdapterError> { + ensure_budget(deadline)?; let temp = TempPng::new()?; let mut command = Command::new(SCREENCAPTURE); - command.args(["-x", "-t", "png"]); - - if let Some(wid) = window_id { - command.args(["-l", &wid.to_string()]); - } - - command.arg(temp.path()); - let output = run_screencapture(&mut command)?; - + command.args(arguments(temp.path())?); + let output = run_screencapture(&mut command, deadline)?; if !output.status.success() { return Err(map_screencapture_error(&output)); } - - read_png(temp.path()) + let mut buffer = read_png(temp.path(), deadline)?; + buffer.scale_factor = scale_factor; + Ok(buffer) } - pub fn capture_app(pid: i32) -> Result<ImageBuffer, AdapterError> { - tracing::debug!("system: screenshot app pid={pid}"); - capture(find_cg_window_id_for_pid(pid)) + fn base_args() -> Vec<OsString> { + vec![ + OsString::from("-x"), + OsString::from("-t"), + OsString::from("png"), + ] } - pub fn capture_screen(_idx: usize) -> Result<ImageBuffer, AdapterError> { - tracing::debug!("system: screenshot screen"); - capture(None) + pub(super) fn display_args(index: usize, output: &Path) -> Result<Vec<OsString>, AdapterError> { + let display_number = index.checked_add(1).ok_or_else(|| { + AdapterError::new(ErrorCode::InvalidArgs, "display index is too large") + })?; + let mut args = base_args(); + args.push(OsString::from("-D")); + args.push(OsString::from(display_number.to_string())); + args.push(output.as_os_str().to_owned()); + Ok(args) + } + + pub(super) fn window_args(window_id: u32, output: &Path) -> Vec<OsString> { + let mut args = base_args(); + args.push(OsString::from("-l")); + args.push(OsString::from(window_id.to_string())); + args.push(output.as_os_str().to_owned()); + args + } + + pub fn capture_screen(index: usize, deadline: Deadline) -> Result<ImageBuffer, AdapterError> { + ensure_budget(deadline)?; + let expected = crate::system::display::display_at(index, deadline)?; + ensure_budget(deadline)?; + capture_display(index, &expected, deadline) + } + + pub fn capture_display( + index: usize, + expected: &DisplayInfo, + deadline: Deadline, + ) -> Result<ImageBuffer, AdapterError> { + ensure_budget(deadline)?; + let (raw_index, current) = crate::system::display::capture_selection(expected, deadline)?; + ensure_budget(deadline)?; + verify_display_identity(index, expected, ¤t)?; + let captured = capture(current.scale, deadline, |path| { + display_args(raw_index, path) + })?; + ensure_budget(deadline)?; + let after = crate::system::display::display_at_capture_index(raw_index, deadline)?; + ensure_budget(deadline)?; + verify_display_identity(index, ¤t, &after)?; + Ok(captured) + } + + pub fn capture_window( + window: &WindowInfo, + deadline: Deadline, + ) -> Result<ImageBuffer, AdapterError> { + if window.process_instance.is_none() { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Exact window screenshot requires a process instance token", + ) + .with_suggestion("Refresh the target with 'list-windows', then retry")); + } + let verified = crate::system::window_resolve::resolve_window_strict( + window, + deadline_instant(deadline)?, + )?; + let window_id = parse_window_id(&verified.id)?; + let scale = crate::system::display::scale_for_bounds(verified.bounds, deadline)?; + ensure_budget(deadline)?; + let captured = capture(scale, deadline, |path| Ok(window_args(window_id, path)))?; + crate::system::window_resolve::resolve_window_strict( + &verified, + deadline_instant(deadline)?, + )?; + Ok(captured) } struct TempPng { @@ -56,11 +124,17 @@ mod imp { impl TempPng { fn new() -> Result<Self, AdapterError> { let mut dir = std::env::temp_dir(); - dir.push(format!("agent-desktop-screenshot-{}", unique_suffix())); + dir.push(format!( + "agent-desktop-screenshot-{}-{}", + std::process::id(), + NEXT_TEMP_ID.fetch_add(1, Ordering::Relaxed) + )); std::fs::DirBuilder::new() .mode(0o700) .create(&dir) - .map_err(|e| AdapterError::internal(format!("create screenshot temp dir: {e}")))?; + .map_err(|error| { + AdapterError::internal(format!("create screenshot temp dir: {error}")) + })?; let path = dir.join("capture.png"); Ok(Self { dir, path }) } @@ -72,40 +146,105 @@ mod imp { impl Drop for TempPng { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - let _ = std::fs::remove_dir(&self.dir); + if let Err(error) = std::fs::remove_file(&self.path) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::debug!(%error, path = %self.path.display(), "screenshot file cleanup failed"); + } + if let Err(error) = std::fs::remove_dir(&self.dir) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::debug!(%error, path = %self.dir.display(), "screenshot directory cleanup failed"); + } } } - fn unique_suffix() -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let seq = NEXT_TEMP_ID.fetch_add(1, Ordering::Relaxed); - format!("{}-{nanos}-{seq}", std::process::id()) + pub(super) fn run_screencapture( + command: &mut Command, + deadline: Deadline, + ) -> Result<Output, AdapterError> { + crate::system::process::run_with_deadline( + command, + "screencapture", + deadline_instant(deadline)?, + ) } - fn run_screencapture(command: &mut Command) -> Result<Output, AdapterError> { - crate::system::process::run_with_timeout(command, "screencapture", SCREENSHOT_TIMEOUT) - } - - fn read_png(path: &Path) -> Result<ImageBuffer, AdapterError> { - let data = std::fs::read(path) - .map_err(|e| AdapterError::internal(format!("read screenshot: {e}")))?; - let (width, height) = png_dimensions(&data); + fn read_png(path: &Path, deadline: Deadline) -> Result<ImageBuffer, AdapterError> { + ensure_budget(deadline)?; + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path) + .map_err(|error| AdapterError::internal(format!("open screenshot: {error}")))?; + let metadata = file + .metadata() + .map_err(|error| AdapterError::internal(format!("stat screenshot: {error}")))?; + if !metadata.file_type().is_file() { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "screencapture output is not a regular file", + )); + } + validate_png_size(metadata.len())?; + let capacity = usize::try_from(metadata.len()) + .map_err(|_| AdapterError::new(ErrorCode::ActionFailed, "screenshot is too large"))?; + let mut data = Vec::new(); + data.try_reserve_exact(capacity).map_err(|_| { + AdapterError::new(ErrorCode::ActionFailed, "screenshot allocation failed") + })?; + file.take(MAX_PNG_BYTES + 1) + .read_to_end(&mut data) + .map_err(|error| AdapterError::internal(format!("read screenshot: {error}")))?; + validate_png_size(data.len() as u64)?; + ensure_budget(deadline)?; + let (width, height) = parse_png_dimensions(&data).ok_or_else(|| { + AdapterError::new( + ErrorCode::ActionFailed, + "screencapture returned an invalid PNG payload", + ) + })?; + validate_pixel_count(width, height)?; Ok(ImageBuffer { data, format: ImageFormat::Png, width, height, + scale_factor: 1.0, }) } - fn map_screencapture_error(output: &Output) -> AdapterError { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - let combined = format!("{stderr}\n{stdout}"); + pub(super) fn validate_png_size(bytes: u64) -> Result<(), AdapterError> { + if (24..=MAX_PNG_BYTES).contains(&bytes) { + Ok(()) + } else { + Err(AdapterError::new( + ErrorCode::ActionFailed, + "screenshot PNG size is outside the supported 24-byte to 64-MiB range", + )) + } + } + + pub(super) fn validate_pixel_count(width: u32, height: u32) -> Result<(), AdapterError> { + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .ok_or_else(|| AdapterError::new(ErrorCode::ActionFailed, "pixel count overflowed"))?; + if width > 0 && height > 0 && pixels <= MAX_IMAGE_PIXELS { + Ok(()) + } else { + Err(AdapterError::new( + ErrorCode::ActionFailed, + "screenshot exceeds the 100-megapixel image budget", + )) + } + } + + pub(super) fn map_screencapture_error(output: &Output) -> AdapterError { + let combined = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stderr), + String::from_utf8_lossy(&output.stdout) + ); let lower = combined.to_lowercase(); if lower.contains("screen recording") || lower.contains("not authorized") @@ -114,88 +253,62 @@ mod imp { { return AdapterError::new(ErrorCode::PermDenied, "Screen Recording permission denied") .with_suggestion( - "Open System Settings > Privacy & Security > Screen Recording and add the app that launches agent-desktop. If macOS lists the built binary separately, add that binary too.", + "Open System Settings > Privacy & Security > Screen Recording and add the app that launches agent-desktop.", ) .with_platform_detail(combined.trim()); } - let detail = combined.trim(); - let detail = if detail.is_empty() { - "screencapture produced no diagnostic output" + AdapterError::internal("screencapture exited with error").with_platform_detail( + if detail.is_empty() { + "screencapture produced no diagnostic output" + } else { + detail + }, + ) + } + + pub(super) fn verify_display_identity( + index: usize, + expected: &DisplayInfo, + current: &DisplayInfo, + ) -> Result<(), AdapterError> { + if expected.id == current.id { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!( + "Display at index {index} changed from '{}' to '{}'", + expected.id, current.id + ), + ) + .with_suggestion("Run 'list-displays' to refresh display indexes, then retry.")) + } + + fn parse_window_id(id: &str) -> Result<u32, AdapterError> { + crate::system::window_resolve::parse_window_number(id) + .and_then(|value| u32::try_from(value).ok()) + .filter(|value| *value > 0) + .ok_or_else(|| { + AdapterError::new(ErrorCode::InvalidArgs, format!("Invalid window id: '{id}'")) + }) + } + + fn deadline_instant(deadline: Deadline) -> Result<Instant, AdapterError> { + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(deadline.timeout_error()); + } + Instant::now() + .checked_add(remaining) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "deadline is out of range")) + } + + fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) } else { - detail - }; - AdapterError::internal("screencapture exited with error").with_platform_detail(detail) - } - - fn png_dimensions(data: &[u8]) -> (u32, u32) { - if data.len() < 24 { - return (0, 0); - } - let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); - let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); - (w, h) - } - - fn find_cg_window_id_for_pid(pid: i32) -> Option<u32> { - let mut best_id: Option<u32> = None; - let mut best_area: f64 = 0.0; - - for record in crate::system::cg_window::visible_window_records() { - if record.pid != pid { - continue; - } - - if record.area > best_area { - best_area = record.area; - best_id = Some(record.window_number as u32); - } - } - - best_id - } - - #[cfg(test)] - mod tests { - use super::*; - use std::os::unix::process::ExitStatusExt; - use std::process::ExitStatus; - - fn output(stderr: &str) -> Output { - Output { - status: ExitStatus::from_raw(1 << 8), - stdout: Vec::new(), - stderr: stderr.as_bytes().to_vec(), - } - } - - #[test] - fn maps_screen_recording_error_to_permission_denied() { - let err = map_screencapture_error(&output("Screen Recording is not authorized")); - - assert_eq!(err.code, ErrorCode::PermDenied); - assert!(err.suggestion.is_some()); - } - - #[test] - fn maps_unknown_screencapture_error_to_internal() { - let err = map_screencapture_error(&output("display server unavailable")); - - assert_eq!(err.code, ErrorCode::Internal); - assert_eq!( - err.platform_detail.as_deref(), - Some("display server unavailable") - ); - } - - #[test] - fn run_screencapture_kills_timed_out_process() { - let mut command = Command::new("/bin/sleep"); - command.arg("10"); - - let err = run_screencapture(&mut command).unwrap_err(); - - assert_eq!(err.code, ErrorCode::Timeout); + Ok(()) } } } @@ -204,13 +317,28 @@ mod imp { mod imp { use super::*; - pub fn capture_app(_pid: i32) -> Result<ImageBuffer, AdapterError> { - Err(AdapterError::not_supported("capture_app")) + pub fn capture_screen(_index: usize, _deadline: Deadline) -> Result<ImageBuffer, AdapterError> { + Err(AdapterError::not_supported("capture_screen")) } - pub fn capture_screen(_idx: usize) -> Result<ImageBuffer, AdapterError> { - Err(AdapterError::not_supported("capture_screen")) + pub fn capture_display( + _index: usize, + _expected: &DisplayInfo, + _deadline: Deadline, + ) -> Result<ImageBuffer, AdapterError> { + Err(AdapterError::not_supported("capture_display")) + } + + pub fn capture_window( + _window: &WindowInfo, + _deadline: Deadline, + ) -> Result<ImageBuffer, AdapterError> { + Err(AdapterError::not_supported("capture_window")) } } -pub use imp::{capture_app, capture_screen}; +pub(crate) use imp::{capture_display, capture_screen, capture_window}; + +#[cfg(all(test, target_os = "macos"))] +#[path = "screenshot_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/screenshot_tests.rs b/crates/macos/src/system/screenshot_tests.rs new file mode 100644 index 0000000..530b474 --- /dev/null +++ b/crates/macos/src/system/screenshot_tests.rs @@ -0,0 +1,113 @@ +use super::imp::{ + display_args, map_screencapture_error, run_screencapture, validate_pixel_count, + validate_png_size, verify_display_identity, window_args, +}; +use agent_desktop_core::{Deadline, DisplayInfo, ErrorCode, Rect}; +use std::ffi::OsString; +use std::os::unix::process::ExitStatusExt; +use std::path::Path; +use std::process::{Command, ExitStatus, Output}; + +fn output(stderr: &str) -> Output { + Output { + status: ExitStatus::from_raw(1 << 8), + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + } +} + +#[test] +fn maps_screen_recording_error_to_permission_denied() { + let err = map_screencapture_error(&output("Screen Recording is not authorized")); + + assert_eq!(err.code, ErrorCode::PermDenied); + assert!(err.suggestion.is_some()); +} + +#[test] +fn maps_unknown_screencapture_error_to_internal() { + let err = map_screencapture_error(&output("display server unavailable")); + + assert_eq!(err.code, ErrorCode::Internal); + assert_eq!( + err.platform_detail.as_deref(), + Some("display server unavailable") + ); +} + +#[test] +fn run_screencapture_kills_timed_out_process() { + let mut command = Command::new("/bin/sleep"); + command.arg("10"); + + let deadline = Deadline::after(20).expect("deadline"); + let err = run_screencapture(&mut command, deadline).unwrap_err(); + + assert_eq!(err.code, ErrorCode::Timeout); +} + +#[test] +fn display_argv_uses_one_based_display_selector_without_a_rect() { + let args = display_args(1, Path::new("/tmp/capture.png")).expect("display args"); + + assert_eq!( + args, + ["-x", "-t", "png", "-D", "2", "/tmp/capture.png"].map(OsString::from) + ); + assert!(!args.contains(&OsString::from("-R"))); +} + +#[test] +fn window_argv_keeps_the_requested_cg_window_number() { + let args = window_args(314, Path::new("/tmp/capture.png")); + + assert_eq!( + args, + ["-x", "-t", "png", "-l", "314", "/tmp/capture.png"].map(OsString::from) + ); +} + +#[test] +fn display_identity_mismatch_is_rejected() { + let expected = display("100", 2.0); + let current = display("200", 1.0); + + let error = + verify_display_identity(1, &expected, ¤t).expect_err("changed display identity"); + + assert_eq!(error.code, ErrorCode::InvalidArgs); + assert!(error.message.contains("100")); + assert!(error.message.contains("200")); +} + +#[test] +fn rejects_png_payloads_outside_the_bounded_size_range() { + let too_small = validate_png_size(23).expect_err("short PNG"); + let too_large = validate_png_size(64 * 1024 * 1024 + 1).expect_err("large PNG"); + + assert_eq!(too_small.code, ErrorCode::ActionFailed); + assert_eq!(too_large.code, ErrorCode::ActionFailed); +} + +#[test] +fn rejects_images_outside_the_bounded_pixel_range() { + let zero = validate_pixel_count(0, 100).expect_err("zero-width image"); + let too_large = validate_pixel_count(10_001, 10_001).expect_err("large image"); + + assert_eq!(zero.code, ErrorCode::ActionFailed); + assert_eq!(too_large.code, ErrorCode::ActionFailed); +} + +fn display(id: &str, scale: f64) -> DisplayInfo { + DisplayInfo { + id: id.into(), + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + is_primary: id == "100", + scale, + } +} diff --git a/crates/macos/src/system/signals.rs b/crates/macos/src/system/signals.rs new file mode 100644 index 0000000..7a37c89 --- /dev/null +++ b/crates/macos/src/system/signals.rs @@ -0,0 +1,167 @@ +use agent_desktop_core::{ + AdapterError, AppInfo, SignalBaseline, SignalCompleteness, SignalFilter, SnapshotSurface, + SurfaceSignal, WindowFilter, +}; +use std::time::{Duration, Instant}; + +#[cfg(target_os = "macos")] +pub(crate) fn supported_surfaces_impl() -> Vec<SnapshotSurface> { + vec![ + SnapshotSurface::Window, + SnapshotSurface::Focused, + SnapshotSurface::Menu, + SnapshotSurface::Menubar, + SnapshotSurface::Sheet, + SnapshotSurface::Popover, + SnapshotSurface::Alert, + ] +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn supported_surfaces_impl() -> Vec<SnapshotSurface> { + vec![SnapshotSurface::Window] +} + +#[cfg(target_os = "macos")] +pub(crate) fn capture_signal_baseline_impl( + filter: &SignalFilter, + deadline: Instant, +) -> Result<SignalBaseline, AdapterError> { + ensure_before_deadline(deadline)?; + let apps = matching_apps(filter, deadline)?; + cap_ax_messaging(&apps, deadline)?; + let windows = if filter.app.is_some() && apps.is_empty() { + Vec::new() + } else { + crate::system::app_inventory::list_windows_until( + &WindowFilter { + focused_only: false, + app: filter.app.clone(), + }, + deadline, + )? + }; + ensure_before_deadline(deadline)?; + let surfaces = surfaces_for_apps(filter, &apps, deadline)?; + ensure_before_deadline(deadline)?; + Ok(SignalBaseline { + windows, + apps, + surfaces, + completeness: SignalCompleteness::complete(), + }) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn capture_signal_baseline_impl( + _filter: &SignalFilter, + _deadline: Instant, +) -> Result<SignalBaseline, AdapterError> { + Err(AdapterError::not_supported("capture_signal_baseline")) +} + +#[cfg(target_os = "macos")] +fn matching_apps(filter: &SignalFilter, deadline: Instant) -> Result<Vec<AppInfo>, AdapterError> { + let all = crate::system::app_inventory::list_apps_complete_until(deadline)?; + ensure_before_deadline(deadline)?; + Ok(filter_apps(filter, all)) +} + +#[cfg(target_os = "macos")] +fn filter_apps(filter: &SignalFilter, all: Vec<AppInfo>) -> Vec<AppInfo> { + if let Some(name) = &filter.app { + return all + .into_iter() + .filter(|app| app.name.eq_ignore_ascii_case(name)) + .collect(); + } + if let Some(process) = &filter.process { + return all + .into_iter() + .filter(|app| { + app.pid == process.pid + && app.process_instance.as_deref() == Some(process.instance.as_str()) + }) + .collect(); + } + all +} + +#[cfg(target_os = "macos")] +fn cap_ax_messaging(apps: &[AppInfo], deadline: Instant) -> Result<(), AdapterError> { + for app in apps { + remaining_before_deadline(deadline)?; + let pid = crate::system::process_identity::to_pid_t(app.pid)?; + let element = crate::tree::element_for_pid(pid); + crate::tree::attributes::set_messaging_timeout(&element, deadline)?; + } + ensure_before_deadline(deadline) +} + +#[cfg(target_os = "macos")] +fn surfaces_for_apps( + filter: &SignalFilter, + apps: &[AppInfo], + deadline: Instant, +) -> Result<Vec<SurfaceSignal>, AdapterError> { + if filter.app.is_none() && filter.process.is_none() { + return Ok(Vec::new()); + } + let mut surfaces = Vec::new(); + for app in apps { + ensure_before_deadline(deadline)?; + let pid = crate::system::process_identity::to_pid_t(app.pid)?; + for info in crate::tree::surface_inventory::list_surfaces_for_pid(pid, deadline)? { + if let Some(kind) = map_surface_kind(&info.kind) { + let process_instance = app.process_instance.clone().ok_or_else(|| { + AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Surface owner lacks a verified process instance", + ) + .with_details(serde_json::json!({ + "pid": app.pid, + "complete": false, + })) + })?; + surfaces.push(SurfaceSignal { + kind, + app: app.name.clone(), + pid: app.pid, + process_instance, + id: info.id, + title: info.title, + }); + } + } + } + Ok(surfaces) +} + +#[cfg(target_os = "macos")] +fn map_surface_kind(raw: &str) -> Option<SnapshotSurface> { + match raw { + "sheet" => Some(SnapshotSurface::Sheet), + "popover" => Some(SnapshotSurface::Popover), + "alert" => Some(SnapshotSurface::Alert), + "menu" | "context_menu" => Some(SnapshotSurface::Menu), + _ => None, + } +} + +#[cfg(target_os = "macos")] +fn remaining_before_deadline(deadline: Instant) -> Result<Duration, AdapterError> { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(AdapterError::timeout("Signal baseline capture timed out")); + } + Ok(remaining) +} + +#[cfg(target_os = "macos")] +fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { + remaining_before_deadline(deadline).map(|_| ()) +} + +#[cfg(test)] +#[path = "signals_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/signals_tests.rs b/crates/macos/src/system/signals_tests.rs new file mode 100644 index 0000000..f41496c --- /dev/null +++ b/crates/macos/src/system/signals_tests.rs @@ -0,0 +1,103 @@ +use super::*; + +#[test] +fn supported_surfaces_are_only_the_implemented_macos_contract() { + assert_eq!( + supported_surfaces_impl(), + vec![ + SnapshotSurface::Window, + SnapshotSurface::Focused, + SnapshotSurface::Menu, + SnapshotSurface::Menubar, + SnapshotSurface::Sheet, + SnapshotSurface::Popover, + SnapshotSurface::Alert, + ] + ); +} + +#[test] +fn map_surface_kind_covers_known_shapes() { + assert_eq!(map_surface_kind("sheet"), Some(SnapshotSurface::Sheet)); + assert_eq!(map_surface_kind("popover"), Some(SnapshotSurface::Popover)); + assert_eq!(map_surface_kind("alert"), Some(SnapshotSurface::Alert)); + assert_eq!(map_surface_kind("menu"), Some(SnapshotSurface::Menu)); + assert_eq!( + map_surface_kind("context_menu"), + Some(SnapshotSurface::Menu) + ); + assert_eq!(map_surface_kind("something_else"), None); +} + +#[test] +fn matching_apps_filters_by_name_case_insensitively() { + let filter = SignalFilter { + app: Some("textedit".into()), + process: None, + }; + let apps = filter_apps( + &filter, + vec![ + AppInfo { + name: "TextEdit".into(), + pid: agent_desktop_core::ProcessId::new(42), + bundle_id: None, + process_instance: None, + }, + AppInfo { + name: "Finder".into(), + pid: agent_desktop_core::ProcessId::new(7), + bundle_id: None, + process_instance: None, + }, + ], + ); + + assert_eq!(apps.len(), 1); + assert_eq!(apps[0].pid, 42); +} + +#[test] +fn app_filter_with_no_constraints_preserves_the_complete_inventory() { + let filter = SignalFilter::default(); + let apps = vec![AppInfo { + name: "Finder".into(), + pid: agent_desktop_core::ProcessId::new(7), + bundle_id: None, + process_instance: None, + }]; + + let filtered = filter_apps(&filter, apps); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].name, "Finder"); + assert_eq!(filtered[0].pid, 7); +} + +#[test] +fn surfaces_for_apps_is_empty_without_an_app_or_pid_filter() { + let filter = SignalFilter::default(); + let apps = vec![AppInfo { + name: "Finder".into(), + pid: agent_desktop_core::ProcessId::new(1), + bundle_id: None, + process_instance: None, + }]; + let deadline = Instant::now() + Duration::from_secs(5); + let surfaces = + surfaces_for_apps(&filter, &apps, deadline).expect("unscoped inventory succeeds"); + assert!( + surfaces.is_empty(), + "unscoped waits must not walk every running app's AX tree for surfaces" + ); +} + +#[test] +fn baseline_capture_rejects_an_expired_deadline() { + let deadline = Instant::now() - Duration::from_millis(1); + + let error = capture_signal_baseline_impl(&SignalFilter::default(), deadline) + .expect_err("an expired baseline capture must time out"); + + assert_eq!(error.code.as_str(), "TIMEOUT"); +} diff --git a/crates/macos/src/system/wait.rs b/crates/macos/src/system/wait.rs index f41a804..9326313 100644 --- a/crates/macos/src/system/wait.rs +++ b/crates/macos/src/system/wait.rs @@ -1,53 +1,39 @@ -use agent_desktop_core::error::AdapterError; -use std::time::{Duration, Instant}; +use agent_desktop_core::{AdapterError, Deadline, ProcessIdentity}; #[cfg(target_os = "macos")] -mod imp { - use super::*; +pub(crate) fn wait_for_menu( + process: ProcessIdentity, + open: bool, + deadline: Deadline, +) -> Result<(), AdapterError> { use crate::tree::surfaces::is_menu_open; + use std::time::Duration; - const DEFAULT_MENU_TIMEOUT_MS: u64 = 750; - const MAX_MENU_TIMEOUT_MS: u64 = 10_000; - - pub fn wait_for_menu(pid: i32, open: bool, timeout_ms: u64) -> Result<(), AdapterError> { - let deadline = Instant::now() + Duration::from_millis(timeout_ms); - loop { - if is_menu_open(pid) == open { - return Ok(()); - } - if Instant::now() >= deadline { - let msg = if open { - format!("No context menu opened within {timeout_ms}ms") - } else { - format!("Context menu did not close within {timeout_ms}ms") - }; - return Err(AdapterError::timeout(msg)); - } - std::thread::sleep(Duration::from_millis(50)); + loop { + let identity = crate::system::process_identity::require_core(&process)?; + let instant = crate::tree::locator_deadline::from_operation(deadline)?; + if is_menu_open(identity.pid(), instant)? == open { + crate::system::process_identity::require_core(&process)?; + return Ok(()); } - } - - pub fn menu_timeout_ms() -> u64 { - std::env::var("AGENT_DESKTOP_MENU_TIMEOUT_MS") - .ok() - .and_then(|raw| raw.parse::<u64>().ok()) - .filter(|ms| *ms > 0) - .map(|ms| ms.min(MAX_MENU_TIMEOUT_MS)) - .unwrap_or(DEFAULT_MENU_TIMEOUT_MS) + if deadline.is_expired() { + let message = if open { + "No context menu opened before the deadline" + } else { + "Context menu did not close before the deadline" + }; + return Err(deadline.timeout_error().with_platform_detail(message)); + } + let pause = deadline.remaining_slice(Duration::from_millis(50))?; + std::thread::sleep(pause); } } #[cfg(not(target_os = "macos"))] -mod imp { - use super::*; - - pub fn wait_for_menu(_pid: i32, _open: bool, _timeout_ms: u64) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("wait_for_menu")) - } - - pub fn menu_timeout_ms() -> u64 { - 750 - } +pub(crate) fn wait_for_menu( + _process: ProcessIdentity, + _open: bool, + _deadline: Deadline, +) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("wait_for_menu")) } - -pub use imp::{menu_timeout_ms, wait_for_menu}; diff --git a/crates/macos/src/system/window_ax_state.rs b/crates/macos/src/system/window_ax_state.rs new file mode 100644 index 0000000..7cc6e79 --- /dev/null +++ b/crates/macos/src/system/window_ax_state.rs @@ -0,0 +1,113 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use rustc_hash::FxHashMap; +use std::time::Instant; + +type AxWindowIdentity = (Option<String>, Option<i64>); + +#[derive(Clone, Debug)] +pub(crate) struct WindowAxState { + pub(crate) focused: Option<AxWindowIdentity>, + pub(crate) minimized_by_id: FxHashMap<i64, bool>, +} + +pub(crate) fn read_until(pid: i32, deadline: Instant) -> Result<WindowAxState, AdapterError> { + let app = crate::tree::element_for_pid(pid); + let focused = focused_identity(&app, pid, deadline)?; + let mut minimized_by_id = FxHashMap::default(); + for window in crate::tree::surface_read::elements(&app, "AXWindows", deadline)? { + if crate::tree::surface_read::string(&window, "AXRole", deadline)?.as_deref() + != Some("AXWindow") + { + continue; + } + let window_id = + match crate::system::window_resolve::ax_window_id_with_deadline(&window, deadline) { + Ok(window_id) => window_id, + Err(error) if crate::system::window_bridge::is_unavailable(&error) => break, + Err(error) => return Err(error), + }; + let Some(window_id) = window_id else { + continue; + }; + if let Some(minimized) = + crate::tree::surface_read::boolean(&window, "AXMinimized", deadline)? + { + minimized_by_id.insert(window_id, minimized); + } + } + Ok(WindowAxState { + focused, + minimized_by_id, + }) +} + +pub(crate) fn read_frontmost_until( + pid: i32, + deadline: Instant, +) -> Result<WindowAxState, AdapterError> { + let app = crate::tree::element_for_pid(pid); + Ok(WindowAxState { + focused: focused_identity(&app, pid, deadline)?, + minimized_by_id: FxHashMap::default(), + }) +} + +fn focused_identity( + app: &crate::tree::AXElement, + pid: i32, + deadline: Instant, +) -> Result<Option<AxWindowIdentity>, AdapterError> { + if crate::tree::surface_read::boolean(app, "AXFrontmost", deadline)? != Some(true) { + return Ok(None); + } + let Some(focused) = crate::tree::surface_read::element(app, "AXFocusedWindow", deadline)? + else { + return Ok(None); + }; + let role = crate::tree::surface_read::string(&focused, "AXRole", deadline)?; + let window = if role.as_deref() == Some("AXWindow") { + focused + } else if role.as_deref().is_some_and(is_transient_surface_role) { + let Some(window) = crate::tree::surface_read::element(&focused, "AXWindow", deadline)? + else { + return Ok(None); + }; + if crate::tree::surface_read::string(&window, "AXRole", deadline)?.as_deref() + != Some("AXWindow") + { + return Ok(None); + } + window + } else { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + "Focused accessibility object was not a window", + ) + .with_details(serde_json::json!({ "pid": pid, "complete": false, "role": role }))); + }; + let title = crate::tree::surface_read::string(&window, "AXTitle", deadline)?; + let number = match crate::system::window_resolve::ax_window_id_with_deadline(&window, deadline) + { + Ok(number) => number, + Err(error) if crate::system::window_bridge::is_unavailable(&error) => None, + Err(error) => return Err(error), + }; + Ok(Some((title, number))) +} + +fn is_transient_surface_role(role: &str) -> bool { + matches!(role, "AXSheet" | "AXDialog" | "AXAlert" | "AXPopover") +} + +#[cfg(test)] +mod tests { + use super::is_transient_surface_role; + + #[test] + fn modal_surfaces_can_temporarily_own_focused_window() { + for role in ["AXSheet", "AXDialog", "AXAlert", "AXPopover"] { + assert!(is_transient_surface_role(role)); + } + assert!(!is_transient_surface_role("AXButton")); + } +} diff --git a/crates/macos/src/system/window_bridge.rs b/crates/macos/src/system/window_bridge.rs new file mode 100644 index 0000000..14ad76d --- /dev/null +++ b/crates/macos/src/system/window_bridge.rs @@ -0,0 +1,149 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; + +use crate::tree::AXElement; + +#[cfg(target_os = "macos")] +type WindowBridge = unsafe extern "C" fn(accessibility_sys::AXUIElementRef, *mut u32) -> i32; + +#[cfg(target_os = "macos")] +pub(crate) fn window_id( + window: &AXElement, + deadline: std::time::Instant, +) -> Result<Option<i64>, AdapterError> { + crate::tree::ax_ipc::prepare(window, deadline)?; + let bridge = bridge().ok_or_else(bridge_unavailable)?; + let mut window_id = 0_u32; + let error = unsafe { bridge(window.0, &mut window_id) }; + classify(error).map(|present| present.then_some(i64::from(window_id))) +} + +#[cfg(target_os = "macos")] +pub(crate) fn is_unavailable(error: &AdapterError) -> bool { + error + .details + .as_ref() + .and_then(|details| details.get("kind")) + .and_then(serde_json::Value::as_str) + == Some("resolution_window_bridge_unavailable") +} + +#[cfg(target_os = "macos")] +fn bridge() -> Option<WindowBridge> { + static BRIDGE: std::sync::OnceLock<Option<WindowBridge>> = std::sync::OnceLock::new(); + *BRIDGE.get_or_init(|| { + let symbol = unsafe { dlsym(RTLD_DEFAULT, c"_AXUIElementGetWindow".as_ptr()) }; + if symbol.is_null() { + None + } else { + Some(unsafe { std::mem::transmute::<*mut std::ffi::c_void, WindowBridge>(symbol) }) + } + }) +} + +#[cfg(target_os = "macos")] +const RTLD_DEFAULT: *mut std::ffi::c_void = -2_isize as *mut std::ffi::c_void; + +#[cfg(target_os = "macos")] +unsafe extern "C" { + fn dlsym( + handle: *mut std::ffi::c_void, + symbol: *const std::ffi::c_char, + ) -> *mut std::ffi::c_void; +} + +#[cfg(target_os = "macos")] +fn classify(error: i32) -> Result<bool, AdapterError> { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, kAXErrorNoValue, kAXErrorSuccess, + }; + + if error == kAXErrorSuccess { + return Ok(true); + } + if error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue { + return Ok(false); + } + if error == kAXErrorAPIDisabled { + return Err( + AdapterError::permission_denied().with_details(serde_json::json!({ + "kind": "resolution_window_bridge", + "ax_error": error, + "complete": false, + "retryable": false, + })), + ); + } + let label = if error == kAXErrorCannotComplete { + "kAXErrorCannotComplete" + } else if error == kAXErrorInvalidUIElement { + "kAXErrorInvalidUIElement" + } else { + "unclassified AXError" + }; + Err(AdapterError::new( + ErrorCode::AppUnresponsive, + format!("Could not bridge AXWindow to CGWindow: {label}"), + ) + .with_suggestion("Retry after the application finishes updating its window inventory") + .with_details(serde_json::json!({ + "kind": "resolution_window_bridge", + "ax_error": error, + "complete": false, + "retryable": true, + }))) +} + +#[cfg(target_os = "macos")] +fn bridge_unavailable() -> AdapterError { + AdapterError::new( + ErrorCode::ActionNotSupported, + "The optional AX-to-CG window bridge is unavailable on this macOS version", + ) + .with_suggestion("Target the uniquely titled verified window, or refresh the window inventory") + .with_details(serde_json::json!({ + "kind": "resolution_window_bridge_unavailable", + "symbol": "_AXUIElementGetWindow", + "complete": false, + "retryable": false, + })) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn window_id( + _window: &AXElement, + _deadline: std::time::Instant, +) -> Result<Option<i64>, AdapterError> { + Ok(None) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn is_unavailable(_error: &AdapterError) -> bool { + false +} + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::*; + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + + #[test] + fn bridge_errors_preserve_permission_and_incomplete_states() { + assert_eq!( + classify(kAXErrorAPIDisabled).unwrap_err().code, + ErrorCode::PermDenied + ); + for error in [kAXErrorCannotComplete, kAXErrorInvalidUIElement] { + let classified = classify(error).unwrap_err(); + assert_eq!(classified.code, ErrorCode::AppUnresponsive); + assert_eq!(classified.details.unwrap()["complete"], false); + } + } + + #[test] + fn unavailable_symbol_is_machine_distinguishable() { + assert!(is_unavailable(&bridge_unavailable())); + } +} diff --git a/crates/macos/src/system/window_inventory.rs b/crates/macos/src/system/window_inventory.rs index 373ace0..a483e28 100644 --- a/crates/macos/src/system/window_inventory.rs +++ b/crates/macos/src/system/window_inventory.rs @@ -1,16 +1,12 @@ -use agent_desktop_core::{ - adapter::WindowFilter, - node::{AppInfo, WindowInfo}, -}; -use std::time::Duration; +use agent_desktop_core::{AdapterError, WindowFilter, WindowInfo}; +use std::time::Instant; use crate::system::cg_window; -pub(crate) fn visible_apps() -> Vec<AppInfo> { - apps_from_window_records(&cg_window::visible_window_records()) -} - -fn apps_from_window_records(records: &[cg_window::WindowRecord]) -> Vec<AppInfo> { +#[cfg(test)] +fn apps_from_window_records( + records: &[cg_window::WindowRecord], +) -> Vec<agent_desktop_core::AppInfo> { let mut seen_pids = rustc_hash::FxHashSet::default(); let mut apps = Vec::new(); @@ -18,201 +14,167 @@ fn apps_from_window_records(records: &[cg_window::WindowRecord]) -> Vec<AppInfo> if !seen_pids.insert(record.pid) { continue; } - apps.push(AppInfo { + apps.push(agent_desktop_core::AppInfo { name: record.app_name.clone(), - pid: record.pid, + pid: agent_desktop_core::ProcessId::try_from(record.pid) + .expect("fixture pid must be positive"), bundle_id: None, + process_instance: record.process_instance.clone(), }); } apps } -pub(crate) fn list_windows( +pub(crate) fn list_windows_until( filter: &WindowFilter, - app_for_name: impl FnMut(&str, &[AppInfo]) -> Option<AppInfo>, -) -> Vec<WindowInfo> { - list_windows_with_sources( - filter, - app_for_name, - cg_window::visible_window_records, - ax_window_for_app, - std::thread::sleep, + deadline: Instant, +) -> Result<Vec<WindowInfo>, AdapterError> { + let app_filter = filter.app.as_deref().unwrap_or("").to_ascii_lowercase(); + if app_filter.is_empty() { + return crate::system::window_inventory_global::list_windows_until( + filter.focused_only, + deadline, + ); + } + let scope = cg_window::WindowRecordScope::App(&app_filter); + let records = cg_window::window_records_until(deadline, scope)?; + let candidates = records; + windows_from_records_with_focus( + candidates, + filter.focused_only, + |pid| crate::system::window_ax_state::read_until(pid, deadline), + crate::system::process_identity::matches_instance, ) } -fn list_windows_with_sources( - filter: &WindowFilter, - mut app_for_name: impl FnMut(&str, &[AppInfo]) -> Option<AppInfo>, - mut visible_records: impl FnMut() -> Vec<cg_window::WindowRecord>, - mut ax_window_for_app: impl FnMut(&AppInfo) -> Option<WindowInfo>, - mut sleep: impl FnMut(Duration), -) -> Vec<WindowInfo> { - let mut app = None; - let mut app_loaded = false; - - for attempt in 0..3 { - let records = visible_records(); - let windows = visible_windows_from_records(filter, &records); - if !windows.is_empty() { - return windows; - } - - if let Some(app_name) = filter.app.as_deref() { - if !app_loaded { - let visible_apps = apps_from_window_records(&records); - app = app_for_name(app_name, &visible_apps); - app_loaded = true; - } - if let Some(app) = app.as_ref() { - if let Some(window) = ax_window_for_app(app) { - if !filter.focused_only || window.is_focused { - return vec![window]; - } - } - } - } - - if attempt == 2 || !should_retry_empty(filter, app.as_ref()) { - break; - } - - sleep(Duration::from_millis(50)); +pub(crate) fn exact_window_for_pid_until( + pid: i32, + deadline: Instant, +) -> Result<WindowInfo, AdapterError> { + let records = + cg_window::window_records_until(deadline, cg_window::WindowRecordScope::Pid(pid))?; + let mut windows = windows_from_records_with_focus( + records, + false, + |owner_pid| crate::system::window_ax_state::read_until(owner_pid, deadline), + |owner_pid, instance| { + crate::system::process_identity::matches_instance(owner_pid, instance) + }, + )?; + if windows.len() == 1 { + return Ok(windows.remove(0)); } - - Vec::new() -} - -fn visible_windows_from_records( - filter: &WindowFilter, - records: &[cg_window::WindowRecord], -) -> Vec<WindowInfo> { - let app_filter = filter.app.as_deref().unwrap_or("").to_ascii_lowercase(); - let candidates = records + let mut focused = windows .iter() - .filter(|record| matches_app_filter(&record.app_name, &app_filter)) + .filter(|window| window.state.is_focused) .cloned() - .collect(); - - windows_from_records_with_focus(candidates, filter.focused_only, focused_window_identity) + .collect::<Vec<_>>(); + if focused.len() == 1 { + return Ok(focused.remove(0)); + } + if windows.is_empty() { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::WindowNotFound, + format!("Process {pid} has no visible window"), + )); + } + Err(AdapterError::ambiguous_target(format!( + "Process {pid} has multiple visible windows and no exact focused window" + )) + .with_details(serde_json::json!({ + "candidate_count": windows.len(), + "candidate_window_ids": windows.iter().map(|window| &window.id).collect::<Vec<_>>(), + }))) } fn windows_from_records_with_focus( records: Vec<cg_window::WindowRecord>, focused_only: bool, - mut focused_identity: impl FnMut(i32) -> FocusedWindowIdentity, -) -> Vec<WindowInfo> { - let candidates: Vec<_> = records + mut ax_state: impl FnMut(i32) -> Result<crate::system::window_ax_state::WindowAxState, AdapterError>, + mut verify_instance: impl FnMut(i32, &str) -> Result<bool, AdapterError>, +) -> Result<Vec<WindowInfo>, AdapterError> { + let candidates = records .into_iter() - .map(|record| { - let title = record.title.unwrap_or_else(|| record.app_name.clone()); - (record.app_name, title, record.pid, record.window_number) - }) - .collect(); + .filter(|record| record.window_number > 0) + .collect::<Vec<_>>(); let mut title_counts = std::collections::HashMap::new(); - for (_, title, pid, _) in &candidates { - *title_counts.entry((*pid, title.clone())).or_insert(0) += 1; + for record in &candidates { + *title_counts + .entry((record.pid, record.display_title().to_owned())) + .or_insert(0) += 1; } - let mut focus_cache = std::collections::HashMap::new(); + let mut state_cache = std::collections::HashMap::new(); let mut windows = Vec::new(); let mut focused_seen = false; - for (app_name, title, pid, window_number) in candidates { + for record in candidates { + let process_instance = record.process_instance.as_deref().ok_or_else(|| { + identity_race_error( + record.pid, + "CoreGraphics record omitted its process instance", + ) + })?; + if !verify_instance(record.pid, process_instance)? { + return Err(identity_race_error( + record.pid, + "window owner changed before accessibility focus read", + )); + } let title_count = title_counts - .get(&(pid, title.clone())) + .get(&(record.pid, record.display_title().to_owned())) .copied() .unwrap_or(0); - let identity = focus_cache - .entry(pid) - .or_insert_with(|| focused_identity(pid)); - let is_focused = - !focused_seen && matches_focused_window(&title, window_number, identity, title_count); + let state = match state_cache.entry(record.pid) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => entry.insert(ax_state(record.pid)?), + }; + if !verify_instance(record.pid, process_instance)? { + return Err(identity_race_error( + record.pid, + "window owner changed after accessibility focus read", + )); + } + let is_focused = !focused_seen + && matches_focused_window( + record.display_title(), + record.window_number, + &state.focused, + title_count, + ); if focused_only && !is_focused { continue; } focused_seen |= is_focused; - - windows.push(WindowInfo { - id: format!("w-{window_number}"), - title, - app: app_name, - pid, - bounds: None, - is_focused, - }); + let minimized = state.minimized_by_id.get(&record.window_number).copied(); + windows.push(record.into_window_info(is_focused, minimized)?); } - windows + Ok(windows) } +#[cfg(test)] fn matches_app_filter(app_name: &str, app_filter: &str) -> bool { app_filter.is_empty() || app_name.eq_ignore_ascii_case(app_filter) } -fn should_retry_empty(filter: &WindowFilter, app: Option<&AppInfo>) -> bool { - filter.app.is_none() || app.is_some() -} - -fn ax_window_for_app(app_info: &AppInfo) -> Option<WindowInfo> { - let app = crate::tree::element_for_pid(app_info.pid); - let window = focused_window_element(&app) - .or_else(|| crate::tree::copy_element_attr(&app, "AXMainWindow")) - .or_else(|| { - let windows = crate::tree::copy_ax_array(&app, "AXWindows")?; - child_bearing_window_index(&windows, |window| { - crate::tree::element::count_children(window, None) - }) - .map(|index| windows[index].clone()) - .or_else(|| windows.into_iter().next()) - })?; - if crate::tree::copy_string_attr(&window, "AXRole").as_deref() != Some("AXWindow") { - return None; - } - let title = - crate::tree::copy_string_attr(&window, "AXTitle").unwrap_or_else(|| app_info.name.clone()); - let window_number = crate::tree::copy_i64_attr(&window, "AXWindowNumber").unwrap_or(0); - let is_focused = crate::tree::copy_bool_attr(&app, "AXFrontmost") == Some(true); - Some(ax_window_info(app_info, title, window_number, is_focused)) -} - -fn ax_window_info( - app_info: &AppInfo, - title: String, - window_number: i64, - is_focused: bool, -) -> WindowInfo { - WindowInfo { - id: format!("w-{window_number}"), - title, - app: app_info.name.clone(), - pid: app_info.pid, - bounds: None, - is_focused, - } -} - -fn child_bearing_window_index<T>( - windows: &[T], - mut count_children: impl FnMut(&T) -> u32, -) -> Option<usize> { - windows.iter().position(|window| count_children(window) > 0) +fn identity_race_error(pid: i32, phase: &str) -> AdapterError { + AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Window owner changed while exact window identity was being assembled", + ) + .with_details(serde_json::json!({ + "kind": "window_identity_race", + "pid": pid, + "phase": phase, + "complete": false, + "retryable": true, + })) } type FocusedWindowIdentity = Option<(Option<String>, Option<i64>)>; -fn focused_window_identity(pid: i32) -> FocusedWindowIdentity { - let app = crate::tree::element_for_pid(pid); - if crate::tree::copy_bool_attr(&app, "AXFrontmost") != Some(true) { - return None; - } - let window = focused_window_element(&app)?; - Some(( - crate::tree::copy_string_attr(&window, "AXTitle"), - crate::tree::copy_i64_attr(&window, "AXWindowNumber"), - )) -} - fn matches_focused_window( title: &str, window_number: i64, @@ -228,33 +190,6 @@ fn matches_focused_window( focused_title.as_deref() == Some(title) && same_title_count == 1 } -fn focused_window_element(app: &crate::tree::AXElement) -> Option<crate::tree::AXElement> { - let focused = crate::tree::copy_element_attr(app, "AXFocusedWindow")?; - window_ancestor(focused, 4) -} - -fn window_ancestor( - mut element: crate::tree::AXElement, - max_depth: usize, -) -> Option<crate::tree::AXElement> { - for _ in 0..=max_depth { - if is_window_element(&element) { - return Some(element); - } - if let Some(window) = crate::tree::copy_element_attr(&element, "AXWindow") { - if is_window_element(&window) { - return Some(window); - } - } - element = crate::tree::copy_element_attr(&element, "AXParent")?; - } - None -} - -fn is_window_element(element: &crate::tree::AXElement) -> bool { - crate::tree::copy_string_attr(element, "AXRole").as_deref() == Some("AXWindow") -} - #[cfg(test)] #[path = "window_inventory_tests.rs"] mod tests; diff --git a/crates/macos/src/system/window_inventory_global.rs b/crates/macos/src/system/window_inventory_global.rs new file mode 100644 index 0000000..fe3f200 --- /dev/null +++ b/crates/macos/src/system/window_inventory_global.rs @@ -0,0 +1,330 @@ +use agent_desktop_core::{AdapterError, ErrorCode, WindowInfo}; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::time::Instant; + +use crate::system::{cg_window::WindowRecord, window_ax_state::WindowAxState}; + +impl OwnerSnapshotView for crate::system::workspace_apps::WindowOwnerSnapshot { + fn eligible_pids(&self) -> FxHashSet<i32> { + crate::system::workspace_apps::WindowOwnerSnapshot::eligible_pids(self) + } + + fn generation(&self, pid: i32) -> Option<OwnerGeneration> { + self.owner(pid).map(|owner| match owner.launch_time { + Some(launch_time) => OwnerGeneration::LaunchTime(launch_time), + None => OwnerGeneration::LiveProcess, + }) + } + + fn frontmost_pid(&self) -> Option<i32> { + self.frontmost().map(|owner| owner.pid) + } + + fn same_generation(&self, other: &Self) -> bool { + crate::system::workspace_apps::WindowOwnerSnapshot::same_generation(self, other) + } +} + +pub(super) trait OwnerSnapshotView { + fn eligible_pids(&self) -> FxHashSet<i32>; + fn generation(&self, pid: i32) -> Option<OwnerGeneration>; + fn frontmost_pid(&self) -> Option<i32>; + fn same_generation(&self, other: &Self) -> bool; +} + +#[derive(Clone, Copy)] +pub(super) enum OwnerGeneration { + LaunchTime(f64), + LiveProcess, +} + +pub(crate) fn list_windows_until( + focused_only: bool, + deadline: Instant, +) -> Result<Vec<WindowInfo>, AdapterError> { + stabilize_global_with(deadline, || capture_global_once(focused_only, deadline)) +} + +fn capture_global_once( + focused_only: bool, + deadline: Instant, +) -> Result<Vec<WindowInfo>, AdapterError> { + capture_global_with( + focused_only, + || crate::system::workspace_apps::window_owner_snapshot_until(deadline), + |eligible_pids| { + crate::system::cg_window::window_records_until( + deadline, + crate::system::cg_window::WindowRecordScope::Pids(eligible_pids), + ) + }, + |pid| crate::system::window_ax_state::read_frontmost_until(pid, deadline), + ) +} + +fn capture_global_with<S: OwnerSnapshotView>( + focused_only: bool, + mut capture_owners: impl FnMut() -> Result<S, AdapterError>, + mut capture_records: impl FnMut(&FxHashSet<i32>) -> Result<Vec<WindowRecord>, AdapterError>, + mut read_frontmost: impl FnMut(i32) -> Result<WindowAxState, AdapterError>, +) -> Result<Vec<WindowInfo>, AdapterError> { + let before = capture_owners()?; + let eligible_pids = OwnerSnapshotView::eligible_pids(&before); + let records = capture_records(&eligible_pids)?; + let assembled = + assemble_global_windows(records.clone(), &before, focused_only, &mut read_frontmost); + let after = capture_owners()?; + validate_snapshot_pair(&before, &after, &records)?; + assembled +} + +pub(super) fn assemble_global_windows<S: OwnerSnapshotView>( + records: Vec<WindowRecord>, + owners: &S, + focused_only: bool, + mut read_frontmost: impl FnMut(i32) -> Result<WindowAxState, AdapterError>, +) -> Result<Vec<WindowInfo>, AdapterError> { + let records = records + .into_iter() + .filter(|record| record.window_number > 0) + .collect::<Vec<_>>(); + validate_record_generations(&records, owners)?; + let frontmost_pid = owners.frontmost_pid(); + let frontmost_window_owner = + frontmost_pid.filter(|pid| records.iter().any(|record| record.pid == *pid)); + let focus_state = match frontmost_window_owner { + Some(pid) => Some(read_frontmost(pid).map_err(frontmost_read_error)?), + None => None, + }; + let focused_window = match matching_focus_windows(&records, frontmost_pid, focus_state.as_ref()) + { + FocusJoin::TitleAmbiguous => None, + FocusJoin::Windows(windows) => { + if frontmost_window_owner.is_some() && windows.len() != 1 { + return Err(focus_join_error(frontmost_pid, windows.len())); + } + windows.first().copied() + } + }; + records + .into_iter() + .filter_map(|record| { + let is_focused = Some((record.pid, record.window_number)) == focused_window; + if focused_only && !is_focused { + return None; + } + let minimized = if Some(record.pid) == frontmost_pid { + focus_state + .as_ref() + .and_then(|state| state.minimized_by_id.get(&record.window_number).copied()) + } else { + None + }; + Some(record.into_window_info(is_focused, minimized)) + }) + .collect::<Result<Vec<_>, _>>() +} + +pub(super) fn validate_snapshot_pair<S: OwnerSnapshotView>( + before: &S, + after: &S, + records: &[WindowRecord], +) -> Result<(), AdapterError> { + if !before.same_generation(after) { + return Err(owner_churn_error(None, "appkit_snapshot_changed")); + } + validate_record_generations(records, after) +} + +pub(super) fn stabilize_global_with( + deadline: Instant, + mut capture: impl FnMut() -> Result<Vec<WindowInfo>, AdapterError>, +) -> Result<Vec<WindowInfo>, AdapterError> { + let mut attempts = 0_u64; + let mut last_error = None; + loop { + if Instant::now() >= deadline { + return Err(global_timeout(attempts, last_error.as_ref())); + } + attempts += 1; + match capture() { + Ok(windows) => return Ok(windows), + Err(error) if retryable_global_error(&error) => { + last_error = Some(error); + } + Err(error) => return Err(error), + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if !remaining.is_zero() { + std::thread::sleep(remaining.min(std::time::Duration::from_millis(5))); + } + } +} + +fn validate_record_generations<S: OwnerSnapshotView>( + records: &[WindowRecord], + owners: &S, +) -> Result<(), AdapterError> { + for (pid, instance) in unique_process_instances(records)? { + let generation = owners + .generation(pid) + .ok_or_else(|| owner_churn_error(Some(pid), "owner_not_eligible"))?; + let matches = match generation { + OwnerGeneration::LaunchTime(launch_time) => { + crate::system::process_identity::instance_matches_launch_time( + pid, + instance, + launch_time, + )? + } + OwnerGeneration::LiveProcess => { + crate::system::process_identity::matches_instance(pid, instance)? + } + }; + if !matches { + return Err(owner_churn_error(Some(pid), "process_generation_changed")); + } + } + Ok(()) +} + +fn unique_process_instances(records: &[WindowRecord]) -> Result<Vec<(i32, &str)>, AdapterError> { + let mut instances = FxHashMap::default(); + for record in records { + let instance = record + .process_instance + .as_deref() + .filter(|instance| !instance.is_empty()) + .ok_or_else(|| owner_churn_error(Some(record.pid), "process_instance_missing"))?; + match instances.entry(record.pid) { + std::collections::hash_map::Entry::Occupied(entry) => { + if *entry.get() != instance { + return Err(owner_churn_error( + Some(record.pid), + "process_instance_conflict", + )); + } + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(instance); + } + } + } + let mut instances = instances.into_iter().collect::<Vec<_>>(); + instances.sort_unstable_by_key(|(pid, _)| *pid); + Ok(instances) +} + +enum FocusJoin { + Windows(Vec<(i32, i64)>), + TitleAmbiguous, +} + +fn matching_focus_windows( + records: &[WindowRecord], + frontmost_pid: Option<i32>, + state: Option<&WindowAxState>, +) -> FocusJoin { + let Some(frontmost_pid) = frontmost_pid else { + return FocusJoin::Windows(Vec::new()); + }; + let Some((focused_title, focused_number)) = state.and_then(|state| state.focused.as_ref()) + else { + return FocusJoin::Windows(Vec::new()); + }; + if let Some(number) = focused_number { + return FocusJoin::Windows( + records + .iter() + .filter(|record| record.pid == frontmost_pid && record.window_number == *number) + .map(|record| (record.pid, record.window_number)) + .collect(), + ); + } + let title_matches = records + .iter() + .filter(|record| { + record.pid == frontmost_pid && focused_title.as_deref() == Some(record.display_title()) + }) + .map(|record| (record.pid, record.window_number)) + .collect::<Vec<_>>(); + if title_matches.len() > 1 { + return FocusJoin::TitleAmbiguous; + } + FocusJoin::Windows(title_matches) +} + +fn frontmost_read_error(error: AdapterError) -> AdapterError { + if !matches!( + error.code, + ErrorCode::Timeout | ErrorCode::AppUnresponsive | ErrorCode::ElementNotFound + ) { + return error; + } + let source_kind = error + .details + .as_ref() + .and_then(|details| details.get("kind")) + .cloned(); + AdapterError::new( + ErrorCode::AppUnresponsive, + "Authoritative frontmost window accessibility read was incomplete", + ) + .with_details(serde_json::json!({ + "kind": "frontmost_window_ax_incomplete", + "source_code": error.code.as_str(), + "source_kind": source_kind, + "complete": false, + "retryable": true, + })) +} + +fn focus_join_error(pid: Option<i32>, match_count: usize) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Authoritative frontmost app did not join to one exact window", + ) + .with_details(serde_json::json!({ + "kind": "frontmost_window_join", + "pid": pid, + "match_count": match_count, + "complete": false, + "retryable": true, + })) +} + +fn owner_churn_error(pid: Option<i32>, phase: &str) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Window owner identity changed during global inventory", + ) + .with_details(serde_json::json!({ + "kind": "global_window_owner_churn", + "pid": pid, + "phase": phase, + "complete": false, + "retryable": true, + })) +} + +fn retryable_global_error(error: &AdapterError) -> bool { + error.code == ErrorCode::AppUnresponsive && error.is_explicitly_retryable() +} + +fn global_timeout(attempts: u64, last_error: Option<&AdapterError>) -> AdapterError { + AdapterError::timeout("Global application window inventory did not stabilize before deadline") + .with_details(serde_json::json!({ + "kind": "global_window_inventory_unstable", + "attempts": attempts, + "last_code": last_error.map(|error| error.code.as_str()), + "last_kind": last_error + .and_then(|error| error.details.as_ref()) + .and_then(|details| details.get("kind")), + "complete": false, + "retryable": true, + })) +} + +#[cfg(test)] +#[path = "window_inventory_global_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/window_inventory_global_tests.rs b/crates/macos/src/system/window_inventory_global_tests.rs new file mode 100644 index 0000000..cd6674a --- /dev/null +++ b/crates/macos/src/system/window_inventory_global_tests.rs @@ -0,0 +1,374 @@ +use super::*; +use agent_desktop_core::Rect; +use rustc_hash::FxHashMap; + +#[derive(Clone)] +struct Owners { + launches: FxHashMap<i32, Option<f64>>, + frontmost: Option<i32>, +} + +impl OwnerSnapshotView for Owners { + fn eligible_pids(&self) -> FxHashSet<i32> { + self.launches.keys().copied().collect() + } + + fn generation(&self, pid: i32) -> Option<OwnerGeneration> { + self.launches.get(&pid).map(|launch| match launch { + Some(launch_time) => OwnerGeneration::LaunchTime(*launch_time), + None => OwnerGeneration::LiveProcess, + }) + } + + fn frontmost_pid(&self) -> Option<i32> { + self.frontmost + } + + fn same_generation(&self, other: &Self) -> bool { + self.launches == other.launches && self.frontmost == other.frontmost + } +} + +#[test] +fn background_owner_is_never_ax_probed_and_has_unknown_minimized_state() { + let owners = owners(&[(10, 100.0), (20, 200.0)], Some(10)); + let records = vec![record(10, 1, "Front"), record(20, 2, "Background")]; + let mut probes = Vec::new(); + + let windows = assemble_global_windows(records, &owners, false, |pid| { + probes.push(pid); + if pid == 20 { + return Err(AdapterError::permission_denied()); + } + Ok(ax_state(Some((Some("Front".into()), Some(1))))) + }) + .unwrap(); + + assert_eq!(probes, [10]); + assert!(windows[0].state.is_focused); + assert!(!windows[1].state.is_focused); + assert_eq!(windows[1].state.minimized, None); +} + +#[test] +fn windowless_owner_without_launch_date_does_not_poison_inventory() { + let owners = Owners { + launches: FxHashMap::from_iter([(10, Some(100.0)), (20, None)]), + frontmost: Some(10), + }; + let records = vec![record(10, 1, "Front")]; + let mut probes = Vec::new(); + + let windows = assemble_global_windows(records, &owners, false, |pid| { + probes.push(pid); + Ok(ax_state(Some((Some("Front".into()), Some(1))))) + }) + .unwrap(); + + assert_eq!(probes, [10]); + assert_eq!(windows.len(), 1); +} + +#[test] +fn window_owner_without_launch_date_uses_live_process_generation() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let process_instance = crate::system::process_identity::token_for_pid(pid) + .unwrap() + .expect("current process token"); + let owners = Owners { + launches: FxHashMap::from_iter([(pid, None)]), + frontmost: None, + }; + let records = vec![WindowRecord { + app_name: "Current".into(), + pid, + title: Some("Current".into()), + window_number: 1, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + visible: true, + process_instance: Some(process_instance), + }]; + + let windows = assemble_global_windows(records, &owners, false, |_| { + panic!("no AX probe without a frontmost owner") + }) + .unwrap(); + + assert_eq!(windows.len(), 1); +} + +#[test] +fn live_generation_rejects_stale_tokens() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let process_instance = crate::system::process_identity::token_for_pid(pid) + .unwrap() + .expect("current process token"); + let mut parts = process_instance.split(':'); + let prefix = parts.next().unwrap(); + let seconds = parts.next().unwrap().parse::<u64>().unwrap(); + let microseconds = parts.next().unwrap(); + let stale = format!("{prefix}:{}:{microseconds}", seconds + 1); + let owners = Owners { + launches: FxHashMap::from_iter([(pid, None)]), + frontmost: None, + }; + let mut stale_record = record(pid, 1, "Current"); + stale_record.process_instance = Some(stale); + + let error = assemble_global_windows(vec![stale_record], &owners, false, |_| { + panic!("no AX probe without a frontmost owner") + }) + .unwrap_err(); + + assert_eq!( + error.details.unwrap()["phase"], + "process_generation_changed" + ); +} + +#[test] +fn duplicate_windows_share_one_consistent_process_instance() { + let records = vec![record(10, 1, "One"), record(10, 2, "Two")]; + let instances = unique_process_instances(&records).unwrap(); + + assert_eq!(instances, [(10, "macos-proc-v1:100:0")]); + + let mut conflicting = records; + conflicting[1].process_instance = Some("macos-proc-v1:101:0".into()); + let error = unique_process_instances(&conflicting).unwrap_err(); + assert_eq!(error.details.unwrap()["phase"], "process_instance_conflict"); +} + +#[test] +fn focused_only_returns_one_authoritatively_joined_window() { + let owners = owners(&[(10, 100.0), (20, 200.0)], Some(10)); + let records = vec![record(10, 1, "Front"), record(20, 2, "Background")]; + + let windows = assemble_global_windows(records, &owners, true, |_| { + Ok(ax_state(Some((Some("Front".into()), Some(1))))) + }) + .unwrap(); + + assert_eq!(windows.len(), 1); + assert_eq!(windows[0].pid, 10); + assert!(windows[0].state.is_focused); +} + +#[test] +fn ineligible_owner_is_rejected_before_ax_join() { + let owners = owners(&[(10, 100.0)], Some(10)); + let mut probed = false; + + let error = assemble_global_windows(vec![record(418, 1, "Protected")], &owners, false, |_| { + probed = true; + Ok(ax_state(None)) + }) + .unwrap_err(); + + assert!(!probed); + assert_eq!(error.details.unwrap()["phase"], "owner_not_eligible"); +} + +#[test] +fn frontmost_ax_timeout_is_retryable_but_permission_denial_is_not_rewritten() { + let owners = owners(&[(10, 100.0)], Some(10)); + let records = vec![record(10, 1, "Front")]; + let timeout = assemble_global_windows(records.clone(), &owners, false, |_| { + Err(AdapterError::timeout("hung")) + }) + .unwrap_err(); + let denied = assemble_global_windows(records, &owners, false, |_| { + Err(AdapterError::permission_denied()) + }) + .unwrap_err(); + + assert_eq!(timeout.code, ErrorCode::AppUnresponsive); + assert_eq!(timeout.details.unwrap()["retryable"], true); + assert_eq!(denied.code, ErrorCode::PermDenied); +} + +#[test] +fn missing_focus_join_fails_incomplete() { + let owners = owners(&[(10, 100.0)], Some(10)); + let records = vec![record(10, 1, "Solo")]; + + let error = assemble_global_windows(records, &owners, false, |_| { + Ok(ax_state(Some((Some("Solo".into()), Some(99))))) + }) + .unwrap_err(); + + assert_eq!(error.details.unwrap()["kind"], "frontmost_window_join"); +} + +#[test] +fn ambiguous_title_without_window_number_degrades_to_unconfirmed_focus() { + let owners = owners(&[(10, 100.0)], Some(10)); + let records = vec![record(10, 1, "Duplicate"), record(10, 2, "Duplicate")]; + + let windows = assemble_global_windows(records, &owners, false, |_| { + Ok(ax_state(Some((Some("Duplicate".into()), None)))) + }) + .unwrap(); + + assert_eq!(windows.len(), 2); + assert!(windows.iter().all(|window| !window.state.is_focused)); +} + +#[test] +fn owner_snapshot_or_process_generation_churn_is_retryable() { + let before = owners(&[(10, 100.0)], Some(10)); + let after = owners(&[(10, 106.0)], Some(10)); + let records = vec![record(10, 1, "Front")]; + + let snapshot_error = validate_snapshot_pair(&before, &after, &records).unwrap_err(); + let generation_error = assemble_global_windows(records, &after, false, |_| { + Ok(ax_state(Some((Some("Front".into()), Some(1))))) + }) + .unwrap_err(); + + assert_eq!( + snapshot_error.details.unwrap()["phase"], + "appkit_snapshot_changed" + ); + assert_eq!( + generation_error.details.unwrap()["phase"], + "process_generation_changed" + ); +} + +#[test] +fn stabilization_retries_churn_and_preserves_strict_failures() { + let mut attempts = 0; + let windows = stabilize_global_with(Instant::now() + std::time::Duration::from_secs(1), || { + attempts += 1; + if attempts == 1 { + Err(owner_churn_error(None, "test")) + } else { + Ok(Vec::new()) + } + }) + .unwrap(); + + assert!(windows.is_empty()); + assert_eq!(attempts, 2); + let denied = stabilize_global_with(Instant::now() + std::time::Duration::from_secs(1), || { + Err(AdapterError::permission_denied()) + }) + .unwrap_err(); + assert_eq!(denied.code, ErrorCode::PermDenied); +} + +#[test] +fn persistent_owner_churn_times_out_with_exact_attempt_metrics() { + let mut attempts = 0_u64; + let error = stabilize_global_with( + Instant::now() + std::time::Duration::from_millis(20), + || { + attempts += 1; + Err(owner_churn_error(None, "test")) + }, + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::Timeout); + let details = error.details.unwrap(); + assert!(attempts >= 1); + assert_eq!(details["attempts"].as_u64(), Some(attempts)); + assert_eq!(details["last_code"], "APP_UNRESPONSIVE"); + assert_eq!(details["last_kind"], "global_window_owner_churn"); +} + +#[test] +fn global_capture_orders_appkit_cg_ax_appkit_and_filters_before_cg() { + let owners = owners(&[(10, 100.0), (20, 200.0)], Some(10)); + let events = std::cell::RefCell::new(Vec::new()); + let snapshot_count = std::cell::Cell::new(0); + + let windows = capture_global_with( + false, + || { + let ordinal = snapshot_count.get(); + events + .borrow_mut() + .push(if ordinal == 0 { "a" } else { "b" }); + snapshot_count.set(ordinal + 1); + Ok(owners.clone()) + }, + |eligible| { + events.borrow_mut().push("cg"); + assert_eq!(eligible, &FxHashSet::from_iter([10, 20])); + Ok(vec![record(10, 1, "Front")]) + }, + |_| { + events.borrow_mut().push("ax"); + Ok(ax_state(Some((Some("Front".into()), Some(1))))) + }, + ) + .unwrap(); + + assert_eq!(&*events.borrow(), &["a", "cg", "ax", "b"]); + assert_eq!(windows.len(), 1); +} + +#[test] +fn post_ax_snapshot_is_captured_and_churn_wins_over_stale_ax_failure() { + let snapshots = std::cell::RefCell::new(std::collections::VecDeque::from([ + owners(&[(10, 100.0)], Some(10)), + owners(&[(10, 106.0)], Some(10)), + ])); + let snapshot_count = std::cell::Cell::new(0); + + let error = capture_global_with( + false, + || { + snapshot_count.set(snapshot_count.get() + 1); + Ok(snapshots.borrow_mut().pop_front().unwrap()) + }, + |_| Ok(vec![record(10, 1, "Front")]), + |_| Err(AdapterError::permission_denied()), + ) + .unwrap_err(); + + assert_eq!(snapshot_count.get(), 2); + assert_eq!(error.details.unwrap()["phase"], "appkit_snapshot_changed"); +} + +fn owners(entries: &[(i32, f64)], frontmost: Option<i32>) -> Owners { + Owners { + launches: entries + .iter() + .map(|(pid, launch)| (*pid, Some(*launch))) + .collect(), + frontmost, + } +} + +fn record(pid: i32, window_number: i64, title: &str) -> WindowRecord { + let launch = if pid == 10 { 100 } else { 200 }; + WindowRecord { + app_name: format!("App-{pid}"), + pid, + title: Some(title.into()), + window_number, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + visible: true, + process_instance: Some(format!("macos-proc-v1:{launch}:0")), + } +} + +fn ax_state(focused: Option<(Option<String>, Option<i64>)>) -> WindowAxState { + WindowAxState { + focused, + minimized_by_id: FxHashMap::default(), + } +} diff --git a/crates/macos/src/system/window_inventory_tests.rs b/crates/macos/src/system/window_inventory_tests.rs index ed697b3..45c3583 100644 --- a/crates/macos/src/system/window_inventory_tests.rs +++ b/crates/macos/src/system/window_inventory_tests.rs @@ -1,5 +1,14 @@ use super::*; use crate::system::cg_window::WindowRecord; +use agent_desktop_core::Rect; +use rustc_hash::FxHashMap; + +fn ax_state(focused: FocusedWindowIdentity) -> crate::system::window_ax_state::WindowAxState { + crate::system::window_ax_state::WindowAxState { + focused, + minimized_by_id: FxHashMap::default(), + } +} #[test] fn apps_from_window_records_deduplicates_by_pid() { @@ -33,27 +42,6 @@ fn matches_app_filter_rejects_substring_match() { assert!(!matches_app_filter("Mail Helper", "mail")); } -#[test] -fn retry_empty_skips_known_missing_app_filter() { - let filter = WindowFilter { - app: Some("Missing".to_string()), - focused_only: false, - }; - - assert!(!should_retry_empty(&filter, None)); -} - -#[test] -fn retry_empty_allows_known_app_filter_for_ax_race() { - let filter = WindowFilter { - app: Some("Mail".to_string()), - focused_only: false, - }; - let app = app("Mail", 42); - - assert!(should_retry_empty(&filter, Some(&app))); -} - #[test] fn windows_from_records_marks_single_focused_window_once() { let windows = windows_from_records_with_focus( @@ -62,11 +50,13 @@ fn windows_from_records_marks_single_focused_window_once() { record("Mail", 10, "Inbox", 2), ], false, - |_| Some((Some("Inbox".to_string()), Some(2))), - ); + |_| Ok(ax_state(Some((Some("Inbox".to_string()), Some(2))))), + |_, _| Ok(true), + ) + .unwrap(); - assert!(!windows[0].is_focused); - assert!(windows[1].is_focused); + assert!(!windows[0].state.is_focused); + assert!(windows[1].state.is_focused); } #[test] @@ -77,13 +67,68 @@ fn windows_from_records_focus_only_filters_unfocused_windows() { record("Mail", 10, "Sent", 2), ], true, - |_| Some((Some("Sent".to_string()), Some(2))), - ); + |_| Ok(ax_state(Some((Some("Sent".to_string()), Some(2))))), + |_, _| Ok(true), + ) + .unwrap(); assert_eq!(windows.len(), 1); assert_eq!(windows[0].title, "Sent"); } +#[test] +fn windows_from_records_preserve_capture_bounds() { + let expected = Rect { + x: 12.0, + y: 34.0, + width: 800.0, + height: 600.0, + }; + let mut source = record("Preview", 10, "Document", 7); + source.bounds = expected; + + let windows = windows_from_records_with_focus( + vec![source], + false, + |_| Ok(ax_state(None)), + |_, _| Ok(true), + ) + .unwrap(); + + assert_eq!(windows[0].bounds, Some(expected)); +} + +#[test] +fn windows_from_records_never_publish_zero_window_ids() { + let windows = windows_from_records_with_focus( + vec![record("Preview", 10, "Unverified", 0)], + false, + |_| Ok(ax_state(None)), + |_, _| Ok(true), + ) + .unwrap(); + + assert!(windows.is_empty()); +} + +#[test] +fn window_inventory_rejects_owner_generation_change_around_focus_read() { + let mut checks = 0; + let error = windows_from_records_with_focus( + vec![record("Preview", 10, "Document", 7)], + false, + |_| Ok(ax_state(None)), + |_, _| { + checks += 1; + Ok(checks == 1) + }, + ) + .expect_err("owner changed after AX join"); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::AppUnresponsive); + assert_eq!(error.details.unwrap()["kind"], "window_identity_race"); +} + #[test] fn matches_focused_window_uses_window_number_when_available() { let identity = Some((Some("Other".to_string()), Some(7))); @@ -102,140 +147,15 @@ fn matches_focused_window_uses_unique_title_without_window_number() { } #[test] -fn list_windows_retries_after_unfocused_ax_fallback_for_focused_filter() { - let filter = WindowFilter { - app: Some("Mail".to_string()), - focused_only: true, - }; - let app = app("Mail", 42); - let mut visible_calls = 0; - let mut ax_calls = 0; - - let windows = list_windows_with_sources( - &filter, - |_, _| Some(app.clone()), - || { - visible_calls += 1; - Vec::new() - }, - |_| { - ax_calls += 1; - Some(window("Mail", 42, "Inbox", 7, ax_calls == 2)) - }, - |_| {}, - ); - - assert_eq!(windows.len(), 1); - assert_eq!(windows[0].title, "Inbox"); - assert_eq!(visible_calls, 2); - assert_eq!(ax_calls, 2); -} - -#[test] -fn list_windows_sleeps_between_unfocused_ax_fallback_retries() { - let filter = WindowFilter { - app: Some("Mail".to_string()), - focused_only: true, - }; - let app = app("Mail", 42); - let mut sleep_calls = 0; - - let windows = list_windows_with_sources( - &filter, - |_, _| Some(app.clone()), - Vec::new, - |_| Some(window("Mail", 42, "Inbox", 7, false)), - |_| sleep_calls += 1, - ); - - assert!(windows.is_empty()); - assert_eq!(sleep_calls, 2); -} - -#[test] -fn list_windows_without_app_filter_retries_empty_records() { +fn deadline_window_inventory_rejects_expiry_before_native_reads() { let filter = WindowFilter { app: None, focused_only: false, }; - let mut visible_calls = 0; - let mut app_resolver_called = false; - let windows = list_windows_with_sources( - &filter, - |_, _| { - app_resolver_called = true; - None - }, - || { - visible_calls += 1; - Vec::new() - }, - |_| None, - |_| {}, - ); + let error = list_windows_until(&filter, Instant::now()).unwrap_err(); - assert!(windows.is_empty()); - assert_eq!(visible_calls, 3); - assert!(!app_resolver_called); -} - -#[test] -fn list_windows_stops_when_named_app_is_missing() { - let filter = WindowFilter { - app: Some("Missing".to_string()), - focused_only: true, - }; - let mut visible_calls = 0; - - let windows = list_windows_with_sources( - &filter, - |_, _| None, - || { - visible_calls += 1; - Vec::new() - }, - |_| None, - |_| {}, - ); - - assert!(windows.is_empty()); - assert_eq!(visible_calls, 1); -} - -#[test] -fn ax_window_info_uses_resolved_app_identity() { - let app = app("Mail", 42); - let window = ax_window_info(&app, "Inbox".to_string(), 7, true); - - assert_eq!(window.app, "Mail"); - assert_eq!(window.pid, 42); - assert_eq!(window.id, "w-7"); -} - -#[test] -fn child_bearing_window_index_prefers_first_window_with_children() { - let windows = [0, 7, 3]; - - assert_eq!( - child_bearing_window_index(&windows, |count| *count), - Some(1) - ); -} - -#[test] -fn child_bearing_window_index_returns_none_when_all_windows_are_empty() { - let windows = [0, 0]; - - assert_eq!(child_bearing_window_index(&windows, |count| *count), None); -} - -fn app(name: &str, pid: i32) -> AppInfo { - AppInfo { - name: name.to_string(), - pid, - bundle_id: None, - } + assert_eq!(error.code.as_str(), "TIMEOUT"); } fn record(app_name: &str, pid: i32, title: &str, window_number: i64) -> WindowRecord { @@ -244,23 +164,13 @@ fn record(app_name: &str, pid: i32, title: &str, window_number: i64) -> WindowRe pid, title: Some(title.to_string()), window_number, - area: 100.0, - } -} - -fn window( - app_name: &str, - pid: i32, - title: &str, - window_number: i64, - is_focused: bool, -) -> WindowInfo { - WindowInfo { - id: format!("w-{window_number}"), - title: title.to_string(), - app: app_name.to_string(), - pid, - bounds: None, - is_focused, + bounds: agent_desktop_core::Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }, + visible: true, + process_instance: Some(format!("instance-{pid}")), } } diff --git a/crates/macos/src/system/window_list.rs b/crates/macos/src/system/window_list.rs deleted file mode 100644 index 5488ba6..0000000 --- a/crates/macos/src/system/window_list.rs +++ /dev/null @@ -1,13 +0,0 @@ -use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo}; - -pub(crate) fn list_windows_impl(filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { - #[cfg(target_os = "macos")] - { - Ok(crate::system::app_inventory::list_windows(filter)) - } - #[cfg(not(target_os = "macos"))] - { - let _ = filter; - Err(AdapterError::not_supported("list_windows")) - } -} diff --git a/crates/macos/src/system/window_ops.rs b/crates/macos/src/system/window_ops.rs index c7e2b44..ef2b8d9 100644 --- a/crates/macos/src/system/window_ops.rs +++ b/crates/macos/src/system/window_ops.rs @@ -1,13 +1,11 @@ -use agent_desktop_core::{action::WindowOp, error::AdapterError, node::WindowInfo}; +use agent_desktop_core::{AdapterError, Deadline, ErrorCode, WindowInfo, WindowOp}; #[cfg(target_os = "macos")] mod imp { use super::*; use accessibility_sys::{ - AXUIElementSetAttributeValue, kAXErrorSuccess, kAXPositionAttribute, kAXSizeAttribute, - kAXValueTypeCGPoint, kAXValueTypeCGSize, + kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint, kAXValueTypeCGSize, }; - use agent_desktop_core::error::ErrorCode; use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; use core_graphics::geometry::{CGPoint, CGSize}; use std::ffi::c_void; @@ -16,30 +14,47 @@ mod imp { fn AXValueCreate(value_type: u32, value_ptr: *const c_void) -> *mut c_void; } - pub fn execute(win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> { + pub fn execute(win: &WindowInfo, op: WindowOp, deadline: Deadline) -> Result<(), AdapterError> { tracing::debug!( "system: window_op {:?} app={:?} title={:?}", op, win.app, win.title ); - let win_el = crate::tree::window_element_for(win.pid, &win.title); + ensure_budget(deadline)?; + let win_el = crate::system::window_resolve::window_element_for_info(win, deadline)?; match op { - WindowOp::Resize { width, height } => set_size(&win_el, width, height), - WindowOp::Move { x, y } => set_position(&win_el, x, y), - WindowOp::Minimize => set_minimized(&win_el, true), - WindowOp::Maximize => maximize_to_main_display(&win_el), - WindowOp::Restore => set_minimized(&win_el, false), + WindowOp::Resize { width, height } => set_size(&win_el, width, height, deadline), + WindowOp::Move { x, y } => set_position(&win_el, x, y, deadline), + WindowOp::Minimize => set_minimized(&win_el, true, deadline), + WindowOp::Maximize => maximize_to_main_display(&win_el, deadline), + WindowOp::Restore => set_minimized(&win_el, false, deadline), } } - fn maximize_to_main_display(el: &crate::tree::AXElement) -> Result<(), AdapterError> { - let bounds = core_graphics::display::CGDisplay::main().bounds(); - set_position(el, bounds.origin.x, bounds.origin.y)?; - set_size(el, bounds.size.width, bounds.size.height) + fn maximize_to_main_display( + el: &crate::tree::AXElement, + deadline: Deadline, + ) -> Result<(), AdapterError> { + let current = crate::tree::element_bounds::read_bounds_with_deadline( + el, + std::time::Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::internal("Window deadline is out of range"))?, + )?; + let work_area = crate::system::display_work_area::for_window(current, deadline)?; + set_position(el, work_area.x, work_area.y, deadline)?; + set_size(el, work_area.width, work_area.height, deadline) } - fn set_size(el: &crate::tree::AXElement, width: f64, height: f64) -> Result<(), AdapterError> { + fn set_size( + el: &crate::tree::AXElement, + width: f64, + height: f64, + deadline: Deadline, + ) -> Result<(), AdapterError> { + validate_size(width, height)?; + prepare(el, deadline)?; let size = CGSize::new(width, height); let ax_value = unsafe { AXValueCreate(kAXValueTypeCGSize, &size as *const _ as *const c_void) }; @@ -47,21 +62,35 @@ mod imp { return Err(AdapterError::internal("Failed to create AXValue for size")); } let cf_attr = CFString::new(kAXSizeAttribute); - let err = unsafe { - AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), ax_value as _) - }; + let err = crate::tree::ax_ipc::set_attribute_value( + el, + cf_attr.as_concrete_TypeRef(), + ax_value as _, + deadline, + )?; unsafe { core_foundation::base::CFRelease(ax_value as _) }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("Resize failed (err={err})"), - ) - .with_suggestion("Window may not support resizing. Try a different size.")); - } - Ok(()) + finish(el, err, "resize window", deadline)?; + crate::system::window_postcondition::wait_for_geometry( + el, + agent_desktop_core::Rect { + x: 0.0, + y: 0.0, + width, + height, + }, + false, + deadline, + ) } - fn set_position(el: &crate::tree::AXElement, x: f64, y: f64) -> Result<(), AdapterError> { + fn set_position( + el: &crate::tree::AXElement, + x: f64, + y: f64, + deadline: Deadline, + ) -> Result<(), AdapterError> { + validate_position(x, y)?; + prepare(el, deadline)?; let point = CGPoint::new(x, y); let ax_value = unsafe { AXValueCreate(kAXValueTypeCGPoint, &point as *const _ as *const c_void) }; @@ -71,91 +100,176 @@ mod imp { )); } let cf_attr = CFString::new(kAXPositionAttribute); - let err = unsafe { - AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), ax_value as _) - }; + let err = crate::tree::ax_ipc::set_attribute_value( + el, + cf_attr.as_concrete_TypeRef(), + ax_value as _, + deadline, + )?; unsafe { core_foundation::base::CFRelease(ax_value as _) }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("Move failed (err={err})"), - ) - .with_suggestion( - "Window may not support repositioning. Verify coordinates are on-screen.", - )); - } - Ok(()) + finish(el, err, "move window", deadline)?; + crate::system::window_postcondition::wait_for_geometry( + el, + agent_desktop_core::Rect { + x, + y, + width: 0.0, + height: 0.0, + }, + true, + deadline, + ) } - fn set_minimized(el: &crate::tree::AXElement, minimized: bool) -> Result<(), AdapterError> { + fn set_minimized( + el: &crate::tree::AXElement, + minimized: bool, + deadline: Deadline, + ) -> Result<(), AdapterError> { + prepare(el, deadline)?; let cf_attr = CFString::new("AXMinimized"); let val = if minimized { CFBoolean::true_value() } else { CFBoolean::false_value() }; - let err = unsafe { - AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), val.as_CFTypeRef()) - }; - if err != kAXErrorSuccess { - let op = if minimized { "Minimize" } else { "Restore" }; + let err = crate::tree::ax_ipc::set_attribute_value( + el, + cf_attr.as_concrete_TypeRef(), + val.as_CFTypeRef(), + deadline, + )?; + finish( + el, + err, + if minimized { + "minimize window" + } else { + "restore window" + }, + deadline, + )?; + crate::system::window_postcondition::wait_for_minimized(el, minimized, deadline) + } + + fn validate_size(width: f64, height: f64) -> Result<(), AdapterError> { + const MAX_DIMENSION: f64 = 1_000_000.0; + if !width.is_finite() + || !height.is_finite() + || width <= 0.0 + || height <= 0.0 + || width > MAX_DIMENSION + || height > MAX_DIMENSION + { return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("{op} failed (err={err})"), - ) - .with_suggestion("Window may not support this operation. Try 'focus-window' first.")); + ErrorCode::InvalidArgs, + "Window width and height must be finite, positive, and at most 1000000", + )); } Ok(()) } + + fn validate_position(x: f64, y: f64) -> Result<(), AdapterError> { + const MAX_COORDINATE: f64 = 1_000_000.0; + if !x.is_finite() || !y.is_finite() || x.abs() > MAX_COORDINATE || y.abs() > MAX_COORDINATE + { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + "Window coordinates must be finite and within -1000000..=1000000", + )); + } + Ok(()) + } + + fn prepare(el: &crate::tree::AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(el, deadline) + } + + fn finish( + element: &crate::tree::AXElement, + error: i32, + operation: &str, + deadline: Deadline, + ) -> Result<(), AdapterError> { + crate::system::focus::finish_mutation(element, error, operation, deadline) + } + + fn ensure_budget(deadline: Deadline) -> Result<(), AdapterError> { + if deadline.is_expired() { + Err(deadline.timeout_error()) + } else { + Ok(()) + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn geometry_rejects_non_finite_and_extreme_values() { + assert!(validate_size(f64::NAN, 100.0).is_err()); + assert!(validate_size(100.0, f64::INFINITY).is_err()); + assert!(validate_size(0.0, 100.0).is_err()); + assert!(validate_position(f64::NEG_INFINITY, 0.0).is_err()); + assert!(validate_position(1_000_001.0, 0.0).is_err()); + } + + #[test] + fn geometry_accepts_negative_screen_coordinates() { + assert!(validate_position(-1920.0, 0.0).is_ok()); + assert!(validate_size(1920.0, 1080.0).is_ok()); + } + } } #[cfg(target_os = "macos")] mod raise { use accessibility_sys::{ - AXUIElementPerformAction, AXUIElementSetAttributeValue, kAXErrorSuccess, + kAXErrorActionUnsupported, kAXErrorAttributeUnsupported, kAXErrorNotImplemented, + kAXErrorSuccess, }; + use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics}; use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; - /// Raises a window element to the top of its app's window stack and - /// confirms with a brief `AXMain` poll. CGEvents land on the topmost - /// window at the click point, so a frontmost app is not enough when the - /// target element lives in a background window. Best-effort: a window - /// that refuses both `AXRaise` and a settable `AXMain` is left as-is. - pub(crate) fn raise_window(window: &crate::tree::AXElement) { + pub(crate) fn raise_window( + window: &crate::tree::AXElement, + deadline: Deadline, + ) -> Result<(), AdapterError> { + prepare(window, deadline)?; let raise = CFString::new("AXRaise"); - let raise_err = unsafe { AXUIElementPerformAction(window.0, raise.as_concrete_TypeRef()) }; - if raise_err != kAXErrorSuccess { - let main_attr = CFString::new("AXMain"); - let ax_err = unsafe { - AXUIElementSetAttributeValue( - window.0, - main_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if ax_err != kAXErrorSuccess { - tracing::debug!("raise_window: AXMain fallback returned err={ax_err}"); - } + let raise_err = + crate::tree::ax_ipc::perform_action(window, raise.as_concrete_TypeRef(), deadline)?; + if raise_err == kAXErrorSuccess { + return crate::system::focus::wait_until_main(window, deadline).map_err(after_delivery); } - wait_until_main(window); + if safe_to_fallback(raise_err) { + prepare(window, deadline)?; + let main_attr = CFString::new("AXMain"); + let ax_err = crate::tree::ax_ipc::set_attribute_value( + window, + main_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + deadline, + )?; + crate::system::focus::finish_mutation(window, ax_err, "raise window", deadline)?; + return crate::system::focus::wait_until_main(window, deadline).map_err(after_delivery); + } + crate::system::focus::finish_mutation(window, raise_err, "raise window", deadline) } - fn wait_until_main(window: &crate::tree::AXElement) { - use std::time::{Duration, Instant}; + fn prepare(window: &crate::tree::AXElement, deadline: Deadline) -> Result<(), AdapterError> { + crate::tree::attributes::set_messaging_timeout(window, deadline) + } - const POLL_INTERVAL: Duration = Duration::from_millis(5); - const MAIN_DEADLINE: Duration = Duration::from_millis(50); + fn safe_to_fallback(error: i32) -> bool { + error == kAXErrorActionUnsupported + || error == kAXErrorAttributeUnsupported + || error == kAXErrorNotImplemented + } - let deadline = Instant::now() + MAIN_DEADLINE; - loop { - if crate::tree::copy_bool_attr(window, "AXMain") == Some(true) { - return; - } - if Instant::now() >= deadline { - return; - } - std::thread::sleep(POLL_INTERVAL); - } + fn after_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) } } @@ -166,9 +280,13 @@ pub(crate) use raise::raise_window; mod imp { use super::*; - pub fn execute(_win: &WindowInfo, _op: WindowOp) -> Result<(), AdapterError> { + pub fn execute( + _win: &WindowInfo, + _op: WindowOp, + _deadline: Deadline, + ) -> Result<(), AdapterError> { Err(AdapterError::not_supported("window_op")) } } -pub use imp::execute; +pub(crate) use imp::execute; diff --git a/crates/macos/src/system/window_postcondition.rs b/crates/macos/src/system/window_postcondition.rs new file mode 100644 index 0000000..27ee831 --- /dev/null +++ b/crates/macos/src/system/window_postcondition.rs @@ -0,0 +1,107 @@ +use agent_desktop_core::{AdapterError, Deadline, DeliverySemantics, Rect}; + +const TOLERANCE: f64 = 2.0; + +pub(crate) fn wait_for_geometry( + element: &crate::tree::AXElement, + expected: Rect, + position: bool, + deadline: Deadline, +) -> Result<(), AdapterError> { + wait_until(deadline, || { + let bounds = crate::tree::element_bounds::read_bounds_with_deadline( + element, + deadline_instant(deadline)?, + )?; + Ok(bounds.is_some_and(|actual| geometry_matches(actual, expected, position))) + }) +} + +pub(crate) fn wait_for_minimized( + element: &crate::tree::AXElement, + expected: bool, + deadline: Deadline, +) -> Result<(), AdapterError> { + wait_until(deadline, || { + Ok( + crate::tree::surface_read::boolean( + element, + "AXMinimized", + deadline_instant(deadline)?, + )? == Some(expected), + ) + }) +} + +fn wait_until( + deadline: Deadline, + mut complete: impl FnMut() -> Result<bool, AdapterError>, +) -> Result<(), AdapterError> { + loop { + if complete().map_err(after_delivery)? { + return Ok(()); + } + if deadline.is_expired() { + return Err(after_delivery(AdapterError::timeout( + "Window operation did not reach its requested postcondition", + ))); + } + let pause = deadline + .remaining_slice(std::time::Duration::from_millis(10)) + .map_err(after_delivery)?; + std::thread::sleep(pause.min(std::time::Duration::from_millis(10))); + } +} + +fn geometry_matches(actual: Rect, expected: Rect, position: bool) -> bool { + let values = if position { + [(actual.x, expected.x), (actual.y, expected.y)] + } else { + [ + (actual.width, expected.width), + (actual.height, expected.height), + ] + }; + values + .into_iter() + .all(|(actual, expected)| (actual - expected).abs() <= TOLERANCE) +} + +fn deadline_instant(deadline: Deadline) -> Result<std::time::Instant, AdapterError> { + std::time::Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::internal("Window operation deadline is out of range")) +} + +fn after_delivery(error: AdapterError) -> AdapterError { + error.with_disposition(DeliverySemantics::delivered_unverified()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn geometry_verification_tolerates_ax_rounding_only() { + let expected = Rect { + x: 10.0, + y: 20.0, + width: 100.0, + height: 80.0, + }; + let rounded = Rect { + x: 11.0, + y: 19.0, + width: 99.0, + height: 81.0, + }; + let wrong = Rect { + width: 90.0, + ..rounded + }; + + assert!(geometry_matches(rounded, expected, true)); + assert!(geometry_matches(rounded, expected, false)); + assert!(!geometry_matches(wrong, expected, false)); + } +} diff --git a/crates/macos/src/system/window_resolve.rs b/crates/macos/src/system/window_resolve.rs new file mode 100644 index 0000000..127653c --- /dev/null +++ b/crates/macos/src/system/window_resolve.rs @@ -0,0 +1,365 @@ +use agent_desktop_core::{AdapterError, ErrorCode, WindowInfo}; + +use crate::system::cg_window::WindowRecord; +use crate::tree::{AXElement, attributes::set_messaging_timeout, element_for_pid}; + +pub(crate) struct WindowIdentityEvidence<'a> { + pub pid: i32, + pub app: Option<&'a str>, + pub process_instance: Option<&'a str>, + pub title: Option<&'a str>, + pub bounds_hash: Option<u64>, +} + +pub(crate) fn window_element_for_info( + win: &WindowInfo, + deadline: agent_desktop_core::Deadline, +) -> Result<AXElement, AdapterError> { + window_element_for_info_with_deadline( + win, + std::time::Instant::now() + .checked_add(deadline.remaining()) + .ok_or_else(|| AdapterError::new(ErrorCode::InvalidArgs, "deadline is out of range"))?, + ) +} + +pub(crate) fn window_element_for_info_with_deadline( + win: &WindowInfo, + deadline: std::time::Instant, +) -> Result<AXElement, AdapterError> { + if win.id.is_empty() { + let pid = crate::system::process_identity::to_pid_t(win.pid)?; + return window_element_by_title(pid, &win.title, deadline); + } + resolve_window_element_strict(win, deadline) +} + +pub(crate) fn resolve_window_strict( + win: &WindowInfo, + deadline: std::time::Instant, +) -> Result<WindowInfo, AdapterError> { + let (window_number, record) = locate_verified_record_until(win, deadline)?; + let ax_state = crate::system::window_ax_state::read_until(record.pid, deadline)?; + verify_window_record(win, &record)?; + let mut resolved = window_info_from_record(win, &record)?; + resolved.state.minimized = ax_state.minimized_by_id.get(&window_number).copied(); + if let Some((_, Some(focused_number))) = ax_state.focused { + resolved.state.is_focused = focused_number == window_number; + } + Ok(resolved) +} + +fn locate_verified_record_until( + win: &WindowInfo, + deadline: std::time::Instant, +) -> Result<(i64, WindowRecord), AdapterError> { + let window_number = parse_window_number(&win.id).ok_or_else(|| invalid_window_id(&win.id))?; + let record = + crate::system::cg_window_exact::exact_window_record_until(window_number, deadline)? + .ok_or_else(|| window_not_found(&win.id))?; + verify_window_record(win, &record)?; + Ok((window_number, record)) +} + +pub(crate) fn verify_window_identity_until( + id: &str, + evidence: WindowIdentityEvidence<'_>, + deadline: std::time::Instant, +) -> Result<(), AdapterError> { + let window_number = parse_window_number(id).ok_or_else(|| invalid_window_id(id))?; + let record = + crate::system::cg_window_exact::exact_window_record_until(window_number, deadline)? + .ok_or_else(|| window_not_found(id))?; + if !window_record_matches_source(&record, &evidence) { + return Err(window_identity_mismatch(id)); + } + let Some(process_instance) = evidence.process_instance else { + return Err(window_identity_mismatch(id)); + }; + if !crate::system::process_identity::matches_instance(evidence.pid, process_instance)? { + return Err(window_identity_mismatch(id)); + } + Ok(()) +} + +fn window_record_matches_source( + record: &WindowRecord, + evidence: &WindowIdentityEvidence<'_>, +) -> bool { + if record.pid != evidence.pid + || evidence + .app + .is_some_and(|app| !app.is_empty() && record.app_name != app) + || evidence.process_instance.is_none() + || record.process_instance.as_deref() != evidence.process_instance + { + return false; + } + let bounds_changed = evidence.bounds_hash.is_some_and(|expected| { + record + .bounds + .bounds_hash() + .is_none_or(|actual| actual != expected) + }); + if bounds_changed { + tracing::debug!( + expected_bounds_hash = ?evidence.bounds_hash, + actual_bounds_hash = ?record.bounds.bounds_hash(), + "window moved or resized while immutable source identity remained valid" + ); + } + let title_changed = evidence.title.is_some_and(|title| { + !title.is_empty() && record.title.as_deref().unwrap_or(record.app_name.as_str()) != title + }); + if title_changed { + tracing::debug!( + expected_title = ?evidence.title, + actual_title = ?record.title, + "window title changed while immutable source identity remained valid" + ); + } + true +} + +fn resolve_window_element_strict( + win: &WindowInfo, + deadline: std::time::Instant, +) -> Result<AXElement, AdapterError> { + crate::tree::locator_deadline::remaining(deadline)?; + let (window_number, record) = locate_verified_record_until(win, deadline)?; + crate::tree::locator_deadline::remaining(deadline)?; + let pid = crate::system::process_identity::to_pid_t(win.pid)?; + ax_window_element_for_number(pid, window_number, record.title.as_deref(), deadline)? + .ok_or_else(|| window_not_found(&win.id)) +} + +pub(crate) fn parse_window_number(id: &str) -> Option<i64> { + id.strip_prefix('w')? + .strip_prefix('-')? + .parse() + .ok() + .filter(|number| *number > 0) +} + +fn verify_window_record(win: &WindowInfo, record: &WindowRecord) -> Result<(), AdapterError> { + let pid = crate::system::process_identity::to_pid_t(win.pid)?; + if record.pid != pid { + return Err(window_identity_mismatch(&win.id)); + } + let Some(instance) = win.process_instance.as_deref() else { + return Err(window_identity_mismatch(&win.id)); + }; + if record.process_instance.as_deref() != Some(instance) + || !crate::system::process_identity::matches_instance(pid, instance)? + { + return Err(window_identity_mismatch(&win.id)); + } + if !win.app.is_empty() && record.app_name != win.app { + return Err(window_identity_mismatch(&win.id)); + } + if !win.title.is_empty() { + let record_title = record.title.as_deref().unwrap_or(record.app_name.as_str()); + if record_title != win.title { + return Err(window_identity_mismatch(&win.id)); + } + } + Ok(()) +} + +fn window_info_from_record( + win: &WindowInfo, + record: &WindowRecord, +) -> Result<WindowInfo, AdapterError> { + Ok(WindowInfo { + id: win.id.clone(), + title: record + .title + .clone() + .unwrap_or_else(|| record.app_name.clone()), + app: record.app_name.clone(), + pid: crate::system::process_identity::from_pid_t(record.pid)?, + process_instance: record.process_instance.clone(), + bounds: Some(record.bounds), + state: agent_desktop_core::WindowState { + is_focused: win.state.is_focused, + minimized: win.state.minimized, + visible: Some(record.visible), + }, + }) +} + +fn ax_window_element_for_number( + pid: i32, + window_number: i64, + fallback_title: Option<&str>, + deadline: std::time::Instant, +) -> Result<Option<AXElement>, AdapterError> { + let windows = windows_for_pid(pid, deadline)?; + let mut unavailable_bridge = None; + for window in &windows { + prepare_for_read(window, deadline)?; + if crate::tree::resolve_ax_read::read_string(window, "AXRole", deadline)?.as_deref() + != Some("AXWindow") + { + continue; + } + match ax_window_id_with_deadline(window, deadline) { + Ok(Some(actual)) if actual == window_number => return Ok(Some(window.clone())), + Ok(_) => {} + Err(error) if crate::system::window_bridge::is_unavailable(&error) => { + unavailable_bridge = Some(error); + break; + } + Err(error) => return Err(error), + } + } + let Some(title) = fallback_title else { + return match unavailable_bridge { + Some(error) => Err(error), + None => Ok(None), + }; + }; + let mut single_match = None; + for window in windows { + prepare_for_read(&window, deadline)?; + if crate::tree::resolve_ax_read::read_string(&window, "AXRole", deadline)?.as_deref() + != Some("AXWindow") + { + continue; + } + if crate::tree::resolve_ax_read::read_string(&window, "AXTitle", deadline)?.as_deref() + != Some(title) + { + continue; + } + if single_match.is_some() { + return Err(AdapterError::ambiguous_target(format!( + "Multiple AX windows matched the verified CoreGraphics title '{title}'" + )) + .with_details(serde_json::json!({ + "kind": "verified_window_title_ambiguous", + "title": title, + }))); + } + single_match = Some(window); + } + match single_match { + Some(window) => Ok(Some(window)), + None => match unavailable_bridge { + Some(error) => Err(error), + None => Ok(None), + }, + } +} + +fn window_element_by_title( + pid: i32, + requested_title: &str, + deadline: std::time::Instant, +) -> Result<AXElement, AdapterError> { + let windows = windows_for_pid(pid, deadline)?; + let mut exact = Vec::new(); + let mut window_count = 0_usize; + let mut only_window = None; + for window in windows { + if crate::tree::resolve_ax_read::read_string(&window, "AXRole", deadline)?.as_deref() + != Some("AXWindow") + { + continue; + } + window_count += 1; + only_window = Some(window.clone()); + let title = crate::tree::resolve_ax_read::read_string(&window, "AXTitle", deadline)?; + if title.as_deref() == Some(requested_title) && !requested_title.is_empty() { + exact.push(window); + } + } + if exact.len() == 1 { + return Ok(exact.remove(0)); + } + if exact.len() > 1 { + return Err(AdapterError::ambiguous_target(format!( + "Multiple windows have the exact title '{requested_title}'" + )) + .with_details(serde_json::json!({ + "kind": "window_title_ambiguous", + "title": requested_title, + "candidate_count": exact.len(), + }))); + } + if requested_title.is_empty() && window_count == 1 { + return only_window.ok_or_else(|| window_not_found(requested_title)); + } + Err(window_not_found(requested_title)) +} + +fn windows_for_pid(pid: i32, deadline: std::time::Instant) -> Result<Vec<AXElement>, AdapterError> { + let app = element_for_pid(pid); + #[cfg(target_os = "macos")] + { + Ok( + crate::tree::resolve_ax_read::read_array(&app, "AXWindows", deadline)? + .unwrap_or_default(), + ) + } + #[cfg(not(target_os = "macos"))] + { + prepare_for_read(&app, deadline)?; + let windows = crate::tree::attributes::copy_ax_array_result(&app, "AXWindows") + .ok() + .flatten() + .unwrap_or_default(); + crate::tree::locator_deadline::remaining(deadline)?; + Ok(windows) + } +} + +fn prepare_for_read(element: &AXElement, deadline: std::time::Instant) -> Result<(), AdapterError> { + crate::tree::locator_deadline::remaining(deadline)?; + set_messaging_timeout(element, deadline)?; + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(crate) fn ax_window_id_with_deadline( + window: &AXElement, + deadline: std::time::Instant, +) -> Result<Option<i64>, AdapterError> { + prepare_for_read(window, deadline)?; + let window_id = crate::system::window_bridge::window_id(window, deadline)?; + crate::tree::locator_deadline::remaining(deadline)?; + Ok(window_id) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn ax_window_id_with_deadline( + _window: &AXElement, + _deadline: std::time::Instant, +) -> Result<Option<i64>, AdapterError> { + Ok(None) +} + +fn invalid_window_id(id: &str) -> AdapterError { + AdapterError::new(ErrorCode::InvalidArgs, format!("Invalid window id: '{id}'")) + .with_suggestion("Window ids come from 'list-windows' (format w-<number>).") +} + +fn window_not_found(id: &str) -> AdapterError { + AdapterError::new( + ErrorCode::WindowNotFound, + format!("Window '{id}' not found"), + ) + .with_suggestion("Run 'list-windows' to see available windows and their IDs.") +} + +fn window_identity_mismatch(id: &str) -> AdapterError { + AdapterError::new( + ErrorCode::WindowNotFound, + format!("Window '{id}' identity mismatch"), + ) + .with_suggestion("Run 'list-windows' to refresh window IDs, then retry.") +} + +#[cfg(test)] +#[path = "window_resolve_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/window_resolve_tests.rs b/crates/macos/src/system/window_resolve_tests.rs new file mode 100644 index 0000000..6f205c6 --- /dev/null +++ b/crates/macos/src/system/window_resolve_tests.rs @@ -0,0 +1,188 @@ +use super::*; +use crate::system::cg_window::WindowRecord; + +fn record(app_name: &str, pid: i32, title: &str, window_number: i64) -> WindowRecord { + WindowRecord { + app_name: app_name.into(), + pid, + title: Some(title.into()), + window_number, + bounds: agent_desktop_core::Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }, + visible: true, + process_instance: Some(instance(pid)), + } +} + +fn win(id: &str, pid: i32, title: &str) -> WindowInfo { + WindowInfo { + id: id.into(), + title: title.into(), + app: "TextEdit".into(), + pid: agent_desktop_core::ProcessId::try_from(pid).unwrap(), + process_instance: Some(instance(pid)), + bounds: None, + state: agent_desktop_core::WindowState::default(), + } +} + +fn current_pid() -> i32 { + i32::try_from(std::process::id()).unwrap() +} + +fn instance(pid: i32) -> String { + if pid != current_pid() { + return format!("instance-{pid}"); + } + crate::system::process_identity::token_for_pid(pid) + .unwrap() + .expect("current test process has an identity token") +} + +#[test] +fn parse_window_number_accepts_w_prefix() { + assert_eq!(parse_window_number("w-42"), Some(42)); + assert_eq!(parse_window_number("w-0"), None); + assert_eq!(parse_window_number("w--1"), None); + assert_eq!(parse_window_number("42"), None); + assert_eq!(parse_window_number("w-bad"), None); +} + +#[test] +fn verify_rejects_recycled_id_with_wrong_pid() { + let requested = win("w-100", 10, "Untitled"); + let live = record("TextEdit", 99, "Untitled", 100); + assert!(verify_window_record(&requested, &live).is_err()); +} + +#[test] +fn verify_rejects_recycled_pid_with_wrong_application_identity() { + let pid = current_pid(); + let requested = win("w-100", pid, "Untitled"); + let live = record("DifferentApp", pid, "Untitled", 100); + + assert!(verify_window_record(&requested, &live).is_err()); +} + +#[test] +fn verify_rejects_title_mismatch_when_title_provided() { + let pid = current_pid(); + let requested = win("w-100", pid, "Doc A"); + let live = record("TextEdit", pid, "Doc B", 100); + assert!(verify_window_record(&requested, &live).is_err()); +} + +#[test] +fn verify_accepts_matching_pid_and_title() { + let pid = current_pid(); + let requested = win("w-100", pid, "Untitled"); + let live = record("TextEdit", pid, "Untitled", 100); + assert!(verify_window_record(&requested, &live).is_ok()); +} + +#[test] +fn resolved_window_preserves_verified_core_graphics_bounds() { + let requested = win("w-100", 10, "Untitled"); + let live = record("TextEdit", 10, "Untitled", 100); + + let resolved = window_info_from_record(&requested, &live).unwrap(); + + assert_eq!(resolved.bounds, Some(live.bounds)); +} + +#[test] +fn source_identity_survives_window_move_and_resize() { + let pid = current_pid(); + let mut live = record("TextEdit", pid, "Untitled", 100); + let original_hash = live.bounds.bounds_hash(); + live.bounds.x += 50.0; + live.bounds.width += 25.0; + live.title = Some("Renamed document".into()); + live.visible = false; + let process_instance = instance(pid); + + assert!(window_record_matches_source( + &live, + &WindowIdentityEvidence { + pid, + app: Some("TextEdit"), + process_instance: Some(process_instance.as_str()), + title: Some("Untitled"), + bounds_hash: original_hash, + }, + )); +} + +#[test] +fn source_identity_still_rejects_pid_and_application_mismatch() { + let pid = current_pid(); + let live = record("TextEdit", pid, "Untitled", 100); + + assert!(!window_record_matches_source( + &live, + &WindowIdentityEvidence { + pid: pid + 1, + app: Some("TextEdit"), + process_instance: Some(instance(pid).as_str()), + title: Some("Untitled"), + bounds_hash: None, + }, + )); + assert!(!window_record_matches_source( + &live, + &WindowIdentityEvidence { + pid, + app: Some("DifferentApp"), + process_instance: Some(instance(pid).as_str()), + title: Some("Untitled"), + bounds_hash: None, + }, + )); +} + +#[test] +fn source_identity_rejects_missing_or_changed_process_generation() { + let pid = current_pid(); + let live = record("TextEdit", pid, "Untitled", 100); + + assert!(!window_record_matches_source( + &live, + &WindowIdentityEvidence { + pid, + app: Some("TextEdit"), + process_instance: None, + title: Some("Untitled"), + bounds_hash: None, + }, + )); + assert!(!window_record_matches_source( + &live, + &WindowIdentityEvidence { + pid, + app: Some("TextEdit"), + process_instance: Some("different-generation"), + title: Some("Untitled"), + bounds_hash: None, + }, + )); +} + +#[test] +fn verify_skips_title_when_request_title_empty() { + let pid = current_pid(); + let requested = WindowInfo { + id: "w-100".into(), + title: String::new(), + app: "TextEdit".into(), + pid: agent_desktop_core::ProcessId::try_from(pid).unwrap(), + process_instance: Some(instance(pid)), + bounds: None, + state: agent_desktop_core::WindowState::default(), + }; + let live = record("TextEdit", pid, "Any Title", 100); + assert!(verify_window_record(&requested, &live).is_ok()); +} diff --git a/crates/macos/src/system/workspace_apps.rs b/crates/macos/src/system/workspace_apps.rs index 5228158..71e7313 100644 --- a/crates/macos/src/system/workspace_apps.rs +++ b/crates/macos/src/system/workspace_apps.rs @@ -1,134 +1,290 @@ -use agent_desktop_core::node::AppInfo; -use core_foundation::base::CFTypeRef; -use std::{ffi::c_void, sync::OnceLock}; +use agent_desktop_core::{AdapterError, AppInfo, ErrorCode}; +use serde::Deserialize; +use std::time::Instant; -type Id = *mut c_void; -type Class = *mut c_void; -type Sel = *mut c_void; -const RTLD_LAZY: i32 = 1; +const MAX_SNAPSHOT_BYTES: usize = 1024 * 1024; +const MAX_FIELD_BYTES: usize = 16 * 1024; -struct AutoreleasePool(Option<Id>); - -impl AutoreleasePool { - unsafe fn new() -> Self { - let pool = unsafe { objc_autoreleasePoolPush() }; - Self((!pool.is_null()).then_some(pool)) - } +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ActivationPolicy { + Regular, + Accessory, + Prohibited, } -impl Drop for AutoreleasePool { - fn drop(&mut self) { - if let Some(pool) = self.0 { - unsafe { objc_autoreleasePoolPop(pool) }; +#[derive(Deserialize)] +struct BridgedApplication { + name: String, + pid: i32, + #[serde(default)] + bundle_id: Option<String>, + launch_time: NullableLaunchTime, + activation_policy: ActivationPolicy, +} + +#[derive(Deserialize)] +struct BridgedWorkspaceSnapshot { + applications: Vec<BridgedApplication>, + frontmost_pid: i32, + frontmost_launch_time: NullableLaunchTime, +} + +#[derive(Deserialize)] +#[serde(transparent)] +struct NullableLaunchTime(serde_json::Value); + +impl NullableLaunchTime { + fn value(&self) -> Option<Option<f64>> { + match &self.0 { + serde_json::Value::Null => Some(None), + serde_json::Value::Number(number) => number.as_f64().map(Some), + _ => None, } } } -pub(crate) fn list_apps() -> Vec<AppInfo> { - if !appkit_loaded() { - return Vec::new(); +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct WindowOwner { + pub(crate) pid: i32, + pub(crate) name: String, + pub(crate) bundle_id: Option<String>, + pub(crate) launch_time: Option<f64>, + pub(crate) activation_policy: ActivationPolicy, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct WindowOwnerSnapshot { + owners: Vec<WindowOwner>, + frontmost_pid: Option<i32>, +} + +impl WindowOwnerSnapshot { + pub(crate) fn eligible_pids(&self) -> rustc_hash::FxHashSet<i32> { + self.owners.iter().map(|owner| owner.pid).collect() } - unsafe { - let _pool = AutoreleasePool::new(); - let workspace_cls = objc_getClass(c"NSWorkspace".as_ptr()); - if workspace_cls.is_null() { - return Vec::new(); - } + pub(crate) fn owner(&self, pid: i32) -> Option<&WindowOwner> { + self.owners.iter().find(|owner| owner.pid == pid) + } - let shared_sel = sel_registerName(c"sharedWorkspace".as_ptr()); - let send_class: unsafe extern "C" fn(Class, Sel) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - let workspace = send_class(workspace_cls, shared_sel); - if workspace.is_null() { - return Vec::new(); - } + pub(crate) fn frontmost(&self) -> Option<&WindowOwner> { + self.frontmost_pid.and_then(|pid| self.owner(pid)) + } - let running_sel = sel_registerName(c"runningApplications".as_ptr()); - let send_id: unsafe extern "C" fn(Id, Sel) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - let running = send_id(workspace, running_sel); - if running.is_null() { - return Vec::new(); - } - - apps_from_running_array(running, send_id) + pub(crate) fn same_generation(&self, other: &Self) -> bool { + self == other } } -fn appkit_loaded() -> bool { - static APPKIT_LOADED: OnceLock<bool> = OnceLock::new(); - *APPKIT_LOADED.get_or_init(|| unsafe { - !dlopen( - c"/System/Library/Frameworks/AppKit.framework/AppKit".as_ptr(), - RTLD_LAZY, - ) - .is_null() +pub(crate) fn list_apps_until(deadline: Instant) -> Result<Vec<AppInfo>, AdapterError> { + list_apps_with(deadline, |_| true) +} + +pub(crate) fn list_apps_scoped_until( + name: &str, + bundle_id: Option<&str>, + deadline: Instant, +) -> Result<Vec<AppInfo>, AdapterError> { + list_apps_with(deadline, |app| { + app.name.eq_ignore_ascii_case(name) + && bundle_id.is_none_or(|bundle| { + app.bundle_id + .as_deref() + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(bundle)) + }) }) } -fn apps_from_running_array( - running: Id, - send_id: unsafe extern "C" fn(Id, Sel) -> Id, -) -> Vec<AppInfo> { - unsafe { - let count_sel = sel_registerName(c"count".as_ptr()); - let send_count: unsafe extern "C" fn(Id, Sel) -> usize = - std::mem::transmute(objc_msgSend as *const c_void); - let count = send_count(running, count_sel); +pub(crate) fn window_owner_snapshot_until( + deadline: Instant, +) -> Result<WindowOwnerSnapshot, AdapterError> { + ensure_before_deadline(deadline)?; + crate::system::cocoa_runtime::ensure_cocoa_multithreaded()?; + let bytes = crate::system::appkit_bridge::workspace_snapshot_json()?; + ensure_before_deadline(deadline)?; + window_owner_snapshot_from_json(&bytes, deadline) +} - let object_sel = sel_registerName(c"objectAtIndex:".as_ptr()); - let send_object: unsafe extern "C" fn(Id, Sel, usize) -> Id = - std::mem::transmute(objc_msgSend as *const c_void); - let policy_sel = sel_registerName(c"activationPolicy".as_ptr()); - let send_policy: unsafe extern "C" fn(Id, Sel) -> isize = - std::mem::transmute(objc_msgSend as *const c_void); - let pid_sel = sel_registerName(c"processIdentifier".as_ptr()); - let send_pid: unsafe extern "C" fn(Id, Sel) -> i32 = - std::mem::transmute(objc_msgSend as *const c_void); - let name_sel = sel_registerName(c"localizedName".as_ptr()); - let bundle_sel = sel_registerName(c"bundleIdentifier".as_ptr()); +fn list_apps_with( + deadline: Instant, + include: impl Fn(&BridgedApplication) -> bool, +) -> Result<Vec<AppInfo>, AdapterError> { + ensure_before_deadline(deadline)?; + crate::system::cocoa_runtime::ensure_cocoa_multithreaded()?; + let bytes = crate::system::appkit_bridge::workspace_snapshot_json()?; + ensure_before_deadline(deadline)?; + apps_from_json_with( + &bytes, + deadline, + include, + crate::system::process_identity::token_for_pid, + ) +} - let mut seen_pids = rustc_hash::FxHashSet::default(); - let mut apps = Vec::new(); - for idx in 0..count { - let app = send_object(running, object_sel, idx); - if app.is_null() || send_policy(app, policy_sel) != 0 { - continue; - } +#[cfg(test)] +fn apps_from_json(bytes: &[u8], deadline: Instant) -> Result<Vec<AppInfo>, AdapterError> { + apps_from_json_with( + bytes, + deadline, + |_| true, + crate::system::process_identity::token_for_pid, + ) +} - let pid = send_pid(app, pid_sel); - if pid <= 0 || !seen_pids.insert(pid) { - continue; - } - - if let Some(name) = ns_string(send_id(app, name_sel)) { - apps.push(AppInfo { - name, - pid, - bundle_id: ns_string(send_id(app, bundle_sel)), - }); - } +fn apps_from_json_with( + bytes: &[u8], + deadline: Instant, + include: impl Fn(&BridgedApplication) -> bool, + mut resolve: impl FnMut(i32) -> Result<Option<String>, AdapterError>, +) -> Result<Vec<AppInfo>, AdapterError> { + let bridged = bridged_snapshot(bytes)?; + let mut seen_pids = rustc_hash::FxHashSet::default(); + let mut apps = Vec::with_capacity(bridged.applications.len()); + for app in bridged + .applications + .into_iter() + .filter(|app| app.activation_policy == ActivationPolicy::Regular) + .filter(include) + { + ensure_before_deadline(deadline)?; + if !valid_application(&app) || !seen_pids.insert(app.pid) { + return Err(inventory_error( + "AppKit returned invalid running-application identity", + )); } - - apps + let process_instance = resolve(app.pid)? + .ok_or_else(|| inventory_error("Selected application exited during inventory"))?; + apps.push(AppInfo { + name: app.name, + pid: crate::system::process_identity::from_pid_t(app.pid)?, + bundle_id: app.bundle_id, + process_instance: Some(process_instance), + }); } + ensure_before_deadline(deadline)?; + Ok(apps) } -unsafe fn ns_string(id: Id) -> Option<String> { - if id.is_null() { - return None; +fn window_owner_snapshot_from_json( + bytes: &[u8], + deadline: Instant, +) -> Result<WindowOwnerSnapshot, AdapterError> { + let bridged = bridged_snapshot(bytes)?; + let mut seen_pids = rustc_hash::FxHashSet::default(); + let mut owners = Vec::with_capacity(bridged.applications.len()); + for app in bridged + .applications + .into_iter() + .filter(|app| app.activation_policy != ActivationPolicy::Prohibited) + { + ensure_before_deadline(deadline)?; + let Some(launch_time) = app.launch_time.value() else { + return Err(inventory_error( + "AppKit returned invalid window-owner identity", + )); + }; + if launch_time.is_some_and(|value| !valid_launch_time(value)) + || !valid_application(&app) + || !seen_pids.insert(app.pid) + { + return Err(inventory_error( + "AppKit returned invalid window-owner identity", + )); + } + owners.push(WindowOwner { + pid: app.pid, + name: app.name, + bundle_id: app.bundle_id, + launch_time, + activation_policy: app.activation_policy, + }); } - crate::cf_type::borrowed_cf_string(id as CFTypeRef).map(|value| value.to_string()) + owners.sort_unstable_by_key(|owner| owner.pid); + let Some(frontmost_launch_time) = bridged.frontmost_launch_time.value() else { + return Err(inventory_error( + "AppKit returned incomplete frontmost-application identity", + )); + }; + let frontmost_pid = + validate_frontmost(&mut owners, bridged.frontmost_pid, frontmost_launch_time)?; + ensure_before_deadline(deadline)?; + Ok(WindowOwnerSnapshot { + owners, + frontmost_pid, + }) } -unsafe extern "C" { - fn objc_autoreleasePoolPush() -> Id; - fn objc_autoreleasePoolPop(pool: Id); - fn dlopen(filename: *const core::ffi::c_char, flag: i32) -> Id; - fn objc_getClass(name: *const core::ffi::c_char) -> Class; - fn sel_registerName(name: *const core::ffi::c_char) -> Sel; - fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; +fn validate_frontmost( + owners: &mut [WindowOwner], + pid: i32, + launch_time: Option<f64>, +) -> Result<Option<i32>, AdapterError> { + if pid == 0 && launch_time.is_none() { + return Ok(None); + } + if pid <= 0 || launch_time.is_some_and(|value| !valid_launch_time(value)) { + return Err(inventory_error( + "AppKit returned incomplete frontmost-application identity", + )); + } + let Some(owner) = owners.iter_mut().find(|owner| owner.pid == pid) else { + return Err(inventory_error( + "AppKit frontmost application did not exactly match an eligible window owner", + )); + }; + match (owner.launch_time, launch_time) { + (Some(owner_launch_time), Some(frontmost_launch_time)) + if owner_launch_time.to_bits() != frontmost_launch_time.to_bits() => + { + return Err(inventory_error( + "AppKit frontmost application did not exactly match an eligible window owner", + )); + } + (None, Some(frontmost_launch_time)) => { + owner.launch_time = Some(frontmost_launch_time); + } + _ => {} + } + Ok(Some(pid)) +} + +fn bridged_snapshot(bytes: &[u8]) -> Result<BridgedWorkspaceSnapshot, AdapterError> { + if bytes.len() > MAX_SNAPSHOT_BYTES { + return Err(inventory_error("AppKit returned an oversized snapshot")); + } + serde_json::from_slice(bytes).map_err(|_| inventory_error("AppKit returned invalid JSON")) +} + +fn valid_application(app: &BridgedApplication) -> bool { + app.pid > 0 + && !app.name.trim().is_empty() + && app.name.len() <= MAX_FIELD_BYTES + && app + .bundle_id + .as_ref() + .is_none_or(|bundle_id| bundle_id.len() <= MAX_FIELD_BYTES) +} + +fn valid_launch_time(value: f64) -> bool { + value.is_finite() && value > 0.0 +} + +fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { + if Instant::now() >= deadline { + return Err(AdapterError::timeout("NSWorkspace app inventory timed out")); + } + Ok(()) +} + +fn inventory_error(message: &str) -> AdapterError { + AdapterError::new(ErrorCode::AppUnresponsive, message) + .with_suggestion("Retry after macOS finishes updating the running-application inventory") + .with_details(serde_json::json!({ + "kind": "inventory_source", + "source": "ns_workspace", + "retryable": true, + })) } #[cfg(test)] diff --git a/crates/macos/src/system/workspace_apps_tests.rs b/crates/macos/src/system/workspace_apps_tests.rs index 4424c7c..36fda35 100644 --- a/crates/macos/src/system/workspace_apps_tests.rs +++ b/crates/macos/src/system/workspace_apps_tests.rs @@ -1,28 +1,291 @@ use super::*; -use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; -#[test] -fn ns_string_rejects_null() { - assert_eq!(unsafe { ns_string(std::ptr::null_mut()) }, None); +fn deadline() -> Instant { + Instant::now() + std::time::Duration::from_secs(1) } #[test] -fn ns_string_rejects_non_string_object() { - let number = CFNumber::from(42); - let value = number.as_CFTypeRef() as Id; +fn malformed_bridge_payload_is_retryable_inventory_failure() { + let error = apps_from_json(b"not-json", deadline()).unwrap_err(); - assert_eq!(unsafe { ns_string(value) }, None); + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.details.unwrap()["retryable"], true); } #[test] -fn ns_string_accepts_cf_string_object() { - let string = CFString::new("Mail"); - let value = string.as_CFTypeRef() as Id; +fn owner_snapshot_includes_regular_and_accessory_but_excludes_prohibited() { + let bytes = br#"{ + "applications":[ + {"name":"Mail","pid":10,"bundle_id":"com.apple.mail","launch_time":100.25,"activation_policy":"regular"}, + {"name":"Palette","pid":11,"launch_time":101.5,"activation_policy":"accessory"}, + {"name":"Daemon","pid":12,"launch_time":102.75,"activation_policy":"prohibited"} + ], + "frontmost_pid":11, + "frontmost_launch_time":101.5 + }"#; + let snapshot = window_owner_snapshot_from_json(bytes, deadline()).unwrap(); - assert_eq!(unsafe { ns_string(value) }.as_deref(), Some("Mail")); + assert_eq!(snapshot.eligible_pids().len(), 2); + assert!(snapshot.eligible_pids().contains(&10)); + assert!(snapshot.eligible_pids().contains(&11)); + assert!(snapshot.owner(12).is_none()); + assert_eq!(snapshot.owner(10).unwrap().name, "Mail"); + assert_eq!( + snapshot.owner(10).unwrap().bundle_id.as_deref(), + Some("com.apple.mail") + ); + assert_eq!( + snapshot.owner(10).unwrap().activation_policy, + ActivationPolicy::Regular + ); + assert_eq!( + snapshot.owner(11).unwrap().activation_policy, + ActivationPolicy::Accessory + ); + assert_eq!(snapshot.frontmost().unwrap().pid, 11); + assert_eq!(snapshot.frontmost().unwrap().launch_time, Some(101.5)); } #[test] -fn autorelease_pool_ignores_null_push_result() { - drop(AutoreleasePool(None)); +fn non_launchservices_accessory_keeps_optional_launch_identity() { + let bytes = br#"{ + "applications":[ + {"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"}, + {"name":"Palette","pid":11,"launch_time":null,"activation_policy":"accessory"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.25 + }"#; + let snapshot = window_owner_snapshot_from_json(bytes, deadline()).unwrap(); + + assert!(snapshot.eligible_pids().contains(&11)); + assert_eq!(snapshot.owner(11).unwrap().launch_time, None); +} + +#[test] +fn frontmost_requires_an_exact_owner_generation_match() { + let bytes = br#"{ + "applications":[ + {"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.2500001 + }"#; + let error = window_owner_snapshot_from_json(bytes, deadline()).unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.details.unwrap()["source"], "ns_workspace"); +} + +#[test] +fn zero_and_null_are_the_explicit_missing_frontmost_pair() { + let bytes = br#"{ + "applications":[ + {"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"} + ], + "frontmost_pid":0, + "frontmost_launch_time":null + }"#; + let snapshot = window_owner_snapshot_from_json(bytes, deadline()).unwrap(); + + assert!(snapshot.frontmost().is_none()); +} + +#[test] +fn non_launchservices_frontmost_app_uses_live_process_identity() { + let bytes = br#"{ + "applications":[ + {"name":"Direct","pid":10,"launch_time":null,"activation_policy":"regular"} + ], + "frontmost_pid":10, + "frontmost_launch_time":null + }"#; + let snapshot = window_owner_snapshot_from_json(bytes, deadline()).unwrap(); + + assert_eq!(snapshot.frontmost().unwrap().pid, 10); + assert_eq!(snapshot.frontmost().unwrap().launch_time, None); +} + +#[test] +fn incomplete_or_contradictory_frontmost_identity_is_rejected() { + for bytes in [ + br#"{"applications":[],"frontmost_pid":0}"#.as_slice(), + br#"{"applications":[],"frontmost_pid":0,"frontmost_launch_time":100.0}"#.as_slice(), + br#"{"applications":[],"frontmost_pid":10,"frontmost_launch_time":null}"#.as_slice(), + ] { + let error = window_owner_snapshot_from_json(bytes, deadline()).unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + } +} + +#[test] +fn duplicate_owner_pid_is_rejected_for_same_or_different_generation() { + for launch_time in [100.25, 101.5] { + let bytes = format!( + r#"{{ + "applications":[ + {{"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"}}, + {{"name":"Palette","pid":10,"launch_time":{launch_time},"activation_policy":"accessory"}} + ], + "frontmost_pid":0, + "frontmost_launch_time":null + }}"# + ); + let error = window_owner_snapshot_from_json(bytes.as_bytes(), deadline()).unwrap_err(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + } +} + +#[test] +fn owner_snapshot_equality_detects_generation_and_frontmost_changes() { + let baseline = window_owner_snapshot_from_json( + br#"{ + "applications":[ + {"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"}, + {"name":"Palette","pid":11,"launch_time":101.5,"activation_policy":"accessory"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.25 + }"#, + deadline(), + ) + .unwrap(); + let reordered = window_owner_snapshot_from_json( + br#"{ + "applications":[ + {"name":"Palette","pid":11,"launch_time":101.5,"activation_policy":"accessory"}, + {"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.25 + }"#, + deadline(), + ) + .unwrap(); + let changed = window_owner_snapshot_from_json( + br#"{ + "applications":[ + {"name":"Mail","pid":10,"launch_time":100.5,"activation_policy":"regular"}, + {"name":"Palette","pid":11,"launch_time":101.5,"activation_policy":"accessory"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.5 + }"#, + deadline(), + ) + .unwrap(); + + assert_eq!(baseline, reordered); + assert!(baseline.same_generation(&reordered)); + assert_ne!(baseline, changed); + assert!(!baseline.same_generation(&changed)); +} + +#[test] +fn app_inventory_keeps_only_regular_activation_policy() { + let bytes = br#"{ + "applications":[ + {"name":"Mail","pid":10,"launch_time":100.25,"activation_policy":"regular"}, + {"name":"Palette","pid":11,"launch_time":101.5,"activation_policy":"accessory"}, + {"name":"Daemon","pid":12,"launch_time":102.75,"activation_policy":"prohibited"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.25 + }"#; + let mut probed = Vec::new(); + let apps = apps_from_json_with( + bytes, + deadline(), + |_| true, + |pid| { + probed.push(pid); + Ok(Some(format!("instance-{pid}"))) + }, + ) + .unwrap(); + + assert_eq!(probed, [10]); + assert_eq!(apps.len(), 1); + assert_eq!(apps[0].name, "Mail"); +} + +#[test] +fn expired_workspace_deadline_is_rejected_before_native_reads() { + let error = list_apps_until(Instant::now()).unwrap_err(); + + assert_eq!(error.code.as_str(), "TIMEOUT"); +} + +#[test] +fn scoped_lookup_never_probes_unrelated_applications() { + let bytes = br#"{ + "applications":[ + {"name":"Target","pid":10,"launch_time":100.25,"activation_policy":"regular"}, + {"name":"Other","pid":418,"launch_time":101.5,"activation_policy":"regular"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.25 + }"#; + let mut probed = Vec::new(); + let apps = apps_from_json_with( + bytes, + deadline(), + |app| app.name == "Target", + |pid| { + probed.push(pid); + if pid == 418 { + Err(AdapterError::permission_denied()) + } else { + Ok(Some(format!("instance-{pid}"))) + } + }, + ) + .unwrap(); + + assert_eq!(probed, [10]); + assert_eq!(apps[0].process_instance.as_deref(), Some("instance-10")); +} + +#[test] +fn scoped_lookup_propagates_selected_identity_denial() { + let bytes = br#"{ + "applications":[ + {"name":"Target","pid":418,"launch_time":100.25,"activation_policy":"regular"} + ], + "frontmost_pid":418, + "frontmost_launch_time":100.25 + }"#; + let error = apps_from_json_with( + bytes, + deadline(), + |_| true, + |_| Err(AdapterError::permission_denied()), + ) + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::PermDenied); +} + +#[test] +fn scoped_lookup_preserves_all_same_name_processes() { + let bytes = br#"{ + "applications":[ + {"name":"Target","pid":10,"launch_time":100.25,"activation_policy":"regular"}, + {"name":"target","pid":11,"launch_time":101.5,"activation_policy":"regular"}, + {"name":"Other","pid":12,"launch_time":102.75,"activation_policy":"regular"} + ], + "frontmost_pid":10, + "frontmost_launch_time":100.25 + }"#; + let apps = apps_from_json_with( + bytes, + deadline(), + |app| app.name.eq_ignore_ascii_case("Target"), + |pid| Ok(Some(format!("instance-{pid}"))), + ) + .unwrap(); + + assert_eq!(apps.iter().map(|app| app.pid).collect::<Vec<_>>(), [10, 11]); } diff --git a/crates/macos/src/tree/action_list.rs b/crates/macos/src/tree/action_list.rs index 88869fb..7679aaa 100644 --- a/crates/macos/src/tree/action_list.rs +++ b/crates/macos/src/tree/action_list.rs @@ -1,61 +1,147 @@ use super::AXElement; -use super::capabilities::{copy_action_names, is_attr_settable}; +use super::capabilities::{copy_action_names_with_status, is_attr_settable_with_status}; use agent_desktop_core::capability; #[cfg(target_os = "macos")] use accessibility_sys::{kAXFocusedAttribute, kAXValueAttribute}; +pub(crate) struct AvailableActionsRead { + pub(crate) actions: Vec<String>, + pub(crate) complete: bool, + pub(crate) cannot_complete: bool, + pub(crate) invalid_element: bool, + pub(crate) api_disabled: bool, + pub(crate) deadline_exhausted: bool, + pub(crate) settable_reads: u64, +} + +impl Default for AvailableActionsRead { + fn default() -> Self { + Self { + actions: Vec::new(), + complete: true, + cannot_complete: false, + invalid_element: false, + api_disabled: false, + deadline_exhausted: false, + settable_reads: 0, + } + } +} + #[cfg(target_os = "macos")] -pub(crate) fn platform_available_actions( +pub(crate) fn read_platform_available_actions( el: &AXElement, role: &str, has_scrollbars: bool, -) -> Vec<String> { - let ax_actions = copy_action_names(el); + deadline: std::time::Instant, + usage: &mut crate::tree::observation_usage::ObservationUsage, +) -> AvailableActionsRead { + let mut read = AvailableActionsRead::default(); + if crate::tree::locator_deadline::prepare(el, deadline).is_err() { + read.complete = false; + read.deadline_exhausted = true; + return read; + } + let native_actions = copy_action_names_with_status(el, deadline, usage); + if let Some(error) = native_actions.error { + record_error(&mut read, error); + } + let ax_actions = native_actions.value.unwrap_or_default(); let has = |name: &str| ax_actions.iter().any(|a| a == name); - let mut actions = Vec::new(); if has("AXPress") { - push_unique(&mut actions, capability::CLICK); + push_unique(&mut read.actions, capability::CLICK); if crate::tree::roles::is_toggleable_role(role) { - push_unique(&mut actions, capability::TOGGLE); - } - if matches!(role, "combobox" | "menuitem" | "tab") { - push_unique(&mut actions, capability::SELECT); + push_unique(&mut read.actions, capability::TOGGLE); } } if has("AXShowMenu") && role_allows_context_menu_action(role) { - push_unique(&mut actions, capability::RIGHT_CLICK); + push_unique(&mut read.actions, capability::RIGHT_CLICK); } if has("AXScrollToVisible") { - push_unique(&mut actions, capability::SCROLL); - push_unique(&mut actions, capability::SCROLL_TO); + push_unique(&mut read.actions, capability::SCROLL_TO); } - if has_scroll_mechanism(role, &has, has_scrollbars) { - push_unique(&mut actions, capability::SCROLL); + if has_scroll_mechanism(&has, has_scrollbars) { + push_unique(&mut read.actions, capability::SCROLL); } - if has("AXIncrement") - || has("AXDecrement") - || (role_may_bear_value(role) && is_attr_settable(el, kAXValueAttribute)) + let value_settable = role_may_bear_value(role) + && read_settable(el, kAXValueAttribute, deadline, &mut read).unwrap_or(false); + if has("AXIncrement") || has("AXDecrement") || value_settable { + push_unique(&mut read.actions, capability::SET_VALUE); + } + if (role == "combobox" && value_settable) || role_supports_collection_select(role) { + push_unique(&mut read.actions, capability::SELECT); + } + if role_may_accept_focus(role) + && read_settable(el, kAXFocusedAttribute, deadline, &mut read).unwrap_or(false) { - push_unique(&mut actions, capability::SET_VALUE); + push_unique(&mut read.actions, capability::SET_FOCUS); } - if role_may_accept_focus(role) && is_attr_settable(el, kAXFocusedAttribute) { - push_unique(&mut actions, capability::SET_FOCUS); + if role_may_insert_text(role) + && read_settable(el, "AXSelectedText", deadline, &mut read).unwrap_or(false) + { + push_unique(&mut read.actions, capability::TYPE_TEXT); } - if (role_may_expand(role) && is_attr_settable(el, "AXExpanded")) + if (role_may_expand(role) + && read_settable(el, "AXExpanded", deadline, &mut read).unwrap_or(false)) || (has("AXPress") && agent_desktop_core::roles::is_expandable_role(role)) { - push_unique(&mut actions, capability::EXPAND); - push_unique(&mut actions, capability::COLLAPSE); + push_unique(&mut read.actions, capability::EXPAND); + push_unique(&mut read.actions, capability::COLLAPSE); } - actions + read +} + +#[cfg(target_os = "macos")] +fn read_settable( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + read: &mut AvailableActionsRead, +) -> Option<bool> { + if crate::tree::locator_deadline::prepare(element, deadline).is_err() { + read.complete = false; + read.deadline_exhausted = true; + return None; + } + read.settable_reads += 1; + let result = is_attr_settable_with_status(element, attribute, deadline); + if let Some(error) = result.error { + record_error(read, error); + } + result.value +} + +#[cfg(target_os = "macos")] +fn record_error(read: &mut AvailableActionsRead, error: i32) { + if is_definitive_absence(error) { + return; + } + read.complete = false; + read.cannot_complete |= error == accessibility_sys::kAXErrorCannotComplete; + read.invalid_element |= error == accessibility_sys::kAXErrorInvalidUIElement; + read.api_disabled |= error == accessibility_sys::kAXErrorAPIDisabled; +} + +/// AppKit's NSAccessibility bridge answers these codes when an element simply +/// does not implement the probed attribute or action list — including +/// `kAXErrorFailure`, which it returns for synthetic `NSAccessibilityElement`s +/// with no action support. They are complete "none present" answers, not +/// transport failures, so they must not mark the read incomplete. +#[cfg(target_os = "macos")] +fn is_definitive_absence(error: i32) -> bool { + matches!( + error, + accessibility_sys::kAXErrorAttributeUnsupported + | accessibility_sys::kAXErrorNoValue + | accessibility_sys::kAXErrorNotImplemented + | accessibility_sys::kAXErrorActionUnsupported + | accessibility_sys::kAXErrorFailure + ) } -/// Whether a role could carry a settable `AXValue`, so the `is_settable` probe -/// is worth an IPC. Click/navigation-only roles never do; `unknown` always -/// probes so an unmapped role never loses a capability. fn role_may_bear_value(role: &str) -> bool { matches!( role, @@ -70,48 +156,42 @@ fn role_may_bear_value(role: &str) -> bool { | "switch" | "colorwell" | "scrollbar" - | "valueindicator" - | "unknown" + | "handle" ) } -/// Whether a role could carry a settable `AXFocused`, so the `is_settable` -/// probe is worth an IPC. Interactive controls and focus-holding containers -/// (tables, outlines, web areas) can; static/decorative roles never do. -/// `unknown` always probes so an unmapped role never loses a capability. fn role_may_accept_focus(role: &str) -> bool { agent_desktop_core::roles::is_interactive_role(role) || matches!( role, - "table" - | "outline" - | "list" - | "browser" - | "webarea" - | "scrollarea" - | "group" - | "row" - | "unknown" + "table" | "outline" | "list" | "browser" | "webarea" | "scrollarea" | "group" | "row" ) } -/// Whether a role could expose a settable `AXExpanded`. Leaf/interactive roles -/// never expand; `unknown` always probes. fn role_may_expand(role: &str) -> bool { agent_desktop_core::roles::is_expandable_role(role) || matches!( role, - "group" | "outline" | "row" | "browser" | "table" | "list" | "cell" | "unknown" + "group" | "outline" | "row" | "browser" | "table" | "list" | "cell" ) } +fn role_may_insert_text(role: &str) -> bool { + matches!(role, "textfield" | "combobox") +} + #[cfg(not(target_os = "macos"))] -pub(crate) fn platform_available_actions( +pub(crate) fn read_platform_available_actions( _el: &AXElement, _role: &str, _has_scrollbars: bool, -) -> Vec<String> { - Vec::new() + _deadline: std::time::Instant, + _usage: &mut crate::tree::observation_usage::ObservationUsage, +) -> AvailableActionsRead { + AvailableActionsRead { + complete: false, + ..AvailableActionsRead::default() + } } fn push_unique(actions: &mut Vec<String>, action: &str) { @@ -124,49 +204,38 @@ fn role_allows_context_menu_action(role: &str) -> bool { !matches!(role, "combobox" | "menubutton") } -fn has_scroll_mechanism(role: &str, has: &impl Fn(&str) -> bool, has_scrollbars: bool) -> bool { - role_supports_scroll(role) - || has("AXScrollDownByPage") +fn role_supports_collection_select(role: &str) -> bool { + matches!(role, "list" | "table" | "outline") +} + +fn has_scroll_mechanism(has: &impl Fn(&str) -> bool, has_scrollbars: bool) -> bool { + has("AXScrollDownByPage") || has("AXScrollUpByPage") || has("AXScrollLeftByPage") || has("AXScrollRightByPage") - || (role_may_own_scrollbars(role) && has_scrollbars) -} - -fn role_supports_scroll(role: &str) -> bool { - matches!( - role, - "scrollarea" | "browser" | "table" | "outline" | "list" - ) -} - -fn role_may_own_scrollbars(role: &str) -> bool { - matches!( - role, - "application" - | "window" - | "sheet" - | "dialog" - | "group" - | "splitter" - | "webarea" - | "grid" - | "unknown" - ) + || has_scrollbars } #[cfg(test)] mod tests { + #[cfg(target_os = "macos")] + use super::{AvailableActionsRead, record_error}; use super::{ - role_allows_context_menu_action, role_may_accept_focus, role_may_own_scrollbars, - role_supports_scroll, + has_scroll_mechanism, role_allows_context_menu_action, role_may_accept_focus, + role_may_bear_value, role_may_insert_text, role_supports_collection_select, }; + #[test] + fn canonical_handle_role_is_probed_for_settable_value() { + assert!(role_may_bear_value("handle")); + assert!(!role_may_bear_value("valueindicator")); + } + #[test] fn focus_probe_is_limited_to_focus_bearing_roles() { assert!(role_may_accept_focus("textfield")); assert!(role_may_accept_focus("webarea")); - assert!(role_may_accept_focus("unknown")); + assert!(!role_may_accept_focus("unknown")); assert!(!role_may_accept_focus("statictext")); assert!(!role_may_accept_focus("image")); } @@ -180,17 +249,72 @@ mod tests { } #[test] - fn scroll_container_roles_advertise_scroll_without_scroll_to() { - assert!(role_supports_scroll("scrollarea")); - assert!(role_supports_scroll("browser")); - assert!(!role_supports_scroll("button")); + fn collection_containers_advertise_the_restored_select_contract() { + assert!(role_supports_collection_select("list")); + assert!(role_supports_collection_select("table")); + assert!(role_supports_collection_select("outline")); + assert!(!role_supports_collection_select("group")); } #[test] - fn scrollbar_probe_is_limited_to_container_like_roles() { - assert!(role_may_own_scrollbars("group")); - assert!(role_may_own_scrollbars("webarea")); - assert!(!role_may_own_scrollbars("button")); - assert!(!role_may_own_scrollbars("cell")); + fn scroll_requires_native_directional_actions_or_concrete_scrollbars() { + assert!(has_scroll_mechanism( + &|name| name == "AXScrollDownByPage", + false + )); + assert!(has_scroll_mechanism(&|_| false, true)); + assert!(!has_scroll_mechanism(&|_| false, false)); + } + + #[test] + fn text_insertion_probe_is_limited_to_editable_text_roles() { + assert!(role_may_insert_text("textfield")); + assert!(role_may_insert_text("combobox")); + assert!(!role_may_insert_text("unknown")); + assert!(!role_may_insert_text("button")); + } + + #[test] + #[cfg(target_os = "macos")] + fn failed_native_action_reads_remain_incomplete() { + let mut read = AvailableActionsRead::default(); + + record_error(&mut read, accessibility_sys::kAXErrorCannotComplete); + + assert!(!read.complete); + assert!(read.cannot_complete); + assert!(read.actions.is_empty()); + } + + #[test] + #[cfg(target_os = "macos")] + fn definitive_absence_codes_are_complete_empty_answers() { + for error in [ + accessibility_sys::kAXErrorAttributeUnsupported, + accessibility_sys::kAXErrorNoValue, + accessibility_sys::kAXErrorNotImplemented, + accessibility_sys::kAXErrorActionUnsupported, + accessibility_sys::kAXErrorFailure, + ] { + let mut read = AvailableActionsRead::default(); + record_error(&mut read, error); + assert!(read.complete, "code {error} must stay complete"); + assert!(!read.cannot_complete); + assert!(read.actions.is_empty()); + } + } + + #[test] + #[cfg(target_os = "macos")] + fn transport_failures_are_never_classified_as_absence() { + for error in [ + accessibility_sys::kAXErrorCannotComplete, + accessibility_sys::kAXErrorInvalidUIElement, + accessibility_sys::kAXErrorAPIDisabled, + ] { + let mut read = AvailableActionsRead::default(); + record_error(&mut read, error); + assert!(!read.complete, "code {error} must mark the read incomplete"); + } } } diff --git a/crates/macos/src/tree/adapter.rs b/crates/macos/src/tree/adapter.rs new file mode 100644 index 0000000..a7d8948 --- /dev/null +++ b/crates/macos/src/tree/adapter.rs @@ -0,0 +1,178 @@ +use crate::adapter::{MacOSAdapter, ax_element}; +use agent_desktop_core::{ + AccessibilityNode, AdapterError, AppInfo, Deadline, ElementState, LiveElement, NativeHandle, + ObservationOps, ObservationRequest, ObservationRoot, ObservedTree, Rect, RefEntry, SurfaceInfo, + TreeOptions, WindowFilter, WindowInfo, +}; + +impl ObservationOps for MacOSAdapter { + fn observe_tree( + &self, + root: ObservationRoot<'_>, + request: &ObservationRequest, + ) -> Result<ObservedTree, AdapterError> { + crate::tree::query::observe_tree(root, request) + } + + fn get_tree( + &self, + window: &WindowInfo, + options: &TreeOptions, + deadline: Deadline, + ) -> Result<AccessibilityNode, AdapterError> { + self.observe_tree( + ObservationRoot::Window(window), + &ObservationRequest::snapshot(options, deadline), + )? + .into_accessibility_tree() + } + + fn resolve_element_strict( + &self, + entry: &RefEntry, + deadline: Deadline, + ) -> Result<NativeHandle, AdapterError> { + crate::tree::resolve::resolve_element_with_deadline(entry, deadline) + } + + fn resolve_locator_anchor( + &self, + entry: &RefEntry, + deadline: Deadline, + ) -> Result<NativeHandle, AdapterError> { + crate::tree::resolve::resolve_locator_anchor_with_deadline(entry, deadline) + } + + fn list_windows( + &self, + filter: &WindowFilter, + deadline: Deadline, + ) -> Result<Vec<WindowInfo>, AdapterError> { + crate::system::app_inventory::list_windows_until( + filter, + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + + fn list_apps(&self, deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError> { + crate::system::app_inventory::list_apps_complete_until( + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + + fn list_apps_scoped( + &self, + name: &str, + bundle_id: Option<&str>, + deadline: Deadline, + ) -> Result<Vec<AppInfo>, AdapterError> { + crate::system::app_inventory::list_apps_scoped_until( + name, + bundle_id, + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + + fn list_surfaces( + &self, + process: agent_desktop_core::ProcessIdentity, + deadline: Deadline, + ) -> Result<Vec<SurfaceInfo>, AdapterError> { + let identity = crate::system::process_identity::require_core(&process)?; + let surfaces = crate::tree::surface_inventory::list_surfaces_for_pid( + identity.pid(), + crate::tree::locator_deadline::from_operation(deadline)?, + )?; + crate::system::process_identity::require_core(&process)?; + Ok(surfaces) + } + + fn hit_test( + &self, + handle: &NativeHandle, + point: agent_desktop_core::Point, + deadline: Deadline, + ) -> Result<agent_desktop_core::hit_test::HitTestResult, AdapterError> { + crate::tree::hit_test::hit_test_impl(handle, point, deadline) + } + + fn get_live_value( + &self, + handle: &NativeHandle, + deadline: Deadline, + ) -> Result<Option<String>, AdapterError> { + #[cfg(target_os = "macos")] + { + Ok( + crate::actions::post_state::read_element_state(ax_element(handle)?, deadline)? + .value, + ) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("get_live_value")) + } + + fn get_live_state( + &self, + handle: &NativeHandle, + deadline: Deadline, + ) -> Result<Option<ElementState>, AdapterError> { + #[cfg(target_os = "macos")] + { + Ok(Some(crate::actions::post_state::read_element_state( + ax_element(handle)?, + deadline, + )?)) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("get_live_state")) + } + + fn get_live_actions( + &self, + handle: &NativeHandle, + deadline: Deadline, + ) -> Result<Option<Vec<String>>, AdapterError> { + #[cfg(target_os = "macos")] + { + Ok(Some(crate::actions::post_state::read_live_actions( + ax_element(handle)?, + deadline, + )?)) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("get_live_actions")) + } + + fn get_live_element( + &self, + handle: &NativeHandle, + deadline: Deadline, + ) -> Result<LiveElement, AdapterError> { + #[cfg(target_os = "macos")] + { + crate::actions::post_state::read_live_element(ax_element(handle)?, deadline) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("get_live_element")) + } + + fn get_element_bounds( + &self, + handle: &NativeHandle, + deadline: Deadline, + ) -> Result<Option<Rect>, AdapterError> { + #[cfg(target_os = "macos")] + { + crate::tree::element_bounds::read_bounds_with_deadline( + ax_element(handle)?, + crate::tree::locator_deadline::from_operation(deadline)?, + ) + } + #[cfg(not(target_os = "macos"))] + { + let _ = handle; + Err(AdapterError::not_supported("get_element_bounds")) + } + } +} diff --git a/crates/macos/src/tree/attributes.rs b/crates/macos/src/tree/attributes.rs index 7314bd2..e8053eb 100644 --- a/crates/macos/src/tree/attributes.rs +++ b/crates/macos/src/tree/attributes.rs @@ -4,171 +4,124 @@ mod imp { cf_type::created_cf_array, tree::{ax_element::AXElement, ax_value}, }; - use accessibility_sys::{ - AXUIElementCopyAttributeValue, AXUIElementCopyAttributeValues, - AXUIElementCopyMultipleAttributeValues, AXUIElementSetMessagingTimeout, kAXErrorSuccess, - kAXValueAttribute, - }; + + const MALFORMED_AX_VALUE: i32 = i32::MIN; + use accessibility_sys::{kAXErrorAttributeUnsupported, kAXErrorNoValue, kAXErrorSuccess}; use core_foundation::{ array::CFArray, base::{CFType, CFTypeRef, TCFType}, boolean::CFBoolean, - number::CFNumber, string::CFString, }; - pub fn copy_string_attr(el: &AXElement, attr: &str) -> Option<String> { + pub fn set_messaging_timeout( + el: &AXElement, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<(), agent_desktop_core::AdapterError> { + crate::tree::ax_ipc::prepare(el, deadline).map(|_| ()) + } + + pub fn copy_bool_attr( + el: &AXElement, + attr: &str, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Option<bool> { + copy_bool_attr_result(el, attr, deadline).ok().flatten() + } + + pub(crate) fn copy_bool_attr_result( + el: &AXElement, + attr: &str, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<bool>, i32> { let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; - cf_type.downcast::<CFString>().map(|s| s.to_string()) - } - - pub fn set_messaging_timeout(el: &AXElement, timeout: std::time::Duration) { - if el.0.is_null() { - return; - } - let seconds = timeout.as_secs_f32().clamp(0.001, 2.0); - unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) }; - } - - pub fn copy_value_typed(el: &AXElement) -> Option<String> { - let cf_attr = CFString::new(kAXValueAttribute); - let mut val_ref: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut val_ref) - }; - if err != kAXErrorSuccess || val_ref.is_null() { - return None; - } - let cf = unsafe { CFType::wrap_under_create_rule(val_ref) }; - if let Some(s) = cf.downcast::<CFString>() { - return Some(s.to_string()); - } - if let Some(b) = cf.downcast::<CFBoolean>() { - return Some(bool::from(b).to_string()); - } - if let Some(n) = cf.downcast::<CFNumber>() { - if let Some(i) = n.to_i64() { - return Some(i.to_string()); - } - if let Some(f) = n.to_f64() { - return Some(format!("{:.2}", f)); + let (err, value) = + crate::tree::ax_ipc::copy_attribute_value(el, cf_attr.as_concrete_TypeRef(), deadline); + if err != kAXErrorSuccess { + if !value.is_null() { + unsafe { core_foundation::base::CFRelease(value) }; } + return if is_absent_error(err) { + Ok(None) + } else { + Err(err) + }; } - None - } - - pub fn copy_bool_attr(el: &AXElement, attr: &str) -> Option<bool> { - let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; + if value.is_null() { + return Ok(None); } let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; - cf_type.downcast::<CFBoolean>().map(|b| b.into()) + cf_type + .downcast::<CFBoolean>() + .map(|value| Some(value.into())) + .ok_or(MALFORMED_AX_VALUE) } - pub fn copy_i64_attr(el: &AXElement, attr: &str) -> Option<i64> { - let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; - cf_type.downcast::<CFNumber>().and_then(|n| n.to_i64()) - } - - pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option<Vec<AXElement>> { - let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let arr = created_cf_array(value)?; - Some(ax_array_items(arr)) - } - - pub fn copy_ax_array_prefix( + pub(crate) fn copy_ax_array_prefix_result( el: &AXElement, attr: &str, max_values: usize, - ) -> Option<Vec<AXElement>> { + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<Vec<AXElement>>, i32> { if max_values == 0 { - return Some(Vec::new()); + return Ok(Some(Vec::new())); } let cf_attr = CFString::new(attr); - let mut value: core_foundation_sys::array::CFArrayRef = std::ptr::null(); - let err = unsafe { - AXUIElementCopyAttributeValues( - el.0, - cf_attr.as_concrete_TypeRef(), - 0, - max_values as core_foundation_sys::base::CFIndex, - &mut value, - ) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; + let (err, value) = crate::tree::ax_ipc::copy_attribute_values( + el, + cf_attr.as_concrete_TypeRef(), + 0, + max_values as core_foundation_sys::base::CFIndex, + deadline, + ); + if err != kAXErrorSuccess { + if !value.is_null() { + drop(created_cf_array(value as CFTypeRef)); + } + return if is_absent_error(err) { + Ok(None) + } else { + Err(err) + }; } - let arr = created_cf_array(value as CFTypeRef)?; - Some(ax_array_items(arr)) + if value.is_null() { + return Ok(None); + } + let Some(arr) = created_cf_array(value as CFTypeRef) else { + return Err(MALFORMED_AX_VALUE); + }; + let expected = arr.len() as usize; + let elements = ax_array_items(arr); + if elements.len() != expected { + return Err(MALFORMED_AX_VALUE); + } + Ok(Some(elements)) } - pub fn copy_element_attr(el: &AXElement, attr: &str) -> Option<AXElement> { + pub(crate) fn copy_element_attr_result( + el: &AXElement, + attr: &str, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<AXElement>, i32> { let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; + let (err, value) = + crate::tree::ax_ipc::copy_attribute_value(el, cf_attr.as_concrete_TypeRef(), deadline); + if err != kAXErrorSuccess { + if !value.is_null() { + unsafe { core_foundation::base::CFRelease(value) }; + } + return if is_absent_error(err) { + Ok(None) + } else { + Err(err) + }; + } + if value.is_null() { + return Ok(None); } ax_value::created_ax_element(value) - } - - pub fn copy_first_element_attr(el: &AXElement, attrs: &[&str]) -> Option<AXElement> { - if attrs.is_empty() { - return None; - } - let cf_names: Vec<CFString> = attrs.iter().map(|attr| CFString::new(attr)).collect(); - let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect(); - let names_arr = CFArray::from_copyable(&cf_refs); - let mut result_ref: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyMultipleAttributeValues( - el.0, - names_arr.as_concrete_TypeRef(), - 0, - &mut result_ref as *mut _ as *mut _, - ) - }; - if !result_ref.is_null() { - let arr = created_cf_array(result_ref); - if err == kAXErrorSuccess - && let Some(arr) = arr - { - return arr - .into_iter() - .find_map(|item| ax_value::retained_ax_element(&item)); - } - } - attrs.iter().find_map(|attr| copy_element_attr(el, attr)) + .map(Some) + .ok_or(MALFORMED_AX_VALUE) } fn ax_array_items(arr: CFArray<CFType>) -> Vec<AXElement> { @@ -176,53 +129,72 @@ mod imp { .filter_map(|item| ax_value::retained_ax_element(&item)) .collect() } + + fn is_absent_error(error: i32) -> bool { + error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue + } } #[cfg(not(target_os = "macos"))] mod imp { use crate::tree::ax_element::AXElement; - pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option<Vec<AXElement>> { + pub(crate) fn copy_ax_array_result( + _el: &AXElement, + _attr: &str, + ) -> Result<Option<Vec<AXElement>>, i32> { + Ok(None) + } + + pub fn copy_bool_attr( + _el: &AXElement, + _attr: &str, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Option<bool> { None } - pub fn copy_ax_array_prefix( + pub(crate) fn copy_bool_attr_result( + _el: &AXElement, + _attr: &str, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<bool>, i32> { + Ok(None) + } + + pub(crate) fn copy_element_attr_result( + _el: &AXElement, + _attr: &str, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<AXElement>, i32> { + Ok(None) + } + + pub(crate) fn copy_ax_array_prefix_result( _el: &AXElement, _attr: &str, _max_values: usize, - ) -> Option<Vec<AXElement>> { - None + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<Vec<AXElement>>, i32> { + Ok(None) } - pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option<String> { - None + pub fn set_messaging_timeout( + _el: &AXElement, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<(), agent_desktop_core::AdapterError> { + Ok(()) } - - pub fn copy_bool_attr(_el: &AXElement, _attr: &str) -> Option<bool> { - None - } - - pub fn copy_i64_attr(_el: &AXElement, _attr: &str) -> Option<i64> { - None - } - - pub fn copy_element_attr(_el: &AXElement, _attr: &str) -> Option<AXElement> { - None - } - - pub fn copy_first_element_attr(_el: &AXElement, _attrs: &[&str]) -> Option<AXElement> { - None - } - - pub fn copy_value_typed(_el: &AXElement) -> Option<String> { - None - } - - pub fn set_messaging_timeout(_el: &AXElement, _timeout: std::time::Duration) {} } +pub(crate) use super::text_attributes::{ + copy_string_attr_bounded_result, copy_string_attr_result, copy_value_typed, + copy_value_typed_bounded_result, copy_value_typed_result, +}; pub(crate) use imp::{ - copy_ax_array, copy_ax_array_prefix, copy_bool_attr, copy_element_attr, - copy_first_element_attr, copy_i64_attr, copy_string_attr, copy_value_typed, + copy_ax_array_prefix_result, copy_bool_attr, copy_bool_attr_result, copy_element_attr_result, set_messaging_timeout, }; + +#[cfg(not(target_os = "macos"))] +pub(crate) use imp::copy_ax_array_result; diff --git a/crates/macos/src/tree/ax_element.rs b/crates/macos/src/tree/ax_element.rs index 73215e1..a0fcaf4 100644 --- a/crates/macos/src/tree/ax_element.rs +++ b/crates/macos/src/tree/ax_element.rs @@ -1,10 +1,17 @@ #[cfg(target_os = "macos")] mod imp { use accessibility_sys::AXUIElementRef; + use agent_desktop_core::NativeHandle; use core_foundation::base::{CFRelease, CFRetain, CFTypeRef}; pub struct AXElement(pub(crate) AXUIElementRef); + impl AXElement { + pub(crate) fn into_native_handle(self) -> NativeHandle { + NativeHandle::new(self) + } + } + impl Drop for AXElement { fn drop(&mut self) { if !self.0.is_null() { @@ -25,8 +32,16 @@ mod imp { #[cfg(not(target_os = "macos"))] mod imp { + use agent_desktop_core::NativeHandle; + pub struct AXElement(pub(crate) *const std::ffi::c_void); + impl AXElement { + pub(crate) fn into_native_handle(self) -> NativeHandle { + NativeHandle::new(self) + } + } + impl Drop for AXElement { fn drop(&mut self) {} } @@ -38,4 +53,16 @@ mod imp { } } -pub use imp::AXElement; +pub(crate) use imp::AXElement; + +#[cfg(test)] +mod tests { + use super::AXElement; + + #[test] + fn converts_to_an_owned_typed_native_handle() { + let handle = AXElement(std::ptr::null_mut()).into_native_handle(); + + assert!(handle.downcast_ref::<AXElement>().is_some()); + } +} diff --git a/crates/macos/src/tree/ax_ipc.rs b/crates/macos/src/tree/ax_ipc.rs new file mode 100644 index 0000000..401501f --- /dev/null +++ b/crates/macos/src/tree/ax_ipc.rs @@ -0,0 +1,269 @@ +use agent_desktop_core::{AdapterError, Deadline, ErrorCode}; +use std::time::{Duration, Instant}; + +#[cfg(target_os = "macos")] +use accessibility_sys::{ + AXUIElementCopyActionNames, AXUIElementCopyAttributeValue, AXUIElementCopyAttributeValues, + AXUIElementCopyElementAtPosition, AXUIElementCopyMultipleAttributeValues, + AXUIElementGetAttributeValueCount, AXUIElementGetPid, AXUIElementIsAttributeSettable, + AXUIElementPerformAction, AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout, + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorFailure, kAXErrorInvalidUIElement, + kAXErrorSuccess, +}; +#[cfg(target_os = "macos")] +use core_foundation_sys::{ + array::CFArrayRef, + base::{CFIndex, CFTypeRef}, + string::CFStringRef, +}; + +const MAX_IPC_SLICE: Duration = Duration::from_millis(250); + +pub(crate) trait AxDeadline: Copy { + fn absolute(self) -> Result<Instant, AdapterError>; +} + +impl AxDeadline for Instant { + fn absolute(self) -> Result<Instant, AdapterError> { + Ok(self) + } +} + +impl AxDeadline for Deadline { + fn absolute(self) -> Result<Instant, AdapterError> { + let remaining = self.remaining(); + if remaining.is_zero() { + return Err(self.timeout_error()); + } + Instant::now() + .checked_add(remaining) + .ok_or_else(|| AdapterError::timeout("Accessibility deadline overflowed")) + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare( + element: &super::AXElement, + deadline: impl AxDeadline, +) -> Result<Duration, AdapterError> { + if element.0.is_null() { + return Err(AdapterError::new( + ErrorCode::ElementNotFound, + "Cannot address a null accessibility element", + )); + } + let remaining = deadline + .absolute()? + .saturating_duration_since(Instant::now()) + .min(MAX_IPC_SLICE); + if remaining.is_zero() { + return Err(AdapterError::timeout( + "Accessibility deadline exhausted before IPC", + )); + } + let error = unsafe { AXUIElementSetMessagingTimeout(element.0, remaining.as_secs_f32()) }; + if error == kAXErrorSuccess { + Ok(remaining) + } else { + Err(timeout_install_error(error)) + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn copy_attribute_value( + element: &super::AXElement, + attribute: CFStringRef, + deadline: impl AxDeadline, +) -> (i32, CFTypeRef) { + let mut value: CFTypeRef = std::ptr::null(); + let error = match prepare(element, deadline) { + Ok(_) => unsafe { AXUIElementCopyAttributeValue(element.0, attribute, &mut value) }, + Err(error) => adapter_error_to_ax(&error), + }; + (error, value) +} + +#[cfg(target_os = "macos")] +pub(crate) fn copy_attribute_values( + element: &super::AXElement, + attribute: CFStringRef, + index: CFIndex, + max_values: CFIndex, + deadline: impl AxDeadline, +) -> (i32, CFArrayRef) { + let mut values = std::ptr::null(); + let error = match prepare(element, deadline) { + Ok(_) => unsafe { + AXUIElementCopyAttributeValues(element.0, attribute, index, max_values, &mut values) + }, + Err(error) => adapter_error_to_ax(&error), + }; + (error, values) +} + +#[cfg(target_os = "macos")] +pub(crate) fn copy_multiple_attribute_values( + element: &super::AXElement, + attributes: CFArrayRef, + deadline: impl AxDeadline, +) -> (i32, CFTypeRef) { + let mut values: CFTypeRef = std::ptr::null(); + let error = match prepare(element, deadline) { + Ok(_) => unsafe { + AXUIElementCopyMultipleAttributeValues( + element.0, + attributes, + 0, + &mut values as *mut _ as *mut _, + ) + }, + Err(error) => adapter_error_to_ax(&error), + }; + (error, values) +} + +#[cfg(target_os = "macos")] +pub(crate) fn attribute_value_count( + element: &super::AXElement, + attribute: CFStringRef, + deadline: impl AxDeadline, +) -> Result<usize, i32> { + let mut count: CFIndex = 0; + let error = match prepare(element, deadline) { + Ok(_) => unsafe { AXUIElementGetAttributeValueCount(element.0, attribute, &mut count) }, + Err(error) => adapter_error_to_ax(&error), + }; + if error != kAXErrorSuccess { + return Err(error); + } + usize::try_from(count).map_err(|_| i32::MIN) +} + +#[cfg(target_os = "macos")] +pub(crate) fn copy_action_names( + element: &super::AXElement, + deadline: impl AxDeadline, +) -> (i32, CFArrayRef) { + let mut actions = std::ptr::null(); + let error = match prepare(element, deadline) { + Ok(_) => unsafe { AXUIElementCopyActionNames(element.0, &mut actions) }, + Err(error) => adapter_error_to_ax(&error), + }; + (error, actions) +} + +#[cfg(target_os = "macos")] +pub(crate) fn is_attribute_settable( + element: &super::AXElement, + attribute: CFStringRef, + deadline: impl AxDeadline, +) -> (i32, bool) { + let mut settable = 0_u8; + let error = match prepare(element, deadline) { + Ok(_) => unsafe { AXUIElementIsAttributeSettable(element.0, attribute, &mut settable) }, + Err(error) => adapter_error_to_ax(&error), + }; + (error, settable != 0) +} + +#[cfg(target_os = "macos")] +pub(crate) fn perform_action( + element: &super::AXElement, + action: CFStringRef, + deadline: impl AxDeadline, +) -> Result<i32, AdapterError> { + prepare(element, deadline).map_err(pre_mutation_error)?; + Ok(unsafe { AXUIElementPerformAction(element.0, action) }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn set_attribute_value( + element: &super::AXElement, + attribute: CFStringRef, + value: CFTypeRef, + deadline: impl AxDeadline, +) -> Result<i32, AdapterError> { + prepare(element, deadline).map_err(pre_mutation_error)?; + Ok(unsafe { AXUIElementSetAttributeValue(element.0, attribute, value) }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn element_at_position( + system: &super::AXElement, + point: (f32, f32), + deadline: impl AxDeadline, +) -> (i32, accessibility_sys::AXUIElementRef) { + let mut element = std::ptr::null_mut(); + let error = match prepare(system, deadline) { + Ok(_) => unsafe { + AXUIElementCopyElementAtPosition(system.0, point.0, point.1, &mut element) + }, + Err(error) => adapter_error_to_ax(&error), + }; + (error, element) +} + +#[cfg(target_os = "macos")] +pub(crate) fn pid(element: &super::AXElement, deadline: impl AxDeadline) -> Result<i32, i32> { + let mut pid = 0_i32; + let error = match prepare(element, deadline) { + Ok(_) => unsafe { AXUIElementGetPid(element.0, &mut pid) }, + Err(error) => adapter_error_to_ax(&error), + }; + if error == kAXErrorSuccess && pid > 0 { + Ok(pid) + } else { + Err(error) + } +} + +#[cfg(target_os = "macos")] +fn timeout_install_error(error: i32) -> AdapterError { + let code = if error == kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == kAXErrorInvalidUIElement { + ErrorCode::ElementNotFound + } else { + ErrorCode::ActionFailed + }; + AdapterError::new( + code, + "Could not install the accessibility messaging timeout", + ) + .with_details(serde_json::json!({ + "ax_error": error, + "kind": "messaging_timeout_install", + })) + .with_suggestion("Refresh the target and retry before issuing another accessibility call") +} + +#[cfg(target_os = "macos")] +fn adapter_error_to_ax(error: &AdapterError) -> i32 { + match error.code { + ErrorCode::PermDenied => kAXErrorAPIDisabled, + ErrorCode::ElementNotFound | ErrorCode::StaleRef => kAXErrorInvalidUIElement, + ErrorCode::Timeout | ErrorCode::AppUnresponsive => kAXErrorCannotComplete, + _ => kAXErrorFailure, + } +} + +fn pre_mutation_error(error: AdapterError) -> AdapterError { + error.with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mutation_preflight_failure_is_explicitly_not_delivered() { + let error = pre_mutation_error(AdapterError::timeout("preflight")); + + assert_eq!( + error.disposition, + agent_desktop_core::DeliverySemantics::not_delivered() + ); + } +} diff --git a/crates/macos/src/tree/ax_value.rs b/crates/macos/src/tree/ax_value.rs index 80d4cd8..68ae227 100644 --- a/crates/macos/src/tree/ax_value.rs +++ b/crates/macos/src/tree/ax_value.rs @@ -5,7 +5,6 @@ mod imp { use core_foundation::base::{CFRetain, CFType, CFTypeRef, TCFType}; use core_foundation_sys::base::{CFGetTypeID, CFRelease}; - /// Takes ownership of a non-null +1 create-rule reference and releases mismatched values. pub(crate) fn created_ax_element(value: CFTypeRef) -> Option<AXElement> { if value.is_null() { return None; diff --git a/crates/macos/src/tree/bounded_string.rs b/crates/macos/src/tree/bounded_string.rs new file mode 100644 index 0000000..051881c --- /dev/null +++ b/crates/macos/src/tree/bounded_string.rs @@ -0,0 +1,119 @@ +use super::observation_usage::ObservationUsage; + +pub(crate) struct BoundedString { + pub(crate) value: String, + pub(crate) complete: bool, +} + +impl BoundedString { + #[cfg(target_os = "macos")] + pub(crate) fn from_cf( + value: &core_foundation::string::CFString, + usage: &mut ObservationUsage, + ) -> Result<Self, ()> { + use core_foundation::base::TCFType; + use core_foundation_sys::{ + base::{CFIndex, CFRange}, + string::{CFStringGetBytes, CFStringGetLength, kCFStringEncodingUTF8}, + }; + + let value_ref = value.as_concrete_TypeRef(); + let length = unsafe { CFStringGetLength(value_ref) }; + let capacity = usage.string_capacity(); + let allocation = utf8_allocation_size(length, capacity)?; + let mut bytes = vec![0_u8; allocation]; + let mut used: CFIndex = 0; + let converted = unsafe { + CFStringGetBytes( + value_ref, + CFRange::init(0, length), + kCFStringEncodingUTF8, + 0, + false.into(), + bytes.as_mut_ptr(), + CFIndex::try_from(allocation).map_err(|_| ())?, + &mut used, + ) + }; + let used = usize::try_from(used).map_err(|_| ())?; + bytes.truncate(used); + let value = String::from_utf8(bytes).map_err(|_| ())?; + usage.claim_text(used); + Ok(Self { + value, + complete: converted == length, + }) + } + + pub(crate) fn from_owned(mut value: String, usage: &mut ObservationUsage) -> Self { + let capacity = usage.string_capacity(); + let complete = value.len() <= capacity; + if !complete { + let mut end = capacity.min(value.len()); + while !value.is_char_boundary(end) { + end = end.saturating_sub(1); + } + value.truncate(end); + } + usage.claim_text(value.len()); + Self { value, complete } + } +} + +#[cfg(target_os = "macos")] +fn utf8_allocation_size( + length: core_foundation_sys::base::CFIndex, + capacity: usize, +) -> Result<usize, ()> { + let maximum = unsafe { + core_foundation_sys::string::CFStringGetMaximumSizeForEncoding( + length, + core_foundation_sys::string::kCFStringEncodingUTF8, + ) + }; + Ok(capacity.min(usize::try_from(maximum).map_err(|_| ())?)) +} + +#[cfg(test)] +mod tests { + use super::*; + use agent_desktop_core::ObservationBudget; + + #[test] + fn owned_text_truncates_only_at_utf8_boundaries() { + let mut usage = ObservationUsage::new(ObservationBudget { + max_field_bytes: 4, + max_text_bytes: 4, + ..ObservationBudget::default() + }); + + let read = BoundedString::from_owned("a🙂z".into(), &mut usage); + + assert_eq!(read.value, "a"); + assert!(!read.complete); + } + + #[cfg(target_os = "macos")] + #[test] + fn short_cf_strings_do_not_allocate_the_full_field_budget() { + let allocation = utf8_allocation_size(4, 64 * 1024).unwrap(); + + assert!(allocation <= 12); + } + + #[cfg(target_os = "macos")] + #[test] + fn exhausted_text_budget_never_reports_a_complete_cf_read() { + let value = core_foundation::string::CFString::new("content"); + let mut usage = ObservationUsage::new(ObservationBudget { + max_field_bytes: 0, + max_text_bytes: 0, + ..ObservationBudget::default() + }); + + let read = BoundedString::from_cf(&value, &mut usage).unwrap(); + + assert!(read.value.is_empty()); + assert!(!read.complete); + } +} diff --git a/crates/macos/src/tree/build_context.rs b/crates/macos/src/tree/build_context.rs index 7640260..449b5de 100644 --- a/crates/macos/src/tree/build_context.rs +++ b/crates/macos/src/tree/build_context.rs @@ -1,30 +1,49 @@ +use agent_desktop_core::Rect; + use super::AXElement; -pub struct TreeBuildContext { +#[derive(Clone)] +pub(crate) struct TreeBuildContext { pub(crate) focused: Option<AXElement>, + pub(crate) window_bounds: Option<Rect>, include_bounds: bool, } impl TreeBuildContext { - pub fn for_pid(pid: i32, include_bounds: bool) -> Self { + pub(crate) fn for_pid_with_deadline( + pid: i32, + include_bounds: bool, + deadline: std::time::Instant, + ) -> Result<Self, agent_desktop_core::AdapterError> { let app = super::element_for_pid(pid); + let focused = super::resolve_ax_read::read_element(&app, "AXFocusedUIElement", deadline)?; + Ok(Self { + focused, + window_bounds: None, + include_bounds, + }) + } + + pub(crate) fn empty(include_bounds: bool) -> Self { Self { - focused: super::copy_element_attr(&app, "AXFocusedUIElement"), + focused: None, + window_bounds: None, include_bounds, } } - pub fn empty(include_bounds: bool) -> Self { + pub(crate) fn child_context(&self, window_bounds: Option<Rect>) -> Self { Self { - focused: None, - include_bounds, + focused: self.focused.clone(), + window_bounds: window_bounds.or(self.window_bounds), + include_bounds: self.include_bounds, } } pub(crate) fn bounds_for( &self, - bounds: Option<agent_desktop_core::node::Rect>, - ) -> Option<agent_desktop_core::node::Rect> { + bounds: Option<agent_desktop_core::Rect>, + ) -> Option<agent_desktop_core::Rect> { if self.include_bounds { bounds } else { None } } } diff --git a/crates/macos/src/tree/builder.rs b/crates/macos/src/tree/builder.rs deleted file mode 100644 index 90f1733..0000000 --- a/crates/macos/src/tree/builder.rs +++ /dev/null @@ -1,361 +0,0 @@ -use agent_desktop_core::capability; -use agent_desktop_core::node::AccessibilityNode; -use rustc_hash::FxHashSet; - -use super::AXElement; -use super::action_list::platform_available_actions; -use super::attributes::{copy_ax_array, copy_ax_array_prefix, copy_bool_attr, copy_string_attr}; -use super::build_context::TreeBuildContext; -use super::capabilities::same_element; -use super::element::{ - ABSOLUTE_MAX_DEPTH, child_attributes, count_children, element_for_pid, fetch_node_attrs, -}; - -#[cfg(target_os = "macos")] -use accessibility_sys::{ - kAXChildrenAttribute, kAXRoleAttribute, kAXTitleAttribute, kAXValueAttribute, - kAXWindowsAttribute, -}; - -#[cfg(target_os = "macos")] -pub fn window_element_for(pid: i32, win_title: &str) -> AXElement { - let app = element_for_pid(pid); - - if let Some(windows) = copy_ax_array(&app, kAXWindowsAttribute) { - let mut first_candidate = None; - let mut child_candidate = None; - let mut partial_candidate = None; - for win in &windows { - if copy_string_attr(win, kAXRoleAttribute).as_deref() != Some("AXWindow") { - continue; - } - first_candidate.get_or_insert_with(|| win.clone()); - let title = copy_string_attr(win, kAXTitleAttribute); - if title - .as_deref() - .is_some_and(|title| window_titles_are_exact_match(title, win_title)) - { - return win.clone(); - } - if title - .as_deref() - .is_some_and(|title| window_titles_are_partial_match(title, win_title)) - { - partial_candidate.get_or_insert_with(|| win.clone()); - } - if child_candidate.is_none() && count_children(win, None) > 0 { - child_candidate = Some(win.clone()); - } - } - if let Some(candidate) = partial_candidate.or(child_candidate).or(first_candidate) { - return candidate; - } - } - - app -} - -#[cfg(target_os = "macos")] -fn window_titles_are_exact_match(candidate_title: &str, requested_title: &str) -> bool { - !candidate_title.is_empty() && !requested_title.is_empty() && candidate_title == requested_title -} - -#[cfg(target_os = "macos")] -fn window_titles_are_partial_match(candidate_title: &str, requested_title: &str) -> bool { - !candidate_title.is_empty() - && !requested_title.is_empty() - && (candidate_title.contains(requested_title) || requested_title.contains(candidate_title)) -} - -#[cfg(target_os = "macos")] -pub fn build_subtree( - el: &AXElement, - depth: u8, - raw_depth: u8, - max_depth: u8, - ancestors: &mut FxHashSet<usize>, - skeleton: bool, - context: &TreeBuildContext, -) -> Option<AccessibilityNode> { - if depth > max_depth { - return None; - } - if raw_depth >= ABSOLUTE_MAX_DEPTH { - if ancestors.contains(&(el.0 as usize)) { - return None; - } - let attrs = fetch_node_attrs(el); - let role = attrs - .role - .as_deref() - .map(crate::tree::roles::ax_role_to_str) - .unwrap_or("unknown") - .to_string(); - let is_secure_text = is_secure_text_role(attrs.role.as_deref()); - let value = redact_secure_value(attrs.role.as_deref(), attrs.value); - let name = attrs.title.or(attrs.description); - let child_count = count_children(el, attrs.role.as_deref()); - let bounds = context.bounds_for(attrs.bounds); - let mut states = Vec::new(); - if is_secure_text { - states.push("secure".into()); - } - return Some(AccessibilityNode { - ref_id: None, - available_actions: platform_available_actions(el, &role, attrs.has_scrollbars), - name, - value, - description: None, - hint: None, - states, - role, - bounds, - children_count: if child_count > 0 { - Some(child_count) - } else { - None - }, - children: vec![], - }); - } - let ptr_key = el.0 as usize; - if !ancestors.insert(ptr_key) { - return None; - } - - let attrs = fetch_node_attrs(el); - - let (role, promoted_label) = - crate::tree::roles::normalized_role_and_label(el, attrs.role.as_deref()); - let is_secure_text = is_secure_text_role(attrs.role.as_deref()); - let value = redact_secure_value(attrs.role.as_deref(), attrs.value); - let is_promoted_item = promoted_label.is_some(); - let available_actions = if is_promoted_item { - vec![capability::CLICK.into(), capability::RIGHT_CLICK.into()] - } else { - platform_available_actions(el, &role, attrs.has_scrollbars) - }; - - let name = promoted_label.or_else(|| attrs.title.clone().or_else(|| attrs.description.clone())); - let description = if attrs.title.is_some() { - attrs.description - } else { - None - }; - - let name = if name.is_none() && attrs.role.as_deref() == Some("AXStaticText") { - value.clone().or(name) - } else { - name - }; - - let mut states = Vec::new(); - if context - .focused - .as_ref() - .is_some_and(|focused| same_element(el, focused)) - { - states.push("focused".into()); - } - if !attrs.states.enabled { - states.push("disabled".into()); - } - if is_secure_text { - states.push("secure".into()); - } - if attrs - .states - .expanded - .or(attrs.states.disclosing) - .unwrap_or_else(|| element_is_expanded(el)) - { - states.push("expanded".into()); - } - if super::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) { - states.push("checked".into()); - } - - let bounds = context.bounds_for(attrs.bounds); - - let is_web_wrapper = matches!( - attrs.role.as_deref(), - Some("AXGroup") | Some("AXGenericElement") - ) && attrs.title.as_deref().is_none_or(str::is_empty) - && value.as_deref().is_none_or(str::is_empty); - - let child_depth = if is_web_wrapper { depth } else { depth + 1 }; - let child_raw_depth = raw_depth + 1; - - let at_skeleton_boundary = - skeleton && (child_depth > max_depth || child_raw_depth >= ABSOLUTE_MAX_DEPTH); - - if at_skeleton_boundary { - let child_count = count_children(el, attrs.role.as_deref()); - let children_count = if child_count > 0 { - Some(child_count) - } else { - None - }; - let name = name.or_else(|| label_from_child_attrs(el, attrs.role.as_deref())); - ancestors.remove(&ptr_key); - return Some(AccessibilityNode { - ref_id: None, - role, - name, - value, - description, - hint: None, - states, - available_actions, - bounds, - children_count, - children: vec![], - }); - } - - let children_raw = copy_children(el, attrs.role.as_deref()).unwrap_or_default(); - let name = name.or_else(|| label_from_children(&children_raw)); - - let children = if is_promoted_item { - Vec::new() - } else { - children_raw - .into_iter() - .filter_map(|child| { - build_subtree( - &child, - child_depth, - child_raw_depth, - max_depth, - ancestors, - skeleton, - context, - ) - }) - .collect() - }; - - ancestors.remove(&ptr_key); - - Some(AccessibilityNode { - ref_id: None, - role, - name, - value, - description, - hint: None, - states, - available_actions, - bounds, - children_count: None, - children, - }) -} - -fn is_secure_text_role(ax_role: Option<&str>) -> bool { - ax_role == Some("AXSecureTextField") -} - -fn redact_secure_value(ax_role: Option<&str>, value: Option<String>) -> Option<String> { - if is_secure_text_role(ax_role) { - None - } else { - value - } -} - -fn element_is_expanded(el: &AXElement) -> bool { - copy_bool_attr(el, "AXExpanded") - .or_else(|| copy_bool_attr(el, "AXDisclosing")) - .unwrap_or(false) -} - -fn value_is_checked(value: Option<&str>) -> bool { - matches!(value, Some("1" | "true")) -} - -pub fn label_from_children(children: &[AXElement]) -> Option<String> { - #[cfg(target_os = "macos")] - { - fn text_of(el: &AXElement) -> Option<String> { - copy_string_attr(el, kAXValueAttribute) - .or_else(|| copy_string_attr(el, kAXTitleAttribute)) - .filter(|s| !s.is_empty()) - } - - for child in children.iter().take(5) { - match copy_string_attr(child, kAXRoleAttribute).as_deref() { - Some("AXStaticText") => { - if let Some(s) = text_of(child) { - return Some(s); - } - } - Some("AXCell") | Some("AXGroup") => { - for gc in - copy_ax_array_prefix(child, kAXChildrenAttribute, 5).unwrap_or_default() - { - if copy_string_attr(&gc, kAXRoleAttribute).as_deref() - == Some("AXStaticText") - { - if let Some(s) = text_of(&gc) { - return Some(s); - } - } - } - } - _ => {} - } - } - None - } - #[cfg(not(target_os = "macos"))] - { - let _ = children; - None - } -} - -#[cfg(target_os = "macos")] -fn label_from_child_attrs(el: &AXElement, ax_role: Option<&str>) -> Option<String> { - for attr in child_attributes(ax_role) { - let children = copy_ax_array_prefix(el, attr, 5).unwrap_or_default(); - if let Some(label) = label_from_children(&children) { - return Some(label); - } - } - None -} - -#[cfg(target_os = "macos")] -fn copy_children(el: &AXElement, ax_role: Option<&str>) -> Option<Vec<AXElement>> { - for attr in child_attributes(ax_role) { - if let Some(v) = copy_ax_array(el, attr) { - if !v.is_empty() { - return Some(v); - } - } - } - None -} - -#[cfg(not(target_os = "macos"))] -pub fn window_element_for(_pid: i32, _win_title: &str) -> AXElement { - element_for_pid(0) -} - -#[cfg(not(target_os = "macos"))] -pub fn build_subtree( - _el: &AXElement, - _depth: u8, - _raw_depth: u8, - _max_depth: u8, - _visited: &mut FxHashSet<usize>, - _skeleton: bool, - _context: &TreeBuildContext, -) -> Option<AccessibilityNode> { - None -} - -#[cfg(test)] -#[path = "builder_tests.rs"] -mod tests; diff --git a/crates/macos/src/tree/builder_tests.rs b/crates/macos/src/tree/builder_tests.rs deleted file mode 100644 index 31c93d0..0000000 --- a/crates/macos/src/tree/builder_tests.rs +++ /dev/null @@ -1,54 +0,0 @@ -use super::{ - child_attributes, redact_secure_value, window_titles_are_exact_match, - window_titles_are_partial_match, -}; - -#[test] -fn test_browser_children_use_columns() { - assert_eq!( - child_attributes(Some("AXBrowser")), - ["AXColumns", "AXContents"] - ); -} - -#[test] -fn test_default_children_follow_fallback_order() { - assert_eq!( - child_attributes(Some("AXGroup")), - ["AXChildren", "AXContents", "AXChildrenInNavigationOrder"] - ); -} - -#[test] -fn test_secure_text_value_is_redacted() { - assert_eq!( - redact_secure_value(Some("AXSecureTextField"), Some("secret".into())), - None - ); - assert_eq!( - redact_secure_value(Some("AXTextField"), Some("visible".into())), - Some("visible".into()) - ); -} - -#[test] -fn window_title_matching_rejects_empty_titles() { - assert!(!window_titles_are_exact_match("", "")); - assert!(!window_titles_are_exact_match("Inbox", "")); - assert!(!window_titles_are_exact_match("", "Inbox")); - assert!(!window_titles_are_partial_match("Inbox", "")); - assert!(!window_titles_are_partial_match("", "Inbox")); -} - -#[test] -fn window_title_matching_accepts_exact_and_truncated_titles() { - assert!(window_titles_are_exact_match("Inbox", "Inbox")); - assert!(window_titles_are_partial_match( - "noy4/agent-desktop: Native desktop automation", - "noy4/agent-desktop" - )); - assert!(window_titles_are_partial_match( - "noy4/agent-desktop: Native desk...", - "noy4/agent-desktop: Native desk" - )); -} diff --git a/crates/macos/src/tree/capabilities.rs b/crates/macos/src/tree/capabilities.rs index dcf08f0..82c6f78 100644 --- a/crates/macos/src/tree/capabilities.rs +++ b/crates/macos/src/tree/capabilities.rs @@ -1,41 +1,83 @@ +pub(crate) struct NativeRead<T> { + pub(crate) value: Option<T>, + pub(crate) error: Option<i32>, +} + +impl<T> NativeRead<T> { + fn success(value: T) -> Self { + Self { + value: Some(value), + error: None, + } + } + + fn failure(error: i32) -> Self { + Self { + value: None, + error: Some(error), + } + } +} + #[cfg(target_os = "macos")] mod imp { + use super::NativeRead; use crate::{cf_type::created_cf_array, tree::AXElement}; - use accessibility_sys::{ - AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, kAXErrorSuccess, - }; + use accessibility_sys::kAXErrorSuccess; use core_foundation::{ base::{CFEqual, CFTypeRef, TCFType}, string::CFString, }; - use std::os::raw::c_uchar; - pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool { + pub(crate) fn is_attr_settable_with_status( + el: &AXElement, + attr: &str, + deadline: std::time::Instant, + ) -> NativeRead<bool> { let cf_attr = CFString::new(attr); - let mut settable: c_uchar = 0; - let err = unsafe { - AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable) - }; - err == kAXErrorSuccess && settable != 0 + let (err, settable) = + crate::tree::ax_ipc::is_attribute_settable(el, cf_attr.as_concrete_TypeRef(), deadline); + if err == kAXErrorSuccess { + NativeRead::success(settable) + } else { + NativeRead::failure(err) + } } - pub fn copy_action_names(el: &AXElement) -> Vec<String> { - let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null(); - let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) }; - if err != kAXErrorSuccess || actions_ref.is_null() { - return Vec::new(); + pub(crate) fn copy_action_names_with_status( + el: &AXElement, + deadline: std::time::Instant, + usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> NativeRead<Vec<String>> { + let (err, actions_ref) = crate::tree::ax_ipc::copy_action_names(el, deadline); + if err != kAXErrorSuccess { + if !actions_ref.is_null() { + drop(created_cf_array(actions_ref as _)); + } + return NativeRead::failure(err); + } + if actions_ref.is_null() { + return NativeRead::failure(err); } let Some(actions) = created_cf_array(actions_ref as _) else { - return Vec::new(); + return NativeRead::failure(i32::MIN); }; + if actions.len() > 256 { + return NativeRead::failure(i32::MIN + 1); + } let mut result = Vec::with_capacity(actions.len() as usize); for i in 0..actions.len() { if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) { - result.push(name.to_string()); + match crate::tree::bounded_string::BoundedString::from_cf(&name, usage) { + Ok(name) if name.complete => result.push(name.value), + _ => return NativeRead::failure(i32::MIN + 2), + } + } else { + return NativeRead::failure(i32::MIN); } } - result + NativeRead::success(result) } pub fn same_element(a: &AXElement, b: &AXElement) -> bool { @@ -45,14 +87,23 @@ mod imp { #[cfg(not(target_os = "macos"))] mod imp { + use super::NativeRead; use crate::tree::AXElement; - pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool { - false + pub(crate) fn is_attr_settable_with_status( + _el: &AXElement, + _attr: &str, + _deadline: std::time::Instant, + ) -> NativeRead<bool> { + NativeRead::failure(i32::MIN) } - pub fn copy_action_names(_el: &AXElement) -> Vec<String> { - Vec::new() + pub(crate) fn copy_action_names_with_status( + _el: &AXElement, + _deadline: std::time::Instant, + _usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> NativeRead<Vec<String>> { + NativeRead::failure(i32::MIN) } pub fn same_element(_a: &AXElement, _b: &AXElement) -> bool { @@ -60,4 +111,21 @@ mod imp { } } -pub use imp::{copy_action_names, is_attr_settable, same_element}; +pub(crate) use imp::same_element; +pub(crate) use imp::{copy_action_names_with_status, is_attr_settable_with_status}; + +#[cfg(test)] +mod tests { + use super::NativeRead; + + #[test] + fn failed_native_read_never_fabricates_an_empty_value() { + let strings = NativeRead::<Vec<String>>::failure(-1); + let flag = NativeRead::<bool>::failure(-1); + + assert!(strings.value.is_none()); + assert!(flag.value.is_none()); + assert_eq!(strings.error, Some(-1)); + assert_eq!(flag.error, Some(-1)); + } +} diff --git a/crates/macos/src/tree/child_labels.rs b/crates/macos/src/tree/child_labels.rs new file mode 100644 index 0000000..5982285 --- /dev/null +++ b/crates/macos/src/tree/child_labels.rs @@ -0,0 +1,259 @@ +use crate::tree::AXElement; + +pub(crate) const MAX_LABEL_ELEMENTS: usize = 5; + +pub(crate) struct NameEvidenceSinks<'a> { + pub(crate) stats: &'a mut agent_desktop_core::LocatorStats, + pub(crate) usage: &'a mut crate::tree::observation_usage::ObservationUsage, +} + +pub(crate) fn complete_name_evidence_with_deadline( + attrs: &crate::tree::NodeAttrs, + role: &str, + children: &[AXElement], + deadline: std::time::Instant, + mut sinks: NameEvidenceSinks<'_>, +) -> Result<(agent_desktop_core::NameEvidence, bool), agent_desktop_core::AdapterError> { + let mut evidence = attrs.name_evidence.clone(); + if !should_read_child_label(role, &evidence) { + return Ok((evidence, true)); + } + let (label, complete) = label_from_children(children, deadline, &mut sinks)?; + evidence.child_label = label; + Ok((evidence, complete)) +} + +fn should_read_child_label(role: &str, evidence: &agent_desktop_core::NameEvidence) -> bool { + names_from_child_content(role) && !has_name_without_child_content(evidence) +} + +fn names_from_child_content(role: &str) -> bool { + agent_desktop_core::roles::INTERACTIVE_ROLES.contains(&role) +} + +fn has_name_without_child_content(evidence: &agent_desktop_core::NameEvidence) -> bool { + [ + evidence.explicit_label.as_deref(), + evidence.labelled_by_text.as_deref(), + evidence.native_title.as_deref(), + evidence.static_value.as_deref(), + evidence.description.as_deref(), + ] + .into_iter() + .any(|value| value.is_some_and(|value| !value.trim().is_empty())) +} + +#[cfg(target_os = "macos")] +fn label_from_children( + children: &[AXElement], + deadline: std::time::Instant, + sinks: &mut NameEvidenceSinks<'_>, +) -> Result<(Option<String>, bool), agent_desktop_core::AdapterError> { + let mut labels = Vec::new(); + note_label_limit(children.len(), sinks.stats); + let mut complete = children.len() <= MAX_LABEL_ELEMENTS; + for child in children.iter().take(MAX_LABEL_ELEMENTS) { + let (role, role_complete) = timed_string(child, "AXRole", deadline, sinks)?; + complete &= role_complete; + match role.as_deref() { + Some("AXStaticText") => { + let (subrole, subrole_complete) = + timed_string(child, "AXSubrole", deadline, sinks)?; + complete &= subrole_complete; + if subrole.as_deref() != Some("AXSecureTextField") { + complete &= push_static_text(&mut labels, child, deadline, sinks)?; + } + } + Some("AXCell") | Some("AXGroup") => { + let (title, title_complete) = timed_string(child, "AXTitle", deadline, sinks)?; + complete &= title_complete; + if let Some(title) = title { + labels.push(title); + } + let grandchildren = crate::tree::query::child_read::read_children( + child, + role.as_deref(), + MAX_LABEL_ELEMENTS, + deadline, + ); + record_child_read(&grandchildren, sinks.stats)?; + complete &= grandchildren.complete + && !grandchildren.truncated() + && !grandchildren.status.invalid_element; + for grandchild in grandchildren.elements { + let (role, role_complete) = + timed_string(&grandchild, "AXRole", deadline, sinks)?; + complete &= role_complete; + if role.as_deref() == Some("AXStaticText") { + let (subrole, subrole_complete) = + timed_string(&grandchild, "AXSubrole", deadline, sinks)?; + complete &= subrole_complete; + if subrole.as_deref() != Some("AXSecureTextField") { + complete &= + push_static_text(&mut labels, &grandchild, deadline, sinks)?; + } + } + } + } + _ => {} + } + } + let (label, join_complete) = join_unique_labels(labels, sinks.usage); + Ok((label, complete && join_complete)) +} + +#[cfg(not(target_os = "macos"))] +fn label_from_children( + _children: &[AXElement], + _deadline: std::time::Instant, + _sinks: &mut NameEvidenceSinks<'_>, +) -> Result<(Option<String>, bool), agent_desktop_core::AdapterError> { + Ok((None, true)) +} + +#[cfg(target_os = "macos")] +fn timed_string( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + sinks: &mut NameEvidenceSinks<'_>, +) -> Result<(Option<String>, bool), agent_desktop_core::AdapterError> { + sinks.stats.semantic_reads.child_label_reads += 1; + let read = crate::tree::attributes::copy_string_attr_bounded_result( + element, + attribute, + deadline, + sinks.usage, + ); + match read { + Ok(value) => Ok(complete_text(value, sinks.stats)), + Err(error) => degrade_transient_read(error, "child_label.text", sinks.stats), + } +} + +#[cfg(target_os = "macos")] +fn push_static_text( + labels: &mut Vec<String>, + element: &AXElement, + deadline: std::time::Instant, + sinks: &mut NameEvidenceSinks<'_>, +) -> Result<bool, agent_desktop_core::AdapterError> { + sinks.stats.semantic_reads.child_label_reads += 1; + let read = + crate::tree::attributes::copy_value_typed_bounded_result(element, deadline, sinks.usage); + let value = match read { + Ok(value) => value, + Err(error) => { + return degrade_transient_read(error, "child_label.value", sinks.stats) + .map(|(_, complete)| complete); + } + }; + let (mut text, mut complete) = complete_text(value, sinks.stats); + if text.is_none() && complete { + let (title, title_complete) = timed_string(element, "AXTitle", deadline, sinks)?; + text = title; + complete &= title_complete; + } + if let Some(text) = text { + labels.push(text); + } + Ok(complete) +} + +#[cfg(target_os = "macos")] +fn degrade_transient_read( + error: i32, + phase: &str, + stats: &mut agent_desktop_core::LocatorStats, +) -> Result<(Option<String>, bool), agent_desktop_core::AdapterError> { + if error == accessibility_sys::kAXErrorAPIDisabled { + return Err(crate::tree::query::read_error::semantic_read(error, phase)); + } + stats.reads.health.cannot_complete += + u64::from(error == accessibility_sys::kAXErrorCannotComplete); + stats.reads.health.native_read_failures += u64::from( + error != accessibility_sys::kAXErrorCannotComplete + && error != accessibility_sys::kAXErrorInvalidUIElement, + ); + Ok((None, false)) +} + +fn complete_text( + value: Option<crate::tree::bounded_string::BoundedString>, + stats: &mut agent_desktop_core::LocatorStats, +) -> (Option<String>, bool) { + match value { + Some(value) if value.complete => (Some(value.value), true), + Some(_) => { + stats.traversal.limits.text_hits += 1; + (None, false) + } + None => (None, true), + } +} + +#[cfg(target_os = "macos")] +fn record_child_read( + read: &crate::tree::query::child_read::ChildRead, + stats: &mut agent_desktop_core::LocatorStats, +) -> Result<(), agent_desktop_core::AdapterError> { + stats.reads.counts.child_reads += read.status.attempts; + stats.reads.health.cannot_complete += read.status.health.cannot_complete; + stats.reads.health.native_read_failures += read.status.health.native_read_failures; + stats.reads.health.deadline_exhausted += read.status.health.deadline_exhausted; + stats.traversal.limits.child_count_changes += u64::from(read.status.count_changed); + stats.traversal.limits.child_label_hits += u64::from(read.truncated()); + stats.traversal.limits.child_hits += u64::from(read.status.cursor_stalled); + if read.status.api_disabled { + return Err(crate::tree::query::read_error::semantic_read( + accessibility_sys::kAXErrorAPIDisabled, + "child_label.children", + )); + } + Ok(()) +} + +fn note_label_limit(child_count: usize, stats: &mut agent_desktop_core::LocatorStats) { + stats.traversal.limits.child_label_hits += u64::from(child_count > MAX_LABEL_ELEMENTS); +} + +fn join_unique_labels( + labels: impl IntoIterator<Item = String>, + usage: &mut crate::tree::observation_usage::ObservationUsage, +) -> (Option<String>, bool) { + let mut joined = String::new(); + let mut seen = Vec::new(); + let capacity = usage.string_capacity(); + let mut complete = true; + for label in labels { + let normalized = label.split_whitespace().collect::<Vec<_>>().join(" "); + if normalized.is_empty() || seen.iter().any(|value| value == &normalized) { + continue; + } + let separator = usize::from(!joined.is_empty()); + let remaining = capacity.saturating_sub(joined.len().saturating_add(separator)); + if remaining == 0 { + complete = false; + break; + } + if separator == 1 { + joined.push(' '); + } + let mut end = remaining.min(normalized.len()); + while !normalized.is_char_boundary(end) { + end = end.saturating_sub(1); + } + joined.push_str(&normalized[..end]); + complete &= end == normalized.len(); + seen.push(normalized); + if !complete { + break; + } + } + usage.claim_text(joined.len()); + ((!joined.is_empty()).then_some(joined), complete) +} + +#[cfg(test)] +#[path = "child_labels_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/child_labels_tests.rs b/crates/macos/src/tree/child_labels_tests.rs new file mode 100644 index 0000000..2ef16fd --- /dev/null +++ b/crates/macos/src/tree/child_labels_tests.rs @@ -0,0 +1,159 @@ +use super::*; +use agent_desktop_core::ObservationBudget; + +fn usage(max_field_bytes: usize) -> crate::tree::observation_usage::ObservationUsage { + crate::tree::observation_usage::ObservationUsage::new(ObservationBudget { + max_field_bytes, + max_text_bytes: max_field_bytes, + ..ObservationBudget::default() + }) +} + +#[test] +fn child_label_cap_is_reported_as_incomplete_traversal() { + let mut stats = agent_desktop_core::LocatorStats::default(); + + note_label_limit(MAX_LABEL_ELEMENTS + 1, &mut stats); + + assert_eq!(stats.traversal.limits.child_label_hits, 1); +} + +#[test] +fn labels_are_normalized_deduplicated_and_joined_in_document_order() { + let mut usage = usage(256); + let labels = vec![ + " Save\n".to_string(), + "Save".to_string(), + " Draft title ".to_string(), + ]; + + assert_eq!( + join_unique_labels(labels, &mut usage), + (Some("Save Draft title".into()), true) + ); +} + +#[test] +fn label_budget_truncation_is_explicit_and_utf8_safe() { + let mut usage = usage(4); + + let (label, complete) = join_unique_labels(["a🙂z".into()], &mut usage); + + assert_eq!(label.as_deref(), Some("a")); + assert!(!complete); +} + +#[test] +fn description_only_name_skips_child_content_fallback() { + let evidence = agent_desktop_core::NameEvidence { + description: Some("scroll-area".into()), + ..Default::default() + }; + + assert!(!should_read_child_label("button", &evidence)); +} + +#[test] +fn unnamed_elements_still_use_bounded_child_content_fallback() { + assert!(should_read_child_label( + "button", + &agent_desktop_core::NameEvidence::default() + )); +} + +#[test] +fn container_roles_never_derive_names_from_children() { + for role in [ + "scrollarea", + "group", + "window", + "list", + "table", + "outline", + "toolbar", + ] { + assert!( + !should_read_child_label(role, &agent_desktop_core::NameEvidence::default()), + "container role {role} must not name itself from descendants" + ); + } +} + +#[test] +fn direct_names_skip_child_label_reads_for_every_role() { + let evidence = agent_desktop_core::NameEvidence { + explicit_label: Some("scroll-area".into()), + ..Default::default() + }; + + assert!(!should_read_child_label("button", &evidence)); +} + +#[cfg(target_os = "macos")] +#[test] +fn transient_child_label_error_degrades_name_to_unknown_and_incomplete() { + let mut stats = agent_desktop_core::LocatorStats::default(); + let mut usage = usage(256); + let children = [AXElement(std::ptr::null_mut())]; + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(250); + + let (evidence, complete) = complete_name_evidence_with_deadline( + &crate::tree::NodeAttrs::default(), + "button", + &children, + deadline, + NameEvidenceSinks { + stats: &mut stats, + usage: &mut usage, + }, + ) + .expect("a transient child-label read error must not abort the traversal"); + + assert_eq!(evidence.child_label, None); + assert!(!complete); +} + +#[cfg(target_os = "macos")] +#[test] +fn api_disabled_child_label_reads_still_fail_closed() { + let mut stats = agent_desktop_core::LocatorStats::default(); + + let error = degrade_transient_read( + accessibility_sys::kAXErrorAPIDisabled, + "child_label.text", + &mut stats, + ) + .expect_err("API disablement must abort the observation"); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::PermDenied); +} + +#[cfg(target_os = "macos")] +#[test] +fn transient_child_label_errors_are_recorded_in_read_health() { + let mut stats = agent_desktop_core::LocatorStats::default(); + + for error in [ + accessibility_sys::kAXErrorInvalidUIElement, + accessibility_sys::kAXErrorCannotComplete, + accessibility_sys::kAXErrorFailure, + ] { + assert_eq!( + degrade_transient_read(error, "child_label.text", &mut stats).unwrap(), + (None, false) + ); + } + + assert_eq!(stats.reads.health.cannot_complete, 1); + assert_eq!(stats.reads.health.native_read_failures, 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn invalid_child_list_degrades_instead_of_aborting() { + let mut stats = agent_desktop_core::LocatorStats::default(); + let mut read = crate::tree::query::child_read::ChildRead::empty(true); + read.status.invalid_element = true; + + assert!(record_child_read(&read, &mut stats).is_ok()); +} diff --git a/crates/macos/src/tree/element.rs b/crates/macos/src/tree/element.rs index c6e3ec1..16bbd38 100644 --- a/crates/macos/src/tree/element.rs +++ b/crates/macos/src/tree/element.rs @@ -4,256 +4,34 @@ pub(crate) fn child_attributes(ax_role: Option<&str>) -> &'static [&'static str] if ax_role == Some("AXBrowser") { &["AXColumns", "AXContents"] } else if ax_role == Some("AXApplication") { - &["AXWindows", "AXFocusedWindow", "AXMainWindow", "AXChildren"] + &["AXWindows", "AXChildren"] } else { - &["AXChildren", "AXContents", "AXChildrenInNavigationOrder"] + &["AXChildren", "AXChildrenInNavigationOrder", "AXContents"] } } #[cfg(target_os = "macos")] mod imp { - use super::*; - use crate::{ - cf_type::created_cf_array, - tree::{ - NodeAttrs, - attributes::{ - copy_ax_array, copy_bool_attr, copy_first_element_attr, copy_string_attr, - copy_value_typed, - }, - ax_element::AXElement, - ax_value, - element_bounds::{read_bounds, rect_from_parts}, - node_attrs::{NodeAttrStates, parse_bool_attr, parse_enabled}, - }, - }; - use accessibility_sys::{ - AXUIElementCopyMultipleAttributeValues, AXUIElementCreateApplication, - AXUIElementGetAttributeValueCount, AXUIElementSetMessagingTimeout, AXValueGetValue, - kAXDescriptionAttribute, kAXEnabledAttribute, kAXErrorSuccess, kAXPositionAttribute, - kAXRoleAttribute, kAXSizeAttribute, kAXTitleAttribute, kAXValueAttribute, - kAXValueTypeCGPoint, kAXValueTypeCGSize, - }; - use core_foundation::{ - array::CFArray, - base::{CFType, CFTypeRef, TCFType}, - boolean::CFBoolean, - number::CFNumber, - string::CFString, - }; - use core_graphics::geometry::{CGPoint, CGSize}; - - const SCROLLBAR_ATTRS: [&str; 2] = ["AXVerticalScrollBar", "AXHorizontalScrollBar"]; + use crate::tree::ax_element::AXElement; + use accessibility_sys::AXUIElementCreateApplication; pub fn element_for_pid(pid: i32) -> AXElement { - let el = AXElement(unsafe { AXUIElementCreateApplication(pid) }); - if !el.0.is_null() { - unsafe { AXUIElementSetMessagingTimeout(el.0, 2.0) }; - } - el - } - - pub fn fetch_node_attrs(el: &AXElement) -> NodeAttrs { - let attr_names = [ - kAXRoleAttribute, - kAXTitleAttribute, - kAXDescriptionAttribute, - kAXValueAttribute, - kAXEnabledAttribute, - "AXFocused", - "AXExpanded", - "AXDisclosing", - kAXPositionAttribute, - kAXSizeAttribute, - SCROLLBAR_ATTRS[0], - SCROLLBAR_ATTRS[1], - ]; - let cf_names: Vec<CFString> = attr_names.iter().map(|a| CFString::new(a)).collect(); - let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect(); - let names_arr = CFArray::from_copyable(&cf_refs); - - let mut result_ref: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyMultipleAttributeValues( - el.0, - names_arr.as_concrete_TypeRef(), - 0, - &mut result_ref as *mut _ as *mut _, - ) - }; - - if err != kAXErrorSuccess || result_ref.is_null() { - return fetch_node_attrs_slow(el); - } - - let Some(arr) = created_cf_array(result_ref) else { - return fetch_node_attrs_slow(el); - }; - - let mut texts: [Option<String>; 8] = Default::default(); - let mut position: Option<CGPoint> = None; - let mut size: Option<CGSize> = None; - let mut has_scrollbars = false; - for (idx, item) in arr.into_iter().enumerate() { - match idx { - 0..=7 => texts[idx] = decode_text_attr(idx, &item), - 8 => position = decode_ax_point(&item), - 9 => size = decode_ax_size(&item), - 10 | 11 => { - has_scrollbars = - has_scrollbars || ax_value::retained_ax_element(&item).is_some(); - } - _ => {} - } - } - - let get = |i: usize| texts.get(i).and_then(|v| v.clone()); - NodeAttrs { - role: get(0), - title: get(1), - description: get(2), - value: get(3), - states: NodeAttrStates { - enabled: parse_enabled(get(4)), - focused: parse_bool_attr(get(5)), - expanded: parse_bool_attr(get(6)), - disclosing: parse_bool_attr(get(7)), - }, - bounds: position.zip(size).and_then(|(p, s)| rect_from_parts(p, s)), - has_scrollbars, - } - } - - fn decode_text_attr(idx: usize, item: &CFType) -> Option<String> { - if let Some(s) = item.downcast::<CFString>() { - return Some(s.to_string()); - } - match idx { - 3 => { - if let Some(b) = item.downcast::<CFBoolean>() { - return Some(bool::from(b).to_string()); - } - if let Some(n) = item.downcast::<CFNumber>() { - if let Some(i) = n.to_i64() { - return Some(i.to_string()); - } - if let Some(f) = n.to_f64() { - return Some(format!("{:.2}", f)); - } - } - None - } - 4..=7 => item - .downcast::<CFBoolean>() - .map(|b| bool::from(b).to_string()), - _ => None, - } - } - - fn decode_ax_point(item: &CFType) -> Option<CGPoint> { - let mut point = CGPoint::new(0.0, 0.0); - let decoded = unsafe { - AXValueGetValue( - item.as_CFTypeRef() as _, - kAXValueTypeCGPoint, - &mut point as *mut _ as *mut std::ffi::c_void, - ) - }; - decoded.then_some(point) - } - - fn decode_ax_size(item: &CFType) -> Option<CGSize> { - let mut size = CGSize::new(0.0, 0.0); - let decoded = unsafe { - AXValueGetValue( - item.as_CFTypeRef() as _, - kAXValueTypeCGSize, - &mut size as *mut _ as *mut std::ffi::c_void, - ) - }; - decoded.then_some(size) - } - - fn fetch_node_attrs_slow(el: &AXElement) -> NodeAttrs { - let role = copy_string_attr(el, kAXRoleAttribute); - let title = copy_string_attr(el, kAXTitleAttribute); - let desc = copy_string_attr(el, kAXDescriptionAttribute); - let val = copy_value_typed(el); - let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true); - NodeAttrs { - role, - title, - description: desc, - value: val, - states: NodeAttrStates { - enabled, - focused: copy_bool_attr(el, "AXFocused"), - expanded: copy_bool_attr(el, "AXExpanded"), - disclosing: copy_bool_attr(el, "AXDisclosing"), - }, - bounds: read_bounds(el), - has_scrollbars: copy_first_element_attr(el, &SCROLLBAR_ATTRS).is_some(), - } - } - - pub fn resolve_element_name(el: &AXElement) -> Option<String> { - let ax_role = copy_string_attr(el, kAXRoleAttribute); - let title = copy_string_attr(el, kAXTitleAttribute); - let desc = copy_string_attr(el, kAXDescriptionAttribute); - - let name = title.or(desc); - let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") { - copy_string_attr(el, kAXValueAttribute).or(name) - } else { - name - }; - - name.or_else(|| { - let children = super::child_attributes(ax_role.as_deref()) - .iter() - .find_map(|attr| copy_ax_array(el, attr).filter(|v| !v.is_empty())) - .unwrap_or_default(); - crate::tree::builder::label_from_children(&children) - }) - } - - pub fn count_children(element: &AXElement, ax_role: Option<&str>) -> u32 { - for attr_name in child_attributes(ax_role) { - let mut count: core_foundation_sys::base::CFIndex = 0; - let attr = CFString::from_static_string(attr_name); - let err = unsafe { - AXUIElementGetAttributeValueCount(element.0, attr.as_concrete_TypeRef(), &mut count) - }; - if err != kAXErrorSuccess { - continue; - } - if count > 0 { - return count as u32; - } - } - 0 + AXElement(unsafe { AXUIElementCreateApplication(pid) }) } } #[cfg(not(target_os = "macos"))] mod imp { - use crate::tree::{NodeAttrs, ax_element::AXElement}; + use crate::tree::ax_element::AXElement; pub fn element_for_pid(_pid: i32) -> AXElement { AXElement(std::ptr::null()) } - - pub fn count_children(_element: &AXElement, _ax_role: Option<&str>) -> u32 { - 0 - } - - pub fn resolve_element_name(_el: &AXElement) -> Option<String> { - None - } - - pub fn fetch_node_attrs(_el: &AXElement) -> NodeAttrs { - NodeAttrs::default() - } } -pub use imp::{count_children, element_for_pid, fetch_node_attrs, resolve_element_name}; +pub(crate) use crate::tree::node_attribute_fetch::fetch_node_attrs_with_status_for; +pub(crate) use imp::element_for_pid; + +#[cfg(test)] +#[path = "element_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/element_bounds.rs b/crates/macos/src/tree/element_bounds.rs index 1fa4df8..ab53812 100644 --- a/crates/macos/src/tree/element_bounds.rs +++ b/crates/macos/src/tree/element_bounds.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::node::Rect; +use agent_desktop_core::Rect; use super::AXElement; @@ -23,65 +23,21 @@ pub(crate) fn rect_from_parts( } #[cfg(target_os = "macos")] -pub fn read_bounds(el: &AXElement) -> Option<Rect> { - use accessibility_sys::{ - AXUIElementCopyAttributeValue, AXValueGetValue, kAXErrorSuccess, kAXPositionAttribute, - kAXSizeAttribute, kAXValueTypeCGPoint, kAXValueTypeCGSize, - }; - use core_foundation::{ - base::{CFRelease, CFTypeRef, TCFType}, - string::CFString, - }; - use core_graphics::geometry::{CGPoint, CGSize}; - use std::ffi::c_void; - - let pos_cf = CFString::new(kAXPositionAttribute); - let mut pos_ref: CFTypeRef = std::ptr::null_mut(); - let pos_ok = - unsafe { AXUIElementCopyAttributeValue(el.0, pos_cf.as_concrete_TypeRef(), &mut pos_ref) }; - if pos_ok != kAXErrorSuccess || pos_ref.is_null() { - return None; - } - - let mut point = CGPoint::new(0.0, 0.0); - let got_pos = unsafe { - AXValueGetValue( - pos_ref as _, - kAXValueTypeCGPoint, - &mut point as *mut _ as *mut c_void, - ) - }; - unsafe { CFRelease(pos_ref) }; - if !got_pos { - return None; - } - - let size_cf = CFString::new(kAXSizeAttribute); - let mut size_ref: CFTypeRef = std::ptr::null_mut(); - let size_ok = unsafe { - AXUIElementCopyAttributeValue(el.0, size_cf.as_concrete_TypeRef(), &mut size_ref) - }; - if size_ok != kAXErrorSuccess || size_ref.is_null() { - return None; - } - - let mut size = CGSize::new(0.0, 0.0); - let got_size = unsafe { - AXValueGetValue( - size_ref as _, - kAXValueTypeCGSize, - &mut size as *mut _ as *mut c_void, - ) - }; - unsafe { CFRelease(size_ref) }; - if !got_size { - return None; - } - - rect_from_parts(point, size) +pub(crate) fn read_bounds_with_deadline( + element: &AXElement, + deadline: std::time::Instant, +) -> Result<Option<Rect>, agent_desktop_core::AdapterError> { + let point = super::resolve_ax_read::read_point(element, "AXPosition", deadline)?; + let size = super::resolve_ax_read::read_size(element, "AXSize", deadline)?; + Ok(point + .zip(size) + .and_then(|(point, size)| rect_from_parts(point, size))) } #[cfg(not(target_os = "macos"))] -pub fn read_bounds(_el: &AXElement) -> Option<Rect> { - None +pub(crate) fn read_bounds_with_deadline( + _element: &AXElement, + _deadline: std::time::Instant, +) -> Result<Option<Rect>, agent_desktop_core::AdapterError> { + Ok(None) } diff --git a/crates/macos/src/tree/element_dedupe.rs b/crates/macos/src/tree/element_dedupe.rs index 7c47b9a..434e780 100644 --- a/crates/macos/src/tree/element_dedupe.rs +++ b/crates/macos/src/tree/element_dedupe.rs @@ -63,7 +63,9 @@ mod tests { #[cfg(target_os = "macos")] fn current_process_element() -> AXElement { AXElement(unsafe { - accessibility_sys::AXUIElementCreateApplication(std::process::id() as i32) + accessibility_sys::AXUIElementCreateApplication( + i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"), + ) }) } diff --git a/crates/macos/src/tree/element_name.rs b/crates/macos/src/tree/element_name.rs new file mode 100644 index 0000000..45bb18d --- /dev/null +++ b/crates/macos/src/tree/element_name.rs @@ -0,0 +1,50 @@ +use agent_desktop_core::{AdapterError, EvidenceRequirements, LocatorField}; + +pub(crate) fn resolve_element_name( + element: &super::AXElement, + deadline: std::time::Instant, + usage: &mut super::observation_usage::ObservationUsage, +) -> Result<Option<String>, AdapterError> { + if !usage.claim_node() { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Element name resolution exhausted its node budget", + ) + .with_details(serde_json::json!({ + "kind": "element_name_node_budget", + "complete": false, + }))); + } + let mut stats = agent_desktop_core::LocatorStats::default(); + let child_plan = super::query::child_read_plan::ChildReadPlan::load(usage.child_capacity()); + let read = super::query::node_read::read_node( + element, + super::query::node_read_context::NodeReadContext { + tree: &super::TreeBuildContext::empty(false), + stats: &mut stats, + usage, + requirements: EvidenceRequirements { + role: true, + name: true, + ..EvidenceRequirements::default() + }, + deadline, + child_plan, + }, + )?; + usage.note_child_demand(read.child_read.total_count, &mut stats); + usage.claim_edges(read.child_read.elements.len()); + match read.evidence.name { + LocatorField::Known(name) => Ok(Some(name)), + LocatorField::Absent => Ok(None), + LocatorField::Unknown => Err(AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Element name evidence was incomplete", + ) + .with_details(serde_json::json!({ + "kind": "element_name_incomplete", + "complete": false, + "query_stats": stats, + }))), + } +} diff --git a/crates/macos/src/tree/element_tests.rs b/crates/macos/src/tree/element_tests.rs new file mode 100644 index 0000000..be331d2 --- /dev/null +++ b/crates/macos/src/tree/element_tests.rs @@ -0,0 +1,45 @@ +const ELEMENT_SOURCE: &str = include_str!("element.rs"); +const NODE_ATTRIBUTE_FETCH_SOURCE: &str = include_str!("node_attribute_fetch.rs"); +const ATTRIBUTE_NAMES_SOURCE: &str = include_str!("node_attribute_names.rs"); +const READONLY_SOURCE: &str = include_str!("readonly.rs"); + +#[test] +fn generic_child_sources_prefer_exhaustive_navigation_order_before_content_subset() { + assert_eq!( + super::child_attributes(None), + ["AXChildren", "AXChildrenInNavigationOrder", "AXContents"] + ); + assert_eq!( + super::child_attributes(Some("AXBrowser")), + ["AXColumns", "AXContents"] + ); + assert_eq!( + super::child_attributes(Some("AXApplication")), + ["AXWindows", "AXChildren"] + ); +} + +#[test] +fn readonly_derivation_has_a_single_owner() { + let editable_role_check_sites = READONLY_SOURCE.matches("editable_ax_role(role)").count(); + assert_eq!( + editable_role_check_sites, 1, + "readonly derivation must be computed once in compute_readonly, not re-derived \ + separately by fetch_node_attrs and fetch_node_attrs_slow" + ); +} + +#[test] +fn fetch_node_attrs_reuses_cached_attr_names_array() { + assert!( + ATTRIBUTE_NAMES_SOURCE.contains("ATTRIBUTE_NAMES.with"), + "fetch_node_attrs must reuse the thread-local cached attribute-name CFArray instead of \ + rebuilding its 22 CFStrings on every call" + ); + assert!( + !ELEMENT_SOURCE.contains(".map(|attribute| CFString::new(attribute))") + && !NODE_ATTRIBUTE_FETCH_SOURCE.contains(".map(|attribute| CFString::new(attribute))"), + "fetch_node_attrs regressed to rebuilding the CFString attr-name array inline per call; \ + that construction belongs solely in AttributeNames::new" + ); +} diff --git a/crates/macos/src/tree/hit_test.rs b/crates/macos/src/tree/hit_test.rs new file mode 100644 index 0000000..d4a0a23 --- /dev/null +++ b/crates/macos/src/tree/hit_test.rs @@ -0,0 +1,374 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::tree::{ + AXElement, capabilities::same_element, element::ABSOLUTE_MAX_DEPTH, + element_bounds::read_bounds_with_deadline, + }; + use accessibility_sys::{ + AXUIElementCreateSystemWide, kAXApplicationRole, kAXErrorSuccess, kAXRoleAttribute, + }; + use agent_desktop_core::{ + AdapterError, Point, Rect, hit_test::HitTestResult, native_handle::NativeHandle, + }; + + /// Hit-tests `point` in the system-wide accessibility hierarchy, then + /// classifies the frontmost result against `target`'s ancestor chain: a hit on + /// `target` or a descendant reaches it, a hit outside that chain names a + /// real occluder, and a hit on `target`'s own ancestor is retried against + /// the owning application before remaining unknown. + pub fn hit_test_impl( + handle: &NativeHandle, + point: Point, + deadline: agent_desktop_core::Deadline, + ) -> Result<HitTestResult, AdapterError> { + let deadline = crate::tree::locator_deadline::from_operation(deadline)?; + hit_test_element(crate::adapter::ax_element(handle)?, point, deadline) + } + + pub(crate) fn hit_test_ax_element( + target: &AXElement, + point: Point, + deadline: agent_desktop_core::Deadline, + ) -> Result<HitTestResult, AdapterError> { + let deadline = crate::tree::locator_deadline::from_operation(deadline)?; + hit_test_element(target, point, deadline) + } + + pub(crate) fn visible_bounds_ax_element( + target: &AXElement, + deadline: agent_desktop_core::Deadline, + ) -> Result<Option<Rect>, AdapterError> { + let deadline = crate::tree::locator_deadline::from_operation(deadline)?; + clipped_bounds(target, deadline) + } + + fn hit_test_element( + target: &AXElement, + point: Point, + deadline: std::time::Instant, + ) -> Result<HitTestResult, AdapterError> { + let Some(bounds) = read_bounds_with_deadline(target, deadline)? else { + return Ok(HitTestResult::Unknown); + }; + if bounds.width <= 0.0 || bounds.height <= 0.0 { + return Ok(HitTestResult::Unknown); + } + if !clipping_is_complete(target, &point, deadline)? { + return Ok(HitTestResult::Unknown); + } + let system = AXElement(unsafe { AXUIElementCreateSystemWide() }); + if system.0.is_null() { + return Ok(HitTestResult::Unknown); + } + let Some(hit) = hit_at_position(&system, &point, deadline) else { + return Ok(HitTestResult::Unknown); + }; + let Some(classification) = classify_hit(target, &hit, deadline) else { + return Ok(HitTestResult::Unknown); + }; + if needs_application_retry(classification) { + return Ok(hit_test_in_application(target, &point, deadline)); + } + Ok(classification_result(classification, &hit, deadline)) + } + + fn hit_at_position( + root: &AXElement, + point: &Point, + deadline: std::time::Instant, + ) -> Option<AXElement> { + let (error, hit) = + crate::tree::ax_ipc::element_at_position(root, ax_point(point), deadline); + (error == kAXErrorSuccess && !hit.is_null()).then(|| AXElement(hit)) + } + + fn hit_test_in_application( + target: &AXElement, + point: &Point, + deadline: std::time::Instant, + ) -> HitTestResult { + let Ok(pid) = crate::tree::ax_ipc::pid(target, deadline) else { + return HitTestResult::Unknown; + }; + let application = crate::tree::element_for_pid(pid); + let Some(hit) = hit_at_position(&application, point, deadline) else { + return HitTestResult::Unknown; + }; + match classify_hit(target, &hit, deadline) { + Some(HitClassification::ReachesTarget) => HitTestResult::ReachesTarget, + _ => HitTestResult::Unknown, + } + } + + fn classify_hit( + target: &AXElement, + hit: &AXElement, + deadline: std::time::Instant, + ) -> Option<HitClassification> { + let limit = ABSOLUTE_MAX_DEPTH as usize; + let reaches_target = if same_element(target, hit) { + Ancestry::Found + } else { + ancestry(hit, target, limit, deadline) + }; + if reaches_target == Ancestry::Incomplete { + return None; + } + let is_ancestor_of_target = if reaches_target == Ancestry::Found { + Ancestry::Absent + } else { + ancestry(target, hit, limit, deadline) + }; + if is_ancestor_of_target == Ancestry::Incomplete { + return None; + } + Some(classify_relation( + reaches_target == Ancestry::Found, + is_ancestor_of_target == Ancestry::Found, + )) + } + + fn classification_result( + classification: HitClassification, + hit: &AXElement, + deadline: std::time::Instant, + ) -> HitTestResult { + match classification { + HitClassification::ReachesTarget => HitTestResult::ReachesTarget, + HitClassification::AncestorOfTarget => HitTestResult::Unknown, + HitClassification::Unrelated => { + intercepted_by(hit, deadline).unwrap_or(HitTestResult::Unknown) + } + } + } + + pub(super) fn ax_point(point: &Point) -> (f32, f32) { + (point.x as f32, point.y as f32) + } + + pub(super) fn needs_application_retry(classification: HitClassification) -> bool { + matches!(classification, HitClassification::AncestorOfTarget) + } + + fn intercepted_by( + hit: &AXElement, + deadline: std::time::Instant, + ) -> Result<HitTestResult, AdapterError> { + let mut usage = crate::tree::observation_usage::ObservationUsage::new( + agent_desktop_core::ObservationBudget::default(), + ); + let ax_role = + read_complete_text(hit, kAXRoleAttribute, deadline, &mut usage, "hit_test.role")?; + let ax_subrole = + read_complete_text(hit, "AXSubrole", deadline, &mut usage, "hit_test.subrole")?; + let role = ax_role + .as_deref() + .map(|role| crate::tree::roles::ax_role_and_subrole_to_str(role, ax_subrole.as_deref())) + .unwrap_or("unknown"); + Ok(HitTestResult::InterceptedBy { + role: Some(role.to_string()), + name: crate::tree::element_name::resolve_element_name(hit, deadline, &mut usage)?, + bounds: read_bounds_with_deadline(hit, deadline)?, + }) + } + + fn read_complete_text( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + usage: &mut crate::tree::observation_usage::ObservationUsage, + phase: &str, + ) -> Result<Option<String>, AdapterError> { + crate::tree::locator_deadline::prepare(element, deadline)?; + match crate::tree::attributes::copy_string_attr_bounded_result( + element, attribute, deadline, usage, + ) + .map_err(|error| crate::tree::query::read_error::semantic_read(error, phase))? + { + Some(value) if value.complete => Ok(Some(value.value)), + Some(_) => Err(AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "Hit-test text evidence exceeded its observation budget", + ) + .with_details(serde_json::json!({ + "kind": "hit_test_text_incomplete", + "attribute": attribute, + "complete": false, + }))), + None => Ok(None), + } + } + + fn ancestry( + start: &AXElement, + expected: &AXElement, + limit: usize, + deadline: std::time::Instant, + ) -> Ancestry { + let mut current = start.clone(); + let mut visited = Vec::new(); + for _ in 0..limit { + if !remember_ancestor(&mut visited, ¤t) { + return Ancestry::Incomplete; + } + let parent = + match crate::tree::resolve_ax_read::read_element(¤t, "AXParent", deadline) { + Ok(Some(parent)) => parent, + Ok(None) => return Ancestry::Absent, + Err(_) => return Ancestry::Incomplete, + }; + if same_element(&parent, expected) { + return Ancestry::Found; + } + current = parent; + } + Ancestry::Incomplete + } + + fn clipping_is_complete( + target: &AXElement, + point: &Point, + deadline: std::time::Instant, + ) -> Result<bool, AdapterError> { + Ok(clipped_bounds(target, deadline)?.is_some_and(|bounds| { + point.x >= bounds.x + && point.y >= bounds.y + && point.x <= bounds.x + bounds.width + && point.y <= bounds.y + bounds.height + })) + } + + fn clipped_bounds( + target: &AXElement, + deadline: std::time::Instant, + ) -> Result<Option<Rect>, AdapterError> { + let Some(mut visible) = read_bounds_with_deadline(target, deadline)? else { + return Ok(None); + }; + let mut current = target.clone(); + let mut visited = Vec::new(); + let mut usage = crate::tree::observation_usage::ObservationUsage::new( + agent_desktop_core::ObservationBudget::default(), + ); + for _ in 0..ABSOLUTE_MAX_DEPTH { + if !remember_ancestor(&mut visited, ¤t) { + return Ok(None); + } + let role = match read_complete_text( + ¤t, + kAXRoleAttribute, + deadline, + &mut usage, + "hit_test.clip_role", + ) { + Ok(role) => role, + Err(_) => return Ok(None), + }; + if ends_clipping_walk(role.as_deref()) { + return Ok(Some(visible)); + } + if role.as_deref().is_some_and(clips_descendants) { + let Some(bounds) = read_bounds_with_deadline(¤t, deadline)? else { + return Ok(None); + }; + let Some(intersection) = intersect_rects(visible, bounds) else { + return Ok(None); + }; + visible = intersection; + } + current = + match crate::tree::resolve_ax_read::read_element(¤t, "AXParent", deadline) { + Ok(Some(parent)) => parent, + Ok(None) => return Ok(Some(visible)), + Err(_) => return Ok(None), + }; + } + Ok(None) + } + + pub(super) fn intersect_rects(left: Rect, right: Rect) -> Option<Rect> { + let x = left.x.max(right.x); + let y = left.y.max(right.y); + let right_edge = (left.x + left.width).min(right.x + right.width); + let bottom_edge = (left.y + left.height).min(right.y + right.height); + (right_edge > x && bottom_edge > y).then_some(Rect { + x, + y, + width: right_edge - x, + height: bottom_edge - y, + }) + } + + fn clips_descendants(role: &str) -> bool { + matches!( + role, + "AXWindow" | "AXScrollArea" | "AXWebArea" | "AXSheet" | "AXPopover" + ) + } + + pub(super) fn remember_ancestor(visited: &mut Vec<AXElement>, current: &AXElement) -> bool { + if visited + .iter() + .any(|ancestor| same_element(ancestor, current)) + { + return false; + } + visited.push(current.clone()); + true + } + + pub(super) fn ends_clipping_walk(role: Option<&str>) -> bool { + role == Some(kAXApplicationRole) + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Ancestry { + Found, + Absent, + Incomplete, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum HitClassification { + ReachesTarget, + AncestorOfTarget, + Unrelated, + } + + pub(super) fn classify_relation( + reaches_target: bool, + is_ancestor_of_target: bool, + ) -> HitClassification { + if reaches_target { + HitClassification::ReachesTarget + } else if is_ancestor_of_target { + HitClassification::AncestorOfTarget + } else { + HitClassification::Unrelated + } + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use agent_desktop_core::{ + AdapterError, Point, hit_test::HitTestResult, native_handle::NativeHandle, + }; + + pub fn hit_test_impl( + _handle: &NativeHandle, + _point: Point, + _deadline: agent_desktop_core::Deadline, + ) -> Result<HitTestResult, AdapterError> { + Err(AdapterError::not_supported("hit_test")) + } +} + +pub(crate) use imp::hit_test_impl; + +#[cfg(target_os = "macos")] +pub(crate) use imp::{hit_test_ax_element, visible_bounds_ax_element}; + +#[cfg(all(test, target_os = "macos"))] +#[path = "hit_test_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/hit_test_tests.rs b/crates/macos/src/tree/hit_test_tests.rs new file mode 100644 index 0000000..462a63c --- /dev/null +++ b/crates/macos/src/tree/hit_test_tests.rs @@ -0,0 +1,101 @@ +use super::imp::{ + HitClassification, ax_point, classify_relation, ends_clipping_walk, intersect_rects, + needs_application_retry, remember_ancestor, +}; +use crate::tree::AXElement; +use accessibility_sys::AXUIElementCreateSystemWide; +use agent_desktop_core::{Point, Rect}; + +#[test] +fn self_hit_reaches_target() { + assert_eq!( + classify_relation(true, false), + HitClassification::ReachesTarget + ); +} + +#[test] +fn reaches_target_takes_priority_over_ancestor() { + assert_eq!( + classify_relation(true, true), + HitClassification::ReachesTarget + ); +} + +#[test] +fn ancestor_hit_is_reported_separately_from_unrelated() { + assert_eq!( + classify_relation(false, true), + HitClassification::AncestorOfTarget + ); +} + +#[test] +fn unrelated_hit_is_neither_target_nor_ancestor() { + assert_eq!( + classify_relation(false, false), + HitClassification::Unrelated + ); +} + +#[test] +fn target_ancestor_hit_retries_in_application_scope() { + assert!(needs_application_retry(HitClassification::AncestorOfTarget)); + assert!(!needs_application_retry(HitClassification::ReachesTarget)); + assert!(!needs_application_retry(HitClassification::Unrelated)); +} + +#[test] +fn external_display_point_stays_in_global_top_left_coordinates() { + assert_eq!( + ax_point(&Point { + x: 2065.0, + y: 636.0, + }), + (2065.0, 636.0) + ); +} + +#[test] +fn application_root_completes_the_clipping_walk_without_a_parent() { + assert!(ends_clipping_walk(Some("AXApplication"))); + assert!(!ends_clipping_walk(Some("AXGroup"))); + assert!(!ends_clipping_walk(None)); +} + +#[test] +fn clipped_bounds_keep_only_the_visible_part_of_a_partially_shown_target() { + let target = Rect { + x: 100.0, + y: 900.0, + width: 220.0, + height: 160.0, + }; + let viewport = Rect { + x: 0.0, + y: 0.0, + width: 1000.0, + height: 950.0, + }; + + assert_eq!( + intersect_rects(target, viewport), + Some(Rect { + x: 100.0, + y: 900.0, + width: 220.0, + height: 50.0, + }) + ); +} + +#[test] +fn ancestor_walk_retains_handles_and_detects_the_same_element() { + let system = AXElement(unsafe { AXUIElementCreateSystemWide() }); + let equivalent = AXElement(unsafe { AXUIElementCreateSystemWide() }); + let mut visited = Vec::new(); + + assert!(remember_ancestor(&mut visited, &system)); + assert!(!remember_ancestor(&mut visited, &equivalent)); + assert_eq!(visited.len(), 1); +} diff --git a/crates/macos/src/tree/locator_deadline.rs b/crates/macos/src/tree/locator_deadline.rs new file mode 100644 index 0000000..eb7f576 --- /dev/null +++ b/crates/macos/src/tree/locator_deadline.rs @@ -0,0 +1,78 @@ +use agent_desktop_core::AdapterError; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub(crate) const MAX_IPC_SLICE: Duration = Duration::from_millis(250); + +#[cfg(test)] +pub(crate) fn from_timeout(started: Instant, timeout: Duration) -> Instant { + started.checked_add(timeout).unwrap_or(started) +} + +pub(crate) fn from_operation( + deadline: agent_desktop_core::Deadline, +) -> Result<Instant, AdapterError> { + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(deadline.timeout_error()); + } + Instant::now() + .checked_add(remaining) + .ok_or_else(|| AdapterError::timeout("Accessibility deadline overflowed")) +} + +pub(crate) fn remaining(deadline: Instant) -> Result<Duration, AdapterError> { + remaining_at(deadline, Instant::now()) +} + +pub(crate) fn prepare( + element: &super::AXElement, + deadline: Instant, +) -> Result<Duration, AdapterError> { + super::ax_ipc::prepare(element, deadline) +} + +fn remaining_at(deadline: Instant, now: Instant) -> Result<Duration, AdapterError> { + let remaining = deadline.saturating_duration_since(now); + if remaining.is_zero() { + return Err( + AdapterError::timeout("Live locator deadline exhausted").with_details(json!({ + "kind": "locator_deadline_exhausted", + "retryable": true, + })), + ); + } + Ok(remaining) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remaining_budget_is_derived_from_one_absolute_deadline() { + let started = Instant::now(); + let deadline = from_timeout(started, Duration::from_millis(100)); + let now = started + Duration::from_millis(40); + + assert_eq!( + remaining_at(deadline, now).unwrap(), + Duration::from_millis(60) + ); + } + + #[test] + fn exhausted_and_overflowing_budgets_fail_closed() { + let started = Instant::now(); + let deadline = from_timeout(started, Duration::from_millis(1)); + + let error = remaining_at(deadline, deadline).expect_err("deadline must be exhausted"); + assert!(error.is_explicitly_retryable()); + assert_eq!(from_timeout(started, Duration::MAX), started); + } + + #[test] + fn operation_deadline_rejects_expired_inputs() { + assert!(from_operation(agent_desktop_core::Deadline::after(0).unwrap()).is_err()); + } +} diff --git a/crates/macos/src/tree/mod.rs b/crates/macos/src/tree/mod.rs index a7cbce9..73b57cb 100644 --- a/crates/macos/src/tree/mod.rs +++ b/crates/macos/src/tree/mod.rs @@ -1,36 +1,53 @@ -pub mod action_list; +pub(crate) mod action_list; +mod adapter; pub(crate) mod attributes; -pub mod ax_element; +pub(crate) mod ax_element; +pub(crate) mod ax_ipc; pub(crate) mod ax_value; -pub mod build_context; -pub mod builder; -pub mod capabilities; -pub mod element; -pub mod element_bounds; +pub(crate) mod bounded_string; +pub(crate) mod build_context; +pub(crate) mod capabilities; +pub(crate) mod child_labels; +pub(crate) mod element; +pub(crate) mod element_bounds; pub(crate) mod element_dedupe; +pub(crate) mod element_name; +pub(crate) mod hit_test; +pub(crate) mod locator_deadline; +pub(crate) mod native_id; +pub(crate) mod node_attr_states; +pub(crate) mod node_attribute_decode; +pub(crate) mod node_attribute_fetch; +pub(crate) mod node_attribute_metrics; +pub(crate) mod node_attribute_names; +pub(crate) mod node_attribute_read; +pub(crate) mod node_attribute_status; pub(crate) mod node_attrs; -pub mod resolve; +pub(crate) mod node_control_states; +pub(crate) mod node_identifiers; +pub(crate) mod node_semantic_states; +pub(crate) mod observation_usage; +pub(crate) mod query; +pub(crate) mod readonly; +mod renderer_probe; +pub(crate) mod resolve; +pub(crate) mod resolve_ax_read; mod resolve_bounds; mod resolve_classify; -mod resolve_deadline; -mod resolve_identity; +mod resolve_errors; +mod resolve_read_context; mod resolve_roots; mod resolve_search; -pub mod roles; -pub mod surfaces; +pub(crate) mod roles; +pub(crate) mod state_reader; +pub(crate) mod surface_inventory; +pub(crate) mod surface_read; +pub(crate) mod surfaces; +pub(crate) mod text_attributes; -pub(crate) use attributes::{ - copy_ax_array, copy_bool_attr, copy_element_attr, copy_i64_attr, copy_string_attr, - copy_value_typed, -}; -pub use ax_element::AXElement; -pub use build_context::TreeBuildContext; -pub use builder::{build_subtree, window_element_for}; -pub use capabilities::same_element; -pub use element::{element_for_pid, resolve_element_name}; -pub use element_bounds::read_bounds; +pub(crate) use attributes::{copy_bool_attr, copy_value_typed}; +pub(crate) use ax_element::AXElement; +pub(crate) use build_context::TreeBuildContext; +pub(crate) use capabilities::same_element; +pub(crate) use element::element_for_pid; pub(crate) use node_attrs::NodeAttrs; -pub use surfaces::{ - alert_for_pid, focused_surface_for_pid, menu_element_for_pid, menubar_for_pid, popover_for_pid, - sheet_for_pid, -}; diff --git a/crates/macos/src/tree/native_id.rs b/crates/macos/src/tree/native_id.rs new file mode 100644 index 0000000..518863e --- /dev/null +++ b/crates/macos/src/tree/native_id.rs @@ -0,0 +1,37 @@ +pub(crate) fn meaningful_native_id(raw: Option<String>) -> Option<String> { + raw.filter(|id| !id.trim().is_empty()) +} + +pub(crate) fn meaningful_dom_id(raw: Option<String>) -> Option<String> { + raw.filter(|id| !id.trim().is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn undocumented_ns_prefix_is_preserved() { + assert_eq!( + meaningful_native_id(Some("_NS:42".into())), + Some("_NS:42".into()) + ); + } + + #[test] + fn developer_assigned_id_is_kept() { + assert_eq!( + meaningful_native_id(Some("submit-btn".into())), + Some("submit-btn".into()) + ); + } + + #[test] + fn chromium_dom_id_is_kept_and_blank_is_rejected() { + assert_eq!( + meaningful_dom_id(Some("compose-message".into())), + Some("compose-message".into()) + ); + assert_eq!(meaningful_dom_id(Some(" ".into())), None); + } +} diff --git a/crates/macos/src/tree/node_attr_states.rs b/crates/macos/src/tree/node_attr_states.rs new file mode 100644 index 0000000..140c4d0 --- /dev/null +++ b/crates/macos/src/tree/node_attr_states.rs @@ -0,0 +1,18 @@ +use super::{node_control_states::NodeControlStates, node_semantic_states::NodeSemanticStates}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NodeAttrStates { + pub(crate) enabled: bool, + pub(crate) control: NodeControlStates, + pub(crate) semantic: NodeSemanticStates, +} + +impl Default for NodeAttrStates { + fn default() -> Self { + Self { + enabled: true, + control: NodeControlStates::default(), + semantic: NodeSemanticStates::default(), + } + } +} diff --git a/crates/macos/src/tree/node_attribute_decode.rs b/crates/macos/src/tree/node_attribute_decode.rs new file mode 100644 index 0000000..fd97768 --- /dev/null +++ b/crates/macos/src/tree/node_attribute_decode.rs @@ -0,0 +1,172 @@ +#[cfg(target_os = "macos")] +mod imp { + use accessibility_sys::{ + AXValueGetType, AXValueGetTypeID, AXValueGetValue, kAXValueTypeAXError, + kAXValueTypeCGPoint, kAXValueTypeCGSize, + }; + use core_foundation::{ + base::{CFType, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + }; + use core_foundation_sys::base::{CFGetTypeID, CFNullGetTypeID}; + use core_foundation_sys::number::{ + CFNumberGetType, CFNumberIsFloatType, kCFNumberFloat32Type, kCFNumberFloatType, + }; + use core_graphics::geometry::{CGPoint, CGSize}; + + pub(crate) fn is_null(item: &CFType) -> bool { + unsafe { CFGetTypeID(item.as_CFTypeRef()) == CFNullGetTypeID() } + } + + pub(crate) fn slot_error(item: &CFType) -> Option<i32> { + let value = item.as_CFTypeRef(); + if unsafe { CFGetTypeID(value) } != unsafe { AXValueGetTypeID() } + || unsafe { AXValueGetType(value as _) } != kAXValueTypeAXError + { + return None; + } + let mut error = 0_i32; + unsafe { + AXValueGetValue( + value as _, + kAXValueTypeAXError, + &mut error as *mut _ as *mut std::ffi::c_void, + ) + } + .then_some(error) + } + + pub(crate) fn text( + index: usize, + item: &CFType, + usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> Option<crate::tree::bounded_string::BoundedString> { + if let Some(value) = item.downcast::<CFString>() { + return crate::tree::bounded_string::BoundedString::from_cf(&value, usage).ok(); + } + let value = match index { + 3 => scalar_text(item), + 4..=12 => item + .downcast::<CFBoolean>() + .map(|value| bool::from(value).to_string()), + _ => None, + }?; + Some(crate::tree::bounded_string::BoundedString::from_owned( + value, usage, + )) + } + + fn scalar_text(item: &CFType) -> Option<String> { + if let Some(value) = item.downcast::<CFBoolean>() { + return Some(bool::from(value).to_string()); + } + let number = item.downcast::<CFNumber>()?; + number_text(&number) + } + + pub(crate) fn number_text(number: &CFNumber) -> Option<String> { + let number_ref = number.as_concrete_TypeRef(); + if unsafe { CFNumberIsFloatType(number_ref) } == 0 { + return number.to_i64().map(|value| value.to_string()); + } + let number_type = unsafe { CFNumberGetType(number_ref) }; + if number_type == kCFNumberFloat32Type || number_type == kCFNumberFloatType { + number.to_f32().map(|value| value.to_string()) + } else { + number.to_f64().map(|value| value.to_string()) + } + } + + pub(crate) fn point(item: &CFType) -> Option<CGPoint> { + let mut point = CGPoint::new(0.0, 0.0); + unsafe { + AXValueGetValue( + item.as_CFTypeRef() as _, + kAXValueTypeCGPoint, + &mut point as *mut _ as *mut std::ffi::c_void, + ) + } + .then_some(point) + } + + pub(crate) fn size(item: &CFType) -> Option<CGSize> { + let mut size = CGSize::new(0.0, 0.0); + unsafe { + AXValueGetValue( + item.as_CFTypeRef() as _, + kAXValueTypeCGSize, + &mut size as *mut _ as *mut std::ffi::c_void, + ) + } + .then_some(size) + } +} + +#[cfg(target_os = "macos")] +pub(crate) use imp::{is_null, number_text, point, size, slot_error, text}; + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::*; + use accessibility_sys::{AXValueCreate, kAXErrorCannotComplete, kAXValueTypeAXError}; + use core_foundation::{ + base::{CFType, TCFType}, + number::CFNumber, + }; + + fn number_item(number: &CFNumber) -> CFType { + unsafe { CFType::wrap_under_get_rule(number.as_CFTypeRef()) } + } + + fn decode_number(number: &CFNumber) -> String { + let item = number_item(number); + let mut usage = crate::tree::observation_usage::ObservationUsage::new( + agent_desktop_core::ObservationBudget::default(), + ); + text(3, &item, &mut usage) + .expect("CFNumber must decode") + .value + } + + #[test] + fn ax_error_slots_decode_without_becoming_empty_values() { + let error = kAXErrorCannotComplete; + let value = unsafe { + AXValueCreate( + kAXValueTypeAXError, + &error as *const _ as *const std::ffi::c_void, + ) + }; + let value = unsafe { CFType::wrap_under_create_rule(value as _) }; + + assert_eq!(slot_error(&value), Some(kAXErrorCannotComplete)); + } + + #[test] + fn cf_null_slots_remain_authoritatively_absent() { + let value = unsafe { CFType::wrap_under_get_rule(core_foundation_sys::base::kCFNull as _) }; + + assert!(is_null(&value)); + assert_eq!(slot_error(&value), None); + } + + #[test] + fn fractional_numbers_keep_shortest_round_trip_text() { + let first = decode_number(&CFNumber::from(1.001_f64)); + let second = decode_number(&CFNumber::from(1.002_f64)); + + assert_eq!(first, "1.001"); + assert_eq!(second, "1.002"); + assert_ne!(first, second); + } + + #[test] + fn large_integers_remain_exact() { + assert_eq!( + decode_number(&CFNumber::from(i64::MAX)), + i64::MAX.to_string() + ); + } +} diff --git a/crates/macos/src/tree/node_attribute_fetch.rs b/crates/macos/src/tree/node_attribute_fetch.rs new file mode 100644 index 0000000..7e2b6f9 --- /dev/null +++ b/crates/macos/src/tree/node_attribute_fetch.rs @@ -0,0 +1,367 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::{ + cf_type::created_cf_array, + tree::{ + NodeAttrs, + ax_element::AXElement, + ax_value, + element_bounds::rect_from_parts, + node_attr_states::NodeAttrStates, + node_attribute_decode, + node_attribute_names::copy_node_attribute_values, + node_attribute_read::NodeAttributeRead, + node_attribute_status::{ + AX_DOM_IDENTIFIER, AX_IDENTIFIER, NodeAttributeStatus, ROLE, SUBROLE, VALUE, + }, + node_attrs::{parse_bool_attr, parse_enabled}, + node_control_states::NodeControlStates, + node_identifiers::NodeIdentifiers, + node_semantic_states::NodeSemanticStates, + }, + }; + use accessibility_sys::{ + kAXDescriptionAttribute, kAXErrorFailure, kAXErrorSuccess, kAXRoleAttribute, + kAXSubroleAttribute, kAXTitleAttribute, + }; + use agent_desktop_core::EvidenceRequirements; + + pub(crate) fn fetch_node_attrs_with_status_for( + element: &AXElement, + requirements: EvidenceRequirements, + deadline: std::time::Instant, + usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> NodeAttributeRead { + let mask = crate::tree::node_attribute_names::safe_attribute_mask(requirements); + let requested = + crate::tree::node_attribute_names::requested_indices(mask).collect::<Vec<_>>(); + let mut requested_count = requested.len() as u64; + if crate::tree::locator_deadline::prepare(element, deadline).is_err() { + return failed_node_attribute_read( + accessibility_sys::kAXErrorCannotComplete, + requested_count, + true, + 1, + ); + } + let (batch_error, batch_result) = copy_node_attribute_values(element, mask, deadline); + if batch_error != kAXErrorSuccess || batch_result.is_null() { + if !batch_result.is_null() { + drop(created_cf_array(batch_result)); + } + return failed_node_attribute_read( + if batch_error == kAXErrorSuccess { + kAXErrorFailure + } else { + batch_error + }, + requested_count, + false, + 1, + ); + } + let Some(attributes) = created_cf_array(batch_result) else { + return failed_node_attribute_read(kAXErrorFailure, requested_count, false, 1); + }; + let mut batch_reads = 1; + let fallback_reads = 0; + + let mut texts: [Option<String>; 17] = Default::default(); + let mut labelled_by_text = None; + let mut position = None; + let mut size = None; + let mut has_scrollbars = false; + let mut subrole = None; + let mut status = NodeAttributeStatus::default(); + let mut deadline_exhausted = false; + for (index, item) in requested.into_iter().zip(attributes.into_iter()) { + if node_attribute_decode::is_null(&item) { + continue; + } + if let Some(error) = node_attribute_decode::slot_error(&item) { + status.record_slot_error(index, error); + continue; + } + match index { + 0..=16 => match node_attribute_decode::text(index, &item, usage) { + Some(value) if value.complete => texts[index] = Some(value.value), + Some(_) => status.record_truncated(index), + None => status.record_slot_error(index, kAXErrorFailure), + }, + 17 => { + if let Some(label) = ax_value::retained_ax_element(&item) { + match title_ui_element_text(&label, deadline, usage) { + Ok(Some(value)) if value.complete => { + labelled_by_text = Some(value.value) + } + Ok(Some(_)) => status.record_truncated(index), + Ok(None) => {} + Err(error) => { + deadline_exhausted |= + error == accessibility_sys::kAXErrorCannotComplete; + status.record_slot_error(index, error); + } + } + } else { + status.record_slot_error(index, kAXErrorFailure); + } + } + 18 => match node_attribute_decode::point(&item) { + Some(value) => position = Some(value), + None => status.record_slot_error(index, kAXErrorFailure), + }, + 19 => match node_attribute_decode::size(&item) { + Some(value) => size = Some(value), + None => status.record_slot_error(index, kAXErrorFailure), + }, + 20 | 21 => { + if ax_value::retained_ax_element(&item).is_some() { + has_scrollbars = true; + } else { + status.record_slot_error(index, kAXErrorFailure); + } + } + 22 => match node_attribute_decode::text(index, &item, usage) { + Some(value) if value.complete => subrole = Some(value.value), + Some(_) => status.record_truncated(index), + None => status.record_slot_error(index, kAXErrorFailure), + }, + _ => {} + } + } + let get = |index: usize| texts.get(index).and_then(Clone::clone); + let role = get(ROLE); + if status.field_unknown(SUBROLE) { + subrole = None; + } + let role_complete = !status.field_unknown(ROLE) && role.is_some(); + let subrole_complete = !status.field_unknown(SUBROLE); + if crate::tree::node_attribute_names::should_read_value( + requirements, + role.as_deref(), + subrole.as_deref(), + role_complete, + subrole_complete, + ) { + batch_reads += 1; + requested_count += 1; + if crate::tree::locator_deadline::prepare(element, deadline).is_err() { + deadline_exhausted = true; + status.record_slot_error(VALUE, accessibility_sys::kAXErrorCannotComplete); + } else { + let value_mask = crate::tree::node_attribute_status::attribute_bit(VALUE); + let (value_error, value_result) = + copy_node_attribute_values(element, value_mask, deadline); + if value_error != kAXErrorSuccess || value_result.is_null() { + if !value_result.is_null() { + drop(created_cf_array(value_result)); + } + status.record_slot_error( + VALUE, + if value_error == kAXErrorSuccess { + kAXErrorFailure + } else { + value_error + }, + ); + } else if let Some(values) = created_cf_array(value_result) { + match values.into_iter().next() { + Some(item) if node_attribute_decode::is_null(&item) => {} + Some(item) => { + if let Some(error) = node_attribute_decode::slot_error(&item) { + status.record_slot_error(VALUE, error); + } else { + match node_attribute_decode::text(VALUE, &item, usage) { + Some(value) if value.complete => { + texts[VALUE] = Some(value.value) + } + Some(_) => status.record_truncated(VALUE), + None => status.record_slot_error(VALUE, kAXErrorFailure), + } + } + } + None => status.record_slot_error(VALUE, kAXErrorFailure), + } + } else { + status.record_slot_error(VALUE, kAXErrorFailure); + } + } + } else if requirements.value && (!role_complete || !subrole_complete) { + status.record_slot_error(VALUE, accessibility_sys::kAXErrorCannotComplete); + } + let get = |index: usize| texts.get(index).and_then(Clone::clone); + let value = get(VALUE); + let readonly = if requirements.states { + crate::tree::readonly::read_readonly(element, role.as_deref(), deadline) + } else { + crate::tree::readonly::ReadonlyRead { + value: None, + error: None, + attempted: false, + deadline_exhausted: false, + } + }; + if let Some(error) = readonly.error { + status.record_readonly_error(error); + } + deadline_exhausted |= readonly.deadline_exhausted; + let identifier_field = |index| { + if status.field_unknown(index) { + agent_desktop_core::LocatorField::Unknown + } else { + get(index) + .map(agent_desktop_core::LocatorField::Known) + .unwrap_or(agent_desktop_core::LocatorField::Absent) + } + }; + NodeAttributeRead { + attrs: NodeAttrs { + name_evidence: agent_desktop_core::NameEvidence { + explicit_label: None, + labelled_by_text, + native_title: get(1), + static_value: (role.as_deref() == Some("AXStaticText")) + .then(|| value.clone()) + .flatten(), + child_label: None, + placeholder: get(16), + description: get(2), + }, + role, + subrole, + value, + states: NodeAttrStates { + enabled: parse_enabled(get(4)), + control: NodeControlStates { + focused: parse_bool_attr(get(5)), + expanded: parse_bool_attr(get(6)), + disclosing: parse_bool_attr(get(7)), + selected: parse_bool_attr(get(8)), + readonly: readonly.value, + }, + semantic: NodeSemanticStates { + hidden: parse_bool_attr(get(9)), + busy: parse_bool_attr(get(10)), + modal: parse_bool_attr(get(11)), + required: parse_bool_attr(get(12)), + }, + }, + bounds: position + .zip(size) + .and_then(|(position, size)| rect_from_parts(position, size)), + has_scrollbars, + }, + identifiers: NodeIdentifiers::from_fields( + identifier_field(AX_IDENTIFIER), + identifier_field(AX_DOM_IDENTIFIER), + ), + metrics: crate::tree::node_attribute_metrics::NodeAttributeMetrics { + batch_reads, + requested_count, + fallback_reads, + settable_reads: u64::from(readonly.attempted), + deadline_exhausted, + }, + status, + } + } + + fn failed_node_attribute_read( + error: i32, + requested_count: u64, + deadline_exhausted: bool, + batch_reads: u64, + ) -> NodeAttributeRead { + let mut status = NodeAttributeStatus::default(); + status.record_batch_error(error); + NodeAttributeRead { + attrs: NodeAttrs::default(), + identifiers: NodeIdentifiers::from_fields( + agent_desktop_core::LocatorField::Unknown, + agent_desktop_core::LocatorField::Unknown, + ), + metrics: crate::tree::node_attribute_metrics::NodeAttributeMetrics { + batch_reads, + requested_count, + fallback_reads: 0, + settable_reads: 0, + deadline_exhausted, + }, + status, + } + } + + fn title_ui_element_text( + element: &AXElement, + deadline: std::time::Instant, + usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> Result<Option<crate::tree::bounded_string::BoundedString>, i32> { + crate::tree::locator_deadline::prepare(element, deadline) + .map_err(|_| accessibility_sys::kAXErrorCannotComplete)?; + let title = crate::tree::attributes::copy_string_attr_bounded_result( + element, + kAXTitleAttribute, + deadline, + usage, + )?; + if let Some(title) = title.filter(|value| !value.value.trim().is_empty()) { + return Ok(Some(title)); + } + let role = read_unbounded_identity(element, kAXRoleAttribute, deadline)?; + let subrole = read_unbounded_identity(element, kAXSubroleAttribute, deadline)?; + let safe_static_text = role.as_deref() == Some("AXStaticText") + && subrole.as_deref() != Some("AXSecureTextField"); + if safe_static_text { + crate::tree::locator_deadline::prepare(element, deadline) + .map_err(|_| accessibility_sys::kAXErrorCannotComplete)?; + if let Some(value) = + crate::tree::attributes::copy_value_typed_bounded_result(element, deadline, usage)? + .filter(|value| !value.value.trim().is_empty()) + { + return Ok(Some(value)); + } + } + crate::tree::locator_deadline::prepare(element, deadline) + .map_err(|_| accessibility_sys::kAXErrorCannotComplete)?; + crate::tree::attributes::copy_string_attr_bounded_result( + element, + kAXDescriptionAttribute, + deadline, + usage, + ) + } + + fn read_unbounded_identity( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + ) -> Result<Option<String>, i32> { + crate::tree::locator_deadline::prepare(element, deadline) + .map_err(|_| accessibility_sys::kAXErrorCannotComplete)?; + crate::tree::attributes::copy_string_attr_result(element, attribute, deadline) + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use crate::tree::{ + NodeAttrs, ax_element::AXElement, node_attribute_read::NodeAttributeRead, + node_identifiers::NodeIdentifiers, + }; + + pub(crate) fn fetch_node_attrs_with_status_for( + _element: &AXElement, + _requirements: agent_desktop_core::EvidenceRequirements, + _deadline: std::time::Instant, + _usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> NodeAttributeRead { + NodeAttributeRead { + attrs: NodeAttrs::default(), + identifiers: NodeIdentifiers::default(), + metrics: crate::tree::node_attribute_metrics::NodeAttributeMetrics::default(), + status: crate::tree::node_attribute_status::NodeAttributeStatus::default(), + } + } +} + +pub(crate) use imp::fetch_node_attrs_with_status_for; diff --git a/crates/macos/src/tree/node_attribute_metrics.rs b/crates/macos/src/tree/node_attribute_metrics.rs new file mode 100644 index 0000000..488d36e --- /dev/null +++ b/crates/macos/src/tree/node_attribute_metrics.rs @@ -0,0 +1,8 @@ +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct NodeAttributeMetrics { + pub(crate) batch_reads: u64, + pub(crate) requested_count: u64, + pub(crate) fallback_reads: u64, + pub(crate) settable_reads: u64, + pub(crate) deadline_exhausted: bool, +} diff --git a/crates/macos/src/tree/node_attribute_names.rs b/crates/macos/src/tree/node_attribute_names.rs new file mode 100644 index 0000000..ae19ff5 --- /dev/null +++ b/crates/macos/src/tree/node_attribute_names.rs @@ -0,0 +1,257 @@ +#[cfg(target_os = "macos")] +mod imp { + use std::{cell::RefCell, collections::HashMap}; + + use accessibility_sys::{ + kAXDescriptionAttribute, kAXEnabledAttribute, kAXPlaceholderValueAttribute, + kAXPositionAttribute, kAXRoleAttribute, kAXSizeAttribute, kAXSubroleAttribute, + kAXTitleAttribute, kAXTitleUIElementAttribute, kAXValueAttribute, + }; + use agent_desktop_core::EvidenceRequirements; + use core_foundation::{ + array::CFArray, + base::{CFTypeRef, TCFType}, + string::{CFString, CFStringRef}, + }; + + pub(crate) const NODE_ATTRIBUTE_COUNT: u64 = 23; + const NAMES: [&str; NODE_ATTRIBUTE_COUNT as usize] = [ + kAXRoleAttribute, + kAXTitleAttribute, + kAXDescriptionAttribute, + kAXValueAttribute, + kAXEnabledAttribute, + "AXFocused", + "AXExpanded", + "AXDisclosing", + "AXSelected", + "AXHidden", + "AXElementBusy", + "AXModal", + "AXRequired", + "AXIdentifier", + "AXDOMIdentifier", + "AXLabelValue", + kAXPlaceholderValueAttribute, + kAXTitleUIElementAttribute, + kAXPositionAttribute, + kAXSizeAttribute, + "AXVerticalScrollBar", + "AXHorizontalScrollBar", + kAXSubroleAttribute, + ]; + + struct AttributeNames { + _names: Vec<CFString>, + arrays: RefCell<HashMap<u32, CFArray<CFStringRef>>>, + } + + impl AttributeNames { + fn new() -> Self { + let names = NAMES + .iter() + .map(|attribute| CFString::new(attribute)) + .collect::<Vec<_>>(); + Self { + _names: names, + arrays: RefCell::new(HashMap::new()), + } + } + + fn with_array<T>(&self, mask: u32, read: impl FnOnce(&CFArray<CFStringRef>) -> T) -> T { + let mut arrays = self.arrays.borrow_mut(); + let array = arrays.entry(mask).or_insert_with(|| { + let refs = self + ._names + .iter() + .enumerate() + .filter(|(index, _)| mask & (1_u32 << index) != 0) + .map(|(_, attribute)| attribute.as_concrete_TypeRef()) + .collect::<Vec<_>>(); + CFArray::from_copyable(&refs) + }); + read(array) + } + } + + thread_local! { + static ATTRIBUTE_NAMES: AttributeNames = AttributeNames::new(); + } + + pub(crate) fn copy_node_attribute_values( + element: &crate::tree::AXElement, + mask: u32, + deadline: std::time::Instant, + ) -> (i32, CFTypeRef) { + ATTRIBUTE_NAMES.with(|names| { + names.with_array(mask, |array| { + crate::tree::ax_ipc::copy_multiple_attribute_values( + element, + array.as_concrete_TypeRef(), + deadline, + ) + }) + }) + } + + pub(crate) fn attribute_mask(requirements: EvidenceRequirements) -> u32 { + use crate::tree::node_attribute_status::{ + AX_DOM_IDENTIFIER, AX_IDENTIFIER, BUSY, DESCRIPTION, DISCLOSING, ENABLED, EXPANDED, + FOCUSED, HIDDEN, HORIZONTAL_SCROLLBAR, MODAL, PLACEHOLDER, POSITION, REQUIRED, ROLE, + SELECTED, SIZE, SUBROLE, TITLE, TITLE_ELEMENT, VALUE, VERTICAL_SCROLLBAR, + attribute_bit, + }; + let mut mask = attribute_bit(ROLE) | attribute_bit(SUBROLE); + if requirements.name || requirements.description { + for index in [TITLE, DESCRIPTION, VALUE, PLACEHOLDER, TITLE_ELEMENT] { + mask |= attribute_bit(index); + } + } + if requirements.value { + mask |= attribute_bit(VALUE); + } + if requirements.identifiers { + mask |= attribute_bit(AX_IDENTIFIER) | attribute_bit(AX_DOM_IDENTIFIER); + } + if requirements.states { + for index in [ + VALUE, ENABLED, FOCUSED, EXPANDED, DISCLOSING, SELECTED, HIDDEN, BUSY, MODAL, + REQUIRED, POSITION, SIZE, + ] { + mask |= attribute_bit(index); + } + } + if requirements.ref_evidence.bounds { + for index in [POSITION, SIZE] { + mask |= attribute_bit(index); + } + } + if requirements.ref_evidence.actions { + for index in [VERTICAL_SCROLLBAR, HORIZONTAL_SCROLLBAR] { + mask |= attribute_bit(index); + } + } + mask + } + + pub(crate) fn safe_attribute_mask(requirements: EvidenceRequirements) -> u32 { + use crate::tree::node_attribute_status::{VALUE, attribute_bit}; + attribute_mask(requirements) & !attribute_bit(VALUE) + } + + pub(crate) fn should_read_value( + requirements: EvidenceRequirements, + role: Option<&str>, + subrole: Option<&str>, + role_complete: bool, + subrole_complete: bool, + ) -> bool { + if !role_complete || !subrole_complete || role.is_none() { + return false; + } + let secure = role == Some("AXSecureTextField") || subrole == Some("AXSecureTextField"); + if secure { + return false; + } + requirements.value + || (requirements.name && role == Some("AXStaticText")) + || (requirements.states + && role.is_some_and(|role| { + let canonical = crate::tree::roles::ax_role_and_subrole_to_str(role, subrole); + crate::tree::roles::is_toggleable_role(canonical) || canonical == "button" + })) + } + + pub(crate) fn requested_indices(mask: u32) -> impl Iterator<Item = usize> { + (0..NODE_ATTRIBUTE_COUNT as usize).filter(move |index| mask & (1_u32 << index) != 0) + } +} + +#[cfg(test)] +pub(crate) use imp::NODE_ATTRIBUTE_COUNT; +#[cfg(target_os = "macos")] +#[cfg(test)] +pub(crate) use imp::attribute_mask; +pub(crate) use imp::{ + copy_node_attribute_values, requested_indices, safe_attribute_mask, should_read_value, +}; + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::*; + use agent_desktop_core::EvidenceRequirements; + + #[test] + fn role_plan_requests_role_and_subrole_instead_of_every_attribute() { + let requirements = EvidenceRequirements { + role: true, + ..EvidenceRequirements::default() + }; + + assert_eq!(attribute_mask(requirements).count_ones(), 2); + assert_eq!( + attribute_mask(EvidenceRequirements::snapshot()).count_ones(), + NODE_ATTRIBUTE_COUNT as u32 - 1 + ); + } + + #[test] + fn name_plan_avoids_actions_bounds_states_and_identifiers() { + let requirements = EvidenceRequirements { + role: true, + name: true, + ..EvidenceRequirements::default() + }; + let requested = requested_indices(attribute_mask(requirements)).collect::<Vec<_>>(); + + assert_eq!(requested, [0, 1, 2, 3, 16, 17, 22]); + } + + #[test] + fn role_only_selected_anchor_plan_is_exactly_six_cheap_attributes() { + let mut requirements = EvidenceRequirements { + role: true, + identifiers: true, + ..EvidenceRequirements::default() + }; + requirements.ref_evidence.bounds = true; + let requested = requested_indices(attribute_mask(requirements)).collect::<Vec<_>>(); + + assert_eq!(requested, [0, 13, 14, 18, 19, 22]); + } + + #[test] + fn value_plan_never_fetches_secure_text_content() { + let requirements = EvidenceRequirements::snapshot(); + + assert!(!should_read_value( + requirements, + Some("AXTextField"), + Some("AXSecureTextField"), + true, + true, + )); + assert!(should_read_value( + requirements, + Some("AXTextField"), + None, + true, + true, + )); + assert!(!should_read_value(requirements, None, None, false, true,)); + assert!(!should_read_value( + requirements, + Some("AXTextField"), + None, + true, + false, + )); + assert_eq!( + safe_attribute_mask(requirements) + & crate::tree::node_attribute_status::attribute_bit( + crate::tree::node_attribute_status::VALUE + ), + 0 + ); + } +} diff --git a/crates/macos/src/tree/node_attribute_read.rs b/crates/macos/src/tree/node_attribute_read.rs new file mode 100644 index 0000000..b8fb303 --- /dev/null +++ b/crates/macos/src/tree/node_attribute_read.rs @@ -0,0 +1,10 @@ +use super::{ + NodeAttrs, node_attribute_status::NodeAttributeStatus, node_identifiers::NodeIdentifiers, +}; + +pub(crate) struct NodeAttributeRead { + pub(crate) attrs: NodeAttrs, + pub(crate) identifiers: NodeIdentifiers, + pub(crate) metrics: super::node_attribute_metrics::NodeAttributeMetrics, + pub(crate) status: NodeAttributeStatus, +} diff --git a/crates/macos/src/tree/node_attribute_status.rs b/crates/macos/src/tree/node_attribute_status.rs new file mode 100644 index 0000000..ed0a735 --- /dev/null +++ b/crates/macos/src/tree/node_attribute_status.rs @@ -0,0 +1,189 @@ +use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, kAXErrorNoValue, +}; + +pub(crate) const ROLE: usize = 0; +pub(crate) const TITLE: usize = 1; +pub(crate) const DESCRIPTION: usize = 2; +pub(crate) const VALUE: usize = 3; +pub(crate) const ENABLED: usize = 4; +pub(crate) const FOCUSED: usize = 5; +pub(crate) const EXPANDED: usize = 6; +pub(crate) const DISCLOSING: usize = 7; +pub(crate) const SELECTED: usize = 8; +pub(crate) const HIDDEN: usize = 9; +pub(crate) const BUSY: usize = 10; +pub(crate) const MODAL: usize = 11; +pub(crate) const REQUIRED: usize = 12; +pub(crate) const AX_IDENTIFIER: usize = 13; +pub(crate) const AX_DOM_IDENTIFIER: usize = 14; +pub(crate) const LABEL: usize = 15; +pub(crate) const PLACEHOLDER: usize = 16; +pub(crate) const TITLE_ELEMENT: usize = 17; +pub(crate) const POSITION: usize = 18; +pub(crate) const SIZE: usize = 19; +pub(crate) const VERTICAL_SCROLLBAR: usize = 20; +pub(crate) const HORIZONTAL_SCROLLBAR: usize = 21; +pub(crate) const SUBROLE: usize = 22; +const READONLY_PROBE: usize = 23; +const ATTRIBUTE_COUNT: usize = 23; + +const ROLE_MASK: u32 = bit(ROLE) | bit(SUBROLE); +const STATE_MASK: u32 = + bit(ROLE) | bit(VALUE) | range_mask(4, 12) | bit(POSITION) | bit(SIZE) | bit(READONLY_PROBE); +const BOUNDS_MASK: u32 = bit(POSITION) | bit(SIZE); +const SCROLLBAR_MASK: u32 = bit(VERTICAL_SCROLLBAR) | bit(HORIZONTAL_SCROLLBAR); +const ALL_ATTRIBUTE_MASK: u32 = range_mask(0, ATTRIBUTE_COUNT - 1); + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct NodeAttributeStatus { + unknown_mask: u32, + pub(crate) cannot_complete: bool, + pub(crate) native_read_failures: u64, + pub(crate) invalid_element: bool, + pub(crate) api_disabled: bool, + pub(crate) text_truncations: u64, +} + +impl NodeAttributeStatus { + pub(crate) fn record_slot_error(&mut self, index: usize, error: i32) { + self.record_error(bit(index), error); + } + + pub(crate) fn record_batch_error(&mut self, error: i32) { + self.record_error(ALL_ATTRIBUTE_MASK, error); + } + + pub(crate) fn record_readonly_error(&mut self, error: i32) { + self.record_error(bit(READONLY_PROBE), error); + } + + pub(crate) fn record_truncated(&mut self, index: usize) { + self.unknown_mask |= bit(index); + self.text_truncations += 1; + } + + pub(crate) fn field_unknown(&self, index: usize) -> bool { + self.unknown_mask & bit(index) != 0 + } + + pub(crate) fn role_unknown(&self) -> bool { + self.unknown_mask & ROLE_MASK != 0 + } + + pub(crate) fn value_unknown(&self) -> bool { + self.field_unknown(VALUE) + } + + pub(crate) fn states_unknown(&self) -> bool { + self.unknown_mask & STATE_MASK != 0 + } + + pub(crate) fn bounds_unknown(&self) -> bool { + self.unknown_mask & BOUNDS_MASK != 0 + } + + pub(crate) fn scrollbars_unknown(&self) -> bool { + self.unknown_mask & SCROLLBAR_MASK != 0 + } + + fn record_error(&mut self, mask: u32, error: i32) { + if is_absent_error(error) { + return; + } + self.unknown_mask |= mask; + self.cannot_complete |= error == kAXErrorCannotComplete; + self.invalid_element |= error == kAXErrorInvalidUIElement; + self.api_disabled |= error == kAXErrorAPIDisabled; + self.native_read_failures += u64::from( + error != kAXErrorCannotComplete + && error != kAXErrorInvalidUIElement + && error != kAXErrorAPIDisabled, + ); + } +} + +pub(crate) const fn attribute_bit(index: usize) -> u32 { + bit(index) +} + +const fn bit(index: usize) -> u32 { + 1_u32 << index +} + +const fn range_mask(start: usize, end: usize) -> u32 { + ((1_u32 << (end + 1)) - 1) & !((1_u32 << start) - 1) +} + +fn is_absent_error(error: i32) -> bool { + error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn per_slot_errors_preserve_absent_unknown_and_terminal_states() { + let fixtures = [ + (TITLE, kAXErrorAttributeUnsupported), + (VALUE, kAXErrorCannotComplete), + (AX_IDENTIFIER, kAXErrorInvalidUIElement), + (AX_DOM_IDENTIFIER, kAXErrorAPIDisabled), + ]; + let mut status = NodeAttributeStatus::default(); + for (index, error) in fixtures { + status.record_slot_error(index, error); + } + + assert!(!status.field_unknown(TITLE)); + assert!(status.field_unknown(VALUE)); + assert!(status.field_unknown(AX_IDENTIFIER)); + assert!(status.field_unknown(AX_DOM_IDENTIFIER)); + assert!(status.cannot_complete); + assert!(status.invalid_element); + assert!(status.api_disabled); + assert_eq!(status.native_read_failures, 0); + } + + #[test] + fn unsupported_readonly_probe_does_not_make_state_unknown() { + let mut status = NodeAttributeStatus::default(); + status.record_readonly_error(kAXErrorAttributeUnsupported); + + assert!(!status.states_unknown()); + } + + #[test] + fn application_only_hidden_absence_differs_from_an_incomplete_hidden_read() { + let mut absent = NodeAttributeStatus::default(); + absent.record_slot_error(HIDDEN, kAXErrorAttributeUnsupported); + assert!(!absent.states_unknown()); + + let mut incomplete = NodeAttributeStatus::default(); + incomplete.record_slot_error(HIDDEN, kAXErrorCannotComplete); + assert!(incomplete.states_unknown()); + assert!(incomplete.cannot_complete); + } + + #[test] + fn truncated_text_is_unknown_instead_of_exact_evidence() { + let mut status = NodeAttributeStatus::default(); + + status.record_truncated(TITLE); + + assert!(status.field_unknown(TITLE)); + assert_eq!(status.text_truncations, 1); + } + + #[test] + fn decode_failure_is_counted_as_an_unclassified_native_read_failure() { + let mut status = NodeAttributeStatus::default(); + + status.record_slot_error(ROLE, accessibility_sys::kAXErrorFailure); + + assert_eq!(status.native_read_failures, 1); + assert!(status.role_unknown()); + } +} diff --git a/crates/macos/src/tree/node_attrs.rs b/crates/macos/src/tree/node_attrs.rs index 92f9440..118b4c9 100644 --- a/crates/macos/src/tree/node_attrs.rs +++ b/crates/macos/src/tree/node_attrs.rs @@ -1,35 +1,17 @@ -use agent_desktop_core::node::Rect; +use super::node_attr_states::NodeAttrStates; +use agent_desktop_core::{NameEvidence, Rect}; #[derive(Debug, Clone, Default)] pub(crate) struct NodeAttrs { pub(crate) role: Option<String>, - pub(crate) title: Option<String>, - pub(crate) description: Option<String>, + pub(crate) subrole: Option<String>, pub(crate) value: Option<String>, + pub(crate) name_evidence: NameEvidence, pub(crate) states: NodeAttrStates, pub(crate) bounds: Option<Rect>, pub(crate) has_scrollbars: bool, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct NodeAttrStates { - pub(crate) enabled: bool, - pub(crate) focused: Option<bool>, - pub(crate) expanded: Option<bool>, - pub(crate) disclosing: Option<bool>, -} - -impl Default for NodeAttrStates { - fn default() -> Self { - Self { - enabled: true, - focused: None, - expanded: None, - disclosing: None, - } - } -} - pub(crate) fn parse_enabled(enabled: Option<String>) -> bool { enabled.map(|s| s == "true").unwrap_or(true) } diff --git a/crates/macos/src/tree/node_control_states.rs b/crates/macos/src/tree/node_control_states.rs new file mode 100644 index 0000000..599737c --- /dev/null +++ b/crates/macos/src/tree/node_control_states.rs @@ -0,0 +1,8 @@ +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct NodeControlStates { + pub(crate) focused: Option<bool>, + pub(crate) expanded: Option<bool>, + pub(crate) disclosing: Option<bool>, + pub(crate) selected: Option<bool>, + pub(crate) readonly: Option<bool>, +} diff --git a/crates/macos/src/tree/node_identifiers.rs b/crates/macos/src/tree/node_identifiers.rs new file mode 100644 index 0000000..4a46fc4 --- /dev/null +++ b/crates/macos/src/tree/node_identifiers.rs @@ -0,0 +1,148 @@ +use agent_desktop_core::LocatorField; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct NodeIdentifiers { + pub(crate) ax_identifier: LocatorField<String>, + pub(crate) ax_dom_identifier: LocatorField<String>, +} + +impl NodeIdentifiers { + #[cfg(test)] + pub(crate) fn from_options( + ax_identifier: Option<String>, + ax_dom_identifier: Option<String>, + ) -> Self { + Self { + ax_identifier: normalize_ax(option_field(ax_identifier)), + ax_dom_identifier: normalize_dom(option_field(ax_dom_identifier)), + } + } + + pub(crate) fn from_fields( + ax_identifier: LocatorField<String>, + ax_dom_identifier: LocatorField<String>, + ) -> Self { + Self { + ax_identifier: normalize_ax(ax_identifier), + ax_dom_identifier: normalize_dom(ax_dom_identifier), + } + } +} + +impl Default for NodeIdentifiers { + fn default() -> Self { + Self { + ax_identifier: LocatorField::Absent, + ax_dom_identifier: LocatorField::Absent, + } + } +} + +#[cfg(all(test, target_os = "macos"))] +fn failed_field(error: i32) -> (LocatorField<String>, bool, bool, bool) { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, kAXErrorNoValue, + }; + + if error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue { + (LocatorField::Absent, false, false, false) + } else { + ( + LocatorField::Unknown, + error == kAXErrorCannotComplete, + error == kAXErrorInvalidUIElement, + error == kAXErrorAPIDisabled, + ) + } +} + +#[cfg(test)] +fn option_field(value: Option<String>) -> LocatorField<String> { + value + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent) +} + +fn normalize_ax(field: LocatorField<String>) -> LocatorField<String> { + normalize(field, crate::tree::native_id::meaningful_native_id) +} + +fn normalize_dom(field: LocatorField<String>) -> LocatorField<String> { + normalize(field, crate::tree::native_id::meaningful_dom_id) +} + +fn normalize( + field: LocatorField<String>, + normalize: impl FnOnce(Option<String>) -> Option<String>, +) -> LocatorField<String> { + match field { + LocatorField::Known(value) => normalize(Some(value)) + .map(LocatorField::Known) + .unwrap_or(LocatorField::Absent), + LocatorField::Absent => LocatorField::Absent, + LocatorField::Unknown => LocatorField::Unknown, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(target_os = "macos")] + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, + }; + + #[test] + fn dual_identifiers_remain_distinct() { + let identifiers = + NodeIdentifiers::from_options(Some("native-save".into()), Some("dom-save".into())); + + assert_eq!( + identifiers.ax_identifier, + LocatorField::Known("native-save".into()) + ); + assert_eq!( + identifiers.ax_dom_identifier, + LocatorField::Known("dom-save".into()) + ); + } + + #[test] + fn native_and_dom_identifiers_are_both_preserved() { + let identifiers = + NodeIdentifiers::from_options(Some("_NS:42".into()), Some("compose".into())); + + assert_eq!( + identifiers.ax_identifier, + LocatorField::Known("_NS:42".into()) + ); + assert_eq!( + identifiers.ax_dom_identifier, + LocatorField::Known("compose".into()) + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn identifier_read_failures_preserve_absent_unknown_and_invalid_states() { + assert_eq!( + failed_field(kAXErrorAttributeUnsupported), + (LocatorField::Absent, false, false, false) + ); + assert_eq!( + failed_field(kAXErrorCannotComplete), + (LocatorField::Unknown, true, false, false) + ); + assert_eq!( + failed_field(kAXErrorInvalidUIElement), + (LocatorField::Unknown, false, true, false) + ); + assert_eq!( + failed_field(kAXErrorAPIDisabled), + (LocatorField::Unknown, false, false, true) + ); + } +} diff --git a/crates/macos/src/tree/node_semantic_states.rs b/crates/macos/src/tree/node_semantic_states.rs new file mode 100644 index 0000000..5f35c37 --- /dev/null +++ b/crates/macos/src/tree/node_semantic_states.rs @@ -0,0 +1,7 @@ +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct NodeSemanticStates { + pub(crate) hidden: Option<bool>, + pub(crate) busy: Option<bool>, + pub(crate) modal: Option<bool>, + pub(crate) required: Option<bool>, +} diff --git a/crates/macos/src/tree/observation_usage.rs b/crates/macos/src/tree/observation_usage.rs new file mode 100644 index 0000000..ede1522 --- /dev/null +++ b/crates/macos/src/tree/observation_usage.rs @@ -0,0 +1,95 @@ +use agent_desktop_core::ObservationBudget; + +pub(crate) struct ObservationUsage { + limits: ObservationBudget, + nodes: usize, + edges: usize, + text_bytes: usize, +} + +impl ObservationUsage { + pub(crate) fn new(limits: ObservationBudget) -> Self { + Self { + limits, + nodes: 0, + edges: 0, + text_bytes: 0, + } + } + + pub(crate) fn claim_node(&mut self) -> bool { + if self.nodes >= self.limits.max_nodes { + return false; + } + self.nodes += 1; + true + } + + pub(crate) fn child_capacity(&self) -> usize { + self.limits + .max_children_per_node + .min(self.limits.max_edges.saturating_sub(self.edges)) + .min(self.limits.max_nodes.saturating_sub(self.nodes)) + } + + pub(crate) fn claim_edges(&mut self, count: usize) { + self.edges = self.edges.saturating_add(count).min(self.limits.max_edges); + } + + pub(crate) fn note_child_demand( + &self, + total_count: usize, + stats: &mut agent_desktop_core::LocatorStats, + ) { + if total_count > self.limits.max_children_per_node { + stats.traversal.limits.child_hits += 1; + } + if total_count > self.limits.max_edges.saturating_sub(self.edges) { + stats.traversal.limits.edge_hits += 1; + } + if total_count > self.limits.max_nodes.saturating_sub(self.nodes) { + stats.traversal.limits.node_hits += 1; + } + } + + pub(crate) fn string_capacity(&self) -> usize { + self.limits + .max_field_bytes + .min(self.limits.max_text_bytes.saturating_sub(self.text_bytes)) + } + + pub(crate) fn claim_text(&mut self, bytes: usize) { + self.text_bytes = self + .text_bytes + .saturating_add(bytes) + .min(self.limits.max_text_bytes); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits() -> ObservationBudget { + ObservationBudget { + max_nodes: 3, + max_edges: 2, + max_children_per_node: 4, + max_field_bytes: 5, + max_text_bytes: 7, + } + } + + #[test] + fn node_edge_and_text_capacities_share_one_observation_budget() { + let mut usage = ObservationUsage::new(limits()); + + assert!(usage.claim_node()); + assert_eq!(usage.child_capacity(), 2); + usage.claim_edges(2); + assert_eq!(usage.child_capacity(), 0); + assert_eq!(usage.string_capacity(), 5); + usage.claim_text(5); + assert_eq!(usage.string_capacity(), 2); + } +} diff --git a/crates/macos/src/tree/query/arena.rs b/crates/macos/src/tree/query/arena.rs new file mode 100644 index 0000000..5a8f4ba --- /dev/null +++ b/crates/macos/src/tree/query/arena.rs @@ -0,0 +1,51 @@ +use agent_desktop_core::{AdapterError, LocatorStats}; +use rustc_hash::FxHashSet; + +pub(crate) struct TraversalArena { + pub(crate) stats: LocatorStats, + pub(crate) ancestors: FxHashSet<usize>, + pub(crate) structurally_complete: bool, + owned_handles: u64, +} + +impl TraversalArena { + pub(crate) fn new() -> Self { + Self { + stats: LocatorStats::default(), + ancestors: FxHashSet::default(), + structurally_complete: true, + owned_handles: 0, + } + } + + pub(crate) fn add_handles(&mut self, count: usize) { + self.owned_handles = self + .owned_handles + .saturating_add(u64::try_from(count).unwrap_or(u64::MAX)); + self.stats.traversal.peak_handles_owned = self + .stats + .traversal + .peak_handles_owned + .max(self.owned_handles); + } + + pub(crate) fn drop_handles(&mut self, count: usize) { + self.owned_handles = self + .owned_handles + .saturating_sub(u64::try_from(count).unwrap_or(u64::MAX)); + } + + pub(crate) fn mark_incomplete(&mut self) { + self.structurally_complete = false; + } + + pub(crate) fn finish(self) -> Result<LocatorStats, AdapterError> { + if self.owned_handles != 0 { + return Err(AdapterError::internal(format!( + "observation retained {} native handles", + self.owned_handles + ))); + } + Ok(self.stats) + } +} diff --git a/crates/macos/src/tree/query/child_page.rs b/crates/macos/src/tree/query/child_page.rs new file mode 100644 index 0000000..2d27b4e --- /dev/null +++ b/crates/macos/src/tree/query/child_page.rs @@ -0,0 +1,31 @@ +pub(crate) struct PagedPrefix<T> { + pub(crate) elements: Vec<T>, + pub(crate) stalled: bool, +} + +pub(crate) fn read_paged_prefix<T>( + requested: usize, + page_size: usize, + mut read_page: impl FnMut(usize, usize) -> Result<Vec<T>, i32>, +) -> Result<PagedPrefix<T>, i32> { + let mut elements = Vec::with_capacity(requested); + while elements.len() < requested { + let page_len = (requested - elements.len()).min(page_size); + let page = read_page(elements.len(), page_len)?; + let returned = page.len(); + if returned > page_len { + return Err(i32::MIN); + } + elements.extend(page); + if returned == 0 { + return Ok(PagedPrefix { + elements, + stalled: true, + }); + } + } + Ok(PagedPrefix { + elements, + stalled: false, + }) +} diff --git a/crates/macos/src/tree/query/child_read.rs b/crates/macos/src/tree/query/child_read.rs new file mode 100644 index 0000000..70bd276 --- /dev/null +++ b/crates/macos/src/tree/query/child_read.rs @@ -0,0 +1,385 @@ +use crate::tree::AXElement; + +use super::{ + child_page::read_paged_prefix, child_read_status::ChildReadStatus, + child_source_availability::ChildSourceAvailability, +}; + +pub(crate) struct ChildRead { + pub(crate) elements: Vec<AXElement>, + pub(crate) total_count: usize, + pub(crate) complete: bool, + pub(crate) source_availability: ChildSourceAvailability, + pub(crate) prefix_certain: bool, + pub(crate) status: ChildReadStatus, +} + +impl ChildRead { + pub(crate) fn empty(complete: bool) -> Self { + Self { + elements: Vec::new(), + total_count: 0, + complete, + source_availability: if complete { + ChildSourceAvailability::Available + } else { + ChildSourceAvailability::Unknown + }, + prefix_certain: complete, + status: ChildReadStatus::default(), + } + } + + fn unavailable(status: ChildReadStatus) -> Self { + Self { + elements: Vec::new(), + total_count: 0, + complete: true, + source_availability: ChildSourceAvailability::Unavailable, + prefix_certain: true, + status, + } + } + + pub(crate) fn truncated(&self) -> bool { + self.elements.len() < self.total_count + } +} + +#[cfg(target_os = "macos")] +mod imp { + use super::super::child_read_telemetry as telemetry; + use super::*; + use crate::{cf_type::created_cf_array, tree::ax_value}; + use accessibility_sys::{kAXErrorAttributeUnsupported, kAXErrorNoValue, kAXErrorSuccess}; + use core_foundation::{base::TCFType, string::CFString}; + use core_foundation_sys::base::{CFIndex, CFTypeRef}; + + const CHILD_PAGE_SIZE: usize = 128; + + pub(crate) fn read_children( + element: &AXElement, + role: Option<&str>, + max_elements: usize, + deadline: std::time::Instant, + ) -> ChildRead { + super::super::child_source::read_first_nonempty( + crate::tree::element::child_attributes(role), + |attribute| read_attribute_children(element, attribute, max_elements, deadline), + ) + } + + pub(crate) fn read_child_at( + element: &AXElement, + role: Option<&str>, + index: usize, + deadline: std::time::Instant, + ) -> ChildRead { + super::super::child_source::read_first_nonempty( + crate::tree::element::child_attributes(role), + |attribute| read_attribute_child_at(element, attribute, index, deadline), + ) + } + + pub(crate) fn read_attribute_children( + element: &AXElement, + attribute: &str, + max_elements: usize, + deadline: std::time::Instant, + ) -> ChildRead { + let mut status = ChildReadStatus::default(); + if prepare(element, deadline, &mut status).is_err() { + return ChildRead { + elements: Vec::new(), + total_count: 0, + complete: false, + source_availability: ChildSourceAvailability::Unknown, + prefix_certain: false, + status, + }; + } + status.attempts += 1; + let count = match child_count(element, attribute, deadline) { + Ok(count) => count, + Err(error) if is_absent_error(error) => { + return ChildRead::unavailable(status); + } + Err(error) => { + telemetry::record(&mut status, attribute, "initial_count", error, None); + return ChildRead { + elements: Vec::new(), + total_count: 0, + complete: false, + source_availability: ChildSourceAvailability::Unknown, + prefix_certain: false, + status, + }; + } + }; + let requested = count.min(max_elements); + let mut complete = true; + let mut elements = match read_prefix(element, attribute, requested, deadline, &mut status) { + Ok(elements) => elements, + Err(error) => { + telemetry::record(&mut status, attribute, "prefix", error, Some(count)); + complete = false; + Vec::new() + } + }; + complete &= elements.len() == requested; + let final_count = match stable_count(element, attribute, deadline, &mut status) { + Ok(final_count) => final_count, + Err(error) => { + telemetry::record(&mut status, attribute, "stable_count", error, Some(count)); + complete = false; + count + } + }; + if count_changed(count, final_count) { + status.count_changed = true; + complete = false; + elements.truncate(final_count); + } + ChildRead { + elements, + total_count: final_count, + complete, + source_availability: ChildSourceAvailability::Available, + prefix_certain: complete, + status, + } + } + + fn read_attribute_child_at( + element: &AXElement, + attribute: &str, + index: usize, + deadline: std::time::Instant, + ) -> ChildRead { + let mut status = ChildReadStatus::default(); + if prepare(element, deadline, &mut status).is_err() { + return ChildRead { + elements: Vec::new(), + total_count: 0, + complete: false, + source_availability: ChildSourceAvailability::Unknown, + prefix_certain: false, + status, + }; + } + status.attempts += 1; + let initial_count = match child_count(element, attribute, deadline) { + Ok(count) => count, + Err(error) if is_absent_error(error) => return ChildRead::unavailable(status), + Err(error) => { + telemetry::record(&mut status, attribute, "initial_count", error, None); + return ChildRead { + elements: Vec::new(), + total_count: 0, + complete: false, + source_availability: ChildSourceAvailability::Unknown, + prefix_certain: false, + status, + }; + } + }; + let mut elements = if index < initial_count { + status.attempts += 1; + match copy_page(element, attribute, index, 1, deadline) { + Ok(elements) => elements, + Err(error) => { + telemetry::record( + &mut status, + attribute, + "indexed_child", + error, + Some(initial_count), + ); + Vec::new() + } + } + } else { + Vec::new() + }; + if index < initial_count && elements.is_empty() { + status.cursor_stalled = true; + } + let final_count = match stable_count(element, attribute, deadline, &mut status) { + Ok(count) => count, + Err(error) => { + telemetry::record( + &mut status, + attribute, + "stable_count", + error, + Some(initial_count), + ); + initial_count + } + }; + if count_changed(initial_count, final_count) { + status.count_changed = true; + } + let expected = usize::from(index < initial_count); + let complete = status.health.deadline_exhausted == 0 + && status.health.cannot_complete == 0 + && !status.invalid_element + && !status.api_disabled + && !status.count_changed + && elements.len() == expected; + elements.truncate(expected); + ChildRead { + elements, + total_count: final_count, + complete, + source_availability: ChildSourceAvailability::Available, + prefix_certain: complete, + status, + } + } + + fn child_count( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + ) -> Result<usize, i32> { + let attribute = CFString::new(attribute); + crate::tree::ax_ipc::attribute_value_count( + element, + attribute.as_concrete_TypeRef(), + deadline, + ) + } + + fn read_prefix( + element: &AXElement, + attribute: &str, + requested: usize, + deadline: std::time::Instant, + status: &mut ChildReadStatus, + ) -> Result<Vec<AXElement>, i32> { + let result = read_paged_prefix(requested, CHILD_PAGE_SIZE, |index, page_len| { + prepare(element, deadline, status) + .map_err(|_| accessibility_sys::kAXErrorCannotComplete)?; + status.attempts += 1; + copy_page(element, attribute, index, page_len, deadline) + })?; + status.cursor_stalled |= result.stalled; + Ok(result.elements) + } + + fn stable_count( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + status: &mut ChildReadStatus, + ) -> Result<usize, i32> { + prepare(element, deadline, status) + .map_err(|_| accessibility_sys::kAXErrorCannotComplete)?; + status.attempts += 1; + child_count(element, attribute, deadline) + } + + fn copy_page( + element: &AXElement, + attribute: &str, + index: usize, + max_values: usize, + deadline: std::time::Instant, + ) -> Result<Vec<AXElement>, i32> { + let attribute = CFString::new(attribute); + let index = CFIndex::try_from(index).map_err(|_| i32::MIN)?; + let max_values = CFIndex::try_from(max_values).map_err(|_| i32::MIN)?; + let (error, result) = crate::tree::ax_ipc::copy_attribute_values( + element, + attribute.as_concrete_TypeRef(), + index, + max_values, + deadline, + ); + if error != kAXErrorSuccess { + if !result.is_null() { + drop(created_cf_array(result as CFTypeRef)); + } + return Err(error); + } + if result.is_null() { + return Ok(Vec::new()); + } + let Some(array) = created_cf_array(result as CFTypeRef) else { + return Err(i32::MIN); + }; + let expected = array.len() as usize; + let elements = array + .into_iter() + .filter_map(|value| ax_value::retained_ax_element(&value)) + .collect::<Vec<_>>(); + if elements.len() != expected { + return Err(i32::MIN); + } + Ok(elements) + } + + fn prepare( + element: &AXElement, + deadline: std::time::Instant, + status: &mut ChildReadStatus, + ) -> Result<(), ()> { + crate::tree::locator_deadline::prepare(element, deadline) + .map(|_| ()) + .map_err(|_| { + status.health.deadline_exhausted = 1; + }) + } + + #[cfg(test)] + pub(super) fn record_error(status: &mut ChildReadStatus, error: i32) { + telemetry::record_status(status, error); + } + + pub(super) fn is_absent_error(error: i32) -> bool { + error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue + } + + pub(super) fn count_changed(initial: usize, final_count: usize) -> bool { + initial != final_count + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use super::*; + + pub(crate) fn read_children( + _element: &AXElement, + _role: Option<&str>, + _max_elements: usize, + _deadline: std::time::Instant, + ) -> ChildRead { + ChildRead::empty(true) + } + + pub(crate) fn read_attribute_children( + _element: &AXElement, + _attribute: &str, + _max_elements: usize, + _deadline: std::time::Instant, + ) -> ChildRead { + ChildRead::empty(true) + } + + pub(crate) fn read_child_at( + _element: &AXElement, + _role: Option<&str>, + _index: usize, + _deadline: std::time::Instant, + ) -> ChildRead { + ChildRead::empty(true) + } +} + +pub(crate) use imp::{read_attribute_children, read_child_at, read_children}; + +#[cfg(all(test, target_os = "macos"))] +#[path = "child_read_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/query/child_read_plan.rs b/crates/macos/src/tree/query/child_read_plan.rs new file mode 100644 index 0000000..261be37 --- /dev/null +++ b/crates/macos/src/tree/query/child_read_plan.rs @@ -0,0 +1,63 @@ +#[derive(Clone, Copy)] +pub(crate) struct ChildReadPlan { + max_elements: usize, + boundary_elements: usize, + logical_depth: Option<u8>, + max_logical_depth: u8, +} + +impl ChildReadPlan { + pub(crate) fn load(max_elements: usize) -> Self { + Self { + max_elements, + boundary_elements: max_elements, + logical_depth: None, + max_logical_depth: u8::MAX, + } + } + + pub(crate) fn boundary_aware( + max_elements: usize, + boundary_elements: usize, + logical_depth: u8, + max_logical_depth: u8, + ) -> Self { + Self { + max_elements, + boundary_elements, + logical_depth: Some(logical_depth), + max_logical_depth, + } + } + + pub(crate) fn max_elements(self, transparent_wrapper: bool) -> usize { + let beyond_boundary = self.logical_depth.is_some_and(|depth| { + depth.saturating_add(u8::from(!transparent_wrapper)) > self.max_logical_depth + }); + if beyond_boundary { + self.boundary_elements + } else { + self.max_elements + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn boundary_nodes_request_only_the_native_child_count() { + let plan = ChildReadPlan::boundary_aware(128, 0, 3, 3); + + assert_eq!(plan.max_elements(false), 0); + assert_eq!(plan.max_elements(true), 128); + } + + #[test] + fn selected_root_boundary_can_load_only_bounded_label_children() { + let plan = ChildReadPlan::boundary_aware(128, 5, 0, 0); + + assert_eq!(plan.max_elements(false), 5); + } +} diff --git a/crates/macos/src/tree/query/child_read_status.rs b/crates/macos/src/tree/query/child_read_status.rs new file mode 100644 index 0000000..9be2771 --- /dev/null +++ b/crates/macos/src/tree/query/child_read_status.rs @@ -0,0 +1,25 @@ +#[derive(Default)] +pub(crate) struct ChildReadStatus { + pub(crate) attempts: u64, + pub(crate) health: agent_desktop_core::LocatorReadHealth, + pub(crate) invalid_element: bool, + pub(crate) api_disabled: bool, + pub(crate) count_changed: bool, + pub(crate) cursor_stalled: bool, +} + +impl ChildReadStatus { + pub(crate) fn merge(&mut self, other: Self) { + self.attempts += other.attempts; + self.health.cannot_complete += other.health.cannot_complete; + self.health.native_read_failures += other.health.native_read_failures; + self.health.deadline_exhausted = self + .health + .deadline_exhausted + .max(other.health.deadline_exhausted); + self.invalid_element |= other.invalid_element; + self.api_disabled |= other.api_disabled; + self.count_changed |= other.count_changed; + self.cursor_stalled |= other.cursor_stalled; + } +} diff --git a/crates/macos/src/tree/query/child_read_telemetry.rs b/crates/macos/src/tree/query/child_read_telemetry.rs new file mode 100644 index 0000000..3caaa94 --- /dev/null +++ b/crates/macos/src/tree/query/child_read_telemetry.rs @@ -0,0 +1,41 @@ +use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, kAXErrorNoValue, kAXErrorSuccess, +}; + +use super::child_read_status::ChildReadStatus; + +pub(crate) fn record_status(status: &mut ChildReadStatus, error: i32) { + status.health.cannot_complete += u64::from(error == kAXErrorCannotComplete); + status.invalid_element |= error == kAXErrorInvalidUIElement; + status.api_disabled |= error == kAXErrorAPIDisabled; + status.health.native_read_failures += u64::from( + error != kAXErrorSuccess + && error != kAXErrorAttributeUnsupported + && error != kAXErrorNoValue + && error != kAXErrorCannotComplete + && error != kAXErrorInvalidUIElement + && error != kAXErrorAPIDisabled, + ); +} + +pub(crate) fn record( + status: &mut ChildReadStatus, + attribute: &str, + phase: &'static str, + error: i32, + child_count: Option<usize>, +) { + record_status(status, error); + tracing::debug!( + attribute, + phase, + ax_error_code = error, + child_count, + cannot_complete_count = status.health.cannot_complete, + native_read_failure_count = status.health.native_read_failures, + invalid_element = status.invalid_element, + api_disabled = status.api_disabled, + "AX child read failed" + ); +} diff --git a/crates/macos/src/tree/query/child_read_tests.rs b/crates/macos/src/tree/query/child_read_tests.rs new file mode 100644 index 0000000..b544ecd --- /dev/null +++ b/crates/macos/src/tree/query/child_read_tests.rs @@ -0,0 +1,144 @@ +use super::imp::{count_changed, is_absent_error, record_error}; +use super::*; +use accessibility_sys::{ + kAXErrorAttributeUnsupported, kAXErrorCannotComplete, kAXErrorFailure, kAXErrorNoValue, +}; + +#[test] +fn unsupported_and_missing_child_attributes_are_authoritatively_absent() { + assert!(is_absent_error(kAXErrorAttributeUnsupported)); + assert!(is_absent_error(kAXErrorNoValue)); + assert!(!is_absent_error(kAXErrorCannotComplete)); +} + +#[test] +fn bounded_child_read_distinguishes_truncation_from_native_failure() { + let read = ChildRead { + elements: Vec::new(), + total_count: 4, + complete: true, + source_availability: ChildSourceAvailability::Available, + prefix_certain: true, + status: ChildReadStatus::default(), + }; + + assert!(read.truncated()); + assert!(read.complete); +} + +#[test] +fn child_count_change_is_not_a_complete_observation() { + assert!(count_changed(4, 3)); + assert!(count_changed(3, 4)); + assert!(!count_changed(4, 4)); +} + +#[test] +fn cursor_stall_is_preserved_when_statuses_merge() { + let mut status = ChildReadStatus::default(); + status.merge(ChildReadStatus { + cursor_stalled: true, + health: agent_desktop_core::LocatorReadHealth { + native_read_failures: 2, + ..Default::default() + }, + ..ChildReadStatus::default() + }); + + assert!(status.cursor_stalled); + assert_eq!(status.health.native_read_failures, 2); +} + +#[test] +fn indexed_child_read_is_not_rejected_for_unread_siblings() { + let read = ChildRead { + elements: vec![AXElement(std::ptr::null_mut())], + total_count: 10_000, + complete: true, + source_availability: ChildSourceAvailability::Available, + prefix_certain: true, + status: ChildReadStatus::default(), + }; + + assert!(read.complete); + assert!(read.truncated()); +} + +#[test] +fn positive_short_pages_continue_from_the_advanced_offset() { + let mut calls = Vec::new(); + let result = read_paged_prefix(4, 128, |offset, maximum| { + calls.push((offset, maximum)); + Ok(if offset == 0 { vec![0, 1] } else { vec![2, 3] }) + }) + .unwrap(); + + assert_eq!(calls, [(0, 4), (2, 2)]); + assert_eq!(result.elements, [0, 1, 2, 3]); + assert!(!result.stalled); +} + +#[test] +fn zero_progress_stalls_and_terminates_before_the_requested_count() { + let mut calls = Vec::new(); + let result = read_paged_prefix(4, 128, |offset, maximum| { + calls.push((offset, maximum)); + Ok(if offset == 0 { vec![0, 1] } else { Vec::new() }) + }) + .unwrap(); + + assert_eq!(calls, [(0, 4), (2, 2)]); + assert_eq!(result.elements, [0, 1]); + assert!(result.stalled); +} + +#[test] +fn multi_page_reads_preserve_order_beyond_the_native_page_size() { + let mut calls = Vec::new(); + let result = read_paged_prefix(260, 128, |offset, maximum| { + calls.push((offset, maximum)); + Ok((offset..offset + maximum).collect::<Vec<_>>()) + }) + .unwrap(); + + assert_eq!(calls, [(0, 128), (128, 128), (256, 4)]); + assert_eq!(result.elements, (0..260).collect::<Vec<_>>()); + assert!(!result.stalled); +} + +#[test] +fn page_error_after_progress_is_not_collapsed_to_a_short_complete_read() { + let error = read_paged_prefix(4, 128, |offset, _| { + if offset == 0 { + Ok(vec![0, 1]) + } else { + Err(kAXErrorCannotComplete) + } + }) + .err() + .expect("a later native error must remain visible"); + + assert_eq!(error, kAXErrorCannotComplete); +} + +#[test] +fn unclassified_native_error_is_counted_explicitly() { + let mut status = ChildReadStatus::default(); + + record_error(&mut status, i32::MIN); + + assert_eq!(status.health.native_read_failures, 1); + assert_eq!(status.health.cannot_complete, 0); + assert!(!status.invalid_element); + assert!(!status.api_disabled); +} + +#[test] +fn ax_failure_remains_a_native_read_failure() { + let mut status = ChildReadStatus::default(); + + record_error(&mut status, kAXErrorFailure); + + assert_eq!(status.health.native_read_failures, 1); + assert_eq!(status.health.cannot_complete, 0); +} diff --git a/crates/macos/src/tree/query/child_source.rs b/crates/macos/src/tree/query/child_source.rs new file mode 100644 index 0000000..15819fe --- /dev/null +++ b/crates/macos/src/tree/query/child_source.rs @@ -0,0 +1,127 @@ +use super::{ + child_read::ChildRead, child_read_status::ChildReadStatus, + child_source_availability::ChildSourceAvailability, +}; + +const CANONICAL_CHILDREN: &str = "AXChildren"; + +pub(super) fn read_first_nonempty( + attributes: &[&str], + mut read_attribute: impl FnMut(&str) -> ChildRead, +) -> ChildRead { + let mut status = ChildReadStatus::default(); + let mut prefix_certain = true; + let mut any_available = false; + for attribute in attributes { + let mut read = read_attribute(attribute); + let terminal = read.status.invalid_element || read.status.api_disabled; + any_available |= read.source_availability == ChildSourceAvailability::Available; + status.merge(read.status); + let source_selected = read.source_availability == ChildSourceAvailability::Available + && (read.total_count > 0 || *attribute == CANONICAL_CHILDREN); + if source_selected { + read.prefix_certain &= prefix_certain; + read.complete &= read.prefix_certain; + read.status = status; + return read; + } + prefix_certain &= read.prefix_certain; + if terminal { + break; + } + } + ChildRead { + elements: Vec::new(), + total_count: 0, + complete: prefix_certain, + source_availability: if any_available { + ChildSourceAvailability::Available + } else if prefix_certain { + ChildSourceAvailability::Unavailable + } else { + ChildSourceAvailability::Unknown + }, + prefix_certain, + status, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::AXElement; + + fn read(count: usize, complete: bool, availability: ChildSourceAvailability) -> ChildRead { + ChildRead { + elements: (count > 0) + .then(|| AXElement(std::ptr::null_mut())) + .into_iter() + .collect(), + total_count: count, + complete, + source_availability: availability, + prefix_certain: complete, + status: ChildReadStatus::default(), + } + } + + #[test] + fn successful_empty_children_stops_fallbacks_as_complete() { + let mut attributes = Vec::new(); + let selected = read_first_nonempty(&["AXChildren", "AXContents"], |attribute| { + attributes.push(attribute.to_string()); + read(0, true, ChildSourceAvailability::Available) + }); + + assert_eq!(attributes, ["AXChildren"]); + assert!(selected.elements.is_empty()); + assert!(selected.complete); + } + + #[test] + fn unsupported_children_reaches_contents() { + let mut attributes = Vec::new(); + let selected = read_first_nonempty(&["AXChildren", "AXContents"], |attribute| { + attributes.push(attribute.to_string()); + if attribute == "AXChildren" { + read(0, true, ChildSourceAvailability::Unavailable) + } else { + read(1, true, ChildSourceAvailability::Available) + } + }); + + assert_eq!(attributes, ["AXChildren", "AXContents"]); + assert_eq!(selected.elements.len(), 1); + assert!(selected.complete); + } + + #[test] + fn failed_children_poisons_nonempty_fallback() { + let selected = read_first_nonempty(&["AXChildren", "AXContents"], |attribute| { + if attribute == "AXChildren" { + let mut failed = read(0, false, ChildSourceAvailability::Unknown); + failed.status.health.native_read_failures = 1; + failed + } else { + read(1, true, ChildSourceAvailability::Available) + } + }); + + assert_eq!(selected.elements.len(), 1); + assert!(!selected.complete); + assert_eq!(selected.status.health.native_read_failures, 1); + } + + #[test] + fn partial_nonempty_primary_stops_fallback_and_remains_incomplete() { + let mut attributes = Vec::new(); + let selected = read_first_nonempty(&["AXChildren", "AXContents"], |attribute| { + attributes.push(attribute.to_string()); + read(1, false, ChildSourceAvailability::Available) + }); + + assert_eq!(attributes, ["AXChildren"]); + assert_eq!(selected.elements.len(), 1); + assert!(!selected.complete); + } +} diff --git a/crates/macos/src/tree/query/child_source_availability.rs b/crates/macos/src/tree/query/child_source_availability.rs new file mode 100644 index 0000000..18e8fbf --- /dev/null +++ b/crates/macos/src/tree/query/child_source_availability.rs @@ -0,0 +1,6 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ChildSourceAvailability { + Available, + Unavailable, + Unknown, +} diff --git a/crates/macos/src/tree/query/evidence_fields.rs b/crates/macos/src/tree/query/evidence_fields.rs new file mode 100644 index 0000000..6e6fa32 --- /dev/null +++ b/crates/macos/src/tree/query/evidence_fields.rs @@ -0,0 +1,232 @@ +use agent_desktop_core::{LocatorField, NameEvidence}; + +use crate::tree::node_attribute_status::{ + DESCRIPTION, LABEL, NodeAttributeStatus, PLACEHOLDER, TITLE, TITLE_ELEMENT, VALUE, +}; + +pub(crate) fn name_field( + evidence: &NameEvidence, + status: &NodeAttributeStatus, + role: Option<&str>, + children_complete: bool, +) -> LocatorField<String> { + let mut uncertain = false; + for (candidate, unknown) in name_sources(evidence, status, role, children_complete) { + uncertain |= unknown; + if let Some(value) = meaningful(candidate) { + return if uncertain { + LocatorField::Unknown + } else { + LocatorField::Known(value.to_string()) + }; + } + } + if uncertain { + LocatorField::Unknown + } else { + LocatorField::Absent + } +} + +pub(crate) fn description_field( + evidence: &NameEvidence, + status: &NodeAttributeStatus, + role: Option<&str>, + _children_complete: bool, +) -> LocatorField<String> { + let sources = stronger_name_sources(evidence, status, role); + let pre_name_known = sources + .iter() + .any(|(candidate, _)| meaningful(*candidate).is_some()); + let pre_name_unknown = sources.iter().any(|(_, unknown)| *unknown); + let description = meaningful(evidence.description.as_deref()); + let description_unknown = status.field_unknown(DESCRIPTION); + match (description, pre_name_known, pre_name_unknown) { + (Some(_), true, _) if description_unknown => LocatorField::Unknown, + (Some(value), true, _) => LocatorField::Known(value.to_string()), + (Some(_), false, true) => LocatorField::Unknown, + (None, true, _) if description_unknown => LocatorField::Unknown, + (None, false, true) if description_unknown => LocatorField::Unknown, + _ => LocatorField::Absent, + } +} + +fn name_sources<'a>( + evidence: &'a NameEvidence, + status: &NodeAttributeStatus, + role: Option<&str>, + children_complete: bool, +) -> [(Option<&'a str>, bool); 7] { + let stronger = stronger_name_sources(evidence, status, role); + [ + stronger[0], + stronger[1], + stronger[2], + stronger[3], + ( + evidence.description.as_deref(), + status.field_unknown(DESCRIPTION), + ), + (evidence.child_label.as_deref(), !children_complete), + ( + evidence.placeholder.as_deref(), + status.field_unknown(PLACEHOLDER), + ), + ] +} + +fn stronger_name_sources<'a>( + evidence: &'a NameEvidence, + status: &NodeAttributeStatus, + role: Option<&str>, +) -> [(Option<&'a str>, bool); 4] { + [ + ( + evidence.explicit_label.as_deref(), + status.field_unknown(LABEL), + ), + ( + evidence.labelled_by_text.as_deref(), + status.field_unknown(TITLE_ELEMENT), + ), + ( + evidence.native_title.as_deref(), + status.field_unknown(TITLE), + ), + ( + (role == Some("AXStaticText")) + .then_some(evidence.static_value.as_deref()) + .flatten(), + role == Some("AXStaticText") && status.field_unknown(VALUE), + ), + ] +} + +fn meaningful(value: Option<&str>) -> Option<&str> { + value.filter(|value| !value.trim().is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transient_higher_priority_name_source_keeps_fallback_unknown() { + let evidence = NameEvidence { + native_title: Some("Save".into()), + ..NameEvidence::default() + }; + let mut status = NodeAttributeStatus::default(); + status.record_slot_error(LABEL, accessibility_sys::kAXErrorCannotComplete); + + assert_eq!( + name_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Unknown + ); + } + + #[test] + fn known_higher_priority_name_ignores_lower_priority_unknowns() { + let evidence = NameEvidence { + explicit_label: Some("Save".into()), + ..NameEvidence::default() + }; + let mut status = NodeAttributeStatus::default(); + status.record_slot_error(TITLE, accessibility_sys::kAXErrorCannotComplete); + + assert_eq!( + name_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Known("Save".into()) + ); + } + + #[test] + fn description_remains_unknown_when_a_name_source_is_transient() { + let evidence = NameEvidence { + description: Some("Saves the draft".into()), + ..NameEvidence::default() + }; + let mut status = NodeAttributeStatus::default(); + status.record_slot_error(TITLE, accessibility_sys::kAXErrorCannotComplete); + + assert_eq!( + name_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Unknown + ); + assert_eq!( + description_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Unknown + ); + } + + #[test] + fn description_only_evidence_is_the_name_not_a_duplicate_description() { + let evidence = NameEvidence { + description: Some("scroll-area".into()), + ..NameEvidence::default() + }; + let status = NodeAttributeStatus::default(); + + assert_eq!( + name_field(&evidence, &status, Some("AXScrollArea"), true), + LocatorField::Known("scroll-area".into()) + ); + assert_eq!( + description_field(&evidence, &status, Some("AXScrollArea"), true), + LocatorField::Absent + ); + } + + #[test] + fn description_stays_separate_when_a_stronger_name_exists() { + let evidence = NameEvidence { + native_title: Some("Save".into()), + description: Some("Saves the draft".into()), + ..NameEvidence::default() + }; + let status = NodeAttributeStatus::default(); + + assert_eq!( + name_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Known("Save".into()) + ); + assert_eq!( + description_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Known("Saves the draft".into()) + ); + } + + #[test] + fn explicit_label_also_preserves_a_separate_description() { + let evidence = NameEvidence { + explicit_label: Some("Save".into()), + description: Some("Saves the draft".into()), + ..NameEvidence::default() + }; + let status = NodeAttributeStatus::default(); + + assert_eq!( + name_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Known("Save".into()) + ); + assert_eq!( + description_field(&evidence, &status, Some("AXButton"), true), + LocatorField::Known("Saves the draft".into()) + ); + } + + #[test] + fn unknown_description_cannot_be_bypassed_by_child_content() { + let evidence = NameEvidence { + child_label: Some("fallback child".into()), + ..NameEvidence::default() + }; + let mut status = NodeAttributeStatus::default(); + status.record_slot_error(DESCRIPTION, accessibility_sys::kAXErrorCannotComplete); + + assert_eq!( + name_field(&evidence, &status, Some("AXGroup"), true), + LocatorField::Unknown + ); + } +} diff --git a/crates/macos/src/tree/query/mod.rs b/crates/macos/src/tree/query/mod.rs new file mode 100644 index 0000000..2de8260 --- /dev/null +++ b/crates/macos/src/tree/query/mod.rs @@ -0,0 +1,241 @@ +mod arena; +mod child_page; +pub(crate) mod child_read; +pub(crate) mod child_read_plan; +mod child_read_status; +mod child_read_telemetry; +mod child_source; +mod child_source_availability; +mod evidence_fields; +mod node_evidence; +pub(crate) mod node_read; +pub(crate) mod node_read_context; +pub(crate) mod read_error; +mod traversal; + +use crate::tree::AXElement; +use agent_desktop_core::{ + AdapterError, ErrorCode, ObservationRequest, ObservationRoot, ObservationSource, ObservedTree, + SnapshotSurface, +}; +use serde_json::json; + +struct ResolvedRoot { + element: AXElement, + source: ObservationSource, + context: crate::tree::TreeBuildContext, + pid: i32, + process_instance: Option<String>, + activation_eligible: bool, +} + +pub(crate) fn observe_tree( + root: ObservationRoot<'_>, + request: &ObservationRequest, +) -> Result<ObservedTree, AdapterError> { + let request = (*request).validate()?; + let deadline = crate::tree::locator_deadline::from_operation(request.deadline)?; + let resolved = resolve_root(root, &request, deadline)?; + let (tree, renderer_ready, stats) = + traversal::LocatorTraversal::new(&request, resolved.context, deadline) + .build(resolved.element, resolved.source)?; + if resolved.activation_eligible && tree.is_complete() && !renderer_ready { + if let Some(instance) = resolved.process_instance.as_deref() { + if crate::tree::renderer_probe::activation_supported(resolved.pid, instance, deadline)? + { + return Err(renderer_activation_required(resolved.pid, stats)); + } + } + } + Ok(tree) +} + +fn renderer_activation_required(pid: i32, stats: agent_desktop_core::LocatorStats) -> AdapterError { + let mut error = AdapterError::renderer_accessibility_activation_required( + "The application exposes renderer activation but no web accessibility surface", + ); + if let Some(details) = error + .details + .as_mut() + .and_then(serde_json::Value::as_object_mut) + { + details.insert("pid".into(), json!(pid)); + details.insert("complete".into(), json!(true)); + details.insert("renderer_ready".into(), json!(false)); + details.insert("query_stats".into(), json!(stats)); + } + error +} + +fn resolve_root( + root: ObservationRoot<'_>, + request: &ObservationRequest, + deadline: std::time::Instant, +) -> Result<ResolvedRoot, AdapterError> { + let source = ObservationSource::from_root(&root); + match root { + ObservationRoot::Window(window) => { + let pid = crate::system::process_identity::to_pid_t(window.pid)?; + let element = resolve_window_surface(window, request.surface, deadline)?; + let context = + crate::tree::TreeBuildContext::for_pid_with_deadline(pid, true, deadline)? + .child_context(window.bounds); + Ok(ResolvedRoot { + element, + source, + context, + pid, + process_instance: window.process_instance.clone(), + activation_eligible: request.surface == SnapshotSurface::Window, + }) + } + ObservationRoot::Element { + handle, + entry, + root_ref, + } => { + verify_entry_process(entry)?; + let pid = crate::system::process_identity::to_pid_t(entry.process.pid)?; + let element = handle.downcast_ref::<AXElement>().cloned().ok_or_else(|| { + AdapterError::new( + ErrorCode::StaleRef, + "Live locator root handle is null or no longer valid", + ) + .with_suggestion("Refresh the source snapshot and retry the locator") + .with_details(json!({ + "kind": "locator_root_invalid", + "root_ref": root_ref, + })) + })?; + let context = + crate::tree::TreeBuildContext::for_pid_with_deadline(pid, true, deadline)? + .child_context(entry.geometry.bounds); + Ok(ResolvedRoot { + element, + source, + context, + pid, + process_instance: entry.process.process_instance.clone(), + activation_eligible: false, + }) + } + } +} + +fn verify_entry_process(entry: &agent_desktop_core::RefEntry) -> Result<(), AdapterError> { + let instance = entry.process.process_instance.as_deref().ok_or_else(|| { + AdapterError::stale_ref("Live locator root has no process instance identity") + })?; + let pid = crate::system::process_identity::to_pid_t(entry.process.pid)?; + match crate::system::process_identity::matches_instance(pid, instance) { + Ok(true) => Ok(()), + Ok(false) => Err(AdapterError::stale_ref( + "Live locator root process instance is no longer running", + )), + Err(error) if error.code == ErrorCode::InvalidArgs => Err(AdapterError::stale_ref( + "Live locator root has a malformed process instance identity", + )), + Err(error) => Err(error), + } +} + +fn resolve_window_surface( + window: &agent_desktop_core::WindowInfo, + surface: SnapshotSurface, + deadline: std::time::Instant, +) -> Result<AXElement, AdapterError> { + crate::tree::locator_deadline::remaining(deadline)?; + let pid = crate::system::process_identity::to_pid_t(window.pid)?; + let element = match surface { + SnapshotSurface::Window => { + crate::system::window_resolve::window_element_for_info_with_deadline(window, deadline)? + } + SnapshotSurface::Focused => crate::tree::surfaces::focused_surface_for_pid(pid, deadline)? + .ok_or_else(|| AdapterError::element_not_found("No focused surface found"))?, + SnapshotSurface::Menu => crate::tree::surfaces::menu_element_for_pid(pid, deadline)? + .ok_or_else(|| AdapterError::element_not_found("No open context menu"))?, + SnapshotSurface::Menubar => crate::tree::surfaces::menubar_for_pid(pid, deadline)? + .ok_or_else(|| AdapterError::element_not_found("No menu bar found"))?, + SnapshotSurface::Sheet => crate::tree::surfaces::sheet_for_pid(pid, deadline)? + .ok_or_else(|| AdapterError::element_not_found("No open sheet"))?, + SnapshotSurface::Popover => crate::tree::surfaces::popover_for_pid(pid, deadline)? + .ok_or_else(|| AdapterError::element_not_found("No visible popover"))?, + SnapshotSurface::Alert => crate::tree::surfaces::alert_for_pid(pid, deadline)? + .ok_or_else(|| AdapterError::element_not_found("No open alert or dialog"))?, + _ => return Err(AdapterError::not_supported("snapshot surface")), + }; + crate::tree::locator_deadline::remaining(deadline)?; + Ok(element) +} + +#[cfg(test)] +mod tests { + use super::*; + use agent_desktop_core::{ + NativeHandle, ObservationRoot, RefCapabilities, RefEntry, RefEntryIdentity, RefGeometry, + RefProcess, RefScope, RefSource, + }; + + #[test] + fn null_element_root_is_a_structured_stale_ref() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let process_instance = crate::system::process_identity::token_for_pid(pid) + .expect("current process identity read") + .expect("current process identity"); + let entry = RefEntry { + process: RefProcess { + pid: agent_desktop_core::ProcessId::try_from(pid).expect("test pid is positive"), + process_instance: Some(process_instance), + }, + identity: RefEntryIdentity { + role: "button".into(), + name: Some("Save".into()), + value: None, + description: None, + native_id: None, + }, + geometry: RefGeometry { + bounds: None, + bounds_hash: None, + }, + capabilities: RefCapabilities { + states: Vec::new(), + available_actions: Vec::new(), + }, + source: RefSource { + source_app: Some("Fixture".into()), + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: Default::default(), + }, + scope: RefScope { + root_ref: None, + path_is_absolute: false, + path: Default::default(), + }, + }; + let handle = NativeHandle::null(); + + let error = resolve_root( + ObservationRoot::Element { + handle: &handle, + entry: &entry, + root_ref: Some("@e1"), + }, + &ObservationRequest::snapshot( + &agent_desktop_core::TreeOptions::default(), + agent_desktop_core::Deadline::after(1_000).unwrap(), + ), + crate::tree::locator_deadline::from_timeout( + std::time::Instant::now(), + std::time::Duration::from_secs(1), + ), + ) + .err() + .expect("null root must fail closed"); + + assert_eq!(error.code, ErrorCode::StaleRef); + assert_eq!(error.details.unwrap()["kind"], "locator_root_invalid"); + } +} diff --git a/crates/macos/src/tree/query/node_evidence.rs b/crates/macos/src/tree/query/node_evidence.rs new file mode 100644 index 0000000..af72743 --- /dev/null +++ b/crates/macos/src/tree/query/node_evidence.rs @@ -0,0 +1,209 @@ +use agent_desktop_core::{ + EvidenceRequirements, IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, + LocatorStats, +}; + +pub(crate) fn is_wrapper_candidate(role: Option<&str>, subrole: Option<&str>) -> bool { + role == Some("AXGenericElement") + && role.is_some_and(|role| { + crate::tree::roles::ax_role_and_subrole_to_str(role, subrole) == "group" + }) +} + +pub(crate) fn is_transparent_wrapper( + role: Option<&str>, + subrole: Option<&str>, + name: &agent_desktop_core::NameEvidence, + value: Option<&str>, + identifiers: &IdentifierEvidence, + actions: &LocatorField<Vec<String>>, +) -> bool { + let strings = [ + name.explicit_label.as_deref(), + name.labelled_by_text.as_deref(), + name.native_title.as_deref(), + name.static_value.as_deref(), + name.child_label.as_deref(), + name.placeholder.as_deref(), + name.description.as_deref(), + value, + ]; + is_wrapper_candidate(role, subrole) + && strings + .into_iter() + .all(|text| text.is_none_or(|text| text.trim().is_empty())) + && identifiers.is_complete() + && identifiers.identifiers().is_empty() + && actions.known().is_some_and(Vec::is_empty) +} + +pub(crate) fn option_field<T>(value: Option<T>, uncertain: bool) -> LocatorField<T> { + match value { + Some(value) => LocatorField::Known(value), + None if uncertain => LocatorField::Unknown, + None => LocatorField::Absent, + } +} + +pub(crate) fn update_identifier_stats(identifiers: &IdentifierEvidence, stats: &mut LocatorStats) { + let count = identifiers.identifiers().len() as u64; + stats.identifiers.values_observed += count; + stats.identifiers.nodes_with_identifiers += u64::from(count > 0); + stats.identifiers.nodes_with_multiple_identifiers += u64::from(count > 1); +} + +pub(crate) fn unknown() -> LocatorEvidence { + LocatorEvidence { + role: LocatorField::Unknown, + name: LocatorField::Unknown, + description: LocatorField::Unknown, + value: LocatorField::Unknown, + identifiers: IdentifierEvidence::unknown(), + states: LocatorField::Unknown, + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Unknown, + available_actions: LocatorField::Unknown, + }, + } +} + +pub(crate) fn identifiers( + identifiers: &crate::tree::node_identifiers::NodeIdentifiers, + requested: bool, +) -> IdentifierEvidence { + if !requested { + return IdentifierEvidence::unknown(); + } + let complete = + !identifiers.ax_identifier.is_unknown() && !identifiers.ax_dom_identifier.is_unknown(); + let mut values = Vec::new(); + if let Some(value) = identifiers.ax_identifier.known() { + values.push(agent_desktop_core::ElementIdentifier { + kind: agent_desktop_core::IdentifierKind::AxIdentifier, + value: value.clone(), + }); + } + let preferred = if let Some(value) = identifiers.ax_dom_identifier.known() { + let index = values.len(); + values.push(agent_desktop_core::ElementIdentifier { + kind: agent_desktop_core::IdentifierKind::AxDomIdentifier, + value: value.clone(), + }); + Some(index) + } else { + (!values.is_empty()).then_some(0) + }; + IdentifierEvidence::typed(values, preferred, complete) +} + +pub(crate) fn required_complete( + evidence: &LocatorEvidence, + requirements: EvidenceRequirements, +) -> bool { + (!requirements.role || !evidence.role.is_unknown()) + && (!requirements.name || !evidence.name.is_unknown()) + && (!requirements.description || !evidence.description.is_unknown()) + && (!requirements.value || !evidence.value.is_unknown()) + && (!requirements.identifiers || evidence.identifiers.is_complete()) + && (!requirements.states || !evidence.states.is_unknown()) + && (!requirements.ref_evidence.bounds || !evidence.ref_evidence.bounds.is_unknown()) + && (!requirements.ref_evidence.actions + || !evidence.ref_evidence.available_actions.is_unknown()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dom_identifier_is_preferred_without_discarding_ax_identifier() { + let identifiers = crate::tree::node_identifiers::NodeIdentifiers::from_fields( + LocatorField::Known("native-save".into()), + LocatorField::Known("dom-save".into()), + ); + let evidence = super::identifiers(&identifiers, true); + + assert_eq!(evidence.identifiers().len(), 2); + assert_eq!( + evidence.preferred_identifier(), + Some(&agent_desktop_core::ElementIdentifier { + kind: agent_desktop_core::IdentifierKind::AxDomIdentifier, + value: "dom-save".into(), + }) + ); + } + + #[test] + fn equal_strings_remain_distinct_across_identifier_kinds() { + let identifiers = crate::tree::node_identifiers::NodeIdentifiers::from_fields( + LocatorField::Known("shared".into()), + LocatorField::Known("shared".into()), + ); + let evidence = super::identifiers(&identifiers, true); + + assert_eq!(evidence.identifiers().len(), 2); + assert_ne!( + evidence.identifiers()[0].kind, + evidence.identifiers()[1].kind + ); + } + + fn inert_generic_wrapper() -> ( + agent_desktop_core::NameEvidence, + IdentifierEvidence, + LocatorField<Vec<String>>, + ) { + ( + agent_desktop_core::NameEvidence::default(), + IdentifierEvidence::absent(), + LocatorField::Known(Vec::new()), + ) + } + + #[test] + fn only_semantically_inert_generic_elements_are_transparent() { + let (name, identifiers, actions) = inert_generic_wrapper(); + + assert!(is_transparent_wrapper( + Some("AXGenericElement"), + None, + &name, + None, + &identifiers, + &actions, + )); + assert!(!is_transparent_wrapper( + Some("AXGroup"), + None, + &name, + None, + &identifiers, + &actions, + )); + } + + #[test] + fn named_or_actionable_generic_elements_consume_logical_depth() { + let (mut name, identifiers, actions) = inert_generic_wrapper(); + name.native_title = Some("Settings".into()); + assert!(!is_transparent_wrapper( + Some("AXGenericElement"), + None, + &name, + None, + &identifiers, + &actions, + )); + + name.native_title = None; + let actions = LocatorField::Known(vec!["AXPress".into()]); + assert!(!is_transparent_wrapper( + Some("AXGenericElement"), + None, + &name, + None, + &identifiers, + &actions, + )); + } +} diff --git a/crates/macos/src/tree/query/node_read.rs b/crates/macos/src/tree/query/node_read.rs new file mode 100644 index 0000000..4999dcc --- /dev/null +++ b/crates/macos/src/tree/query/node_read.rs @@ -0,0 +1,307 @@ +use agent_desktop_core::{ + AdapterError, ErrorCode, LocatorEvidence, LocatorField, LocatorRefEvidence, LocatorStats, +}; +use serde_json::json; + +use crate::tree::query::evidence_fields::{description_field, name_field}; +use crate::tree::query::node_evidence::{ + identifiers as identifier_evidence, is_transparent_wrapper, is_wrapper_candidate, option_field, + required_complete, unknown as unknown_evidence, update_identifier_stats, +}; +use crate::tree::query::node_read_context::NodeReadContext; +use crate::tree::{AXElement, query::child_read::ChildRead}; + +pub(crate) struct NodeRead { + pub(crate) attrs: crate::tree::NodeAttrs, + pub(crate) evidence: LocatorEvidence, + pub(crate) web_wrapper: bool, + pub(crate) invalid_element: bool, + pub(crate) child_read: ChildRead, + pub(crate) evidence_complete: bool, +} + +pub(crate) fn read_node( + element: &AXElement, + context: NodeReadContext<'_>, +) -> Result<NodeRead, AdapterError> { + let NodeReadContext { + tree, + stats, + usage, + requirements, + deadline, + child_plan, + } = context; + crate::tree::locator_deadline::prepare(element, deadline)?; + let read = crate::tree::element::fetch_node_attrs_with_status_for( + element, + requirements, + deadline, + usage, + ); + record_attribute_read(stats, &read.metrics, &read.status); + stats.semantic_reads.settable_reads += read.metrics.settable_reads; + stats.reads.health.deadline_exhausted += u64::from(read.metrics.deadline_exhausted); + stats.traversal.limits.text_hits += read.status.text_truncations; + if read.status.api_disabled { + return Err(permission_error("attributes")); + } + let identifiers = identifier_evidence(&read.identifiers, requirements.identifiers); + update_identifier_stats(&identifiers, stats); + if read.status.invalid_element { + return Ok(NodeRead { + attrs: crate::tree::NodeAttrs::default(), + evidence: unknown_evidence(), + web_wrapper: false, + invalid_element: true, + child_read: ChildRead::empty(false), + evidence_complete: false, + }); + } + let attrs = read.attrs; + let wrapper_candidate = is_wrapper_candidate(attrs.role.as_deref(), attrs.subrole.as_deref()); + let child_read = crate::tree::query::child_read::read_children( + element, + attrs.role.as_deref(), + child_plan.max_elements(wrapper_candidate), + deadline, + ); + stats.reads.counts.child_reads += child_read.status.attempts; + stats.reads.health.cannot_complete += child_read.status.health.cannot_complete; + stats.reads.health.native_read_failures += child_read.status.health.native_read_failures; + stats.reads.health.deadline_exhausted += child_read.status.health.deadline_exhausted; + stats.traversal.limits.child_count_changes += u64::from(child_read.status.count_changed); + stats.traversal.limits.child_hits += u64::from(child_read.status.cursor_stalled); + if child_read.status.api_disabled { + return Err(permission_error("children")); + } + if child_read.status.invalid_element { + return Ok(NodeRead { + attrs: crate::tree::NodeAttrs::default(), + evidence: unknown_evidence(), + web_wrapper: false, + invalid_element: true, + child_read: ChildRead::empty(false), + evidence_complete: false, + }); + } + + let role = attrs + .role + .as_deref() + .map(|role| crate::tree::roles::ax_role_and_subrole_to_str(role, attrs.subrole.as_deref())) + .unwrap_or("unknown") + .to_string(); + let secure = attrs.role.as_deref() == Some("AXSecureTextField") + || attrs.subrole.as_deref() == Some("AXSecureTextField"); + let value = (!secure).then(|| attrs.value.clone()).flatten(); + let (name_evidence, child_label_complete) = if requirements.name || requirements.description { + crate::tree::child_labels::complete_name_evidence_with_deadline( + &attrs, + &role, + &child_read.elements, + deadline, + crate::tree::child_labels::NameEvidenceSinks { + stats: &mut *stats, + usage: &mut *usage, + }, + )? + } else { + (attrs.name_evidence.clone(), true) + }; + let children_complete = child_read.complete && !child_read.truncated() && child_label_complete; + let name_field = if !requirements.name { + LocatorField::Unknown + } else { + name_field( + &name_evidence, + &read.status, + attrs.role.as_deref(), + children_complete, + ) + }; + let description_field = if !requirements.description { + LocatorField::Unknown + } else { + description_field( + &name_evidence, + &read.status, + attrs.role.as_deref(), + children_complete, + ) + }; + let state_context = crate::tree::state_reader::StateReaderContext { + focused: tree.focused.as_ref(), + window_bounds: tree.window_bounds, + is_secure_text: secure, + }; + let states = requirements.states.then(|| { + crate::tree::state_reader::states_from_element(element, &attrs, &role, &state_context) + }); + let actions = if let Some(actions) = + read_native_actions_if(requirements.ref_evidence.actions, || { + crate::tree::action_list::read_platform_available_actions( + element, + &role, + attrs.has_scrollbars, + deadline, + usage, + ) + }) { + stats.reads.counts.action_reads += 1; + stats.reads.health.cannot_complete += u64::from(actions.cannot_complete); + stats.reads.health.deadline_exhausted += u64::from(actions.deadline_exhausted); + stats.semantic_reads.settable_reads += actions.settable_reads; + if actions.api_disabled { + return Err(permission_error("actions")); + } + if actions.invalid_element { + return Ok(NodeRead { + attrs: crate::tree::NodeAttrs::default(), + evidence: unknown_evidence(), + web_wrapper: false, + invalid_element: true, + child_read: ChildRead::empty(false), + evidence_complete: false, + }); + } + if actions.complete && !actions.deadline_exhausted && !read.status.scrollbars_unknown() { + LocatorField::Known(actions.actions) + } else { + LocatorField::Unknown + } + } else { + LocatorField::Unknown + }; + let web_wrapper = is_transparent_wrapper( + attrs.role.as_deref(), + attrs.subrole.as_deref(), + &name_evidence, + value.as_deref(), + &identifiers, + &actions, + ); + let role_field = if attrs.role.is_some() { + LocatorField::Known(role) + } else { + option_field(None, read.status.role_unknown()) + }; + let evidence = LocatorEvidence { + role: role_field, + name: name_field, + description: description_field, + value: if requirements.value { + option_field(value, read.status.value_unknown()) + } else { + LocatorField::Unknown + }, + identifiers, + states: if !requirements.states || read.status.states_unknown() { + LocatorField::Unknown + } else { + LocatorField::Known(states.unwrap_or_default()) + }, + ref_evidence: LocatorRefEvidence { + bounds: if requirements.ref_evidence.bounds { + option_field(tree.bounds_for(attrs.bounds), read.status.bounds_unknown()) + } else { + LocatorField::Unknown + }, + available_actions: actions, + }, + }; + let evidence_complete = required_complete(&evidence, requirements) + && !read.metrics.deadline_exhausted + && child_read.status.health.deadline_exhausted == 0; + Ok(NodeRead { + attrs, + evidence, + web_wrapper, + invalid_element: false, + child_read, + evidence_complete, + }) +} + +fn read_native_actions_if<T>(include_actions: bool, reader: impl FnOnce() -> T) -> Option<T> { + include_actions.then(reader) +} + +fn permission_error(phase: &str) -> AdapterError { + AdapterError::new( + ErrorCode::PermDenied, + "Accessibility API is disabled while reading live locator evidence", + ) + .with_suggestion("Grant Accessibility permission, then retry") + .with_details(json!({ "kind": "locator_api_disabled", "phase": phase })) +} + +fn record_attribute_read( + stats: &mut LocatorStats, + metrics: &crate::tree::node_attribute_metrics::NodeAttributeMetrics, + status: &crate::tree::node_attribute_status::NodeAttributeStatus, +) { + stats.reads.counts.attribute_batches += metrics.batch_reads; + stats.reads.counts.attributes_requested += metrics.requested_count; + stats.reads.counts.fallback_reads += metrics.fallback_reads; + stats.reads.health.cannot_complete += u64::from(status.cannot_complete); + stats.reads.health.native_read_failures += status.native_read_failures; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn attribute_stats_record_actual_batch_and_field_counts() { + let mut stats = LocatorStats::default(); + let metrics = crate::tree::node_attribute_metrics::NodeAttributeMetrics { + batch_reads: 1, + requested_count: 22, + ..Default::default() + }; + + record_attribute_read( + &mut stats, + &metrics, + &crate::tree::node_attribute_status::NodeAttributeStatus::default(), + ); + + assert_eq!(stats.reads.counts.attribute_batches, 1); + assert_eq!(stats.reads.counts.attributes_requested, 22); + assert_eq!(stats.reads.counts.fallback_reads, 0); + } + + #[test] + fn cannot_complete_keeps_missing_fields_unknown() { + let mut stats = LocatorStats::default(); + let metrics = crate::tree::node_attribute_metrics::NodeAttributeMetrics { + batch_reads: 1, + requested_count: 22, + fallback_reads: 22, + ..Default::default() + }; + let mut status = crate::tree::node_attribute_status::NodeAttributeStatus::default(); + status.record_batch_error(accessibility_sys::kAXErrorCannotComplete); + + record_attribute_read(&mut stats, &metrics, &status); + + assert_eq!(stats.reads.health.cannot_complete, 1); + assert_eq!(stats.reads.counts.fallback_reads, 22); + assert_eq!(option_field::<String>(None, true), LocatorField::Unknown); + assert_eq!( + option_field(Some("Save".to_string()), true), + LocatorField::Known("Save".into()) + ); + } + + #[test] + fn omitted_ref_evidence_performs_zero_native_action_reads() { + let calls = std::cell::Cell::new(0); + + let read = read_native_actions_if(false, || calls.set(calls.get() + 1)); + + assert!(read.is_none()); + assert_eq!(calls.get(), 0); + } +} diff --git a/crates/macos/src/tree/query/node_read_context.rs b/crates/macos/src/tree/query/node_read_context.rs new file mode 100644 index 0000000..28a33cf --- /dev/null +++ b/crates/macos/src/tree/query/node_read_context.rs @@ -0,0 +1,8 @@ +pub(crate) struct NodeReadContext<'a> { + pub(crate) tree: &'a crate::tree::TreeBuildContext, + pub(crate) stats: &'a mut agent_desktop_core::LocatorStats, + pub(crate) usage: &'a mut crate::tree::observation_usage::ObservationUsage, + pub(crate) requirements: agent_desktop_core::EvidenceRequirements, + pub(crate) deadline: std::time::Instant, + pub(crate) child_plan: super::child_read_plan::ChildReadPlan, +} diff --git a/crates/macos/src/tree/query/read_error.rs b/crates/macos/src/tree/query/read_error.rs new file mode 100644 index 0000000..10601e5 --- /dev/null +++ b/crates/macos/src/tree/query/read_error.rs @@ -0,0 +1,26 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use serde_json::json; + +pub(crate) fn semantic_read(error: i32, phase: &str) -> AdapterError { + let details = json!({ "ax_error": error, "kind": "observation_semantic_read", "phase": phase }); + if error == accessibility_sys::kAXErrorAPIDisabled { + return AdapterError::new( + ErrorCode::PermDenied, + "Accessibility API was disabled during observation", + ) + .with_suggestion("Grant Accessibility permission, then retry") + .with_details(details); + } + AdapterError::new( + ErrorCode::AppUnresponsive, + "Accessibility evidence read did not complete", + ) + .with_suggestion("Retry within the existing operation deadline") + .with_details(json!({ + "ax_error": error, + "complete": false, + "kind": "observation_semantic_read", + "phase": phase, + "retryable": true, + })) +} diff --git a/crates/macos/src/tree/query/traversal.rs b/crates/macos/src/tree/query/traversal.rs new file mode 100644 index 0000000..0d0a674 --- /dev/null +++ b/crates/macos/src/tree/query/traversal.rs @@ -0,0 +1,276 @@ +use super::{arena::TraversalArena, child_read::ChildRead, node_read::read_node}; +use crate::tree::AXElement; +use agent_desktop_core::{ + AdapterError, ErrorCode, ObservationRequest, ObservationSource, ObservedSubtree, ObservedTree, +}; +use serde_json::json; +use std::time::{Duration, Instant}; + +pub(crate) struct LocatorTraversal { + request: ObservationRequest, + context: crate::tree::TreeBuildContext, + deadline: Instant, + arena: TraversalArena, + usage: crate::tree::observation_usage::ObservationUsage, +} + +impl LocatorTraversal { + pub(crate) fn new( + request: &ObservationRequest, + context: crate::tree::TreeBuildContext, + deadline: Instant, + ) -> Self { + Self { + request: *request, + context, + deadline, + arena: TraversalArena::new(), + usage: crate::tree::observation_usage::ObservationUsage::new(request.budget), + } + } + + pub(crate) fn build( + mut self, + root: AXElement, + source: ObservationSource, + ) -> Result<(ObservedTree, bool, agent_desktop_core::LocatorStats), AdapterError> { + self.arena.add_handles(1); + let root = self.visit(root, 0, 0)?; + let complete = self.arena.structurally_complete; + let renderer_ready = self.arena.stats.activation.ready; + let stats = self.arena.finish()?; + let root = root.ok_or_else(|| { + let code = if stats.reads.health.deadline_exhausted > 0 { + ErrorCode::Timeout + } else { + ErrorCode::AppUnresponsive + }; + AdapterError::new( + code, + "Accessibility observation ended before reading its root", + ) + .with_details(json!({ + "kind": "observation_root_incomplete", + "complete": false, + "query_stats": stats, + })) + })?; + let tree = ObservedTree::from_roots(vec![root], source, stats.clone(), complete)?; + Ok((tree, renderer_ready, stats)) + } + + fn visit( + &mut self, + element: AXElement, + logical_depth: u8, + raw_depth: u8, + ) -> Result<Option<ObservedSubtree>, AdapterError> { + let Some(_) = self.remaining_budget() else { + self.note_deadline_exhausted(); + self.arena.drop_handles(1); + return Ok(None); + }; + let pointer = element.0 as usize; + if !self.arena.ancestors.insert(pointer) { + self.arena.stats.traversal.cycles_skipped += 1; + self.arena.drop_handles(1); + return Ok(None); + } + if !self.usage.claim_node() { + self.arena.stats.traversal.limits.node_hits += 1; + self.arena.mark_incomplete(); + self.arena.ancestors.remove(&pointer); + self.arena.drop_handles(1); + return Ok(None); + } + self.note_visit(logical_depth, raw_depth); + let requirements = self.request.evidence_for_raw_depth(raw_depth); + let boundary_elements = if raw_depth == 0 && self.request.hydrates_root_name_from_children() + { + crate::tree::child_labels::MAX_LABEL_ELEMENTS + } else { + 0 + }; + let child_plan = super::child_read_plan::ChildReadPlan::boundary_aware( + self.usage.child_capacity(), + boundary_elements, + logical_depth, + self.request.max_logical_depth, + ); + let read = read_node( + &element, + super::node_read_context::NodeReadContext { + tree: &self.context, + stats: &mut self.arena.stats, + usage: &mut self.usage, + requirements, + deadline: self.deadline, + child_plan, + }, + )?; + let renderer_surface_observed = read + .evidence + .role + .known() + .is_some_and(|role| role == "webarea"); + if renderer_surface_observed { + self.arena.stats.activation.ready = true; + } + if read.invalid_element { + self.arena.ancestors.remove(&pointer); + self.arena.drop_handles(1); + if raw_depth == 0 { + return Err(AdapterError::new( + ErrorCode::StaleRef, + "Live locator root is no longer a valid accessibility element", + ) + .with_suggestion("Refresh the source snapshot and retry the locator") + .with_details(json!({ "kind": "locator_root_invalid" }))); + } + self.arena.mark_incomplete(); + return Ok(None); + } + let child_logical_depth = logical_depth + u8::from(!read.web_wrapper); + if read.web_wrapper { + self.arena.stats.traversal.web_wrapper_nodes += 1; + } + self.usage + .note_child_demand(read.child_read.total_count, &mut self.arena.stats); + let loaded_child_count = read.child_read.elements.len(); + self.usage.claim_edges(loaded_child_count); + self.arena.add_handles(loaded_child_count); + let at_requested_boundary = child_logical_depth > self.request.max_logical_depth; + let (children, children_count, subtree_complete) = if at_requested_boundary { + self.arena.drop_handles(loaded_child_count); + ( + Vec::new(), + u32::try_from(read.child_read.total_count) + .ok() + .filter(|count| *count > 0), + read.child_read.complete, + ) + } else { + let (children, complete) = + self.visit_children(read.child_read, (child_logical_depth, raw_depth))?; + (children, None, complete) + }; + self.arena.ancestors.remove(&pointer); + self.arena.drop_handles(1); + let subtree_complete = structural_completeness(subtree_complete, read.evidence_complete); + if !subtree_complete { + self.arena.mark_incomplete(); + } + Ok(Some(ObservedSubtree::new( + read.evidence, + children, + subtree_complete, + children_count, + ))) + } + + fn visit_children( + &mut self, + read: ChildRead, + depths: (u8, u8), + ) -> Result<(Vec<ObservedSubtree>, bool), AdapterError> { + let mut complete = read.complete && !read.truncated(); + if !complete { + self.arena.mark_incomplete(); + } + if depths.1 >= self.request.max_raw_depth && !read.elements.is_empty() { + self.arena.stats.traversal.limits.depth_hits += 1; + self.arena.drop_handles(read.elements.len()); + self.arena.mark_incomplete(); + return Ok((Vec::new(), false)); + } + let total = read.elements.len(); + let mut children = Vec::new(); + let mut predecessors_complete = read.prefix_certain; + for (child_index, child) in read.elements.into_iter().enumerate() { + if self.remaining_budget().is_none() { + self.note_deadline_exhausted(); + self.arena.drop_handles(total.saturating_sub(child_index)); + complete = false; + break; + } + match self.visit(child, depths.0, depths.1.saturating_add(1))? { + Some(subtree) => { + complete &= subtree.is_complete(); + let edge_complete = retained_edge_certainty(&mut predecessors_complete, true); + children.push( + subtree + .with_source_child_index(child_index) + .with_predecessors_complete(edge_complete), + ); + } + None => { + retained_edge_certainty(&mut predecessors_complete, false); + } + } + } + if !complete { + self.arena.mark_incomplete(); + } + Ok((children, complete)) + } + + fn remaining_budget(&self) -> Option<Duration> { + self.deadline + .checked_duration_since(Instant::now()) + .map(|remaining| remaining.min(crate::tree::locator_deadline::MAX_IPC_SLICE)) + .filter(|remaining| !remaining.is_zero()) + } + + fn note_deadline_exhausted(&mut self) { + self.arena.stats.reads.health.deadline_exhausted += 1; + self.arena.mark_incomplete(); + } + + fn note_visit(&mut self, logical_depth: u8, raw_depth: u8) { + self.arena.stats.traversal.nodes_visited += 1; + self.arena.stats.traversal.max_logical_depth = self + .arena + .stats + .traversal + .max_logical_depth + .max(logical_depth); + self.arena.stats.traversal.max_raw_depth = + self.arena.stats.traversal.max_raw_depth.max(raw_depth); + } +} + +fn retained_edge_certainty(prefix_certain: &mut bool, retained: bool) -> bool { + let edge_certain = *prefix_certain; + *prefix_certain &= retained; + edge_certain +} + +fn structural_completeness(topology_complete: bool, _evidence_complete: bool) -> bool { + topology_complete +} + +#[cfg(test)] +mod tests { + use super::retained_edge_certainty; + + #[test] + fn uncertain_source_prefix_marks_retained_edge_uncertain() { + let mut prefix_certain = false; + + assert!(!retained_edge_certainty(&mut prefix_certain, true)); + } + + #[test] + fn omitted_native_predecessor_marks_later_retained_edge_uncertain() { + let mut prefix_certain = true; + + assert!(retained_edge_certainty(&mut prefix_certain, false)); + assert!(!retained_edge_certainty(&mut prefix_certain, true)); + } + + #[test] + fn unknown_semantic_evidence_does_not_poison_complete_topology() { + assert!(super::structural_completeness(true, false)); + assert!(!super::structural_completeness(false, true)); + } +} diff --git a/crates/macos/src/tree/readonly.rs b/crates/macos/src/tree/readonly.rs new file mode 100644 index 0000000..0882aea --- /dev/null +++ b/crates/macos/src/tree/readonly.rs @@ -0,0 +1,55 @@ +pub(crate) struct ReadonlyRead { + pub(crate) value: Option<bool>, + pub(crate) error: Option<i32>, + pub(crate) attempted: bool, + pub(crate) deadline_exhausted: bool, +} + +#[cfg(target_os = "macos")] +pub(crate) fn read_readonly( + element: &super::AXElement, + role: Option<&str>, + deadline: std::time::Instant, +) -> ReadonlyRead { + if !editable_ax_role(role) { + return ReadonlyRead { + value: None, + error: None, + attempted: false, + deadline_exhausted: false, + }; + } + if super::locator_deadline::prepare(element, deadline).is_err() { + return ReadonlyRead { + value: None, + error: None, + attempted: false, + deadline_exhausted: true, + }; + } + let read = super::capabilities::is_attr_settable_with_status(element, "AXValue", deadline); + ReadonlyRead { + value: read.value.map(|settable| !settable), + error: read.error, + attempted: true, + deadline_exhausted: false, + } +} + +#[cfg(target_os = "macos")] +fn editable_ax_role(role: Option<&str>) -> bool { + matches!( + role, + Some( + "AXTextField" + | "AXTextArea" + | "AXSearchField" + | "AXComboBox" + | "AXPopUpButton" + | "AXIncrementor" + | "AXStepper" + | "AXSlider" + | "AXValueIndicator" + ) + ) +} diff --git a/crates/macos/src/tree/renderer_probe.rs b/crates/macos/src/tree/renderer_probe.rs new file mode 100644 index 0000000..12bf411 --- /dev/null +++ b/crates/macos/src/tree/renderer_probe.rs @@ -0,0 +1,86 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; + +pub(crate) fn activation_supported( + pid: i32, + process_instance: &str, + deadline: std::time::Instant, +) -> Result<bool, AdapterError> { + if !crate::system::process_identity::matches_instance(pid, process_instance)? { + return Err(AdapterError::stale_ref( + "Renderer process instance changed before activation probing", + )); + } + let application = super::element_for_pid(pid); + super::locator_deadline::prepare(&application, deadline)?; + let read = super::capabilities::is_attr_settable_with_status( + &application, + "AXManualAccessibility", + deadline, + ); + super::locator_deadline::remaining(deadline)?; + match (read.value, read.error) { + (Some(value), None) => Ok(value), + (None, Some(error)) if is_unsupported(error) => Ok(false), + (None, Some(error)) if error == accessibility_sys::kAXErrorAPIDisabled => { + Err(AdapterError::new( + ErrorCode::PermDenied, + "Accessibility API is disabled while probing renderer support", + )) + } + (None, Some(error)) => Err(inconclusive_probe(error)), + _ => Err(inconclusive_probe(accessibility_sys::kAXErrorFailure)), + } +} + +fn inconclusive_probe(error: i32) -> AdapterError { + let code = match error { + accessibility_sys::kAXErrorCannotComplete => ErrorCode::Timeout, + accessibility_sys::kAXErrorInvalidUIElement => ErrorCode::StaleRef, + _ => ErrorCode::AppUnresponsive, + }; + AdapterError::new( + code, + "Renderer accessibility capability probe was inconclusive", + ) + .with_details(serde_json::json!({ + "kind": "renderer_capability_probe", + "ax_error": error, + "complete": false, + "retryable": true, + })) + .with_suggestion("Retry after the renderer accessibility tree finishes updating") +} + +fn is_unsupported(error: i32) -> bool { + matches!( + error, + accessibility_sys::kAXErrorAttributeUnsupported + | accessibility_sys::kAXErrorNoValue + | accessibility_sys::kAXErrorNotImplemented + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_probe_results_do_not_request_activation() { + for error in [ + accessibility_sys::kAXErrorAttributeUnsupported, + accessibility_sys::kAXErrorNoValue, + accessibility_sys::kAXErrorNotImplemented, + ] { + assert!(is_unsupported(error)); + } + assert!(!is_unsupported(accessibility_sys::kAXErrorCannotComplete)); + } + + #[test] + fn transient_probe_failures_are_never_collapsed_to_unsupported() { + let error = inconclusive_probe(accessibility_sys::kAXErrorCannotComplete); + + assert_eq!(error.code, ErrorCode::Timeout); + assert_eq!(error.details.unwrap()["complete"], false); + } +} diff --git a/crates/macos/src/tree/resolve.rs b/crates/macos/src/tree/resolve.rs index 4f1c4f1..7f9a8d4 100644 --- a/crates/macos/src/tree/resolve.rs +++ b/crates/macos/src/tree/resolve.rs @@ -1,139 +1,325 @@ -use agent_desktop_core::{ - adapter::NativeHandle, - error::{AdapterError, ErrorCode}, - refs::RefEntry, -}; +#[cfg(target_os = "macos")] +use agent_desktop_core::ref_identity::has_meaningful_identity; +use agent_desktop_core::{AdapterError, ErrorCode, NativeHandle, RefEntry, SnapshotSurface}; use std::time::{Duration, Instant}; #[cfg(target_os = "macos")] use super::resolve_classify::identity_summary_for_message; #[cfg(target_os = "macos")] -use super::resolve_deadline::sleep_before_retry; +use super::resolve_read_context::ResolveReadContext; #[cfg(target_os = "macos")] -use super::resolve_identity::has_meaningful_identity; -#[cfg(target_os = "macos")] -use super::resolve_roots::{ - candidate_roots, path_candidate_roots, source_window_number, source_window_scope_required, -}; +use super::resolve_roots::{candidate_roots, source_window_scope_required}; #[cfg(target_os = "macos")] use super::resolve_search::{find_entry_by_path, find_entry_in_roots}; +const MAX_RESOLVE_DEPTH: u8 = 50; + #[cfg(target_os = "macos")] -pub fn resolve_element_impl(entry: &RefEntry) -> Result<NativeHandle, AdapterError> { - resolve_element_with_timeout(entry, Duration::from_secs(5)) +pub(crate) fn resolve_element_with_deadline( + entry: &RefEntry, + operation_deadline: agent_desktop_core::Deadline, +) -> Result<NativeHandle, AdapterError> { + verify_process_instance(entry)?; + let deadline = crate::tree::locator_deadline::from_operation(operation_deadline)?; + let result = retry_incomplete_until(deadline, || resolve_once(entry, deadline)); + match result { + Err(error) if error.code == ErrorCode::ElementNotFound => { + Err(stale_ref_error(entry, &error)) + } + other => other, + } } -/// Resolves a ref to a live handle, retrying until the deadline. Once any -/// complete search pass concludes the element is absent, a later attempt that -/// merely runs out of deadline budget is still a stale ref — not an -/// indeterminate timeout. Tracking `proved_absent` keeps STALE_REF -/// deterministic regardless of how big or slow the surrounding tree is. #[cfg(target_os = "macos")] -pub fn resolve_element_with_timeout( +pub(crate) fn resolve_locator_anchor_with_deadline( entry: &RefEntry, - timeout: Duration, + operation_deadline: agent_desktop_core::Deadline, ) -> Result<NativeHandle, AdapterError> { - let (resolve_depth, attempts) = (50, 4); - let deadline = Instant::now() + timeout; - let mut proved_absent = false; - for attempt in 0..attempts { - if can_use_path_fast_path(entry) { - let path_roots = match path_candidate_roots(entry, deadline) { - Ok(roots) => roots, - Err(err) => return Err(downgrade_timeout(err, entry, proved_absent)), - }; - let scope_verified = path_roots.scope_verified; - match find_entry_by_path(&path_roots.roots, entry, scope_verified, deadline) { - Ok(handle) => { - return Ok(handle); - } - Err(err) if is_retryable_resolution_error(&err) => proved_absent = true, - Err(err) => return Err(downgrade_timeout(err, entry, proved_absent)), - } - if requires_scoped_path_resolution(entry) { - if attempt + 1 < attempts { - sleep_before_retry(deadline); - } - continue; - } - } - if !can_use_broad_search(entry) { - if attempt + 1 < attempts { - sleep_before_retry(deadline); - } - continue; - } - let roots = match candidate_roots(entry, deadline) { - Ok(roots) => roots, - Err(err) => return Err(downgrade_timeout(err, entry, proved_absent)), - }; - let scope_verified = roots.scope_verified; - match find_entry_in_roots(&roots.roots, entry, resolve_depth, scope_verified, deadline) { - Ok(handle) => { - return Ok(handle); - } - Err(err) if is_retryable_resolution_error(&err) => proved_absent = true, - Err(err) => return Err(downgrade_timeout(err, entry, proved_absent)), + verify_process_instance(entry)?; + let deadline = crate::tree::locator_deadline::from_operation(operation_deadline)?; + let result = retry_incomplete_until(deadline, || resolve_locator_anchor_once(entry, deadline)); + match result { + Err(error) if error.code == ErrorCode::ElementNotFound => { + Err(stale_ref_error(entry, &error)) } + other => other, + } +} - if attempt + 1 < attempts { - sleep_before_retry(deadline); +#[cfg(target_os = "macos")] +fn verify_process_instance(entry: &RefEntry) -> Result<(), AdapterError> { + let Some(instance) = entry.process.process_instance.as_deref() else { + return Err(AdapterError::stale_ref( + "Saved target has no process instance identity", + )); + }; + let pid = crate::system::process_identity::to_pid_t(entry.process.pid)?; + match crate::system::process_identity::matches_instance(pid, instance) { + Ok(true) => Ok(()), + Ok(false) => Err(AdapterError::stale_ref( + "Saved target belongs to a process instance that is no longer running", + )), + Err(error) if error.code == ErrorCode::InvalidArgs => Err(AdapterError::stale_ref( + "Saved target carries a malformed process instance identity", + )), + Err(error) => Err(error), + } +} + +#[cfg(target_os = "macos")] +fn resolve_once(entry: &RefEntry, deadline: Instant) -> Result<NativeHandle, AdapterError> { + let started = Instant::now(); + crate::tree::locator_deadline::remaining(deadline)?; + let mut read_context = ResolveReadContext::new(deadline); + if can_use_path_fast_path(entry) { + let roots = match candidate_roots(entry, &mut read_context) { + Ok(roots) => roots, + Err(error) => { + return finish_resolution( + Err(error), + &mut read_context, + started, + "path_roots", + true, + ); + } + }; + match find_entry_by_path(&roots.roots, entry, roots.scope_verified, &mut read_context) { + Ok(handle) => { + return finish_resolution( + Ok(handle), + &mut read_context, + started, + "path_success", + true, + ); + } + Err(error) if error.code == ErrorCode::ElementNotFound => { + tracing::debug!(stats = ?read_context.stats, "strict resolution path fast-path fell back to broad search"); + } + Err(error) => { + return finish_resolution( + Err(error), + &mut read_context, + started, + "path_error", + true, + ); + } } } - - Err(stale_ref_error(entry)) + if !can_use_broad_search(entry) { + return finish_resolution( + Err(AdapterError::element_not_found("element")), + &mut read_context, + started, + "identity_insufficient", + false, + ); + } + let roots = match candidate_roots(entry, &mut read_context) { + Ok(roots) => roots, + Err(error) => { + return finish_resolution(Err(error), &mut read_context, started, "broad_roots", false); + } + }; + let result = find_entry_in_roots( + &roots.roots, + entry, + MAX_RESOLVE_DEPTH, + roots.scope_verified, + &mut read_context, + ); + finish_resolution(result, &mut read_context, started, "broad_search", false) } #[cfg(target_os = "macos")] -fn stale_ref_error(entry: &RefEntry) -> AdapterError { +fn resolve_locator_anchor_once( + entry: &RefEntry, + deadline: Instant, +) -> Result<NativeHandle, AdapterError> { + let started = Instant::now(); + crate::tree::locator_deadline::remaining(deadline)?; + let mut read_context = ResolveReadContext::new(deadline); + if !can_use_locator_anchor_path(entry) { + return finish_resolution( + Err(AdapterError::element_not_found("locator anchor")), + &mut read_context, + started, + "locator_anchor_insufficient", + true, + ); + } + let roots = match candidate_roots(entry, &mut read_context) { + Ok(roots) => roots, + Err(error) => { + return finish_resolution( + Err(error), + &mut read_context, + started, + "locator_anchor_roots", + true, + ); + } + }; + let result = find_entry_by_path(&roots.roots, entry, roots.scope_verified, &mut read_context); + finish_resolution( + result, + &mut read_context, + started, + "locator_anchor_path", + true, + ) +} + +#[cfg(target_os = "macos")] +fn finish_resolution<T>( + result: Result<T, AdapterError>, + context: &mut ResolveReadContext, + started: Instant, + phase: &str, + path_fast: bool, +) -> Result<T, AdapterError> { + context.stats.elapsed_us = started.elapsed().as_micros() as u64; + match result { + Ok(value) => { + tracing::debug!(phase, path_fast, stats = ?context.stats, "strict resolution completed"); + Ok(value) + } + Err(mut error) => { + let mut details = error + .details + .take() + .unwrap_or_else(|| serde_json::json!({})); + if let Some(object) = details.as_object_mut() { + object.insert("resolution_phase".into(), serde_json::json!(phase)); + object.insert("path_fast".into(), serde_json::json!(path_fast)); + object.insert("query_stats".into(), serde_json::json!(&context.stats)); + } + tracing::debug!(phase, path_fast, code = error.code.as_str(), stats = ?context.stats, "strict resolution failed"); + Err(error.with_details(details)) + } + } +} + +#[cfg(target_os = "macos")] +fn retry_incomplete_until<T>( + deadline: Instant, + mut operation: impl FnMut() -> Result<T, AdapterError>, +) -> Result<T, AdapterError> { + let mut last_incomplete = None; + loop { + if let Err(timeout) = crate::tree::locator_deadline::remaining(deadline) { + return Err(last_incomplete + .map(mark_deadline_elapsed) + .unwrap_or(timeout)); + } + match operation() { + Ok(value) => return Ok(value), + Err(error) if is_retryable_resolution_error(&error) => { + last_incomplete = Some(error); + sleep_before_retry(deadline); + } + Err(error) if error.code == ErrorCode::Timeout => match last_incomplete { + Some(incomplete) => return Err(mark_deadline_elapsed(incomplete)), + None => return Err(error), + }, + Err(error) => return Err(error), + } + } +} + +#[cfg(target_os = "macos")] +fn sleep_before_retry(deadline: Instant) { + if let Ok(remaining) = crate::tree::locator_deadline::remaining(deadline) { + std::thread::sleep(remaining.min(Duration::from_millis(25))); + } +} + +#[cfg(target_os = "macos")] +fn mark_deadline_elapsed(mut error: AdapterError) -> AdapterError { + let mut details = error + .details + .take() + .unwrap_or_else(|| serde_json::json!({})); + if let Some(object) = details.as_object_mut() { + object.insert("deadline_elapsed".into(), serde_json::json!(true)); + } else { + details = serde_json::json!({ + "evidence": details, + "deadline_elapsed": true, + }); + } + error.with_details(details) +} + +#[cfg(target_os = "macos")] +fn stale_ref_error(entry: &RefEntry, cause: &AdapterError) -> AdapterError { + let retryable = cause.permits_retry_by_default(); AdapterError::new( ErrorCode::StaleRef, format!("Element not found: {}", identity_summary_for_message(entry)), ) .with_suggestion("Run 'snapshot' to refresh, then retry with the updated ref.") -} - -/// A deadline `TIMEOUT` becomes `STALE_REF` once a prior complete pass proved -/// the element absent: the element is gone, the timeout is incidental. -#[cfg(target_os = "macos")] -fn downgrade_timeout(err: AdapterError, entry: &RefEntry, proved_absent: bool) -> AdapterError { - if proved_absent && err.code == ErrorCode::Timeout { - stale_ref_error(entry) - } else { - err - } + .with_details(serde_json::json!({ + "kind": "resolution_complete_absence", + "complete": true, + "retryable": retryable, + "pid": entry.process.pid, + "source_window_id": entry.source.source_window_id, + "cause": cause.details, + })) } #[cfg(target_os = "macos")] -fn is_retryable_resolution_error(err: &AdapterError) -> bool { - err.code == ErrorCode::ElementNotFound +fn is_retryable_resolution_error(error: &AdapterError) -> bool { + error.code == ErrorCode::AppUnresponsive && error.is_explicitly_retryable() } #[cfg(target_os = "macos")] fn can_use_path_fast_path(entry: &RefEntry) -> bool { - (entry.root_ref.is_none() || entry.path_is_absolute) - && !entry.path.is_empty() - && (entry.bounds_hash.is_some() || source_window_number(entry).is_some()) + let source_scoped = if entry.source.source_surface == SnapshotSurface::Window { + source_window_scope_required(entry) + } else { + true + }; + source_scoped + && (entry.scope.root_ref.is_none() || entry.scope.path_is_absolute) + && !entry.scope.path.is_empty() + && (entry.geometry.bounds_hash.is_some() || has_meaningful_identity(entry)) } #[cfg(target_os = "macos")] -fn requires_scoped_path_resolution(entry: &RefEntry) -> bool { - (entry.root_ref.is_none() || entry.path_is_absolute) - && entry.bounds_hash.is_none() - && !entry.path.is_empty() - && source_window_scope_required(entry) +fn can_use_locator_anchor_path(entry: &RefEntry) -> bool { + let source_scoped = if entry.source.source_surface == SnapshotSurface::Window { + source_window_scope_required(entry) + } else { + true + }; + source_scoped + && (entry.scope.root_ref.is_none() || entry.scope.path_is_absolute) + && (entry.geometry.bounds_hash.is_some() || has_meaningful_identity(entry)) } #[cfg(target_os = "macos")] fn can_use_broad_search(entry: &RefEntry) -> bool { - entry.bounds_hash.is_some() || has_meaningful_identity(entry) + entry.geometry.bounds_hash.is_some() || has_meaningful_identity(entry) } #[cfg(test)] #[path = "resolve_tests.rs"] mod tests; +#[cfg(test)] +#[path = "resolve_tests_more.rs"] +mod tests_more; + #[cfg(not(target_os = "macos"))] -pub fn resolve_element_impl(_entry: &RefEntry) -> Result<NativeHandle, AdapterError> { - Err(AdapterError::not_supported("resolve_element")) +pub(crate) fn resolve_element_with_deadline( + _entry: &RefEntry, + _deadline: agent_desktop_core::Deadline, +) -> Result<NativeHandle, AdapterError> { + Err(AdapterError::not_supported( + "resolve_element_strict_with_timeout", + )) } diff --git a/crates/macos/src/tree/resolve_ax_read.rs b/crates/macos/src/tree/resolve_ax_read.rs new file mode 100644 index 0000000..1554697 --- /dev/null +++ b/crates/macos/src/tree/resolve_ax_read.rs @@ -0,0 +1,208 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; + +use super::AXElement; + +#[cfg(target_os = "macos")] +pub(crate) fn read_string_with_usage( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + usage: &mut crate::tree::observation_usage::ObservationUsage, +) -> Result<Option<String>, AdapterError> { + let value = read_value(element, attribute, deadline, |value| { + use core_foundation::{ + base::{CFType, TCFType}, + string::CFString, + }; + + let value = unsafe { CFType::wrap_under_create_rule(value) }; + let value = value.downcast::<CFString>()?; + crate::tree::bounded_string::BoundedString::from_cf(&value, usage).ok() + })?; + match value { + Some(value) if value.complete => Ok(Some(value.value)), + Some(_) => Err(incomplete_read_error( + attribute, + "text budget exhausted", + i32::MIN, + )), + None => Ok(None), + } +} + +pub(crate) fn read_string( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, +) -> Result<Option<String>, AdapterError> { + crate::tree::surface_read::string(element, attribute, deadline) +} + +pub(crate) fn read_array( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, +) -> Result<Option<Vec<AXElement>>, AdapterError> { + crate::tree::surface_read::elements(element, attribute, deadline) + .map(|elements| (!elements.is_empty()).then_some(elements)) +} + +#[cfg(target_os = "macos")] +pub(super) fn read_element( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, +) -> Result<Option<AXElement>, AdapterError> { + read_value(element, attribute, deadline, |value| { + crate::tree::ax_value::created_ax_element(value) + }) +} + +#[cfg(target_os = "macos")] +pub(super) fn read_point( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, +) -> Result<Option<core_graphics::geometry::CGPoint>, AdapterError> { + read_value(element, attribute, deadline, |value| { + use accessibility_sys::{AXValueGetValue, kAXValueTypeCGPoint}; + use core_graphics::geometry::CGPoint; + + let mut point = CGPoint::new(0.0, 0.0); + let decoded = unsafe { + AXValueGetValue( + value as _, + kAXValueTypeCGPoint, + &mut point as *mut _ as *mut std::ffi::c_void, + ) + }; + unsafe { core_foundation::base::CFRelease(value) }; + decoded.then_some(point) + }) +} + +#[cfg(target_os = "macos")] +pub(super) fn read_size( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, +) -> Result<Option<core_graphics::geometry::CGSize>, AdapterError> { + read_value(element, attribute, deadline, |value| { + use accessibility_sys::{AXValueGetValue, kAXValueTypeCGSize}; + use core_graphics::geometry::CGSize; + + let mut size = CGSize::new(0.0, 0.0); + let decoded = unsafe { + AXValueGetValue( + value as _, + kAXValueTypeCGSize, + &mut size as *mut _ as *mut std::ffi::c_void, + ) + }; + unsafe { core_foundation::base::CFRelease(value) }; + decoded.then_some(size) + }) +} + +#[cfg(target_os = "macos")] +fn read_value<T>( + element: &AXElement, + attribute: &str, + deadline: std::time::Instant, + decode: impl FnOnce(core_foundation::base::CFTypeRef) -> Option<T>, +) -> Result<Option<T>, AdapterError> { + use core_foundation::{base::TCFType, string::CFString}; + + let attribute_name = CFString::new(attribute); + let (error, value) = crate::tree::ax_ipc::copy_attribute_value( + element, + attribute_name.as_concrete_TypeRef(), + deadline, + ); + if std::time::Instant::now() >= deadline { + release_if_present(value); + return Err( + AdapterError::timeout("Strict element resolution deadline exhausted").with_details( + serde_json::json!({ + "kind": "resolution_deadline_exhausted", + "complete": false, + }), + ), + ); + } + if !classify_ax_read(error, attribute)? { + release_if_present(value); + return Ok(None); + } + if value.is_null() { + return Err(incomplete_read_error( + attribute, + "kAXErrorSuccess returned a null value", + error, + )); + } + decode(value).map(Some).ok_or_else(|| { + incomplete_read_error(attribute, "unexpected accessibility value type", error) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn classify_ax_read(error: i32, attribute: &str) -> Result<bool, AdapterError> { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, kAXErrorNoValue, kAXErrorSuccess, + }; + + if error == kAXErrorSuccess { + return Ok(true); + } + if error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue { + return Ok(false); + } + if error == kAXErrorAPIDisabled { + return Err(AdapterError::new( + ErrorCode::PermDenied, + "Accessibility API is disabled during strict element resolution", + ) + .with_suggestion("Grant Accessibility permission, then retry") + .with_details(serde_json::json!({ + "kind": "resolution_ax_read", + "attribute": attribute, + "ax_error": error, + "complete": false, + "retryable": false, + }))); + } + let label = if error == kAXErrorCannotComplete { + "kAXErrorCannotComplete" + } else if error == kAXErrorInvalidUIElement { + "kAXErrorInvalidUIElement" + } else { + "unclassified AXError" + }; + Err(incomplete_read_error(attribute, label, error)) +} + +#[cfg(target_os = "macos")] +fn incomplete_read_error(attribute: &str, reason: &str, error: i32) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!("Strict element resolution could not read {attribute}: {reason}"), + ) + .with_suggestion("Retry after the target application finishes updating its accessibility tree") + .with_details(serde_json::json!({ + "kind": "resolution_ax_read", + "attribute": attribute, + "ax_error": error, + "reason": reason, + "complete": false, + "retryable": true, + })) +} + +#[cfg(target_os = "macos")] +fn release_if_present(value: core_foundation::base::CFTypeRef) { + if !value.is_null() { + unsafe { core_foundation::base::CFRelease(value) }; + } +} diff --git a/crates/macos/src/tree/resolve_bounds.rs b/crates/macos/src/tree/resolve_bounds.rs index 2e83046..69c70f0 100644 --- a/crates/macos/src/tree/resolve_bounds.rs +++ b/crates/macos/src/tree/resolve_bounds.rs @@ -1,85 +1,38 @@ -use agent_desktop_core::{node::Rect, refs::RefEntry}; +use agent_desktop_core::{Rect, RefEntry}; use super::AXElement; -pub(super) fn bounds_match(el: &AXElement, entry: &RefEntry) -> bool { - match entry.bounds_hash { - Some(expected) => { - let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash()); - actual.map(|h| h == expected).unwrap_or(false) - } - None => true, - } -} - -pub(super) fn should_prune_by_bounds(el: &AXElement, entry: &RefEntry, depth: u8) -> bool { - if depth == 0 || entry.bounds.is_none() || entry.bounds_hash.is_none() { - return false; - } - let Some(candidate) = crate::tree::read_bounds(el) else { - return false; +#[cfg(target_os = "macos")] +pub(super) fn bounds_match_with_deadline( + element: &AXElement, + entry: &RefEntry, + deadline: std::time::Instant, +) -> Result<bool, agent_desktop_core::AdapterError> { + let Some(expected) = entry.geometry.bounds_hash else { + return Ok(true); }; - let Some(target) = entry.bounds.as_ref() else { - return false; - }; - !rects_overlap(&candidate, target) + Ok(read_bounds_with_deadline(element, deadline)? + .map(|actual| actual.bounds_hash() == Some(expected)) + .unwrap_or(false)) } -fn rects_overlap(candidate: &Rect, target: &Rect) -> bool { - let candidate_right = candidate.x + candidate.width; - let candidate_bottom = candidate.y + candidate.height; - let target_right = target.x + target.width; - let target_bottom = target.y + target.height; - candidate.x <= target_right - && candidate_right >= target.x - && candidate.y <= target_bottom - && candidate_bottom >= target.y -} - -#[cfg(test)] -mod tests { - use agent_desktop_core::node::Rect; - - use super::rects_overlap; - - fn r(x: f64, y: f64, w: f64, h: f64) -> Rect { - Rect { - x, - y, - width: w, - height: h, - } - } - - #[test] - fn overlapping_rects_intersect() { - assert!(rects_overlap( - &r(0.0, 0.0, 10.0, 10.0), - &r(5.0, 5.0, 10.0, 10.0) - )); - } - - #[test] - fn touching_edge_rects_are_considered_overlapping() { - assert!(rects_overlap( - &r(0.0, 0.0, 10.0, 10.0), - &r(10.0, 0.0, 10.0, 10.0) - )); - } - - #[test] - fn non_overlapping_rects_do_not_intersect() { - assert!(!rects_overlap( - &r(0.0, 0.0, 5.0, 5.0), - &r(10.0, 10.0, 5.0, 5.0) - )); - } - - #[test] - fn contained_rect_is_always_overlapping() { - assert!(rects_overlap( - &r(2.0, 2.0, 3.0, 3.0), - &r(0.0, 0.0, 10.0, 10.0) - )); - } +#[cfg(target_os = "macos")] +fn read_bounds_with_deadline( + element: &AXElement, + deadline: std::time::Instant, +) -> Result<Option<Rect>, agent_desktop_core::AdapterError> { + #[cfg(test)] + if element.0.is_null() { + return Ok(None); + } + let position = super::resolve_ax_read::read_point( + element, + accessibility_sys::kAXPositionAttribute, + deadline, + )?; + let size = + super::resolve_ax_read::read_size(element, accessibility_sys::kAXSizeAttribute, deadline)?; + Ok(position + .zip(size) + .and_then(|(position, size)| crate::tree::element_bounds::rect_from_parts(position, size))) } diff --git a/crates/macos/src/tree/resolve_classify.rs b/crates/macos/src/tree/resolve_classify.rs index 5527bf5..69f82c4 100644 --- a/crates/macos/src/tree/resolve_classify.rs +++ b/crates/macos/src/tree/resolve_classify.rs @@ -1,53 +1,84 @@ -use agent_desktop_core::{adapter::NativeHandle, error::AdapterError, refs::RefEntry}; +use agent_desktop_core::{ + AdapterError, NativeHandle, RefEntry, ref_identity::has_meaningful_identity, +}; use super::AXElement; #[cfg(target_os = "macos")] -use super::attributes::copy_string_attr; -#[cfg(target_os = "macos")] -use super::element::resolve_element_name; -#[cfg(target_os = "macos")] -use super::resolve_bounds::bounds_match; +use super::resolve_bounds::bounds_match_with_deadline; #[cfg(target_os = "macos")] pub(super) fn classify_candidates( mut matches: Vec<AXElement>, entry: &RefEntry, source_window_verified: bool, + deadline: std::time::Instant, ) -> Result<NativeHandle, AdapterError> { match matches.len() { 0 => Err(AdapterError::element_not_found("element")), 1 => { let candidate = matches.remove(0); - if source_window_verified || verified_bounds_match(&candidate, entry) { + if candidate_is_sufficiently_verified( + &candidate, + entry, + source_window_verified, + deadline, + )? { retained_handle(candidate) } else { Err(AdapterError::element_not_found("element")) } } - _ => classify_ambiguous_candidates(matches, entry), + _ => classify_ambiguous_candidates(matches, entry, deadline), } } #[cfg(target_os = "macos")] -fn verified_bounds_match(candidate: &AXElement, entry: &RefEntry) -> bool { - entry.bounds_hash.is_some() && bounds_match(candidate, entry) +fn candidate_is_sufficiently_verified( + candidate: &AXElement, + entry: &RefEntry, + source_window_verified: bool, + deadline: std::time::Instant, +) -> Result<bool, AdapterError> { + if source_window_verified && !candidate_requires_bounds(entry) { + return Ok(true); + } + verified_bounds_match(candidate, entry, deadline) +} + +fn candidate_requires_bounds(entry: &RefEntry) -> bool { + !has_meaningful_identity(entry) +} + +#[cfg(target_os = "macos")] +fn verified_bounds_match( + candidate: &AXElement, + entry: &RefEntry, + deadline: std::time::Instant, +) -> Result<bool, AdapterError> { + if entry.geometry.bounds_hash.is_none() { + return Ok(false); + } + bounds_match_with_deadline(candidate, entry, deadline) } #[cfg(target_os = "macos")] fn classify_ambiguous_candidates( matches: Vec<AXElement>, entry: &RefEntry, + deadline: std::time::Instant, ) -> Result<NativeHandle, AdapterError> { - let mut bounds_matches: Vec<_> = matches - .iter() - .filter(|candidate| verified_bounds_match(candidate, entry)) - .cloned() - .collect(); - if entry.bounds_hash.is_some() && bounds_matches.is_empty() { - return Err(AdapterError::element_not_found("element")); - } - if bounds_matches.len() == 1 { - return retained_handle(bounds_matches.remove(0)); + if entry.geometry.bounds_hash.is_some() { + let mut bounds_matches = Vec::new(); + for candidate in &matches { + if verified_bounds_match(candidate, entry, deadline)? { + bounds_matches.push(candidate.clone()); + } + } + match bounds_matches.len() { + 0 => {} + 1 => return retained_handle(bounds_matches.remove(0)), + _ => {} + } } let count = matches.len(); Err(AdapterError::ambiguous_target(format!( @@ -56,13 +87,14 @@ fn classify_ambiguous_candidates( )) .with_details(serde_json::json!({ "candidate_count": count, - "role": entry.role, - "name": entry.name, - "description": entry.description, - "source_app": entry.source_app, - "source_window_id": entry.source_window_id, - "source_window_title": entry.source_window_title, - "candidates": candidate_summaries(&matches) + "candidate_summaries_truncated": count > 10, + "role": entry.identity.role, + "name": entry.identity.name, + "description": entry.identity.description, + "source_app": entry.source.source_app, + "source_window_id": entry.source.source_window_id, + "source_window_title": entry.source.source_window_title, + "candidates": candidate_summaries(&matches, entry) }))) } @@ -70,9 +102,9 @@ fn classify_ambiguous_candidates( pub(super) fn identity_summary_for_message(entry: &RefEntry) -> String { format!( "role={}, name_chars={}, description_chars={}", - entry.role, - text_len(entry.name.as_deref()), - text_len(entry.description.as_deref()) + entry.identity.role, + text_len(entry.identity.name.as_deref()), + text_len(entry.identity.description.as_deref()) ) } @@ -83,37 +115,26 @@ fn text_len(value: Option<&str>) -> usize { #[cfg(target_os = "macos")] fn retained_handle(candidate: AXElement) -> Result<NativeHandle, AdapterError> { - use core_foundation::base::{CFRetain, CFTypeRef}; if candidate.0.is_null() { #[cfg(test)] return Ok(NativeHandle::null()); #[cfg(not(test))] return Err(AdapterError::element_not_found("element")); } - unsafe { CFRetain(candidate.0 as CFTypeRef) }; - Ok(unsafe { NativeHandle::from_ptr(candidate.0 as *const _) }) + Ok(candidate.into_native_handle()) } #[cfg(target_os = "macos")] -fn candidate_summaries(matches: &[AXElement]) -> Vec<serde_json::Value> { +fn candidate_summaries(matches: &[AXElement], entry: &RefEntry) -> Vec<serde_json::Value> { matches .iter() .take(10) .enumerate() - .map(|(index, element)| { - let ax_role = copy_string_attr(element, accessibility_sys::kAXRoleAttribute); - let (role, label) = - crate::tree::roles::normalized_role_and_label(element, ax_role.as_deref()); - let name = label.or_else(|| resolve_element_name(element)); - let description = copy_string_attr(element, accessibility_sys::kAXDescriptionAttribute); - let bounds = crate::tree::read_bounds(element); + .map(|(index, _)| { serde_json::json!({ "index": index, - "role": role, - "name": name, - "description": description, - "bounds": bounds, - "bounds_hash": bounds.as_ref().map(|bounds| bounds.bounds_hash()) + "role": entry.identity.role, + "identity": "matched", }) }) .collect() diff --git a/crates/macos/src/tree/resolve_deadline.rs b/crates/macos/src/tree/resolve_deadline.rs deleted file mode 100644 index 5e13a73..0000000 --- a/crates/macos/src/tree/resolve_deadline.rs +++ /dev/null @@ -1,31 +0,0 @@ -use agent_desktop_core::error::{AdapterError, ErrorCode}; -use std::time::{Duration, Instant}; - -#[cfg(target_os = "macos")] -pub(super) fn remaining_before_deadline(deadline: Instant) -> Result<Duration, AdapterError> { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Err(timeout_error()); - } - Ok(remaining) -} - -#[cfg(target_os = "macos")] -pub(super) fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { - remaining_before_deadline(deadline).map(|_| ()) -} - -#[cfg(target_os = "macos")] -pub(super) fn timeout_error() -> AdapterError { - AdapterError::new(ErrorCode::Timeout, "Element resolution timed out") - .with_suggestion("Retry the command, or run 'snapshot' if the UI changed.") -} - -#[cfg(target_os = "macos")] -pub(super) fn sleep_before_retry(deadline: Instant) { - if let Ok(remaining) = remaining_before_deadline(deadline) { - if remaining > Duration::from_millis(100) { - std::thread::sleep(remaining.min(Duration::from_millis(75))); - } - } -} diff --git a/crates/macos/src/tree/resolve_errors.rs b/crates/macos/src/tree/resolve_errors.rs new file mode 100644 index 0000000..906ae79 --- /dev/null +++ b/crates/macos/src/tree/resolve_errors.rs @@ -0,0 +1,44 @@ +use agent_desktop_core::{AdapterError, ErrorCode, RefEntry}; + +pub(super) fn native_identifier_role_reuse(entry: &RefEntry) -> AdapterError { + AdapterError::stale_ref( + "Saved native identifier was found only on a different accessibility role", + ) + .with_details(serde_json::json!({ + "kind": "native_identifier_role_reuse", + "expected_role": entry.identity.role, + "complete": true, + "retryable": false, + })) +} + +pub(super) fn identity_unknown(entry: &RefEntry) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Strict resolution could not determine candidate identity from the live accessibility evidence", + ) + .with_suggestion("Retry after the target application finishes updating its accessibility tree") + .with_details(serde_json::json!({ + "kind": "resolution_identity_unknown", + "role": entry.identity.role, + "complete": false, + "retryable": true, + })) +} + +pub(super) fn incomplete_traversal(phase: &str, depth: u8) -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + format!( + "Strict element resolution observed an incomplete accessibility tree during {phase}" + ), + ) + .with_suggestion("Retry after the target application finishes updating its accessibility tree") + .with_details(serde_json::json!({ + "kind": "resolution_traversal_incomplete", + "phase": phase, + "depth": depth, + "complete": false, + "retryable": true, + })) +} diff --git a/crates/macos/src/tree/resolve_identity.rs b/crates/macos/src/tree/resolve_identity.rs deleted file mode 100644 index 95de21c..0000000 --- a/crates/macos/src/tree/resolve_identity.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(super) use agent_desktop_core::ref_identity::{ - bounded_window_fallback_allowed, has_meaningful_identity, identity_matches, -}; diff --git a/crates/macos/src/tree/resolve_read_context.rs b/crates/macos/src/tree/resolve_read_context.rs new file mode 100644 index 0000000..578ce6f --- /dev/null +++ b/crates/macos/src/tree/resolve_read_context.rs @@ -0,0 +1,17 @@ +pub(super) struct ResolveReadContext { + pub(super) stats: agent_desktop_core::LocatorStats, + pub(super) usage: crate::tree::observation_usage::ObservationUsage, + pub(super) deadline: std::time::Instant, +} + +impl ResolveReadContext { + pub(super) fn new(deadline: std::time::Instant) -> Self { + Self { + stats: agent_desktop_core::LocatorStats::default(), + usage: crate::tree::observation_usage::ObservationUsage::new( + agent_desktop_core::ObservationBudget::default(), + ), + deadline, + } + } +} diff --git a/crates/macos/src/tree/resolve_roots.rs b/crates/macos/src/tree/resolve_roots.rs index c4fc1d0..6d5fe5f 100644 --- a/crates/macos/src/tree/resolve_roots.rs +++ b/crates/macos/src/tree/resolve_roots.rs @@ -1,14 +1,9 @@ -use agent_desktop_core::{adapter::SnapshotSurface, error::AdapterError, refs::RefEntry}; -use std::time::Instant; +use agent_desktop_core::{AdapterError, RefEntry, SnapshotSurface}; use super::AXElement; -use super::attributes::{ - copy_ax_array, copy_element_attr, copy_i64_attr, copy_string_attr, set_messaging_timeout, -}; use super::element::element_for_pid; use super::element_dedupe::ElementDedupe; -use super::resolve_deadline::{ensure_before_deadline, remaining_before_deadline}; -use super::resolve_identity::bounded_window_fallback_allowed; +use super::resolve_read_context::ResolveReadContext; #[cfg(target_os = "macos")] pub(super) struct CandidateRoots { @@ -17,237 +12,379 @@ pub(super) struct CandidateRoots { } #[cfg(target_os = "macos")] -pub(super) fn path_candidate_roots( +pub(super) fn candidate_roots( entry: &RefEntry, - deadline: Instant, + context: &mut ResolveReadContext, ) -> Result<CandidateRoots, AdapterError> { - if entry.bounds_hash.is_some() { - return candidate_roots(entry, deadline); + let deadline = context.deadline; + crate::tree::locator_deadline::remaining(deadline)?; + let pid = crate::system::process_identity::to_pid_t(entry.process.pid)?; + let application = element_for_pid(pid); + if entry.source.source_surface != SnapshotSurface::Window { + verify_source_application(&application, entry, context)?; + return source_surface_scoped_roots(entry, deadline); } - let roots: Vec<_> = scoped_surface_root(entry, deadline)?.into_iter().collect(); + if source_window_scope_required(entry) && entry.source.source_window_id.is_some() { + return source_window_scoped_roots(&application, entry, context); + } + verify_source_application(&application, entry, context)?; + if source_window_scope_required(entry) { + return source_window_scoped_roots(&application, entry, context); + } + + let mut roots = Vec::new(); + let mut dedupe = ElementDedupe; + let windows = read_root_array(&application, "AXWindows", context)?; + if windows.as_ref().is_some_and(|windows| !windows.is_empty()) { + add_array(&mut roots, &mut dedupe, windows); + } else { + add_optional_element( + &mut roots, + &mut dedupe, + super::resolve_ax_read::read_element(&application, "AXFocusedWindow", deadline)?, + ); + add_optional_element( + &mut roots, + &mut dedupe, + super::resolve_ax_read::read_element(&application, "AXMainWindow", deadline)?, + ); + } + add_array( + &mut roots, + &mut dedupe, + read_root_array(&application, "AXMenus", context)?, + ); + add_array( + &mut roots, + &mut dedupe, + read_root_array(&application, "AXChildren", context)?, + ); + crate::tree::locator_deadline::remaining(deadline)?; Ok(CandidateRoots { - scope_verified: source_window_scope_required(entry) && !roots.is_empty(), roots, + scope_verified: false, }) } #[cfg(target_os = "macos")] -pub(super) fn candidate_roots( +fn source_surface_scoped_roots( entry: &RefEntry, - deadline: Instant, + deadline: std::time::Instant, ) -> Result<CandidateRoots, AdapterError> { - if source_window_scope_required(entry) { - return source_window_scoped_roots(entry, deadline); - } - - let root = element_for_pid(entry.pid); - prepare_for_read(&root, deadline)?; - let mut roots = Vec::new(); - let mut dedupe = ElementDedupe; - let windows = copy_ax_array(&root, "AXWindows").unwrap_or_default(); - if let Some(window) = exact_source_window_from_windows(&windows, entry, deadline)? { - dedupe.push(&mut roots, window); - } - prepare_for_read(&root, deadline)?; - if let Some(focused) = copy_element_attr(&root, "AXFocusedWindow") { - dedupe.push(&mut roots, focused); - } - prepare_for_read(&root, deadline)?; - if let Some(main) = copy_element_attr(&root, "AXMainWindow") { - dedupe.push(&mut roots, main); - } - for window in windows { - dedupe.push(&mut roots, window); - } - ensure_before_deadline(deadline)?; - if let Some(menubar) = crate::tree::menubar_for_pid(entry.pid) { - dedupe.push(&mut roots, menubar); - } - ensure_before_deadline(deadline)?; - if let Some(menu) = crate::tree::menu_element_for_pid(entry.pid) { - dedupe.push(&mut roots, menu); - } - if roots.is_empty() { - roots.push(root); - } + let pid = crate::system::process_identity::to_pid_t(entry.process.pid)?; + let root = match entry.source.source_surface { + SnapshotSurface::Focused => super::surfaces::focused_surface_for_pid(pid, deadline)?, + SnapshotSurface::Menu => super::surfaces::menu_element_for_pid(pid, deadline)?, + SnapshotSurface::Menubar => super::surfaces::menubar_for_pid(pid, deadline)?, + SnapshotSurface::Sheet => super::surfaces::sheet_for_pid(pid, deadline)?, + SnapshotSurface::Popover => super::surfaces::popover_for_pid(pid, deadline)?, + SnapshotSurface::Alert => super::surfaces::alert_for_pid(pid, deadline)?, + SnapshotSurface::Window => None, + _ => None, + }; + let Some(root) = root else { + return Err( + AdapterError::element_not_found("saved source surface").with_details( + serde_json::json!({ + "kind": "source_surface_absent", + "surface": entry.source.source_surface.as_str(), + "complete": true, + "retryable": true, + }), + ), + ); + }; Ok(CandidateRoots { - roots, - scope_verified: false, + roots: vec![root], + scope_verified: true, }) } #[cfg(target_os = "macos")] fn source_window_scoped_roots( + application: &AXElement, entry: &RefEntry, - deadline: Instant, + context: &mut ResolveReadContext, ) -> Result<CandidateRoots, AdapterError> { - let Some(windows) = windows_for_pid(entry.pid, deadline)? else { - return Ok(CandidateRoots { - roots: Vec::new(), - scope_verified: false, - }); - }; - if let Some(window) = window_by_number(&windows, source_window_number(entry), deadline)? { - return Ok(CandidateRoots { - roots: vec![window], - scope_verified: true, - }); - } - if let Some(window) = window_by_title(&windows, entry.source_window_title.as_deref(), deadline)? - { - return Ok(CandidateRoots { - roots: vec![window], - scope_verified: false, - }); - } - if bounded_window_fallback_allowed(entry) { - let roots = fallback_replacement_window_roots(&windows, deadline)?; - if !roots.is_empty() { - return Ok(CandidateRoots { - roots, - scope_verified: false, - }); + let deadline = context.deadline; + if let Some(id) = entry.source.source_window_id.as_deref() { + if source_window_number(entry).is_none() { + return Err( + AdapterError::element_not_found("saved source window").with_details( + serde_json::json!({ + "kind": "source_window_identity_invalid", + "source_window_id": id, + "complete": true, + "retryable": false, + }), + ), + ); } + crate::system::window_resolve::verify_window_identity_until( + id, + crate::system::window_resolve::WindowIdentityEvidence { + pid: crate::system::process_identity::to_pid_t(entry.process.pid)?, + app: entry.source.source_app.as_deref(), + process_instance: entry.process.process_instance.as_deref(), + title: entry.source.source_window_title.as_deref(), + bounds_hash: entry.source.source_window_bounds_hash, + }, + deadline, + )?; } + let windows = read_root_array(application, "AXWindows", context)?.unwrap_or_default(); + let window = if entry.source.source_window_id.is_some() { + window_by_number(&windows, entry, context)? + } else { + window_by_title( + &windows, + entry.source.source_window_title.as_deref(), + context, + )? + }; + let scope_verified = + source_scope_verified(entry.source.source_window_id.as_deref(), window.is_some()); Ok(CandidateRoots { - roots: Vec::new(), - scope_verified: false, + scope_verified, + roots: window.into_iter().collect(), }) } -#[cfg(target_os = "macos")] -fn scoped_surface_root( - entry: &RefEntry, - deadline: Instant, -) -> Result<Option<AXElement>, AdapterError> { - ensure_before_deadline(deadline)?; - let root = match entry.source_surface { - SnapshotSurface::Window if source_window_scope_required(entry) => { - exact_source_window_number_root(entry, deadline)? - } - SnapshotSurface::Window => exact_source_window_root(entry, deadline)?, - SnapshotSurface::Focused => crate::tree::focused_surface_for_pid(entry.pid), - SnapshotSurface::Menu => crate::tree::menu_element_for_pid(entry.pid), - SnapshotSurface::Menubar => crate::tree::menubar_for_pid(entry.pid), - SnapshotSurface::Sheet => crate::tree::sheet_for_pid(entry.pid), - SnapshotSurface::Popover => crate::tree::popover_for_pid(entry.pid), - SnapshotSurface::Alert => crate::tree::alert_for_pid(entry.pid), - _ => return Err(AdapterError::not_supported("snapshot surface")), - }; - Ok(root) -} - -#[cfg(target_os = "macos")] -fn exact_source_window_number_root( - entry: &RefEntry, - deadline: Instant, -) -> Result<Option<AXElement>, AdapterError> { - let Some(windows) = windows_for_pid(entry.pid, deadline)? else { - return Ok(None); - }; - window_by_number(&windows, source_window_number(entry), deadline) -} - -#[cfg(target_os = "macos")] -fn exact_source_window_root( - entry: &RefEntry, - deadline: Instant, -) -> Result<Option<AXElement>, AdapterError> { - let Some(windows) = windows_for_pid(entry.pid, deadline)? else { - return Ok(None); - }; - exact_source_window_from_windows(&windows, entry, deadline) -} - -#[cfg(target_os = "macos")] -fn exact_source_window_from_windows( - windows: &[AXElement], - entry: &RefEntry, - deadline: Instant, -) -> Result<Option<AXElement>, AdapterError> { - if let Some(window) = window_by_number(windows, source_window_number(entry), deadline)? { - return Ok(Some(window)); - } - window_by_title(windows, entry.source_window_title.as_deref(), deadline) -} - -#[cfg(target_os = "macos")] -fn windows_for_pid(pid: i32, deadline: Instant) -> Result<Option<Vec<AXElement>>, AdapterError> { - let root = element_for_pid(pid); - prepare_for_read(&root, deadline)?; - Ok(copy_ax_array(&root, "AXWindows")) +pub(super) fn source_scope_verified(source_window_id: Option<&str>, matched: bool) -> bool { + source_window_id.is_some() && matched } #[cfg(target_os = "macos")] fn window_by_number( windows: &[AXElement], - source_window_number: Option<i64>, - deadline: Instant, + entry: &RefEntry, + context: &mut ResolveReadContext, ) -> Result<Option<AXElement>, AdapterError> { - let Some(source_window_number) = source_window_number else { + let deadline = context.deadline; + let Some(source_window_number) = source_window_number(entry) else { return Ok(None); }; - for win in windows { - prepare_for_read(win, deadline)?; - if copy_i64_attr(win, "AXWindowNumber") == Some(source_window_number) { - return Ok(Some(win.clone())); + let mut bridge_unavailable = None; + let mut found = None; + let mut match_count = 0_usize; + for window in windows { + crate::tree::locator_deadline::remaining(deadline)?; + match crate::system::window_resolve::ax_window_id_with_deadline(window, deadline) { + Ok(Some(actual)) if actual == source_window_number => { + match_count += 1; + if found.is_none() { + found = Some(window.clone()); + } + } + Ok(_) => {} + Err(error) if crate::system::window_bridge::is_unavailable(&error) => { + bridge_unavailable = Some(error); + break; + } + Err(error) => return Err(error), } } - Ok(None) + if let Some(error) = bridge_unavailable { + return Err(error); + } + require_unique_window_number_match(match_count, source_window_number)?; + found + .map(Some) + .ok_or_else(|| window_bridge_miss_error(entry)) +} + +#[cfg(target_os = "macos")] +pub(super) fn require_unique_window_number_match( + match_count: usize, + window_number: i64, +) -> Result<(), AdapterError> { + if match_count > 1 { + return Err(AdapterError::ambiguous_target(format!( + "Multiple AX windows matched verified CoreGraphics window w-{window_number}" + )) + .with_details(serde_json::json!({ + "kind": "source_window_number_ambiguous", + "source_window_id": format!("w-{window_number}"), + "candidate_count": match_count, + }))); + } + Ok(()) } #[cfg(target_os = "macos")] fn window_by_title( windows: &[AXElement], source_window_title: Option<&str>, - deadline: Instant, + context: &mut ResolveReadContext, ) -> Result<Option<AXElement>, AdapterError> { - let Some(source_window_title) = source_window_title else { + let Some(source_window_title) = source_window_title.filter(|title| !title.is_empty()) else { return Ok(None); }; let mut found = None; - for win in windows { - prepare_for_read(win, deadline)?; - if copy_string_attr(win, "AXTitle").as_deref() == Some(source_window_title) { - if found.is_some() { - return Ok(None); - } - found = Some(win.clone()); + for window in windows { + crate::tree::locator_deadline::remaining(context.deadline)?; + if super::resolve_ax_read::read_string_with_usage( + window, + "AXTitle", + context.deadline, + &mut context.usage, + )? + .as_deref() + != Some(source_window_title) + { + continue; } + if found.is_some() { + return Err(AdapterError::ambiguous_target(format!( + "Multiple windows matched the saved title '{source_window_title}'" + )) + .with_details(serde_json::json!({ + "kind": "source_window_title_ambiguous", + "title": source_window_title, + "candidate_count_at_least": 2, + }))); + } + found = Some(window.clone()); } Ok(found) } #[cfg(target_os = "macos")] -pub(super) fn fallback_replacement_window_roots( - windows: &[AXElement], - deadline: Instant, -) -> Result<Vec<AXElement>, AdapterError> { - let mut roots = Vec::new(); - let mut dedupe = ElementDedupe; - for win in windows { - prepare_for_read(win, deadline)?; - dedupe.push(&mut roots, win.clone()); +fn window_bridge_miss_error(entry: &RefEntry) -> AdapterError { + AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + "The verified CoreGraphics window could not be matched to one live AXWindow", + ) + .with_suggestion("Retry after the application finishes updating its accessibility windows") + .with_details(serde_json::json!({ + "kind": "resolution_window_bridge_miss", + "source_window_id": entry.source.source_window_id, + "complete": false, + "retryable": true, + })) +} + +#[cfg(target_os = "macos")] +fn verify_source_application( + application: &AXElement, + entry: &RefEntry, + context: &mut ResolveReadContext, +) -> Result<(), AdapterError> { + let Some(expected) = entry + .source + .source_app + .as_deref() + .filter(|name| !name.is_empty()) + else { + return Ok(()); + }; + let actual = super::resolve_ax_read::read_string_with_usage( + application, + "AXTitle", + context.deadline, + &mut context.usage, + )?; + if actual + .as_deref() + .is_some_and(|actual| actual.eq_ignore_ascii_case(expected)) + { + return Ok(()); } - Ok(roots) + Err( + AdapterError::element_not_found("source application").with_details(serde_json::json!({ + "kind": "source_process_identity", + "pid": entry.process.pid, + "expected_app": expected, + "actual_app": actual, + "complete": true, + "retryable": false, + })), + ) +} + +#[cfg(target_os = "macos")] +fn add_optional_element( + roots: &mut Vec<AXElement>, + dedupe: &mut ElementDedupe, + element: Option<AXElement>, +) { + if let Some(element) = element { + dedupe.push(roots, element); + } +} + +#[cfg(target_os = "macos")] +fn add_array( + roots: &mut Vec<AXElement>, + dedupe: &mut ElementDedupe, + elements: Option<Vec<AXElement>>, +) { + for element in elements.unwrap_or_default() { + dedupe.push(roots, element); + } +} + +#[cfg(target_os = "macos")] +fn read_root_array( + element: &AXElement, + attribute: &str, + context: &mut ResolveReadContext, +) -> Result<Option<Vec<AXElement>>, AdapterError> { + let max_elements = context.usage.child_capacity(); + let read = super::query::child_read::read_attribute_children( + element, + attribute, + max_elements, + context.deadline, + ); + context.stats.reads.counts.child_reads += read.status.attempts; + context.stats.reads.health.cannot_complete += read.status.health.cannot_complete; + context.stats.reads.health.native_read_failures += read.status.health.native_read_failures; + context.stats.reads.health.deadline_exhausted += read.status.health.deadline_exhausted; + context.stats.traversal.limits.child_count_changes += u64::from(read.status.count_changed); + context + .usage + .note_child_demand(read.total_count, &mut context.stats); + context.usage.claim_edges(read.elements.len()); + if read.status.api_disabled { + return Err(AdapterError::permission_denied()); + } + if read.status.invalid_element || !read.complete || read.truncated() { + return Err(AdapterError::new( + agent_desktop_core::ErrorCode::AppUnresponsive, + format!("Strict resolution could not read {attribute} completely"), + ) + .with_details(serde_json::json!({ + "kind": "resolution_root_array_incomplete", + "attribute": attribute, + "complete": false, + "total_count": read.total_count, + "loaded_count": read.elements.len(), + "count_changed": read.status.count_changed, + "retryable": true, + }))); + } + Ok((read.total_count > 0).then_some(read.elements)) } #[cfg(target_os = "macos")] pub(super) fn source_window_scope_required(entry: &RefEntry) -> bool { - matches!(entry.source_surface, SnapshotSurface::Window) && source_window_number(entry).is_some() + matches!(entry.source.source_surface, SnapshotSurface::Window) + && (entry.source.source_window_id.is_some() + || entry + .source + .source_window_title + .as_deref() + .is_some_and(|title| !title.is_empty())) } pub(super) fn source_window_number(entry: &RefEntry) -> Option<i64> { - entry + let number = entry + .source .source_window_id .as_deref()? .strip_prefix("w-")? .parse() - .ok() -} - -#[cfg(target_os = "macos")] -fn prepare_for_read(element: &AXElement, deadline: Instant) -> Result<(), AdapterError> { - set_messaging_timeout(element, remaining_before_deadline(deadline)?); - Ok(()) + .ok()?; + (number > 0).then_some(number) } diff --git a/crates/macos/src/tree/resolve_search.rs b/crates/macos/src/tree/resolve_search.rs index c5d4425..b29ee27 100644 --- a/crates/macos/src/tree/resolve_search.rs +++ b/crates/macos/src/tree/resolve_search.rs @@ -1,101 +1,180 @@ -use agent_desktop_core::{adapter::NativeHandle, error::AdapterError, refs::RefEntry}; +use agent_desktop_core::{ + AdapterError, EvidenceRequirements, IdentityMatch, NativeHandle, RefEntry, + ref_identity::{has_stable_text_identity, identity_match}, +}; use rustc_hash::FxHashSet; -use std::time::Instant; use super::AXElement; -#[cfg(target_os = "macos")] -use super::attributes::{copy_ax_array, copy_string_attr, set_messaging_timeout}; -#[cfg(target_os = "macos")] -use super::element::{child_attributes, resolve_element_name}; use super::element_dedupe::ElementDedupe; -use super::resolve_bounds::should_prune_by_bounds; -#[cfg(target_os = "macos")] use super::resolve_classify::classify_candidates; -#[cfg(target_os = "macos")] -use super::resolve_deadline::{ensure_before_deadline, remaining_before_deadline}; -use super::resolve_identity::has_meaningful_identity; -#[cfg(target_os = "macos")] -use super::resolve_identity::identity_matches; +pub(super) use super::resolve_errors::incomplete_traversal as incomplete_traversal_error; +use super::resolve_errors::{ + identity_unknown as identity_unknown_error, + native_identifier_role_reuse as native_identifier_role_reuse_error, +}; +use super::resolve_read_context::ResolveReadContext; #[cfg(target_os = "macos")] pub(super) fn find_entry_by_path( roots: &[AXElement], entry: &RefEntry, source_window_verified: bool, - deadline: Instant, + read_context: &mut ResolveReadContext, ) -> Result<NativeHandle, AdapterError> { - if entry.path.is_empty() { - return Err(AdapterError::element_not_found("element")); - } - ensure_before_deadline(deadline)?; - + crate::tree::locator_deadline::remaining(read_context.deadline)?; let mut matches = Vec::new(); let mut dedupe = ElementDedupe; + let mut incomplete = None; for root in roots { - ensure_before_deadline(deadline)?; + crate::tree::locator_deadline::remaining(read_context.deadline)?; + let Some(candidate) = element_at_path(root, &entry.scope.path, read_context)? else { + continue; + }; + let identity = candidate_identity(&candidate, entry, read_context)?; + match identity { + IdentityMatch::Match => { + dedupe.push(&mut matches, candidate); + } + IdentityMatch::Unknown => { + incomplete.get_or_insert_with(|| identity_unknown_error(entry)); + } + IdentityMatch::NoMatch => {} + } if should_stop_collecting(matches.len(), entry) { break; } - let Some(candidate) = element_at_path(root, &entry.path, deadline)? else { - continue; - }; - if element_matches_entry(&candidate, entry) { - dedupe.push(&mut matches, candidate); - } } - - classify_candidates(matches, entry, source_window_verified) + if should_stop_collecting(matches.len(), entry) { + return classify_candidates( + matches, + entry, + source_window_verified, + read_context.deadline, + ); + } + if let Some(error) = incomplete { + return Err(error); + } + classify_candidates( + matches, + entry, + source_window_verified, + read_context.deadline, + ) } #[cfg(target_os = "macos")] fn element_at_path( root: &AXElement, path: &[usize], - deadline: Instant, + context: &mut ResolveReadContext, ) -> Result<Option<AXElement>, AdapterError> { - let mut current = root.clone(); - let mut seen = FxHashSet::default(); - for idx in path { - ensure_before_deadline(deadline)?; - set_messaging_timeout(¤t, remaining_before_deadline(deadline)?); - let ax_role = copy_string_attr(¤t, accessibility_sys::kAXRoleAttribute); - let children = resolve_children(¤t, ax_role.as_deref(), deadline, &mut seen)?; - let Some(child) = children.get(*idx) else { + walk_finite_path(root.clone(), path, |current, index| { + crate::tree::locator_deadline::remaining(context.deadline)?; + if !context.usage.claim_node() { + context.stats.traversal.limits.node_hits += 1; + return Err(incomplete_traversal_error("path_node_budget", 0)); + } + context.stats.reads.counts.attribute_batches += 1; + context.stats.reads.counts.attributes_requested += 1; + let role = super::resolve_ax_read::read_string(¤t, "AXRole", context.deadline)?; + let mut read = crate::tree::query::child_read::read_child_at( + ¤t, + role.as_deref(), + index, + context.deadline, + ); + record_path_child_read(&read, context); + crate::tree::locator_deadline::remaining(context.deadline)?; + if read.status.api_disabled { + return Err(AdapterError::permission_denied()); + } + if !read.complete { + return Err(incomplete_traversal_error("path_children", 0)); + } + context + .usage + .claim_edges(usize::from(!read.elements.is_empty())); + Ok(read.elements.pop()) + }) +} + +pub(super) fn walk_finite_path<T>( + mut current: T, + path: &[usize], + mut read_child: impl FnMut(T, usize) -> Result<Option<T>, AdapterError>, +) -> Result<Option<T>, AdapterError> { + for index in path { + let Some(child) = read_child(current, *index)? else { return Ok(None); }; - current = child.clone(); + current = child; } Ok(Some(current)) } +#[cfg(target_os = "macos")] +fn record_path_child_read( + read: &crate::tree::query::child_read::ChildRead, + context: &mut ResolveReadContext, +) { + context.stats.reads.counts.child_reads += read.status.attempts; + context.stats.reads.health.cannot_complete += read.status.health.cannot_complete; + context.stats.reads.health.native_read_failures += read.status.health.native_read_failures; + context.stats.reads.health.deadline_exhausted += read.status.health.deadline_exhausted; + context.stats.traversal.limits.child_count_changes += u64::from(read.status.count_changed); + context.stats.traversal.limits.child_hits += u64::from(read.status.cursor_stalled); +} + #[cfg(target_os = "macos")] pub(super) fn find_entry_in_roots( roots: &[AXElement], entry: &RefEntry, resolve_depth: u8, source_window_verified: bool, - deadline: Instant, + read_context: &mut ResolveReadContext, ) -> Result<NativeHandle, AdapterError> { let mut matches = Vec::new(); let mut seen_matches = ElementDedupe; - let mut child_scratch = FxHashSet::default(); + let mut incomplete = None; + let mut identifier_role_reuse = false; for root in roots { if should_stop_collecting(matches.len(), entry) { break; } - let mut visited = FxHashSet::default(); + let mut ancestors = FxHashSet::default(); let mut context = CollectContext { entry, max_depth: resolve_depth, - ancestors: &mut visited, + ancestors: &mut ancestors, seen_matches: &mut seen_matches, matches: &mut matches, - child_scratch: &mut child_scratch, - deadline, + incomplete: &mut incomplete, + identifier_role_reuse: &mut identifier_role_reuse, + read_context, }; collect_elements_recursive(root, 0, &mut context)?; } - classify_candidates(matches, entry, source_window_verified) + if should_stop_collecting(matches.len(), entry) { + return classify_candidates( + matches, + entry, + source_window_verified, + read_context.deadline, + ); + } + if let Some(error) = incomplete { + return Err(error); + } + if identifier_role_reuse && matches.is_empty() { + return Err(native_identifier_role_reuse_error(entry)); + } + classify_candidates( + matches, + entry, + source_window_verified, + read_context.deadline, + ) } #[cfg(target_os = "macos")] @@ -105,109 +184,203 @@ struct CollectContext<'a> { ancestors: &'a mut FxHashSet<usize>, seen_matches: &'a mut ElementDedupe, matches: &'a mut Vec<AXElement>, - child_scratch: &'a mut FxHashSet<usize>, - deadline: Instant, + incomplete: &'a mut Option<AdapterError>, + identifier_role_reuse: &'a mut bool, + read_context: &'a mut ResolveReadContext, } #[cfg(target_os = "macos")] fn collect_elements_recursive( - el: &AXElement, + element: &AXElement, depth: u8, context: &mut CollectContext<'_>, ) -> Result<(), AdapterError> { - use accessibility_sys::kAXRoleAttribute; - if should_stop_collecting(context.matches.len(), context.entry) { return Ok(()); } - ensure_before_deadline(context.deadline)?; - - let ptr_key = el.0 as usize; - if !context.ancestors.insert(ptr_key) { + crate::tree::locator_deadline::remaining(context.read_context.deadline)?; + let pointer = element.0 as usize; + if !context.ancestors.insert(pointer) { return Ok(()); } - let ax_role = copy_string_attr(el, kAXRoleAttribute); - let (normalized, promoted_label) = - crate::tree::roles::normalized_role_and_label(el, ax_role.as_deref()); - - if normalized == context.entry.role - && element_identity_matches(el, context.entry, promoted_label) - && context.seen_matches.push_clone(context.matches, el) - && should_stop_collecting(context.matches.len(), context.entry) - { - context.ancestors.remove(&ptr_key); + let read = read_node( + element, + identity_requirements(context.entry), + context.read_context, + )?; + if read.invalid_element { + mark_incomplete( + context, + incomplete_traversal_error("invalid_element", depth), + ); + context.ancestors.remove(&pointer); return Ok(()); } - if depth < context.max_depth && !should_prune_for_resolution(el, context.entry, depth) { - let children = resolve_children( - el, - ax_role.as_deref(), - context.deadline, - context.child_scratch, - )?; - for child in &children { - collect_elements_recursive(child, depth + 1, context)?; + if native_identifier_reused_by_different_role(context.entry, &read.evidence) { + *context.identifier_role_reuse = true; + } + + match read.evidence.role.known() { + Some(role) => { + if role == &context.entry.identity.role { + match match_native_or_text_identity(context.entry, &read.evidence) { + IdentityMatch::Match => { + context.seen_matches.push_clone(context.matches, element); + } + IdentityMatch::Unknown => { + if provisional_geometry_candidate(context.entry) { + context.seen_matches.push_clone(context.matches, element); + } else { + mark_incomplete(context, identity_unknown_error(context.entry)); + } + } + IdentityMatch::NoMatch => {} + } + } + } + None if read.evidence.role.is_unknown() => { + mark_incomplete(context, incomplete_traversal_error("role", depth)); + } + None => {} + } + + if depth >= context.max_depth { + if !read.child_read.elements.is_empty() + || !read.child_read.complete + || read.child_read.truncated() + { + mark_incomplete(context, incomplete_traversal_error("depth_limit", depth)); + } + } else { + if !read.child_read.complete || read.child_read.truncated() { + mark_incomplete(context, incomplete_traversal_error("children", depth)); + } + for child in &read.child_read.elements { + collect_elements_recursive(child, depth.saturating_add(1), context)?; + if should_stop_collecting(context.matches.len(), context.entry) { + break; + } } } - context.ancestors.remove(&ptr_key); + context.ancestors.remove(&pointer); Ok(()) } #[cfg(target_os = "macos")] -fn element_matches_entry(el: &AXElement, entry: &RefEntry) -> bool { - let ax_role = copy_string_attr(el, accessibility_sys::kAXRoleAttribute); - let (normalized, promoted_label) = - crate::tree::roles::normalized_role_and_label(el, ax_role.as_deref()); - normalized == entry.role && element_identity_matches(el, entry, promoted_label) -} - -pub(super) fn should_stop_collecting(match_count: usize, entry: &RefEntry) -> bool { - match_count > 1 && entry.bounds_hash.is_none() -} - -fn should_prune_for_resolution(el: &AXElement, entry: &RefEntry, depth: u8) -> bool { - !has_meaningful_identity(entry) && should_prune_by_bounds(el, entry, depth) -} - -#[cfg(target_os = "macos")] -fn element_identity_matches( - el: &AXElement, +fn candidate_identity( + element: &AXElement, entry: &RefEntry, - promoted_label: Option<String>, -) -> bool { - let elem_name = promoted_label.or_else(|| resolve_element_name(el)); - let elem_value = crate::tree::copy_value_typed(el); - let elem_description = copy_string_attr(el, accessibility_sys::kAXDescriptionAttribute); - identity_matches( + context: &mut ResolveReadContext, +) -> Result<IdentityMatch, AdapterError> { + let read = read_node(element, identity_requirements(entry), context)?; + if read.invalid_element { + return Ok(IdentityMatch::Unknown); + } + let matched = match read.evidence.role.known() { + Some(role) if role == &entry.identity.role => { + let identity = match_native_or_text_identity(entry, &read.evidence); + if identity == IdentityMatch::Unknown && provisional_geometry_candidate(entry) { + IdentityMatch::Match + } else { + identity + } + } + Some(_) => IdentityMatch::NoMatch, + None if read.evidence.role.is_unknown() => IdentityMatch::Unknown, + None => IdentityMatch::NoMatch, + }; + Ok(matched) +} + +pub(super) fn match_native_or_text_identity( + entry: &RefEntry, + evidence: &agent_desktop_core::LocatorEvidence, +) -> IdentityMatch { + identity_match( entry, - elem_name.as_deref(), - elem_value.as_deref(), - elem_description.as_deref(), + &evidence.name, + &evidence.value, + &evidence.description, + &evidence.identifiers, ) } -#[cfg(target_os = "macos")] -fn resolve_children( - el: &AXElement, - ax_role: Option<&str>, - deadline: Instant, - seen: &mut FxHashSet<usize>, -) -> Result<Vec<AXElement>, AdapterError> { - seen.clear(); - set_messaging_timeout(el, remaining_before_deadline(deadline)?); - let mut result = Vec::new(); - for attr in child_attributes(ax_role) { - ensure_before_deadline(deadline)?; - if let Some(children) = copy_ax_array(el, attr) { - for child in children { - if seen.insert(child.0 as usize) { - result.push(child); - } - } - } - } - Ok(result) +pub(super) fn native_identifier_reused_by_different_role( + entry: &RefEntry, + evidence: &agent_desktop_core::LocatorEvidence, +) -> bool { + let Some(expected) = entry.identity.native_id.as_ref() else { + return false; + }; + evidence + .role + .known() + .is_some_and(|role| role != &entry.identity.role) + && evidence.identifiers.is_complete() + && evidence + .identifiers + .identifiers() + .iter() + .any(|identifier| identifier == expected) +} + +fn provisional_geometry_candidate(entry: &RefEntry) -> bool { + entry.geometry.bounds_hash.is_some() + && !agent_desktop_core::ref_identity::has_meaningful_identity(entry) +} + +#[cfg(target_os = "macos")] +fn read_node( + element: &AXElement, + requirements: EvidenceRequirements, + context: &mut ResolveReadContext, +) -> Result<crate::tree::query::node_read::NodeRead, AdapterError> { + if !context.usage.claim_node() { + context.stats.traversal.limits.node_hits += 1; + return Err(incomplete_traversal_error("node_budget", 0)); + } + let child_plan = + crate::tree::query::child_read_plan::ChildReadPlan::load(context.usage.child_capacity()); + let read = crate::tree::query::node_read::read_node( + element, + crate::tree::query::node_read_context::NodeReadContext { + tree: &crate::tree::TreeBuildContext::empty(false), + stats: &mut context.stats, + usage: &mut context.usage, + requirements, + deadline: context.deadline, + child_plan, + }, + )?; + context + .usage + .note_child_demand(read.child_read.total_count, &mut context.stats); + context.usage.claim_edges(read.child_read.elements.len()); + Ok(read) +} + +fn identity_requirements(entry: &RefEntry) -> EvidenceRequirements { + let semantic_text = has_stable_text_identity(entry) || entry.identity.role == "cell"; + EvidenceRequirements { + role: true, + name: semantic_text, + description: semantic_text, + value: semantic_text, + identifiers: entry.identity.native_id.is_some(), + states: false, + ref_evidence: Default::default(), + } +} + +pub(super) fn should_stop_collecting(match_count: usize, entry: &RefEntry) -> bool { + match_count > 1 && entry.geometry.bounds_hash.is_none() +} + +fn mark_incomplete(context: &mut CollectContext<'_>, error: AdapterError) { + if context.incomplete.is_none() { + *context.incomplete = Some(error); + } } diff --git a/crates/macos/src/tree/resolve_tests.rs b/crates/macos/src/tree/resolve_tests.rs index 0e60ea8..a77be28 100644 --- a/crates/macos/src/tree/resolve_tests.rs +++ b/crates/macos/src/tree/resolve_tests.rs @@ -1,39 +1,62 @@ use super::*; use crate::tree::AXElement; use crate::tree::resolve_classify::classify_candidates; -use crate::tree::resolve_identity::bounded_window_fallback_allowed; -use crate::tree::resolve_roots::{fallback_replacement_window_roots, source_window_number}; -use crate::tree::resolve_search::should_stop_collecting; -use agent_desktop_core::adapter::SnapshotSurface; +use crate::tree::resolve_roots::{ + require_unique_window_number_match, source_scope_verified, source_window_number, + source_window_scope_required, +}; +use crate::tree::resolve_search::{ + find_entry_by_path, incomplete_traversal_error, match_native_or_text_identity, + native_identifier_reused_by_different_role, walk_finite_path, +}; +use agent_desktop_core::{ + RefCapabilities, RefEntryIdentity, RefGeometry, RefProcess, RefScope, RefSource, + SnapshotSurface, +}; -fn entry( +pub(super) fn entry( bounds_hash: Option<u64>, source_window_id: Option<&str>, source_window_title: Option<&str>, root_ref: Option<&str>, ) -> RefEntry { RefEntry { - pid: 1, - role: "cell".into(), - name: Some("Investors".into()), - value: None, - description: None, - states: vec![], - bounds: None, - bounds_hash, - available_actions: vec![], - source_app: None, - source_window_id: source_window_id.map(String::from), - source_window_title: source_window_title.map(String::from), - source_surface: SnapshotSurface::Window, - root_ref: root_ref.map(String::from), - path_is_absolute: false, - path: smallvec::smallvec![0, 1], + process: RefProcess { + pid: agent_desktop_core::ProcessId::new(1), + process_instance: Some("test-instance".into()), + }, + identity: RefEntryIdentity { + role: "cell".into(), + name: Some("Investors".into()), + value: None, + description: None, + native_id: None, + }, + geometry: RefGeometry { + bounds: None, + bounds_hash, + }, + capabilities: RefCapabilities { + states: vec![], + available_actions: vec![], + }, + source: RefSource { + source_app: None, + source_window_id: source_window_id.map(String::from), + source_window_title: source_window_title.map(String::from), + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: RefScope { + root_ref: root_ref.map(String::from), + path_is_absolute: false, + path: smallvec::smallvec![0, 1], + }, } } #[test] -fn path_fast_path_accepts_bounds_or_source_window_identity() { +fn path_fast_path_requires_a_hard_window_scope() { assert!(!can_use_path_fast_path(&entry(None, None, None, None))); assert!(can_use_path_fast_path(&entry( None, @@ -41,92 +64,69 @@ fn path_fast_path_accepts_bounds_or_source_window_identity() { None, None ))); - assert!(!can_use_path_fast_path(&entry( + assert!(can_use_path_fast_path(&entry( None, None, Some("Documents"), None ))); - assert!(can_use_path_fast_path(&entry(Some(42), None, None, None))); + let mut menu = entry(Some(42), Some("w-10"), None, None); + menu.source.source_surface = SnapshotSurface::Menu; + assert!(can_use_path_fast_path(&menu)); +} + +#[test] +fn locator_anchor_path_requires_scope_and_identity_or_geometry() { + let mut unscoped = entry(Some(42), None, None, None); + assert!(!can_use_locator_anchor_path(&unscoped)); + + unscoped.source.source_window_id = Some("w-10".into()); + assert!(can_use_locator_anchor_path(&unscoped)); + + unscoped.geometry.bounds_hash = None; + unscoped.identity.name = None; + assert!(!can_use_locator_anchor_path(&unscoped)); +} + +#[test] +fn drill_down_path_must_be_absolute() { assert!(!can_use_path_fast_path(&entry( Some(42), Some("w-10"), - Some("Documents"), + None, Some("@e1") ))); - let mut absolute_drill = entry(Some(42), Some("w-10"), Some("Documents"), Some("@e1")); - absolute_drill.path_is_absolute = true; - assert!(can_use_path_fast_path(&absolute_drill)); + let mut absolute = entry(Some(42), Some("w-10"), None, Some("@e1")); + absolute.scope.path_is_absolute = true; + assert!(can_use_path_fast_path(&absolute)); } #[test] -fn no_bounds_source_window_refs_require_scoped_path_resolution() { - assert!(!requires_scoped_path_resolution(&entry( - None, None, None, None - ))); - assert!(requires_scoped_path_resolution(&entry( - None, - Some("w-10"), - None, - None - ))); - assert!(!requires_scoped_path_resolution(&entry( - None, - None, - Some("Documents"), - None - ))); - assert!(!requires_scoped_path_resolution(&entry( - Some(42), - Some("w-10"), - Some("Documents"), - None - ))); - assert!(!requires_scoped_path_resolution(&entry( - None, - Some("w-10"), - Some("Documents"), - Some("@e1") - ))); - let mut absolute_drill = entry(None, Some("w-10"), Some("Documents"), Some("@e1")); - absolute_drill.path_is_absolute = true; - assert!(requires_scoped_path_resolution(&absolute_drill)); -} - -#[test] -fn scoped_path_retry_fails_closed_when_scope_is_unresolved() { - let no_bounds_entry = entry(None, Some("w-10"), Some("Freeform"), None); - - assert!(requires_scoped_path_resolution(&no_bounds_entry)); - assert!(requires_scoped_path_resolution(&description_entry())); - assert!(!requires_scoped_path_resolution(&entry( - Some(42), - Some("w-10"), - Some("Freeform"), - None - ))); -} - -#[test] -fn scoped_path_retry_fails_closed_for_blank_identity_without_bounds() { - let mut blank = entry(None, Some("w-10"), Some("Freeform"), None); - blank.name = None; - - assert!(requires_scoped_path_resolution(&blank)); -} - -#[test] -fn broad_search_requires_bounds_or_meaningful_identity() { +fn broad_search_requires_bounds_or_stable_identity() { let mut blank = entry(None, None, None, None); - blank.name = None; + blank.identity.name = None; assert!(!can_use_broad_search(&blank)); - assert!(can_use_broad_search(&description_entry())); - assert!(can_use_broad_search(&entry(Some(42), None, None, None))); + blank.identity.native_id = Some(agent_desktop_core::ElementIdentifier { + kind: agent_desktop_core::IdentifierKind::AxDomIdentifier, + value: "dom-id".into(), + }); + assert!(can_use_broad_search(&blank)); + blank.identity.native_id = None; + blank.identity.description = Some("Insert Text Box".into()); + assert!(can_use_broad_search(&blank)); } #[test] -fn source_window_number_parses_window_ids_only() { +fn source_window_scope_rejects_malformed_ids_instead_of_broadening() { + let malformed = entry(Some(42), Some("not-a-window"), None, None); + + assert!(source_window_scope_required(&malformed)); + assert_eq!(source_window_number(&malformed), None); +} + +#[test] +fn source_window_number_parses_only_canonical_ids() { assert_eq!( source_window_number(&entry(None, Some("w-42"), None, None)), Some(42) @@ -136,178 +136,251 @@ fn source_window_number_parses_window_ids_only() { None ); assert_eq!( - source_window_number(&entry(None, Some("w-bad"), None, None)), + source_window_number(&entry(None, Some("w-0"), None, None)), + None + ); + assert_eq!( + source_window_number(&entry(None, Some("w--1"), None, None)), None ); } #[test] -fn only_element_not_found_is_retryable_resolution_error() { - assert!(is_retryable_resolution_error( +fn duplicate_ax_window_bridge_matches_are_ambiguous() { + let error = require_unique_window_number_match(2, 42).unwrap_err(); + + assert_eq!(error.code, ErrorCode::AmbiguousTarget); + assert_eq!(error.details.unwrap()["candidate_count"], 2); +} + +#[test] +fn mutable_title_scope_never_claims_exact_verification() { + assert!(!source_scope_verified(None, true)); + assert!(!source_scope_verified(Some("w-42"), false)); + assert!(source_scope_verified(Some("w-42"), true)); +} + +#[test] +fn strict_resolution_requires_the_exact_process_instance() { + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let token = crate::system::process_identity::token_for_pid(pid) + .unwrap() + .expect("current process identity"); + let mut stored = entry(None, None, None, None); + stored.process.pid = agent_desktop_core::ProcessId::try_from(pid).unwrap(); + stored.process.process_instance = Some(token.clone()); + + assert!(verify_process_instance(&stored).is_ok()); + + stored.process.process_instance = None; + assert_eq!( + verify_process_instance(&stored).unwrap_err().code, + ErrorCode::StaleRef + ); + + stored.process.process_instance = Some(token); + stored.process.pid = agent_desktop_core::ProcessId::try_from(i32::MAX).unwrap(); + assert_eq!( + verify_process_instance(&stored).unwrap_err().code, + ErrorCode::StaleRef + ); +} + +#[test] +fn only_explicitly_retryable_incomplete_errors_retry() { + let retryable = incomplete_traversal_error("children", 3); + assert!(is_retryable_resolution_error(&retryable)); + assert!(!is_retryable_resolution_error( &AdapterError::element_not_found("element") )); assert!(!is_retryable_resolution_error( - &AdapterError::ambiguous_target("2 candidates matched") + &AdapterError::permission_denied() )); - assert!(!is_retryable_resolution_error(&AdapterError::new( - ErrorCode::Timeout, - "resolution timed out" - ))); } #[test] -fn expired_deadline_fails_before_path_resolution_reads() { - let err = match find_entry_by_path( +fn cannot_complete_retry_can_recover_before_one_deadline() { + let attempts = std::cell::Cell::new(0); + let deadline = Instant::now() + Duration::from_millis(250); + + let result = retry_incomplete_until(deadline, || { + attempts.set(attempts.get() + 1); + if attempts.get() == 1 { + Err(incomplete_traversal_error("children", 1)) + } else { + Ok(7_u8) + } + }); + + assert_eq!(result.unwrap(), 7); + assert_eq!(attempts.get(), 2); +} + +#[test] +fn incomplete_traversal_is_never_element_not_found() { + let error = incomplete_traversal_error("depth_limit", 50); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.details.unwrap()["complete"], false); +} + +#[test] +fn expired_deadline_fails_before_path_reads() { + let mut context = ResolveReadContext::new(Instant::now()); + let error = find_entry_by_path( &[], - &entry(Some(42), Some("w-42"), Some("Documents"), None), - false, - std::time::Instant::now(), - ) { - Ok(_) => panic!("expected timeout"), - Err(err) => err, - }; - - assert_eq!(err.code, ErrorCode::Timeout); -} - -#[test] -fn ambiguous_candidate_classification_reports_structured_details() { - let err = match classify_candidates( - vec![ - AXElement(std::ptr::null_mut()), - AXElement(std::ptr::null_mut()), - ], - &entry(None, Some("w-42"), Some("Documents"), None), - true, - ) { - Ok(_) => panic!("expected ambiguous target"), - Err(err) => err, - }; - - assert_eq!(err.code, ErrorCode::AmbiguousTarget); - assert!(!err.message.contains("Investors")); - assert!(err.message.contains("name_chars=9")); - let details = err.details.unwrap(); - assert_eq!(details["candidate_count"], 2); - assert_eq!(details["role"], "cell"); - assert_eq!(details["source_window_id"], "w-42"); -} - -#[test] -fn multiple_identity_candidates_without_bounds_match_are_stale_not_ambiguous() { - let err = match classify_candidates( - vec![ - AXElement(std::ptr::null_mut()), - AXElement(std::ptr::null_mut()), - ], - &entry(Some(42), Some("w-42"), Some("Documents"), None), - true, - ) { - Ok(_) => panic!("expected stale moved target"), - Err(err) => err, - }; - - assert_eq!(err.code, ErrorCode::ElementNotFound); -} - -#[test] -fn single_meaningful_identity_candidate_resolves_after_bounds_change() { - let _handle = classify_candidates( - vec![AXElement(std::ptr::null_mut())], - &entry(None, Some("w-42"), Some("Documents"), None), + &entry(Some(42), Some("w-42"), None, None), true, + &mut context, ) - .expect("unique identity should resolve within the verified source window"); + .err() + .expect("expired path resolution must fail"); + + assert_eq!(error.code, ErrorCode::Timeout); } #[test] -fn cross_window_replacement_without_verified_source_window_fails_closed() { - let err = match classify_candidates( +fn finite_path_walk_allows_reused_native_handle_identity() { + let calls = std::cell::Cell::new(0); + let result = walk_finite_path(7_u8, &[0, 0, 0], |handle, _| { + calls.set(calls.get() + 1); + Ok(Some(handle)) + }) + .unwrap(); + + assert_eq!(result, Some(7)); + assert_eq!(calls.get(), 3); +} + +#[test] +fn preserved_native_path_replays_nested_target_in_one_read_per_level() { + let reads = std::cell::RefCell::new(Vec::new()); + let result = walk_finite_path(0_u8, &[2, 0], |parent, index| { + reads.borrow_mut().push(index); + Ok(match (parent, index) { + (0, 2) => Some(20), + (20, 0) => Some(99), + _ => None, + }) + }) + .unwrap(); + + assert_eq!(result, Some(99)); + assert_eq!(reads.into_inner(), [2, 0]); +} + +#[test] +fn ambiguity_reports_bounded_candidate_summaries() { + let candidates = (0..11).map(|_| AXElement(std::ptr::null_mut())).collect(); + let error = classify_candidates( + candidates, + &entry(None, Some("w-42"), None, None), + true, + Instant::now() + Duration::from_secs(1), + ) + .err() + .expect("duplicate candidates must be ambiguous"); + + assert_eq!(error.code, ErrorCode::AmbiguousTarget); + let details = error.details.unwrap(); + assert_eq!(details["candidate_count"], 11); + assert_eq!(details["candidate_summaries_truncated"], true); + assert_eq!(details["candidates"].as_array().unwrap().len(), 10); +} + +#[test] +fn unique_scoped_candidate_resolves_even_after_bounds_move() { + let handle = classify_candidates( vec![AXElement(std::ptr::null_mut())], - &entry(None, Some("w-42"), Some("Documents"), None), - false, - ) { - Ok(_) => panic!("expected stale candidate to fail closed"), - Err(err) => err, + &entry(Some(42), Some("w-42"), None, None), + true, + Instant::now() + Duration::from_secs(1), + ); + + assert!(handle.is_ok()); +} + +#[test] +fn scoped_role_only_candidate_does_not_bypass_bounds_verification() { + let mut weak = entry(None, Some("w-42"), None, None); + weak.identity.name = None; + + let error = classify_candidates( + vec![AXElement(std::ptr::null_mut())], + &weak, + true, + Instant::now() + Duration::from_secs(1), + ) + .err() + .expect("window scope alone must not identify a role-only candidate"); + + assert_eq!(error.code, ErrorCode::ElementNotFound); +} + +#[test] +fn authoritatively_absent_native_identifier_is_a_definitive_mismatch() { + use agent_desktop_core::{ + IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, }; - assert_eq!(err.code, ErrorCode::ElementNotFound); -} - -#[test] -fn non_window_identity_candidate_without_bounds_fails_closed() { - let mut menu_entry = entry(None, None, None, None); - menu_entry.source_surface = SnapshotSurface::Menu; - - let err = match classify_candidates(vec![AXElement(std::ptr::null_mut())], &menu_entry, false) { - Ok(_) => panic!("expected stale candidate to fail closed"), - Err(err) => err, + let mut stored = entry(Some(42), Some("w-42"), None, None); + stored.identity.native_id = Some(agent_desktop_core::ElementIdentifier { + kind: agent_desktop_core::IdentifierKind::AxIdentifier, + value: "stable-id".into(), + }); + let live = LocatorEvidence { + role: LocatorField::Known(stored.identity.role.clone()), + name: LocatorField::Known("Investors".into()), + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::absent(), + states: LocatorField::Absent, + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Absent, + }, }; - assert_eq!(err.code, ErrorCode::ElementNotFound); + assert_eq!( + match_native_or_text_identity(&stored, &live), + agent_desktop_core::IdentityMatch::NoMatch + ); } #[test] -fn bounded_window_fallback_requires_untitled_window_ref_with_bounds_hash() { - assert!(bounded_window_fallback_allowed(&entry( - Some(42), - Some("w-10"), - None, - None - ))); - assert!(!bounded_window_fallback_allowed(&entry( - Some(42), - Some("w-10"), - Some("Stale Title"), - None - ))); - assert!(!bounded_window_fallback_allowed(&entry( - None, - Some("w-10"), - Some("Documents"), - None - ))); - let mut menu_entry = entry(Some(42), Some("w-10"), Some("Documents"), None); - menu_entry.source_surface = SnapshotSurface::Menu; - assert!(!bounded_window_fallback_allowed(&menu_entry)); -} - -#[test] -fn bounds_hash_keeps_collecting_to_disambiguate_identity_matches() { - assert!(!should_stop_collecting( - 2, - &entry(Some(42), None, None, None) - )); - assert!(should_stop_collecting(2, &entry(None, None, None, None))); -} - -#[test] -fn bounded_window_fallback_propagates_expired_deadline() { - let err = match fallback_replacement_window_roots( - &[AXElement(std::ptr::null_mut())], - std::time::Instant::now() - std::time::Duration::from_millis(1), - ) { - Ok(_) => panic!("expected timeout"), - Err(err) => err, +fn native_identifier_reuse_by_a_different_role_is_definitively_stale() { + use agent_desktop_core::{ + ElementIdentifier, IdentifierEvidence, IdentifierKind, LocatorEvidence, LocatorField, + LocatorRefEvidence, }; - assert_eq!(err.code, ErrorCode::Timeout); + let mut stored = entry(None, Some("w-42"), None, None); + let identifier = ElementIdentifier { + kind: IdentifierKind::AxDomIdentifier, + value: "compose".into(), + }; + stored.identity.native_id = Some(identifier.clone()); + let live = LocatorEvidence { + role: LocatorField::Known("textfield".into()), + name: LocatorField::Absent, + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::typed([identifier], Some(0), true), + states: LocatorField::Absent, + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Absent, + }, + }; + + assert!(native_identifier_reused_by_different_role(&stored, &live)); } #[test] -fn bounded_window_fallback_must_not_stop_after_first_match() { - let mut bounded_entry = entry(Some(42), Some("w-10"), Some("Documents"), None); - bounded_entry.role = "textfield".into(); - bounded_entry.name = Some("00:01".into()); - bounded_entry.value = Some("00:01".into()); +fn complete_absence_remains_retryable_for_renderer_detach_and_reattach() { + let cause = AdapterError::element_not_found("element"); + let error = stale_ref_error(&entry(None, Some("w-42"), None, None), &cause); - assert!(!should_stop_collecting(2, &bounded_entry)); -} - -fn description_entry() -> RefEntry { - let mut entry = entry(None, Some("w-10"), Some("Freeform"), None); - entry.role = "button".into(); - entry.name = None; - entry.description = Some("Insert Text Box".into()); - entry + assert_eq!(error.code, ErrorCode::StaleRef); + assert_eq!(error.details.unwrap()["retryable"], true); } diff --git a/crates/macos/src/tree/resolve_tests_more.rs b/crates/macos/src/tree/resolve_tests_more.rs new file mode 100644 index 0000000..30c5b11 --- /dev/null +++ b/crates/macos/src/tree/resolve_tests_more.rs @@ -0,0 +1,109 @@ +use super::tests::entry; +use super::*; +use crate::tree::AXElement; +use crate::tree::resolve_classify::classify_candidates; +use crate::tree::resolve_search::{match_native_or_text_identity, should_stop_collecting}; + +#[test] +fn identifier_kind_mismatch_is_not_an_exact_match() { + use agent_desktop_core::{ + ElementIdentifier, IdentifierEvidence, IdentifierKind, LocatorEvidence, LocatorField, + LocatorRefEvidence, + }; + + let mut stored = entry(Some(42), Some("w-42"), None, None); + stored.identity.native_id = Some(ElementIdentifier { + kind: IdentifierKind::AxIdentifier, + value: "stable-id".into(), + }); + let live = LocatorEvidence { + role: LocatorField::Known(stored.identity.role.clone()), + name: LocatorField::Known("Investors".into()), + description: LocatorField::Absent, + value: LocatorField::Absent, + identifiers: IdentifierEvidence::typed( + [ElementIdentifier { + kind: IdentifierKind::AxDomIdentifier, + value: "stable-id".into(), + }], + Some(0), + true, + ), + states: LocatorField::Absent, + ref_evidence: LocatorRefEvidence { + bounds: LocatorField::Absent, + available_actions: LocatorField::Absent, + }, + }; + + assert_eq!( + match_native_or_text_identity(&stored, &live), + agent_desktop_core::IdentityMatch::NoMatch + ); +} + +#[test] +fn duplicate_identity_candidates_remain_ambiguous_after_bounds_drift() { + let error = classify_candidates( + vec![ + AXElement(std::ptr::null_mut()), + AXElement(std::ptr::null_mut()), + ], + &entry(Some(42), Some("w-42"), None, None), + true, + Instant::now() + Duration::from_secs(1), + ) + .err() + .expect("duplicate live identities must remain ambiguous"); + + assert_eq!(error.code, ErrorCode::AmbiguousTarget); +} + +#[test] +fn unscoped_candidate_requires_matching_bounds() { + let error = classify_candidates( + vec![AXElement(std::ptr::null_mut())], + &entry(Some(42), Some("w-42"), None, None), + false, + Instant::now() + Duration::from_secs(1), + ) + .err() + .expect("unscoped candidate without matching bounds must fail"); + + assert_eq!(error.code, ErrorCode::ElementNotFound); +} + +#[test] +fn bounds_search_keeps_collecting_for_disambiguation() { + assert!(!should_stop_collecting( + 2, + &entry(Some(42), None, None, None) + )); + assert!(should_stop_collecting(2, &entry(None, None, None, None))); +} + +#[test] +fn ax_read_errors_distinguish_absent_unknown_and_permission_denied() { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorAttributeUnsupported, kAXErrorCannotComplete, + kAXErrorInvalidUIElement, + }; + + assert!( + !crate::tree::resolve_ax_read::classify_ax_read( + kAXErrorAttributeUnsupported, + "AXIdentifier" + ) + .unwrap() + ); + for error in [kAXErrorCannotComplete, kAXErrorInvalidUIElement] { + let classified = + crate::tree::resolve_ax_read::classify_ax_read(error, "AXIdentifier").unwrap_err(); + assert_eq!(classified.code, ErrorCode::AppUnresponsive); + assert_eq!(classified.details.unwrap()["complete"], false); + } + let denied = + crate::tree::resolve_ax_read::classify_ax_read(kAXErrorAPIDisabled, "AXIdentifier") + .unwrap_err(); + assert_eq!(denied.code, ErrorCode::PermDenied); +} diff --git a/crates/macos/src/tree/roles.rs b/crates/macos/src/tree/roles.rs index ef614d0..0223904 100644 --- a/crates/macos/src/tree/roles.rs +++ b/crates/macos/src/tree/roles.rs @@ -1,4 +1,4 @@ -pub fn ax_role_to_str(ax_role: &str) -> &'static str { +pub(crate) fn ax_role_to_str(ax_role: &str) -> &'static str { match ax_role { "AXApplication" => "application", "AXButton" => "button", @@ -9,24 +9,30 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str { "AXLink" => "link", "AXMenuItem" | "AXMenuBarItem" => "menuitem", "AXRadioButton" => "radiobutton", - "AXTab" | "AXTabGroup" => "tab", - "AXSlider" | "AXValueIndicator" => "slider", + "AXTab" => "tab", + "AXTabGroup" => "tablist", + "AXSlider" => "slider", + "AXValueIndicator" => "handle", "AXComboBox" | "AXPopUpButton" => "combobox", - "AXOutlineRow" | "AXRow" => "treeitem", + "AXOutlineRow" => "treeitem", + "AXRow" => "row", "AXCell" => "cell", "AXColumn" => "column", "AXWindow" => "window", "AXSheet" => "sheet", "AXDialog" => "dialog", - "AXGroup" | "AXGenericElement" => "group", + "AXGroup" | "AXGenericElement" | "AXSplitGroup" => "group", + "AXRadioGroup" => "radiogroup", "AXToolbar" => "toolbar", "AXStaticText" => "statictext", "AXImage" => "image", "AXTable" => "table", "AXList" => "list", "AXOutline" => "outline", - "AXScrollArea" | "AXScrollBar" => "scrollarea", - "AXSplitter" | "AXSplitGroup" => "splitter", + "AXScrollArea" => "scrollarea", + "AXScrollBar" => "scrollbar", + "AXSplitter" => "splitter", + "AXSeparator" => "separator", "AXMenu" | "AXMenuBar" => "menu", "AXIncrementor" | "AXStepper" => "incrementor", "AXDisclosureTriangle" => "disclosure", @@ -48,53 +54,63 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str { "AXLayoutArea" | "AXLayoutItem" => "layoutitem", "AXLevelIndicator" => "levelindicator", "AXRelevanceIndicator" => "relevanceindicator", + "AXDocument" => "document", + "AXHeading" => "heading", + "AXParagraph" => "paragraph", + "AXStatus" => "status", + "AXToolTip" => "tooltip", _ => "unknown", } } -pub fn normalized_role_and_label( - el: &crate::tree::AXElement, - ax_role: Option<&str>, -) -> (String, Option<String>) { - let promoted_label = promoted_item_label(ax_role, el); - let role = if promoted_label.is_some() { - "cell" - } else { - ax_role.map(ax_role_to_str).unwrap_or("unknown") - }; - (role.to_string(), promoted_label) +pub(crate) fn ax_role_and_subrole_to_str(ax_role: &str, ax_subrole: Option<&str>) -> &'static str { + match ax_subrole { + Some("AXApplicationAlert") => "alert", + Some("AXApplicationAlertDialog") => "alertdialog", + Some("AXApplicationDialog") => "dialog", + Some("AXApplicationLog") => "log", + Some("AXApplicationMarquee") => "marquee", + Some("AXApplicationStatus") => "status", + Some("AXApplicationTimer") => "timer", + Some("AXDocumentArticle") => "article", + Some("AXDocumentMath") => "math", + Some("AXDocumentNote") => "note", + Some("AXDocumentRegion") => "region", + Some("AXLandmarkBanner") => "banner", + Some("AXLandmarkComplementary") => "complementary", + Some("AXLandmarkContentInfo") => "contentinfo", + Some("AXLandmarkForm") => "form", + Some("AXLandmarkMain") => "main", + Some("AXLandmarkNavigation") => "navigation", + Some("AXLandmarkSearch") => "search", + Some("AXDefinition") => "definition", + Some("AXTerm") => "term", + Some("AXTabPanel") => "tabpanel", + Some("AXUserInterfaceTooltip") => "tooltip", + Some("AXToggleButton") => match ax_role_to_str(ax_role) { + primary @ ("checkbox" | "switch" | "radiobutton") => primary, + _ => "button", + }, + Some("AXOutlineRow") => "treeitem", + Some("AXTableRow") => "row", + Some("AXSecureTextField" | "AXSearchField") => "textfield", + Some("AXDialog" | "AXSystemDialog") => "dialog", + Some( + "AXCloseButton" | "AXMinimizeButton" | "AXZoomButton" | "AXToolbarButton" + | "AXFullScreenButton" | "AXSortButton", + ) => "button", + Some("AXToggle" | "AXSwitch") => "switch", + Some("AXContentList" | "AXDefinitionList" | "AXDescriptionList") => "list", + Some("AXSeparatorDockItem") => "separator", + _ => ax_role_to_str(ax_role), + } } -pub fn promoted_item_label(ax_role: Option<&str>, el: &crate::tree::AXElement) -> Option<String> { - if ax_role != Some("AXGroup") { - return None; - } - let children = crate::tree::element::child_attributes(ax_role) - .iter() - .find_map(|attr| { - crate::tree::copy_ax_array(el, attr).filter(|children| !children.is_empty()) - }) - .unwrap_or_default(); - let has_icon = children - .iter() - .any(|child| crate::tree::copy_string_attr(child, "AXRole").as_deref() == Some("AXImage")); - if !has_icon { - return None; - } - children.iter().find_map(|child| { - if crate::tree::copy_string_attr(child, "AXRole").as_deref() == Some("AXTextField") { - crate::tree::copy_string_attr(child, "AXValue").filter(|value| !value.is_empty()) - } else { - None - } - }) -} - -pub use agent_desktop_core::roles::is_toggleable_role; +pub(crate) use agent_desktop_core::roles::is_toggleable_role; #[cfg(test)] mod tests { - use super::ax_role_to_str; + use super::{ax_role_and_subrole_to_str, ax_role_to_str}; #[test] fn interactive_ax_roles_map_to_exact_normalized_roles() { @@ -121,9 +137,6 @@ mod tests { assert_eq!(ax_role_to_str("AXToggle"), "switch"); assert_eq!(ax_role_to_str("AXOutlineRow"), "treeitem"); - assert_eq!(ax_role_to_str("AXRow"), "treeitem"); - - assert_eq!(ax_role_to_str("AXScrollBar"), "scrollarea"); } #[test] @@ -132,4 +145,166 @@ mod tests { assert_eq!(ax_role_to_str(""), "unknown"); assert_eq!(ax_role_to_str("button"), "unknown"); } + + #[test] + fn every_emitted_role_is_in_the_core_vocabulary() { + for native in [ + "AXApplication", + "AXButton", + "AXTextField", + "AXCheckBox", + "AXSwitch", + "AXLink", + "AXMenuItem", + "AXRadioButton", + "AXTab", + "AXTabGroup", + "AXSlider", + "AXValueIndicator", + "AXComboBox", + "AXOutlineRow", + "AXRow", + "AXCell", + "AXColumn", + "AXWindow", + "AXSheet", + "AXDialog", + "AXGroup", + "AXToolbar", + "AXStaticText", + "AXImage", + "AXTable", + "AXList", + "AXOutline", + "AXScrollArea", + "AXScrollBar", + "AXSplitter", + "AXSplitGroup", + "AXSeparator", + "AXMenu", + "AXIncrementor", + "AXDisclosureTriangle", + "AXProgressIndicator", + "AXColorWell", + "AXWebArea", + "AXBrowser", + "AXGrid", + "AXHandle", + "AXPopover", + "AXDockItem", + "AXRuler", + "AXRulerMarker", + "AXTimeField", + "AXDateField", + "AXHelpTag", + "AXMatte", + "AXDrawer", + "AXLayoutArea", + "AXLevelIndicator", + "AXRelevanceIndicator", + "AXDocument", + "AXHeading", + "AXParagraph", + "AXStatus", + "AXToolTip", + ] { + assert!( + agent_desktop_core::roles::is_canonical_role(ax_role_to_str(native)), + "{native} emitted a noncanonical role" + ); + } + } + + #[test] + fn container_and_control_roles_do_not_collapse_into_interactive_siblings() { + assert_eq!(ax_role_to_str("AXTabGroup"), "tablist"); + assert_eq!(ax_role_to_str("AXTab"), "tab"); + assert_eq!(ax_role_to_str("AXRow"), "row"); + assert_eq!(ax_role_to_str("AXOutlineRow"), "treeitem"); + assert_eq!(ax_role_to_str("AXValueIndicator"), "handle"); + assert_eq!(ax_role_to_str("AXSlider"), "slider"); + assert_eq!(ax_role_to_str("AXScrollBar"), "scrollbar"); + assert_eq!(ax_role_to_str("AXScrollArea"), "scrollarea"); + assert_eq!(ax_role_to_str("AXSplitGroup"), "group"); + assert_eq!(ax_role_to_str("AXSplitter"), "splitter"); + } + + #[test] + fn subroles_preserve_semantics_hidden_by_generic_native_roles() { + assert_eq!( + ax_role_and_subrole_to_str("AXRow", Some("AXOutlineRow")), + "treeitem" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXRow", Some("AXTableRow")), + "row" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXWindow", Some("AXDialog")), + "dialog" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXTextField", Some("AXSecureTextField")), + "textfield" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXDockItem", Some("AXSeparatorDockItem")), + "separator" + ); + } + + #[test] + fn button_subrole_does_not_erase_a_primary_checkbox_role() { + assert_eq!( + ax_role_and_subrole_to_str("AXCheckBox", Some("AXToggleButton")), + "checkbox" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXButton", Some("AXToggleButton")), + "button" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXSwitch", Some("AXToggleButton")), + "switch" + ); + assert_eq!( + ax_role_and_subrole_to_str("AXRadioButton", Some("AXToggleButton")), + "radiobutton" + ); + } + + #[test] + fn chromium_subroles_preserve_web_semantics_hidden_by_ax_group() { + let mappings = [ + ("AXApplicationAlert", "alert"), + ("AXApplicationAlertDialog", "alertdialog"), + ("AXApplicationDialog", "dialog"), + ("AXApplicationLog", "log"), + ("AXApplicationMarquee", "marquee"), + ("AXApplicationStatus", "status"), + ("AXApplicationTimer", "timer"), + ("AXDocumentArticle", "article"), + ("AXDocumentMath", "math"), + ("AXDocumentNote", "note"), + ("AXDocumentRegion", "region"), + ("AXLandmarkBanner", "banner"), + ("AXLandmarkComplementary", "complementary"), + ("AXLandmarkContentInfo", "contentinfo"), + ("AXLandmarkForm", "form"), + ("AXLandmarkMain", "main"), + ("AXLandmarkNavigation", "navigation"), + ("AXLandmarkSearch", "search"), + ("AXDefinition", "definition"), + ("AXTerm", "term"), + ("AXTabPanel", "tabpanel"), + ("AXUserInterfaceTooltip", "tooltip"), + ("AXToggleButton", "button"), + ]; + + for (subrole, expected) in mappings { + let mapped = ax_role_and_subrole_to_str("AXGroup", Some(subrole)); + assert_eq!(mapped, expected, "unexpected mapping for {subrole}"); + assert!(agent_desktop_core::roles::is_canonical_role(mapped)); + } + } } diff --git a/crates/macos/src/tree/state_reader.rs b/crates/macos/src/tree/state_reader.rs new file mode 100644 index 0000000..8a3f463 --- /dev/null +++ b/crates/macos/src/tree/state_reader.rs @@ -0,0 +1,99 @@ +use agent_desktop_core::Rect; +use agent_desktop_core::state; + +use super::{AXElement, NodeAttrs}; + +pub(crate) struct StateReaderContext<'a> { + pub focused: Option<&'a AXElement>, + pub window_bounds: Option<Rect>, + pub is_secure_text: bool, +} + +pub(crate) fn states_from_element( + el: &AXElement, + attrs: &NodeAttrs, + role: &str, + ctx: &StateReaderContext<'_>, +) -> Vec<String> { + let mut states = Vec::new(); + if ctx + .focused + .is_some_and(|focused| super::capabilities::same_element(el, focused)) + || attrs.states.control.focused == Some(true) + { + states.push(state::FOCUSED.into()); + } + if !attrs.states.enabled { + states.push(state::DISABLED.into()); + } + if ctx.is_secure_text { + states.push(state::SECURE.into()); + } + if is_expanded(attrs) { + states.push(state::EXPANDED.into()); + } + if super::roles::is_toggleable_role(role) { + if value_is_checked(attrs.value.as_deref()) { + states.push(state::CHECKED.into()); + } else if value_is_indeterminate(attrs.value.as_deref()) { + states.push(state::INDETERMINATE.into()); + } + } + if attrs.states.control.selected == Some(true) { + states.push(state::SELECTED.into()); + } + if attrs.states.semantic.hidden == Some(true) { + states.push(state::HIDDEN.into()); + } + if attrs.states.semantic.busy == Some(true) { + states.push(state::BUSY.into()); + } + if attrs.states.semantic.modal == Some(true) { + states.push(state::MODAL.into()); + } + if attrs.states.semantic.required == Some(true) { + states.push(state::REQUIRED.into()); + } + if role == "button" && value_is_checked(attrs.value.as_deref()) { + states.push(state::PRESSED.into()); + } + if attrs.states.control.readonly == Some(true) { + states.push(state::READONLY.into()); + } + if is_offscreen(attrs.bounds, ctx.window_bounds) { + states.push(state::OFFSCREEN.into()); + } + states +} + +fn is_expanded(attrs: &NodeAttrs) -> bool { + attrs + .states + .control + .expanded + .or(attrs.states.control.disclosing) + .unwrap_or(false) +} + +fn value_is_checked(value: Option<&str>) -> bool { + matches!(value, Some("1" | "true")) +} + +fn value_is_indeterminate(value: Option<&str>) -> bool { + matches!(value, Some("2" | "mixed")) +} + +fn is_offscreen(bounds: Option<Rect>, window_bounds: Option<Rect>) -> bool { + let (Some(el), Some(win)) = (bounds, window_bounds) else { + return false; + }; + let el_right = el.x + el.width; + let el_bottom = el.y + el.height; + let win_right = win.x + win.width; + let win_bottom = win.y + win.height; + el_right <= win.x || el.x >= win_right || el_bottom <= win.y || el.y >= win_bottom +} + +#[cfg(test)] +#[path = "state_reader_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/state_reader_tests.rs b/crates/macos/src/tree/state_reader_tests.rs new file mode 100644 index 0000000..dbe96dc --- /dev/null +++ b/crates/macos/src/tree/state_reader_tests.rs @@ -0,0 +1,144 @@ +use super::*; +use crate::tree::{node_attr_states::NodeAttrStates, node_attrs::NodeAttrs}; +use agent_desktop_core::Rect; + +fn sample_attrs() -> NodeAttrs { + NodeAttrs { + role: Some("AXCheckBox".into()), + subrole: None, + value: Some("2".into()), + name_evidence: agent_desktop_core::NameEvidence::default(), + states: NodeAttrStates::default(), + bounds: Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }), + has_scrollbars: false, + } +} + +fn ctx_with(window_bounds: Option<Rect>, is_secure_text: bool) -> StateReaderContext<'static> { + StateReaderContext { + focused: None, + window_bounds, + is_secure_text, + } +} + +#[test] +fn mixed_checkbox_emits_indeterminate_not_checked() { + let attrs = sample_attrs(); + let ctx = ctx_with(None, false); + let el = AXElement(std::ptr::null_mut()); + let states = states_from_element(&el, &attrs, "checkbox", &ctx); + assert!(states.contains(&state::INDETERMINATE.to_string())); + assert!(!states.contains(&state::CHECKED.to_string())); +} + +#[test] +fn hidden_attr_emits_hidden_token() { + let mut attrs = sample_attrs(); + attrs.states.semantic.hidden = Some(true); + let ctx = ctx_with(None, false); + let el = AXElement(std::ptr::null_mut()); + let states = states_from_element(&el, &attrs, "button", &ctx); + assert!(states.contains(&state::HIDDEN.to_string())); +} + +#[test] +fn clipped_bounds_emit_offscreen() { + let mut attrs = sample_attrs(); + attrs.bounds = Some(Rect { + x: 100.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + let window = Rect { + x: 0.0, + y: 0.0, + width: 50.0, + height: 50.0, + }; + let ctx = ctx_with(Some(window), false); + let el = AXElement(std::ptr::null_mut()); + let states = states_from_element(&el, &attrs, "button", &ctx); + assert!(states.contains(&state::OFFSCREEN.to_string())); +} + +#[test] +fn emitted_tokens_over_representative_inputs_are_vocabulary_members() { + let el = AXElement(std::ptr::null_mut()); + let window = Rect { + x: 0.0, + y: 0.0, + width: 50.0, + height: 50.0, + }; + let mut cases: Vec<(NodeAttrs, &str, StateReaderContext<'static>)> = Vec::new(); + + let mut disabled = sample_attrs(); + disabled.states.enabled = false; + cases.push((disabled, "button", ctx_with(None, false))); + + cases.push((sample_attrs(), "textfield", ctx_with(None, true))); + + let mut expanded = sample_attrs(); + expanded.states.control.expanded = Some(true); + cases.push((expanded, "disclosure", ctx_with(None, false))); + + let mut checked = sample_attrs(); + checked.value = Some("1".into()); + cases.push((checked, "checkbox", ctx_with(None, false))); + + let mut selected = sample_attrs(); + selected.states.control.selected = Some(true); + cases.push((selected, "cell", ctx_with(None, false))); + + let mut busy = sample_attrs(); + busy.states.semantic.busy = Some(true); + cases.push((busy, "button", ctx_with(None, false))); + + let mut modal = sample_attrs(); + modal.states.semantic.modal = Some(true); + cases.push((modal, "window", ctx_with(None, false))); + + let mut required = sample_attrs(); + required.states.semantic.required = Some(true); + cases.push((required, "textfield", ctx_with(None, false))); + + let mut pressed = sample_attrs(); + pressed.value = Some("1".into()); + cases.push((pressed, "button", ctx_with(None, false))); + + let mut readonly = sample_attrs(); + readonly.states.control.readonly = Some(true); + cases.push((readonly, "textfield", ctx_with(None, false))); + + let mut offscreen = sample_attrs(); + offscreen.bounds = Some(Rect { + x: 1000.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + cases.push((offscreen, "button", ctx_with(Some(window), false))); + + let mut emitted: Vec<String> = Vec::new(); + for (attrs, role, ctx) in &cases { + emitted.extend(states_from_element(&el, attrs, role, ctx)); + } + assert!( + !emitted.is_empty(), + "representative inputs should exercise at least one producer branch" + ); + state::assert_states_in_vocabulary(&emitted); +} + +#[test] +#[should_panic(expected = "is not in STATE_VOCABULARY")] +fn assert_states_in_vocabulary_rejects_bogus_token() { + state::assert_states_in_vocabulary(&["zzz_bogus_state_token".to_string()]); +} diff --git a/crates/macos/src/tree/surface_inventory.rs b/crates/macos/src/tree/surface_inventory.rs new file mode 100644 index 0000000..6daebd9 --- /dev/null +++ b/crates/macos/src/tree/surface_inventory.rs @@ -0,0 +1,209 @@ +use crate::tree::{AXElement, element::element_for_pid}; +use agent_desktop_core::{AdapterError, SurfaceInfo}; +use std::time::Instant; + +pub(crate) fn list_surfaces_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Vec<SurfaceInfo>, AdapterError> { + ensure_before_deadline(deadline)?; + let app = element_for_pid(pid); + let mut surfaces = Vec::new(); + + if let Some(children) = read_array(&app, "AXChildren", deadline)? { + collect_app_surfaces(&children, deadline, &mut surfaces)?; + } + if let Some(focused) = read_element(&app, "AXFocusedUIElement", deadline)? { + collect_context_menus(&focused, deadline, &mut surfaces)?; + } + if let Some(window) = read_element(&app, "AXFocusedWindow", deadline)? { + collect_window_surfaces(&window, "focused-window", deadline, &mut surfaces)?; + } + + ensure_before_deadline(deadline)?; + Ok(surfaces) +} + +fn collect_app_surfaces( + children: &[AXElement], + deadline: Instant, + surfaces: &mut Vec<SurfaceInfo>, +) -> Result<(), AdapterError> { + for (index, child) in children.iter().enumerate() { + let id = format!("app/children/{index}"); + match read_string(child, "AXRole", deadline)?.as_deref() { + Some("AXMenuBar") => collect_menubar_surfaces(child, &id, deadline, surfaces)?, + Some("AXMenu") => push_menu_surface(child, "context_menu", id, deadline, surfaces)?, + _ => {} + } + } + Ok(()) +} + +fn collect_menubar_surfaces( + menubar: &AXElement, + menubar_id: &str, + deadline: Instant, + surfaces: &mut Vec<SurfaceInfo>, +) -> Result<(), AdapterError> { + let Some(items) = read_array(menubar, "AXChildren", deadline)? else { + return Ok(()); + }; + for (item_index, item) in items.iter().enumerate() { + if read_string(item, "AXRole", deadline)?.as_deref() != Some("AXMenuBarItem") + || read_bool(item, "AXSelected", deadline)? != Some(true) + { + continue; + } + let Some(children) = read_array(item, "AXChildren", deadline)? else { + continue; + }; + for (menu_index, menu) in children.iter().enumerate() { + if read_string(menu, "AXRole", deadline)?.as_deref() == Some("AXMenu") { + push_menu_surface( + menu, + "menu", + format!("{menubar_id}/items/{item_index}/children/{menu_index}"), + deadline, + surfaces, + )?; + } + } + } + Ok(()) +} + +fn collect_context_menus( + element: &AXElement, + deadline: Instant, + surfaces: &mut Vec<SurfaceInfo>, +) -> Result<(), AdapterError> { + let Some(children) = read_array(element, "AXChildren", deadline)? else { + return Ok(()); + }; + for (index, child) in children.iter().enumerate() { + if read_string(child, "AXRole", deadline)?.as_deref() == Some("AXMenu") { + push_menu_surface( + child, + "context_menu", + format!("focused/children/{index}"), + deadline, + surfaces, + )?; + } + } + Ok(()) +} + +fn push_menu_surface( + menu: &AXElement, + kind: &str, + id: String, + deadline: Instant, + surfaces: &mut Vec<SurfaceInfo>, +) -> Result<(), AdapterError> { + let title = read_title(menu, deadline)?; + let item_count = read_array(menu, "AXChildren", deadline)?.map(|items| items.len()); + surfaces.push(SurfaceInfo { + id, + kind: kind.into(), + title, + item_count, + }); + Ok(()) +} + +fn collect_window_surfaces( + window: &AXElement, + window_id: &str, + deadline: Instant, + surfaces: &mut Vec<SurfaceInfo>, +) -> Result<(), AdapterError> { + if let Some(kind) = surface_kind(window, deadline)? { + surfaces.push(SurfaceInfo { + id: window_id.to_string(), + kind: kind.into(), + title: read_title(window, deadline)?, + item_count: None, + }); + } + let Some(children) = read_array(window, "AXChildren", deadline)? else { + return Ok(()); + }; + for (index, child) in children.iter().enumerate() { + if let Some(kind) = surface_kind(child, deadline)? { + surfaces.push(SurfaceInfo { + id: format!("{window_id}/children/{index}"), + kind: kind.into(), + title: read_title(child, deadline)?, + item_count: None, + }); + } + } + Ok(()) +} + +fn surface_kind( + element: &AXElement, + deadline: Instant, +) -> Result<Option<&'static str>, AdapterError> { + let role = read_string(element, "AXRole", deadline)?; + let subrole = read_string(element, "AXSubrole", deadline)?; + Ok(match subrole.as_deref() { + Some("AXSheet") => Some("sheet"), + Some("AXPopover") => Some("popover"), + Some("AXDialog") | Some("AXAlert") => Some("alert"), + _ => match role.as_deref() { + Some("AXSheet") => Some("sheet"), + Some("AXPopover") => Some("popover"), + _ => None, + }, + }) +} + +fn read_title(element: &AXElement, deadline: Instant) -> Result<Option<String>, AdapterError> { + match read_string(element, "AXTitle", deadline)? { + Some(title) => Ok(Some(title)), + None => read_string(element, "AXDescription", deadline), + } +} + +fn read_string( + element: &AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<String>, AdapterError> { + crate::tree::surface_read::string(element, attribute, deadline) +} + +fn read_bool( + element: &AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<bool>, AdapterError> { + crate::tree::surface_read::boolean(element, attribute, deadline) +} + +fn read_array( + element: &AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<Vec<AXElement>>, AdapterError> { + crate::tree::surface_read::elements(element, attribute, deadline).map(Some) +} + +fn read_element( + element: &AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + crate::tree::surface_read::element(element, attribute, deadline) +} + +fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { + crate::tree::surface_read::ensure_before_deadline(deadline) +} + +#[cfg(test)] +#[path = "surface_inventory_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/surface_inventory_tests.rs b/crates/macos/src/tree/surface_inventory_tests.rs new file mode 100644 index 0000000..e300f06 --- /dev/null +++ b/crates/macos/src/tree/surface_inventory_tests.rs @@ -0,0 +1,13 @@ +use super::*; +use std::time::Duration; + +#[test] +fn expired_deadline_fails_before_accessibility_inventory() { + let deadline = Instant::now() - Duration::from_millis(1); + + let pid = i32::try_from(std::process::id()).expect("test pid fits macOS pid_t"); + let error = list_surfaces_for_pid(pid, deadline) + .expect_err("an expired surface inventory must time out"); + + assert_eq!(error.code.as_str(), "TIMEOUT"); +} diff --git a/crates/macos/src/tree/surface_read.rs b/crates/macos/src/tree/surface_read.rs new file mode 100644 index 0000000..b2af1e1 --- /dev/null +++ b/crates/macos/src/tree/surface_read.rs @@ -0,0 +1,164 @@ +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::time::Instant; + +pub(crate) fn elements( + element: &super::AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Vec<super::AXElement>, AdapterError> { + let read = super::query::child_read::read_attribute_children( + element, + attribute, + agent_desktop_core::ObservationBudget::default().max_children_per_node, + deadline, + ); + ensure_before_deadline(deadline)?; + if read.status.api_disabled { + return Err(map_error(attribute, accessibility_sys::kAXErrorAPIDisabled)); + } + if read.status.invalid_element { + return Err(map_error( + attribute, + accessibility_sys::kAXErrorInvalidUIElement, + )); + } + if !read.complete || read.truncated() { + return Err(AdapterError::new( + ErrorCode::AppUnresponsive, + format!("Accessibility surface read was incomplete for {attribute}"), + ) + .with_details(serde_json::json!({ + "kind": "surface_array_incomplete", + "attribute": attribute, + "complete": false, + "total_count": read.total_count, + "loaded_count": read.elements.len(), + "count_changed": read.status.count_changed, + }))); + } + Ok(read.elements) +} + +pub(crate) fn element( + source: &super::AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<super::AXElement>, AdapterError> { + prepare(source, deadline)?; + finish( + super::attributes::copy_element_attr_result(source, attribute, deadline), + attribute, + deadline, + ) +} + +pub(crate) fn string( + element: &super::AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<String>, AdapterError> { + prepare(element, deadline)?; + finish( + super::attributes::copy_string_attr_result(element, attribute, deadline), + attribute, + deadline, + ) +} + +pub(crate) fn boolean( + element: &super::AXElement, + attribute: &str, + deadline: Instant, +) -> Result<Option<bool>, AdapterError> { + prepare(element, deadline)?; + finish( + super::attributes::copy_bool_attr_result(element, attribute, deadline), + attribute, + deadline, + ) +} + +pub(crate) fn ensure_before_deadline(deadline: Instant) -> Result<(), AdapterError> { + if Instant::now() >= deadline { + return Err(deadline_error()); + } + Ok(()) +} + +fn prepare(element: &super::AXElement, deadline: Instant) -> Result<(), AdapterError> { + if deadline.saturating_duration_since(Instant::now()).is_zero() { + return Err(deadline_error()); + } + super::attributes::set_messaging_timeout(element, deadline)?; + Ok(()) +} + +fn finish<T>( + result: Result<Option<T>, i32>, + attribute: &str, + deadline: Instant, +) -> Result<Option<T>, AdapterError> { + ensure_before_deadline(deadline)?; + result.map_err(|error| map_error(attribute, error)) +} + +fn map_error(attribute: &str, error: i32) -> AdapterError { + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + + let code = if error == kAXErrorAPIDisabled { + ErrorCode::PermDenied + } else if error == kAXErrorCannotComplete { + ErrorCode::Timeout + } else if error == kAXErrorInvalidUIElement { + ErrorCode::ElementNotFound + } else { + ErrorCode::ActionFailed + }; + AdapterError::new( + code, + format!("Accessibility surface read failed for {attribute}"), + ) + .with_details(serde_json::json!({ + "attribute": attribute, + "ax_error": error, + "kind": "surface_read", + })) + .with_suggestion("Retry after the application finishes updating its accessibility surfaces") +} + +fn deadline_error() -> AdapterError { + AdapterError::timeout("Accessibility surface resolution exceeded its absolute deadline") +} + +#[cfg(test)] +mod tests { + use super::*; + use accessibility_sys::{ + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorInvalidUIElement, + }; + + #[test] + fn classifier_preserves_permission_timeout_and_stale_states() { + assert_eq!( + map_error("AXRole", kAXErrorAPIDisabled).code, + ErrorCode::PermDenied + ); + assert_eq!( + map_error("AXRole", kAXErrorCannotComplete).code, + ErrorCode::Timeout + ); + assert_eq!( + map_error("AXRole", kAXErrorInvalidUIElement).code, + ErrorCode::ElementNotFound + ); + } + + #[test] + fn expired_deadline_fails_without_native_reads() { + let error = ensure_before_deadline(Instant::now()).expect_err("expired deadline"); + + assert_eq!(error.code, ErrorCode::Timeout); + } +} diff --git a/crates/macos/src/tree/surfaces.rs b/crates/macos/src/tree/surfaces.rs index c60f313..2e7797e 100644 --- a/crates/macos/src/tree/surfaces.rs +++ b/crates/macos/src/tree/surfaces.rs @@ -1,328 +1,269 @@ -use super::AXElement; -use super::attributes::{copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr}; -use super::element::element_for_pid; -use agent_desktop_core::node::SurfaceInfo; +use super::{AXElement, element::element_for_pid, surface_read}; +use agent_desktop_core::{AdapterError, ErrorCode}; +use std::time::Instant; + +const MAX_SURFACE_NODES: usize = 2_048; #[cfg(target_os = "macos")] -mod imp { - use super::*; +pub(crate) fn focused_surface_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + let app = element_for_pid(pid); + surface_read::element(&app, "AXFocusedWindow", deadline) +} - fn focused_window_element(pid: i32) -> Option<AXElement> { - let app = element_for_pid(pid); - copy_element_attr(&app, "AXFocusedWindow") - } - - fn open_menubar_menu(pid: i32) -> Option<AXElement> { - let app = element_for_pid(pid); - let app_children = copy_ax_array(&app, "AXChildren")?; - let menubar = app_children - .into_iter() - .find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenuBar"))?; - let items = copy_ax_array(&menubar, "AXChildren")?; - for item in &items { - if copy_string_attr(item, "AXRole").as_deref() != Some("AXMenuBarItem") { - continue; - } - if !copy_bool_attr(item, "AXSelected").unwrap_or(false) { - continue; - } - if let Some(children) = copy_ax_array(item, "AXChildren") { - return children - .into_iter() - .find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu")); - } +#[cfg(target_os = "macos")] +pub(crate) fn menubar_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + let app = element_for_pid(pid); + for child in surface_read::elements(&app, "AXChildren", deadline)? { + if has_role(&child, "AXMenuBar", deadline)? { + return Ok(Some(child)); } - None } + Ok(None) +} - fn context_menu_from_app(pid: i32) -> Option<AXElement> { - let app = element_for_pid(pid); - if let Some(menu) = - copy_ax_array(&app, "AXMenus").and_then(|menus| menus.into_iter().find(is_menu)) +#[cfg(target_os = "macos")] +pub(crate) fn menu_element_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + if let Some(menu) = open_menubar_menu(pid, deadline)? { + return Ok(Some(menu)); + } + context_menu_from_app(pid, deadline) +} + +#[cfg(target_os = "macos")] +fn open_menubar_menu(pid: i32, deadline: Instant) -> Result<Option<AXElement>, AdapterError> { + let Some(menubar) = menubar_for_pid(pid, deadline)? else { + return Ok(None); + }; + for item in surface_read::elements(&menubar, "AXChildren", deadline)? { + if !has_role(&item, "AXMenuBarItem", deadline)? + || surface_read::boolean(&item, "AXSelected", deadline)? != Some(true) { - return Some(menu); + continue; } - if let Some(focused) = copy_element_attr(&app, "AXFocusedUIElement") { - if let Some(menu) = find_menu_descendant(&focused, 0) { - return Some(menu); + for child in surface_read::elements(&item, "AXChildren", deadline)? { + if has_role(&child, "AXMenu", deadline)? { + return Ok(Some(child)); } } - let children = copy_ax_array(&app, "AXChildren")?; - for child in children { - if let Some(menu) = find_menu_descendant(&child, 0) { - return Some(menu); - } + } + Ok(None) +} + +#[cfg(target_os = "macos")] +fn context_menu_from_app(pid: i32, deadline: Instant) -> Result<Option<AXElement>, AdapterError> { + let app = element_for_pid(pid); + for menu in surface_read::elements(&app, "AXMenus", deadline)? { + if displayed_menu(&menu, deadline)? { + return Ok(Some(menu)); } - None } - - fn is_menu(el: &AXElement) -> bool { - copy_string_attr(el, "AXRole").as_deref() == Some("AXMenu") - && copy_bool_attr(el, "AXVisible").unwrap_or(true) + if let Some(focused) = surface_read::element(&app, "AXFocusedUIElement", deadline)? + && let Some(menu) = find_menu_descendant(focused, deadline)? + { + return Ok(Some(menu)); } + for child in surface_read::elements(&app, "AXChildren", deadline)? { + if let Some(menu) = find_menu_descendant(child, deadline)? { + return Ok(Some(menu)); + } + } + Ok(None) +} - /// Searches for a *displayed* menu under `el`. The menu bar is skipped: - /// every `AXMenuBarItem` carries a latent `AXMenu` child that exists even - /// when the dropdown is closed, so descending into the menu bar would make - /// `is_menu_open` permanently true for any app with a menu bar. Open - /// menu-bar dropdowns are detected separately by `open_menubar_menu` via - /// the `AXSelected` gate. - fn find_menu_descendant(el: &AXElement, depth: usize) -> Option<AXElement> { +#[cfg(target_os = "macos")] +fn find_menu_descendant( + root: AXElement, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + let mut stack = vec![(root, 0_u8)]; + let mut visited = 0_usize; + while let Some((element, depth)) = stack.pop() { + surface_read::ensure_before_deadline(deadline)?; + visited += 1; + if visited > MAX_SURFACE_NODES { + return Err(surface_limit_error()); + } if depth > 8 { - return None; + continue; } - if copy_string_attr(el, "AXRole").as_deref() == Some("AXMenuBar") { - return None; + let role = surface_read::string(&element, "AXRole", deadline)?; + if role.as_deref() == Some("AXMenuBar") { + continue; } - if is_menu(el) { - return Some(el.clone()); - } - for child in copy_ax_array(el, "AXChildren").unwrap_or_default() { - if let Some(menu) = find_menu_descendant(&child, depth + 1) { - return Some(menu); - } - } - None - } - - pub fn menu_element_for_pid(pid: i32) -> Option<AXElement> { - open_menubar_menu(pid).or_else(|| context_menu_from_app(pid)) - } - - pub fn menubar_for_pid(pid: i32) -> Option<AXElement> { - let app = element_for_pid(pid); - let app_children = copy_ax_array(&app, "AXChildren")?; - app_children - .into_iter() - .find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenuBar")) - } - - pub fn focused_surface_for_pid(pid: i32) -> Option<AXElement> { - focused_window_element(pid) - } - - /// Returns the focused window or its first child whose role or subrole matches `target`. - /// - /// The focused window itself may be the target (e.g. Electron sheets). - fn first_child_with_role_or_subrole(pid: i32, target: &str) -> Option<AXElement> { - let win = focused_window_element(pid)?; - if copy_string_attr(&win, "AXRole").as_deref() == Some(target) - || copy_string_attr(&win, "AXSubrole").as_deref() == Some(target) + if role.as_deref() == Some("AXMenu") + && surface_read::boolean(&element, "AXVisible", deadline)? != Some(false) { - return Some(win); + return Ok(Some(element)); } - let children = copy_ax_array(&win, "AXChildren")?; - children.into_iter().find(|child| { - copy_string_attr(child, "AXSubrole").as_deref() == Some(target) - || copy_string_attr(child, "AXRole").as_deref() == Some(target) - }) + stack.extend( + surface_read::elements(&element, "AXChildren", deadline)? + .into_iter() + .rev() + .map(|child| (child, depth.saturating_add(1))), + ); } + Ok(None) +} - pub fn sheet_for_pid(pid: i32) -> Option<AXElement> { - first_child_with_role_or_subrole(pid, "AXSheet") +#[cfg(target_os = "macos")] +fn displayed_menu(element: &AXElement, deadline: Instant) -> Result<bool, AdapterError> { + Ok(has_role(element, "AXMenu", deadline)? + && surface_read::boolean(element, "AXVisible", deadline)? != Some(false)) +} + +#[cfg(target_os = "macos")] +fn has_role(element: &AXElement, expected: &str, deadline: Instant) -> Result<bool, AdapterError> { + Ok(surface_read::string(element, "AXRole", deadline)?.as_deref() == Some(expected)) +} + +#[cfg(target_os = "macos")] +fn first_child_with_role_or_subrole( + pid: i32, + target: &str, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + let Some(window) = focused_surface_for_pid(pid, deadline)? else { + return Ok(None); + }; + if role_or_subrole_matches(&window, target, deadline)? { + return Ok(Some(window)); } - - pub fn popover_for_pid(pid: i32) -> Option<AXElement> { - first_child_with_role_or_subrole(pid, "AXPopover") + for child in surface_read::elements(&window, "AXChildren", deadline)? { + if role_or_subrole_matches(&child, target, deadline)? { + return Ok(Some(child)); + } } + Ok(None) +} - pub fn alert_for_pid(pid: i32) -> Option<AXElement> { - if let Some(win) = focused_window_element(pid) { - let children = copy_ax_array(&win, "AXChildren").unwrap_or_default(); - if let Some(found) = children.into_iter().find(|child| { - let subrole = copy_string_attr(child, "AXSubrole"); - matches!( - subrole.as_deref(), - Some("AXDialog") | Some("AXAlert") | Some("AXSheet") - ) - }) { - return Some(found); +#[cfg(target_os = "macos")] +fn role_or_subrole_matches( + element: &AXElement, + target: &str, + deadline: Instant, +) -> Result<bool, AdapterError> { + if surface_read::string(element, "AXRole", deadline)?.as_deref() == Some(target) { + return Ok(true); + } + Ok(surface_read::string(element, "AXSubrole", deadline)?.as_deref() == Some(target)) +} + +#[cfg(target_os = "macos")] +pub(crate) fn sheet_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + first_child_with_role_or_subrole(pid, "AXSheet", deadline) +} + +#[cfg(target_os = "macos")] +pub(crate) fn popover_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + first_child_with_role_or_subrole(pid, "AXPopover", deadline) +} + +#[cfg(target_os = "macos")] +pub(crate) fn alert_for_pid( + pid: i32, + deadline: Instant, +) -> Result<Option<AXElement>, AdapterError> { + let app = element_for_pid(pid); + let mut windows = surface_read::elements(&app, "AXWindows", deadline)?; + if let Some(focused) = focused_surface_for_pid(pid, deadline)? { + windows.insert(0, focused); + } + for window in windows { + if is_alert(&window, deadline)? { + return Ok(Some(window)); + } + for child in surface_read::elements(&window, "AXChildren", deadline)? { + if is_alert(&child, deadline)? { + return Ok(Some(child)); } } - - let app = element_for_pid(pid); - let windows = copy_ax_array(&app, "AXWindows")?; - for win in &windows { - let role = copy_string_attr(win, "AXRole"); - let subrole = copy_string_attr(win, "AXSubrole"); - if matches!( - subrole.as_deref(), - Some("AXDialog") | Some("AXAlert") | Some("AXSheet") - ) || matches!(role.as_deref(), Some("AXSheet")) - { - return Some(win.clone()); - } - let children = copy_ax_array(win, "AXChildren").unwrap_or_default(); - if let Some(found) = children.into_iter().find(|child| { - let sr = copy_string_attr(child, "AXSubrole"); - matches!( - sr.as_deref(), - Some("AXDialog") | Some("AXAlert") | Some("AXSheet") - ) - }) { - return Some(found); - } - } - None } + Ok(None) +} - pub fn is_menu_open(pid: i32) -> bool { - open_menubar_menu(pid).is_some() || context_menu_from_app(pid).is_some() - } +#[cfg(target_os = "macos")] +fn is_alert(element: &AXElement, deadline: Instant) -> Result<bool, AdapterError> { + let role = surface_read::string(element, "AXRole", deadline)?; + let subrole = surface_read::string(element, "AXSubrole", deadline)?; + Ok(matches!(role.as_deref(), Some("AXSheet")) + || matches!( + subrole.as_deref(), + Some("AXDialog") | Some("AXAlert") | Some("AXSheet") + )) +} - /// Lists open UI surfaces for the app. The focused window itself counts when its role/subrole matches (e.g. Electron sheets). - pub fn list_surfaces_for_pid(pid: i32) -> Vec<SurfaceInfo> { - let mut surfaces = Vec::new(); - let app = element_for_pid(pid); +#[cfg(target_os = "macos")] +pub(crate) fn is_menu_open(pid: i32, deadline: Instant) -> Result<bool, AdapterError> { + Ok(menu_element_for_pid(pid, deadline)?.is_some()) +} - if let Some(app_children) = copy_ax_array(&app, "AXChildren") { - for ch in &app_children { - match copy_string_attr(ch, "AXRole").as_deref() { - Some("AXMenuBar") => { - if let Some(items) = copy_ax_array(ch, "AXChildren") { - for item in &items { - if copy_string_attr(item, "AXRole").as_deref() - != Some("AXMenuBarItem") - { - continue; - } - if !copy_bool_attr(item, "AXSelected").unwrap_or(false) { - continue; - } - let title = copy_string_attr(item, "AXTitle"); - if let Some(menu_children) = copy_ax_array(item, "AXChildren") { - for menu in &menu_children { - if copy_string_attr(menu, "AXRole").as_deref() - == Some("AXMenu") - { - let item_count = - copy_ax_array(menu, "AXChildren").map(|v| v.len()); - surfaces.push(SurfaceInfo { - kind: "menu".into(), - title: title.clone(), - item_count, - }); - } - } - } - } - } - } - Some("AXMenu") => { - let title = copy_string_attr(ch, "AXTitle") - .or_else(|| copy_string_attr(ch, "AXDescription")); - let item_count = copy_ax_array(ch, "AXChildren").map(|v| v.len()); - surfaces.push(SurfaceInfo { - kind: "context_menu".into(), - title, - item_count, - }); - } - _ => {} - } - } - } - - if let Some(focused) = copy_element_attr(&app, "AXFocusedUIElement") { - if let Some(children) = copy_ax_array(&focused, "AXChildren") { - for ch in &children { - if copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu") { - let title = copy_string_attr(ch, "AXTitle") - .or_else(|| copy_string_attr(ch, "AXDescription")); - let item_count = copy_ax_array(ch, "AXChildren").map(|v| v.len()); - surfaces.push(SurfaceInfo { - kind: "context_menu".into(), - title, - item_count, - }); - } - } - } - } - - if let Some(win) = focused_window_element(pid) { - let win_role = copy_string_attr(&win, "AXRole"); - let win_subrole = copy_string_attr(&win, "AXSubrole"); - let win_kind = match win_subrole.as_deref() { - Some("AXSheet") => Some("sheet"), - Some("AXPopover") => Some("popover"), - Some("AXDialog") | Some("AXAlert") => Some("alert"), - _ => match win_role.as_deref() { - Some("AXSheet") => Some("sheet"), - Some("AXPopover") => Some("popover"), - _ => None, - }, - }; - if let Some(kind) = win_kind { - let title = copy_string_attr(&win, "AXTitle") - .or_else(|| copy_string_attr(&win, "AXDescription")); - surfaces.push(SurfaceInfo { - kind: kind.into(), - title, - item_count: None, - }); - } - - if let Some(children) = copy_ax_array(&win, "AXChildren") { - for child in &children { - let role = copy_string_attr(child, "AXRole"); - let subrole = copy_string_attr(child, "AXSubrole"); - let kind = match subrole.as_deref() { - Some("AXSheet") => "sheet", - Some("AXPopover") => "popover", - Some("AXDialog") | Some("AXAlert") => "alert", - _ => match role.as_deref() { - Some("AXSheet") => "sheet", - Some("AXPopover") => "popover", - _ => continue, - }, - }; - let title = copy_string_attr(child, "AXTitle") - .or_else(|| copy_string_attr(child, "AXDescription")); - surfaces.push(SurfaceInfo { - kind: kind.into(), - title, - item_count: None, - }); - } - } - } - - surfaces - } +fn surface_limit_error() -> AdapterError { + AdapterError::new( + ErrorCode::AppUnresponsive, + "Accessibility surface search exceeded its node budget", + ) + .with_details(serde_json::json!({ + "kind": "surface_search_limit", + "limit": MAX_SURFACE_NODES, + "complete": false, + })) + .with_suggestion("Retry with the target application in a more stable UI state") } #[cfg(not(target_os = "macos"))] -mod imp { - use super::*; - - pub fn menu_element_for_pid(_pid: i32) -> Option<AXElement> { - None - } - pub fn menubar_for_pid(_pid: i32) -> Option<AXElement> { - None - } - pub fn focused_surface_for_pid(_pid: i32) -> Option<AXElement> { - None - } - pub fn sheet_for_pid(_pid: i32) -> Option<AXElement> { - None - } - pub fn popover_for_pid(_pid: i32) -> Option<AXElement> { - None - } - pub fn alert_for_pid(_pid: i32) -> Option<AXElement> { - None - } - pub fn is_menu_open(_pid: i32) -> bool { - false - } - pub fn list_surfaces_for_pid(_pid: i32) -> Vec<SurfaceInfo> { - Vec::new() - } +macro_rules! unsupported_surface { + ($name:ident) => { + pub(crate) fn $name( + _pid: i32, + _deadline: Instant, + ) -> Result<Option<AXElement>, AdapterError> { + Ok(None) + } + }; } -pub use imp::{ - alert_for_pid, focused_surface_for_pid, is_menu_open, list_surfaces_for_pid, - menu_element_for_pid, menubar_for_pid, popover_for_pid, sheet_for_pid, -}; +#[cfg(not(target_os = "macos"))] +unsupported_surface!(focused_surface_for_pid); +#[cfg(not(target_os = "macos"))] +unsupported_surface!(menubar_for_pid); +#[cfg(not(target_os = "macos"))] +unsupported_surface!(menu_element_for_pid); +#[cfg(not(target_os = "macos"))] +unsupported_surface!(sheet_for_pid); +#[cfg(not(target_os = "macos"))] +unsupported_surface!(popover_for_pid); +#[cfg(not(target_os = "macos"))] +unsupported_surface!(alert_for_pid); + +#[cfg(not(target_os = "macos"))] +pub(crate) fn is_menu_open(_pid: i32, _deadline: Instant) -> Result<bool, AdapterError> { + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn surface_node_limit_is_explicitly_incomplete() { + let error = surface_limit_error(); + + assert_eq!(error.code, ErrorCode::AppUnresponsive); + assert_eq!(error.details.expect("limit details")["complete"], false); + } +} diff --git a/crates/macos/src/tree/text_attributes.rs b/crates/macos/src/tree/text_attributes.rs new file mode 100644 index 0000000..2404596 --- /dev/null +++ b/crates/macos/src/tree/text_attributes.rs @@ -0,0 +1,183 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::tree::{AXElement, bounded_string::BoundedString}; + use accessibility_sys::{ + kAXErrorAttributeUnsupported, kAXErrorNoValue, kAXErrorSuccess, kAXValueAttribute, + }; + use core_foundation::{ + base::{CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + }; + + const MALFORMED_AX_VALUE: i32 = i32::MIN; + const TEXT_TRUNCATED: i32 = i32::MIN + 1; + + pub(crate) fn copy_string_attr_result( + el: &AXElement, + attr: &str, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<String>, i32> { + let mut usage = crate::tree::observation_usage::ObservationUsage::new( + agent_desktop_core::ObservationBudget::default(), + ); + match copy_string_attr_bounded_result(el, attr, deadline, &mut usage)? { + Some(value) if value.complete => Ok(Some(value.value)), + Some(_) => Err(TEXT_TRUNCATED), + None => Ok(None), + } + } + + pub(crate) fn copy_string_attr_bounded_result( + el: &AXElement, + attr: &str, + deadline: impl crate::tree::ax_ipc::AxDeadline, + usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> Result<Option<BoundedString>, i32> { + let attribute = CFString::new(attr); + let (error, value) = crate::tree::ax_ipc::copy_attribute_value( + el, + attribute.as_concrete_TypeRef(), + deadline, + ); + if error != kAXErrorSuccess { + release_if_present(value); + return if is_absent_error(error) { + Ok(None) + } else { + Err(error) + }; + } + if value.is_null() { + return Ok(None); + } + let value = unsafe { CFType::wrap_under_create_rule(value) } + .downcast::<CFString>() + .ok_or(MALFORMED_AX_VALUE)?; + BoundedString::from_cf(&value, usage) + .map(Some) + .map_err(|_| MALFORMED_AX_VALUE) + } + + pub(crate) fn copy_value_typed( + el: &AXElement, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Option<String> { + copy_value_typed_result(el, deadline).ok().flatten() + } + + pub(crate) fn copy_value_typed_result( + el: &AXElement, + deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<String>, i32> { + let mut usage = crate::tree::observation_usage::ObservationUsage::new( + agent_desktop_core::ObservationBudget::default(), + ); + match copy_value_typed_bounded_result(el, deadline, &mut usage)? { + Some(value) if value.complete => Ok(Some(value.value)), + Some(_) => Err(TEXT_TRUNCATED), + None => Ok(None), + } + } + + pub(crate) fn copy_value_typed_bounded_result( + el: &AXElement, + deadline: impl crate::tree::ax_ipc::AxDeadline, + usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> Result<Option<BoundedString>, i32> { + let attribute = CFString::new(kAXValueAttribute); + let (error, value) = crate::tree::ax_ipc::copy_attribute_value( + el, + attribute.as_concrete_TypeRef(), + deadline, + ); + if error != kAXErrorSuccess { + release_if_present(value); + return if is_absent_error(error) { + Ok(None) + } else { + Err(error) + }; + } + if value.is_null() { + return Ok(None); + } + let value = unsafe { CFType::wrap_under_create_rule(value) }; + if let Some(text) = value.downcast::<CFString>() { + return BoundedString::from_cf(&text, usage) + .map(Some) + .map_err(|_| MALFORMED_AX_VALUE); + } + if let Some(boolean) = value.downcast::<CFBoolean>() { + return Ok(Some(BoundedString::from_owned( + bool::from(boolean).to_string(), + usage, + ))); + } + if let Some(number) = value.downcast::<CFNumber>() { + let text = crate::tree::node_attribute_decode::number_text(&number); + return Ok(text.map(|text| BoundedString::from_owned(text, usage))); + } + Err(MALFORMED_AX_VALUE) + } + + fn is_absent_error(error: i32) -> bool { + error == kAXErrorAttributeUnsupported || error == kAXErrorNoValue + } + + fn release_if_present(value: CFTypeRef) { + if !value.is_null() { + drop(unsafe { CFType::wrap_under_create_rule(value) }); + } + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use crate::tree::{AXElement, bounded_string::BoundedString}; + + pub(crate) fn copy_string_attr_result( + _el: &AXElement, + _attr: &str, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<String>, i32> { + Ok(None) + } + + pub(crate) fn copy_string_attr_bounded_result( + _el: &AXElement, + _attr: &str, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + _usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> Result<Option<BoundedString>, i32> { + Ok(None) + } + + pub(crate) fn copy_value_typed( + _el: &AXElement, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Option<String> { + None + } + + pub(crate) fn copy_value_typed_result( + _el: &AXElement, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + ) -> Result<Option<String>, i32> { + Ok(None) + } + + pub(crate) fn copy_value_typed_bounded_result( + _el: &AXElement, + _deadline: impl crate::tree::ax_ipc::AxDeadline, + _usage: &mut crate::tree::observation_usage::ObservationUsage, + ) -> Result<Option<BoundedString>, i32> { + Ok(None) + } +} + +pub(crate) use imp::{ + copy_string_attr_bounded_result, copy_string_attr_result, copy_value_typed, + copy_value_typed_bounded_result, copy_value_typed_result, +}; diff --git a/crates/windows/src/adapter.rs b/crates/windows/src/adapter.rs index b3fe3b1..bef0afb 100644 --- a/crates/windows/src/adapter.rs +++ b/crates/windows/src/adapter.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::adapter::PlatformAdapter; +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; pub struct WindowsAdapter; @@ -14,4 +14,50 @@ impl Default for WindowsAdapter { } } -impl PlatformAdapter for WindowsAdapter {} +impl ObservationOps for WindowsAdapter {} +impl ActionOps for WindowsAdapter {} +impl InputOps for WindowsAdapter {} +impl SystemOps for WindowsAdapter {} + +#[cfg(test)] +mod tests { + use super::*; + use agent_desktop_core::{AppError, CommandContext, ErrorCode, SnapshotSurface}; + + #[test] + fn snapshot_surfaces_fail_closed_until_windows_implements_them() { + let adapter = WindowsAdapter::new(); + assert!(adapter.supported_surfaces().is_empty()); + + let error = agent_desktop_core::commands::snapshot::execute( + agent_desktop_core::commands::snapshot::SnapshotArgs { + app: None, + window_id: None, + max_depth: 1, + include_bounds: false, + interactive_only: false, + compact: true, + surface: SnapshotSurface::Window, + skeleton: false, + root_ref: None, + snapshot_id: None, + }, + &adapter, + &CommandContext::default(), + ) + .expect_err("an unimplemented surface must fail at validation"); + + let AppError::Adapter(error) = error else { + panic!("surface validation must return an adapter error") + }; + assert_eq!(error.code, ErrorCode::PlatformNotSupported); + assert!( + error + .details + .as_ref() + .and_then(|details| details.get("supported_surfaces")) + .and_then(|surfaces| surfaces.as_array()) + .is_some_and(Vec::is_empty) + ); + } +} diff --git a/crates/windows/src/lib.rs b/crates/windows/src/lib.rs index 8e29ad6..bb0f54f 100644 --- a/crates/windows/src/lib.rs +++ b/crates/windows/src/lib.rs @@ -1,3 +1,5 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + mod actions; mod adapter; mod input; diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..325c82c --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,87 @@ +# agent-desktop FAQ + +## What is agent-desktop? + +agent-desktop is a native desktop automation CLI for AI agents. It lets agents observe and control desktop apps through OS accessibility trees, using structured JSON instead of screenshots, pixel matching, or browser-only automation. + +## Does agent-desktop require screenshots or pixel matching? + +No. The core workflow reads native accessibility trees and assigns refs to interactive elements. Screenshots are available as a separate command, but agents do not need screenshots or pixel matching to click buttons, type into fields, inspect menus, or navigate app windows. + +## How does agent-desktop work? + +| Component | Function | +|-----------|----------| +| **Native Rust CLI** | Fast, single binary, no runtime dependencies | +| **C-ABI cdylib** | Load once from Python, Swift, Go, Ruby, Node, or C instead of forking | +| **58 Commands** | Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, and bundled `skills` docs | +| **Snapshot & Refs** | Compact snapshot IDs and deterministic element refs like `@e1`, `@e2` | +| **Structured JSON** | Machine-readable responses with error codes and recovery hints | + +## What makes agent-desktop useful for AI agents? + +| Feature | Benefit | +|---------|---------| +| **Progressive Skeleton Traversal** | 78–96% token reduction on dense apps | +| **Headless-by-Default Actions** | Ref actions use accessibility APIs and block unintended physical side effects | +| **Snapshot Refs** | Agents act on stable refs within a snapshot instead of guessing coordinates | +| **Recovery Hints** | Errors include machine-readable codes and suggestions for the next agent step | +| **Cross-Language FFI** | Python, Swift, Go, Ruby, Node, C, and C++ hosts can call the native library directly | + +## Which platforms are supported? + +| Feature | macOS | Windows | Linux | +|---------|:-----:|:-------:|:-----:| +| Accessibility tree | **Yes** | Planned | Planned | +| Click/type/keyboard | **Yes** | Planned | Planned | +| Mouse input | **Yes** | Planned | Planned | +| Screenshot | **Yes** | Planned | Planned | +| Clipboard | **Yes** | Planned | Planned | +| App/window management | **Yes** | Planned | Planned | +| Notifications | **Yes** | Planned | Planned | + +## How do I install agent-desktop? + +Install the CLI from npm: + +```bash +npm install -g agent-desktop +agent-desktop snapshot --app Safari +``` + +Build the FFI library from source: + +```bash +cargo build --profile release-ffi -p agent-desktop-ffi +# Outputs under target/release-ffi/: libagent_desktop_ffi.dylib/.so or agent_desktop_ffi.dll +``` + +## What is the ref system? + +`snapshot` assigns qualified refs to interactive elements in depth-first order, such as `@s8f3k2p9:e1`. The embedded snapshot ID removes the separate `--snapshot` flag. Snapshot lookup remains confined to the selected namespace, so refs created under a session still require that session's `--session` or `AGENT_DESKTOP_SESSION` scope. Legacy bare refs such as `@e1` require an explicit `--snapshot` in the same namespace. + +Interactive roles that receive refs: + +`button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `incrementor`, `menubutton`, `switch`, `colorwell`, `dockitem`. + +Stale ref recovery: + +```text +snapshot -> act -> STALE_REF? -> snapshot again -> retry +``` + +## Is agent-desktop free and open source? + +Yes. agent-desktop is Apache-2.0 licensed for personal and commercial use. + +## Where can I get help? + +| Resource | Link | +|----------|------| +| **Repository** | [github.com/lahfir/agent-desktop](https://github.com/lahfir/agent-desktop) | +| **ClawHub Skill** | [clawhub.ai/lahfir/agent-desktop](https://clawhub.ai/lahfir/agent-desktop) | +| **skills.sh Listing** | [skills.sh/lahfir/agent-desktop/agent-desktop](https://skills.sh/lahfir/agent-desktop/agent-desktop) | +| **npm Package** | [npmjs.com/package/agent-desktop](https://www.npmjs.com/package/agent-desktop) | +| **CI Status** | [GitHub Actions](https://github.com/lahfir/agent-desktop/actions/workflows/ci.yml?query=branch%3Amain) | +| **Releases** | [GitHub Releases](https://github.com/lahfir/agent-desktop/releases) | +| **Issues** | [GitHub Issues](https://github.com/lahfir/agent-desktop/issues) | diff --git a/docs/json-output.md b/docs/json-output.md new file mode 100644 index 0000000..3f35989 --- /dev/null +++ b/docs/json-output.md @@ -0,0 +1,64 @@ +# JSON output contract + +Every command returns structured JSON: + +```json +{ + "version": "2.1", + "ok": true, + "command": "click", + "data": { "action": "click" } +} +``` + +Errors include machine-readable codes and recovery hints: + +```json +{ + "version": "2.1", + "ok": false, + "command": "click", + "error": { + "code": "STALE_REF", + "message": "Element at @e7 no longer matches the last snapshot", + "suggestion": "Run 'snapshot' to refresh refs, then retry", + "recovery": { + "strategy": "refresh_snapshot_then_retry_original", + "retryable": true, + "requires_fresh_snapshot": true + }, + "disposition": { + "delivery": "not_delivered", + "retry": "safe" + } + } +} +``` + +Version 2.1 replaces the 2.0 `error.retry_command` string with the structured +`error.recovery` object and adds `error.disposition`. Consumers must select a +recovery strategy from `recovery.strategy` only when `disposition.retry` is +`safe`; command strings from older envelopes must not be executed blindly. +The removed `retry_command` field has no compatibility alias. + +## Error codes + +| Code | Meaning | +|------|---------| +| `PERM_DENIED` | Accessibility permission not granted | +| `ELEMENT_NOT_FOUND` | No element matched the ref or query | +| `APP_NOT_FOUND` | Application not running or no windows | +| `STALE_REF` | Ref could not be re-identified in the live UI | +| `AMBIGUOUS_TARGET` | Ref recovery matched multiple plausible targets | +| `SNAPSHOT_NOT_FOUND` | Snapshot ID is missing or expired | +| `POLICY_DENIED` | Physical/headed path blocked by policy | +| `ACTION_FAILED` | The OS rejected the action | +| `ACTION_NOT_SUPPORTED` | The target does not expose the requested action | +| `APP_UNRESPONSIVE` | The matching application stopped responding | +| `PLATFORM_NOT_SUPPORTED` | Adapter method not implemented on this platform | +| `TIMEOUT` | Wait condition expired | +| `INVALID_ARGS` | Invalid argument values | + +## Exit codes + +`0` success, `1` structured error (JSON on stdout), `2` argument parse error. diff --git a/docs/phases.md b/docs/phases.md index 6349b3e..fe48942 100644 --- a/docs/phases.md +++ b/docs/phases.md @@ -8,25 +8,37 @@ Most recent shipments against this roadmap: -| Version | Date | What shipped | -|---------|------------|--------------| -| v0.3.0 | 2026-06-20 | Playwright-grade reliability hardening on the Phase 1 contracts: session-scoped latest snapshot pointers, explicit snapshot IDs usable across sessions, actionability checks, headed/headless policy, JSONL `--trace`, stale-ref diagnostics, and refstore symlink hardening | +| Version | Date | What shipped | +|---------|------|---------------| +| **Unreleased** | — | Foundation reliability contract (Playwright-grade, U0–U19) completed on `feat/foundation-playwright-grade-contract` (PR #93, `feat!`) — capability-supertrait `PlatformAdapter` split, canonical role/state vocabulary, live `is --property visible`, `list-displays` + honest `--screen` + scale factor, truthful Automation permission, `native_id` identity spine, window-id-first resolution, `LocatorQuery` + live `find`, default-on auto-wait, three-way occlusion gate, `scroll_into_view`, core accname computation, `supported_surfaces` introspection, typed `ActionStep` delivery tier, `ProcessState` + `APP_UNRESPONSIVE`, `LaunchOptions`, `SignalBaseline` + `wait --event`, typed clipboard, mouse modifiers/wheel, FFI ABI major 3, envelope 2.1. Will cut the next minor per pre-1.0 policy. See [Phase 1.6](#phase-16--playwright-grade-foundation-contract-completed). | +| v0.4.7 | 2026-07-02 | Trace viewer and replay artifacts | +| v0.4.6 | 2026-07-02 | Sessions promoted to the first-class trace container: `session start/end/list/gc`, manifest-gated automatic JSONL segments under `sessions/<id>/trace/` | +| v0.4.5 | 2026-06-30 | `--wait-for` selector polling flags | +| v0.4.4 | 2026-06-29 | macOS + core adapter hardening with caller-controllable guardrails | +| v0.4.3 | 2026-06-28 | macOS `retained_handle` null-guard hardening against release-only `CFRetain(null)` | +| v0.4.2 | 2026-06-27 | FFI Phase B and C: Python smoke harness, parity gates, command-wrapper groundwork | +| v0.4.1 | 2026-06-26 | FFI C-ABI surface complete (Phase A): load-time ABI handshake, session-scoped adapter, JSON-envelope command entrypoints (version, status, snapshot, wait, execute-by-ref), tracing log callback | +| v0.4.0 | 2026-06-24 | **Breaking:** `version` command always emits the JSON envelope (drops `--json`); over-engineering cleanup | +| v0.3.1 | 2026-06-21 | macOS stale-ref resolution hardening | +| v0.3.0 | 2026-06-20 | Playwright-grade reliability hardening on the Phase 1 contracts: session-scoped latest snapshot pointers, qualified snapshot refs within explicit namespaces, actionability checks, headed/headless policy, JSONL `--trace`, stale-ref diagnostics, and refstore symlink hardening | | v0.2.3 | 2026-06-06 | macOS AX window fallback hardening and fullscreen AX tree retrieval fixes | | v0.2.2 | 2026-06-02 | macOS CFArray type-safety fix for Mail.app snapshot stability | | v0.2.1 | 2026-05-23 | Empty accessibility-identity ref stability fix | | v0.2.0 | 2026-05-20 | Unified command execution contracts; chain deadlines now return structured `TIMEOUT` instead of `ACTION_FAILED` | | v0.1.14 | 2026-05 | Phase 1 unified core: typed batch/CLI path, `CommandPolicy`, `PermissionReport`, snapshot-scoped `RefStore`, headless `ActionRequest`, macOS screenshot backend boundary | -| v0.1.13 | 2026-04-17 | FFI cdylib on 5 platforms (aarch64/x86_64 macOS + Linux, x86_64 Windows MSVC), Sigstore build-provenance attestations, FFI review fixes (#26 — 50 commits) | +| v0.1.13 | 2026-04-17 | FFI cdylib on 5 platforms (aarch64/x86_64 macOS + Linux, x86_64 Windows MSVC), Sigstore build-provenance attestations, FFI review fixes (#26 — 66 commits) | | v0.1.12 | 2026-03–04 | Progressive skeleton traversal + ref-rooted drill-down (#20) | | v0.1.11 | 2026-02–03 | Skill-install prompt fix on all success paths | | v0.1.9 | 2026-01–02 | Scalable skill architecture + ClawHub auto-publish (#14) | | v0.1.8 | 2026-01 | `--compact` flag to collapse single-child unnamed nodes | | v0.1.7 | 2025-12 | Electron / web app accessibility-tree compatibility | -- Phase 1 completion: incremental across v0.1.0 – v0.1.14 (macOS MVP, 54 commands, unified core engine). -- v0.3.0 reliability hardening extends the Phase 1 contracts; it does not change the planned Windows/Linux adapter sequence. -- Phase 1.5 completion: v0.1.13 (FFI cdylib on 5 platforms). -- Phase 2: planned. Public scope is summarized in the Phase 2 section below. +- Phase 1 completion: incremental across v0.1.0 – v0.1.14 (macOS MVP, 58 shipped command names — 54 operational, 4 fail-closed pending daemon-owned held input — unified core engine). +- v0.2.0 – v0.3.1 unify command execution contracts and harden Playwright-grade ref reliability on top of the Phase 1 contracts. +- v0.4.0 – v0.4.7 complete the FFI C-ABI surface (load-time handshake, session-scoped adapter, JSON-envelope entrypoints), then land session-first tracing and a trace viewer. +- Foundation reliability contract (Playwright-grade, U0–U19) completed on `feat/foundation-playwright-grade-contract` — see [Phase 1.6](#phase-16--playwright-grade-foundation-contract-completed). +- Phase 1.5 completion: v0.1.13 (FFI cdylib on 5 platforms); the FFI C-ABI surface itself (`ad_snapshot` / `ad_execute_by_ref` / `ad_wait` / `ad_version` / `ad_status` / `ad_init` / `ad_abi_version`) completed in v0.4.1. +- Phase 2: planned. Delivered as dependency-ordered sub-phases 2.0–2.15 into the `feat/windows-adapter` integration branch — see [Platform Delivery Model](#platform-delivery-model--sub-phases-and-integration-branches) and the Phase 2 section below. - Phase 3+: planned. See each phase section below for the additive platform work and trait defaults that later phases backfill. --- @@ -36,13 +48,14 @@ Most recent shipments against this roadmap: | Phase | Name | Status | Platforms | |-------|------|--------|-----------| | 1 | Foundation + macOS MVP | **Completed** (v0.1.0 – v0.1.14) | macOS | -| 1.5 | FFI Distribution (C-ABI cdylib) | **Completed** (v0.1.13) | macOS, Windows, Linux | -| 2 | Windows Adapter | Planned | macOS, Windows | -| 3 | Linux Adapter | Planned | macOS, Windows, Linux | +| 1.5 | FFI Distribution (C-ABI cdylib) | **Completed** (v0.1.13; C-ABI surface completed v0.4.1) | macOS, Windows, Linux | +| 1.6 | Playwright-grade Foundation Contract | **Completed** (PR #93) | macOS (contract in core) | +| 2 | Windows Adapter | Planned — sub-phases 2.0–2.15 | macOS, Windows | +| 3 | Linux Adapter | Planned — sub-phases 3.0–3.15 | macOS, Windows, Linux | | 4 | MCP Server Mode | Planned | All | | 5 | Production Readiness | Planned | All | -Future platform phases are additive against the Phase 1 + v0.3.0 reliability contracts: typed command args, `CommandPolicy`, `PermissionReport`, snapshot-scoped refs, session-scoped latest snapshot pointers, `ActionRequest`, headed/headless interaction policy, JSONL reliability tracing, and the `PlatformAdapter` boundary. Core can still gain explicitly planned additive trait methods, but Windows/Linux should not fork command semantics or duplicate transport dispatch. +Future platform phases are additive against the Phase 1 + 1.5 + 1.6 contracts: typed command args, `CommandPolicy`, `PermissionReport`, snapshot-scoped refs, session-scoped latest snapshot pointers, `ActionRequest`, headed/headless interaction policy, JSONL reliability tracing, the capability-supertrait `PlatformAdapter` boundary (`ActionOps` / `InputOps` / `ObservationOps` / `SystemOps`), default-on auto-wait, and the occlusion gate. Core can still gain explicitly planned additive trait methods, but Windows/Linux implement — never fork — command semantics, and never duplicate transport dispatch. Phase 2 and Phase 3 ship as dependency-ordered sub-phases against a per-platform integration branch; see [Platform Delivery Model](#platform-delivery-model--sub-phases-and-integration-branches). --- @@ -61,7 +74,7 @@ Current shipped code uses explicit match arms, not a runtime command registry. L | `src/command_policy/` | Permissions, ref usage, side-effect classification | One policy source of truth for CLI, batch, and tests | | `src/batch/` | JSON batch parser and executor | Deserializes into `Commands`; no separate command interpretation | | `src/dispatch/` | Direct command match plus parse helpers | Shared CLI/batch execution path | -| `crates/{macos,windows,linux}/` | Adapter method implementations | Same trait signatures across platforms | +| `crates/{macos,windows,linux}/` | Adapter method implementations across four capability traits (`ActionOps`, `InputOps`, `ObservationOps`, `SystemOps`) | Same trait signatures across platforms | | `crates/ffi/` | C ABI wrappers around adapter/core types | ABI marshaling only | ### Add a Command @@ -71,7 +84,7 @@ Current shipped code uses explicit match arms, not a runtime command registry. L 3. Add the CLI args/variant in `src/cli_args/` and `src/cli/mod.rs`. 4. Add a single arm in `src/dispatch/mod.rs`. 5. Add a `CommandPolicy` arm. -6. If needed, add one `PlatformAdapter` method with a `not_supported()` default, then implement it per adapter. +6. If needed, add one method to the relevant `PlatformAdapter` capability trait (`ActionOps` / `InputOps` / `ObservationOps` / `SystemOps`) with a `not_supported()` default, then implement it per adapter. Batch receives the command automatically once `src/batch/mod.rs` maps the JSON command name to that same CLI enum variant. There is no separate batch-only behavior. @@ -83,6 +96,7 @@ Ref actions use `ActionRequest { action, policy }`. The default `InteractionPoli - Physical fallbacks are explicit and policy-gated. - Raw cursor commands (`hover`, `drag`, `mouse-*`) require `--headed`; other commands must not silently focus apps or move the cursor. - Expected OS denials return specific error codes such as `PERM_DENIED`, `SNAPSHOT_NOT_FOUND`, or `POLICY_DENIED`, not generic `INTERNAL`. +- **Auto-wait is on by default (Phase 1.6).** Every ref-consuming action waits, bounded, for its target to become actionable — visible, enabled, stable, unoccluded, and receiving events — before acting, the same default-on model as Playwright's actionability checks. The bound is 5000ms; `--timeout-ms 0` restores pre-1.6 single-shot behavior (act immediately, fail fast, no retry loop). A three-way occlusion gate (`hit_test` → `ReachesTarget` / `InterceptedBy { role, name, bounds }` / `Unknown`) blocks delivery when another element visibly intercepts the target point, but an inconclusive probe reports `Unknown` and never false-fails the action. Every action step reports a typed `ActionStep { label, outcome, mechanism: Option<StepMechanism>, verified: Option<bool> }`, so callers can see whether delivery was `SemanticApi` or `PhysicalSynthetic` and whether the effect was independently verified — no command claims success it did not observe. Windows and Linux should implement the same signatures rather than copying macOS-specific fallback decisions. @@ -90,7 +104,7 @@ Windows and Linux should implement the same signatures rather than copying macOS ## Phase 1 — Foundation + macOS MVP -**Status: Completed** — shipped incrementally across v0.1.0 – v0.1.14. +**Status: Completed** — shipped incrementally across v0.1.0 – v0.1.14, with the contract surface further hardened through v0.4.7 and the Phase 1.6 foundation contract (below). Phase 1 is the load-bearing phase. It establishes the shared command path, trait boundaries, output contract, error types, permission model, ref lifecycle, and full workspace structure. All subsequent platform phases build on top of this foundation without duplicating command semantics. @@ -100,46 +114,68 @@ Phase 1 is the load-bearing phase. It establishes the shared command path, trait |----|-----------|----------------| | P1-O1 | Working macOS snapshot CLI | `snapshot --app Finder` returns valid JSON with refs for all interactive elements | | P1-O2 | Platform adapter trait | Trait compiles with mock adapter; macOS adapter satisfies all trait methods | -| P1-O3 | Ref-based interaction | `click @e3` successfully invokes AXPress on the resolved element | +| P1-O3 | Ref-based interaction | `click @s8f3k2p9:e3` successfully invokes AXPress on the resolved element | | P1-O4 | Context efficiency | Typical Finder snapshot < 500 tokens (measured via tiktoken) | -| P1-O5 | Typed JSON contract | Output envelope carries `version: "2.0"`. **Partial**: dedicated standalone JSON-Schema files were never delivered — deferred to later quality gates. | +| P1-O5 | Typed JSON contract | Output envelope carries `version: "2.1"`. **Partial**: dedicated standalone JSON-Schema files were never delivered — deferred to later quality gates. | | P1-O6 | Permission detection | Permission report covers Accessibility, Screen Recording, and Automation with recovery suggestions | | P1-O7 | Command extensibility | Adding a new command follows the current shared path: `commands/{name}.rs` + `commands/mod.rs` + `src/cli_args/` + `src/cli/mod.rs` + `src/dispatch/mod.rs` + `src/command_policy/mod.rs` | -| P1-O8 | 54 working commands | All commands pass integration tests | +| P1-O8 | 58 shipped command names (54 operational) | All commands pass integration tests; `key-down` / `key-up` / `mouse-down` / `mouse-up` fail closed pending daemon-owned held input | | P1-O9 | CI pipeline | GitHub Actions macOS runner executes full test suite on every PR | | P1-O10 | Progressive skeleton traversal | Skeleton + drill-down workflow achieves 78%+ token savings on Electron apps | +P1-O3's `@s8f3k2p9:e3` example is the snapshot-qualified ref form current commands accept and emit. Other historical Phase 1 prose in this document may use the shorter legacy bare `@e3` form for brevity — see the Ref System note under [Phase 1.6](#phase-16--playwright-grade-foundation-contract-completed). + ### Workspace Structure ``` agent-desktop/ ├── Cargo.toml # workspace: members, shared deps +├── CONCEPTS.md # shared domain vocabulary for refs, snapshots, sessions, actionability ├── rust-toolchain.toml # pinned Rust version ├── clippy.toml # project-wide lint config ├── LICENSE # Apache-2.0 (shipped in every release tarball) ├── crates/ │ ├── core/ # agent-desktop-core (platform-agnostic) │ │ └── src/ -│ │ ├── lib.rs # public re-exports only -│ │ ├── node.rs # AccessibilityNode, Rect, WindowInfo -│ │ ├── adapter.rs # PlatformAdapter trait -│ │ ├── action.rs # Action enum -│ │ ├── action_request.rs / action_result.rs / action_step*.rs -│ │ ├── actionability/ # Live actionability checks and reports -│ │ ├── refs.rs # RefMap and RefEntry -│ │ ├── refs_store.rs # Snapshot/session-scoped ref persistence -│ │ ├── refs_lock.rs # RefStore write lock -│ │ ├── ref_alloc.rs # INTERACTIVE_ROLES, allocate_refs, is_collapsible, transform_tree -│ │ ├── snapshot_ref.rs # Ref-rooted drill-down (run_from_ref) -│ │ ├── snapshot.rs # SnapshotEngine (filter, allocate, serialize) -│ │ ├── trace.rs # JSONL reliability trace support -│ │ ├── error.rs # ErrorCode enum, AdapterError, AppError -│ │ ├── notification.rs # NotificationInfo, NotificationFilter, NotificationIdentity -│ │ └── commands/ # one file per command (direct match, no Command trait) +│ │ ├── lib.rs # public re-exports only +│ │ ├── node.rs # AccessibilityNode, Rect, WindowInfo +│ │ ├── adapter/ # capability traits + composed PlatformAdapter (actions.rs, input.rs, observation.rs, system.rs) +│ │ ├── adapter_session.rs # AdapterSession trait (open_session's Send+Sync return type) +│ │ ├── action.rs # Action enum +│ │ ├── action_request.rs / action_result.rs / action_step.rs +│ │ ├── actionability/ # Live actionability checks, occlusion gate, stability sampling +│ │ ├── live_locator/ # LocatorQuery evaluation, hydration, evidence-based resolution +│ │ ├── locator.rs # LocatorQuery, IdentityPredicate, StatePredicate +│ │ ├── hit_test.rs # HitTestResult (ReachesTarget / InterceptedBy / Unknown) +│ │ ├── process_state.rs # ProcessState (Running / Exited / Crashed / Unresponsive) +│ │ ├── launch_options.rs # LaunchOptions (args, env, cwd, timeout_ms, attach_if_running) +│ │ ├── clipboard_content.rs # ClipboardContent (Text / Image / FileUrls) +│ │ ├── display_info.rs # DisplayInfo (id, bounds, is_primary, scale) +│ │ ├── accname.rs # Core accessible-name computation over NameEvidence +│ │ ├── name_evidence.rs # NameEvidence supplied by adapters +│ │ ├── role.rs / state.rs / state_predicate.rs # canonical role/state vocabulary +│ │ ├── signal_baseline.rs / signals.rs # SignalBaseline capture + diff_signals for wait --event +│ │ ├── deadline.rs # Deadline + thread_local INHERITED_DEADLINE propagation +│ │ ├── refs.rs # RefMap and RefEntry +│ │ ├── refs_store.rs # Snapshot/session-scoped ref persistence +│ │ ├── refs_lock.rs # RefStore write lock +│ │ ├── ref_action.rs # Ref-consuming action pipeline (poll/wait/exactly-once family) +│ │ ├── ref_alloc.rs # INTERACTIVE_ROLES, allocate_refs, is_collapsible, transform_tree +│ │ ├── snapshot_ref.rs # Ref-rooted drill-down (run_from_ref) +│ │ ├── snapshot_surface.rs # SnapshotSurface (predeclared cross-platform surface vocabulary) +│ │ ├── snapshot.rs # SnapshotEngine (filter, allocate, serialize) +│ │ ├── session/ # session start/end/list/gc, manifest, liveness +│ │ ├── private_file*.rs # 0600-equivalent private artifact writing (Unix perms + Windows ACL family) +│ │ ├── trace.rs / trace_read/ # JSONL reliability trace, segment merge, HTML viewer +│ │ ├── error_code.rs # ErrorCode enum +│ │ ├── adapter_error.rs / app_error.rs # AdapterError, AppError +│ │ ├── output.rs # ENVELOPE_VERSION ("2.1") + envelope builders +│ │ ├── notification.rs # NotificationInfo, NotificationFilter, NotificationIdentity +│ │ └── commands/ # one file per command (direct match, no Command trait) │ ├── macos/ # agent-desktop-macos (Phase 1, shipped) │ ├── windows/ # agent-desktop-windows (stub → Phase 2) │ ├── linux/ # agent-desktop-linux (stub → Phase 3) -│ └── ffi/ # agent-desktop-ffi (cdylib, shipped v0.1.13; see Phase 1.5) +│ └── ffi/ # agent-desktop-ffi (cdylib, shipped v0.1.13; C-ABI surface completed v0.4.1; see Phase 1.5) ├── src/ # agent-desktop binary (entry point) │ ├── main.rs │ ├── batch/ # JSON batch -> typed Commands @@ -148,128 +184,154 @@ agent-desktop/ │ ├── command_policy/ # Permission/ref/side-effect policy │ ├── dispatch/ # Command dispatcher and parse helpers │ └── tests/ # Binary-level conformance tests +├── docs/ +│ └── solutions/ # documented solutions to past problems, tagged by module/problem_type └── tests/ ├── fixtures/ + ├── fixture-app/ # SwiftUI/AppKit fixture app for live e2e (AgentDeskFixture.swift + build.sh) └── integration/ ``` +This tree is representative, not exhaustive — `crates/core/src/` alone holds roughly 200 files (one struct/enum/fn-group per file) plus the subdirectories named above. + ### PlatformAdapter Trait -The single most important abstraction. Every platform-specific operation goes through this trait. Core never imports platform crates. The canonical definition is `crates/core/src/adapter.rs`; this roadmap lists representative method groups only, because the trait grows additively as reliability and platform parity work lands. +The single most important abstraction. Every platform-specific operation goes through this trait. Core never imports platform crates. Since Phase 1.6, `PlatformAdapter` is a **composed capability supertrait**, not one flat trait: ```rust -pub trait PlatformAdapter: Send + Sync { - // Core observation - fn list_windows(&self, filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError>; - fn list_apps(&self) -> Result<Vec<AppInfo>, AdapterError>; - fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError>; - fn get_tree(&self, win: &WindowInfo, opts: &TreeOptions) -> Result<AccessibilityNode, AdapterError>; - fn get_subtree(&self, handle: &NativeHandle, opts: &TreeOptions) -> Result<AccessibilityNode, AdapterError>; - fn list_surfaces(&self, pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError>; - - // Interaction - fn execute_action(&self, handle: &NativeHandle, request: ActionRequest) -> Result<ActionResult, AdapterError>; - fn resolve_element_strict(&self, entry: &RefEntry) -> Result<NativeHandle, AdapterError>; - fn resolve_element_strict_with_timeout(&self, entry: &RefEntry, timeout: Duration) -> Result<NativeHandle, AdapterError>; - fn release_handle(&self, handle: &NativeHandle) -> Result<(), AdapterError>; - fn mouse_event(&self, event: MouseEvent) -> Result<(), AdapterError>; - fn drag(&self, params: DragParams) -> Result<(), AdapterError>; - fn key_event(&self, combo: &KeyCombo, down: bool) -> Result<(), AdapterError>; - fn press_key_for_app(&self, app_name: &str, combo: &KeyCombo) -> Result<ActionResult, AdapterError>; - - // Lifecycle + windowing - fn permission_report(&self) -> PermissionReport; - fn request_permissions(&self) -> PermissionReport; - fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError>; - fn focus_app(&self, pid: i32) -> Result<(), AdapterError>; - fn launch_app(&self, id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError>; - fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError>; - fn is_protected_process(&self, identifier: &str) -> bool; - fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError>; - - // Capture + clipboard - fn screenshot(&self, target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError>; - fn get_clipboard(&self) -> Result<String, AdapterError>; - fn set_clipboard(&self, text: &str) -> Result<(), AdapterError>; - fn clear_clipboard(&self) -> Result<(), AdapterError>; - - // Notifications (macOS shipped; Windows/Linux planned) - fn list_notifications(&self, filter: &NotificationFilter) -> Result<Vec<NotificationInfo>, AdapterError>; - fn dismiss_notification(&self, index: usize, app_filter: Option<&str>) -> Result<NotificationInfo, AdapterError>; - fn dismiss_all_notifications(&self, app_filter: Option<&str>) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError>; - fn notification_action(&self, index: usize, identity: Option<&NotificationIdentity>, action_name: &str) -> Result<ActionResult, AdapterError>; - - // Live evidence and wait probes - fn get_live_value(&self, handle: &NativeHandle) -> Result<Option<String>, AdapterError>; - fn get_live_state(&self, handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError>; - fn get_live_actions(&self, handle: &NativeHandle) -> Result<Option<Vec<String>>, AdapterError>; - fn get_live_element(&self, handle: &NativeHandle) -> Result<LiveElement, AdapterError>; - fn get_element_bounds(&self, handle: &NativeHandle) -> Result<Option<Rect>, AdapterError>; - fn wait_for_menu(&self, pid: i32, open: bool, timeout_ms: u64) -> Result<(), AdapterError>; -} +// crates/core/src/adapter/mod.rs +pub trait PlatformAdapter: ObservationOps + ActionOps + InputOps + SystemOps {} +impl<T: ObservationOps + ActionOps + InputOps + SystemOps> PlatformAdapter for T {} ``` +Each capability lives in its own file under `crates/core/src/adapter/`, and every method defaults to `Err(AdapterError::not_supported(..))` unless noted — this is what lets a platform crate implement only what it supports while the workspace stays green. `crates/core/src/lib.rs` re-exports the four traits and `PlatformAdapter`. The method lists below are representative; the four files are the source of truth. + +**`ActionOps`** (`adapter/actions.rs`) — element interaction: +```rust +fn execute_action(&self, handle: &NativeHandle, request: ActionRequest, lease: &InteractionLease) -> Result<ActionResult, AdapterError>; +fn scroll_into_view(&self, handle: &NativeHandle, lease: &InteractionLease) -> Result<(), AdapterError>; +``` + +**`InputOps`** (`adapter/input.rs`) — raw OS input and clipboard: +```rust +fn mouse_event(&self, event: MouseEvent, lease: &InteractionLease) -> Result<(), AdapterError>; +fn key_event(&self, combo: &KeyCombo, down: bool, lease: &InteractionLease) -> Result<(), AdapterError>; +fn drag(&self, params: DragParams, lease: &InteractionLease) -> Result<(), AdapterError>; +fn get_clipboard_content(&self, format: ClipboardFormat, deadline: Deadline) -> Result<Option<ClipboardContent>, AdapterError>; +fn set_clipboard_content(&self, content: &ClipboardContent, lease: &InteractionLease) -> Result<(), AdapterError>; +fn clear_clipboard(&self, lease: &InteractionLease) -> Result<(), AdapterError>; +``` +`get_clipboard` / `set_clipboard` (untyped `String`) were removed pre-1.0 in favor of the typed `ClipboardContent` methods above; the C ABI (`ad_get_clipboard` / `ad_set_clipboard`) marshals the typed content and is unaffected by the Rust-side rename. + +**`ObservationOps`** (`adapter/observation.rs`) — reading the tree and live element state: +```rust +fn observe_tree(&self, root: ObservationRoot<'_>, request: &ObservationRequest) -> Result<ObservedTree, AdapterError>; +fn list_windows(&self, filter: &WindowFilter, deadline: Deadline) -> Result<Vec<WindowInfo>, AdapterError>; +fn list_apps(&self, deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError>; +fn get_tree(&self, win: &WindowInfo, opts: &TreeOptions, deadline: Deadline) -> Result<AccessibilityNode, AdapterError>; +fn get_subtree(&self, handle: &NativeHandle, opts: &TreeOptions, deadline: Deadline) -> Result<AccessibilityNode, AdapterError>; +fn resolve_element_strict(&self, entry: &RefEntry, deadline: Deadline) -> Result<NativeHandle, AdapterError>; +fn resolve_locator_anchor(&self, entry: &RefEntry, deadline: Deadline) -> Result<NativeHandle, AdapterError>; +fn list_surfaces(&self, process: ProcessIdentity, deadline: Deadline) -> Result<Vec<SurfaceInfo>, AdapterError>; +fn get_live_value(&self, handle: &NativeHandle, deadline: Deadline) -> Result<Option<String>, AdapterError>; +fn get_live_state(&self, handle: &NativeHandle, deadline: Deadline) -> Result<Option<ElementState>, AdapterError>; +fn get_live_actions(&self, handle: &NativeHandle, deadline: Deadline) -> Result<Option<Vec<String>>, AdapterError>; +fn get_live_element(&self, handle: &NativeHandle, deadline: Deadline) -> Result<LiveElement, AdapterError>; +fn get_element_bounds(&self, handle: &NativeHandle, deadline: Deadline) -> Result<Option<Rect>, AdapterError>; +fn hit_test(&self, handle: &NativeHandle, point: Point, deadline: Deadline) -> Result<HitTestResult, AdapterError>; +``` +`list_apps_scoped` is the one method in this trait with a real default (filters `list_apps()`), rather than `not_supported()`. + +**`SystemOps`** (`adapter/system.rs`) — lifecycle, windows, permissions, capture, notifications: +```rust +fn acquire_interaction_lease(&self, deadline: Deadline) -> Result<InteractionLease, AdapterError>; +fn permission_report(&self, deadline: Deadline) -> Result<PermissionReport, AdapterError>; +fn focus_window(&self, win: &WindowInfo, lease: &InteractionLease) -> Result<(), AdapterError>; +fn launch_app(&self, id: &str, options: &LaunchOptions, lease: &InteractionLease) -> Result<WindowInfo, AdapterError>; +fn close_app(&self, app: &AppInfo, force: bool, lease: &InteractionLease) -> Result<(), AdapterError>; +fn process_state(&self, process: ProcessIdentity, deadline: Deadline) -> Result<ProcessState, AdapterError>; +fn is_protected_process(&self, identifier: &str) -> bool; +fn window_op(&self, win: &WindowInfo, op: WindowOp, lease: &InteractionLease) -> Result<(), AdapterError>; +fn screenshot(&self, target: ScreenshotTarget, deadline: Deadline) -> Result<ImageBuffer, AdapterError>; +fn list_displays(&self, deadline: Deadline) -> Result<Vec<DisplayInfo>, AdapterError>; +fn focused_window(&self, deadline: Deadline) -> Result<Option<WindowInfo>, AdapterError>; +fn press_key_for_app(&self, process: ProcessIdentity, combo: &KeyCombo, policy: InteractionPolicy, lease: &InteractionLease) -> Result<ActionResult, AdapterError>; +fn supported_surfaces(&self) -> Vec<SnapshotSurface>; +fn open_session(&self, affinity: &SessionAffinity, deadline: Deadline) -> Result<Box<dyn AdapterSession>, AdapterError>; +fn capture_signal_baseline(&self, filter: &SignalFilter, deadline: Deadline) -> Result<SignalBaseline, AdapterError>; +fn list_notifications(&self, filter: &NotificationFilter, policy: InteractionPolicy, deadline: Deadline, lease: Option<&InteractionLease>) -> Result<Vec<NotificationInfo>, AdapterError>; +fn dismiss_notification(&self, request: DismissNotificationRequest<'_>, lease: &InteractionLease) -> Result<NotificationInfo, AdapterError>; +fn dismiss_all_notifications(&self, request: DismissAllNotificationsRequest<'_>, lease: &InteractionLease) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError>; +fn notification_action(&self, request: NotificationActionRequest<'_>, lease: &InteractionLease) -> Result<ActionResult, AdapterError>; +``` +`permission_report`, `request_permissions`, `unknown_accessibility_means_unsupported`, `supported_surfaces`, `is_protected_process`, and `is_blocked_combo` carry real (non-`not_supported`) defaults, so a bare-minimum adapter still reports something sane. + +`open_session` is the Send+Sync landing zone later phases use for platform session affinity — the Windows COM-MTA worker thread and the Linux D-Bus connection both attach through `AdapterSession::close(self: Box<Self>)`, not through `PlatformAdapter` itself. + ### Key Supporting Types -- `Action` — closed core enum whose platform dispatch arms must stay exhaustive. Current variants: Click, DoubleClick, TripleClick, RightClick, SetValue(String), SetFocus, Expand, Collapse, Select(String), Toggle, Check, Uncheck, Scroll(Direction, Amount), ScrollTo, PressKey(KeyCombo), KeyDown(KeyCombo), KeyUp(KeyCombo), TypeText(String), Clear, Hover, Drag(DragParams) +- `Action` — closed core enum whose platform dispatch arms must stay exhaustive. Current 21 variants (`crates/core/src/action.rs`): `Click`, `DoubleClick`, `RightClick`, `TripleClick`, `SetValue(String)`, `SetFocus`, `Expand`, `Collapse`, `Select(String)`, `Toggle`, `Check`, `Uncheck`, `Scroll(Direction, u32)`, `ScrollTo`, `PressKey(KeyCombo)`, `KeyDown(KeyCombo)`, `KeyUp(KeyCombo)`, `TypeText(String)`, `Clear`, `Hover`, `Drag(DragParams)`. Also carries `headed_requirement()`, `name()`, `requires_cursor_policy()`, `requires_hit_test()`, `requires_scroll_into_view()`, `may_use_focus_fallback()`, `base_interaction_policy()`. - `ActionRequest` — `{ action, policy }`; default policy forbids focus stealing and cursor movement +- `ActionStep` — `{ label: String, outcome: ActionStepOutcome, mechanism: Option<StepMechanism>, verified: Option<bool> }`; `StepMechanism` distinguishes `SemanticApi` delivery from `PhysicalSynthetic` delivery so every reported step is honest about how it reached the target +- `LocatorQuery` — `{ identity: IdentityPredicate, has_text: Option<String>, exact: bool, states: Vec<StatePredicate>, containment: ContainmentPredicate }`; evaluated by `resolve_query` and the live `find` command +- `HitTestResult` — enum `ReachesTarget | InterceptedBy { role, name, bounds } | Unknown`; `Unknown` on probe failure, never a false negative +- `ProcessState` — enum `Running | Exited { code: Option<i32> } | Crashed { signal_or_code: i32 } | Unresponsive` +- `LaunchOptions` — `{ args: Vec<String>, env: BTreeMap<String,String>, cwd: Option<PathBuf>, timeout_ms: u64, attach_if_running: bool }` (default `timeout_ms` 5000, `attach_if_running` true) +- `ClipboardContent` — enum `Text(String) | Image(ImageBuffer) | FileUrls(Vec<String>)` +- `DisplayInfo` — `{ id: String, bounds: Rect, is_primary: bool, scale: f64 }` +- `NameEvidence` — `{ explicit_label, labelled_by_text, native_title, static_value, child_label, placeholder, description: all Option<String> }`; adapters supply evidence, core computes accessible-name precedence +- `SignalBaseline` — `{ windows: Vec<WindowInfo>, apps: Vec<AppInfo>, surfaces: Vec<SurfaceSignal>, completeness: SignalCompleteness }`; `diff_signals(baseline, current) -> Vec<UiEvent>` is a pure function that never touches the adapter +- `Deadline` — `{ started_at, expires_at, timeout_ms, capped }`; thread-local `INHERITED_DEADLINE` propagation via `enter_scope` / `min_expiry` so nested reads share one budget +- `AdapterSession` — `trait AdapterSession: Send + Sync { fn close(self: Box<Self>) -> Result<(), AdapterError>; }`, returned by `open_session` +- `SurfaceWait` — command-local enum (`crates/core/src/commands/wait_surface.rs`) used by `wait`: `Menu | MenuClosed | Notification`. Not a shared core type — scoped to the `wait` command's own predicate handling. +- `ProcessId` — transparent `u32` process identifier shared by core, every adapter, JSON, and the C ABI. Platform adapters perform checked conversion only at native boundaries such as macOS `pid_t`; core never narrows a Windows `DWORD` process ID to a signed integer - `PermissionReport` — `{ accessibility, screen_recording, automation }`, each `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }` -- `MouseEvent`, `DragParams`, `KeyCombo` — dedicated types (not unified under an `InputEvent` enum) +- `MouseEvent`, `DragParams`, `KeyCombo` — dedicated types (not unified under an `InputEvent` enum). The CLI-facing `DragArgs` pairs `DragEndpoint { ref_id: Option<String>, xy: Option<(f64,f64)> }` values for `from`/`to`. - `WindowOp` — Resize{w,h}, Move{x,y}, Minimize, Maximize, Restore, Close -- `ScreenshotTarget` — Screen(usize), Window(pid), FullScreen +- `ScreenshotTarget` — Screen(usize), Display { index, expected }, ExactWindow(WindowInfo), FullScreen - `NotificationInfo` — index, app_name, title, body, actions: Vec<String> - `NotificationIdentity` — expected_app, expected_title (used for NC-reorder-safe `notification_action`) - `SurfaceInfo` — kind, label, bounds (for `list-surfaces` command) +- `SnapshotSurface` — predeclared cross-platform surface vocabulary: `Window`, `Focused`, `Menu`, `Menubar`, `Sheet`, `Popover`, `Alert`, `Desktop`, `Toolbar`, `NotificationCenter`, plus shell-specific variants already predeclared for later phases (`Dock`, `Spotlight`, `MenuBarExtras` for macOS; `Taskbar`, `SystemTray`, `SystemTrayOverflow`, `QuickSettings`, `StartMenu`, `ActionCenter` for Windows). Declaring a variant does not imply any adapter implements it — see `supported_surfaces()` introspection. - `TreeOptions` — max_depth, include_bounds, interactive_only, compact, surface, skeleton (root is CLI-only via `SnapshotArgs.root_ref`, not plumbed into `TreeOptions`) ### macOS Adapter Implementation -Located in `crates/macos/src/` following the platform crate folder structure: +Located in `crates/macos/src/` following the platform crate folder structure (49 files under `tree/`, 31 under `actions/`, 30 under `input/`, 47 under `system/`, 12 under `notifications/` — the listing below is a representative subset): ``` crates/macos/src/ -├── lib.rs # mod declarations + re-exports only -├── adapter.rs # MacOSAdapter: PlatformAdapter impl +├── lib.rs / adapter.rs / cf_type.rs # mod glue + MacOSAdapter: PlatformAdapter impl + CFType helpers ├── tree/ -│ ├── mod.rs # re-exports -│ ├── ax_element.rs # AXElement ownership wrapper -│ ├── attributes.rs # Batched AX attribute reads -│ ├── capabilities.rs # AX-supported actions and settable attributes -│ ├── builder.rs # build_subtree, tree traversal -│ ├── node_attrs.rs # Node metadata extraction -│ ├── roles.rs # AXRole string -> unified role enum mapping -│ ├── resolve*.rs # Element re-identification for ref resolution -│ └── surfaces.rs # Surface detection (menu, sheet, alert, popover) +│ ├── mod.rs, element.rs, ax_element.rs, ax_value.rs, attributes.rs, capabilities.rs +│ ├── build_context.rs, element_bounds.rs, element_dedupe.rs, element_name.rs, child_labels.rs +│ ├── hit_test.rs, native_id.rs, roles.rs, state_reader.rs, surfaces.rs, surface_inventory.rs +│ ├── resolve.rs, resolve_classify.rs, resolve_search.rs, resolve_bounds.rs, resolve_roots.rs, locator_deadline.rs +│ ├── query/ # live-locator query evaluation glue +│ └── node_attribute_*.rs, node_attr_states.rs, node_control_states.rs, node_semantic_states.rs, node_identifiers.rs (batched attribute read/decode family) ├── actions/ -│ ├── mod.rs # re-exports -│ ├── dispatch.rs # perform_action match arms -│ ├── chain*.rs # policy-aware AX-first activation chain -│ ├── discovery.rs # Live capability discovery -│ ├── extras.rs # select_value helpers -│ ├── post_state.rs # Post-action state reads -│ ├── scroll.rs # scroll semantics and explicit physical policy paths -│ └── type_text.rs # focus-fallback text insertion and physical typing +│ ├── mod.rs, dispatch.rs, adapter.rs, ax_helpers.rs, ax_mutation.rs +│ ├── chain.rs, chain_context.rs, chain_def.rs, chain_defs.rs, chain_delivery.rs, chain_disclosure_steps.rs, chain_menu_steps.rs, chain_step.rs, chain_step_exec.rs, chain_value_write.rs, chain_verify.rs +│ ├── delivery_tracker.rs, post_state.rs, extras.rs, select_menu.rs, toggle_state.rs +│ ├── physical_click.rs, physical_keyboard.rs, physical_target.rs, type_text.rs +│ └── scroll.rs, scroll_into_view.rs, scroll_read.rs ├── input/ -│ ├── mod.rs # re-exports -│ ├── keyboard.rs # CGEventCreateKeyboardEvent, key synthesis, text typing -│ ├── keyboard_map.rs # Key name mapping -│ ├── mouse.rs # CGEventCreateMouseEvent, mouse events -│ └── clipboard.rs # NSPasteboard.generalPasteboard read/write +│ ├── mod.rs, adapter.rs, keyboard.rs, keyboard_event.rs, keyboard_map.rs +│ ├── mouse.rs, mouse_drag.rs, mouse_drag_state.rs, mouse_move.rs, mouse_scroll.rs +│ ├── blocked_combo.rs, owned_object.rs +│ └── clipboard.rs, clipboard_file_urls.rs, clipboard_image_io.rs, clipboard_rich.rs, clipboard_runtime.rs, clipboard_transaction.rs, clipboard_helper_{client,dl,entry,identity,process,protocol}.rs # typed clipboard + isolated-helper protocol family ├── notifications/ -│ ├── mod.rs # re-exports -│ ├── list.rs # List notifications via Notification Center AX tree -│ ├── dismiss.rs # Dismiss individual or all notifications via AXPress -│ └── actions.rs # Click notification action buttons (identity-verified) +│ ├── mod.rs, list.rs, actions.rs, dismiss_verify.rs, read.rs, scan.rs +│ └── nc_session.rs # NcSession / NcSessionOps RAII lifecycle (open/wait-ready/close/reactivate) └── system/ - ├── mod.rs # re-exports - ├── app_ops.rs # launch, close, focus via NSWorkspace - ├── app_list.rs # Running app inventory - ├── window_list.rs # Window inventory - ├── window_ops.rs # window resize, move, minimize, maximize, restore - ├── key_dispatch.rs # app-targeted key press - ├── permissions.rs # PermissionReport probe/request - ├── screenshot.rs # ScreenshotBackend + secure screencapture path - └── wait.rs # wait utilities + ├── mod.rs, adapter.rs, app_inventory.rs, app_ops.rs, process.rs, process_apps.rs, process_identity.rs, process_state.rs + ├── window_inventory.rs, window_inventory_global.rs, window_ops.rs, window_ax_state.rs, window_bridge.rs, window_postcondition.rs, window_resolve.rs + ├── display.rs, display_work_area.rs, cg_window.rs, cg_window_exact.rs + ├── focus.rs, key_dispatch.rs, renderer_activation.rs, launch.rs, launch_bridge.m, launch_workspace.rs, launch_completion.rs, launch_callback_result.rs + ├── permission_helper.rs, permission_operation.rs, permissions.rs + ├── screenshot.rs, screen_bridge.m, screen_bridge_contract_tests.rs + ├── signals.rs, wait.rs + └── appkit_bridge.m / appkit_bridge.rs, cocoa_runtime.rs, workspace_apps.rs ``` **Tree traversal:** @@ -278,29 +340,31 @@ crates/macos/src/ - Batch fetch: `AXUIElementCopyMultipleAttributeValues` for 3-5x faster attribute reads - Role mapping: AXRole strings → unified role enum in `tree/roles.rs` - Max depth default: 10, configurable via `--max-depth` -- Name: `kAXTitleAttribute` / `kAXDescriptionAttribute`. Value: `kAXValueAttribute` +- Name: core `accname.rs` computes accessible-name precedence over the `NameEvidence` the adapter supplies (`kAXTitleAttribute` / `kAXDescriptionAttribute` / static value / child label / placeholder). Value: `kAXValueAttribute` - Bounds: `kAXPositionAttribute` + `kAXSizeAttribute` combined to Rect +- Identity: `native_id` reads macOS `AXIdentifier`, falling back to `AXDOMIdentifier` for web content **Action execution:** -- Ref actions take `ActionRequest`, not bare `Action` +- Ref actions take `ActionRequest`, not bare `Action`, and auto-wait for actionability before dispatch (Phase 1.6) - Default policy forbids focus stealing and cursor movement -- Click/right-click/scroll chains run semantic AX steps first and return structured errors instead of silently using physical/headed paths +- Click/right-click/scroll use semantic AX delivery headlessly and prefer physical delivery only under explicit `--headed` - Type uses the focus-fallback policy floor; SetValue/Clear are the pure headless AX value-mutation paths - SetValue/Clear: `AXUIElementSetAttributeValue(kAXValueAttribute, value)` - SetFocus/Press/Hover/Drag/Mouse: explicit focus/cursor/physical commands -- Keyboard/Mouse: `CGEventCreateKeyboardEvent` / `CGEventCreateMouseEvent` via CoreGraphics -- Clipboard: `NSPasteboard.generalPasteboard` read/write via Cocoa FFI +- Keyboard/Mouse: `CGEventCreateKeyboardEvent` / `CGEventCreateMouseEvent` via CoreGraphics; mouse events carry modifier chords and wheel deltas +- Clipboard: `NSPasteboard.generalPasteboard` read/write via Cocoa FFI, marshaled through typed `ClipboardContent` (`Text` / `Image` / `FileUrls`) via the isolated clipboard-helper protocol - Screenshot: `ScreenshotBackend` boundary with secure temporary files; Screen Recording denial maps to `PERM_DENIED` +- Every step reports a typed `ActionStep` (`SemanticApi` vs `PhysicalSynthetic` mechanism, `verified: Option<bool>`) — see Phase 1.6 **Permission detection:** - Probe once per CLI process into `PermissionReport` - Accessibility: `AXIsProcessTrusted()` / `AXIsProcessTrustedWithOptions(prompt: true)` - Screen Recording: platform screen-capture preflight/request path -- Automation: currently `{ "state": "not_required" }` because the shipped command set does not use Apple Events; future Apple Event paths must report a real granted/denied probe +- Automation: probed against System Events without prompting for `permissions` / `status`; `permissions --request` may prompt through the bounded isolated permission helper. The probe is truthful (Phase 1.6 U4) rather than optimistically assumed. - `status`, `permissions`, preflight, and `batch` share the same report; `permissions --request` invokes the request path **Notification management:** -- Open Notification Center via AX: target the `NotificationCenter` process (bundleId: `com.apple.notificationcenterui`) +- Open Notification Center through the guarded System Events path to Control Center's Clock item; authorization is probed without prompting before the Apple Event is sent - List notifications: traverse the Notification Center AX tree — each notification is an `AXGroup` with title, subtitle, and action buttons - Dismiss: perform `AXPress` on the notification's close button, or `AXRemoveFromParent` if supported - Interact: resolve action buttons within a notification group and perform `AXPress` @@ -324,11 +388,11 @@ Platform-agnostic, lives in `agent-desktop-core`: 1. Raw tree: Call `adapter.get_tree(window, opts)` 2. Filter: Remove invisible/offscreen. Remove empty groups with no interactive descendants. Prune beyond max_depth -3. Allocate refs: Depth-first. Interactive roles get `@e1`, `@e2`, etc. Store in RefMap +3. Allocate refs: Depth-first. Interactive roles get sequence numbers `e1`, `e2`, etc., emitted snapshot-qualified as `@<snapshot_id>:e1`, `@<snapshot_id>:e2` (e.g. `@s8f3k2p9:e1`). Legacy bare `@e1` remains valid input only with an explicit `--snapshot <id>`. Store in RefMap 4. Serialize: Omit null fields. Omit empty arrays. Omit bounds in compact mode 5. Estimate tokens: Optionally warn if exceeding threshold -Snapshot refs persist through `RefStore`. The default namespace stores snapshots under `~/.agent-desktop/snapshots/{snapshot_id}/refmap.json`; `--session <id>` stores the same shape under `~/.agent-desktop/sessions/{id}/snapshots/{snapshot_id}/refmap.json`. Each namespace owns one `latest_snapshot_id` pointer for commands that omit `--snapshot`. Explicit `--snapshot <id>` is a direct snapshot handle and can be used without repeating `--session`; if the same snapshot ID appears in multiple sessions, callers pass the matching session to disambiguate. `~/.agent-desktop/last_refmap.json` remains only as a latest-snapshot inspection artifact. Action commands resolve through `RefStore` and use `ResolvedElement` RAII so native handles are released after ref-consuming commands. Return `STALE_REF` on live re-identification mismatch and `SNAPSHOT_NOT_FOUND` when the requested snapshot does not exist. +Snapshot refs persist through `RefStore`. The default namespace stores snapshots under `~/.agent-desktop/snapshots/{snapshot_id}/refmap.json`; `--session <id>` stores the same shape under `~/.agent-desktop/sessions/{id}/snapshots/{snapshot_id}/refmap.json`. Each namespace owns one `latest_snapshot_id` pointer for commands that omit `--snapshot`. A qualified ref or explicit `--snapshot <id>` identifies the snapshot within the selected namespace; session-owned snapshots still require the matching `--session` or `AGENT_DESKTOP_SESSION` scope, and lookup never searches another namespace. `~/.agent-desktop/last_refmap.json` remains only as a latest-snapshot inspection artifact. Action commands resolve through `RefStore` using strict re-identification from platform-neutral `RefEntry` evidence — pid, role, path/source surface, role-conditional stable text identity, and bounds hash; mutable control values are volatile and are never treated as stable text identity. Return `STALE_REF` on 0 live candidates and `AMBIGUOUS_TARGET` on 2+ plausible live candidates. **Progressive Skeleton Traversal:** - `--skeleton` flag clamps depth to `min(max_depth, 3)`, annotates truncated containers with `children_count` for agent discovery @@ -361,40 +425,46 @@ Snapshot refs persist through `RefStore`. The default namespace stores snapshots | `click-tray-item` | Click a system tray item by ID or app name | `<tray-item-id>` | | `open-tray-menu` | Click a tray item and snapshot its resulting menu for ref-based interaction | `<tray-item-id>` | -#### Wait Command Update (Notification — Completed, Menu — Completed) +#### Wait Command Update (Notification — Completed, Menu — Completed, Event/Selector — Completed in 1.6) -The `wait` command has been extended with notification and menu support: +The `wait` command has been extended with notification, menu, event-diff, and selector-polling support: - `wait --notification` — Wait for any new notification to appear (index-diff based detection) - `wait --notification --app Safari` — Wait for a notification from a specific app - `wait --notification --text "Download complete"` — Wait for a notification containing specific text - `wait --menu` / `wait --menu-closed` — Wait for context menu open/close +- `wait --event <kind> [--app ...] [--window-id ...]` — Wait for a `SignalBaseline` diff event (window-opened/closed, app-launched/terminated, focus-changed, surface-appeared); see [Phase 1.6](#phase-16--playwright-grade-foundation-contract-completed) +- `--wait-for <selector>` — Poll a `LocatorQuery`-shaped selector until it matches (v0.4.5) -### Commands Shipped (54) +### Commands Shipped (58 names — 54 operational, 4 fail closed) | Category | Commands | Count | |----------|----------|-------| -| App / Window | `launch`, `close-app`, `list-windows`, `list-apps`, `focus-window`, `resize-window`, `move-window`, `minimize`, `maximize`, `restore` | 10 | -| Observation | `snapshot`, `screenshot`, `find`, `get` (text, value, title, bounds, role, states, tree-stats), `is` (visible, enabled, checked, focused, expanded), `list-surfaces` | 6 | +| App / Window | `launch`, `close-app`, `list-windows`, `list-displays`, `list-apps`, `focus-window`, `resize-window`, `move-window`, `minimize`, `maximize`, `restore` | 11 | +| Observation | `snapshot`, `screenshot`, `find` (live `LocatorQuery` lookup), `get` (text, value, title, bounds, role, states, tree-stats), `is` (live property reads: visible, enabled, checked, focused, expanded, ...), `list-surfaces` | 6 | | Interaction | `click`, `double-click`, `triple-click`, `right-click`, `type`, `set-value`, `clear`, `focus`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse` | 14 | | Scroll | `scroll`, `scroll-to` | 2 | -| Keyboard | `press`, `key-down`, `key-up` | 3 | -| Mouse | `hover`, `drag`, `mouse-move`, `mouse-click`, `mouse-down`, `mouse-up` | 6 | +| Keyboard | `press`, `key-down`\*, `key-up`\* | 3 | +| Mouse | `hover`, `drag`, `mouse-move`, `mouse-click`, `mouse-down`\*, `mouse-up`\*, `mouse-wheel` | 7 | | Clipboard | `clipboard-get`, `clipboard-set`, `clipboard-clear` | 3 | | Notification (macOS) | `list-notifications`, `dismiss-notification`, `dismiss-all-notifications`, `notification-action` | 4 | -| Wait | `wait` (with `--element`, `--window`, `--text`, `--menu`, `--notification` flags) | 1 | -| System | `status`, `permissions`, `version`, `skills` | 4 | +| Wait | `wait` (`--element`, `--window`, `--text`, `--menu`, `--notification`, `--event <kind>`, `--app`, `--window-id` flags) | 1 | +| System | `status`, `permissions`, `version`, `skills`, `session` (start/end/list/gc), `trace` (export/show) | 6 | | Batch | `batch` | 1 | +11+6+14+2+3+7+3+4+1+6+1 = 58 (54 operational + 4 fail-closed). + +\* **Fail closed:** `key-down`, `key-up`, `mouse-down`, `mouse-up` parse and validate their arguments, then unconditionally return an error through `input_hold_policy::reject(...)` before ever calling the adapter (`crates/core/src/commands/key_down.rs`, `key_up.rs`, `mouse_down.rs`, `mouse_up.rs`). Held input (press without an immediate matching release) needs an owner that outlives a single CLI invocation to guarantee release — that owner is the Phase 5 daemon. Until then, these four names exist in the CLI/FFI/MCP surface — so scripts and generated schemas can reference them by a stable name — but they always fail with a suggestion to use `press`, `mouse-click`, or `drag` instead. + > System Tray / Menu Bar Extras commands are listed under "Not Yet Implemented" above — they never shipped in Phase 1. ### JSON Output Contract -All commands produce a response envelope with `version: "2.0"`. Standalone schema files are still deferred; the current contract is enforced by Rust serde types, CLI conformance tests, and documented examples. +All commands produce a response envelope with `version: "2.1"`. Standalone schema files are still deferred; the current contract is enforced by Rust serde types, CLI conformance tests, and documented examples. Success: ```json { - "version": "2.0", + "version": "2.1", "ok": true, "command": "snapshot", "data": { @@ -409,27 +479,38 @@ Success: Error: ```json { - "version": "2.0", + "version": "2.1", "ok": false, "command": "click", "error": { "code": "STALE_REF", "message": "Element could not be resolved from the requested snapshot", - "suggestion": "Run 'snapshot' to refresh, then retry with updated ref" + "suggestion": "Run 'snapshot' to refresh, then retry with updated ref", + "recovery": { + "strategy": "refresh_snapshot_then_retry_original", + "retryable": true, + "requires_fresh_snapshot": true + }, + "disposition": { + "delivery": "not_delivered", + "retry": "safe" + } } } ``` +The `error` object may also carry an optional `details` object. `recovery` and `disposition` are both present whenever the error type has a well-defined recovery path; consumers use `recovery.strategy` only when `disposition.retry` is `"safe"`. Envelope 2.1 removed the 2.0 `retry_command` string outright — there is no compatibility alias, so 2.0 consumers reading `retry_command` must migrate to `recovery.strategy`. + Serialization rules: omit null/None fields (`skip_serializing_if`), omit empty arrays, omit bounds in compact mode, `ref_count` and `tree` inside `data`. ### Error Taxonomy -The `ErrorCode` enum in `crates/core/src/error.rs` exposes these machine-readable variants: +The `ErrorCode` enum in `crates/core/src/error_code.rs` (`AdapterError` lives in `adapter_error.rs`, `AppError` in `app_error.rs` — there is no `error.rs`) exposes these machine-readable variants: | Code | Category | Example | Recovery Suggestion | |------|----------|---------|---------------------| | `PERM_DENIED` | Permission | Accessibility not granted | Open System Settings > Privacy > Accessibility and add the app that launches agent-desktop | -| `ELEMENT_NOT_FOUND` | Ref | @e12 could not be resolved | Run 'snapshot' to refresh, then retry with updated ref | +| `ELEMENT_NOT_FOUND` | Ref | @s8f3k2p9:e12 could not be resolved | Run 'snapshot' to refresh, then retry with updated ref | | `APP_NOT_FOUND` | Application | --app 'Photoshop' not running | Launch the application first | | `ACTION_FAILED` | Execution | AXPress returned error on disabled button | Element may be disabled. Check states before acting | | `ACTION_NOT_SUPPORTED` | Execution | Expand on a button | This element does not support the requested action | @@ -437,16 +518,17 @@ The `ErrorCode` enum in `crates/core/src/error.rs` exposes these machine-readabl | `AMBIGUOUS_TARGET` | Ref | Ref identity maps to more than one live candidate | Run 'snapshot' to refresh, then retry with a more specific ref | | `WINDOW_NOT_FOUND` | Window | --window w-999 does not exist | Run 'list-windows' to see available windows | | `PLATFORM_NOT_SUPPORTED` | Platform | Windows/Linux adapter not yet shipped | This platform ships in Phase 2/3 | -| `TIMEOUT` | Wait / Traversal | wait --element exceeded timeout | Increase --timeout or check app state | +| `TIMEOUT` | Wait / Traversal / Auto-wait | wait --element exceeded timeout; auto-wait exceeded 5000ms without the target becoming actionable | Increase --timeout or --timeout-ms, or check app state | | `INVALID_ARGS` | Input | Bad CLI argument or unknown ref format | Fix the argument per CLI help | | `NOTIFICATION_NOT_FOUND` | Notification | Notification ID not found / NC reordered | Run 'list-notifications' to see current notifications | | `SNAPSHOT_NOT_FOUND` | Ref | Requested snapshot ID is missing | Run 'snapshot' again and use the returned snapshot_id | | `POLICY_DENIED` | Action policy | Physical input blocked by headless policy | Retry with `--headed` for explicit cursor movement, or use a semantic AX action when available | +| `APP_UNRESPONSIVE` | Liveness | Target process failed a terminal read-only liveness probe (hung app window, not-responding) | Wait for the app to recover, or close/relaunch it — this is a terminal enrichment only; transient AX messaging blips degrade gracefully and never by themselves fail a whole command | | `INTERNAL` | Internal | Unexpected error or caught panic | Re-run with verbose logging | Exit codes: `0` success, `1` structured error (JSON on stdout), `2` argument/parse error. -> Codes the earlier draft listed but that **do not exist** in the codebase: `TREE_TIMEOUT` (use `TIMEOUT`), `CLIPBOARD_EMPTY` (no special code; empty clipboard returns empty string), `NOTIFICATION_UNSUPPORTED` (use `PLATFORM_NOT_SUPPORTED`), `TRAY_NOT_FOUND` / `TRAY_UNSUPPORTED` (tray commands never shipped). Deferred-work additions (see Gap Analysis at bottom): `PERMISSION_REVOKED`, `RESOURCE_EXHAUSTED`, `AX_MESSAGING_TIMEOUT`, `AUTOMATION_PERMISSION_DENIED`. +> Codes the earlier draft listed but that **do not exist** in the codebase: `TREE_TIMEOUT` (use `TIMEOUT`), `CLIPBOARD_EMPTY` (no special code; empty clipboard returns an empty string / `None` content), `NOTIFICATION_UNSUPPORTED` (use `PLATFORM_NOT_SUPPORTED`), `TRAY_NOT_FOUND` / `TRAY_UNSUPPORTED` (tray commands never shipped). The liveness enrichment shipped in Phase 1.6: `APP_UNRESPONSIVE` is now a real code (a failed read-only liveness probe upgrades a hang via `ProcessState::Unresponsive`), while ordinary AX messaging exhaustion still reports plain `TIMEOUT` — the once-mooted `AX_MESSAGING_TIMEOUT` was never added as a variant, and `AUTOMATION_PERMISSION_DENIED` folded into `PERM_DENIED` with platform detail. Remaining future-candidate codes: `PERMISSION_REVOKED` (TCC yanked mid-session, distinct from `PERM_DENIED`), `RESOURCE_EXHAUSTED` (refmap >1MB, tree node-count cap). ### Testing @@ -480,18 +562,30 @@ Exit codes: `0` success, `1` structured error (JSON on stdout), `2` argument/par - Real snapshots from Finder, TextEdit, etc. checked into repo - Regression-test serialization format changes +**Live e2e (`tests/e2e/run.sh`, Phase 1.6):** +- Requires macOS + `AGENT_DESKTOP_E2E_EXCLUSIVE=1`; builds the release binary, macOS helper, and `release-ffi` cdylib +- Re-execs itself through `interaction_lock.py run` for exclusive-desktop serialization +- Drives the release binary against the SwiftUI/AppKit fixture app (`tests/fixture-app/AgentDeskFixture.swift`) and asserts every effect by independent observation — never the command's own `ok:true` +- Covers every ref action in both headless and `--headed` mode + ### CI -Current `.github/workflows/ci.yml` on every PR: -- `fmt` job on `ubuntu-latest`: `cargo fmt --all -- --check` -- `test` job on `macos-latest`: - - `cargo tree -p agent-desktop-core` must contain zero platform crate names (dependency isolation) - - `cargo clippy --all-targets -- -D warnings` - - `cargo test --lib --workspace` - - `cargo test -p agent-desktop-ffi --tests` (c_abi_harness + c_header_compile + error_lifetime integration suites) - - `cargo build --profile ci` (fast CLI binary) + 15 MB size check - - `cargo build --profile release-ffi -p agent-desktop-ffi` (the shipped cdylib profile) - - FFI header contract check — compiles `crates/ffi/include/agent_desktop.h` from C tests and keeps header regeneration out of the default build graph +Current `.github/workflows/ci.yml` runs on push to main/master, pull_request, and workflow_dispatch: + +| Job | Runner | What it checks | +|-----|--------|-----------------| +| `fmt` | ubuntu-latest | `cargo fmt --all -- --check`; shellcheck + bash3-compat on e2e scripts; `py_compile` + unittest on `tests/e2e/*.py`; actionlint on workflow files | +| `msrv` | ubuntu-latest | `cargo +1.89.0 check` (pinned MSRV) on core, linux, and the binary crate | +| `platform-check` | matrix: Linux / Windows / macOS | `cargo check --all-targets` per platform crate + binary — proves the stub crates compile on their target before any adapter code lands | +| `test` | macos-latest | Dependency isolation check (`cargo tree -p agent-desktop-core` has zero platform crate names), release-consistency check, file-size rule check, `cargo clippy --all-targets -- -D warnings`, core+macos unit tests, `locator_benchmark` example, `permission-contract.sh`, binary command tests, FFI integration tests, release binary build + version-flag check + 15MB size check, FFI cdylib build (`release-ffi` profile), FFI helper-discovery smoke, npm package tests + wrapper smoke | +| `ffi-python-smoke` | macos-latest | Builds the FFI dylib with the `stub-adapter` feature, runs `tests/ffi-python/smoke.py` against it | +| `ffi-header-drift` | macos-latest | `cbindgen 0.29.4 --verify` against the committed `crates/ffi/include/agent_desktop.h` | +| `ffi-panic-guard` | macos-latest | Asserts `profile.release-ffi` keeps `panic = "unwind"`; runs `crates/ffi/tests/run_cdylib_panic_probe.sh` | +| `ffi-passthrough` | ubuntu-latest | `cargo test -p agent-desktop-ffi --features stub-adapter --test c_abi_passthrough` — Family-B entrypoints (`ad_snapshot` / `ad_status` / `ad_wait` / `ad_execute_by_ref` / `ad_version` + `ad_init` / `ad_destroy` / `ad_check_permissions`) against the stub adapter | + +Three more workflows run outside `ci.yml`: `native-e2e.yml` (workflow_dispatch only, self-hosted `[self-hosted, macOS, agent-desktop-e2e]` — builds and runs the exclusive native E2E suite via `scripts/run-native-e2e-ci.sh`), `codeql.yml` (pull_request + push to main/master + weekly cron + workflow_dispatch — CodeQL analysis for GitHub Actions/JS-TS and Rust), and `supply-chain.yml` (pull_request + push to main/master + weekly cron + workflow_dispatch — release-metadata consistency, npm package/publish policy, `cargo-deny`, `zizmor` workflow security audit). `release.yml` (push to main + workflow_dispatch) runs `release-please`, the per-target `build` and `build-ffi` matrices, `ffi-release-gates`, and the `publish-github` / `publish-npm` / `publish-skills` jobs. + +Every substantive change also runs the performance baseline gate — `bash scripts/perf-baseline-compare.sh` against the merge-base, with the resulting `report.html` reviewed for unintentional latency deltas before merge. This is part of `CLAUDE.md`'s Definition of Done as of the Phase 1.6 branch tip, and is treated as a per-sub-phase gate in Phase 2/3 (see [Platform Delivery Model](#platform-delivery-model--sub-phases-and-integration-branches)). ### Dependencies @@ -520,7 +614,7 @@ Current `.github/workflows/ci.yml` on every PR: ## Phase 1.5 — FFI Distribution (C-ABI cdylib) -**Status: Completed — v0.1.13 (2026-04-17).** +**Status: Completed — v0.1.13 (2026-04-17).** The C-ABI surface itself (load-time handshake, session-scoped adapter, JSON-envelope command entrypoints) completed later, in v0.4.1. Phase 1.5 ships `crates/ffi/` as a first-class distribution target. The CLI stays the primary surface; the cdylib lets Python (ctypes), Swift, Node (ffi-napi), Go (cgo), Ruby (fiddle), and C consumers call `PlatformAdapter` directly without spawning `agent-desktop` per call. @@ -544,14 +638,16 @@ Phase 1.5 ships `crates/ffi/` as a first-class distribution target. The CLI stay crates/ffi/ ├── Cargo.toml # crate-type = ["cdylib", "rlib"] ├── cbindgen.toml # maintainer-only header regeneration config -├── build.rs # bakes install_name = @rpath/libagent_desktop_ffi.dylib on macOS +├── build.rs # 5 lines: bakes install_name = @rpath/libagent_desktop_ffi.dylib on macOS only — no codegen step +├── codegen_templates/ # empty, untracked — reserved for the P2-O16 build.rs codegen migration, not wired up today ├── include/ -│ └── agent_desktop.h # committed, drift-checked against the OUT_DIR output +│ └── agent_desktop.h # committed, drift-checked against the OUT_DIR output; AD_ABI_VERSION_MAJOR = 3 ├── src/ # ad_* extern "C" entrypoints, organized by domain │ ├── types/ # 34 one-type-per-file modules (AdAction, AdRect, AdWindowList, ...) │ ├── convert/ # string / rect / window / app / surface / notification helpers │ ├── tree/ # BFS flat-tree layout (flatten.rs, get.rs, free.rs) │ ├── actions/ # conversion, resolve, execute, result, native_handle +│ ├── commands/ # 8 hand-written ad_* command-backed wrappers (see below); `generated/` subdir is empty/untracked and not compiled — no `mod generated;` in `lib.rs` │ ├── apps/ windows/ input/ screenshot/ surfaces/ notifications/ observation/ │ ├── error.rs # AdResult, errno-style TLS last-error (message/suggestion/platform_detail) │ ├── ffi_try.rs # panic boundary helpers (trap_panic_*) @@ -560,11 +656,29 @@ crates/ffi/ ├── tests/ │ ├── c_abi_harness.rs # raw extern "C" decls, enum fuzzing, out-param zeroing, null tolerance │ ├── c_header_compile.rs # shells out to `cc` to verify every AD_* constant is usable from C +│ ├── c_abi_passthrough.rs # Family-B command entrypoints vs stub adapter (ffi-passthrough CI job) │ └── error_lifetime.rs # last-error pointer stability across successful follow-up calls └── examples/ └── panic_spike.rs # demonstrates panic boundary on the release-ffi profile ``` +**Command-backed entrypoints** (`crates/ffi/src/commands/`, all 8 hand-written today): + +| File | Entrypoint | +|------|------------| +| `execute_by_ref.rs` | `ad_execute_by_ref` | +| `execute_by_ref_timeout.rs` | `ad_execute_by_ref_timeout` | +| `snapshot.rs` | `ad_snapshot` | +| `status.rs` | `ad_status` | +| `trace_export.rs` | `ad_trace_export` | +| `trace_show.rs` | `ad_trace_show` | +| `version.rs` | `ad_version` | +| `wait.rs` | `ad_wait` | + +`envelope_out.rs` and `timeout.rs` in the same directory are shared helpers, not entrypoints; `mod.rs` is module glue. The `ffi-passthrough` CI job exercises the original five Family-B entrypoints (`ad_snapshot`, `ad_status`, `ad_wait`, `ad_execute_by_ref`, `ad_version`) plus `ad_init` / `ad_destroy` / `ad_check_permissions` against the stub adapter; `ad_execute_by_ref_timeout`, `ad_trace_export`, and `ad_trace_show` joined the entrypoint set afterward. + +`wait`'s core event-wait mode (`--event` / `--window-id`) is intentionally **not** exposed over FFI in this release — `wait_args_from_ffi` (`crates/ffi/src/types/wait_args.rs`) always forwards `event: None` and `window_id: None` to core, documented inline. + ### Release Artifacts Shipped via `.github/workflows/release.yml` `build-ffi` matrix job: @@ -591,14 +705,15 @@ Regular `release` profile keeps `panic = "abort"` for the CLI binary, so a panic ### CI Hooks Added -Current gates in `.github/workflows/ci.yml` on every PR: +Current gates in `.github/workflows/ci.yml`: -- `cargo build --profile release-ffi -p agent-desktop-ffi` + `cargo test -p agent-desktop-ffi --tests` (`c_abi_harness`, `c_header_compile`, `error_lifetime` integration suites) in the main `test` job -- `ffi-python-smoke` — Python ctypes smoke harness (`tests/ffi-python/smoke.py`) with `AD_EXPECT_STUB=1` -- `ffi-header-drift` — `cbindgen --verify` (pinned 0.29.4) against the committed header; exits non-zero on any diff -- `ffi-panic-guard` — asserts `panic = "unwind"` in `Cargo.toml` and runs the `panic_spike` example to prove `catch_unwind` survives the `release-ffi` profile -- `ffi-passthrough` — `--test c_abi_passthrough --features stub-adapter` confirms command entrypoints return `PLATFORM_NOT_SUPPORTED` envelopes -- `ffi-codegen-drift` — regenerates `src/commands/generated.rs` and checks for drift against the committed file +- `cargo build --profile release-ffi -p agent-desktop-ffi` + FFI integration suites (`c_abi_harness`, `c_header_compile`, `error_lifetime`) inside the main `test` job (macos-latest) +- `ffi-python-smoke` (macos-latest) — Python ctypes smoke harness (`tests/ffi-python/smoke.py`) against a dylib built with the `stub-adapter` feature +- `ffi-header-drift` (macos-latest) — `cbindgen --verify` (pinned 0.29.4) against the committed header; exits non-zero on any diff +- `ffi-panic-guard` (macos-latest) — asserts `panic = "unwind"` in `Cargo.toml` and runs the `panic_spike` example to prove `catch_unwind` survives the `release-ffi` profile +- `ffi-passthrough` (ubuntu-latest) — `--test c_abi_passthrough --features stub-adapter` confirms the command entrypoints round-trip against the stub adapter + +There is no `ffi-codegen-drift` job. An earlier draft of this document described one; it does not exist because there is no codegen step to drift-check yet — see Gap Status below. ### New Dependencies @@ -609,40 +724,101 @@ Current gates in `.github/workflows/ci.yml` on every PR: ### Forward Compatibility -- Pre-1.0 the ABI is explicitly unstable; consumers pin the artifact version alongside the cdylib version. +- Pre-1.0 the ABI is explicitly unstable; consumers pin the artifact version alongside the cdylib version. `AD_ABI_VERSION_MAJOR` is currently `3` and evolves append-only (Phase 1.6). - Any new `PlatformAdapter` method that lands in Phase 2/3 must add a matching `ad_*` FFI wrapper in the same PR that adds the adapter method. - MCP server mode (Phase 4) is a parallel transport, not an FFI consumer — it calls `PlatformAdapter` directly. -### Gap Status (from 2026-04-17 research) +### Gap Status (updated against current HEAD on `feat/foundation-playwright-grade-contract`) **Resolved:** -- `ad_abi_version()` and `ad_init(expected_major)` now ship; consumers call `ad_init` after `dlopen` for a runtime compat check. -- `ad_snapshot`, `ad_execute_by_ref`, `ad_wait`, `ad_version`, and `ad_status` are now exported. The 5 command-backed wrappers are generated by `build.rs` from `codegen_templates/*.rs.in` into the committed `src/commands/generated.rs`; do not hand-edit that file. +- `ad_abi_version()` and `ad_init(expected_major)` ship; consumers call `ad_init` after `dlopen` for a runtime compat check. `AD_ABI_VERSION_MAJOR` is currently `3`. +- `ad_snapshot`, `ad_execute_by_ref`, `ad_wait`, `ad_version`, and `ad_status` are exported, joined since by `ad_execute_by_ref_timeout`, `ad_trace_export`, and `ad_trace_show`. - `ad_set_log_callback(fn(level, msg))` ships; in-process consumers can install a tracing layer for debug output. +**Correction (this section previously claimed the opposite):** all 8 command-backed wrappers under `crates/ffi/src/commands/` are **hand-written today**, not generated. `crates/ffi/build.rs` is 5 lines and only emits the macOS linker `-install_name` argument. `crates/ffi/codegen_templates/` and `crates/ffi/src/commands/generated/` both exist but are empty and untracked, and `crates/ffi/src/lib.rs` has no `mod generated;` — that directory is not compiled. No `src/commands/generated.rs` file exists anywhere in the repository. The deterministic `build.rs`-driven codegen migration remains open work — see P2-O16. + **Still open:** - No `pyo3` / `maturin` wheel or `cffi` wrapper ships with the repo — the Python consumer path is ctypes. Potential Phase 2 follow-up. - -P2-O16 (below) scopes to the remaining registry migration: full `CommandDescriptor`-registry codegen so adding a CLI command automatically generates an FFI entry, versus the current 5-command template set. +- P2-O16 (below) scopes the full registry migration: `build.rs` codegen walks a compile-time command registry and emits one `ad_<name>` wrapper per command, replacing the current 8 hand-written wrappers. --- -## Phase 2 — Windows Adapter + Cross-Platform Feature Parity +## Phase 1.6 — Playwright-grade Foundation Contract (Completed) -**Status: Planned** — this section is the public objective catalogue and implementation contract. +**Status: Completed** — unreleased on `feat/foundation-playwright-grade-contract` (PR #93, `feat!`); will cut the next minor per the pre-1.0 versioning policy in `CLAUDE.md`. -### Core invariants (research-driven — from Phase 2 plan §Headless-First Invariant) +Phase 1.6 is a breaking hardening pass over the Phase 1 contracts, modeled on Playwright's actionability/auto-wait discipline. It does not add a platform or a command surface — every change lands in `crates/core/` (plus the macOS backfill in the same PR) and tightens what the existing 58 command names guarantee. Twenty units (U0–U19) shipped; Windows and Linux (Phase 2/3) inherit this contract rather than re-deriving it. + +### What shipped + +| Unit | What it is | +|------|------------| +| U0 | Capability-supertrait restructure: `PlatformAdapter` becomes `ObservationOps + ActionOps + InputOps + SystemOps` with a blanket impl; every method defaults to `not_supported()` unless a real default is documented | +| U1 | Canonical role/state vocabulary + a genuinely live `is --property visible` (previously derived from stale tree data) | +| U2 | macOS state-producer expansion onto the new vocabulary | +| U3 | Display contract: `list_displays` command, honest `--screen` targeting, per-display `scale_factor` | +| U4 | Truthful Automation permission — the probe no longer optimistically assumes granted | +| U5 | `native_id` identity spine end-to-end (macOS `AXIdentifier`) | +| U6 | Window identity promoted to a primary key for resolution, with recycled-window-id fail-closed handling | +| U7 | `LocatorQuery` + `resolve_query` + a live `find` command (not snapshot-only) | +| U8 | Default-on auto-wait pre-action gate for every ref action — 5000ms bound, `--timeout-ms 0` restores single-shot behavior (**breaking**) | +| U9 | Three-way `hit_test` / `receives_events` occlusion detection (`ReachesTarget` / `InterceptedBy` / `Unknown`) | +| U10 | `scroll_into_view` promoted to a core-owned contract, not an adapter-local convenience | +| U11 | Core accessible-name computation takes precedence over adapter-supplied `NameEvidence` | +| U12 | `supported_surfaces()` introspection ratifies `SnapshotSurface` as genuinely platform-neutral | +| U13 | Typed `ActionStep` delivery tier (`SemanticApi` vs `PhysicalSynthetic`) with a `verified` flag | +| U14 | `ProcessState` (`Running` / `Exited` / `Crashed` / `Unresponsive`) + `APP_UNRESPONSIVE` error code + envelope 2.1 | +| U15 | `LaunchOptions` (`--arg` / `--env` / `--cwd` / `--no-attach`) | +| U16 | `open_session` adapter-affinity hook returning `Box<dyn AdapterSession>` (Send+Sync) — the landing zone for the Windows COM-MTA worker thread and the Linux D-Bus connection | +| U17 | `SignalBaseline` capture + `diff_signals` + `wait --event` (window-opened/closed, app-launched/terminated, focus-changed, surface-appeared) — an in-invocation baseline-diff, **not** a push subscription | +| U18 | Typed clipboard content (`Text` / `Image` / `FileUrls`); the legacy untyped string clipboard API was removed | +| U19 | Mouse modifier chords + a mouse-wheel primitive | + +Also landed in the same branch, cutting across the units above: + +- `key-down` / `key-up` / `mouse-down` / `mouse-up` fail closed (`input_hold_policy::reject`) — held input is reserved for the Phase 5 daemon, which is the only thing that can guarantee a matching release. +- FFI ABI major bumped to `3`; the ABI evolves append-only. `wait`'s event-wait mode is intentionally not exposed over FFI yet (documented in `wait_args.rs`). +- Live e2e harness (`tests/e2e/run.sh`) against the SwiftUI/AppKit fixture app, both headless and `--headed`, verify-by-observation, currently 109 checks — plus a performance baseline harness (`scripts/perf-baseline-compare.sh` → `report.html`) and a performance-baseline line in `CLAUDE.md`'s Definition of Done. + +### Ref System note + +Every command that emits or accepts a ref now uses the snapshot-qualified form `@<snapshot_id>:e<n>` (e.g. `@s8f3k2p9:e5`). The bare legacy form `@e5` is still accepted, but only together with an explicit `--snapshot <id>`. Historical prose earlier in this document that predates the qualification (written when refs were process-global) may still show a bare `@e5` for brevity; treat it as shorthand for the qualified form. + +--- + +## Platform Delivery Model — Sub-phases and Integration Branches + +Phase 2 (Windows) and Phase 3 (Linux) do not ship as one monolithic implementation PR. Each platform ships as a sequence of dependency-ordered sub-phases against its own integration branch. + +- One integration branch per platform: `feat/windows-adapter`, later `feat/linux-adapter`. +- Each sub-phase is one PR into the integration branch, sized at or under 2,000 changed lines (excluding `Cargo.lock`, the generated FFI header, and vendored fixtures), reviewed on its own. +- **Lifecycle per sub-phase:** plan (via `ce-plan`, written to `docs/plans/`) → implement → per-sub-phase review → merge to the integration branch. There is explicitly **no brainstorming stage** — this document is the finalized product contract; sub-phase planning documents implementation, not product scope. +- **The workspace stays green at every merge.** Unimplemented capabilities return `not_supported()`; the CLI ships honest `PLATFORM_NOT_SUPPORTED` envelopes on the target OS until the capability lands. No sub-phase merge may regress `main`'s CI. +- **Evidence-first rule:** adapter decisions are anchored in committed probe outputs — raw UIA / AT-SPI dumps checked in alongside the sub-phase plan — never docs-only assumptions about how a platform API behaves. The exploration sub-phases (2.0, 3.0) build the initial corpus; later sub-phases extend it. +- **Source-of-truth feedback rule:** this document was authored from documentation research in a single pass; the exploration sub-phases (2.0, 3.0) exist because platform reality outranks documentation. When a committed probe proves a documented approach behaves differently on the real platform — an API that answers differently, a pattern that is unavailable, an event that never fires — the finding ledger entry and the amendment to this document land **in the same PR**. The source of truth tracks proven platform behavior; it is never defended against evidence. +- **Integration branch → main:** only after every sub-phase for that platform has merged. Promotion to `main` runs a full multi-agent review of the whole branch, live e2e (both headless and headed) on the platform runner, a performance-baseline comparison against `main`, and the standard verification contract. It lands as one release-noted `feat!` merge — the same conventional-commit discipline as PR #93. +- **Windows first.** Phase 2 (2.x) completes before Phase 3 (3.x) starts; Linux reuses the same sub-phase template with AT-SPI2/D-Bus substituted for UIA. + +Every sub-phase below follows the same rendering shape: **Goal** (one or two sentences), **Scope** (what lands), **Key APIs** (platform surface touched), **Depends on** (prior sub-phase), **Exit criteria** (what proves it's done), **Est. PR size**. + +--- + +## Phase 2 — Windows Adapter + +**Status: Planned** — delivered as sub-phases 2.0–2.15 into the `feat/windows-adapter` integration branch per the [Platform Delivery Model](#platform-delivery-model--sub-phases-and-integration-branches). This section is the public objective catalogue, the sub-phase implementation contract, and the preserved research (API mappings, capability maps, notification/tray approaches, Electron guidance) that grounds it. + +### Core invariants (research-driven — from the Phase 2 plan's Headless-First Invariant) 1. **Headless-first inside the active desktop session.** Every command — existing and Phase 2 — must run without an agent-desktop GUI, foreground activation, focus steal, or physical cursor movement unless `--headed` explicitly opts into cursor input. Windows, macOS, and Linux still require the target app to exist in the current user's interactive desktop/display session for accessibility and capture APIs. Session 0, Server Core, secure desktops, locked desktops, and other-user sessions return `PLATFORM_NOT_SUPPORTED`, `PERM_DENIED`, or `WINDOW_NOT_FOUND` with `platform_detail`, not silent best effort. The invariant is enforced by integration tests: target window is NOT focused at test entry; `list-windows --focused-only` returns the same window before/after; cursor position unchanged for headless commands. 2. **Skeleton traversal is platform-agnostic.** The novel progressive skeleton pattern (depth-3 clamp + `children_count` annotation + drill-down via `--root @ref` + scoped invalidation via `RefMap::remove_by_root_ref`) lives entirely in `crates/core/src/snapshot_ref.rs`. Windows adapter contributes ~50 LOC glue: `ControlViewWalker` (NOT `RawViewWalker` or `ContentViewWalker`) + `FindAll(TreeScope_Children, TrueCondition)` for `children_count` + fresh `UICacheRequest` per drill-down. -3. **Asymmetric event threading.** `watch_element` uses main-thread `AXObserver` on macOS (research-confirmed: Apple DTS says all AX is main-thread-only; AXSwift / Hammerspoon / Phoenix all do this); worker-thread MTA `IUIAutomation` event handler on Windows (Microsoft 2025 threading doc: UIA supports cross-thread event delivery). -4. **No `inventory` / `linkme` command registry.** Research confirmed neither survives link-GC reliably across ld64, ld-prime, GNU ld, lld, MSVC for cdylib consumers. Phase 2 uses `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` — deterministic, cdylib-safe, zero linker magic. The repository's "one command per file" rule becomes the codegen contract. -5. **FFI compatibility gates.** v0.1.14 adds explicit FFI result codes for snapshot-not-found and policy-denied paths. Phase 2 still owns `ad_abi_version()`, `ad_init(expected_major)`, and any broader ABI-version handshake before new cross-platform ABI surface ships. +3. **Asymmetric event threading.** The future push-based `watch` command (P2-O11) uses main-thread `AXObserver` on macOS (research-confirmed: Apple DTS says all AX is main-thread-only; AXSwift / Hammerspoon / Phoenix all do this); worker-thread MTA `IUIAutomation` event handler on Windows (Microsoft 2025 threading doc: UIA supports cross-thread event delivery). This is distinct from the already-shipped `wait --event`, which is an in-invocation `SignalBaseline` diff, not a subscription — see the naming note under 2.11 below. +4. **No `inventory` / `linkme` command registry.** Research confirmed neither survives link-GC reliably across ld64, ld-prime, GNU ld, lld, MSVC for cdylib consumers. Any future registry uses `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` — deterministic, cdylib-safe, zero linker magic. The repository's "one command per file" rule becomes the codegen contract when that migration (P2-O16) lands. +5. **FFI compatibility gates: shipped.** The ABI handshake (`ad_abi_version()`, `ad_init(expected_major)`) shipped in Phase 1.6; `AD_ABI_VERSION_MAJOR` is currently `3` and evolves append-only. Any new cross-platform ABI surface Phase 2 adds must preserve that handshake, not re-invent it. 6. **`DeliverFiles` replaces `FileDrop`.** Headless-first forbids `NSDraggingSession` on macOS; the new action uses a 4-tier fallback (URL scheme → `NSWorkspace.open` with `activates: false` → pasteboard + `Cmd-V` → AppleScript). Windows primary delivery is app/shell delivery (`ShellExecuteEx`, app URI handlers, `IFileOperation` for filesystem destinations, and `CF_HDROP` clipboard paste where accepted). `IDataObject + DoDragDrop` is an explicit policy-gated fallback/spike for targets that require drag semantics; it is never the default headless path. -### Windows Engineering Invariants (from Phase 2 plan Unit 3) +### Windows Engineering Invariants (from the Phase 2 plan, Unit 3) 1. `SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)` at startup. 2. `CoInitializeEx(NULL, COINIT_MULTITHREADED)` on main thread and on every dedicated UIA worker thread (UIA prefers MTA). @@ -658,15 +834,13 @@ P2-O16 (below) scopes to the remaining registry migration: full `CommandDescript 12. Session isolation: cannot drive windows in other user sessions. 13. `SetForegroundWindow` / `SetWindowPos(HWND_TOP)` is allowed only for explicit focus/window commands whose `InteractionPolicy` permits focus steal. It is never a fallback for semantic ref actions. - - -Phase 2 brings agent-desktop to Windows. It is also the phase that closes the cross-platform feature-parity gaps surfaced after the v0.1.13 FFI ship — shipping Windows meaningfully requires new core abstractions (stable identifiers, event subscriptions, text-range primitives, shell surfaces, and Windows-specific tray/taskbar affordances) that Windows UIA exposes natively and the macOS adapter currently does not surface. Every new trait method added here is implemented on both platforms in the same PR pair when there is a real cross-platform analogue. True Windows shell concepts return `PLATFORM_NOT_SUPPORTED` on other adapters through the same core command path, never through side-channel code. Linux (Phase 3) mirrors the portable parts against AT-SPI2. +Phase 2 brings agent-desktop to Windows. It is also the phase that closes the cross-platform feature-parity gaps surfaced after the v0.1.13 FFI ship — shipping Windows meaningfully requires new core abstractions (event subscriptions, text-range primitives, shell surfaces, and Windows-specific tray/taskbar affordances) that Windows UIA exposes natively and the macOS adapter does not yet surface. Phase 1.6 already delivered several of the abstractions this phase originally scoped as net-new — see the "shipped in 1.6 / remaining" split in the objectives table below. Every new trait method added here is implemented on both platforms in the same PR pair when there is a real cross-platform analogue. True Windows shell concepts return `PLATFORM_NOT_SUPPORTED` on other adapters through the same core command path, never through side-channel code. Linux (Phase 3) mirrors the portable parts against AT-SPI2. Core engine, CLI parser, JSON contract invariants, and command-registration pattern are preserved. What Phase 2 legitimately changes: `AccessibilityNode` field set, `Action` enum variants, `ErrorCode` variants, `PlatformAdapter` trait size. Every new `Action` variant must update core actionability, capability maps, platform dispatch, CLI/FFI conversion, and contract tests in the same change; exhaustive compiler checks are the guard against adapter drift. Every macOS backfill lands atomically with the Windows implementation so the two platforms never drift. -Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, every new command added in Phase 2 (`watch`, `text select-range`, `text get-selection`, `text insert-at-caret`, etc.) lives in **exactly one file** under `crates/core/src/commands/` and is wired through the shared typed command path. If Phase 2 adds codegen, it uses deterministic `build.rs` filesystem enumeration, not linker registries. The per-platform work is the three `PlatformAdapter` method implementations (one each in `crates/macos/`, `crates/windows/`, `crates/linux/`) — nothing repeats across transports. +Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, every new command added in Phase 2 (`watch`, `text select-range`, `text get-selection`, `text insert-at-caret`, etc.) lives in **exactly one file** under `crates/core/src/commands/` and is wired through the shared typed command path. If Phase 2 adds codegen, it uses deterministic `build.rs` filesystem enumeration, not linker registries. The per-platform work is the `PlatformAdapter` capability-trait method implementations (one each in `crates/macos/`, `crates/windows/`, `crates/linux/`) — nothing repeats across transports. -P2-O16 (FFI parity expansion) also migrates the FFI wrappers from hand-written to codegen: a `build.rs` step in `crates/ffi/` walks the registry and emits one `ad_<name>` extern "C" function per `CommandDescriptor`, using the per-type marshaling helpers in `crates/ffi/src/convert/`. After this migration, the FFI crate holds marshaling primitives, not command wrappers. The `crates/mcp/` crate follows the same walk-the-registry pattern with `rmcp`'s `#[tool]` shape — so Phase 4 can ship its MCP server without hand-maintaining the tool list. +P2-O16 (FFI registry migration) also migrates the FFI wrappers from hand-written to codegen: a `build.rs` step in `crates/ffi/` walks the registry and emits one `ad_<name>` extern "C" function per command, using the per-type marshaling helpers in `crates/ffi/src/convert/`. After this migration, the FFI crate holds marshaling primitives, not command wrappers. The `crates/mcp/` crate (Phase 4) follows the same walk-the-registry pattern with `rmcp`'s `#[tool]` shape — so Phase 4 can ship its MCP server without hand-maintaining the tool list. ### Objectives @@ -678,33 +852,33 @@ Core + Windows parity (original scope): | P2-O2 | All existing commands cross-platform | Identical JSON contract output on macOS and Windows for every command | | P2-O3 | Windows input synthesis | `click`, `type`, `press`, all mouse commands working via UIA + SendInput | | P2-O4 | Windows screenshot | `screenshot` produces PNG via `Windows.Graphics.Capture` API | -| P2-O5 | Windows clipboard | `clipboard-get` / `clipboard-set` / `clipboard-clear` working via Win32 Clipboard API | +| P2-O5 | Windows clipboard | `clipboard-get` / `clipboard-set` / `clipboard-clear` working via typed `ClipboardContent` over the Win32 Clipboard API | | P2-O6 | Windows CI | GitHub Actions Windows runner executes build, clippy, unit, contract, and non-interactive tests on every PR. UIA/shell integration tests that require Explorer, Start, Action Center, or an unlocked desktop run on a labeled interactive/self-hosted Windows job or are skipped with explicit `PLATFORM_NOT_SUPPORTED` assertions | | P2-O7 | Windows binary release | Prebuilt `.exe` published via GitHub Releases and npm; Phase 1.5 FFI cdylib for Windows already ships | -Cross-platform core extensions (new, landed alongside Windows): +Cross-platform core extensions (new, landed alongside Windows — each restated with its shipped/remaining split against Phase 1.6): -| ID | Objective | Metric | -|----|-----------|--------| -| P2-O8 | `AccessibilityNode` stable-selector fields | Nodes may carry `identifier`, `subrole`, `role_description`, `placeholder`, `dom_id`, `dom_classes` (all `Option<String>` / `Vec<String>` with `skip_serializing_if`). Populated where the platform/app exposes stable selectors: Windows UIA `AutomationId` / `LocalizedControlType` / `HelpText`; macOS `kAXIdentifierAttribute` / `kAXSubroleAttribute` / `kAXRoleDescriptionAttribute` / `kAXPlaceholderValueAttribute` / `kAXDOMIdentifierAttribute` / `kAXDOMClassListAttribute`. Resolver prefers stable selectors when present and falls back to the existing fingerprint; tests require known controls with explicit IDs to preserve them across re-drills, not every real-app node | -| P2-O9 | `Action` enum expansion for 2026 agent workloads | New variants: `LongPress { duration_ms }`, `ForceClick`, `ShowMenu`, `DeliverFiles(Vec<PathBuf>)` (renamed from `FileDrop` — the original name implied `NSDraggingSession` which is not headless-compatible on macOS; see Phase 2 plan §Headless-First Invariant and Unit 12), `WindowRaise`, `Cancel`, `SelectRange { start, len }`, `InsertAtCaret(String)`. `watch_element` is an adapter method, **not** an `Action` variant (plan §KD8 + origin brainstorm §D8). Each has a macOS AX API mapping (all AX calls on main thread per plan §KD9), a Windows UIA pattern mapping, a new CLI subcommand, FFI conversion coverage where applicable, and exhaustive platform-dispatch tests in the same change. | -| P2-O10 | `ErrorCode` expansion | Add `PermissionRevoked` (distinct from `PermDenied` — TCC yanked mid-session), `ResourceExhausted` (refmap >1 MB, tree node-count cap), `AxMessagingTimeout` (AX-specific timeout separate from orchestration `Timeout`), `AutomationPermissionDenied` (macOS `osascript` grant). Tri-state permission probe at startup distinguishes "never granted" from "revoked" | -| P2-O11 | Event-subscription primitive (push, not poll) | New trait method `watch_element(handle, events: &[EventKind], timeout_ms: u64) -> Result<Vec<ElementEvent>>`. macOS: `AXObserverCreate` + `AXObserverAddNotification` + `CFRunLoopSource` (no more polling in `system/wait.rs`). Windows: `IUIAutomation.AddAutomationEventHandler` + `AddFocusChangedEventHandler` + `AddPropertyChangedEventHandler`. New `wait --event value-changed --ref @e5 --timeout 3000` CLI flag. Linux mirrors in Phase 3 via AT-SPI2 D-Bus signals | -| P2-O12 | Text range primitives | Read caret, read selection, select a range by offsets, read text at range, insert at caret. macOS: `kAXSelectedTextRangeAttribute` (settable), `AXStringForRangeParameterizedAttribute`, `AXBoundsForRangeParameterizedAttribute`, `AXRangeForLineParameterizedAttribute`, `AXValueCreate(kAXValueCFRangeType, …)`. Windows: `TextPattern.GetSelection`, `TextPattern.DocumentRange`, `TextRange.Select`, `TextRange.Move`, `TextRange.GetText`, `TextRange.GetBoundingRectangles`. Commands: `text get-selection`, `text select-range <ref> <start> <len>`, `text insert-at-caret <ref> <string>`, `text at-offset <ref> <start> <len>` | -| P2-O13 | Modern per-window screenshot APIs | macOS: replace `/usr/sbin/screencapture` subprocess with `SCScreenshotManager.captureImage(contentFilter:config:)` filtered to a specific `CGWindowID` from `SCShareableContent.windows`. Windows: `Windows.Graphics.Capture` via `GraphicsCaptureItem.CreateFromWindowHandle(HWND)` + `Direct3D11CaptureFramePool` when supported by the OS/session. No subprocess on the modern path, explicit fallback to legacy capture when unavailable, and permission/support failures map to structured `PERM_DENIED` / `PLATFORM_NOT_SUPPORTED` with `platform_detail` | -| P2-O14 | Toolbar and missing surfaces | Both platforms add `SnapshotSurface::Toolbar`. macOS additionally adds `Spotlight` (pid of `/System/Library/CoreServices/Spotlight.app`), `Dock` (pid of `/System/Library/CoreServices/Dock.app`), and `MenuBarExtras` (enumerates `SystemUIServer`, `ControlCenter`, and per-app `AXExtrasMenuBar`). Windows adds structured shell surfaces for `Taskbar`, `SystemTray`, `SystemTrayOverflow`, `StartMenu`, `ActionCenter`, and `QuickSettings` where the current Windows build/session exposes them | -| P2-O15 | Electron / WebView2 deep-tree toggles | macOS: `build_subtree` writes `AXEnhancedUserInterface = YES` on app root for known Electron bundle IDs (VS Code, Cursor, Slack post-Sept-2024, Teams, Discord, Figma Desktop, Notion). Windows: detect Edge WebView2 via UIA `ClassName = "Chrome_WidgetWin_1"` and the equivalent flag; apply same web-wrapper depth-skip. Both: new `--force-electron-a11y` CLI override | -| P2-O16 | FFI registry migration + parity expansion | Migrate `crates/ffi/` from hand-written `ad_*` wrappers to a `build.rs` codegen step that walks the compile-time `CommandDescriptor` registry and emits one wrapper per command. After this, adding a CLI command automatically produces the FFI entry and the same descriptor metadata can feed JSON Schema / MCP generation in Phase 4. Marshaling helpers stay in `crates/ffi/src/convert/` — these are per-type, not per-command. In the same migration: backfill `ad_snapshot` (full refmap pipeline), `ad_execute_by_ref(adapter, "@e5", action, out)`, `ad_wait(…)`, `ad_version`, `ad_abi_version() -> u32` with `AD_ABI_VERSION_MAJOR` cbindgen `[defines]` export, `ad_status`, `ad_set_log_callback(fn(level, msg))` installing a `tracing_subscriber` layer so dlopen consumers see debug output | -| P2-O17 | Screen Recording / Automation permission detection | macOS Phase 1 already exposes `PermissionReport { accessibility, screen_recording, automation }`. Phase 2 decides whether a distinct `AutomationPermissionDenied` code is still needed once Apple Event automation paths exist | -| P2-O18 | Windows shell surface coverage | Add explicit shell coverage for Start menu/search, taskbar, system tray/overflow, Action Center/notification center, Quick Settings, multi-monitor/DPI, virtual desktop detection, UAC/elevated targets, RDP/locked-session behavior, and Explorer-specific file destinations. New commands are added only where a ref-based `snapshot --surface …` loop cannot expose the surface first; Windows-only behavior still routes through core command files and adapter trait defaults | +| ID | Objective | Metric | Shipped in 1.6 / Remaining | +|----|-----------|--------|------------------------------| +| P2-O8 | Stable-selector expansion | `AccessibilityNode.native_id` remains the portable stable-ID field. Platform adapters preserve their strongest developer ID there (Windows UIA `AutomationId`, macOS `AXIdentifier` or `AXDOMIdentifier`, Linux AT-SPI `accessible-id`), while live locator traversal may retain both native IDs internally for strict matching. Phase 2 may add separately named `subrole`, `role_description`, `placeholder`, and `dom_classes` evidence without renaming or duplicating `native_id`. Resolver tests require controls with explicit IDs to survive re-drills; controls without one continue through the fingerprint fallback | Core contract + macOS `native_id` (`AXIdentifier`) shipped (U5). **Remaining:** Windows `AutomationId` implementation (sub-phase 2.3) | +| P2-O9 | `Action` enum expansion for 2026 agent workloads | New variants: `LongPress { duration_ms }`, `ForceClick`, `ShowMenu`, `DeliverFiles(Vec<PathBuf>)` (renamed from `FileDrop` — the original name implied `NSDraggingSession` which is not headless-compatible on macOS; see Core invariant 6), `WindowRaise`, `Cancel`, `SelectRange { start, len }`, `InsertAtCaret(String)`. `watch` is a new **command**, not an `Action` variant. Each has a macOS AX API mapping (all AX calls on main thread), a Windows UIA pattern mapping, a new CLI subcommand, FFI conversion coverage where applicable, and exhaustive platform-dispatch tests in the same change | **Remaining in full** — none of these variants exist yet; still Phase 2 scope | +| P2-O10 | `ErrorCode` expansion | Consider `PermissionRevoked` (distinct from `PermDenied` — TCC yanked mid-session) and `ResourceExhausted` (refmap >1 MB, tree node-count cap) | `APP_UNRESPONSIVE` shipped (U14) — a failed read-only liveness probe upgrades a hang to `APP_UNRESPONSIVE`; ordinary AX messaging exhaustion still reports plain `TIMEOUT`. **Remaining:** `PermissionRevoked`, `ResourceExhausted` | +| P2-O11 | Event-subscription primitive (push, not poll) | New **command** `watch --event <kind> --ref @s8f3k2p9:e5 --timeout 3000`, backed by a new adapter method distinct from `capture_signal_baseline`. macOS: `AXObserverCreate` + `AXObserverAddNotification` + `CFRunLoopSource`. Windows: `IUIAutomation.AddAutomationEventHandler` + `AddFocusChangedEventHandler` + `AddPropertyChangedEventHandler`. Linux mirrors in Phase 3 via AT-SPI2 D-Bus signals | `wait --event <kind>` (baseline-diff desktop-signal wait, U17) shipped and is **not** this objective — see the naming note under sub-phase 2.11. **Remaining in full:** the push `watch` command itself | +| P2-O12 | Text range primitives | Read caret, read selection, select a range by offsets, read text at range, insert at caret. macOS: `kAXSelectedTextRangeAttribute` (settable), `AXStringForRangeParameterizedAttribute`, `AXBoundsForRangeParameterizedAttribute`, `AXRangeForLineParameterizedAttribute`, `AXValueCreate(kAXValueCFRangeType, …)`. Windows: `TextPattern.GetSelection`, `TextPattern.DocumentRange`, `TextRange.Select`, `TextRange.Move`, `TextRange.GetText`, `TextRange.GetBoundingRectangles`. Commands: `text get-selection`, `text select-range <ref> <start> <len>`, `text insert-at-caret <ref> <string>`, `text at-offset <ref> <start> <len>` | **Remaining in full** — still Phase 2 scope | +| P2-O13 | Modern per-window screenshot APIs | macOS: replace `/usr/sbin/screencapture` subprocess with `SCScreenshotManager.captureImage(contentFilter:config:)` filtered to a specific `CGWindowID` from `SCShareableContent.windows`. Windows: `Windows.Graphics.Capture` via `GraphicsCaptureItem.CreateFromWindowHandle(HWND)` + `Direct3D11CaptureFramePool` when supported by the OS/session. No subprocess on the modern path, explicit fallback to legacy capture when unavailable, and permission/support failures map to structured `PERM_DENIED` / `PLATFORM_NOT_SUPPORTED` with `platform_detail` | `list_displays` + honest `--screen` targeting + `scale_factor` shipped (U3). **Remaining:** ScreenCaptureKit modern macOS capture, `Windows.Graphics.Capture` Windows capture | +| P2-O14 | Toolbar and missing surfaces | Implement the core-predeclared surface vocabulary without changing core: `Toolbar` on both platforms; `Spotlight`, `Dock`, and `MenuBarExtras` on macOS; `Taskbar`, `SystemTray`, `SystemTrayOverflow`, `StartMenu`, `ActionCenter`, and `QuickSettings` on Windows where the current build/session exposes them. `NotificationCenter` remains the portable notification surface while `ActionCenter` names the distinct Windows shell entry point | `supported_surfaces()` introspection shipped (U12) — `SnapshotSurface` is ratified as genuinely platform-neutral, and every variant above already exists as a predeclared enum member. **Remaining:** the Windows shell surface implementations themselves | +| P2-O15 | Electron / WebView2 deep-tree toggles | macOS: `build_subtree` writes `AXEnhancedUserInterface = YES` on app root for known Electron bundle IDs (VS Code, Cursor, Slack post-Sept-2024, Teams, Discord, Figma Desktop, Notion). Windows: detect Edge WebView2 via UIA `ClassName = "Chrome_WidgetWin_1"` and the equivalent flag; apply same web-wrapper depth-skip. Both: new `--force-electron-a11y` CLI override | **Remaining in full** on Windows — macOS depth-skip already exists from Phase 1; the Windows implementation is sub-phase 2.4 | +| P2-O16 | FFI registry migration + parity expansion | Migrate `crates/ffi/` from hand-written `ad_*` wrappers to a `build.rs` codegen step that walks a compile-time command registry and emits one wrapper per command. After this, adding a CLI command automatically produces the FFI entry and the same descriptor metadata can feed JSON Schema / MCP generation in Phase 4. Marshaling helpers stay in `crates/ffi/src/convert/` — per-type, not per-command | **Remaining in full.** The ABI handshake (`ad_abi_version`, `ad_init`) and 8 command-backed entrypoints (`ad_snapshot`, `ad_execute_by_ref`, `ad_execute_by_ref_timeout`, `ad_wait`, `ad_version`, `ad_status`, `ad_trace_export`, `ad_trace_show`) shipped, but all 8 are hand-written — see Phase 1.5 Gap Status. This objective is exactly the codegen migration that turns those hand-written files into generated output | +| P2-O17 | Screen Recording / Automation permission detection | macOS exposes `PermissionReport { accessibility, screen_recording, automation }`. Automation is probed noninteractively for the remaining System Events-backed Notification Center opener; Accessibility and Screen Recording retain explicit preflight states | **Shipped** (U4 truthful Automation permission) | +| P2-O18 | Windows shell surface coverage | Add explicit shell coverage for Start menu/search, taskbar, system tray/overflow, Action Center/notification center, Quick Settings, multi-monitor/DPI, virtual desktop detection, UAC/elevated targets, RDP/locked-session behavior, and Explorer-specific file destinations. New commands are added only where a ref-based `snapshot --surface …` loop cannot expose the surface first; Windows-only behavior still routes through core command files and adapter trait defaults | **Remaining in full** — sub-phase 2.14, explicitly deferrable stretch scope | ### Cross-Platform Trait Extensions -All methods land as `#[non_exhaustive]` additions in `crates/core/src/adapter.rs` with default implementations returning `AdapterError::not_supported(method)`. Windows implements them natively. macOS backfills in the same PR pair. Linux (Phase 3) adds the AT-SPI2 implementations. +New methods land in the appropriate capability trait under `crates/core/src/adapter/`, with default implementations returning `AdapterError::not_supported(method)`. Windows implements them natively. macOS backfills in the same PR pair. Linux (Phase 3) adds the AT-SPI2 implementations; public trait access remains through the crate-root re-exports in `crates/core/src/lib.rs`. ```rust impl PlatformAdapter for … { - // P2-O11 — event subscription + // P2-O11 — event subscription (new `watch` command; distinct from the shipped `wait --event`) fn watch_element( &self, handle: &NativeHandle, @@ -723,7 +897,7 @@ impl PlatformAdapter for … { // native modern API; a `Legacy` fallback preserves the Phase 1 subprocess path.) // P2-O14 — new surfaces - fn list_surfaces(&self, pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError> // extended kinds + fn list_surfaces(&self, process: ProcessIdentity, deadline: Deadline) -> Result<Vec<SurfaceInfo>, AdapterError> // extended kinds via already-shipped SnapshotSurface variants } ``` @@ -733,23 +907,24 @@ New supporting types (land in `crates/core/src/`): - `ElementEvent` — `{ kind, handle_ref_id: Option<String>, timestamp, attr_snapshot: Option<AccessibilityNode> }` - `TextRange` — `{ start: u32, length: u32 }` (UTF-16 code units to match both AX CFRange and UIA TextRange conventions) - `TextSelection` — `{ range: TextRange, caret_offset: u32, lines_in_view: Vec<TextRange> }` -- `ScreenshotBackend` — `Modern` (ScreenCaptureKit / Windows.Graphics.Capture / PipeWire) or `Legacy` (preserves Phase 1 subprocess path as fallback for restricted environments) -- `PermissionReport` is `{ accessibility, screen_recording, automation }` where each field is `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }` +- `ScreenshotBackend` — `Modern` (ScreenCaptureKit / Windows.Graphics.Capture / PipeWire) or `Legacy` (preserves the Phase 1 subprocess path as fallback for restricted environments) + +`PermissionReport` (`{ accessibility, screen_recording, automation }`, each `{ "state": "granted" | "denied" | "not_required" | "unknown" }`) already shipped in Phase 1 and needs no change here. ### Cross-platform capability map (P2-O8 through O17) | Capability | macOS API | Windows API | Linux API (Phase 3) | |------------|-----------|-------------|----------------------| -| Stable `identifier` | `kAXIdentifierAttribute` | UIA `AutomationId` | AT-SPI2 `accessible-id` + GTK `gtk-id` | +| Stable `native_id` — **shipped on macOS (U5)** | `kAXIdentifierAttribute` / `AXDOMIdentifier` | UIA `AutomationId` | AT-SPI2 `accessible-id` + GTK `gtk-id` | | `subrole` | `kAXSubroleAttribute` | UIA `LocalizedControlType` + pattern-based heuristic | AT-SPI2 `role-name` + `state-set` | | `role_description` | `kAXRoleDescriptionAttribute` | UIA `LocalizedControlType` | AT-SPI2 `role-description` | | `placeholder` | `kAXPlaceholderValueAttribute` | UIA `HelpText` + `IsTextEditPatternAvailable` placeholder | AT-SPI2 `description` + HTML `placeholder` via `object-attributes` | | `dom_id` / `dom_classes` | `kAXDOMIdentifierAttribute` / `kAXDOMClassListAttribute` | Edge WebView2 UIA `HtmlId` / `HtmlClass` properties | AT-SPI2 `object-attributes` HTML keys | -| Event subscription | `AXObserverCreate` + `AXObserverAddNotification` on `CFRunLoop` | `IUIAutomation.AddAutomationEventHandler` + `AddFocusChangedEventHandler` + `AddPropertyChangedEventHandler` | AT-SPI2 D-Bus signals via `zbus::StreamFactory` | +| Event subscription (`watch`) | `AXObserverCreate` + `AXObserverAddNotification` on `CFRunLoop` | `IUIAutomation.AddAutomationEventHandler` + `AddFocusChangedEventHandler` + `AddPropertyChangedEventHandler` | AT-SPI2 D-Bus signals via `zbus::StreamFactory` | | Text range read | `AXStringForRangeParameterizedAttribute` + `AXSelectedTextRangeAttribute` | `TextPattern.GetSelection`, `TextPattern.DocumentRange.GetText` | AT-SPI2 `Text.GetText(start, end)` + `Text.GetCaretOffset` | | Text range write | `AXSelectedTextRange = AXValueCreate(kAXValueCFRangeType, …)` | `TextRange.Select` + `TextRange.Move` | AT-SPI2 `EditableText.InsertText` + `Text.SetCaretOffset` | | Modern per-window screenshot | `SCScreenshotManager.captureImage(contentFilter:config:)` | `GraphicsCaptureItem.CreateFromWindowHandle` + `Direct3D11CaptureFramePool` | PipeWire `org.freedesktop.portal.ScreenCast` | -| Toolbar surface | `AXRole == AXToolbar` or `AXUnifiedTitleAndToolbar` | UIA `ControlType.ToolBar` | AT-SPI2 `Role::ToolBar` | +| Toolbar surface — **predeclared in `SnapshotSurface` (U12)** | `AXRole == AXToolbar` or `AXUnifiedTitleAndToolbar` | UIA `ControlType.ToolBar` | AT-SPI2 `Role::ToolBar` | | Menu-bar extras surface | `SystemUIServer` + `ControlCenter` pid walk | UIA `Shell_TrayWnd` + `NotifyIconOverflowWindow` | AT-SPI2 `StatusNotifierWatcher` D-Bus | | Dock / taskbar surface | `Dock.app` pid walk | UIA `Shell_TrayWnd` `TaskListButton` children | AT-SPI2 per-DE panel walk | | `LongPress` | `CGEventCreateMouseEvent(…Down…)` + sleep + `…Up` | `SendInput` hold + release | Coordinate via `ydotool/xdotool` | @@ -757,56 +932,170 @@ New supporting types (land in `crates/core/src/`): | `ShowMenu` action | `AXPerformAction(kAXShowMenuAction)` | `ExpandCollapsePattern.Expand` + UIA right-click fallback | AT-SPI2 `Action.DoAction("popup")` | | `WindowRaise` | `AXUIElementSetAttributeValue(kAXRaiseAction)` | `SetForegroundWindow` + `SetWindowPos(HWND_TOP)` only under explicit focus/window policy | `wmctrl -a` / `xdotool windowactivate` only under explicit focus/window policy | | `Cancel` | `AXPerformAction(kAXCancelAction)` | UIA `WindowPattern.Close` on dialog or `InvokePattern` on cancel button | AT-SPI2 `Action.DoAction("cancel")` or synthesize Escape | -| `DeliverFiles(Vec<PathBuf>)` | 4-tier headless fallback: (1) app-native URL scheme, (2) `NSWorkspace.open(urls:withApplicationAt:configuration:)` with `activates: false`, (3) `NSPasteboard.public.file-url` + `CGEventPostToPid(cmd+v)`, (4) `osascript open`. NEVER `NSDraggingSession` (not headless-compatible — Phase 2 plan Unit 12 research) | App/shell delivery first: app URI handlers, `ShellExecuteEx`, `IFileOperation` for filesystem destinations, and `CF_HDROP` clipboard paste where accepted. `IDataObject + DoDragDrop` is policy-gated fallback/spike only | Portal/native file-transfer path where available; XDND is Phase 3 research, not default | +| `DeliverFiles(Vec<PathBuf>)` | 4-tier headless fallback: (1) app-native URL scheme, (2) `NSWorkspace.open(urls:withApplicationAt:configuration:)` with `activates: false`, (3) `NSPasteboard.public.file-url` + `CGEventPostToPid(cmd+v)`, (4) `osascript open`. NEVER `NSDraggingSession` (not headless-compatible — Core invariant 6) | App/shell delivery first: app URI handlers, `ShellExecuteEx`, `IFileOperation` for filesystem destinations, and `CF_HDROP` clipboard paste where accepted. `IDataObject + DoDragDrop` is policy-gated fallback/spike only | Portal/native file-transfer path where available; XDND is Phase 3 research, not default | | Screen Recording permission | `CGPreflightScreenCaptureAccess` / `CGRequestScreenCaptureAccess` | No macOS-style TCC field. Use `GraphicsCaptureSession::IsSupported` / capture API failures to report `not_required`, `unknown`, `PERM_DENIED`, or `PLATFORM_NOT_SUPPORTED` with `platform_detail` | PipeWire portal permission dialog | -| Automation permission | `AEDeterminePermissionToAutomateTarget` | N/A (no equivalent restriction) | N/A | +| Automation permission — **shipped, truthful (U4)** | Nonprompting System Events probe; explicit request uses the bounded isolated helper | N/A (no equivalent restriction) | N/A | -### Windows Adapter Implementation +### Cross-cutting sub-phase DoD -Full `WindowsAdapter` in `crates/windows/src/` following the identical platform crate folder structure: +Every sub-phase 2.0–2.15 below is held to the same definition of done, stated once here rather than repeated per sub-phase: -``` -crates/windows/src/ -├── lib.rs # mod declarations + re-exports only -├── adapter.rs # WindowsAdapter: PlatformAdapter impl -├── tree/ -│ ├── mod.rs # re-exports -│ ├── element.rs # UIA element wrapper + attribute readers -│ ├── builder.rs # IUIAutomationTreeWalker traversal with CacheRequest -│ ├── roles.rs # UIA ControlType → unified role enum mapping -│ ├── resolve.rs # Element re-identification for ref resolution -│ └── surfaces.rs # Surface detection (menus, dialogs, flyouts) -├── actions/ -│ ├── mod.rs # re-exports -│ ├── dispatch.rs # perform_action match arms via UIA patterns -│ ├── activate.rs # Smart activation chain (InvokePattern → Toggle → coordinate) -│ └── extras.rs # SelectionPattern, ScrollPattern helpers -├── input/ -│ ├── mod.rs # re-exports -│ ├── keyboard.rs # SendInput keyboard synthesis -│ ├── mouse.rs # SendInput mouse events -│ └── clipboard.rs # OpenClipboard / GetClipboardData / SetClipboardData Win32 APIs -├── notifications/ -│ ├── mod.rs # re-exports -│ ├── list.rs # List toast/Action Center notifications via UIA -│ ├── dismiss.rs # Dismiss individual or all notifications -│ └── interact.rs # Click notification action buttons -├── tray/ -│ ├── mod.rs # re-exports -│ ├── list.rs # List system tray items via Shell_TrayWnd UIA tree -│ └── interact.rs # Click tray items, open tray menus -└── system/ - ├── mod.rs # re-exports - ├── app_ops.rs # Process launch via CreateProcess, close via TerminateProcess - ├── window_ops.rs # SetWindowPos, ShowWindow for resize/move/minimize/maximize/restore - ├── key_dispatch.rs # Explicit focus-policy key press via SetForegroundWindow + SendInput - ├── permissions.rs # COM security check, UAC elevation detection - ├── screenshot.rs # Windows.Graphics.Capture modern backend + PrintWindow legacy - ├── shell_surfaces.rs # Start, taskbar, Action Center, Quick Settings - └── wait.rs # wait utilities (polling UIA element existence) -``` +- `cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings`, `cargo test --lib --workspace`, and the relevant conformance suites (role/state vocabulary, contract tests) are green. +- Probe evidence — raw UIA dumps of the target app — is committed alongside the sub-phase's plan doc under `docs/plans/`. +- Adapters keep `not_supported()` defaults for every capability not yet landed in that sub-phase; the CLI surfaces `PLATFORM_NOT_SUPPORTED` honestly rather than a stub success. +- No core rewrites. Core changes land only via explicitly planned additive trait methods — never a signature change to something Phase 1/1.6 already shipped. +- Each sub-phase gets its own review before merging to `feat/windows-adapter`. +- `scripts/perf-baseline-compare.sh` runs on any sub-phase that touches a hot path (tree traversal, resolution, action dispatch). +- Commits follow the repository's Conventional Commits requirement. -### Windows API Mapping +### 2.0 — Platform Exploration & Raw Scripting (pre-Rust) + +**Goal:** empirically map Windows accessibility reality with raw, no-Rust scripts before any adapter code exists, producing a committed evidence corpus the Rust sub-phases implement against — and feeding every contradiction back into this document. + +**Scope:** a `probes/windows/` directory of raw scripts — PowerShell using .NET managed UIA (`System.Windows.Automation`, preinstalled with .NET Framework 4.8) plus small C# programs compiled with `csc.exe` where UIA3 COM specifics differ from the managed wrapper. The corpus must cover, each as a runnable script with captured JSON/text output committed beside it: (1) full-tree dumps of Notepad, Explorer, Settings, and one Electron app (VS Code or Slack) including every property read per node; (2) a pattern-availability census per ControlType (Invoke, Toggle, Value, RangeValue, ExpandCollapse, SelectionItem, Scroll, ScrollItem, Text, Window, LegacyIAccessible); (3) every interaction exercised raw — invoke, toggle, set value, select, expand/collapse, scroll via pattern AND wheel, text get/selection/caret/insert, focus; (4) SendInput synthesis experiments — keyboard incl. modifier chords and UTF-16 chunking limits, mouse click/move/wheel/drag; (5) `ElementFromPoint` hit-testing incl. deliberately occluded and zero-size targets; (6) `CacheRequest` batched reads timed against per-property reads; (7) AutomationId coverage census across Win32 / WinForms / WPF / Electron; (8) event-handler observations (which UIA events actually fire, ordering, MTA threading behavior); (9) elevation/UIPI behavior against an elevated process; (10) RDP-session and DPI/multi-monitor bounds behavior. Alongside the scripts: `probes/windows/FINDINGS.md` — a findings ledger mapping every experiment to observed behavior and a doc-alignment verdict (confirms this document / contradicts it / new edge case). + +**Key APIs:** System.Windows.Automation, UIA3 COM (IUIAutomation) via csc.exe shims, SendInput, ElementFromPoint, CacheRequest. + +**Depends on:** nothing — this is the entry point; the dev VM needs only its preinstalled .NET, PowerShell, and git. + +**Exit criteria:** the script corpus and captured outputs are committed and re-runnable on the dev VM; the findings ledger covers tree, patterns, interactions, input, hit-testing, batching, identity, events, elevation, and session behavior with no open "unknown" rows; every ledger entry that contradicts this document has a matching amendment to this document landed in the same PR (see the source-of-truth feedback rule in the Platform Delivery Model); no Rust adapter sub-phase (2.2 onward) starts until this exit gate is green. + +**Est. PR size:** ~1.5k lines (scripts + ledger; no Rust). + +### 2.1 — Toolchain, CI & COM Bootstrap + +**Goal:** Stand up the Windows build/CI/session substrate so every later sub-phase lands on green CI and a constructible (if functionally empty) `WindowsAdapter`. + +**Scope:** +- `windows-latest` CI job: build + clippy + lib tests, core-isolation check, size check (mirrors the existing `platform-check` matrix job, promoted to a real Windows test lane) +- Self-hosted interactive Windows runner registration for later UIA/shell integration tests, with RDP/session-isolation documented (an interactive session is required for UIA to see a real desktop; `tscon` is the documented console-reattach workaround — see Risk Register) +- `CoInitializeEx(NULL, COINIT_MULTITHREADED)` + `SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)` bootstrap at process start +- `WindowsAdapterSession` implementing `AdapterSession` via `open_session` — owns COM apartment state so later sub-phases don't reinvent COM lifecycle +- Re-verify the dependency pins below against crates.io + supply-chain policy (pinned at 2026-04 research time — see New Dependencies) +- Fix the `private_file_windows_security` `AceSize` validation gap (`crates/core/src/private_file_windows_security.rs`, currently cfg-gated dead code on non-Windows CI) so the private-artifact-writing path is real before any Windows code writes a refmap or trace file + +**Key APIs:** `CoInitializeEx`, `SetProcessDpiAwarenessContext`, Win32 ACL / `AceSize` validation (private-file security) + +**Depends on:** nothing (opening sub-phase) + +**Exit criteria:** workspace green on Windows CI; `WindowsAdapter` constructs and satisfies the trait; every command returns honest `PLATFORM_NOT_SUPPORTED` on Windows; the permission probe is unit-tested against mocked COM security state. + +**Est. PR size:** ~0.8k LOC + +### 2.2 — UIA Element Wrapper & Tree Walk + +**Goal:** Own an `AXElement`-equivalent wrapper for UIA elements and prove raw tree traversal against a real Windows app before any semantics land on top. + +**Scope:** +- `UIAElement` ownership wrapper — `AddRef`/`Release`, `Clone`/`Drop` safety mirroring the `AXElement` pattern (`pub(crate)` inner field to prevent double-free via raw pointer extraction) +- `ElementFromHandle` roots for window entry +- `TreeWalker` traversal with an ancestor-path cycle guard (mirrors macOS: reused pointers across sibling branches, not a global visited set) +- `CacheRequest` batched attribute reads (the UIA analogue of `AXUIElementCopyMultipleAttributeValues`) +- Committed probe examples: raw UIA dumps of Notepad and Explorer, checked in as evidence alongside the sub-phase plan + +**Key APIs:** `IUIAutomation.ElementFromHandle()`, `IUIAutomationTreeWalker.GetFirstChild`/`GetNextSibling`, `CacheRequest` (`uiautomation` crate 0.24+ wrapping the `windows` crate's COM bindings) + +**Depends on:** 2.1 + +**Exit criteria:** an internal tree-dump binary prints Notepad and Explorer trees with batched reads; `CacheRequest` attribute-batching correctness is unit-tested. + +**Est. PR size:** ~2k LOC + +### 2.3 — Vocabulary: Roles, States, native_id, Name Evidence + +**Goal:** Map UIA's vocabulary onto the canonical role/state contract Phase 1.6 (U1/U2) already established, and wire `native_id` end-to-end for Windows. + +**Scope:** +- `ControlType` → unified role enum map in `tree/roles.rs` +- UIA states → canonical state vocabulary — must pass the same conformance tests U1/U2 already run for macOS, parameterized over `WindowsAdapter` +- `AutomationId` → `native_id` (completes P2-O8 on Windows; macOS/core side already shipped) +- `NameEvidence` supplier feeding the core `accname.rs` computation (`LabeledBy`, `Name`, `HelpText` precedence evidence supplied by the adapter; core computes the final name, per U11) + +**Key APIs:** UIA `ControlType` enum, UIA property IDs for state, `AutomationId`, `Name`/`LabeledBy`/`HelpText` properties + +**Depends on:** 2.2 + +**Exit criteria:** vocabulary conformance tests span every UIA `ControlType` (complete mapping coverage, not a sample) and accname tests pass on Windows. + +**Est. PR size:** ~1.5k LOC + +### 2.4 — Observation: Snapshot, Windows, Apps, Displays + +**Goal:** Land the full read path — `snapshot`, `list-windows`, `list-apps`, `list-displays`, `focused_window` — including the Electron/WebView2 web-wrapper depth-skip that makes dense apps usable. + +**Scope:** +- `get_tree` / `get_subtree` wired to the shared `SnapshotEngine` +- Skeleton glue: `ControlViewWalker` (not `RawViewWalker`/`ContentViewWalker`) + `FindAll(TreeScope_Children, TrueCondition)` for `children_count` + fresh `UICacheRequest` per drill-down (Core invariant 2 — ~50 LOC, core owns the rest) +- `list_windows` — HWND-first identity with recycled-window-id corroboration (mirrors the macOS U6 window-identity-as-primary-key pattern) +- `list_apps`, `focused_window`, `list_displays` + per-monitor DPI `scale_factor` +- **Web/Electron web-wrapper depth-skip** (Windows implementation of the pattern macOS already ships): non-semantic wrapper elements (`UIA_GroupControlTypeId` / `UIA_CustomControlTypeId`) with empty `Name` AND empty `Value` do not consume depth budget. Without this, default `--max-depth 10` finds ~3 refs in Slack; with it, 100+. Implement in `crates/windows/src/tree/builder.rs` as `is_web_wrapper`, matching the macOS logic +- **Chromium detection:** detect Chromium-based windows via UIA process name or `Chrome_WidgetWin_1` window class; if the tree is empty/minimal, the error `platform_detail` guides toward `--force-renderer-accessibility` +- **Resolver depth:** element re-identification searches to `ABSOLUTE_MAX_DEPTH` (50) — Electron elements commonly sit at depth 25+; implement in `crates/windows/src/tree/resolve.rs` +- **Surface detection for Electron:** an Electron modal (file picker, dialog) may report as the focused window itself rather than a child; surface detection must check whether the focused window IS the target surface, checking both `ControlType` and `LocalizedControlType`/UIA patterns (analogous to macOS AXRole + AXSubrole); implement in `crates/windows/src/tree/surfaces.rs` +- Progressive skeleton traversal (`--skeleton`, `--root`) needs no Windows-specific work beyond `get_subtree()` — core owns the flow + +**Key APIs:** `ControlViewWalker`, `FindAll`, `UICacheRequest`, `Chrome_WidgetWin_1` class match, `IVirtualDesktopManager` (read-only, for diagnostics) + +**Depends on:** 2.3 + +**Exit criteria:** `snapshot --app Notepad -i` and Explorer return reffed trees on the runner; skeleton drill-down works; a VS Code snapshot at default depth finds 50+ refs through web-aware depth-skip alone (no force flag), and ≥100 refs with `--force-electron-a11y`; an Electron file-picker dialog is detected as a sheet surface. + +**Est. PR size:** ~2k LOC + +### 2.5 — Resolution & Live Locator + +**Goal:** Make refs and the live `find`/`get`/`is` commands (U7) work on Windows with the same strict-resolution guarantees macOS ships. + +**Scope:** +- `resolve_element_strict*` from `RefEntry` evidence — `AutomationId`-first, fingerprint fallback, 0/1/N classification into `STALE_REF`/success/`AMBIGUOUS_TARGET` +- `get_live_value` / `get_live_state` / `get_live_actions` / `get_live_element` / `get_element_bounds` +- `resolve_query` — the `LocatorQuery` evaluator backing live `find` +- `resolve_locator_anchor` + selected-hydration completeness: actions that read state must classify a **definitive absence** separately from a **transport failure** — port the lesson already encoded in macOS's `is_definitive_absence` (`crates/macos/src/tree/action_list.rs`) rather than re-deriving it from scratch + +**Key APIs:** `AutomationId` lookup, `CacheRequest`-scoped re-reads, UIA property read failure classification (COM HRESULT vs "element gone") + +**Depends on:** 2.4 + +**Exit criteria:** `find`/`get`/`is` are live on Windows; `STALE_REF`/`AMBIGUOUS_TARGET` semantics are proven with committed probe evidence (0/1/N candidate cases). + +**Est. PR size:** ~2k LOC + +### 2.6 — Actionability & Occlusion + +**Goal:** Port the Phase 1.6 auto-wait/occlusion gate (U8/U9) onto Windows so every ref action gets the same actionability guarantees before it fires. + +**Scope:** +- `hit_test` three-way result via `ElementFromPoint` + window corroboration — `Unknown` on probe failure, never a false negative, matching `HitTestResult`'s `ReachesTarget | InterceptedBy | Unknown` contract +- `receives_events` evidence +- Visibility / enabled / offscreen evidence feeding the core auto-wait gate (U8) — no Windows-specific auto-wait logic; core drives the loop, the adapter supplies live reads +- `scroll_into_view` via `ScrollItemPattern` + +**Key APIs:** `ElementFromPoint`, `ScrollItemPattern.ScrollIntoView` + +**Depends on:** 2.5 + +**Exit criteria:** zero-bounds / disabled / occluded fixture cases produce the same envelopes as macOS (same error codes, same `disposition`). + +**Est. PR size:** ~1.2k LOC + +### 2.7 — Semantic Action Tier + +**Goal:** Land the UIA pattern-based semantic action dispatch, with the same typed `ActionStep` delivery reporting (U13) macOS already produces. + +**Scope:** +- `perform_action` via UIA patterns: `InvokePattern` (click), `TogglePattern` (toggle/check/uncheck), `ValuePattern` (set-value), `ExpandCollapsePattern` (expand/collapse), `SelectionItemPattern` (select), `RangeValuePattern`, `ScrollPattern` +- Activation chain with `ActionStep` delivery reporting + post-verification reads — honest `verified: Option<bool>` semantics, no step claims an effect it did not observe +- See the Windows API Mapping table below for the full click/set-text/expand/select/toggle/scroll pattern list + +**Key APIs:** `InvokePattern.Invoke()`, `TogglePattern.Toggle()`, `ValuePattern.SetValue()`, `ExpandCollapsePattern.Expand()/.Collapse()`, `SelectionItemPattern.Select()`, `ScrollPattern.Scroll()`/`.SetScrollPercent()` + +**Depends on:** 2.6 + +**Exit criteria:** click / set-value / clear / select / toggle / expand / collapse work headless on the fixture app via the e2e analog (sub-phase 2.12 supplies the fixture; this sub-phase's own tests use ad hoc Notepad/Explorer targets until then). + +**Est. PR size:** ~2k LOC + +#### Windows API Mapping (reference table for sub-phases 2.2–2.10) | Capability | Technology | Details | |------------|-----------|---------| @@ -821,26 +1110,151 @@ crates/windows/src/ | Scroll | `ScrollPattern.Scroll()` / `ScrollPattern.SetScrollPercent()` | Native UIA scroll; mouse wheel only under explicit physical policy | | Keyboard | `SendInput` API | `INPUT_KEYBOARD` structs with virtual key codes and scan codes | | Mouse | `SendInput` API | `INPUT_MOUSE` structs with `MOUSEEVENTF_*` flags | -| Clipboard | `OpenClipboard` / `GetClipboardData` / `SetClipboardData` | Win32 APIs, handle `CF_UNICODETEXT` format | +| Clipboard | `OpenClipboard` / `GetClipboardData` / `SetClipboardData` | Win32 APIs; `CF_UNICODETEXT` for text, `CF_DIB`/PNG for image, `CF_HDROP` for file lists — marshaled through typed `ClipboardContent` | | Screenshot | `Windows.Graphics.Capture` | Modern per-window capture via `GraphicsCaptureItem.CreateFromWindowHandle` + `Direct3D11CaptureFramePool` when WGC is supported by the OS/session. No subprocess, respects DWM compositing. `BitBlt` / `PrintWindow` retained as `ScreenshotBackend::Legacy` fallback for pre-Windows-10 1903 or unavailable WGC environments | -| App launch | `CreateProcess` / `ShellExecuteEx` | Launch by name or path, wait for main window | -| App close | `WM_CLOSE` / `TerminateProcess` | Graceful close first, force kill with `--force` | +| App launch | `CreateProcess` / `ShellExecuteEx` | Launch by name or path via `LaunchOptions` (args/env/cwd/attach-if-running), wait for main window | +| App close | `WM_CLOSE` / `TerminateProcess` | Graceful close first, force kill with `--force`; verified via `ProcessState` | | Window ops | `SetWindowPos` / `ShowWindow` | Resize, move, minimize (`SW_MINIMIZE`), maximize (`SW_MAXIMIZE`), restore (`SW_RESTORE`) | | Permissions | COM security / UAC | Detect elevation requirements; return `PERM_DENIED` if UIA access blocked | -| Notifications | UserNotificationListener + UIA Action Center fallback | Prefer `UserNotificationListener` where app identity/capability and explicit user permission are available. Otherwise Action Center UIA traversal is best-effort fallback: list/dismiss/interact only when the shell exposes stable UIA elements. Do Not Disturb (Focus Assist) state via supported shell APIs or documented registry fallback | -| System tray | UIA + Shell_TrayWnd | System tray items accessible via UIA tree of `Shell_TrayWnd` class. Overflow items in `NotifyIconOverflowWindow`. List via `IUIAutomationTreeWalker` on tray area. Click via `InvokePattern` or coordinate-based `SendInput`. Expand overflow via click on chevron button | -| Start menu / search | UIA + explicit shell open command | `open-system-surface --surface start-menu` opens the Start surface under explicit shell-surface policy, then agents use `snapshot --surface start-menu` + refs. App launching remains `launch`; Start is for shell workflows and search results | -| Taskbar | UIA + Shell_TrayWnd task list | `snapshot --surface taskbar` exposes pinned/running app buttons as refs. Taskbar button invocation uses `InvokePattern` when available; focus-changing activation is allowed only for explicit `focus-window` / `WindowRaise` policy | -| Quick Settings | UIA shell flyout | `open-system-surface --surface quick-settings` exposes Wi-Fi, Bluetooth, audio, display, and accessibility toggles as refs when the shell exports them. Unsupported Windows builds return `PLATFORM_NOT_SUPPORTED` | -| Virtual desktops | `IVirtualDesktopManager` detection | Use public COM detection for "current desktop" filtering and diagnostics. Moving windows between virtual desktops is deferred unless a stable public API path is validated | +| Notifications | UserNotificationListener + UIA Action Center fallback | See Notification Management approach under 2.14 | +| System tray | UIA + Shell_TrayWnd | See System Tray approach under 2.14 | +| Start menu / search | UIA + explicit shell open command | See Windows-specific command surface under 2.14 | +| Taskbar | UIA + Shell_TrayWnd task list | See Windows-specific command surface under 2.14 | +| Quick Settings | UIA shell flyout | See Windows-specific command surface under 2.14 | +| Virtual desktops | `IVirtualDesktopManager` detection | Public COM detection for "current desktop" filtering and diagnostics only. Moving windows between virtual desktops is deferred unless a stable public API path is validated | | Multi-monitor / DPI | Per-monitor DPI + Win32 monitor APIs | All bounds are physical pixels normalized by the same DPI-aware process mode; tests cover mixed-DPI monitor layouts before any coordinate fallback ships | -### Windows-specific command surface (P2-O18) +### 2.8 — Input Synthesis + +**Goal:** Land raw OS input (keyboard, mouse, drag) matching the macOS delivery-tracking and headed/headless policy contract. + +**Scope:** +- `SendInput` keyboard map + `type_text` (UTF-16 chunking for surrogate pairs) +- Mouse events + modifier chords + wheel (mirrors macOS U19) +- Drag with delivery tracking + release guard +- Headed/headless policy parity — raw cursor commands (`hover`, `drag`, `mouse-*`) require `--headed`, same as macOS +- UIPI elevation detection (`GetTokenInformation(TokenIntegrityLevel)`) → `PERM_DENIED` with `platform_detail` in the `COM HRESULT 0x80070005 (E_ACCESSDENIED: ...)` format + +**Key APIs:** `SendInput` (`INPUT_KEYBOARD` / `INPUT_MOUSE`), `GetTokenInformation` + +**Depends on:** 2.7 + +**Exit criteria:** headed e2e gesture cases pass (once 2.12's fixture exists; interim coverage via Notepad/Explorer). + +**Est. PR size:** ~2k LOC + +### 2.9 — System Lifecycle + +**Goal:** Land process/window lifecycle — launch, close, resize/move/minimize/maximize/restore — with the same `ProcessState` liveness contract (U14) macOS ships. + +**Scope:** +- `launch_app` with `LaunchOptions` (`CreateProcess` args/env/cwd, attach-vs-fail policy via `attach_if_running`) +- `close_app` with verified termination (not an optimistic `closed: true` before the process actually exits — mirrors the v0.3.0 macOS `close-app` correction) +- `window_op` (`SetWindowPos`/`ShowWindow` for resize/move/minimize/maximize/restore) +- `ProcessState` probes: `IsHungAppWindow`/`SendMessageTimeout` → `Unresponsive`; exit-code inspection → `Exited`/`Crashed` +- `is_protected_process` +- `press_key_for_app` under the same focus policy macOS enforces + +**Key APIs:** `CreateProcess`, `TerminateProcess`, `SetWindowPos`, `ShowWindow`, `IsHungAppWindow`, `SendMessageTimeout` + +**Depends on:** 2.4 (window identity), 2.8 (input for `press_key_for_app`) + +**Exit criteria:** lifecycle e2e (launch → interact → close) passes; `APP_UNRESPONSIVE` is reachable against a deliberately hung fixture window. + +**Est. PR size:** ~1.8k LOC + +### 2.10 — Capture & Clipboard + +**Goal:** Ship screenshot and typed clipboard, with the modern-capture-first / legacy-fallback split P2-O13 specifies. + +**Scope:** +- `ScreenshotBackend::Legacy` first (`PrintWindow(hwnd, hdc, PW_RENDERFULLCONTENT)` — mitigates DWM black frames), then `ScreenshotBackend::Modern` via `windows-capture`/WGC where the session supports it (P2-O13) +- `screenshot --screen` honest display targeting (pairs with `list_displays` from 2.4) +- Typed clipboard: `CF_UNICODETEXT` → `ClipboardContent::Text`, image via `CF_DIB`/PNG → `ClipboardContent::Image`, `CF_HDROP` file lists → `ClipboardContent::FileUrls`, all written through 0600-equivalent private files (see 2.1's ACL fix) + +**Key APIs:** `PrintWindow`, `windows-capture`/`Windows.Graphics.Capture`, `OpenClipboard`/`GetClipboardData`/`SetClipboardData` + +**Depends on:** 2.1 (private-file ACL fix), 2.4 (displays) + +**Exit criteria:** screenshot + clipboard e2e pass (clipboard tests hermetic, no real-desktop clipboard pollution); WGC gracefully degrades (falls back to `Legacy`, does not fail) in RDP-limited sessions. + +**Est. PR size:** ~1.8k LOC + +### 2.11 — Signals & Wait Parity + +**Goal:** Port `SignalBaseline`/`diff_signals`/`wait --event` (U17) to Windows so the existing `wait` command works identically cross-platform — this is explicitly NOT the future push-based `watch` command (P2-O11). + +**Scope:** +- Windows `SignalBaseline` producers: windows / apps / focus / surfaces +- `wait --event` parity including `surface-appeared` +- Wait utilities operating within `Deadline` budgets, matching the core-owned deadline propagation + +**Naming note:** `wait --event <kind>` is the already-shipped baseline-diff desktop-signal wait (U17) — an in-invocation snapshot-diff, not a subscription. The future push element-event subscription primitive (P2-O11) is a **different, not-yet-built command** named `watch` (e.g. `watch --event value-changed --ref @s8f3k2p9:e5`), landing later in this sub-phase sequence once `watch_element` exists on the adapter trait. Do not conflate the two. + +**Key APIs:** UIA property snapshots for baseline capture (no event handlers yet — that's `watch`, still future) + +**Depends on:** 2.4 (windows/apps/displays), 2.9 (process lifecycle for app-launched/terminated signals) + +**Exit criteria:** an AE6-analog e2e passes — an unnamed dialog is discovered purely by baseline diff. + +**Est. PR size:** ~1k LOC + +### 2.12 — Fixture App & Live E2E Harness + +**Goal:** Give Windows the same verify-by-observation live e2e discipline the macOS SwiftUI fixture already provides. + +**Scope:** +- WinForms fixture app compiled with `csc.exe` (.NET 4.8 preinstalled on the runner — no new toolchain dependency) +- `AutomationId` set on every interactive target from day one (unlike macOS, which had to retrofit `AXIdentifier` — Windows gets this right from the start) +- Fixture targets mirroring `AgentDeskFixture.swift`: delayed-enable, zero-bounds, duplicate-title, occlusion, disclosure +- Harness port (bash via Git-Bash, or a PowerShell driver) asserting every effect by independent re-observation, never the command's own `ok:true` — same contract as `tests/e2e/run.sh` +- `windows-e2e` workflow_dispatch job on the self-hosted interactive Windows runner (registered in 2.1) + +**Key APIs:** `csc.exe`, WinForms `AutomationProperties.AutomationId` + +**Depends on:** 2.7, 2.8, 2.9, 2.10, 2.11 (everything the harness exercises) + +**Exit criteria:** the full Windows live gate is green in both headless and headed tiers. + +**Est. PR size:** ~2k LOC (mostly C#/scripts, not adapter Rust) + +### 2.13 — FFI, npm, Release + +**Goal:** Make the Windows adapter reachable through every distribution channel that already ships for macOS. + +**Scope:** +- FFI real-adapter path validated on Windows (non-stub tests — the stub-adapter tests already run cross-platform in CI, but the real `WindowsAdapter` behind the C ABI needs its own pass) +- npm `postinstall.js` gains a `win32-x64` (+ `win32-arm64` once an ARM64 runner exists) branch +- Release matrix: `.exe` zip + attestation, using the same tarball + sha256 + Sigstore pipeline Phase 1.5 already ships +- `skills/agent-desktop-windows/SKILL.md` — see Skill Update below +- README platform table: Windows column → **Yes** + +**Key APIs:** none new — this sub-phase is packaging, not adapter code + +**Depends on:** 2.2 through 2.11 (needs a working adapter to package) + +**Exit criteria:** `npm install -g` works on Windows; release dry-run artifacts verified. + +**Est. PR size:** ~1.2k LOC + +### 2.14 — Shell Surfaces & Notifications Spike (stretch, explicitly deferrable) + +**Goal:** Cover the Windows-only shell surface (P2-O18) and notification/tray (P2-O14 Windows half) scope that has no macOS analogue to backfill against. + +**Scope:** P2-O18 shell coverage, notification management, and system tray — all three folded in below rather than duplicated. This sub-phase **may ship after the 2.15 integration merge as its own follow-up `feat`** without blocking Phase 3 — it is explicitly the stretch/deferrable sub-phase in the sequence. + +**Key APIs:** see the three subsections immediately below. + +**Depends on:** 2.4 (observation), 2.7 (semantic actions) + +**Exit criteria:** `open-system-surface --surface <kind>` + `snapshot --surface <kind>` round-trips for Start menu, taskbar, Quick Settings, and Action Center where the current shell exposes them, with explicit `PLATFORM_NOT_SUPPORTED` assertions (clear `platform_detail`) where it does not; notification list/dismiss/action work through at least one of the two documented paths; tray list/click work through SNI-equivalent UIA traversal. + +**Est. PR size:** ~2k LOC + +#### Windows-specific command surface (P2-O18) Windows-specific commands are allowed when the operating-system concept has no portable equivalent, but they still follow the repository rules: one core command file, typed CLI/batch dispatch, adapter trait default returning `PLATFORM_NOT_SUPPORTED`, skill docs, and tests. The preferred path remains generic: expose shell UI as a surface, then let agents interact with refs. -Planned Windows shell commands: - | Command | Purpose | Platform behavior | |---------|---------|-------------------| | `open-system-surface --surface <kind>` | Opens an OS shell surface so agents can immediately call `snapshot --surface <kind>` and act by refs | Windows kinds: `start-menu`, `taskbar`, `system-tray`, `system-tray-overflow`, `action-center`, `quick-settings`. macOS may support `spotlight`, `dock`, `menu-bar-extras`, `notification-center`. Unsupported kinds return `PLATFORM_NOT_SUPPORTED` | @@ -848,54 +1262,44 @@ Planned Windows shell commands: No Windows-specific command bypasses refs for ordinary app controls. If a Windows workflow can be represented as `snapshot --app`, `snapshot --surface`, `find`, `click`, `type`, `press`, or `wait`, it uses the existing command surface. -### Notification Management (New Feature — Windows Implementation) +#### Notification Management (Windows Implementation) -Windows notification management must be implemented from scratch as part of Phase 2. The macOS notification implementation (completed as a follow-up to Phase 1) serves as the reference pattern — same `PlatformAdapter` trait methods (`list_notifications`, `dismiss_notification`, `dismiss_all_notifications`, `notification_action`), same JSON output contract, same 1-based indexing. Full notification parity is gated on a spike because Windows has two materially different surfaces: notification-listener APIs that require user permission/app identity, and shell UIA traversal that is best effort. +Windows notification management is built from scratch here. The macOS notification implementation (completed as a Phase 1 follow-up) is the reference pattern — same `PlatformAdapter` trait methods (`list_notifications`, `dismiss_notification`, `dismiss_all_notifications`, `notification_action`), same JSON output contract, same 1-based indexing. Full parity is gated on this being a spike because Windows has two materially different surfaces: notification-listener APIs that require user permission/app identity, and shell UIA traversal that is best effort. -**Implementation approach:** -- **Primary list path:** Use `UserNotificationListener` when package identity/capability and explicit user permission are available. If permission is denied, return `PERM_DENIED` with a permission-specific suggestion. -- **Fallback list path:** Open Action Center with `open-system-surface --surface action-center`; traverse exposed shell UIA elements only when they provide stable names/descriptions/action buttons. -- **Dismiss:** Prefer notification-listener APIs where supported; otherwise invoke the notification's dismiss/close button through UIA. For "dismiss all", invoke the shell's "Clear all" control only when present. -- **Interact with actions:** Resolve action buttons within the notification tree and invoke via the primary API or `InvokePattern`. -- **Focus Assist / Do Not Disturb:** Query through supported shell APIs first. Registry/WNF probes are best-effort diagnostics, not the sole source of truth. -- **Edge case:** Some notifications may be transient (disappear after timeout). The `wait --notification` command should monitor for new toasts via event subscription where supported; otherwise it polls the notification-listener or Action Center fallback within the normal wait deadline. +- **Primary list path:** `UserNotificationListener` when package identity/capability and explicit user permission are available. If permission is denied, return `PERM_DENIED` with a permission-specific suggestion. +- **Fallback list path:** open Action Center with `open-system-surface --surface action-center`; traverse exposed shell UIA elements only when they provide stable names/descriptions/action buttons. +- **Dismiss:** prefer notification-listener APIs where supported; otherwise invoke the notification's dismiss/close button through UIA. For "dismiss all", invoke the shell's "Clear all" control only when present. +- **Interact with actions:** resolve action buttons within the notification tree and invoke via the primary API or `InvokePattern`. +- **Focus Assist / Do Not Disturb:** query through supported shell APIs first; registry/WNF probes are best-effort diagnostics, not the sole source of truth. +- **Edge case:** some notifications are transient (disappear after timeout). `wait --notification` monitors via event subscription where supported; otherwise it polls the notification-listener or Action Center fallback within the normal wait deadline. -### System Tray (New Feature — Windows Implementation) +#### System Tray (Windows Implementation) -System tray interaction must be implemented from scratch as part of Phase 2. +System tray interaction is built from scratch here. -**Implementation approach:** -- **List items:** Access the system tray via UIA tree of `Shell_TrayWnd` window class. Tray items are children of the notification area. Overflow items live in `NotifyIconOverflowWindow` -- **Click:** `InvokePattern` on tray items, falling back to coordinate-based `SendInput` for items that don't expose UIA patterns -- **Open menu:** After clicking a tray item, detect the resulting popup menu via UIA focus-changed events and expose it for ref-based interaction +- **List items:** UIA tree of the `Shell_TrayWnd` window class; tray items are children of the notification area. Overflow items live in `NotifyIconOverflowWindow`. +- **Click:** `InvokePattern` on tray items, falling back to coordinate-based `SendInput` for items that don't expose UIA patterns. +- **Open menu:** after clicking a tray item, detect the resulting popup menu via UIA focus-changed events and expose it for ref-based interaction. -### Web/Electron App Compatibility +### 2.15 — Hardening & Integration Review -Chromium-based apps (Electron, Chrome, Edge, VS Code) expose deep, noisy accessibility trees where every HTML `<div>` becomes a UIA Group element. The macOS adapter solved this with three patterns that must be replicated identically on Windows. +**Goal:** Prove the assembled `feat/windows-adapter` branch is production-grade as a whole, then merge it. -**Chromium detection:** -- Detect Chromium-based windows via UIA process name or `Chrome_WidgetWin_1` window class matching -- If tree is empty or minimal for a Chromium window, warn: "This appears to be a Chromium app. Run the app with `--force-renderer-accessibility` to expose the accessibility tree" -- Include this guidance in the `platform_detail` field of the error response +**Scope:** +- Full-branch multi-agent review (small diff itself — this sub-phase is mostly verification, not new code) +- Live e2e in both headless and headed modes on the Windows runner +- Performance baseline vs `main` (`scripts/perf-baseline-compare.sh` run on Windows) +- LOC/size/isolation audits (`cargo tree -p agent-desktop-core` still zero platform crate names; binary still under 15MB) +- Docs/skills sync (this document, `skills/agent-desktop-windows/`, README) +- Merge `feat/windows-adapter` → `main` as one release-noted `feat!` -**Web-aware tree traversal (depth-skip):** -- Non-semantic wrapper elements (`UIA_GroupControlTypeId` / `UIA_CustomControlTypeId`) with empty `Name` AND empty `Value` properties do NOT consume depth budget during tree traversal -- This matches the macOS pattern where `AXGroup`/`AXGenericElement` wrappers are skipped -- Without this, default `--max-depth 10` finds ~3 refs in Slack; with it, finds 100+ refs -- Implement in `crates/windows/src/tree/builder.rs` with the same `is_web_wrapper` logic +**Key APIs:** none — verification and merge only -**Resolver depth:** -- Element re-identification must search up to `ABSOLUTE_MAX_DEPTH` (50), not a lower hardcoded limit -- Electron elements commonly sit at depth 25+ in the raw tree; a shallow resolver cap causes `STALE_REF` errors -- Implement in `crates/windows/src/tree/resolve.rs` matching the macOS pattern +**Depends on:** 2.0 through 2.14 (2.14 may lag as a follow-up per its own note; 2.15 does not have to wait on it if 2.14 explicitly ships later) -**Surface detection for Electron:** -- When an Electron app opens a modal (file picker, dialog), UIA may report the dialog as the focused window itself rather than a child of the parent window -- Surface detection (`list-surfaces`, `--surface sheet/alert`) must check if the focused window IS the target surface, not only search its children -- Check both `ControlType` and `LocalizedControlType` / UIA patterns (analogous to macOS checking both AXRole and AXSubrole) -- Implement in `crates/windows/src/tree/surfaces.rs` +**Exit criteria:** every item in the Cross-cutting sub-phase DoD holds for the whole branch; `main` gains Windows support in one commit. -**Progressive skeleton traversal** works identically on Windows — `--skeleton` and `--root` flags are platform-agnostic, handled entirely by core. The Windows adapter only needs to implement `get_subtree()` (which delegates to the same `build_subtree()` as `get_tree()`). Token savings for Electron apps (VS Code, Slack) apply equally. +**Est. PR size:** small diff, large verification effort ### Minimum OS Requirements @@ -910,11 +1314,13 @@ Chromium-based apps (Electron, Chrome, Edge, VS Code) expose deep, noisy accessi | Crate | Version | Scope | Purpose | |-------|---------|-------|---------| | `uiautomation` | 0.24+ | Windows | UIA client wrapper, tree walker, patterns | -| `windows` | 0.62.2 | Windows | Raw Win32 / WinRT bindings for SendInput, clipboard, `Windows.Graphics.Capture`, D3D11 frame pool. Pinned to 0.62.2 to match `windows-capture 1.5.x`'s own pin. | -| `windows-capture` | 1.5.4 | Windows | Modern per-window screenshot via `Windows.Graphics.Capture` in supported interactive sessions. Replaces `PrintWindow + PW_RENDERFULLCONTENT` as default, keeps legacy fallback. | -| `screencapturekit` | 1.5 (crates.io) | macOS | Published crates.io canonical crate — the doom-fish fork is the maintained successor, NOT a git-SHA pin. | +| `windows` | 0.62.2 | Windows | Raw Win32 / WinRT bindings for SendInput, clipboard, `Windows.Graphics.Capture`, D3D11 frame pool. Pinned to 0.62.2 to match `windows-capture 1.5.x`'s own pin | +| `windows-capture` | 1.5.4 | Windows | Modern per-window screenshot via `Windows.Graphics.Capture` in supported interactive sessions. Replaces `PrintWindow + PW_RENDERFULLCONTENT` as default, keeps legacy fallback | +| `screencapturekit` | 1.5 (crates.io) | macOS | Published crates.io canonical crate — the doom-fish fork is the maintained successor, NOT a git-SHA pin | | `objc2` | 0.6 | macOS (new for P2-O13 / O17) | Safe bridging to `SCScreenshotManager`, `CGPreflightScreenCaptureAccess`, and AppKit/Foundation calls scoped to screenshot/permissions code | +All five pins above were recorded at research time (2026-04); re-verify each against crates.io and the repository's supply-chain policy at the opening sub-phase of the consuming platform (2.1 for Windows, 3.1 for Linux) before adding them to `Cargo.toml`. + Added as target-gated dependencies in the owning platform crates. The binary crate only depends on the platform crate for the current target. ```toml # src/Cargo.toml @@ -936,112 +1342,60 @@ objc2 = { version = "0.6", features = ["Foundation", "AppKit"] } screencapturekit = "1.5" ``` -### Testing - -**Unit tests (windows):** -- UIA ControlType → role mapping coverage for all control types -- Permission check with mocks (COM security state) -- CacheRequest attribute batching correctness -- Element resolution round-trip (pid, role, name, bounds_hash) - -**Integration tests (Windows CI):** -- Snapshot Explorer — non-empty tree with refs, buttons, text fields -- Snapshot Notepad — text area with value, menu items -- Snapshot Settings — modern WinUI controls -- Snapshot Taskbar / Start / Quick Settings / Action Center surfaces where the runner exposes an interactive Explorer shell; otherwise assert `PLATFORM_NOT_SUPPORTED` with a clear `platform_detail` -- Click button in test app — verify action succeeded -- Type text into Notepad via ref — verify content changed -- Set-value on a text field — verify value set via UIA -- Clipboard get/set/clear roundtrip -- Wait for window title pattern -- Launch + close app lifecycle (Notepad: launch, type, close) -- Resize, move, minimize, maximize, restore window operations -- Screenshot produces valid PNG -- Large tree snapshot performance validation -- Chromium detection — verify warning when tree is empty -- Electron app snapshot (VS Code) — default depth finds 50+ refs via web-aware depth-skip -- Electron surface detection — file picker dialog detected as sheet surface -- List notifications — primary listener path when permission/app identity is available; Action Center UIA fallback otherwise -- Dismiss notification — verify removal through listener or Action Center fallback; skip with `PLATFORM_NOT_SUPPORTED` on unsupported shell builds -- Notification action — click action button on a test toast notification when the platform exposes one -- List tray items — returns known system tray entries (volume, network, clock) -- Click tray item — verify tray menu opens +### Testing (cross-platform validation, beyond each sub-phase's own exit criteria) **Cross-platform validation:** - Same snapshot of a cross-platform app (e.g., VS Code) produces structurally identical JSON on macOS and Windows - All error codes produce identical JSON envelope format **Cross-platform extension tests (P2-O8 through O17):** -- Stable-selector fields: known interactive controls emit `identifier` on both platforms when the app exposes one (UIA `AutomationId` on Windows, `AXIdentifier` on macOS); controls without stable IDs omit the field and still resolve through the fingerprint fallback -- Event subscription: `watch --event value-changed --ref @e3 --timeout 2000` receives an event within 500 ms of a programmatic value change on both platforms -- Text ranges: `text select-range @e1 5 10` + `text get-selection @e1` round-trips to `{start:5, length:10}` on both platforms for a multi-line text editor (TextEdit / Notepad) -- Text insert-at-caret: `text insert-at-caret @e1 "hello"` produces matching `value` on both platforms with the caret advanced correctly +- Stable-selector fields: known interactive controls emit `native_id` on both platforms when the app exposes one (UIA `AutomationId` on Windows, `AXIdentifier` or `AXDOMIdentifier` on macOS); controls without stable IDs omit the field and still resolve through the fingerprint fallback +- Event subscription: `watch --event value-changed --ref @s8f3k2p9:e3 --timeout 2000` receives an event within 500 ms of a programmatic value change on both platforms +- Text ranges: `text select-range @s8f3k2p9:e1 5 10` + `text get-selection @s8f3k2p9:e1` round-trips to `{start:5, length:10}` on both platforms for a multi-line text editor (TextEdit / Notepad) +- Text insert-at-caret: `text insert-at-caret @s8f3k2p9:e1 "hello"` produces matching `value` on both platforms with the caret advanced correctly - Modern screenshot: `screenshot --window <id>` PNG matches a reference capture within SSIM threshold on supported OS/session combinations; cold latency <50 ms on both platforms where modern capture is available (vs ~300 ms macOS subprocess baseline) - Toolbar surface: `snapshot --surface toolbar` on Safari (macOS) and Edge (Windows) returns the toolbar's children with refs - Electron deep-tree: VS Code snapshot with `--force-electron-a11y` exposes ≥100 refs at default depth on both platforms - Screen Recording permission: on a macOS runner without Screen Recording, `screenshot --window` returns `PermDenied` with the Screen Recording suggestion (distinct from AX denial) -- Automation permission: on a macOS runner without Automation for a target app, `close-app` returns `AutomationPermissionDenied` rather than squeezing into `ActionFailed` +- Automation permission: `permissions` reports `granted`, `denied`, or `unknown` without prompting; explicit requests run in the bounded isolated helper, while native `close-app` needs no Apple Event authorization **FFI parity tests (P2-O16):** -- `ad_abi_version()` returns a packed `u32` matching the Cargo version; consumer built against 0.2.0 refuses to load 0.3.0 -- `ad_snapshot` writes a refmap and the same `@e5` resolves via `ad_execute_by_ref` without a prior CLI snapshot on disk -- `ad_execute_by_ref(adapter, "@e5", AD_ACTION_KIND_CLICK, &out)` produces identical `AdActionResult` to `ad_resolve_element` + `ad_execute_action` -- `ad_set_log_callback` receives at least one `tracing::debug!` event during a `ad_get_tree` call +- `ad_abi_version()` returns a packed `u32` matching the Cargo version; a consumer built against an older ABI major refuses to load a newer one +- `ad_snapshot` writes a refmap and the same qualified ref resolves via `ad_execute_by_ref` without a prior CLI snapshot on disk +- `ad_execute_by_ref(adapter, "@s8f3k2p9:e5", AD_ACTION_KIND_CLICK, &out)` produces identical `AdActionResult` to `ad_resolve_element` + `ad_execute_action` +- `ad_set_log_callback` receives at least one `tracing::debug!` event during an `ad_get_tree` call - Every new `Action` variant round-trips through the `AdAction.kind` i32 → Rust enum conversion without UB on arbitrary bit patterns (extends the existing `fuzz_arbitrary_bit_patterns_never_panic_across_all_enums` suite) +- After the P2-O16 codegen migration: adding a command file automatically produces its `ad_<name>` wrapper — a regression test asserts the generated wrapper count matches the command registry count -### CI +Integration-level tests (Explorer/Notepad/Settings snapshots, click/type/clipboard/wait/lifecycle round-trips, Chromium detection, notification/tray list-and-act) are covered as exit criteria on their owning sub-phase above (2.4, 2.5, 2.7, 2.9, 2.10, 2.11, 2.14) rather than repeated here. -- Add GitHub Actions Windows runner alongside existing macOS runner -- Both runners execute: `cargo clippy --all-targets -- -D warnings`, `cargo test --workspace` -- `cargo tree -p agent-desktop-core` continues to contain zero platform crate names -- Binary size check: Windows `.exe` must be under 15MB +### Release, Skill & Docs (folds into 2.13 / 2.15) -### Release +**Release:** +- [ ] Prebuilt Windows `.exe` binary added to the existing `.github/workflows/release.yml` `build` matrix (alongside the macOS CLI targets), using the same tarball + sha256 + attestation pipeline Phase 1.5 ships +- [ ] npm `postinstall.js` gains a `win32-x64` / `win32-arm64` branch so `npm install -g agent-desktop` works on Windows without changes to package shape +- [ ] The Phase 1.5 FFI cdylib for Windows (`x86_64-pc-windows-msvc`) already ships; Phase 2 adds `aarch64-pc-windows-msvc` for ARM64 parity +- [ ] Every new `ad_*` FFI entrypoint (P2-O16) is included in the `release-ffi` build and CI header drift check +- [ ] GitHub Release notes document Windows support and installation -- [ ] Prebuilt Windows `.exe` binary added to the existing `.github/workflows/release.yml` `build` matrix (alongside the macOS CLI targets). Uses the same tarball + sha256 + attestation pipeline shipped in Phase 1.5. -- [ ] npm `postinstall.js` gains a `win32-x64` / `win32-arm64` branch so `npm install -g agent-desktop` works on Windows without changes to package shape. -- [ ] The Phase 1.5 FFI cdylib for Windows (`x86_64-pc-windows-msvc`) is already shipping; Phase 2 adds `aarch64-pc-windows-msvc` for ARM64 parity. -- [ ] Every new `ad_*` FFI entrypoint (P2-O16) is included in the `release-ffi` build and CI header drift check. -- [ ] GitHub Release notes document Windows support and installation. +**Skill Update:** +- [ ] Create `skills/agent-desktop-windows/SKILL.md`: UIA permission model and UAC handling; Windows-specific behaviors (UIA patterns, WinUI3 quirks, COM initialization, Start/taskbar/Action Center/Quick Settings shell surfaces, virtual desktop detection, mixed-DPI coordinates); Chromium/Electron compatibility (depth-skip, resolver depth, surface detection patterns); `--force-renderer-accessibility` guidance for empty trees; Windows error codes and `platform_detail` examples (HRESULT codes); troubleshooting guide (empty trees, COM errors, elevation failures) +- [ ] Update core `SKILL.md`: add Windows platform skill to the skill graph table; update platform support section +- [ ] Update `workflows.md`: add cross-platform patterns noting Windows-specific differences; add Windows-specific workflow examples (e.g., navigating UWP apps) -### Skill Update - -Skill docs are part of the release surface and must stay in sync with command behavior. - -- [ ] Create `skills/agent-desktop-windows/SKILL.md`: - - UIA permission model and UAC handling - - Windows-specific behaviors (UIA patterns, WinUI3 quirks, COM initialization, Start/taskbar/Action Center/Quick Settings shell surfaces, virtual desktop detection, mixed-DPI coordinates) - - Chromium/Electron compatibility: depth-skip, resolver depth, surface detection patterns - - `--force-renderer-accessibility` guidance for empty trees - - Windows error codes and `platform_detail` examples (HRESULT codes) - - Troubleshooting guide (empty trees, COM errors, elevation failures) -- [ ] Update core `SKILL.md`: - - Add Windows platform skill to skill graph table - - Update platform support section -- [ ] Update `workflows.md`: - - Add cross-platform patterns noting Windows-specific differences - - Add Windows-specific workflow examples (e.g., navigating UWP apps) - -### README Update - -- [ ] Update Platform Support table: Windows column → **Yes** -- [ ] Add Windows installation instructions: - - npm (same command, auto-detects platform) - - Direct `.exe` download from GitHub Releases - - From source: `cargo build --release` on Windows (note: requires MSVC toolchain) -- [ ] Add Windows permissions section: - - UIA works without special permissions for most apps - - UAC elevation may be required for elevated processes - - Chromium apps need `--force-renderer-accessibility` -- [ ] Update "From source" section with Windows build requirements (Rust + MSVC) +**README Update:** +- [ ] Platform Support table: Windows column → **Yes** +- [ ] Windows installation instructions: npm (same command, auto-detects platform); direct `.exe` download from GitHub Releases; from source: `cargo build --release` on Windows (requires MSVC toolchain) +- [ ] Windows permissions section: UIA works without special permissions for most apps; UAC elevation may be required for elevated processes; Chromium apps need `--force-renderer-accessibility` +- [ ] "From source" section updated with Windows build requirements (Rust + MSVC) --- -## Phase 3 — Linux Adapter + Cross-Platform Parity Completion +## Phase 3 — Linux Adapter -**Status: Planned** +**Status: Planned** — delivered as sub-phases 3.0–3.15 into the `feat/linux-adapter` integration branch, mirroring the Windows sub-phase template ([Platform Delivery Model](#platform-delivery-model--sub-phases-and-integration-branches)) one-to-one with AT-SPI2/D-Bus specifics substituted for UIA. Phase 3 begins only after Phase 2 (2.0–2.15) has merged to `main`. -Phase 3 completes the three-platform story. The Linux adapter implements the original adapter surface **plus** every cross-platform extension landed in Phase 2 (event subscriptions, text ranges, modern screenshot, stable-selector fields, Toolbar surface, new Action variants, new ErrorCode variants). Each has a canonical AT-SPI2 / D-Bus / Wayland-portal implementation. Core engine, trait contract, command-registry, CLI dispatch, FFI wrappers, and MCP transport are all untouched — per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, Phase 3 is **pure `PlatformAdapter` trait implementation code**, nothing else. No new command files, no CLI dispatch changes, no FFI wrappers, no MCP tool registrations. +Phase 3 completes the three-platform story. The Linux adapter implements the original adapter surface **plus** every cross-platform extension landed in Phase 2 (event subscriptions via `watch`, text ranges, modern screenshot, stable-selector fields, the predeclared toolbar and shell-surface vocabulary, new Action variants, new ErrorCode variants). Each has a canonical AT-SPI2 / D-Bus / Wayland-portal implementation. Core engine, trait contract, command-registry, CLI dispatch, FFI wrappers, and MCP transport are all untouched — per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, Phase 3 is **pure `PlatformAdapter` trait implementation code**, nothing else. No new command files, no CLI dispatch changes, no FFI wrappers, no MCP tool registrations. "Foundation contract" below means Phase 1 + Phase 1.6 + whatever Phase 2 landed on the trait — Phase 3 implements against that settled contract, it does not renegotiate it. ### Objectives @@ -1053,7 +1407,7 @@ Linux parity (original scope): | P3-O2 | All commands cross-platform | Identical JSON contract output on all 3 platforms for every command | | P3-O3 | Linux input synthesis | `click`, `type`, `press`, all mouse commands via AT-SPI actions + xdotool/ydotool | | P3-O4 | Linux screenshot | `screenshot` produces PNG via PipeWire ScreenCast portal (Wayland) / XGetImage (X11) | -| P3-O5 | Linux clipboard | `clipboard-get` / `clipboard-set` / `clipboard-clear` via `wl-clipboard` (Wayland) / `xclip` (X11) | +| P3-O5 | Linux clipboard | `clipboard-get` / `clipboard-set` / `clipboard-clear` via `wl-clipboard` (Wayland) / `xclip` (X11), marshaled through typed `ClipboardContent` | | P3-O6 | Cross-platform CI | GitHub Actions matrix: macOS + Windows + Ubuntu | | P3-O7 | Linux binary release | Prebuilt CLI binary added to the release pipeline (Phase 1.5 already ships the Linux FFI cdylib) | @@ -1061,65 +1415,175 @@ Cross-platform extensions (Linux implementations of Phase 2 primitives): | ID | Objective | Metric | |----|-----------|--------| -| P3-O8 | Stable-selector fields on Linux | `AccessibilityNode.identifier` populated from AT-SPI2 `accessible-id` attribute (standard since AT-SPI 2.18) with GTK `gtk-id` / Qt `objectName` fallback; `dom_id` / `dom_classes` populated from AT-SPI2 `object-attributes` HTML keys (`id`, `class`) on `WebKitGTK` / `Chromium-Content` embeds | -| P3-O9 | AT-SPI2 event subscriptions (P2-O11 parity) | `watch_element` implemented via `zbus::Proxy::receive_signal` on AT-SPI2 signals: `org.a11y.atspi.Event.Object.PropertyChange`, `ChildrenChanged`, `StateChanged:focused`, `Window:Create`, `Window:Destroy`. Same `wait --event` CLI shape as macOS/Windows. Replaces polling in `crates/linux/src/system/wait.rs` before it's even written | +| P3-O8 | Stable-selector fields on Linux | `AccessibilityNode.native_id` populated from AT-SPI2 `accessible-id` (standard since AT-SPI 2.18) with GTK `gtk-id` / Qt `objectName` fallback; `dom_classes` may be populated from AT-SPI2 `object-attributes` HTML keys on `WebKitGTK` / `Chromium-Content` embeds | +| P3-O9 | AT-SPI2 event subscriptions (`watch`, P2-O11 parity) | `watch_element` implemented via `zbus::Proxy::receive_signal` on AT-SPI2 signals: `org.a11y.atspi.Event.Object.PropertyChange`, `ChildrenChanged`, `StateChanged:focused`, `Window:Create`, `Window:Destroy`. Same `watch --event` CLI shape as macOS/Windows (see the `wait --event` vs `watch` naming note carried over from sub-phase 2.11). Replaces polling in `crates/linux/src/system/wait.rs` before it's even written | | P3-O10 | AT-SPI2 Text interface (P2-O12 parity) | Text range primitives via `org.a11y.atspi.Text` D-Bus methods: `GetText(start, end)`, `GetCaretOffset`, `SetCaretOffset`, `GetNSelections`, `GetSelection(n)`, `AddSelection(start, end)`, `RemoveSelection(n)`. `InsertAtCaret` uses `org.a11y.atspi.EditableText.InsertText(position, text, length)` | | P3-O11 | PipeWire modern screenshot (P2-O13 parity) | `screenshot --window <id>` via `org.freedesktop.portal.ScreenCast` (Wayland) + `org.freedesktop.portal.RemoteDesktop` for capture permission flow. XDG desktop portal handles the user consent dialog exactly like `SCScreenshotManager` does on macOS. X11 fallback uses `XGetImage` for the lowest-permission path | -| P3-O12 | Toolbar + surfaces (P2-O14 parity) | `SnapshotSurface::Toolbar` via AT-SPI2 `Role::ToolBar`. Dock / taskbar surface via per-DE panel process walk (GNOME Shell process for gnome-shell extensions, Plasma `plasmashell` for KDE). StatusNotifierWatcher already scoped in the original Phase 3 tray spec | +| P3-O12 | Toolbar + surfaces (P2-O14 parity) | `SnapshotSurface::Toolbar` via AT-SPI2 `Role::ToolBar` — already predeclared core-side (U12), same as Windows. Dock / taskbar surface via per-DE panel process walk (GNOME Shell process for gnome-shell extensions, Plasma `plasmashell` for KDE). StatusNotifierWatcher already scoped in the original Phase 3 tray spec (3.14) | | P3-O13 | Action variants on Linux (P2-O9 parity) | `Action::LongPress` via timed `xdotool/ydotool` button-hold; `Action::ShowMenu` via `org.a11y.atspi.Action.DoAction("popup")`; `Action::Cancel` via `Action.DoAction("cancel")` or Escape synthesis; `Action::DeliverFiles` via portal/native file-transfer where available with XDND as a researched fallback; `Action::ForceClick` returns `ActionNotSupported` on Linux (no pressure input primitive) | -| P3-O14 | FFI cdylib continues to ship | Phase 1.5 already publishes Linux FFI for x86_64 + aarch64; Phase 3 adds each new `ad_*` entrypoint's Linux implementation and extends the header drift check. No new FFI bindings to design — just implementations for the platform-specific methods under the existing trait | +| P3-O14 | FFI cdylib continues to ship | Phase 1.5 already publishes Linux FFI for x86_64 + aarch64; Phase 3 adds each new `ad_*` entrypoint's Linux implementation and extends the header drift check. No new FFI bindings to design — implementations only; the same P2-O16 registry migration applies once it lands | | P3-O15 | Flatpak / Snap compatibility note | AT-SPI2 requires `--talk-name=org.a11y.Bus` permission inside sandboxed runtimes. Skill docs include the exact Flatpak override and Snap plug grants, so sandboxed consumers aren't silently empty-tree | -### Linux Adapter Implementation +### Sub-phase decomposition (3.0–3.15, mirrors Phase 2 one-to-one) -Full `LinuxAdapter` in `crates/linux/src/` following the identical platform crate folder structure: +Same rendering shape, same [Cross-cutting sub-phase DoD](#cross-cutting-sub-phase-dod) as Phase 2 — restated once here rather than per sub-phase: fmt/clippy/`-D warnings`/lib tests/conformance green; probe evidence (AT-SPI2 D-Bus introspection dumps, not screenshots) committed with the plan; adapters keep `not_supported()` defaults for everything not yet landed; no core rewrites; per-sub-phase review; perf baseline on perf-touching sub-phases; Conventional Commits. Every "foundation contract" reference below means Phase 1 + 1.6 + Phase 2's settled trait surface. -``` -crates/linux/src/ -├── lib.rs # mod declarations + re-exports only -├── adapter.rs # LinuxAdapter: PlatformAdapter impl -├── tree/ -│ ├── mod.rs # re-exports -│ ├── element.rs # AT-SPI Accessible wrapper + attribute readers -│ ├── builder.rs # D-Bus tree traversal via GetChildren -│ ├── roles.rs # AT-SPI Role enum → unified role enum mapping -│ ├── resolve.rs # Element re-identification for ref resolution -│ └── surfaces.rs # Surface detection (menus, dialogs, popovers) -├── actions/ -│ ├── mod.rs # re-exports -│ ├── dispatch.rs # perform_action via AT-SPI Action interface -│ ├── activate.rs # Smart activation chain (DoAction → coordinate fallback) -│ └── extras.rs # Text.InsertText, Selection helpers -├── input/ -│ ├── mod.rs # re-exports -│ ├── keyboard.rs # xdotool (X11) / ydotool (Wayland) keyboard synthesis -│ ├── mouse.rs # xdotool (X11) / ydotool (Wayland) mouse events -│ └── clipboard.rs # wl-clipboard (Wayland) / xclip (X11) clipboard ops -├── notifications/ -│ ├── mod.rs # re-exports -│ ├── list.rs # List notifications via D-Bus org.freedesktop.Notifications or daemon-specific API -│ ├── dismiss.rs # Dismiss/close notifications via CloseNotification D-Bus method -│ └── interact.rs # Invoke notification actions via ActionInvoked D-Bus signal -├── tray/ -│ ├── mod.rs # re-exports -│ ├── list.rs # List tray items via StatusNotifierItem D-Bus interface or AT-SPI -│ └── interact.rs # Activate/context-menu tray items via D-Bus methods -└── system/ - ├── mod.rs # re-exports - ├── app_ops.rs # App launch via xdg-open / process spawn, close via SIGTERM/SIGKILL - ├── window_ops.rs # xdotool / wmctrl for resize/move/minimize/maximize/restore - ├── key_dispatch.rs # App-targeted key press via window focus + input synthesis - ├── permissions.rs # AT-SPI2 bus availability check, DBUS_SESSION_BUS_ADDRESS detection - ├── screenshot.rs # PipeWire ScreenCast portal (Wayland) / XGetImage (X11) / xcap crate - └── wait.rs # wait utilities (polling AT-SPI element existence) -``` +### 3.0 — Platform Exploration & Raw Scripting (pre-Rust) -### Linux API Mapping +**Goal:** empirically map Linux/AT-SPI2 accessibility reality with raw, no-Rust scripts before any adapter code exists, producing a committed evidence corpus the Rust sub-phases implement against — and feeding every contradiction back into this document. Mirror of 2.0 for Linux/AT-SPI2. + +**Scope:** a `probes/linux/` directory of raw scripts in Python (direct D-Bus via python-dbus/`busctl`, and pyatspi2 where it clarifies) on GNOME (X11 and Wayland sessions both): full-tree dumps (GNOME Files, Terminal, Text Editor, one Electron app), interface census per Role (Action, Text, EditableText, Value, Selection, Component, StateSet), every interaction exercised raw (DoAction by index/name, text get/caret/insert, value set, selection), input-synthesis experiments via xdotool (X11) and ydotool (Wayland), `Component.Contains`/bounds hit-testing incl. occlusion, event-signal observations over the a11y bus, an accessible-id coverage census (GTK3/GTK4/Qt/Electron), the portal screenshot consent flow, a Flatpak sandbox probe (`--talk-name=org.a11y.Bus` on/off), and bus-bootstrap behavior when at-spi is cold. Alongside the scripts: `probes/linux/FINDINGS.md` — the same findings-ledger shape as 2.0, mapping every experiment to observed behavior and a doc-alignment verdict (confirms this document / contradicts it / new edge case). + +**Key APIs:** python-dbus, `busctl`, pyatspi2, `Component.Contains`, xdotool, ydotool. + +**Depends on:** completion of Phase 2 (Windows ships first). + +**Exit criteria:** the script corpus and captured outputs are committed and re-runnable on the dev VM; the findings ledger covers tree, interfaces, interactions, input, hit-testing, identity, events, portal-consent, sandbox, and bus-bootstrap behavior with no open "unknown" rows; every ledger entry that contradicts this document has a matching amendment to this document landed in the same PR (see the source-of-truth feedback rule in the Platform Delivery Model); no Rust adapter sub-phase (3.2 onward) starts until the ledger is complete and every contradiction has amended this document. + +**Est. PR size:** ~1.5k lines (scripts + ledger; no Rust). + +### 3.1 — Toolchain, CI & Bus Bootstrap + +**Goal:** Stand up the Linux build/CI/session substrate, including the async runtime the rest of Phase 3 depends on. + +**Scope:** +- `ubuntu-latest` CI job promoted to a real Linux test lane (build + clippy + lib tests, core-isolation, size), mirroring 2.1's Windows promotion +- AT-SPI2 `org.a11y.Bus` session-bus availability detection at adapter construction +- `zbus` (5.x) / `atspi` (0.28+) / `tokio` (1.x) dependency pins re-verified against crates.io + supply-chain policy (pinned at 2026-04 research time, same policy as Windows) +- `LinuxAdapterSession` implementing `AdapterSession` via `open_session` — owns the D-Bus connection state so later sub-phases share one connection rather than reconnecting per call +- **`tokio` enters the workspace here for the first time.** The workspace is synchronous through Phase 2 (Windows adds no async runtime — UIA is a synchronous COM API); Linux is the first platform requiring async D-Bus calls via `zbus`/`atspi` + +**Key APIs:** `org.a11y.Bus` presence check, `zbus::Connection` + +**Depends on:** Phase 2 fully merged to `main` + +**Exit criteria:** workspace green on Ubuntu CI; `LinuxAdapter` constructs and satisfies the trait; every command returns honest `PLATFORM_NOT_SUPPORTED` on Linux; missing-bus case returns `PLATFORM_NOT_SUPPORTED` with distro/DE-specific enable instructions (see AT-SPI2 Bus Detection below); bus-availability detection is unit-tested against mocked D-Bus responses. + +**Est. PR size:** ~0.8k LOC + +#### AT-SPI2 Bus Detection + +- Check for `org.a11y.Bus` presence on the D-Bus session bus +- If the bus is not running, return `PLATFORM_NOT_SUPPORTED` with instructions: + - GNOME: "AT-SPI2 should be enabled by default. Check `gsettings get org.gnome.desktop.interface toolkit-accessibility`" + - Other DEs: "Install `at-spi2-core` and ensure `at-spi-bus-launcher` is running" + - Flatpak/Snap: "Ensure the app has `--talk-name=org.a11y.Bus` permission" (P3-O15) + +### 3.2 — Accessible Wrapper & Tree Walk + +**Goal:** Own an `AXElement`-equivalent async wrapper for AT-SPI2 `Accessible` objects and prove raw tree traversal against a real Linux app. + +**Scope:** +- `Accessible` D-Bus proxy wrapper (async, via `zbus`) with the same cycle-guard discipline as macOS/Windows (ancestor-path set, not a global visited set) +- Batched-fetch strategy — AT-SPI2 has no single "copy multiple attributes" call like `AXUIElementCopyMultipleAttributeValues`/`CacheRequest`, so this sub-phase designs the concurrent-D-Bus-call batching strategy (e.g. `futures::join_all` over property-get calls) that stands in for it +- Committed probe examples: raw AT-SPI2 D-Bus introspection dumps (`busctl --user introspect` or the `atspi` crate's own dump) of GNOME Files / GNOME Terminal, checked in as evidence + +**Key APIs:** `org.a11y.atspi.Accessible.GetChildren`, `atspi` crate (v0.28+) + `zbus` (5.x) — pure Rust, no libatspi/GLib dependency + +**Depends on:** 3.1 + +**Exit criteria:** an internal async tree-dump binary prints GNOME Files and Terminal trees with the batching strategy in place. + +**Est. PR size:** ~2k LOC + +### 3.3 — Vocabulary: Role, StateSet, native_id, Name Evidence + +**Goal:** Map AT-SPI2's vocabulary onto the canonical role/state contract (U1/U2), completing `native_id` (P2-O8/P3-O8) on the third platform. + +**Scope:** +- AT-SPI `Role` enum → unified role enum in `tree/roles.rs` +- AT-SPI `StateSet` → canonical state vocabulary — same conformance tests U1/U2 already run, parameterized over `LinuxAdapter` +- `accessible-id` (standard since AT-SPI 2.18) → `native_id`, with GTK `gtk-id` / Qt `objectName` fallback (P3-O8) +- `NameEvidence` supplier feeding core `accname.rs` (AT-SPI `Name`, `LabelledBy` relation, `Description`) + +**Key APIs:** AT-SPI `Role` enum, `StateSet`, `accessible-id`, `Name`/`LabelledBy`/`Description` + +**Depends on:** 3.2 + +**Exit criteria:** vocabulary conformance tests span every AT-SPI `Role` (complete mapping coverage, not a sample) and accname tests pass on Linux. + +**Est. PR size:** ~1.5k LOC + +### 3.4 — Observation: Snapshot, Windows, Apps, Displays + +**Goal:** Land the full read path via portals/randr and the web-wrapper depth-skip for Electron/WebKitGTK content. + +**Scope:** +- `get_tree`/`get_subtree` wired to the shared `SnapshotEngine` via async D-Bus `GetChildren` walks +- `list_windows`/`list_apps`/`focused_window` via the window-manager-neutral AT-SPI2 `Window` interface where available, falling back to `wmctrl`/`xdg-desktop-portal` introspection +- `list_displays` via XRandR (X11) / the Wayland portal's equivalent query + per-monitor scale +- **Web-wrapper depth-skip:** non-semantic wrapper elements with AT-SPI roles `ROLE_PANEL`, `ROLE_SECTION`, or `ROLE_FILLER` that have empty `Name` AND empty `Value` do not consume depth budget — the AT-SPI equivalent of macOS `AXGroup`/`AXGenericElement` and Windows `UIA_GroupControlTypeId` skipping. Implement in `crates/linux/src/tree/builder.rs` as `is_web_wrapper` +- **Chromium detection:** process name matching (electron, chrome, chromium, code); Linux Chromium additionally respects `ACCESSIBILITY_ENABLED=1` as an alternative to `--force-renderer-accessibility` +- **Resolver depth:** element re-identification searches to `ABSOLUTE_MAX_DEPTH` (50), matching macOS/Windows; implement in `crates/linux/src/tree/resolve.rs` +- **Surface detection for Electron:** an Electron modal may report as the active window itself rather than a child; check both `Role` and `RelationSet`/`RELATION_EMBEDS` (analogous to macOS AXRole + AXSubrole); implement in `crates/linux/src/tree/surfaces.rs` + +**Key APIs:** `org.a11y.atspi.Window`, XRandR / portal display enumeration, `wmctrl` (fallback) + +**Depends on:** 3.3 + +**Exit criteria:** `snapshot --app "GNOME Files"` and GNOME Terminal return reffed trees; a VS Code snapshot at default depth finds 50+ refs through web-aware depth-skip alone (no force flag); an Electron file-picker dialog is detected as a sheet surface. + +**Est. PR size:** ~2k LOC + +### 3.5 — Resolution & Live Locator + +**Goal:** Make refs and the live `find`/`get`/`is` commands (U7) work on Linux with the same strict-resolution guarantees. + +**Scope:** +- `resolve_element_strict*` from `RefEntry` evidence — `accessible-id`-first, fingerprint fallback, 0/1/N classification +- `get_live_value`/`get_live_state`/`get_live_actions`/`get_live_element`/`get_element_bounds` via async D-Bus property reads +- `resolve_query` — `LocatorQuery` evaluator backing live `find` +- `resolve_locator_anchor` + selected-hydration completeness — the same definitive-absence-vs-transport-failure classification ported from macOS's `is_definitive_absence`, this time distinguishing "object no longer on the bus" (D-Bus `UnknownObject`/`UnknownMethod` errors) from a genuine timeout + +**Key APIs:** `accessible-id` lookup, AT-SPI2 D-Bus error classification (`org.freedesktop.DBus.Error.UnknownObject` vs timeout) + +**Depends on:** 3.4 + +**Exit criteria:** `find`/`get`/`is` are live on Linux; `STALE_REF`/`AMBIGUOUS_TARGET` semantics proven with committed probe evidence. + +**Est. PR size:** ~2k LOC + +### 3.6 — Actionability & Occlusion + +**Goal:** Port the auto-wait/occlusion gate (U8/U9) onto Linux using AT-SPI2's `Component` interface. + +**Scope:** +- `hit_test` three-way result via `Component.ContainsPoint` + bounds corroboration — `Unknown` on probe failure, never a false negative +- `receives_events` evidence +- Visibility/enabled/offscreen evidence feeding the core auto-wait gate — no Linux-specific auto-wait logic, core drives the loop +- `scroll_into_view` — AT-SPI2 has no native scroll-into-view primitive; implement via `Component.GetPosition` + coordinate-based scroll synthesis, following the same policy gating as the Phase 1 scroll command + +**Key APIs:** `org.a11y.atspi.Component.ContainsPoint`, `Component.GetExtents` + +**Depends on:** 3.5 + +**Exit criteria:** zero-bounds/disabled/occluded fixture cases produce the same envelopes as macOS/Windows. + +**Est. PR size:** ~1.2k LOC + +### 3.7 — Semantic Action Tier + +**Goal:** Land AT-SPI2 `Action`/`EditableText`-based semantic dispatch with the same typed `ActionStep` delivery reporting. + +**Scope:** +- `perform_action` via `org.a11y.atspi.Action.DoAction(0)` (click), name-based dispatch for expand/collapse/toggle (`DoAction("expand")`/`"collapse"`/`"toggle"`), `Selection.SelectChild` (select), `EditableText.InsertText` (set text, falling back to clipboard paste) +- Activation chain with `ActionStep` delivery reporting + post-verification reads, same honest `verified` semantics as macOS/Windows +- See the Linux API Mapping table below for the full pattern list + +**Key APIs:** `Action.DoAction`, `Selection.SelectChild`, `EditableText.InsertText` + +**Depends on:** 3.6 + +**Exit criteria:** click/set-value/clear/select/toggle/expand/collapse work headless on the fixture app via the e2e analog (3.12 supplies the fixture; interim coverage via GNOME Files/Terminal). + +**Est. PR size:** ~2k LOC + +#### Linux API Mapping (reference table for sub-phases 3.2–3.10) | Capability | Technology | Details | |------------|-----------|---------| | Tree root | `atspi Accessible` on bus | Via `atspi` crate (v0.28+) + `zbus` (5.x) — pure Rust, no libatspi/GLib dependency | -| Children | `org.a11y.atspi.Accessible.GetChildren` | Async D-Bus calls to AT-SPI2 registry daemon | +| Children | `org.a11y.atspi.Accessible.GetChildren` | Async D-Bus calls to the AT-SPI2 registry daemon | | Role mapping | AT-SPI `Role` enum | Map to unified role enum in `tree/roles.rs` — e.g. `Role::PushButton` → `button` | | Click | `org.a11y.atspi.Action.DoAction(0)` | AT-SPI actions preferred over coordinate-based input | | Set text | `org.a11y.atspi.Text.InsertText` | AT-SPI text interface; falls back to clipboard paste | @@ -1129,88 +1593,194 @@ crates/linux/src/ | Scroll | Coordinate-based scroll events via xdotool/ydotool | AT-SPI has no native scroll pattern | | Keyboard | `xdotool key` (X11) / `ydotool key` (Wayland) | Shelling out for input synthesis | | Mouse | `xdotool mousemove/click` (X11) / `ydotool mousemove/click` (Wayland) | Display server detected at runtime | -| Clipboard | `wl-copy` / `wl-paste` (Wayland) / `xclip` (X11) | Shelling out; display server detected at runtime | -| Screenshot | PipeWire ScreenCast portal (Wayland) / `XGetImage` (X11) | Or `xcap` crate for consistency | -| App launch | `xdg-open` / direct process spawn | Launch by .desktop file or command name | -| App close | `SIGTERM` / `SIGKILL` | Graceful close first, force with `--force` | +| Clipboard | `wl-copy` / `wl-paste` (Wayland) / `xclip` (X11) | Shelling out; display server detected at runtime; marshaled through typed `ClipboardContent` | +| Screenshot | PipeWire ScreenCast portal (Wayland) / `XGetImage` (X11) | Or the `xcap` crate for consistency | +| App launch | `xdg-open` / direct process spawn | Launch by `.desktop` file or command name, via `LaunchOptions` | +| App close | `SIGTERM` / `SIGKILL` | Graceful close first, force with `--force`; verified via `ProcessState` | | Window ops | `xdotool` / `wmctrl` | Window resize, move, minimize, maximize, restore | -| Permissions | AT-SPI2 bus availability | Check for `org.a11y.Bus` on D-Bus session bus. Return `PLATFORM_UNSUPPORTED` with enable instructions if missing | -| Notifications | D-Bus `org.freedesktop.Notifications` | List via `GetServerInformation` + monitoring `Notify` signals. History varies by daemon: GNOME uses `org.gnome.Shell.Notifications`, KDE uses `org.freedesktop.Notifications` with `GetNotifications`. Dismiss via `CloseNotification(id)`. Interact via `ActionInvoked` signal. Do Not Disturb: GNOME `org.gnome.desktop.notifications.show-banners`, KDE `org.kde.notificationmanager` | -| System tray | D-Bus `org.kde.StatusNotifierWatcher` | SNI (StatusNotifierItem) protocol for modern tray items. Legacy XEmbed tray items via AT-SPI tree of the tray window. List via `RegisteredStatusNotifierItems` property. Activate via `Activate(x, y)` method. Context menu via `ContextMenu(x, y)` method. Fallback: coordinate-based click for XEmbed items | +| Permissions | AT-SPI2 bus availability | Check for `org.a11y.Bus` on the D-Bus session bus. Return `PLATFORM_NOT_SUPPORTED` with enable instructions if missing | +| Notifications | D-Bus `org.freedesktop.Notifications` | See Notification Management approach under 3.14 | +| System tray | D-Bus `org.kde.StatusNotifierWatcher` | See System Tray approach under 3.14 | -### Notification Management (New Feature — Linux Implementation) +### 3.8 — Input Synthesis -Linux notification management must be implemented from scratch as part of Phase 3. The macOS implementation (completed) and Windows implementation (Phase 2) serve as reference patterns — same trait methods, same JSON output contract, same 1-based indexing. +**Goal:** Land raw OS input across the X11/Wayland split, matching the delivery-tracking and headed/headless policy contract. -**Implementation approach:** -- **List notifications:** The standard `org.freedesktop.Notifications` D-Bus interface does NOT provide a "list current notifications" method. Approach varies by desktop environment: - - GNOME: `org.gnome.Shell` exposes `org.gnome.Shell.Notifications` interface with `GetNotifications()` method (returns array of notification dicts) - - KDE Plasma: `org.freedesktop.Notifications` with `GetNotifications()` extension, or `org.kde.notificationmanager` D-Bus interface - - Other DEs: Monitor `Notify` D-Bus signals to maintain an in-memory notification history within the daemon session -- **Dismiss:** `org.freedesktop.Notifications.CloseNotification(id)` D-Bus method call. Works across all notification daemons -- **Interact with actions:** Listen for user-triggered actions or programmatically invoke via `ActionInvoked` signal. Note: the D-Bus spec does not define a method to programmatically trigger actions — coordinate-based click on the notification popup via AT-SPI may be needed as a fallback -- **Do Not Disturb:** - - GNOME: `gsettings get org.gnome.desktop.notifications show-banners` (boolean) - - KDE: `org.kde.notificationmanager` D-Bus interface, `inhibited` property -- **Edge case:** Notification daemon varies by DE — detect via `GetServerInformation()` D-Bus method. Return `PLATFORM_UNSUPPORTED` with daemon-specific guidance if the notification interface is unreachable +**Scope:** +- Display-server-detected keyboard synthesis: `xdotool key` (X11) / `ydotool key` (Wayland) +- Mouse events + modifier chords + wheel via the same detected tool +- Drag with delivery tracking + release guard +- Headed/headless policy parity — raw cursor commands require `--headed`, same as macOS/Windows +- **`libei`** (the newer input-emulation portal API) is documented here as a researched future alternative to shelling out to xdotool/ydotool, not adopted as a dependency in this sub-phase — it would remove the subprocess dependency but isn't mature enough across desktop environments yet to be the default -### System Tray (New Feature — Linux Implementation) +**Key APIs:** `xdotool`, `ydotool` (subprocess), `libei` (documented, not used) -System tray interaction must be implemented from scratch as part of Phase 3. +**Depends on:** 3.7 -**Implementation approach:** -- **Modern tray (SNI):** Most modern Linux apps use the `StatusNotifierItem` (SNI) D-Bus protocol. Discover items via `org.kde.StatusNotifierWatcher.RegisteredStatusNotifierItems` property -- **Legacy tray (XEmbed):** Older apps use XEmbed protocol. Access via AT-SPI tree of the tray window, or coordinate-based interaction -- **List items:** Query `StatusNotifierWatcher` for registered items. Each item exposes `Title`, `IconName`, `ToolTip`, `Menu` (D-Bus menu path) properties -- **Activate:** Call `Activate(x, y)` method on the `StatusNotifierItem` D-Bus interface -- **Context menu:** Call `ContextMenu(x, y)` method, or read the `Menu` property to get the `com.canonical.dbusmenu` path and traverse the menu tree -- **Edge case:** GNOME does not natively support SNI (requires `AppIndicator` extension). Detect and report via error suggestion if no tray is available +**Exit criteria:** headed e2e gesture cases pass on at least GNOME/X11 and GNOME/Wayland (once 3.12's fixture exists). + +**Est. PR size:** ~2k LOC + +### 3.9 — System Lifecycle + +**Goal:** Land process/window lifecycle with the same `ProcessState` liveness contract. + +**Scope:** +- `launch_app` with `LaunchOptions` (spawn via `xdg-open` or direct `Command::spawn`, args/env/cwd, attach-vs-fail policy) +- `close_app` with verified termination (`SIGTERM` then `SIGKILL` under `--force`, confirmed via a follow-up liveness probe, not assumed) +- `window_op` via `xdotool`/`wmctrl` for resize/move/minimize/maximize/restore +- `ProcessState` probes: `/proc/<pid>/stat` state character for hung detection where meaningful, exit-code inspection → `Exited`/`Crashed` +- `is_protected_process` +- `press_key_for_app` under the same focus policy + +**Key APIs:** `xdg-open`, `std::process::Command`, `xdotool`/`wmctrl`, `/proc/<pid>/stat` + +**Depends on:** 3.4 (window identity), 3.8 (input for `press_key_for_app`) + +**Exit criteria:** lifecycle e2e (launch → interact → close) passes. + +**Est. PR size:** ~1.8k LOC + +### 3.10 — Capture & Clipboard + +**Goal:** Ship screenshot and typed clipboard across the Wayland-portal / X11 split. + +**Scope:** +- `screenshot` via `org.freedesktop.portal.ScreenCast` (Wayland, P3-O11) with `org.freedesktop.portal.RemoteDesktop` for the permission flow, falling back to `XGetImage` (X11) for the lowest-permission path +- `screenshot --screen` honest display targeting (pairs with `list_displays` from 3.4) +- Typed clipboard: `wl-clipboard` (Wayland) / `xclip` (X11) subprocess calls, marshaled into `ClipboardContent::Text`/`Image`/`FileUrls`, written through 0600-equivalent private files + +**Key APIs:** `org.freedesktop.portal.ScreenCast`, `org.freedesktop.portal.RemoteDesktop`, `XGetImage`, `wl-clipboard`/`xclip` + +**Depends on:** 3.1 (private-file handling — Unix permissions already real, unlike the Windows ACL gap), 3.4 (displays) + +**Exit criteria:** screenshot + clipboard e2e pass (clipboard tests hermetic); the PipeWire portal flow is proven — the user approves via the XDG portal dialog once, subsequent calls bypass the dialog within the session grant window. + +**Est. PR size:** ~1.8k LOC + +### 3.11 — Signals & Wait Parity + +**Goal:** Port `SignalBaseline`/`diff_signals`/`wait --event` (U17) to Linux, with the same naming distinction from `watch` as Windows 2.11. + +**Scope:** +- Linux `SignalBaseline` producers: windows/apps/focus/surfaces via AT-SPI2 polling snapshots +- `wait --event` parity including `surface-appeared` +- Wait utilities operating within `Deadline` budgets + +**Key APIs:** AT-SPI2 property snapshots for baseline capture (no signal subscription yet — that's `watch`, P3-O9; sub-phase 3.2's D-Bus infrastructure feeds it but the command itself is still future scope beyond this sub-phase's baseline-diff wait) + +**Depends on:** 3.4 (windows/apps/displays), 3.9 (process lifecycle) + +**Exit criteria:** an AE6-analog e2e passes — an unnamed dialog is discovered purely by baseline diff. + +**Est. PR size:** ~1k LOC + +### 3.12 — GTK4 Fixture App & Live E2E Harness + +**Goal:** Give Linux the same verify-by-observation live e2e discipline, with explicit per-desktop-environment notes since Linux fragments more than macOS/Windows. + +**Scope:** +- GTK4 fixture app (GNOME primary target) with `accessible-id`/`gtk-id` set on every interactive target from day one +- Fixture targets mirroring `AgentDeskFixture.swift`/the Windows WinForms fixture: delayed-enable, zero-bounds, duplicate-title, occlusion, disclosure +- Harness port asserting every effect by independent re-observation, same contract as `tests/e2e/run.sh` +- Per-DE notes: GNOME is the primary CI target (best AT-SPI2 support); KDE Plasma 5.24+ is a secondary, manually-verified target, not a CI gate in this sub-phase +- `linux-e2e` workflow_dispatch job on an interactive Ubuntu GNOME runner + +**Key APIs:** GTK4 `Accessible` widget properties (`accessible-id`) + +**Depends on:** 3.7, 3.8, 3.9, 3.10, 3.11 + +**Exit criteria:** the full Linux live gate is green on GNOME, both headless and headed tiers. + +**Est. PR size:** ~2k LOC (mostly fixture app + scripts, not adapter Rust) + +### 3.13 — FFI, npm, Release + +**Goal:** Make the Linux adapter reachable through every distribution channel already shipping for macOS/Windows. + +**Scope:** +- FFI real-adapter path validated on Linux (non-stub tests) for both x86_64 and aarch64 +- npm `postinstall.js` gains `linux-x64`/`linux-arm64` branches +- Release matrix: CLI binary added for `x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu`, reusing the runners Phase 1.5 already builds the FFI cdylib on +- `skills/agent-desktop-linux/SKILL.md` — see Skill Update below +- README platform table: Linux column → **Yes**; minimum glibc (2.35, Ubuntu 22.04 baseline) documented + +**Key APIs:** none new — packaging only + +**Depends on:** 3.2 through 3.11 + +**Exit criteria:** `npm install -g` works on Ubuntu; release dry-run artifacts verified for both architectures. + +**Est. PR size:** ~1.2k LOC + +### 3.14 — Shell Surfaces & Notifications Spike (stretch, explicitly deferrable) + +**Goal:** Cover the Linux-only shell surface, notification, and tray scope — the DE-specific half of parity that has no single canonical implementation across GNOME/KDE/other DEs. + +**Scope:** notification management and system tray, folded in below. Same stretch/deferrable status as Windows 2.14 — **may ship after the 3.15 integration merge as its own follow-up `feat`.** + +**Key APIs:** see the two subsections immediately below. + +**Depends on:** 3.4 (observation), 3.7 (semantic actions) + +**Exit criteria:** notification list/dismiss/action work on at least GNOME via `org.gnome.Shell.Notifications`; tray list/click work via `StatusNotifierWatcher` on a DE that supports SNI (KDE, or GNOME + AppIndicator extension); unsupported DEs assert `PLATFORM_NOT_SUPPORTED` with daemon/DE-specific guidance rather than silently passing. + +**Est. PR size:** ~2k LOC + +#### Notification Management (Linux Implementation) + +Linux notification management is built from scratch here. The macOS (completed) and Windows (Phase 2) implementations are the reference patterns — same trait methods, same JSON output contract, same 1-based indexing. + +- **List notifications:** the standard `org.freedesktop.Notifications` D-Bus interface does NOT provide a "list current notifications" method; the approach varies by desktop environment: + - GNOME: `org.gnome.Shell` exposes `org.gnome.Shell.Notifications` with `GetNotifications()` (returns an array of notification dicts) + - KDE Plasma: `org.freedesktop.Notifications` with a `GetNotifications()` extension, or the `org.kde.notificationmanager` D-Bus interface + - Other DEs: monitor `Notify` D-Bus signals to maintain an in-memory notification history within the daemon session +- **Dismiss:** `org.freedesktop.Notifications.CloseNotification(id)` — works across all notification daemons +- **Interact with actions:** listen for user-triggered actions, or programmatically invoke via the `ActionInvoked` signal; the D-Bus spec does not define a method to trigger actions programmatically — coordinate-based click on the notification popup via AT-SPI may be needed as a fallback +- **Do Not Disturb:** GNOME `gsettings get org.gnome.desktop.notifications show-banners`; KDE `org.kde.notificationmanager`'s `inhibited` property +- **Edge case:** notification daemon varies by DE — detect via `GetServerInformation()`; return `PLATFORM_NOT_SUPPORTED` with daemon-specific guidance if the notification interface is unreachable + +#### System Tray (Linux Implementation) + +System tray interaction is built from scratch here. + +- **Modern tray (SNI):** most modern Linux apps use the `StatusNotifierItem` D-Bus protocol; discover items via `org.kde.StatusNotifierWatcher.RegisteredStatusNotifierItems` +- **Legacy tray (XEmbed):** older apps use the XEmbed protocol; access via the AT-SPI tree of the tray window, or coordinate-based interaction +- **List items:** query `StatusNotifierWatcher` for registered items — `Title`, `IconName`, `ToolTip`, `Menu` (D-Bus menu path) properties +- **Activate:** call `Activate(x, y)` on the `StatusNotifierItem` D-Bus interface +- **Context menu:** call `ContextMenu(x, y)`, or read the `Menu` property to get the `com.canonical.dbusmenu` path and traverse the menu tree +- **Edge case:** GNOME does not natively support SNI (requires the AppIndicator extension); detect and report via an error suggestion if no tray is available + +### 3.15 — Hardening & Integration Review + +**Goal:** Prove the assembled `feat/linux-adapter` branch is production-grade as a whole, then merge it — closing out the three-platform story. + +**Scope:** +- Full-branch multi-agent review +- Live e2e in both headless and headed modes on the GNOME runner +- Performance baseline vs `main` (`scripts/perf-baseline-compare.sh` run on Linux) +- LOC/size/isolation audits +- Docs/skills sync +- Merge `feat/linux-adapter` → `main` as one release-noted `feat!` + +**Key APIs:** none — verification and merge only + +**Depends on:** 3.0 through 3.14 (3.14 may lag as a follow-up per its own note) + +**Exit criteria:** every item in the Cross-cutting sub-phase DoD holds for the whole branch; `main` gains Linux support in one commit, completing the three-platform matrix. + +**Est. PR size:** small diff, large verification effort ### Display Server Detection Runtime detection required for input, clipboard, and screenshot since Linux runs either X11 or Wayland: -- Check `$WAYLAND_DISPLAY` environment variable — if set, use Wayland path -- Check `$DISPLAY` environment variable — if set and no Wayland, use X11 path -- If neither, return `PLATFORM_UNSUPPORTED` with guidance to check display server configuration +- Check `$WAYLAND_DISPLAY` environment variable — if set, use the Wayland path +- Check `$DISPLAY` environment variable — if set and no Wayland, use the X11 path +- If neither, return `PLATFORM_NOT_SUPPORTED` with guidance to check the display server configuration - Input tools: verify `xdotool` (X11) or `ydotool` (Wayland) is installed; error with install instructions if missing - Clipboard tools: verify `xclip` (X11) or `wl-clipboard` (Wayland) is installed; error with install instructions if missing -### Web/Electron App Compatibility - -Same Chromium/Electron compatibility patterns as Phase 2 (Windows), adapted for AT-SPI2. These patterns ensure default `--max-depth 10` works with Electron apps like Slack, VS Code, and Chrome. - -**Web-aware tree traversal (depth-skip):** -- Non-semantic wrapper elements with AT-SPI roles `ROLE_PANEL`, `ROLE_SECTION`, or `ROLE_FILLER` that have empty `Name` AND empty `Value` do NOT consume depth budget during tree traversal -- This is the AT-SPI equivalent of macOS `AXGroup`/`AXGenericElement` and Windows `UIA_GroupControlTypeId` skipping -- Implement in `crates/linux/src/tree/builder.rs` with the same `is_web_wrapper` logic - -**Resolver depth:** -- Element re-identification must search up to `ABSOLUTE_MAX_DEPTH` (50), not a lower hardcoded limit -- Electron elements commonly sit at depth 25+ in the raw AT-SPI tree -- Implement in `crates/linux/src/tree/resolve.rs` matching the macOS/Windows pattern - -**Surface detection for Electron:** -- When an Electron app opens a modal (file picker, dialog), AT-SPI may report the dialog as the active window itself rather than a child of the parent window -- Surface detection must check if the focused window IS the target surface, not only search its children -- Check both `Role` and `RelationSet` / `RELATION_EMBEDS` for dialog detection (analogous to macOS AXRole + AXSubrole) -- Implement in `crates/linux/src/tree/surfaces.rs` - -**Chromium detection:** -- Detect Chromium-based apps via process name matching (electron, chrome, chromium, code) -- If AT-SPI tree is empty for a Chromium app, warn about `--force-renderer-accessibility` -- On Linux, Chromium respects `ACCESSIBILITY_ENABLED=1` environment variable as an alternative - -**Progressive skeleton traversal** works identically on Linux — `--skeleton` and `--root` flags are platform-agnostic, handled entirely by core. The Linux adapter only needs to implement `get_subtree()` (which delegates to the same async tree walker). Token savings for Electron apps apply equally. - -### AT-SPI2 Bus Detection - -- Check for `org.a11y.Bus` presence on the D-Bus session bus -- If bus is not running, return `PLATFORM_UNSUPPORTED` with instructions: - - GNOME: "AT-SPI2 should be enabled by default. Check `gsettings get org.gnome.desktop.interface toolkit-accessibility`" - - Other DEs: "Install `at-spi2-core` and ensure `at-spi-bus-launcher` is running" - - Flatpak/Snap: "Ensure the app has `--talk-name=org.a11y.Bus` permission" - ### Minimum OS Requirements - Ubuntu 22.04+ / Fedora 38+ @@ -1218,15 +1788,15 @@ Same Chromium/Electron compatibility patterns as Phase 2 (Windows), adapted for - `at-spi2-core` package installed (default on GNOME) - X11: `xdotool` installed. Wayland: `ydotool` installed -### Key Risks and Mitigations +### Key Risks and Mitigations (Linux-specific — folds into the Risk Register) | Risk | Mitigation | |------|------------| | Wayland a11y gaps | Focus on GNOME (best AT-SPI2 support). Prefer AT-SPI actions over coordinate input. Document known gaps clearly in skill and README. | | AT-SPI2 bus not running | Detect on first command. Return clear enable instructions specific to the detected distro/DE. | | Display server fragmentation | Runtime detection (X11 vs Wayland). Separate code paths for input/clipboard/screenshot. Test both. | -| Rust a11y crate maintenance stalls | Pin `atspi` and `zbus` versions. `atspi` crate backed by Odilia accessibility project. Maintain patches if upstream stalls. | -| Input tool availability | Check for xdotool/ydotool on first use. Provide package manager install commands in error suggestion. | +| Rust a11y crate maintenance stalls | Pin `atspi` and `zbus` versions. `atspi` crate backed by the Odilia accessibility project. Maintain patches if upstream stalls. | +| Input tool availability | Check for xdotool/ydotool on first use. Provide package manager install commands in the error suggestion. | ### New Dependencies @@ -1236,41 +1806,17 @@ Same Chromium/Electron compatibility patterns as Phase 2 (Windows), adapted for | `zbus` | 5.x | D-Bus connection | MIT/Apache-2.0 | | `tokio` | 1.x | Async runtime (required by atspi/zbus for async D-Bus) | MIT | -Added to `Cargo.toml` as target-gated dependency: +All three pins above were recorded at research time (2026-04); re-verify against crates.io and the repository's supply-chain policy at sub-phase 3.1 before adding them to `Cargo.toml` — same policy as the Windows dependency pins. + +Added to `Cargo.toml` as a target-gated dependency: ```toml [target.'cfg(target_os = "linux")'.dependencies] agent-desktop-linux = { path = "crates/linux" } ``` -Note: `tokio` is introduced here for the first time. Phases 1-2 are fully synchronous. The Linux adapter requires async D-Bus calls via zbus. +`tokio` is introduced here for the first time. The workspace is synchronous through Phase 2 — Windows adds no async runtime, since UIA's COM APIs are synchronous. Linux is the first platform requiring async D-Bus calls via `zbus`/`atspi`. -### Testing - -**Unit tests (linux):** -- AT-SPI Role → role mapping coverage for all role types -- Bus availability detection (mock D-Bus responses) -- Display server detection logic (Wayland vs X11 env vars) -- Element resolution round-trip (pid, role, name, bounds_hash) - -**Integration tests (Ubuntu CI):** -- Snapshot GNOME Files — non-empty tree with refs, buttons, text fields -- Snapshot GNOME Terminal — text area, menu items -- Snapshot GNOME Settings — modern GTK4 controls -- Click button in test app — verify action succeeded -- Type text into GNOME Text Editor via ref — verify content changed -- Clipboard get/set/clear roundtrip (test both X11 and Wayland if CI supports) -- Wait for window title pattern -- Launch + close app lifecycle -- Resize, move, minimize, maximize, restore window operations -- Screenshot produces valid PNG -- AT-SPI2 bus not running — correct error code and guidance -- Electron app snapshot (VS Code) — default depth finds 50+ refs via web-aware depth-skip -- Electron surface detection — file picker dialog detected as sheet surface -- List notifications — returns non-empty list when notifications exist (GNOME) -- Dismiss notification — verify notification dismissed via D-Bus `CloseNotification` -- List tray items — returns known SNI items (if running under KDE or with AppIndicator extension) -- Click tray item — verify tray menu opens via `Activate` D-Bus method -- Notification daemon detection — correct `GetServerInformation` result +### Testing (cross-platform validation, beyond each sub-phase's own exit criteria) **Cross-platform validation:** - Same snapshot of a cross-platform app (e.g., VS Code) produces structurally identical JSON on all 3 platforms @@ -1282,54 +1828,29 @@ Note: `tokio` is introduced here for the first time. Phases 1-2 are fully synchr - AT-SPI `accessible-id` populated for every interactive node in GNOME Calculator, GNOME Files, Firefox (with `ACCESSIBILITY_ENABLED=1`) - `watch --event value-changed` via `zbus` signal subscription delivers an event within 500 ms for a programmatic value change in a test harness app (GTK4 + pygobject) - `text select-range` / `get-selection` / `insert-at-caret` round-trips correctly in GNOME Text Editor via `org.a11y.atspi.Text` + `EditableText` -- PipeWire portal screenshot flow: user approves via XDG portal dialog, subsequent calls bypass the dialog within the session grant window; screenshot matches reference +- PipeWire portal screenshot flow: the user approves via the XDG portal dialog, subsequent calls bypass the dialog within the session grant window; screenshot matches reference - Toolbar surface: Firefox toolbar + GNOME Files toolbar both enumerate via `Role::ToolBar` -- Flatpak compatibility: a Flatpak-packaged GNOME Text Editor snapshot is non-empty when `--talk-name=org.a11y.Bus` is granted; returns clear diagnostic otherwise +- Flatpak compatibility: a Flatpak-packaged GNOME Text Editor snapshot is non-empty when `--talk-name=org.a11y.Bus` is granted; returns a clear diagnostic otherwise -### CI +Integration-level tests (Files/Terminal/Settings snapshots, click/type/clipboard/wait/lifecycle round-trips, bus-not-running error path, notification/tray list-and-act) are covered as exit criteria on their owning sub-phase above (3.4, 3.5, 3.7, 3.9, 3.10, 3.11, 3.14) rather than repeated here. -- GitHub Actions matrix: macOS + Windows + Ubuntu (all three on every PR) -- All runners execute: `cargo clippy --all-targets -- -D warnings`, `cargo test --workspace` -- `cargo tree -p agent-desktop-core` continues to contain zero platform crate names -- Binary size check: all platform binaries must be under 15MB - -### Release +### Release, Skill & Docs (folds into 3.13 / 3.15) +**Release:** - [ ] Prebuilt Linux CLI binary added to `.github/workflows/release.yml` matrix for `x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu` (Phase 1.5 already builds the FFI cdylib for both triples on the same runners — Phase 3 reuses those runners) - [ ] npm `postinstall.js` gains `linux-x64` / `linux-arm64` branches - [ ] Every new `ad_*` Linux implementation from P3-O9 / O10 / O11 is covered by the existing FFI drift check + Sigstore attestation pipeline - [ ] GitHub Release notes document Linux support, minimum glibc (2.35, Ubuntu 22.04 baseline), display-server requirements, and Flatpak/Snap compatibility -### Skill Update +**Skill Update:** +- [ ] Create `skills/agent-desktop-linux/SKILL.md`: AT-SPI2/D-Bus setup and bus detection; Wayland vs X11 differences (input via xdotool/ydotool, clipboard via wl-clipboard/xclip, screenshot via PipeWire/XGetImage); required system tools (`xdotool` or `ydotool`, `xclip` or `wl-clipboard`); Linux error codes and `platform_detail` examples (D-Bus errors, bus not found); troubleshooting guide (bus not running, empty trees, missing tools, Flatpak/Snap permissions) +- [ ] Update core `SKILL.md`: add Linux platform skill to the skill graph table; update platform support section to show all 3 platforms +- [ ] Update `workflows.md`: add cross-platform patterns noting Linux-specific differences; add Linux-specific workflow examples (e.g., GNOME app automation); document display server detection behavior -Skill docs are part of the release surface and must stay in sync with command behavior. - -- [ ] Create `skills/agent-desktop-linux/SKILL.md`: - - AT-SPI2/D-Bus setup and bus detection - - Wayland vs X11 differences (input via xdotool/ydotool, clipboard via wl-clipboard/xclip, screenshot via PipeWire/XGetImage) - - Required system tools: `xdotool` or `ydotool`, `xclip` or `wl-clipboard` - - Linux error codes and `platform_detail` examples (D-Bus errors, bus not found) - - Troubleshooting guide (bus not running, empty trees, missing tools, Flatpak/Snap permissions) -- [ ] Update core `SKILL.md`: - - Add Linux platform skill to skill graph table - - Update platform support section to show all 3 platforms -- [ ] Update `workflows.md`: - - Add cross-platform patterns noting Linux-specific differences - - Add Linux-specific workflow examples (e.g., GNOME app automation) - - Document display server detection behavior - -### README Update - -- [ ] Update Platform Support table: Linux column → **Yes** -- [ ] Add Linux installation instructions: - - npm (same command, auto-detects platform) - - Direct binary download from GitHub Releases - - From source: `cargo build --release` on Linux (note: requires `pkg-config`, `libdbus-1-dev`) -- [ ] Add Linux permissions section: - - AT-SPI2 bus must be running (default on GNOME, may need enabling on other DEs) - - Required tools: `xdotool` (X11) or `ydotool` (Wayland) for input synthesis - - Required tools: `xclip` (X11) or `wl-clipboard` (Wayland) for clipboard - - How to check: `busctl --user list | grep a11y` +**README Update:** +- [ ] Platform Support table: Linux column → **Yes** +- [ ] Linux installation instructions: npm (same command, auto-detects platform); direct binary download from GitHub Releases; from source: `cargo build --release` on Linux (requires `pkg-config`, `libdbus-1-dev`) +- [ ] Linux permissions section: AT-SPI2 bus must be running (default on GNOME, may need enabling on other DEs); required tools `xdotool` (X11) or `ydotool` (Wayland) for input synthesis; required tools `xclip` (X11) or `wl-clipboard` (Wayland) for clipboard; how to check: `busctl --user list | grep a11y` - [ ] Update minimum OS versions: Ubuntu 22.04+ / Fedora 38+ - [ ] Update "From source" section with Linux build requirements @@ -1341,19 +1862,19 @@ Skill docs are part of the release surface and must stay in sync with command be Phase 4 adds a new I/O layer. Core engine and all three platform adapters are unchanged. The MCP server wraps existing command logic in JSON-RPC tool definitions, enabling agent-desktop to work as an MCP-native desktop automation server for Claude Desktop, Cursor, VS Code Copilot, Gemini CLI, Microsoft Agent Framework 1.0, and any other MCP-compatible host. -By Phase 4 the CLI already covers the shared command surface on three platforms, the FFI ships as a shared library for in-process consumers, and the cross-platform event / text-range / stable-selector primitives from Phase 2 / 3 are in place. MCP mode is a **transport + discovery layer**, nothing more. Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant at the top of this document, the MCP crate contains zero per-tool and zero per-platform code — it walks the same deterministic command descriptor registry the CLI and FFI use, and dispatches to the same `execute(args, adapter)` functions. New commands added in Phase 2 or Phase 5 (e.g. `watch_element`, `text select-range`, `find --visual`) become MCP tools automatically with no changes to `crates/mcp/`. +By Phase 4 the CLI already covers the shared command surface on three platforms, the FFI ships as a shared library for in-process consumers, and the cross-platform event / text-range / stable-selector primitives from Phase 2 / 3 are in place. MCP mode is a **transport + discovery layer**, nothing more. Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant at the top of this document, the MCP crate contains zero per-tool and zero per-platform code — it walks the same deterministic command descriptor registry the CLI and FFI use, and dispatches to the same `execute(args, adapter)` functions. New commands added in Phase 2 or Phase 5 (e.g. `watch`, `text select-range`, `find --visual`) become MCP tools automatically with no changes to `crates/mcp/`. ### Objectives | ID | Objective | Metric | |----|-----------|--------| | P4-O1 | MCP server mode via `--mcp` | Responds to MCP `initialize` handshake, reports capabilities, per-host hello-world passes | -| P4-O2 | All commands as MCP tools | `tools/list` returns 54+ tools with JSON Schemas generated from the CLI arg structs via `schemars`; tool names prefixed `desktop_` | +| P4-O2 | All commands as MCP tools | `tools/list` returns all 58 shipped command names as tools (54 operational + 4 fail-closed, matching the CLI surface 1:1) with JSON Schemas generated from the CLI arg structs via `schemars`; tool names prefixed `desktop_` | | P4-O3 | Claude Desktop + Cursor + VS Code + Gemini CLI + MS Agent Framework validated | Each host invokes tools to control a desktop app end-to-end on all three platforms; repo ships `mcp.json` / `claude_desktop_config.json` / `.cursor/mcp.json` examples per host | | P4-O4 | Tool annotations | `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint` on every tool; Claude Desktop surfaces destructive tools with a confirmation prompt | -| P4-O5 | Ref-based MCP tool shape (Playwright-MCP idiom) | Tools take `{ref: "e5"}` not raw `element_handle`, matching Playwright MCP so agents can swap between the two without relearning selectors. Tree snapshots return as MCP resources with refs inline | +| P4-O5 | Ref-based MCP tool shape (Playwright-MCP idiom) | Tools take `{ref: "@s8f3k2p9:e5"}` (the qualified snapshot-ref form) not raw `element_handle`, matching Playwright MCP's ref idiom so agents can swap between the two without relearning selectors. Tree snapshots return as MCP resources with refs inline | | P4-O6 | MCP resource types | `agent-desktop://refmap/current`, `agent-desktop://snapshot/latest`, `agent-desktop://audit/{trace_id}` (audit log under Phase 5). `resources/list` + `resources/read` expose the current RefMap and last snapshot without re-running the command | -| P4-O7 | Tree-diff notifications | `watch_element` events (Phase 2 P2-O11) stream as MCP `notifications/message` during a long-running wait, so the host sees value-changed / focus-changed events as they happen rather than polling | +| P4-O7 | Tree-diff notifications | `watch` events (Phase 2 P2-O11 push subscription — distinct from the already-shipped `wait --event` baseline diff) stream as MCP `notifications/message` during a long-running wait, so the host sees value-changed / focus-changed events as they happen rather than polling | | P4-O8 | Progress notifications | `notifications/progress` for `wait`, `snapshot --skeleton` → `--root` drill-down chains, and large-tree traversals. Agents surface progress to users instead of hanging | | P4-O9 | Tool-level permission tiers | Observation tools (`desktop_snapshot`, `desktop_find`, `desktop_get`, `desktop_is`, `desktop_list_*`) are freely callable. Interaction tools (`desktop_click`, `desktop_type_text`, `desktop_set_value`, `desktop_drag`) are gated behind an `interactive` capability negotiated at `initialize`. Destructive tools (`desktop_close_app`, `desktop_dismiss_all_notifications`) require the `destructive` capability plus the Phase 5 audit log | | P4-O10 | Session-scoped RefMap | Each MCP session has its own in-memory RefMap keyed by `session_id` — no conflict with the on-disk CLI RefMap, no cross-session leakage when a host runs multiple agent-desktop-mcp instances | @@ -1380,10 +1901,10 @@ crates/mcp/src/ ├── capability.rs # P4-O9 tier gating (observation / interactive / destructive) ├── resources.rs # P4-O6 resource types (refmap / snapshot / permissions / events / audit) ├── notifications.rs # P4-O7 watch event forwarder, P4-O8 progress forwarder -└── schema.rs # Translates CommandDescriptor → rmcp tool definition +└── schema.rs # Translates CommandDescriptor → rmcp tool definition ``` -That's the whole crate. It doesn't know what `desktop_click` does — it reads generated command descriptors and forwards invocations through the same command execution function the CLI uses. Adding a command in Phase 2 (`text select-range`, `watch_element`) or Phase 5 (`find --visual`, `audit tail`) should mean **zero lines of MCP-specific behavior** — only shared command metadata and adapter methods change. +That's the whole crate. It doesn't know what `desktop_click` does — it reads generated command descriptors and forwards invocations through the same command execution function the CLI uses. Adding a command in Phase 2 (`text select-range`, `watch`) or Phase 5 (`find --visual`, `audit tail`) should mean **zero lines of MCP-specific behavior** — only shared command metadata and adapter methods change. ### MCP tool registration — the one-time rewrite @@ -1435,32 +1956,36 @@ Observation tools (always available): | `desktop_is` | `is <state> <ref>` | Boolean | | `desktop_list_windows` | `list-windows` | Array of windows | | `desktop_list_apps` | `list-apps` | Array of apps | +| `desktop_list_displays` | `list-displays` | Array of displays with `scale_factor` | | `desktop_list_surfaces` | `list-surfaces` | Array of surfaces (incl. Toolbar / Spotlight / Dock / MenuBarExtras and Windows shell surfaces from P2-O14/P2-O18) | | `desktop_list_notifications` | `list-notifications` | Array of notifications | | `desktop_screenshot` | `screenshot` | Base64 PNG (or MCP resource link) | -| `desktop_clipboard_get` | `clipboard-get` | Clipboard text | +| `desktop_clipboard_get` | `clipboard-get` | Typed clipboard content | | `desktop_permissions` | `permissions` | Tri-state permission report (AX + Screen Recording + Automation) | | `desktop_status` | `status` | Daemon + adapter status | | `desktop_version` | `version` | Version + ABI version | +| `desktop_session` | `session start / end / list / gc` | Session lifecycle (manifest, trace segments) | +| `desktop_trace` | `trace export / show` | Reliability trace export / inspection | Interaction tools (gated by `interactive` capability): | MCP Tool | CLI | Shape | |----------|-----|-------| -| `desktop_click` / `desktop_double_click` / `desktop_triple_click` / `desktop_right_click` | `click @e5` (and variants) | `{ref: "e5"}` | -| `desktop_type_text` | `type @e5 "hello"` | `{ref: "e5", text: "hello"}` | -| `desktop_set_value` | `set-value @e5 "hello"` | `{ref: "e5", value: "hello"}` | -| `desktop_clear` | `clear @e5` | `{ref: "e5"}` | -| `desktop_focus` | `focus @e5` | `{ref: "e5"}` | -| `desktop_select` / `desktop_toggle` / `desktop_check` / `desktop_uncheck` / `desktop_expand` / `desktop_collapse` | — | `{ref: "e5"}` (+ `value` for select) | -| `desktop_scroll` / `desktop_scroll_to` | `scroll <dir>` | `{ref: "e5", direction, amount}` | -| `desktop_press_key` / `desktop_key_down` / `desktop_key_up` | `press <keys>` | `{key, modifiers}` | -| `desktop_hover` / `desktop_drag` | `hover`/`drag` | `{ref: "e5"}` or `{from, to}` | -| `desktop_mouse_move` / `desktop_mouse_click` / `desktop_mouse_down` / `desktop_mouse_up` | — | `{x, y, button}` | -| `desktop_wait` | `wait --element / --window / --text / --menu / --notification` | `{condition, timeout_ms}` | -| `desktop_watch_element` (P2-O11) | `watch --event …` | `{ref: "e5", events: [EventKind], timeout_ms}` — streams via `notifications/message` | +| `desktop_click` / `desktop_double_click` / `desktop_triple_click` / `desktop_right_click` | `click @s8f3k2p9:e5` (and variants) | `{ref: "@s8f3k2p9:e5"}` | +| `desktop_type_text` | `type @s8f3k2p9:e5 "hello"` | `{ref: "@s8f3k2p9:e5", text: "hello"}` | +| `desktop_set_value` | `set-value @s8f3k2p9:e5 "hello"` | `{ref: "@s8f3k2p9:e5", value: "hello"}` | +| `desktop_clear` | `clear @s8f3k2p9:e5` | `{ref: "@s8f3k2p9:e5"}` | +| `desktop_focus` | `focus @s8f3k2p9:e5` | `{ref: "@s8f3k2p9:e5"}` | +| `desktop_select` / `desktop_toggle` / `desktop_check` / `desktop_uncheck` / `desktop_expand` / `desktop_collapse` | — | `{ref: "@s8f3k2p9:e5"}` (+ `value` for select) | +| `desktop_scroll` / `desktop_scroll_to` | `scroll <dir>` | `{ref: "@s8f3k2p9:e5", direction, amount}` | +| `desktop_press_key` | `press <keys>` | `{key, modifiers}` | +| `desktop_hover` / `desktop_drag` | `hover`/`drag` | `{ref: "@s8f3k2p9:e5"}` or `{from, to}` | +| `desktop_mouse_move` / `desktop_mouse_click` / `desktop_mouse_wheel` | — | `{x, y, button}` | +| `desktop_key_down` / `desktop_key_up` / `desktop_mouse_down` / `desktop_mouse_up` | — | `{key, modifiers}` / `{x, y, button}` — listed for CLI parity but fail closed identically to their CLI counterparts (see Phase 1) until the Phase 5 daemon owns held input | +| `desktop_wait` | `wait --element / --window / --text / --menu / --notification / --event` | `{condition, timeout_ms}` | +| `desktop_watch` (P2-O11) | `watch --event …` | `{ref: "@s8f3k2p9:e5", events: [EventKind], timeout_ms}` — streams via `notifications/message`; distinct from `desktop_wait`'s `--event` mode, which is a single baseline-diff read | | `desktop_launch_app` / `desktop_focus_window` / `desktop_resize_window` / `desktop_move_window` / `desktop_minimize` / `desktop_maximize` / `desktop_restore` | app / window ops | App / window args | -| `desktop_clipboard_set` / `desktop_clipboard_clear` | — | `{text}` / `{}` | +| `desktop_clipboard_set` / `desktop_clipboard_clear` | — | typed clipboard content / `{}` | | `desktop_notification_action` | `notification-action <idx> <action>` | `{index, expected_app?, expected_title?, action}` (NC-reorder safe) | | `desktop_text_select_range` / `desktop_text_get_selection` / `desktop_text_insert_at_caret` / `desktop_text_at_offset` (P2-O12) | `text …` subcommands | `{ref, start, length, text?}` | @@ -1482,7 +2007,7 @@ Resources let hosts pull structured state without re-issuing a tool call: | `agent-desktop://refmap/current` | JSON RefMap for the current MCP session (not the on-disk CLI refmap) | Replaced on every `desktop_snapshot` invocation; subscribable via `notifications/resources/updated` | | `agent-desktop://snapshot/latest` | Last `desktop_snapshot` response as JSON (tree + refmap + metadata) | Same update model | | `agent-desktop://permissions/current` | Tri-state permission report (AX, Screen Recording, Automation, display-server) | Refreshed on request; subscribable when Phase 2 P2-O17 permission observer is available | -| `agent-desktop://events/stream` | Merged `watch_element` event stream for the session | Real-time, subscribable | +| `agent-desktop://events/stream` | Merged `watch` event stream for the session | Real-time, subscribable | | `agent-desktop://audit/{trace_id}` | Phase 5 append-only audit log entries for a trace | Growable; new entries as `notifications/resources/updated` | ### Framework Integration Targets @@ -1506,7 +2031,7 @@ Each host gets a ~30-line config + a 60-second "hello agent" demo (launch Calcul - **Stdio (primary):** MCP host spawns `agent-desktop --mcp` as a child process. JSON-RPC over stdin/stdout. Required; validated against all hosts in the Framework Integration table. - **Streamable HTTP (P4-O12, required for MS Agent Framework):** Single HTTP endpoint at `POST /mcp` with chunked response streaming; replaces the pre-March-2025 SSE transport. Used when the host declares `transport: http` in its MCP config. Binds to `127.0.0.1` by default; `--mcp-bind <addr:port>` CLI flag overrides. - **SSE (legacy):** Retained for hosts that haven't migrated to Streamable HTTP. Gated on `--mcp-transport sse`. -- **Session:** On `initialize`, detect platform, probe permissions (AX + Screen Recording + Automation tri-state), report tool capabilities given current permissions. The current CLI already supports `--session <id>` as an on-disk latest-snapshot namespace. MCP adds per-host in-memory session state keyed by `session_id`; it must not use the legacy `~/.agent-desktop/last_refmap.json` artifact and should bridge to the same explicit snapshot semantics as the CLI. +- **Session:** On `initialize`, detect platform, probe permissions (AX + Screen Recording + Automation tri-state), report tool capabilities given current permissions. The current CLI already supports `--session <id>` as an on-disk latest-snapshot namespace and a manifest-gated session with automatic trace segments (Phase 1.6). MCP adds per-host in-memory session state keyed by `session_id`; it must not use the legacy `~/.agent-desktop/last_refmap.json` artifact and should bridge to the same explicit snapshot semantics as the CLI. ### Initialize Handler @@ -1524,7 +2049,7 @@ On receiving MCP `initialize`: | `schemars` | 1.2+ | JSON Schema generation for tool parameter definitions | MIT/Apache-2.0 | | `tokio` | 1.x | Async runtime (required by rmcp for MCP server event loop) | MIT | -Note: If `tokio` was already introduced in Phase 3 (Linux), it is already available. Otherwise, it is introduced here. +`tokio` enters the workspace no later than Phase 3 (Linux, sub-phase 3.1); by Phase 4 it is already available. ### Binary Crate Changes @@ -1565,7 +2090,7 @@ Note: If `tokio` was already introduced in Phase 3 (Linux), it is already availa - `initialize` response's `supported_tools` list shrinks correctly when AX permission is denied (only `desktop_permissions`, `desktop_version`, `desktop_status` remain) **Event streaming tests (P4-O7):** -- `desktop_watch_element` subscription receives `notifications/message` events for a programmatic value change within 500 ms of the change on all three platforms +- `desktop_watch` subscription receives `notifications/message` events for a programmatic value change within 500 ms of the change on all three platforms - Two concurrent watches on different refs get their events routed to the correct subscription ID ### MCP Config Examples @@ -1662,7 +2187,7 @@ The daemon is a long-running process that maintains state between CLI/MCP invoca - Any new connection resets the idle timer **Session multiplexing:** -- The current CLI `--session <id>` persists snapshots on disk and scopes only the latest-snapshot pointer. +- The current CLI `--session <id>` persists snapshots on disk, scopes the latest-snapshot pointer, and (Phase 1.6) gates a manifest with automatic JSONL trace segments. - The daemon upgrades that model to warm, in-memory per-session RefMaps while preserving explicit snapshot IDs as deterministic handles. - Sessions are isolated: agent A's latest pointer never collides with agent B's latest pointer. - Session destroyed on disconnect or explicit `session kill`. @@ -1672,6 +2197,8 @@ The daemon is a long-running process that maintains state between CLI/MCP invoca ### New Commands +> `session start` / `session end` / `session list` / `session gc` and `trace export` / `trace show` already ship as of Phase 1.6 (see Phase 1's Commands Shipped table). The rows below describe the **remaining** daemon-era additions layered on top of those — `session kill` (daemon-specific teardown, distinct from the already-shipped `session end`), a friendlier `trace view` pretty-printer, and OTLP/HAR export flags — not a wholesale reimplementation of session or trace management. + | Command | Description | |---------|-------------| | `session list` | List active daemon sessions with IDs, creation time, last activity | @@ -1715,7 +2242,7 @@ Every destructive operation — `close-app`, `dismiss-all-notifications`, `set-v Defaults: CLI = off (opt-in), MCP `destructive` capability = on (opt-out via `skipConfirm: true` at init). 3. **Append-only audit log** at `~/.agent-desktop/audit.jsonl`: ```json - {"ts":"2026-05-…","trace_id":"9f3c…","actor":"cli|mcp:claude-desktop","tool":"close-app","args":{"app":"Finder"},"policy_decision":"allowed","user_decision":"confirmed","exit":0,"prev_hash":"sha256:…","entry_hash":"sha256:…"} + {"ts":"2026-…","trace_id":"9f3c…","actor":"cli|mcp:claude-desktop","tool":"close-app","args":{"app":"Finder"},"policy_decision":"allowed","user_decision":"confirmed","exit":0,"prev_hash":"sha256:…","entry_hash":"sha256:…"} ``` Hash-chained (Merkle-style) so `agent-desktop audit verify` detects tampering. File mode `0o600`, directory `0o700`. Rotated at 100 MB via `audit.jsonl.{N}.gz`. @@ -1761,7 +2288,7 @@ Matchers: `tool` (glob), `bundle` (exact or glob), `pid`, `trace_mcp_host` (`cli ### Trace Export + OpenTelemetry (P5-O8) -Today, callers opt into a redacted reliability trace with `--trace <path>` and may add `--trace-strict` to fail on setup or pre-action trace write errors. Phase 5 layers trace IDs and span/export tooling on top of that existing JSONL event stream: +Today, callers opt into a redacted reliability trace with `--trace <path>` and may add `--trace-strict` to fail on setup or pre-action trace write errors; Phase 1.6 also made per-session tracing automatic under a manifest-gated `session start`. Phase 5 layers trace IDs and span/export tooling on top of that existing JSONL event stream: ```json {"ts":"…","trace_id":"9f3c…","span_id":"…","parent_span_id":"…","name":"cli.snapshot","kind":"internal","attributes":{"app":"Finder","skeleton":true,"ref_count":14,"duration_ms":87}} @@ -1778,11 +2305,11 @@ Phase 5 spans are OpenTelemetry-compliant so `agent-desktop trace export <uuid> | Safety | Every destructive command supports `--dry-run`; every MCP destructive tool requires the `destructive` capability + audit log; the audit log is hash-chained and tamper-detectable; policy engine evaluated on every invocation. | | Performance | Cold start <200ms. Warm snapshot <50ms via daemon. Tree traversal timeout 5s default, configurable. `watch --event` latency <500ms (push, not poll) per P2-O11. | | Reliability | Zero panics in non-test code. Graceful daemon recovery on crash. Stale socket cleanup on startup. FFI panic boundary in release-ffi profile (already shipping). | -| Observability | Current commands can opt into redacted JSONL via `--trace`. Phase 5 adds daemon metrics, trace IDs, and OpenTelemetry OTLP export via `trace export --otlp`. | +| Observability | Current commands can opt into redacted JSONL via `--trace`, with automatic per-session segments under Phase 1.6. Phase 5 adds daemon metrics, trace IDs, and OpenTelemetry OTLP export via `trace export --otlp`. | | Compatibility | Tested against target app matrix: Finder, TextEdit, Xcode, VS Code, Chrome, Slack (macOS); Explorer, Notepad, Settings, VS Code, Edge (Windows); Nautilus, Terminal, Firefox, VS Code (Linux). | | Distribution | Single binary per platform. No runtime dependencies for the CLI. FFI cdylib tarballs signed via Sigstore (already shipping as of Phase 1.5). Formula / manifest verify Sigstore attestation before installing (P5-O10). | | Documentation | README, CLI reference, MCP reference, per-platform setup guides, troubleshooting, audit-log format reference, policy-file reference, OpenTelemetry trace schema. | -| FFI stability | Header drift check green on every PR. ABI version exported via `ad_abi_version()`. Pre-1.0: minor version bump for any public struct field add; major version bump for any removed or changed signature. | +| FFI stability | Header drift check green on every PR. ABI version exported via `ad_abi_version()` (major currently 3, append-only since Phase 1.6). Pre-1.0: minor version bump for any public struct field add; major version bump for any removed or changed signature. | ### Performance Optimizations @@ -1912,8 +2439,9 @@ The README is updated at the end of each phase to reflect the current state: | Phase | README Changes | |-------|---------------| -| Phase 1 | Initial README: npm + source installation, core workflow, all 54 commands, JSON output, ref system, error codes, platform support table (macOS only) | +| Phase 1 | Initial README: npm + source installation, core workflow, all 58 shipped command names (54 operational + 4 fail-closed), JSON output, ref system, error codes, platform support table (macOS only) | | Phase 1.5 | Add "Language bindings (FFI)" section: platform→artifact table, 5-line Python dlopen snippet, `shasum -a 256 -c checksums.txt` + `gh attestation verify` verification, link to `skills/agent-desktop-ffi/` | +| Phase 1.6 | Note the breaking default-on auto-wait behavior and the qualified `@<snapshot_id>:e<n>` ref form in the CLI reference; no new distribution surface | | Phase 2 | Add Windows: `.exe` installation, Windows permissions, update platform table, Windows build instructions | | Phase 3 | Add Linux: binary installation, AT-SPI2 setup, update platform table, Linux build instructions, minimum OS versions | | Phase 4 | Add MCP Server: `--mcp` usage, Claude Desktop config, Cursor config, tool-to-CLI mapping | @@ -1936,40 +2464,44 @@ See [Command Surface Architecture](#command-surface-architecture-dry-invariant) - A new command creates exactly **one** file under `crates/core/src/commands/`. - CLI and batch must share the typed `Commands` enum, `CommandPolicy`, and `dispatch()` path. - Any future registry/codegen must be deterministic `build.rs` filesystem enumeration, not `inventory` or `linkme`. -- Per-platform work is limited to the `PlatformAdapter` trait implementations in `crates/{macos,windows,linux}/` — never per-transport, never per-command. +- Per-platform work is limited to the `PlatformAdapter` capability-trait implementations in `crates/{macos,windows,linux}/` — never per-transport, never per-command. - PRs that add a command to a single transport without updating the shared registry fail review. If a task in this document sounds like it requires per-transport duplication, it's a wording bug — the actual implementation follows the registry pattern. ### CI Matrix Evolution -| Phase | CI Runners | +| Phase | CI Runners / Jobs | |-------|-----------| | Phase 1 | `macos-latest` (tests + CLI build) + `ubuntu-latest` (`fmt` job) | | Phase 1.5 | Same as Phase 1 on PRs; release workflow fans out to `macos-latest` × 2 darwin arches + `ubuntu-22.04` + `ubuntu-22.04-arm` + `windows-latest` for the FFI matrix | -| Phase 2 | macOS + Windows (CLI tests on Windows) | -| Phase 3 | macOS + Windows + Ubuntu | +| Phase 1.6 (current) | `ci.yml`: `fmt` (ubuntu-latest), `msrv` (ubuntu-latest, Rust 1.89.0), `platform-check` (matrix Linux/Windows/macOS, `cargo check` only — proves the stub crates compile before any adapter lands), `test` (macos-latest, full suite), `ffi-python-smoke`, `ffi-header-drift`, `ffi-panic-guard`, `ffi-passthrough` (ubuntu-latest). Outside `ci.yml`: `native-e2e.yml` (self-hosted macOS, workflow_dispatch), `codeql.yml`, `supply-chain.yml`. `scripts/perf-baseline-compare.sh` is currently a per-PR Definition-of-Done review step, not yet a blocking CI job | +| Phase 2 | `platform-check`'s Windows leg is promoted to a real `windows-latest` test lane (build, clippy, lib tests) at sub-phase 2.1; a self-hosted interactive Windows runner is added for UIA/shell integration tests at 2.12 | +| Phase 3 | `platform-check`'s Linux leg is promoted to a real `ubuntu-latest` test lane at sub-phase 3.1; an interactive Ubuntu GNOME runner is added for AT-SPI2/shell integration tests at 3.12 | | Phase 4 | macOS + Windows + Ubuntu (+ MCP protocol tests) | | Phase 5 | macOS + Windows + Ubuntu (+ daemon tests, package build verification) | -All runners enforce: `cargo clippy --all-targets -- -D warnings`, `cargo test --workspace`, `cargo tree -p agent-desktop-core` contains zero platform crate names, binary size <15MB. +All runners enforce: `cargo clippy --all-targets -- -D warnings`, `cargo test --workspace`, `cargo tree -p agent-desktop-core` contains zero platform crate names, binary size <15MB. Every Phase 2/3 sub-phase additionally runs `scripts/perf-baseline-compare.sh` on hot-path changes — see the [Cross-cutting sub-phase DoD](#cross-cutting-sub-phase-dod). ### Dependency Introduction Schedule | Dependency | Introduced In | Purpose | |------------|---------------|---------| -| `clap` 4.x, `serde` 1.x, `thiserror` 2.x, `tracing` 0.1+, `base64` 0.22+ | Phase 1 | Core: CLI, JSON, errors, logging, encoding | -| `tracing-subscriber` 0.3, `rustc-hash` 2.1 | Phase 1 | Log formatter + fast hashing | -| `accessibility-sys` 0.1+, `core-foundation` 0.10+, `core-graphics` 0.24+ | Phase 1 | macOS AX API FFI | -| `cbindgen` maintainer tool, `libc` 0.2+ | Phase 1.5 | explicit C header regeneration + macOS `pthread_main_np` for FFI main-thread guard | +| `clap` 4.6, `serde`/`serde_json` 1.x, `thiserror` 2.0, `tracing` 0.1+, `base64` 0.22+ | Phase 1 | Core: CLI, JSON, errors, logging, encoding | +| `tracing-subscriber` 0.3, `rustc-hash` 2.1, `smallvec` 1.13 | Phase 1 | Log formatter, fast hashing, small vectors in hot paths | +| `accessibility-sys` 0.2.0, `core-foundation` 0.10.1, `core-foundation-sys` 0.8.7, `core-graphics` 0.25.0 | Phase 1 | macOS AX API FFI | +| `cbindgen` maintainer tool, `libc` 0.2+ | Phase 1.5 | Explicit C header regeneration + macOS `pthread_main_np` for FFI main-thread guard | +| *(no new external crates — a contract-hardening pass over existing dependencies)* | Phase 1.6 | — | | `uiautomation` 0.24+ | Phase 2 | Windows UIA wrapper | | `windows` 0.62.2 | Phase 2 | Win32 / WinRT bindings (pinned to match `windows-capture 1.5` pin) | | `windows-capture` 1.5.4 | Phase 2 | Modern `Windows.Graphics.Capture` screenshot | | `objc2` 0.6 | Phase 2 | macOS safe Objective-C bridging (scoped to `system/screenshot.rs` + `system/permissions.rs`; CI grep guard) | -| `screencapturekit` 1.5 (crates.io) | Phase 2 | ScreenCaptureKit wrapper — published canonical crate, not git fork | +| `screencapturekit` 1.5 (crates.io) | Phase 2 | ScreenCaptureKit wrapper — published canonical crate, not a git fork | | `atspi` 0.28+ + `zbus` 5.x | Phase 3 | Linux AT-SPI2 client via D-Bus | -| `tokio` 1.x | Phase 3 | Async runtime (required by atspi/zbus) | +| `tokio` 1.x | Phase 3 | Async runtime (required by atspi/zbus) — the first async runtime in the workspace; the codebase is synchronous through Phase 2 | | `rmcp` 0.15.0+ | Phase 4 | Official MCP Rust SDK | | `schemars` 1.2 | Phase 4 | JSON Schema generation for MCP tool parameters (deferred from Phase 2 per plan §KD15 — no Phase 2 consumer) | +All Phase 2/3 pins above were recorded at 2026-04 research time; re-verify against crates.io and the repository's supply-chain policy at the opening sub-phase of the consuming platform, per the [Platform Delivery Model](#platform-delivery-model--sub-phases-and-integration-branches). + ### Explicitly NOT Added (research-rejected) | Crate | Rejected at | Reason | @@ -1984,11 +2516,12 @@ All runners enforce: `cargo clippy --all-targets -- -D warnings`, `cargo test -- |------------|-------|---------|-------| | Tree root | `AXUIElementCreateApp(pid)` | `IUIAutomation.ElementFromHandle()` | `atspi Accessible` on bus | | Children | `kAXChildrenAttribute` | `TreeWalker.GetFirstChild` | `GetChildren` D-Bus | +| Stable ID | `AXIdentifier` / `AXDOMIdentifier` (shipped, U5) | UIA `AutomationId` (Phase 2, sub-phase 2.3) | AT-SPI2 `accessible-id` (Phase 3, sub-phase 3.3) | | Click | `AXPress` | `InvokePattern.Invoke()` | `Action.DoAction(0)` | | Set text | `AXValue = val` | `ValuePattern.SetValue()` | `Text.InsertText` | | Keyboard | `CGEventCreateKeyboard` | `SendInput` | `xdotool` / `ydotool` | -| Clipboard | `NSPasteboard` | Win32 Clipboard API | `wl-clipboard` / `xclip` | -| Screenshot | `ScreenshotBackend` over secure `screencapture` path today; ScreenCaptureKit planned | `BitBlt` / `PrintWindow` legacy, Windows.Graphics.Capture planned | `PipeWire` / `XGetImage` | +| Clipboard | `NSPasteboard`, typed `ClipboardContent` (shipped, U18) | Win32 Clipboard API, typed (Phase 2) | `wl-clipboard` / `xclip`, typed (Phase 3) | +| Screenshot | `ScreenshotBackend` over secure `screencapture` path today; ScreenCaptureKit planned (P2-O13) | `BitBlt` / `PrintWindow` legacy, `Windows.Graphics.Capture` planned (P2-O13) | `PipeWire` / `XGetImage` (P3-O11) | | Permissions | `AXIsProcessTrusted()` | COM security / UAC | Bus availability | | Notifications | Notification Center AX tree (`com.apple.notificationcenterui`) | UIA tree of Action Center / Toast Manager | D-Bus `org.freedesktop.Notifications` + daemon-specific history | | System tray | `SystemUIServer` AX tree + `ControlCenter` AX tree | UIA tree of `Shell_TrayWnd` + overflow window | D-Bus `StatusNotifierWatcher` + XEmbed fallback | @@ -2003,10 +2536,13 @@ All runners enforce: `cargo clippy --all-targets -- -D warnings`, `cargo test -- | R2 | Electron/Chrome no a11y tree by default | High | Medium | Detect Chromium windows. Print `--force-renderer-accessibility` guidance in error response. | | R3 | Custom-rendered UIs invisible to a11y | Medium | High | Phase 5 stretch: vision fallback. Short-term: document limitation in README and skills. | | R4 | Wayland a11y gaps | Medium | Medium | Focus on GNOME (best AT-SPI2 support). Prefer AT-SPI actions over coordinate input. Document gaps. | -| R5 | Rust a11y crate maintenance stalls | Low | High | Pin versions, maintain patches. `atspi` backed by Odilia project. Fork-ready. | +| R5 | Rust a11y crate maintenance stalls | Low | High | Pin versions, maintain patches. `atspi` backed by the Odilia project. Fork-ready. | | R6 | MCP spec changes break compat | Low | Medium | Pin `rmcp` version. Monitor spec under Linux Foundation governance. | | R7 | Tree traversal too slow (>5s) | Medium | Medium | Depth limiting via `--max-depth`. Focused-window-only. Cached subtrees in Phase 5 daemon. Progressive skeleton traversal (`--skeleton` + `--root`) reduces token consumption 78-96% for dense apps. | -| R8 | Ref instability confuses agents | Medium | High | Clear docs: refs are snapshot-scoped. `STALE_REF` error with recovery hint. Stable hashing in Phase 5. Progressive skeleton traversal with scoped invalidation provides a stable drill-down workflow for navigating complex UIs. **Phase 2**: stable-selector fields (`identifier`, `subrole`, `role_description`, `placeholder`, `dom_id`, `dom_classes` via `StableSelectors` flatten) + identifier-preferred resolver drop `STALE_REF` rate on Electron / localized apps. | -| R9 | Headless operation requirement | High | Critical | Phase 1 introduced `ActionRequest`/`InteractionPolicy`, default no focus steal/cursor movement, and explicit physical/headed policy paths. Phase 2 must preserve the same contract for Windows/Linux. | -| R10 | Command registry link-GC | Medium | High | Research Topic B confirmed `inventory`/`linkme` are unreliable across linkers for cdylib consumers. Resolved by pure `build.rs` filesystem enumeration — zero linker magic. | +| R8 | Ref instability confuses agents | Medium | High | Clear docs: refs are snapshot-scoped and snapshot-qualified (`@<snapshot_id>:e<n>`). `STALE_REF` error with recovery hint. Progressive skeleton traversal with scoped invalidation provides a stable drill-down workflow. Stable `native_id` evidence (shipped on macOS; Windows/Linux land it in their own vocabulary sub-phase) reduces stale-resolution failures on Electron and localized apps. | +| R9 | Headless operation requirement | High | Critical | Phase 1 introduced `ActionRequest`/`InteractionPolicy`, default no focus steal/cursor movement, and explicit physical/headed policy paths; Phase 1.6 added default-on auto-wait and the occlusion gate on top. Phase 2/3 preserve the same contract for Windows/Linux. | +| R10 | Command registry link-GC | Medium | High | Research Topic B confirmed `inventory`/`linkme` are unreliable across linkers for cdylib consumers. Resolved by pure `build.rs` filesystem enumeration — zero linker magic, once that registry migration (P2-O16) lands. | | R11 | Skeleton traversal cross-platform | Low | High | Core is already platform-agnostic (`crates/core/src/snapshot_ref.rs`); Windows needs ~50 LOC glue (`ControlViewWalker` + `FindAll(TreeScope_Children, TrueCondition)` + fresh `UICacheRequest` per drill-down). Research Topic 4 confirmed `ElementFromHandle(hwnd)` is headless-safe. | +| R12 | RDP / session-isolation blocks Windows dev and CI | Medium | High | UIA requires an interactive session — an RDP disconnect can drop the console session to a non-interactive state. Document the `tscon` console-reattach workaround for the self-hosted runner (sub-phase 2.1); mirrors the macOS exclusive-desktop gate (`AGENT_DESKTOP_E2E_EXCLUSIVE=1` + `interaction_lock.py`) that already serializes native e2e runs. | +| R13 | UIA event handler MTA lifecycle leaks | Medium | Medium | `RemoveAutomationEventHandler` races the final in-flight callback dispatch on the MTA worker thread if torn down naively. Use the post-remove-barrier pattern (`Arc<Handler>` outlives the final callback) documented in the Windows Engineering Invariants — apply it from the first sub-phase that registers a handler (`watch`, once P2-O11 lands), not retrofitted after a leak is observed. | +| R14 | Merge-train discipline: integration branch drifts from `main` | Medium | Medium | 16 sub-phases landing serially into `feat/windows-adapter` (then `feat/linux-adapter`) is a long-lived branch by construction. Mitigate with a rebase cadence (rebase onto `main` at the start of each sub-phase, not just before the final merge) and treat each sub-phase's own review as a checkpoint rather than deferring all review to the 2.15/3.15 hardening pass. | diff --git a/docs/solutions/best-practices/abort-state-guidance-multi-step-physical-input.md b/docs/solutions/best-practices/abort-state-guidance-multi-step-physical-input.md index 928c44b..76afdea 100644 --- a/docs/solutions/best-practices/abort-state-guidance-multi-step-physical-input.md +++ b/docs/solutions/best-practices/abort-state-guidance-multi-step-physical-input.md @@ -1,124 +1,52 @@ --- -title: Abort-state guidance on multi-step physical input errors -date: 2026-06-10 +title: Make interrupted physical drags end in a known safe state +date: 2026-07-11 category: best-practices -module: crates/macos +module: crates/macos input problem_type: best_practice -component: macos-adapter +component: tooling severity: high applies_when: - - Implementing a multi-step physical input sequence (drag, gesture, press-hold) using CGEvent synthesis - - An early return or error exit can leave cursor or button state at an intermediate position - - A Drop guard is introduced to cancel a partially committed OS operation - - An error from a multi-phase input helper needs recovery guidance attached - - The caller must distinguish "no drop committed" from "drop committed at the wrong target" -tags: - - drag - - cgevent - - abort-state - - error-recovery - - mouse-input - - headless-headed - - macos - - physical-fallback + - "Implementing a multi-event physical input sequence" + - "An event may fail after mouse-down or another irreversible input event" + - "Reporting whether a physical action may have been delivered" +tags: [drag, cgevent, delivery, cleanup, physical-input] --- -# Abort-state guidance on multi-step physical input errors +# Make interrupted physical drags end in a known safe state ## Context -A synthetic drag is a multi-step OS mutation: `LeftMouseDown` at the origin, interpolated `LeftMouseDragged` events, a dwell over the destination, then a final `LeftMouseUp`. Any step after the mouse-down can fail, and the original `MouseUpGuard` in `crates/macos/src/input/mouse.rs` had two bugs at once. The happy path disarmed the guard *before* posting the final `LeftMouseUp`, so a failure on that last post left the button logically held down system-wide — the exact defect the guard existed to prevent. And the guard's corrective release on `Drop` fired at the *destination*: CGEvents resolve at the coordinates embedded in the event, so a "corrective" up at the destination is indistinguishable from a successful drop. The file landed in the target folder while the command reported failure — the agent's perception (error) and the system's state (dropped) irreconcilably diverged. +A physical drag starts before its final drop. Once mouse-down is posted, later +deadline, event, or dwell failures cannot be represented as an ordinary +pre-dispatch failure: the system may still believe the button is down. ## Guidance -The pattern has three coordinated legs. +`crates/macos/src/input/mouse_drag.rs` owns a `DragReleaseGuard` for the whole +sequence. It arms only after mouse-down is posted, retains the destination +release until that post succeeds, and uses `Drop` to post a dragged and up +event at the origin when the sequence aborts. The happy path disarms only +after the destination up event has been posted. -**Leg 1 — the guard owns the release, and disarms only after the post succeeds.** +The guard also tracks delivery. Public errors are enriched from that state, so +callers can distinguish a pre-delivery failure from an interrupted physical +operation and avoid an unsafe automatic retry. -```rust -// crates/macos/src/input/mouse.rs -impl MouseUpGuard { - fn release_at(&mut self, point: CGPoint) -> Result<(), AdapterError> { - post_event(CGEventType::LeftMouseUp, point, CGMouseButton::Left)?; - self.armed = false; - Ok(()) - } -} -``` +The origin is deliberate: an abort must not become a destination drop. Cleanup +is still best effort; if the operating system cannot accept the corrective +events, the error must preserve that uncertainty rather than claiming no input +was delivered. -The happy path calls `release.release_at(to)` as its final statement. If that post fails, `armed` stays `true` and `Drop` still runs the abort path. +## Prevention -**Leg 2 — abort at the origin, never the unreached destination.** - -```rust -impl Drop for MouseUpGuard { - fn drop(&mut self) { - if self.armed { - let _ = post_event(CGEventType::LeftMouseDragged, self.origin, CGMouseButton::Left); - let _ = post_event(CGEventType::LeftMouseUp, self.origin, CGMouseButton::Left); - } - } -} -``` - -`origin` is captured at `LeftMouseDown`. Releasing where the gesture picked up is a self-drop — a no-op to most drop targets — which gives the abort genuine cancel semantics. There is no "current cursor position" escape hatch: the coordinate in the event is where the OS resolves the release. - -**Leg 3 — every error from the sequence carries the end state.** - -```rust -pub fn synthesize_drag(params: DragParams) -> Result<(), AdapterError> { - drag_sequence(params).map_err(|err| { - if err.suggestion.is_some() { - return err; - } - err.with_suggestion( - "The drag was aborted: the button was released back at the origin (best-effort) and no drop was committed at the destination. The cursor ends at the origin. Re-check the source state before retrying.", - ) - }) -} -``` - -One `map_err` at the public boundary; inner errors that already carry a tailored suggestion pass through unchanged. The guard's doc comment also states the best-effort limits honestly: the corrective posts can themselves fail (typically the same systemic CGEventSource failure that aborted the drag), and a drop target sitting under the origin still sees a self-drop. - -## Why This Matters - -Without leg 1, a failure on the final post leaves the button logically held: the OS treats every subsequent cursor movement as a drag and every click target as a drop target, corrupting unrelated interactions until something releases the button. - -Without leg 2, an aborted drag silently commits as a real drop. This is the worst failure class for an agent caller: the error says "retry", but the filesystem or UI already changed — retrying duplicates the operation. - -Without leg 3, an agent holding `ACTION_FAILED`/`INTERNAL` cannot tell whether the world changed. With the suggestion, it knows: button released at origin, no drop committed, cursor at origin — and whether a re-snapshot is needed before retry. - -## When to Apply - -- A command posts a sequence of OS-level events where a later step can fail after an earlier irreversible step succeeded -- The abort path could itself mutate state (a corrective event at the wrong coordinates commits, not cancels) -- Errors must communicate post-failure system state, not just the failure cause, for callers to retry safely - -Specifically necessary for anything backed by `CGEventPost`-style APIs that resolve coordinates at event-creation time. - -## Examples - -Before (both bugs, as originally shipped): - -```rust -// happy path disarmed BEFORE the final fallible post -release.armed = false; -post_event(CGEventType::LeftMouseUp, to, CGMouseButton::Left) - -// and Drop released at the DESTINATION, committing the aborted drop -impl Drop for MouseUpGuard { - fn drop(&mut self) { - if self.armed { - let _ = post_event(CGEventType::LeftMouseUp, self.to, CGMouseButton::Left); - } - } -} -``` - -The fixed version is the Guidance section above. Verified by the E2E drag scenario observing the canvas effect, not the command's `ok:true`. +- Allocate and arm cleanup before the first committed input event. +- Do not disarm a cleanup guard before the final event post succeeds. +- Test failures before mouse-down, after mouse-down, and during final release. +- Keep sequence cleanup local to the input primitive; command code should not + need to reconstruct partial pointer state. ## Related -- `best-practices/macos-gesture-headless-capability-2026-06-10.md` — drag/hover/mouse-* are always physical and policy-gated; establishes when this multi-step path runs at all -- `best-practices/playwright-grade-desktop-reliability-2026-06-02.md` — the upstream principle that fallbacks must be explicit and failures honest; abort-state cleanup is that principle applied to mid-sequence failure -- `best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md` — when the physical path is even entered (policy ownership) — context for why aborts are headed-mode territory +- [Document pointer actions from their own reliability pipeline](../documentation-gaps/hover-drag-skip-the-actionability-battery.md) +- [Build desktop actions as an observe-resolve-preflight-dispatch contract](playwright-grade-desktop-reliability-2026-06-02.md) diff --git a/docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md b/docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md index 3409615..29b8187 100644 --- a/docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md +++ b/docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md @@ -1,273 +1,60 @@ --- -title: Deduplicate ref allocator via RefAllocConfig instead of a _with_X copy -date: 2026-04-14 -last_updated: 2026-06-10 +title: Keep ref allocation in one recursive owner +date: 2026-07-11 category: best-practices -module: crates/core +module: crates/core ref allocation problem_type: best_practice component: tooling -severity: medium +severity: high applies_when: - - Two functions share identical bodies differing only in one Optional/nullable field (classic _with_X naming) - - A positional parameter list carries five or more args, including bool flags - - A patch commit touches only one copy of duplicated traversal logic, causing silent drift in the other - - A code review flags a DRY violation on ref-allocation or tree-traversal paths - - Adding a new per-call config axis (e.g. root_ref_id) would require another _with_X variant -tags: - - dry - - ref-allocation - - config-struct - - skeleton-traversal - - drill-down - - deduplication - - rust-patterns - - code-review + - "Changing which nodes receive refs" + - "Changing compact, bounds, skeleton, or drill-down allocation behavior" + - "Adding source or scope evidence to RefEntry" +tags: [refs, snapshot, drill-down, recursion, config-struct] --- -# Deduplicate ref allocator via RefAllocConfig instead of a _with_X copy +# Keep ref allocation in one recursive owner ## Context -The progressive skeleton traversal feature shipped two near-identical ref allocators into `crates/core/`: - -- `allocate_refs` in `crates/core/src/snapshot.rs` — 7 positional parameters, called from the full-snapshot path -- `allocate_refs_with_root` in `crates/core/src/snapshot_ref.rs` — took a local `DrillDownConfig<'a>` struct, called from the drill-down path - -Both bodies were ~95% identical: same `INTERACTIVE_ROLES` check, same `ref_entry_from_node` call, same skeleton-anchor detection (`!is_interactive && node.children_count.is_some() && has_label`), same bounds filtering, same compact-collapse, same `interactive_only` pruning. They differed in exactly one thing: whether each allocated `RefEntry` carried a `root_ref: None` or `root_ref: Some(root_ref_id.to_string())` tag. - -The duplication did not land because an author was lazy. (session history) It landed because the March 10 implementation session used two parallel worktree agents — one owning `snapshot.rs` + `builder.rs`, the other owning `snapshot_ref.rs` + the macOS adapter. Each agent wrote its own allocator in isolation. When the merge agent integrated both worktrees, it hit the project's 400 LOC file limit on `snapshot.rs` and extracted `ref_alloc.rs` purely as a LOC-budget fix — pulling out the *small* helpers (`INTERACTIVE_ROLES`, `actions_for_role`, `ref_entry_from_node`, `is_collapsible`) while leaving the two full allocator bodies in their original files. The merge agent's summary explicitly listed `allocate_refs_with_root` and `DrillDownConfig` as distinct items belonging to `snapshot_ref.rs`. No decision was ever made to leave them separate for design reasons; it was the path of least resistance under a LOC deadline. - -The brainstorm at `docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md` described `ref_alloc.rs` as the home for "shared ref-allocation logic" but only named `INTERACTIVE_ROLES` and `is_collapsible`. It did not specify that the full allocator body should live there, leaving a gap the implementation filled with a copy. +Full snapshots and rooted drill-downs both turn an observed tree into refs. +They differ only in their allocation options, source evidence, and scope. Two +recursive implementations would inevitably make the same UI allocate different +refs depending on how it was observed. ## Guidance -**Any two function bodies that are structurally identical except for a small number of parameter values or literals are a single function waiting to be written.** The correct fix is a shared config struct whose fields default to — or accept an `Option<T>` for — the differentiating value. Copy-and-modify with a name suffix (`foo_with_X`, `foo_v2`, `foo_alternate`, `do_thing_for_Y`) is never the right tool; it is technical debt that generates silent divergence under the first patch. +`allocate_refs` in `crates/core/src/ref_alloc.rs` is the only recursive allocator. +Callers supply a `RefAllocConfig` composed of `RefAllocOptions`, +`RefAllocSource`, and `RefAllocScope` rather than adding another allocator or a +positional argument list. -**Concrete rule for this codebase:** `crates/core/src/ref_alloc.rs` is the single source of truth for the ref-allocation pass. `RefAllocConfig::root_ref_id` is `Option<&'a str>`: +The allocator owns these semantics together: -- `snapshot.rs::build` (and `append_surface_refs`) passes `root_ref_id: None` -- `snapshot_ref.rs::run_from_ref` passes `root_ref_id: Some(root_ref_id)` +- ref eligibility: interactive roles or primary advertised actions, never a + bare focus affordance; +- identity, geometry, capability, source, and scope evidence in `RefEntry`; +- bounds omission, compact collapse, and interactive-only pruning; +- named skeleton anchors that are legitimate drill-down targets. -The function body is identical for both callers. Any future change to allocation logic — adding a new role to `INTERACTIVE_ROLES`, changing skeleton-anchor detection, altering `is_collapsible`, tuning the `interactive_only` pruning rule — is made once in `ref_alloc::allocate_refs` and immediately applies to both the full-snapshot and the drill-down path. - -**The threshold to extract a config struct:** (auto memory [claude]) when a function already takes four or more positional parameters and the new variant adds one more distinguishing value. Do not add an eighth positional parameter. Do not copy the body. Extract a config struct with all fields (including the new one as `Option<T>`) and unify. - -**Applies to tests too.** Two test helper closures that differ only in a flag value are a single parametrised helper. The drill-down tests in `snapshot_ref.rs` already follow this via `drill_config(source_app, pid, root_ref_id, interactive_only, compact)` returning a `RefAllocConfig`. +Full snapshots use an empty root scope. Drill-down passes the root ref and its +path prefix, then updates only that root's descendants in the stored map. ## Why This Matters -The divergence was not theoretical. (session history) Two separate commits drifted the copies in opposite directions before the duplication was even two weeks old: +Ref allocation is an identity boundary, not presentation formatting. A change +to one path but not the other can create stale refs, lose scope evidence, or +make an action available in one observation mode but impossible in another. -**Incident 1 — `cec176d fix: preserve skeleton drill-down anchors`**: repaired logic that ensured skeleton anchors retained their `root_ref` tag correctly. The fix was applied only to `allocate_refs_with_root` in `snapshot_ref.rs`. The matching logic in `allocate_refs` in `snapshot.rs` was not touched. Snapshot and drill-down paths became silently inconsistent on anchor tagging. +## Prevention -**Incident 2 — `0fcf4e8 fix: bypass AXConfirm on web elements and add skeleton anchors to drill-down`**: added the `!is_interactive && node.children_count.is_some() && has_label` skeleton-anchor detection to `allocate_refs_with_root`. The same logic already existed in `allocate_refs` — it had been added there in the original `b13dc69 feat: implement progressive skeleton traversal` commit. The author of `0fcf4e8` had to check both copies, noticed the gap in the drill-down copy, and patched it. But the patch direction was reversed from `cec176d`: this time the snapshot copy was ahead and the drill-down copy was behind. - -Two patches, two weeks, opposite directions of drift, within the same feature branch. That is the expected rate of rot for any copied core algorithm in this codebase. - -**Review cost**: every reviewer on PR #20 had to understand both bodies to confirm they were equivalent. Because they lived in different files with different struct names (`window_pid` vs `pid`, seven positional args vs `&DrillDownConfig`), confirming equivalence required careful side-by-side reading rather than a single glance. The Cursor Bugbot eventually flagged it as "Low Severity". **Low was wrong.** (auto memory [claude]) Per `feedback_dry_is_core.md`, any duplicated core algorithm is P1 minimum, because the next fix will always land in exactly one copy. - -## When to Apply - -**Trigger 1 — identical recursive body.** If you are writing a recursive tree-walk and the only difference from an existing recursive tree-walk is one field on the entries it produces, unify. Recursive duplicates are particularly high-risk because the recursive self-call must also be updated in both copies on every future change. - -**Trigger 2 — name suffix pattern.** Function names ending in `_with_X`, `_v2`, `_alternate`, `_for_Y` are red flags. Before adding such a function, ask: can the original accept an `Option<X>` parameter or a config struct with `X` as an optional field? - -**Trigger 3 — config struct threshold.** Four or more positional parameters + one more distinguishing value = extract a config struct. This rule applies even if the existing function "looks fine" with seven args today; adding an eighth makes the call sites unreadable and invites the copy-and-modify shortcut at the next variant. - -**Trigger 4 — parallel worktree seams.** (session history) When an implementation plan fans out into parallel worktree agents by file, be explicit in the plan about which functions are shared and name the exact helpers that will live in the shared module *before* the agents diverge. The March 10 plan named `INTERACTIVE_ROLES` and `is_collapsible` as shared but omitted `allocate_refs` — and the duplication followed from that omission. - -**Trigger 5 — LOC-budget extraction.** When extracting code into a new module purely to fit a file-size limit, resist the urge to stop at the smallest extraction that clears the limit. Check whether the code that *stays behind* on one side now mirrors code on the other side. If it does, the LOC fix is also a DRY fix and both extractions belong together. - -**Applies to review too.** (auto memory [claude]) Any two function bodies flagged by review tooling as "duplicated" get P1 severity minimum in this project. Low or P3 severity labels on duplication findings are wrong and should be escalated. - -## Examples - -### Before — two independent bodies, two files - -**`crates/core/src/snapshot.rs`** (pre-`d06a6c2`, 7 positional params): - -```rust -fn allocate_refs( - mut node: AccessibilityNode, - refmap: &mut RefMap, - include_bounds: bool, - interactive_only: bool, - compact: bool, - window_pid: i32, - source_app: Option<&str>, -) -> AccessibilityNode { - let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str()); - - if is_interactive { - let entry = ref_entry_from_node(&node, window_pid, source_app, None); - node.ref_id = Some(refmap.allocate(entry)); - } - - let has_label = node.name.as_deref().is_some_and(|n| !n.is_empty()) - || node.description.as_deref().is_some_and(|d| !d.is_empty()); - let is_skeleton_anchor = !is_interactive && node.children_count.is_some() && has_label; - - if is_skeleton_anchor { - let mut entry = ref_entry_from_node(&node, window_pid, source_app, None); - entry.available_actions = vec![]; - node.ref_id = Some(refmap.allocate(entry)); - } - - if !include_bounds { node.bounds = None; } - - node.children = node.children.into_iter() - .filter_map(|child| { - let child = allocate_refs(child, refmap, include_bounds, - interactive_only, compact, window_pid, source_app); - if compact && is_collapsible(&child) { - return child.children.into_iter().next(); - } - if interactive_only && child.ref_id.is_none() - && child.children.is_empty() && child.children_count.is_none() { - None - } else { Some(child) } - }) - .collect(); - node -} -``` - -**`crates/core/src/snapshot_ref.rs`** (pre-`d06a6c2`, local `DrillDownConfig`): - -```rust -struct DrillDownConfig<'a> { - include_bounds: bool, - interactive_only: bool, - compact: bool, - pid: i32, - source_app: Option<&'a str>, - root_ref_id: &'a str, -} - -fn allocate_refs_with_root( - mut node: AccessibilityNode, - refmap: &mut RefMap, - config: &DrillDownConfig, -) -> AccessibilityNode { - let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str()); - - if is_interactive { - let entry = ref_entry_from_node( - &node, config.pid, config.source_app, - Some(config.root_ref_id.to_string()), // <-- the only substantive difference - ); - node.ref_id = Some(refmap.allocate(entry)); - } - - // ... 40+ more lines structurally identical to allocate_refs -} -``` - -The only substantive difference between the two bodies was `None` vs `Some(config.root_ref_id.to_string())` in exactly two call sites. - -### After — one body, one file, two callers - -**`crates/core/src/ref_alloc.rs`** (post-`d06a6c2`): - -```rust -pub(crate) struct RefAllocConfig<'a> { - pub include_bounds: bool, - pub interactive_only: bool, - pub compact: bool, - pub pid: i32, - pub source_app: Option<&'a str>, - pub root_ref_id: Option<&'a str>, // None = full snapshot, Some(_) = drill-down -} - -pub(crate) fn allocate_refs( - mut node: AccessibilityNode, - refmap: &mut RefMap, - config: &RefAllocConfig, -) -> AccessibilityNode { - let root_ref_owned = config.root_ref_id.map(str::to_string); - let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str()); - - if is_interactive { - let entry = ref_entry_from_node(&node, config.pid, config.source_app, root_ref_owned.clone()); - node.ref_id = Some(refmap.allocate(entry)); - } - - let has_label = node.name.as_deref().is_some_and(|n| !n.is_empty()) - || node.description.as_deref().is_some_and(|d| !d.is_empty()); - let is_skeleton_anchor = !is_interactive && node.children_count.is_some() && has_label; - - if is_skeleton_anchor { - let mut entry = ref_entry_from_node(&node, config.pid, config.source_app, root_ref_owned); - entry.available_actions = vec![]; - node.ref_id = Some(refmap.allocate(entry)); - } - - if !config.include_bounds { node.bounds = None; } - - node.children = node.children.into_iter() - .filter_map(|child| { - let child = allocate_refs(child, refmap, config); - if config.compact && is_collapsible(&child) { - return child.children.into_iter().next(); - } - if config.interactive_only - && child.ref_id.is_none() - && child.children.is_empty() - && child.children_count.is_none() - { - None - } else { - Some(child) - } - }) - .collect(); - - node -} -``` - -**`crates/core/src/snapshot.rs`** caller: - -```rust -let config = RefAllocConfig { - include_bounds: opts.include_bounds, - interactive_only: opts.interactive_only, - compact: opts.compact, - pid: window.pid, - source_app: Some(window.app.as_str()), - root_ref_id: None, -}; -let mut tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); -``` - -**`crates/core/src/snapshot_ref.rs`** caller: - -```rust -let config = RefAllocConfig { - include_bounds: opts.include_bounds, - interactive_only: opts.interactive_only, - compact: opts.compact, - pid: entry.pid, - source_app, - root_ref_id: Some(root_ref_id), -}; -let mut tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); -``` - -`DrillDownConfig` is deleted. `allocate_refs_with_root` is deleted. A future change to `INTERACTIVE_ROLES`, the skeleton-anchor condition, the `is_collapsible` rule, or the `interactive_only` pruning touches exactly one place. - -**Net diff** (`d06a6c2 refactor: unify allocate_refs across snapshot and drill-down paths`): +139 / −190 across `ref_alloc.rs`, `snapshot.rs`, `snapshot_ref.rs`. 51 net LOC removed. `grep allocate_refs_with_root` returns zero matches. The follow-up `7c7837a chore: drop with_root from drill test names after allocator unification` renamed the leftover `test_allocate_refs_with_root_*` helpers to `test_drill_alloc_*` so the name is gone from the codebase entirely. - -### Current shape (as of 2026-06-10) - -The code blocks above are the historical record of that refactor; the unified design has since absorbed further growth in exactly the way the pattern promised — all in one place. `RefAllocConfig` now carries ten fields (the originals plus `source_window_id`, `source_window_title`, `source_surface`, and `path_prefix` for element-identity evidence), ref eligibility generalized from `INTERACTIVE_ROLES.contains(...)` to `is_ref_able(node)` (interactive role **or** a non-`SetFocus` advertised action), the skeleton-anchor condition additionally requires `root_ref_id.is_none()`, and `allocate_refs` delegates to a private `allocate_refs_at_path` that threads tree-position tracking. Each of those changes touched the single shared body — none re-introduced a second copy. See `crates/core/src/ref_alloc.rs` for the current implementation. +- Add behavior once in `ref_alloc.rs` and cover both full and rooted snapshots. +- Prefer a nested config type when a new allocation dimension appears; do not + add `_with_root` or `_for_skeleton` copies. +- Treat a new recursive allocator as an architecture violation unless its + output contract is genuinely different and documented. ## Related -- `crates/core/src/ref_alloc.rs` — single source of truth for ref allocation -- `crates/core/src/snapshot.rs` — full-snapshot caller -- `crates/core/src/snapshot_ref.rs` — drill-down caller -- Commit `d06a6c2` — the unifying refactor -- Commit `7c7837a` — test name cleanup -- `docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md` — original plan that split into parallel worktrees but did not name `allocate_refs` as a shared helper (moderate overlap but predates the duplication problem) -- `docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md` — feature brainstorm; named `INTERACTIVE_ROLES` and `is_collapsible` as shared helpers but omitted the allocator body +- [Keep progressive snapshots namespace-scoped and ref-safe](../logic-errors/progressive-snapshot-review-contract-2026-04-16.md) +- [Playwright-grade desktop reliability contract](playwright-grade-desktop-reliability-2026-06-02.md) diff --git a/docs/solutions/best-practices/deterministic-build-artifact-marker-2026-04-16.md b/docs/solutions/best-practices/deterministic-build-artifact-marker-2026-04-16.md deleted file mode 100644 index 332e7c6..0000000 --- a/docs/solutions/best-practices/deterministic-build-artifact-marker-2026-04-16.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -title: Stamp build artifacts at a deterministic path for CI and scripts -date: 2026-04-16 -category: best-practices -module: crates/ffi -problem_type: best_practice -component: tooling -severity: medium -status: superseded_for_agent_desktop -applies_when: - - A build script (cbindgen, bindgen, codegen) writes an artifact to $OUT_DIR whose path includes a cargo-generated hash - - CI or a developer script needs to locate that artifact after the build finishes - - The obvious shell-out (`find target -path '*/out/<artifact>' | head -1`) silently picks the wrong file when multiple cached build dirs coexist - - You want a drift check (`diff committed_copy generated_copy`) that reliably fails when the committed copy is stale -tags: - - build-rs - - cbindgen - - ci - - reproducibility - - determinism - - cargo - - rust-patterns ---- - -## Problem - -Update, 2026-05-19: `agent-desktop` no longer uses this pattern for the FFI -header. `cbindgen` is intentionally denied from the Cargo dependency graph, and -`crates/ffi/include/agent_desktop.h` is treated as the committed C ABI -contract. The pattern below still applies to projects that deliberately run a -build-script generator in the normal Cargo build graph. - -Cargo puts build-script artifacts under a hash-randomized directory: - -``` -target/<profile>/build/<crate-name>-<hash>/out/<artifact> -``` - -The `<hash>` is not documented, not stable across rebuilds of the same -code on different toolchain minor versions, and you can end up with -several `<crate-name>-<hash>/` dirs in the same warm `target/` cache -(e.g. after a `cargo clean -p` and rebuild, or rustup flip). Any CI -step or developer script that resolves the artifact with -`find target -path '.../<artifact>' | head -1` is picking arbitrarily -among those dirs. Under drift check: - -- If `head -1` picks the current build's artifact → drift correctly - surfaces when the committed copy is stale. -- If `head -1` picks a stale leftover → you either self-heal (false - green) or report stale-vs-stale (useless). - -The failure mode is silent: a CI step says "OK: header in sync" while -the committed header is actually out of date, and the bad ABI ships. - -## Solution - -Have `build.rs` write a stable marker file containing the absolute path -of the just-generated artifact. Downstream consumers read the marker -instead of guessing: - -```rust -// crates/<your-crate>/build.rs -fn main() { - // ... generate $OUT_DIR/artifact ... - - if let Some(target_root) = target_root_from_out_dir(Path::new(&out_dir)) { - let stamp = target_root.join("ffi-header-path.txt"); - let _ = std::fs::write(&stamp, out_path.to_string_lossy().as_bytes()); - } -} - -/// OUT_DIR = {target}/{profile}/build/{pkg-hash}/out -fn target_root_from_out_dir(out_dir: &Path) -> Option<PathBuf> { - let mut current = out_dir; - for _ in 0..4 { - current = current.parent()?; - } - Some(current.to_path_buf()) -} -``` - -CI and scripts read the marker: - -```yaml -- name: Drift check - run: | - STAMP=target/ffi-header-path.txt - test -f "$STAMP" || { echo "FAIL: stamp missing"; exit 1; } - GENERATED=$(cat "$STAMP") - test -f "$GENERATED" || { echo "FAIL: stamped path missing"; exit 1; } - diff -u crates/ffi/include/artifact.h "$GENERATED" -``` - -## Why this works - -- **One writer**: the build script's own invocation knows exactly which - `OUT_DIR` it ran in. Stamping the path at that moment captures it - authoritatively; no later tool has to reconstruct it. -- **Stable path**: `target/<marker>.txt` lives one directory above the - hashed build dirs and is overwritten each build. CI's cache system - sees it as content of `target/` — no special allowlisting needed. -- **Fail-fast wrapper**: the marker file's absence is itself a signal - that the build script didn't run (e.g. `cargo check` instead of - `cargo build`). CI should fail rather than fall back to a wrong - default. - -## When NOT to use this - -- If your build script generates artifacts deterministically **at a - fixed location outside `OUT_DIR`** (e.g. directly into a committed - dir), you don't need a marker — the path itself is stable. -- If the artifact lifecycle is driven by `cargo metadata` queries - (e.g. `cargo metadata --format-version=1 | jq .target_directory`), - that's already deterministic. Stamping is only needed when the - specific pkg-hash subdirectory matters. - -## Sibling anti-pattern to avoid - -Do **not** have `build.rs` copy the generated artifact into the source -tree (e.g. `fs::copy(&out_path, &committed_path)`). That mutates the -working tree during every build, which means: - -- `git diff` is polluted by invisible copies. -- The drift check (`git diff --exit-code committed_path`) can - self-heal a stale committed copy instead of catching it. - -The committed copy is the ABI contract; updating it should be an -explicit developer action (dedicated script) or CI-only step. - -## References - -- `scripts/update-ffi-header.sh` — explicit maintainer refresh script -- Todo `006-ready-p2-deterministic-ffi-header-drift-path` — the bug this - pattern resolves diff --git a/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md b/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md index 49b5034..352d378 100644 --- a/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md +++ b/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md @@ -34,7 +34,7 @@ pinned clients cannot distinguish an old response from a new one. ## Guidance -Keep `crates/core/src/output.rs::ENVELOPE_VERSION` as the single source of +Keep `ENVELOPE_VERSION` in `crates/core/src/output.rs` as the single source of truth. Bump it whenever a client that already parses successful JSON needs a different parser branch for the same command name. @@ -59,4 +59,5 @@ the release note. ## Tests Tests should assert the version through `ENVELOPE_VERSION`, not string literals, -so `main`, `batch`, and unit-level envelope tests stay aligned after a bump. +so `main`, `batch`, FFI JSON-command tests, and unit-level envelope tests stay +aligned after a bump. diff --git a/docs/solutions/best-practices/exhaustiveness-guards-over-catch-alls-in-policy-mirrors.md b/docs/solutions/best-practices/exhaustiveness-guards-over-catch-alls-in-policy-mirrors.md index 3fdc7a5..c3f63bf 100644 --- a/docs/solutions/best-practices/exhaustiveness-guards-over-catch-alls-in-policy-mirrors.md +++ b/docs/solutions/best-practices/exhaustiveness-guards-over-catch-alls-in-policy-mirrors.md @@ -1,122 +1,46 @@ --- -title: Use named arms and exhaustiveness guard tests instead of catch-alls in policy and dispatch mirrors -date: 2026-06-10 +title: Use explicit arms for string-keyed policy mirrors +date: 2026-07-11 category: best-practices -module: crates/core, src +module: crates/core command policy problem_type: best_practice -component: command-policy +component: tooling severity: high applies_when: - - A helper mirrors the per-case behavior of real command dispatch (a preflight reproducing each command's policy) - - A new command or action name is added and a parallel mapping must stay in sync - - A match over named semantic cases is tempted to use a catch-all arm - - A registry or test list must provably cover everything the codebase actually contains -tags: - - exhaustiveness - - command-policy - - dispatch - - wait-predicate - - regression-prevention - - contract-tests - - actionability - - rust-patterns + - "Mapping a command-name string to an action or policy" + - "Adding a wait predicate that mirrors command behavior" + - "Reviewing a catch-all branch in a policy mapping" +tags: [policy, match, actionability, wait, regression] --- -# Use named arms and exhaustiveness guard tests instead of catch-alls in policy and dispatch mirrors +# Use explicit arms for string-keyed policy mirrors ## Context -`wait --predicate actionable --action <name>` answers "would this action succeed" by running the same actionability preflight the real command runs — which means it must mirror each command's interaction policy exactly. The first implementation centralized that mirror behind a catch-all: - -```rust -fn actionability_request(action: Action) -> ActionRequest { - match action { - Action::TypeText(_) => ActionRequest::focus_fallback(action), - _ => ActionRequest::headless(action), - } -} -``` - -Correct for the four actions that existed — and a trap for the fifth. Any future action would silently inherit headless policy, right or wrong, with no compiler complaint and no failing test. In a headless-first codebase where policy must flow explicitly from each command's declared base, a mirror that *infers* policy is exactly the drift the project's own learnings warn against. Code review flagged the catch-all as a silent-drift risk (auto memory [claude]: the repo treats policy inference as a standing hazard). +The compiler cannot prove exhaustiveness when user strings select behavior. +A fallback that assigns a default policy can therefore become silently wrong +when a new action is added. ## Guidance -Three legs: explicit arms, a machine-derived universe guard, and per-case value pins. +`commands/wait_predicate.rs::parse_actionability_action` names every supported +`--action` value and constructs the exact `ActionRequest` used for its +preflight. The final branch rejects unknown input; it must never infer a policy +for an unrecognized action. -**Leg 1 — one arm per name; the only "default" is an error.** +For any similar mirror, use one explicit arm per supported value and tests that +pin both the accepted vocabulary and the policy-sensitive cases. Prefer a typed +enum and compiler exhaustiveness when the external string boundary can be +converted first. -```rust -// crates/core/src/commands/wait_predicate.rs -/// Maps each `--action` name to the exact request its real command would -/// run with — policy included — so the preflight answers "would this action -/// succeed". Every name is an explicit arm: a catch-all here would let a -/// new action silently inherit the wrong policy. -fn parse_actionability_action(action: Option<&str>) -> Result<ActionRequest, AppError> { - match action.unwrap_or("click") { - "click" => Ok(ActionRequest::headless(Action::Click)), - "type" => Ok(ActionRequest::focus_fallback(Action::TypeText(String::new()))), - "set-value" => Ok(ActionRequest::headless(Action::SetValue(String::new()))), - "clear" => Ok(ActionRequest::headless(Action::Clear)), - other => Err(AppError::invalid_input_with_suggestion( - format!("Unknown actionability action '{other}'"), - "Use one of: click, type, set-value, clear.", - )), - } -} -``` +## Prevention -The `other` arm rejects unrecognized input — it is input validation, not a semantic catch-all. Adding an action *requires* adding an arm; forgetting produces a user-visible error at this match, never a silent wrong-policy preflight. - -**Leg 2 — a guard test that derives the case universe mechanically.** - -```rust -// crates/core/src/commands/ref_policy_tests.rs -const POLICY_TESTED_COMMANDS: &[&str] = &[ - "check", "clear", "click", "collapse", "double_click", "expand", - "focus", "right_click", "scroll", "scroll_to", "select", "set_value", - "toggle", "triple_click", "type_text", "uncheck", -]; - -#[test] -fn all_context_request_callers_are_policy_tested() { - // scans crates/core/src/commands/*.rs (excluding *_tests) for files - // containing `context.request(` and fails, naming each stem, when one - // is absent from POLICY_TESTED_COMMANDS -} -``` - -The universe is not hand-maintained: the test scans the filesystem for the call-site signature every ref-action command shares. A new command file that calls `context.request(` without a registered policy assertion fails CI with a message naming the stem and the required follow-up. - -**Leg 3 — per-case value pins.** - -Each listed case is pinned to its exact mirrored value, never covered by an "everything else" loop — `type` is asserted `focus_fallback` specifically, each remaining command asserted headless by name. The same shape guards CLI registration in `src/cli/contract_tests.rs`: `NON_COMMAND_MODULES` plus a filesystem scan asserts every command module is either a registered CLI subcommand or an explicitly declared helper. - -## Why This Matters - -Mirrors rot silently. When real dispatch gains a case but the mirror does not, nothing fails — the mirror's catch-all *answers confidently and wrongly*. Here that means a false "actionable: true" for an action whose real command would run a different policy: the agent's wait reports ready, the action then fails or behaves differently. The compiler cannot help across a string-keyed boundary, so the guard test substitutes for exhaustiveness checking by deriving the universe from the same source of truth the dispatcher uses (the files that exist, the call signature they share). - -## When to Apply - -- A function maps symbolic names to typed behavior where cases must genuinely differ (policy, routing, config) -- A test list or registry mirrors real per-case code and grows as the system grows -- The case universe is file-shaped or string-keyed, so `match` exhaustiveness cannot be compiler-enforced — derive it mechanically instead -- When the universe IS a closed enum, prefer matching on the enum directly and let the compiler enforce exhaustiveness; the guard-test pattern is for universes the compiler cannot see - -## Examples - -Before — the policy mirror with a catch-all (silently wrong for the next action): - -```rust -match action { - Action::TypeText(_) => ActionRequest::focus_fallback(action), - _ => ActionRequest::headless(action), -} -``` - -After — the Guidance section above; the guard is `all_context_request_callers_are_policy_tested`, the value pin is `actionable_parse_mirrors_each_real_command_policy`. +- Reject unknown strings with `INVALID_ARGS` and an enumerated suggestion. +- Add a test before extending the accepted vocabulary. +- Do not use `_ => headless(...)` or another semantic default in a + string-to-policy mapping. ## Related -- `best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md` — the same risk class (per-case policy flattened by a structural abstraction) from the shared-helper angle; complementary defenses -- `best-practices/playwright-grade-desktop-reliability-2026-06-02.md` — the dispatch-correctness and test-ownership contract this pattern enforces at the match level -- `best-practices/macos-gesture-headless-capability-2026-06-10.md` — the per-gesture policy table whose explicitness named arms preserve +- [Preserve command policy semantics during refactor](preserve-command-policy-semantics-during-refactor-2026-05-12.md) +- [Make FFI ref actions share CLI policy semantics](keep-ffi-action-policy-aligned-with-cli-2026-05-12.md) diff --git a/docs/solutions/best-practices/ffi-repr-c-struct-size-pinning.md b/docs/solutions/best-practices/ffi-repr-c-struct-size-pinning.md index f0e27b8..a206253 100644 --- a/docs/solutions/best-practices/ffi-repr-c-struct-size-pinning.md +++ b/docs/solutions/best-practices/ffi-repr-c-struct-size-pinning.md @@ -49,17 +49,21 @@ pub extern "C" fn ad_action_size() -> usize { The anonymous const assert fails the Rust build the moment the layout drifts. The extern function lets any binding language query the true size at startup and compare it against its own layout computation. -**Layer 2 — C header: macro + C11-gated `_Static_assert` + layout history.** +**Layer 2 — C header: literal macro + C11-gated `_Static_assert`.** ```c /* crates/ffi/include/agent_desktop.h */ -#define AD_ACTION_SIZE (sizeof(AdAction)) +#define AD_ACTION_SIZE 96 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -_Static_assert(sizeof(AdAction) == 96, "AdAction ABI size changed"); +_Static_assert(sizeof(AdAction) == AD_ACTION_SIZE, "AdAction ABI size changed"); #endif ``` -C11 consumers fail at their own compile time when the header and the pinned literal diverge; pre-C11 consumers verify at runtime by comparing their own layout against `ad_action_size()` (the macro is a sizing shorthand, not a pin — `sizeof` always tracks the current struct). A layout-history comment in the header records past size changes (40→48, the AdAction propagation, renames) so fresh reviewers and upgrading callers stop re-discovering adjudicated breaks. +C11 consumers fail at their own compile time when the generated literal and the +actual C layout diverge. Pre-C11 and runtime-layout consumers compare their own +layout against `ad_action_size()`. The committed header is generated from the +Rust definitions, so the literal macro remains the single C-side pin rather +than tracking `sizeof` automatically. **Layer 3 — integration test: size, alignment, offset ordering, zeroed-read, const-vs-extern agreement.** @@ -106,7 +110,7 @@ Adding a field to a pinned struct forces this sequence, and any step done wrong 1. Add the field to the Rust struct → the `const _` assert fails 2. Update the Rust const to the new size → build green -3. Update the header `_Static_assert` literal and the layout-history comment +3. Regenerate the committed header so its literal size macro is updated 4. Update the integration test size assertion and extend the zeroed-read check to the new field 5. The header-compile test and `c_abi_layout` test confirm both sides agree @@ -114,6 +118,6 @@ The process is self-documenting: the failing assert names the struct and the exp ## Related -- `best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md` — the behavioral-parity companion; this doc is the structural-parity half of the same FFI review discipline -- `best-practices/playwright-grade-desktop-reliability-2026-06-02.md` — "FFI and CLI divergence makes language bindings less reliable" is the motivation the pins serve -- `best-practices/deterministic-build-artifact-marker-2026-04-16.md` — records that the header is a hand-committed ABI contract, the discipline these pins extend +- [Keep FFI action policy aligned with CLI action policy](keep-ffi-action-policy-aligned-with-cli-2026-05-12.md) — the behavioral-parity companion; this document is the structural-parity half of the same FFI review discipline. +- [Playwright-grade desktop reliability contract](playwright-grade-desktop-reliability-2026-06-02.md) — ABI pins prevent bindings from silently diverging from the reliability contract. +- The committed `crates/ffi/include/agent_desktop.h`, its generated header checks, and the Rust layout tests are the current source of truth; this repository intentionally does not use a Cargo build-artifact marker for header generation. diff --git a/docs/solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md b/docs/solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md index 30b13bf..2e7fe82 100644 --- a/docs/solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md +++ b/docs/solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md @@ -1,138 +1,46 @@ --- -title: Guard OS-reordered resources with an identity fingerprint, not a raw index -date: 2026-04-16 -last_updated: 2026-06-10 +title: Guard reordered notifications with required identity evidence +date: 2026-07-11 category: best-practices -module: crates/core, crates/macos, crates/ffi +module: notification commands and FFI problem_type: best_practice -component: notifications +component: tooling severity: high applies_when: - - The API exposes a numeric index (or handle) obtained from a list-then-act flow - - The underlying OS can reorder, add, or remove entries between the list and act calls - - Acting on the wrong entry has user-visible consequences (Reply to the wrong sender, Dismiss the wrong notification, Press the wrong button) - - Existing design assumed lists are stable across calls and did not re-verify -tags: - - notification-center - - confused-deputy - - fingerprint - - identity - - ffi - - reordering - - fail-closed + - "Acting on an item selected by an OS-managed list index" + - "Designing an action API for notifications or another reorderable surface" + - "Adding a notification mutation through CLI or FFI" +tags: [notifications, identity, reordering, fail-closed, ffi] --- -## Problem +# Guard reordered notifications with required identity evidence -macOS Notification Center reassigns the `index` of every visible -notification on each listing. Between `list_notifications()` and -`notification_action(index, ...)`: +## Context -- a new notification arrives → everything shifts down by 1 -- an unrelated notification is dismissed → everything shifts up by 1 -- the user opens the Notification Center sidebar → grouping changes - can renumber entries entirely +Notification Center can insert, remove, regroup, or reorder entries between a +list call and a mutation. An index alone is not an identity and can target a +different notification by the time it is used. -Any tool that round-trips `(app, title) → index → "press Reply"` is a -**confused deputy** at the OS boundary: the tool thinks it's acting on -the notification it showed to the user, but by the time the action -call reaches NC the slot points elsewhere. +## Guidance -This class of bug exists for any OS API that: +Notification mutation commands require at least one identity field from the +same list result: expected app or expected title. Core constructs this with +`commands::notification_identity::required_identity`; an empty identity is +`INVALID_ARGS`. The macOS adapter checks the live entry against +`NotificationIdentity` immediately before it acts and fails closed on mismatch. -- returns an ordered list whose ordering is defined by "current state", - not by a stable identity -- accepts an index (or some other positional handle) as a subsequent - parameter +FFI follows the same rule. Invalid or missing identity data is rejected instead +of being coerced into an unconstrained action. Callers recover by listing again +and selecting a fresh entry. -Examples beyond NC: running-process lists by PID reuse, window lists -after `raise()`, clipboard history, filesystem listings used for bulk -operations. +## Prevention -## Solution +- Never expose a mutating list-index API without an identity precondition. +- Prefer a stable opaque handle when the platform supplies one; otherwise use + the smallest reliable fingerprint and verify immediately before mutation. +- Test insertion, removal, and title/app mismatch between list and act. -Pass an optional identity fingerprint alongside the index. Verify the -row at that index against the fingerprint **before** acting. Fail -closed if it doesn't match. +## Related -```rust -pub struct NotificationIdentity { - pub expected_app: Option<String>, - pub expected_title: Option<String>, -} - -impl NotificationIdentity { - pub fn matches(&self, info: &NotificationInfo) -> bool { - if let Some(ref app) = self.expected_app { - if app != &info.app_name { return false; } - } - if let Some(ref title) = self.expected_title { - if title != &info.title { return false; } - } - true - } -} -``` - -Adapter layer: - -```rust -let entry = list_entries(&filter)?.into_iter() - .find(|e| e.info.index == index) - .ok_or_else(|| AdapterError::notification_not_found(index))?; - -if let Some(id) = identity { - if !id.is_empty() && !id.matches(&entry.info) { - return Err(AdapterError::new( - ErrorCode::NotificationNotFound, - "row at this index does not match the expected fingerprint — NC likely reordered", - )); - } -} - -// safe to press -``` - -## Design choices - -**Optional fields, not mandatory.** The fingerprint is a safety feature, -not a required parameter. Hosts that already reconcile (e.g. by -re-listing just before acting) can leave both fields null and get -the legacy behavior. This keeps the API ergonomic for simple scripts -while making the safe path available. - -**Re-use an existing error code** (`NotificationNotFound`) rather than -adding a new one. From the host's perspective, "the notification I -intended to act on is gone or moved" is semantically the same as "the -notification at that index disappeared": in both cases the right -recovery is re-list and retry. Introducing a distinct -`IDENTITY_MISMATCH` code would fork callers' error handling without a -real behavioral difference. - -**Tri-state UTF-8 decoding at the FFI boundary.** The identity strings -come in as `*const c_char`. Null means "no fingerprint"; invalid UTF-8 -must NOT be silently coerced to "no fingerprint" (that would defeat -the guard). Use `try_c_to_string` (or the `decode_optional_filter!` -macro that wraps it) which returns -`Ok(None)` / `Ok(Some(_))` / `Err(CStrDecodeError)` — the error also -covers strings exceeding the byte cap — and map `Err` to -`InvalidArgs`. - -## When NOT to use this - -- If the OS API returns a stable opaque handle that the caller can - hold across calls (e.g. `HWND` on Windows), plumb the handle - through instead of an index. Fingerprinting is a fallback for - APIs whose handles we can't or shouldn't persist. -- If the operation is idempotent and harmless (e.g. "list children - of this group"), mismatch handling adds cost without value. - -## References - -- `crates/core/src/notification.rs` — `NotificationIdentity` type and - tests -- `crates/macos/src/notifications/actions.rs` — adapter-layer check -- `crates/ffi/src/notifications/action.rs` — C-ABI surface with - optional app/title pointers -- Todo `003-ready-p1-stable-notification-action-identity` — the bug - this pattern resolves +- `crates/core/src/notification_identity.rs` +- `crates/macos/src/notifications/actions.rs` diff --git a/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md b/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md index 904db38..9f3fbee 100644 --- a/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md +++ b/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md @@ -1,55 +1,52 @@ --- -title: Keep FFI action policy aligned with CLI action policy -date: 2026-05-12 -last_updated: 2026-06-10 +title: Make FFI ref actions share CLI policy semantics +date: 2026-07-11 category: best-practices -module: crates/ffi +module: crates/ffi action execution problem_type: best_practice -component: ffi +component: tooling severity: high applies_when: - - FFI exposes an action path that mirrors CLI ref commands - - A core action gains an InteractionPolicy or other side-effect gate - - A new FFI wrapper calls PlatformAdapter directly instead of command dispatch -tags: - - ffi - - interaction-policy - - cli-parity - - abi + - "Adding an FFI action entrypoint" + - "Changing Action base policy or headed behavior" + - "Documenting a low-level native-handle escape hatch" +tags: [ffi, interaction-policy, cli-parity, actions] --- -# Keep FFI action policy aligned with CLI action policy +# Make FFI ref actions share CLI policy semantics ## Context -The CLI action path moved to `ActionRequest { action, policy }`, but the FFI -`ad_execute_action` wrapper initially constructed the cursor/focus policy (then -named `physical`, since renamed `ActionRequest::headed`) for every action. That -meant C, Swift, Python, Go, and Node consumers received focus-stealing and -cursor-moving behavior for actions that the CLI treats as headless by default. +FFI exposes both high-level ref commands and low-level native-handle calls. +They must not be described as having the same safety contract: only the +high-level ref path can reproduce CLI observation-to-action behavior. ## Guidance -FFI wrappers that mirror CLI actions must use the same default side-effect -contract as the CLI. For direct action execution, the default is headless: -no focus stealing and no cursor movement. +`ad_execute_by_ref` and its timeout variant use the core command path. They +load the ref map, resolve strictly, apply actionability, and compute policy +through `Action::base_interaction_policy` joined with the caller's explicit +policy. Ref actions, including `type`, have a headless base; explicit `press` +retains its focus fallback. With the headless policy, natural-input commands +use semantic accessibility delivery. A caller-selected headed policy makes +`click`, `right-click`, `type`, `clear`, and `scroll` physical-first, matching +the CLI `--headed` contract, while semantic-only commands remain semantic. -When a lower-level FFI consumer intentionally wants broader behavior, expose the -policy as an explicit ABI value. Do not hide it inside a wrapper default. +`ad_execute_action` and struct-based direct action entrypoints are deliberately +lower-level escape hatches. They operate on a caller-held native handle or +entry, may apply supplied policy verbatim, and do not claim RefStore-backed CLI +parity. Their safety docs must say so. -## Review Rule +## Prevention -Any change to `ActionRequest`, `InteractionPolicy`, or command preflight must -include a pass over `crates/ffi/src/actions/`. If the CLI and FFI can perform the -same action, they must document the same default and expose any divergence as an -explicit parameter. - -Behavioral parity is only half the FFI review: any structural change to a public -`repr(C)` type — adding, removing, or reordering fields, or growing a struct that -is embedded by value inside another — must also update the three-layer size pin -(Rust const assert, header `_Static_assert`, layout integration test). Size drift -in an embedded struct silently propagates to every outer struct that embeds it. +- Route new language-binding equivalents of CLI commands through the high-level + core command path. +- Test both the observed effect and reported mechanism for headed and headless + natural-input actions, plus every action with a non-headless base. +- Make any skipped boundary—strict resolution, preflight, or base-policy + elevation—explicit in the public FFI documentation. ## Related -- `best-practices/ffi-repr-c-struct-size-pinning.md` — the structural-parity companion: the full three-layer pinning protocol and the AdAction silent-growth incident that motivated it. +- [FFI repr(C) struct size pinning](ffi-repr-c-struct-size-pinning.md) +- [Build desktop actions as an observe-resolve-preflight-dispatch contract](playwright-grade-desktop-reliability-2026-06-02.md) diff --git a/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md b/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md index 4c9f67b..79ba3f0 100644 --- a/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md +++ b/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md @@ -1,103 +1,54 @@ --- -title: Which desktop gestures have a headless path (macOS) and why the command never decides -date: 2026-06-10 +title: Keep gesture capability in the platform adapter and policy in core +date: 2026-07-11 category: best-practices -module: crates/macos -problem_type: best_practice -component: macos-adapter +module: core actions and macos adapter +problem_type: architecture_pattern +component: tooling severity: high applies_when: - - Implementing or using a ref-action command and deciding whether it needs --headed - - A headless ref action returns POLICY_DENIED and you are unsure if that is correct - - Adding a Windows (UIA) or Linux (AT-SPI) adapter and mapping gesture commands - - Writing an E2E that drives a gesture and must assert the real effect, not ok:true -tags: - - interaction-policy - - headless-headed - - accessibility - - macos - - platform-adapter - - swiftui + - "Adding a new action or physical fallback" + - "Porting an action to Windows or Linux" + - "Explaining a headless policy failure" +tags: [interaction-policy, actions, macos, adapters, headless] --- -# Which desktop gestures have a headless path (macOS) and why the command never decides +# Keep gesture capability in the platform adapter and policy in core ## Context -Ref actions run in two modes (Playwright-style): **headless** (default — accessibility-only, no cursor, fail closed with `POLICY_DENIED`) and **headed** (`--headed` — permits cursor movement and focus stealing so physical fallbacks can complete). A recurring question when using or extending the tool is: *which* interactions actually work headlessly, and which inherently need `--headed`? Getting this wrong leads to either surprise `POLICY_DENIED` errors or the false assumption that a command physically interacted when it only did an AX no-op. - -The answer is **per-gesture and per-platform**, because a gesture is headless-capable only when the OS accessibility API exposes a semantic action for it. On macOS the reality is: - -| Gesture / control | Headless path (macOS) | Notes | -|-------------------|-----------------------|-------| -| `click`, `set-value`, `type`, `check`, `select`, `scroll`, `expand`, `toggle`, … | yes | semantic AX actions; the default and most reliable surface | -| `double-click` | partial | `AXOpen` works headless on elements that advertise it (Finder/list/outline rows, table cells). Falls back to `--headed` only for gesture-only targets with no `AXOpen`. | -| `triple-click` | no | macOS exposes no triple-click action; it is purely 3 physical clicks → `--headed` only | -| `hover` | no | hovering *is* moving the cursor over an element; no AX equivalent | -| `drag` / drop | no | dragging *is* a cursor press-move-release; no general AX drag. Native cross-app drop needs the OS `NSDraggingSession`/pasteboard protocol that synthetic CGEvents cannot start (works for same-view source-tracked gestures and web/Electron mouse-DnD) | -| menu bar | enumerate / open | readable and openable via `snapshot --surface menubar`; **SwiftUI `CommandMenu` items accept AXPress but do not route to their action closure** (a SwiftUI limitation, like its `Slider`) — native AppKit menu items fire. `.contextMenu` item selection works. | -| SwiftUI `Slider` / `Stepper` / `DisclosureGroup` | no | not AX-actionable; the native AppKit `NSSlider`/`NSStepper` equivalents are (so `set-value`/`expand` work on those) | +Whether a platform can perform an intent semantically is platform-specific; +whether the caller permitted focus stealing or cursor movement is a portable +side-effect contract. Mixing those decisions into a CLI command would make +future adapters copy macOS assumptions. ## Guidance -1. **The command is platform-agnostic; the adapter owns headless-vs-physical.** A ref-action command builds an `Action` (e.g. `Action::TripleClick`) and calls `adapter.execute_action`. The macOS adapter's dispatch decides how to perform it (AX action vs policy-gated CGEvent). Core never encodes platform behavior — it cannot, because core may never import a platform crate (CI enforces this with `cargo tree -p agent-desktop-core`). +Core creates an `ActionRequest` with the action's least-permissive base policy. +Semantic actions, including `type`, start strictly headless. Explicit `press` +may use focus fallback. Headed mode can elevate policy but may not weaken it. -2. **A new platform that exposes a headless path lights it up automatically — adapter-only change.** If a future Windows (UIA) or Linux (AT-SPI) adapter has a headless action for `double-click`/`triple-click`, it maps the `Action` there and the command succeeds headlessly on that platform with **zero change to the command or core**. The `InteractionPolicy` flows through the request; each adapter honors it per its own capabilities. The agent just sees success (or `POLICY_DENIED` → retry `--headed`) — it never needs to know the platform. +The adapter chooses the requested legal implementation: strict headless uses +semantic accessibility APIs, while headed natural-input commands prefer +physical delivery. `hover` and +`drag` are physical pointer commands by definition and require a cursor-moving +policy before resolving or moving the pointer. A semantic reorder capability, +if a future platform offers one, is a distinct action contract rather than a +hidden change to physical drag. -3. **`hover`/`drag`/`mouse-*` are modeled as raw cursor gestures, not semantic `Action`s** (they call `adapter.mouse_event`/`adapter.drag` with coordinates). They stay physical on every platform by design, because hovering/dragging *are* cursor operations universally. A semantic drag (AX reorder) would be a *new* `Action`, not a change to `drag`. Headless gestures return `POLICY_DENIED` before resolving refs or moving the cursor. Under `--headed`, ref-addressed gestures first ensure the target app is frontmost (`focus_for_physical_input`, gated on `InteractionPolicy::allow_focus_steal`; the response reports `"focused": true` when confirmed — already-frontmost apps skip the raise). `--xy` input never focuses — the caller owns the target there. The chain's physical click fallback goes one step further than the gesture focus path: it also raises the target element's **own window** (AXRaise, AXMain fallback) before posting events, because CGEvents land on the topmost window at the click point and app-frontmost alone is not enough when the element lives in a background window of that app. +Tests must verify the observed effect, not merely native API success. Native +controls and accessibility implementations vary; a successful AX call is not +proof that an application executed its handler. -4. **`POLICY_DENIED` on a headless gesture is correct, not a bug** — it is the fail-closed signal that the headless AX path is unavailable and the caller must opt into `--headed`. Never widen the default policy to make it disappear. +## Prevention -5. **When verifying a gesture in a test, observe the real effect, never the command's `ok:true`.** An AX action can report `verified_press:succeeded` while the underlying control's handler never ran (SwiftUI `CommandMenu`, `Slider`). Re-read the target's state to confirm. - -## Why This Matters - -- It keeps the **headless-first reliability guarantee** honest: the tool only claims a headless effect when the OS actually provides one, and fails closed otherwise. -- It preserves **cross-platform extensibility**: the same 54-command surface works identically across macOS/Windows/Linux, and each adapter contributes whatever headless capability its platform has — without touching the command layer. -- It prevents the **vacuous-success trap**: assuming `ok:true` means the gesture happened, when an AX action succeeded at the API layer but the control ignored it. - -## When to Apply - -- Deciding whether a command needs `--headed` (consult the matrix; `POLICY_DENIED` headless = needs `--headed` or has no headless path). -- Implementing a new platform adapter: map each `Action` to the platform's best headless path first, gate physical fallbacks on the policy, and return `POLICY_DENIED` when only a physical gesture would work. -- Building fixtures/tests: use **native AppKit** controls (`NSSlider`/`NSStepper`) when you need a genuinely AX-actionable target; SwiftUI equivalents will not validate the AX path. - -## Examples - -Double-click is headless-capable only via `AXOpen`: - -```bash -# Finder list row advertises AXOpen -> headless double-click opens it -agent-desktop double-click @e12 - -# A plain button with no AXOpen -> headless fails closed, needs --headed -agent-desktop double-click @e3 # POLICY_DENIED -agent-desktop --headed double-click @e3 # physical double-click completes -``` - -The macOS dispatch gates the physical path on the policy (so it is reachable only under `--headed`): - -```rust -// crates/macos/src/actions/chain_defs.rs -pub(crate) fn double_click(el, _caps, policy) -> Result<(), AdapterError> { - if ax_helpers::has_ax_action(el, "AXOpen") && ax_helpers::try_ax_action(el, "AXOpen") { - return Ok(()); // headless AX path - } - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2, policy) // gated; POLICY_DENIED headless -} -``` - -The menu bar is readable/openable but SwiftUI `CommandMenu` items do not fire via AX: - -```bash -agent-desktop snapshot --app MyApp --surface menubar # enumerates the full menu bar with refs -# clicking a SwiftUI CommandMenu item: verified_press:succeeded, but the action closure never runs -# native AppKit menu items DO fire; .contextMenu item selection DOES fire -``` +- Put shared policy and action vocabulary in core; do not import platform + capability details into core. +- Keep adapter dispatch explicit about semantic versus physical mechanisms. +- Add a real-fixture assertion for every new physical fallback. +- Return `POLICY_DENIED` when the requested fallback is not authorized. ## Related -- `best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md` — why `type` keeps a `focus_fallback` base and the shared helper takes the caller's policy. -- `best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md` — FFI and CLI run the same `ref_action::execute_resolved` ladder, so policy semantics stay identical. -- `best-practices/playwright-grade-desktop-reliability-2026-06-02.md` — strict late resolution, actionability preflight, and the headless-first contract this builds on. -- `best-practices/abort-state-guidance-multi-step-physical-input.md` — what happens when a physical drag sequence fails mid-flight: the button is released back at the origin (never the unreached destination) and the error describes the end state. +- [Preserve command policy semantics during refactor](preserve-command-policy-semantics-during-refactor-2026-05-12.md) +- [Document pointer actions from their own reliability pipeline](../documentation-gaps/hover-drag-skip-the-actionability-battery.md) diff --git a/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md b/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md index 5f98d02..23f47d9 100644 --- a/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md +++ b/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md @@ -1,295 +1,107 @@ --- -title: Playwright-grade desktop reliability contract -date: 2026-06-02 -last_updated: 2026-06-10 +title: Build desktop actions as an observe-resolve-preflight-dispatch contract +date: 2026-07-11 category: best-practices -module: crates/core, crates/macos, crates/ffi, src -problem_type: best_practice +module: core reliability contracts +problem_type: architecture_pattern component: tooling severity: high applies_when: - - Ref resolution, action dispatch, wait semantics, session scope, trace output, or FFI ref-action paths change - - Platform adapters add live state, bounds, action, or resolution behavior - - CLI or FFI command paths are refactored around shared helpers - - Adapter conformance tests are added or changed -tags: - - reliability - - ref-actions - - strict-resolution - - actionability - - capabilities - - wait - - sessions - - tracing - - ffi-parity + - "Adding or changing a ref-based command" + - "Changing platform resolution, actionability, or physical fallback" + - "Adding CLI or FFI automation surfaces" +tags: [reliability, refs, actionability, sessions, ffi, tracing] --- -# Playwright-grade desktop reliability contract +# Build desktop actions as an observe-resolve-preflight-dispatch contract ## Context -Playwright is reliable because actions flow through a consistent ladder: -resolve a locator, prove it is actionable, wait when the state is changing, and -fail with structured recovery when the target is stale or ambiguous. Desktop -automation cannot copy browser semantics directly, but it can use the same -engineering shape. - -The reliability enhancement moved agent-desktop toward that model without -turning the CLI into a browser tool. Core owns the contract. Platform adapters -provide native evidence. CLI and FFI callers share the same ref-action semantics -unless a lower-level API explicitly documents that it bypasses the strict ref -path. - -This learning updates the original contract doc with the implementation details -that matter for future changes. +Desktop accessibility trees are live, mutable, and platform-specific. A ref +from an earlier observation is evidence, not a native handle. Reliable +automation therefore has to make each boundary explicit and reject uncertainty +instead of guessing. ## Guidance -### Keep Hidden Identity Evidence +### 1. Scope observation before storing it -Snapshot output may hide pixel bounds for compact, agent-friendly responses, but -the persisted ref identity still needs bounds evidence. The safe pattern is: +A snapshot is stored in either the global namespace or a selected session. +Explicit `--session` wins over `AGENT_DESKTOP_SESSION`; no process-global +“current session” exists. Qualified refs embed their producing snapshot. Bare +refs are accepted only with an explicit snapshot ID. Never search another +session when the selected namespace has no match. -1. Ask the adapter for bounds when building a ref-bearing snapshot. -2. Allocate refs and store `bounds_hash` from that full internal tree. -3. Strip visible `bounds` only from the returned output when the caller did not - request them. -4. Keep `bounds_hash` in the refmap so later actionability can detect stale refs. +### 2. Re-resolve strictly at action time -Do not let presentation options erase identity evidence. Compact output is a -serialization concern, not a weaker ref contract. +Load the saved `RefEntry`, then let the adapter re-identify the live element +from role, process generation, source, stable text identity, geometry, path, +and native evidence. Zero matches is `STALE_REF`; more than one plausible match +is `AMBIGUOUS_TARGET`. Mutable field values are not stable identity. -### Treat Mutable Values As Volatile Identity +### 3. Separate actionability from dispatch -Strict ref resolution should separate stable labels from mutable control values. -Text field content, selected combobox values, slider values, and incrementor -values can change between snapshot and action without changing the target -element. Core should own that role-conditional identity policy, and platform -adapters should only supply native candidates, live attributes, and primitive -actions. +Ref actions use the shared auto-wait and live actionability checks. Each command +owns its least-permissive base interaction policy: ref actions, including +typing, start headless, while explicit ref-targeted key presses retain their +focus fallback. Headed mode is an opt-in policy elevation, not a different +resolution path. A failed preflight must say why and preserve retry safety. -### Centralize Strict Ref Actions +On macOS, headless natural-input commands use semantic accessibility delivery. +Headed `click`, `right-click`, `type`, `clear`, and `scroll` prefer physical +delivery; semantic-only commands remain semantic. Double- and triple-click, +hover, and drag are physical-only and fail closed without headed permission. +Results expose the delivered step mechanism so tests and callers can verify +this split instead of inferring it from a successful response. -Ref actions must pass through the same ladder: +Pointer commands are a separate physical family. They resolve a live point, +verify visibility, geometry stability, and hit-test receipt, then require an +explicit cursor-moving policy before dispatch. Their terminal errors are not +silently converted into retries. -1. Load the refmap from the caller's session. -2. Resolve the ref with strict platform identity checks. -3. Return `STALE_REF` when the old element no longer matches. -4. Return `AMBIGUOUS_TARGET` when multiple candidates match. -5. Run live actionability checks before adapter dispatch. -6. Dispatch the adapter action chosen by the command's policy. -7. Release native handles through the adapter boundary. +### 4. Track delivery honestly -The CLI path and FFI strict ref-action path should share this model. A lower -level native-handle FFI call may remain available, but it must be documented as -lower-level: it acts on a handle the caller already resolved and therefore does -not have ref identity evidence to re-check. +Every adapter error carries structured delivery semantics. If a physical action +has started, an error must not claim a safe retry. Multi-step input owns its +cleanup guard and reports the best-known final state. Trace failures are +observability failures; they must not change whether an action was delivered. -Command-specific policy still belongs at the command edge. Centralization should -remove duplicated mechanics; it should not flatten `ActionRequest` choices or -post-condition verification. +### 5. Keep every transport on the same core contract -### Treat Actionability as a Core Contract +CLI, batch, and high-level FFI ref actions use the same strict resolution, +policy elevation, actionability, and envelope semantics. Low-level FFI native +handle calls are intentionally escape hatches and must document that they skip +those guarantees. ABI structs are versioned and layout-pinned alongside the +committed C header. -Actionability is not an adapter-specific convenience. Core decides whether a -resolved ref is safe to act on using native evidence supplied by the adapter: +### 6. Verify at three layers -- visibility from live bounds -- stability from live bounds hash versus snapshot bounds hash -- enabled state from live state -- supported action from live or snapshot actions -- interaction policy from the command request -- editability from role and supported actions - -Unavailable native evidence should be `unknown`, not a false failure, when the -platform cannot provide it. A non-empty live action list can narrow capabilities; -an empty transient live action list should not erase snapshot capabilities. - -### Own Capability Vocabulary in Core - -Supported action names are part of the cross-platform contract, not incidental -strings. Put the canonical vocabulary, action-to-capability mapping, role -defaults, and membership helpers in one core module. Actionability, ref -allocation, `is` predicates, FFI tests, and platform adapters should refer to -that vocabulary instead of re-declaring string literals. - -Platform adapters may discover capabilities differently. macOS maps AX actions -and settable attributes; Windows should map UIA patterns; Linux should map -AT-SPI actions and states. Those native differences must converge into the same -core capability names before core evaluates actionability. - -Do not keep pass-through wrappers such as `Action::semantic_capabilities()` when -call sites can use the canonical capability helper directly. Thin wrappers make -future command additions touch multiple files without clarifying ownership. - -### Keep Waits Bounded and Honest - -Wait commands must not hide permanent adapter failures behind timeout polling. -The reliable split is: - -- Retry transient resolution states such as stale, not found, ambiguous, or - timeout while the caller's timeout budget remains. -- Propagate permanent adapter errors immediately. -- Preserve the last observed retryable state in timeout details. TIMEOUT - details carry a `kind` discriminant: `"wait_timeout"` for wait-loop expiry - (predicate, timeout_ms, last observed state) and `"chain_deadline"` for a - chain step expiring mid-increment or mid-disclosure (observed value or - expanded state, plus a `mutated` flag) — agents key on `kind` before - inspecting other fields. -- For `wait --element` without `--snapshot`, refresh the latest-ref cache on a - bounded cadence; for a fixed `--snapshot`, treat missing refs as invalid input - instead of silently switching snapshots. -- `--predicate actionable` checks readiness for a specific action via - `--action` (`click` default, `type`, `set-value`, `clear`); each name maps to - the exact request its real command runs — policy included — through explicit - per-name arms, so an unknown name errors instead of silently inheriting a - default policy. - -This keeps `wait` useful for changing desktop state without making it a blanket -error suppressor. - -Resolver deadline checks should also have one owner. If both root selection and -tree traversal need the same timeout error and remaining-budget logic, share a -small resolver-deadline helper instead of copying the deadline branch into each -module. That keeps `wait --element` bounded through every native AX read without -growing resolver files toward the size limit. - -### Make Tracing Diagnostic, Not Behavioral - -Trace output belongs in the requested JSONL file and never in stdout. It must be -safe for agents to use with machine-readable command output: - -- create private trace files -- reject symlink trace paths on Unix -- redact sensitive fields by key name (`text`/`value`/`name`/…); note that - `message` is NOT key-redacted, so raw caller arguments must never be - interpolated into it — see - `docs/solutions/conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md` -- let `--trace-strict` fail on setup and pre-action trace writes -- keep post-action success trace writes best-effort after the desktop mutation - has already happened -- with a trace-enabled session manifest from `session start`, write per-process - JSONL segments under `sessions/<id>/trace/<pid>-*.jsonl` automatically; - `--trace <path>` still overrides to one file for CI or one-offs - -Reporting a successful desktop mutation as failed because the final trace write -failed is worse than losing that final diagnostic event. - -Do not describe the current trace as equivalent to Playwright Trace Viewer. It -is a JSONL diagnostic stream, not a bundled timeline artifact with before/after -snapshots, screenshots, source metadata, and an inspection UI. A Playwright-like -desktop trace should be introduced as a separate artifact format if needed. - -### Keep the Foundation Cross-Platform - -Core owns the contract; adapters own native evidence. Windows and Linux should -not fork CLI semantics. UIA and AT-SPI implementations must map their native -identity fields into the same `RefEntry` concepts: role, name, value, -description, state, bounds, supported actions, source surface, root ref, and -tree path. - -Actionability should prefer one native live-state read that returns state, -bounds, and supported actions together. Platform adapters may fall back to -separate reads, but the CLI behavior must remain identical: empty transient -action reads do not erase snapshot capabilities, while a non-empty live action -set that lacks the required action can block dispatch. - -Do not add macOS-only assumptions to core to solve a macOS bug. If a behavior is -part of the CLI or FFI contract, core should express the contract and each -adapter should provide native evidence or return a structured unsupported error. - -### Triage Review Findings Against Project Constraints - -Not every simplification suggested by a reviewer is a good simplification. In -this repo: - -- One command per file is intentional; collapsing small command wrappers fights - the project structure. -- Keeping helpers extracted is correct when inlining would push a file toward - the 400 LOC limit or duplicate a shared contract. -- Centralizing strict ref-action behavior in core is better than moving it only - into FFI because CLI and FFI parity is the invariant. -- Splitting tests by owner is better than letting one broad test file become the - dumping ground for ref maps, ref stores, sessions, and legacy migration. -- Windows/Linux portability means platform-neutral contracts in core, not - pretending current macOS AX evidence already exists on other platforms. - -False positives should be marked as such. Do not add code only to satisfy a -review comment that contradicts the architecture. +Use deterministic core tests for identity, policy, retry, and delivery rules; +C/C++ header compilation and layout tests for FFI; and a permissioned release +binary against the SwiftUI fixture for native observation and harmless +interaction. Real-app ignored tests protect platform seams such as window +identity and accessible-name agreement. Headed and headless fixture scenarios +must assert both the observed effect and the reported delivery mechanism. ## Why This Matters -The user-visible failure modes are severe: +The strongest failure mode is a plausible success against the wrong live +element. This design makes uncertainty visible: callers receive a structured +stale, ambiguous, policy, timeout, or delivery-unknown result instead of a +best-effort click. -- acting on a stale ref can mutate the wrong UI element -- hidden bounds in compact snapshots can accidentally weaken future stale-ref - detection -- wait loops that swallow permanent errors waste time and obscure permissions or - adapter failures -- FFI and CLI divergence makes language bindings less reliable than the command - line -- trace failures after mutation can make successful desktop actions look failed +## Prevention -Playwright's reliability comes from a predictable action pipeline. agent-desktop -needs the same predictability across desktop platforms, even though native -accessibility APIs expose weaker and less uniform evidence than a browser DOM. - -## When to Apply - -Any change to ref resolution or action dispatch must include tests for: - -- stale ref rejection -- ambiguous target rejection -- actionability failure before dispatch -- retrying waits that honor timeout and report last observed state -- session isolation -- FFI parity when the behavior is exposed through C ABI -- compact snapshot output preserving hidden identity evidence in the refmap -- capability mappings using the central vocabulary instead of copied strings -- resolver deadlines applied before native reads and shared by resolver modules -- trace strictness without post-mutation false failures - -If a platform needs a coordinate fallback, the fallback must be explicit and -lower confidence. Do not silently replace a failed semantic action with a pixel -click. - -## Examples - -Snapshot construction should request identity bounds internally, then hide only -presentation bounds when needed: - -```rust -let identity_opts = opts.with_ref_identity_bounds(); -let tree = adapter.get_tree(&window, &identity_opts)?; -let (tree, refmap) = allocate_refs(tree, opts)?; -let tree = strip_ref_bounds_when_hidden(tree, opts); -``` - -Ref action execution should keep the command-selected policy while centralizing -the strict ladder: - -```rust -let (entry, handle) = resolve_ref_with_context(ref_id, snapshot_id, adapter, context)?; -check_actionability_with_trace(ref_id, &entry, handle.handle(), adapter, &request, context)?; -let result = adapter.execute_action(handle.handle(), request)?; -``` - -The platform adapter should expose a single live element read when possible: - -```rust -LiveElement { - state: live_state, - bounds: live_bounds, - available_actions: live_actions, -} -``` - -Windows UIA and Linux AT-SPI adapters can fill those fields differently, but the -core actionability decision must stay the same. +- Put platform-neutral rules in core; adapters translate native evidence only. +- Add a regression test at the boundary that failed, then run the fixture or a + safe read-only native probe for adapter changes. +- Keep docs and help text in the same change as a command-contract change. +- Do not add a fallback that bypasses strict resolution, policy, or delivery + accounting merely to improve apparent success rate. ## Related -- [Keep FFI action policy aligned with CLI action policy](keep-ffi-action-policy-aligned-with-cli-2026-05-12.md) -- [Preserve command policy semantics during shared ref-action refactors](preserve-command-policy-semantics-during-refactor-2026-05-12.md) -- [Guard OS-reordered resources with an identity fingerprint, not a raw index](identity-fingerprint-against-os-reorder-2026-04-16.md) -- [Progressive snapshot contract fixes after review](../logic-errors/progressive-snapshot-review-contract-2026-04-16.md) +- [Keep progressive snapshots namespace-scoped and ref-safe](../logic-errors/progressive-snapshot-review-contract-2026-04-16.md) +- [Document pointer actions from their own reliability pipeline](../documentation-gaps/hover-drag-skip-the-actionability-battery.md) +- [Real-app tests are the platform-adapter gate](real-app-tests-are-the-platform-adapter-gate.md) +- [Keep raw caller arguments out of trace-reachable error messages](../conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md) diff --git a/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md b/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md index b86064e..eec91b8 100644 --- a/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md +++ b/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md @@ -1,7 +1,7 @@ --- title: Preserve command policy semantics during shared ref-action refactors date: 2026-05-12 -last_updated: 2026-06-10 +last_updated: 2026-07-12 category: best-practices module: crates/core, crates/macos problem_type: best_practice @@ -25,43 +25,42 @@ tags: ## Context The unified ref-action helper removed repeated `resolve_ref + execute_action + to_value` -boilerplate across commands. That cleanup was correct, but two command-specific -policy choices were accidentally flattened during earlier review rounds: +boilerplate across commands. That cleanup was correct, but command-specific +policy and verification choices were accidentally flattened during earlier +review rounds: - `clear` initially reported success after AXValue writes without verifying app-observable state, so web-backed controls could remain unchanged. -- `type` initially rejected non-ASCII text after an AXValue failure instead of - using the explicit focus-permitted paste path. +- `type` and `clear` lost the distinction between default headless delivery and + explicitly headed natural input. Both commands still compiled and returned structured responses. The regression -was semantic: web-backed fields can report AXValue success while leaving the app's -JS model unchanged, so post-condition verification is part of the command -contract. +was semantic: the selected mechanism and its app-observable post-condition are +part of the command contract, not interchangeable helper details. ## Guidance Shared ref-action dispatch should only remove repeated mechanics. It must not choose the `InteractionPolicy` for a command, and it must not drop post-condition verification that decides whether fallback steps should run. -Default CLI ref commands must stay headless: no focus stealing, no cursor -movement, and no synthetic keyboard or pasteboard use unless an explicit policy -path opted into it. +Default CLI ref commands start from their least-permissive base policy. The +global `--headed` flag joins an explicit focus-and-cursor permission onto that +base; it changes delivery preference only for commands with a natural physical +equivalent. Each command owns its policy: -- Use `ActionRequest::headless` when the command is purely semantic AX work and - must not focus, move the cursor, or synthesize input. -- Use `ActionRequest::focus_fallback` only for APIs that have explicitly opted - into focus-changing behavior, such as CLI `type` after AXValue failure or FFI - callers selecting `AD_POLICY_KIND_FOCUS_FALLBACK`. -- Use `ActionRequest::headed` (formerly `physical`) only for explicit physical - interaction commands or FFI callers selecting `AD_POLICY_KIND_HEADED`. Ref - commands no longer select it directly — the global `--headed` flag upgrades - any command's base policy to headed via `CommandContext::request`. Note the - headed physical path's side effects go beyond app-level focus stealing: the - physical click fallback also raises the target element's own window (AXRaise, - AXMain fallback) before posting events, gated on the same - `allow_cursor_move && allow_focus_steal` policy as the rest of that path. +- `Action::base_interaction_policy` is headless for ref actions, including + `type` and `clear`; explicit `press` retains its focus-fallback base. +- Headless `type` writes `AXSelectedText`, while headed `type` uses PID-targeted + physical text delivery. Headless `clear` uses verified `AXValue`; headed + `clear` prefers the focus-and-keyboard path before the semantic step. +- Headed `click`, `right-click`, and `scroll` likewise prefer their physical + mechanisms. Semantic-only commands stay semantic, and physical-only commands + still fail closed without headed authorization. +- High-level FFI ref actions join the caller's explicit policy with the same + action base. Low-level native-handle entrypoints remain escape hatches and do + not imply CLI policy parity. Do not infer policy from the fact that a command consumes a ref. `click`, `check`, `expand`, `collapse`, `scroll-to`, `clear`, and `type` all consume refs, @@ -79,20 +78,21 @@ constructs a default policy for many commands, treat that as a regression risk. Backfill tests at the command or adapter boundary for commands whose policy is part of correctness: -- `clear` must dispatch headlessly from the CLI and must verify AXValue writes - before reporting success. -- `type` must attempt AXValue first from the CLI, then use only its explicit - focus-fallback tier when AXValue cannot update the target. -- FFI policy-specific tests must prove focus and physical paths are available - only when the caller explicitly selects that policy. +- `clear` must report semantic delivery headlessly and physical delivery when + headed, while verifying the resulting empty value in both modes. +- `type` must report `AXSelectedText` delivery headlessly and PID-targeted + physical delivery when headed. +- FFI policy-specific tests must prove the same headless-versus-headed + mechanism split as the CLI high-level path. - A generic ref-action helper should preserve both the `Action` variant and the caller's `InteractionPolicy`. For AX value writes, treat "set returned success" as incomplete evidence on web-backed controls. Read back the value when the field is not secure; a -mismatch must be a failed step so the next command-specific fallback can run. +mismatch must be a failed step so the command-specific chain can continue or +report an honest failure. ## Related -- `best-practices/exhaustiveness-guards-over-catch-alls-in-policy-mirrors.md` — the same risk class (per-case policy flattened by a structural abstraction) from the string-keyed dispatch-mirror angle: named arms plus machine-derived guard tests where the compiler cannot enforce exhaustiveness. -- `best-practices/macos-gesture-headless-capability-2026-06-10.md` — the per-gesture policy table whose explicitness this guidance preserves. +- [Exhaustiveness guards over catch-alls in policy mirrors](exhaustiveness-guards-over-catch-alls-in-policy-mirrors.md) — named arms and guard tests protect string-keyed mirrors where the compiler cannot prove coverage. +- [macOS gesture headless capability](macos-gesture-headless-capability-2026-06-10.md) — the per-gesture policy table whose explicitness this guidance preserves. diff --git a/docs/solutions/best-practices/real-app-tests-are-the-platform-adapter-gate.md b/docs/solutions/best-practices/real-app-tests-are-the-platform-adapter-gate.md new file mode 100644 index 0000000..220a102 --- /dev/null +++ b/docs/solutions/best-practices/real-app-tests-are-the-platform-adapter-gate.md @@ -0,0 +1,64 @@ +--- +title: Make permissioned fixture and real-app checks the adapter gate +date: 2026-07-11 +category: best-practices +module: platform adapter verification +problem_type: best_practice +component: testing_framework +severity: high +applies_when: + - "Changing a platform adapter's tree, resolution, window, input, or accessibility code" + - "Changing release-binary integration behavior" + - "Fixing a bug that a mock adapter could not expose" +tags: [e2e, fixture-app, accessibility, macos, regression, adapters] +--- + +# Make permissioned fixture and real-app checks the adapter gate + +## Context + +Core unit tests prove shared contracts, but they cannot prove that a native +accessibility API exposes the same tree, identity, or input behavior on a real +desktop. A mock cannot reproduce a bad AX-to-window bridge or disagreement +between independently implemented native readers. + +## Guidance + +The macOS adapter has two complementary native gates: + +- `tests/e2e/run.sh` builds a temporary SwiftUI fixture, drives the release + binary, and independently observes each harmless effect. It requires macOS, + a release binary, and Accessibility permission; it exits with a clear + prerequisite failure when one is unavailable. +- `src/tests/snapshot_test.rs` contains three `#[ignore]` Finder probes: + snapshot window identity must agree with `list-windows`; a reported button + name must be findable by that same name; and a fresh find ref must re-resolve + through `get`. The registration test prevents these guards from silently + disappearing. + +Run normal tests first, then run the native gate on a permissioned machine for +any adapter change. Use the fixture for mutation coverage. Treat real user apps +as observation-only unless a narrowly scoped, reversible interaction is +explicitly authorized. + +## Why This Matters + +The fixture provides deterministic, safe interaction coverage. Finder probes +cover native seams that the fixture cannot emulate: AX window identity, +accessible-name derivation, and strict re-resolution against a real system +application. Together they catch both behavior regressions and accidental test +deletion without treating a mock as a platform oracle. + +## Prevention + +- For every native regression, add a deterministic unit test where possible + and a fixture or safe real-app assertion at the affected seam. +- Verify effects independently of an `ok: true` response. +- Keep native tests opt-in and prerequisite-aware; never make them operate on + user data or depend on an arbitrary foreground application. +- Record a skipped native gate as skipped, not green. + +## Related + +- [Build desktop actions as an observe-resolve-preflight-dispatch contract](playwright-grade-desktop-reliability-2026-06-02.md) +- [Guard OS-reordered resources with an identity fingerprint](identity-fingerprint-against-os-reorder-2026-04-16.md) diff --git a/docs/solutions/conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md b/docs/solutions/conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md index 11da121..b17d758 100644 --- a/docs/solutions/conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md +++ b/docs/solutions/conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md @@ -69,9 +69,12 @@ return Err(AdapterError::new( )); ``` -Fixed sites (commit `b35eea9`): `crates/core/src/commands/wait_timeout.rs` (window/text/selector builders), `crates/macos/src/actions/extras.rs` (`select_value` list arm, `wait_for_value`, `option_not_found`), `crates/macos/src/actions/ax_helpers.rs` (`number_cf_from_str`). Regression guard: `wait_text_timeout_message_omits_raw_text_from_trace_segment` asserts a unique marker passed as `wait --text` never appears in the written segment. +The regression guard `wait_text_timeout_message_omits_raw_text_from_trace_segment` +asserts that a unique marker passed as `wait --text` never appears in the +written trace segment. Keep that test focused on the observable privacy +contract, not a historical list of implementation sites. ## Related -- `docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md` — the broad tracing/reliability contract; its redaction guidance is refined by this convention (key-name redaction does not cover free-text `message` content). +- [Playwright-grade desktop reliability contract](../best-practices/playwright-grade-desktop-reliability-2026-06-02.md) — the broad tracing/reliability contract; this convention refines its redaction guidance. - `crates/core/src/trace_sanitize.rs` — the `SENSITIVE_KEYS` allowlist this convention works around. diff --git a/docs/solutions/documentation-gaps/hover-drag-skip-the-actionability-battery.md b/docs/solutions/documentation-gaps/hover-drag-skip-the-actionability-battery.md new file mode 100644 index 0000000..c4ddac7 --- /dev/null +++ b/docs/solutions/documentation-gaps/hover-drag-skip-the-actionability-battery.md @@ -0,0 +1,55 @@ +--- +title: Document pointer actions from their own reliability pipeline +date: 2026-07-11 +category: documentation-gaps +module: crates/core pointer actions +problem_type: documentation_gap +component: documentation +severity: high +applies_when: + - "Documenting hover or drag behavior" + - "Adding a ref action that resolves a screen point" + - "Explaining timeout or occlusion failures to an agent caller" +tags: [pointer-actions, hover, drag, actionability, documentation] +--- + +# Document pointer actions from their own reliability pipeline + +## Context + +`hover` and `drag` are physical, cursor-moving commands. They do not use the +same dispatch ladder as semantic ref actions, so prose that says all ref +actions have identical actionability behavior becomes wrong as the pointer +path evolves. + +## Guidance + +Follow the code path, not a command-family generalization: + +`hover` and `drag` enter `pointer_action::resolve_point_with_deadline`. For a +ref target it retries strict resolution within the command deadline and reads +live bounds and state. It scrolls a non-visible target into view once, verifies +valid bounds, requires a stable bounds hash across attempts, and then performs +the `receives_events` hit-test before physical input. + +An occluded, invalid, or permanently non-visible target returns a structured +terminal error. Only transient resolution and stability conditions consume the +retry budget. `hover` and `drag` also require headed policy because they move +the cursor; headless mode must reject them before any physical input. + +Do not describe these commands as either “full semantic actionability” or +“occlusion only.” Their contract is a dedicated point-resolution pipeline. + +## Prevention + +- Document pointer commands as a separate family and link to + `crates/core/src/commands/pointer_action.rs`. +- When changing its checks, update the agent-facing interaction reference in + the same change and add a regression test for the terminal/retry boundary. +- Keep the final physical dispatch separate from resolution so an error can + state whether delivery may have started. + +## Related + +- [Playwright-grade desktop reliability contract](../best-practices/playwright-grade-desktop-reliability-2026-06-02.md) +- [Abort-state guidance on multi-step physical input errors](../best-practices/abort-state-guidance-multi-step-physical-input.md) diff --git a/docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md b/docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md index b3b8be8..3e4fae4 100644 --- a/docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md +++ b/docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md @@ -1,70 +1,67 @@ --- - -## title: Progressive snapshot contract fixes after review (skeleton reset, drill window, validation, depth) -date: 2026-04-16 +title: Keep progressive snapshots namespace-scoped and ref-safe +date: 2026-07-11 category: logic-errors -module: agent-desktop-core +module: crates/core snapshot and refs problem_type: logic_error component: tooling symptoms: - - Repeated `snapshot --skeleton` runs grew `ref_count` and retained stale drill-down refs because skeleton reused a merged on-disk `RefMap` instead of replacing it. - - `snapshot --root @eN` returned `window.id` empty and a synthetic title, breaking the same JSON contract as window-rooted snapshots. - - Malformed `--root` values surfaced as `STALE_REF` after a failed map lookup instead of `INVALID_ARGS` from format validation. - - Deep non-skeleton subtree builds could return `None` at `ABSOLUTE_MAX_DEPTH` with no truncation marker, so agents saw missing branches without explanation. + - "A drill-down mutates refs outside its requested subtree or reports a window unrelated to its root ref." + - "Malformed or cross-namespace refs are classified as stale instead of invalid input." + - "Skeleton output hides truncation or creates refs that cannot be resolved safely." root_cause: logic_error resolution_type: code_fix severity: high -tags: - - progressive-snapshot - - skeleton - - drill-down - - refmap - - snapshot - - macos - - agent-contract +tags: [snapshot, drill-down, refs, sessions, skeleton, contracts] +--- -# Progressive snapshot contract fixes after review (skeleton reset, drill window, validation, depth) +# Keep progressive snapshots namespace-scoped and ref-safe ## Problem -A focused review of the `feat/progressive-skeleton-traversal` work found several **contract and state** issues: skeleton mode did not behave like a clean refresh of overview refs, drill-down responses did not carry real window identity, invalid `--root` strings were classified like missing refs, and the macOS tree builder could silently stop at the absolute depth cap. Follow-up work aligned behavior with agent expectations and locked the behavior with tests and docs. - -## Symptoms - -- Refmap growth and stale drill refs after multiple skeleton snapshots (merged load + selective removal). -- Drill-down JSON with empty `window.id` and generic title. -- `bad-ref` style `--root` values reported as stale rather than invalid input. -- Possible empty/missing subtrees at extreme depth without `children_count` or a boundary node. - -## What Didn't Work - -- **Treating skeleton as incremental merge** — convenient for caching, but agents and token budgets assume a fresh overview map unless documented otherwise. -- **Synthetic `WindowInfo` only** — fast to ship, but breaks any workflow that keys on `window.id` or correlates with `list-windows`. -- **Lookup-before-validate for `--root`** — reuses `stale_ref` for every missing key, including syntactically invalid IDs. +Progressive observation has two different operations: a full window snapshot +creates a new ref map, while `snapshot --root` replaces only the selected +subtree in an existing snapshot. Treating either as an unconstrained merge +makes old refs appear valid after their evidence has changed. ## Solution -1. **Full refmap replace on window snapshot** — `snapshot::build` always starts from `RefMap::new()` before `allocate_refs`, then `run` persists the result. Skeleton mode still passes `TreeOptions.skeleton` into `get_tree` for shallow traversal and boundary labeling; it no longer rehydrates prior refs for the overview path. -2. **Resolve real window for drill-down** — `snapshot_ref::run_from_ref` calls `adapter.list_windows` and picks the window matching `entry.pid`, with a fallback to the previous synthetic `WindowInfo` if listing fails. -3. **Validate `--root` early** — `commands/snapshot::execute` calls `validate_ref_id(root)` before `run_from_ref`, so malformed IDs return `invalid_input` / `INVALID_ARGS` instead of `STALE_REF`. -4. **Observable cap at `ABSOLUTE_MAX_DEPTH`** — `build_subtree` returns a **leaf boundary node** with `children_count` when `raw_depth >= ABSOLUTE_MAX_DEPTH` instead of `None`, so deep drill-downs are not silently dropped. -5. **Docs and tests** — `docs/phases.md` clarifies that `root` is CLI-only (`SnapshotArgs`), not `TreeOptions`; skills document `ref_id` in examples and batch `snapshot` args for `skeleton` / `root`. Integration tests cover skeleton depth, ref-count stability across refresh, invalid `--root`, and skeleton→drill flow. +- Resolve `--root` with `ref_token::resolve_ref_target` before loading a map. + A qualified ref supplies its snapshot; a bare ref requires the explicit + snapshot argument. Both are looked up only through + `RefStore::for_session(context.session_id())`. +- Re-resolve the saved root entry strictly before observing its subtree. A + missing, stale, or ambiguous live root fails closed; it is never replaced by + a positional guess. +- Use `RefStore::update_existing_snapshot` to remove only descendants owned by + that root, then allocate replacements through the shared + `ref_alloc::allocate_refs` owner. `RefAllocScope` preserves the root and + absolute path evidence for the replacement entries. +- A normal snapshot writes a fresh map. Skeleton mode is only a shallow + observation policy: it clamps depth to three and leaves `children_count` on + truncated nodes. Named structural anchors can receive drill-down refs, but + inert containers do not become actionable merely because they were visible. +- Derive the response window from the root entry's process identity with + `window_lookup::find_window_for_process`; never synthesize a plausible + window response. ## Why This Works -- **Reset semantics** match the mental model “snapshot the window again” — one `RefMap` per successful window snapshot save, no hidden accumulation from prior sessions on the same tree. -- **Window resolution** restores a stable JSON shape for automation that round-trips with window listing APIs. -- **Validation order** separates **bad syntax** from **stale or missing** refs, which is what agents need for recovery. -- **Boundary nodes** make the absolute depth cap **visible** in the tree instead of a silent prune. +The selected session is a hard namespace boundary, qualification removes +snapshot ambiguity, and strict resolution protects the gap between observation +and a later read. Replacing one rooted subtree preserves unrelated refs while +ensuring no stale descendants survive a re-drill. ## Prevention -- When changing snapshot or ref persistence, add or extend an integration test that asserts `**ref_count` is stable** across two identical `snapshot --skeleton` runs for the same app/window. -- Any new CLI flag that accepts a ref id should call `**validate_ref_id`** before map or adapter lookups. -- For subtree-only APIs, if a hard depth cap exists, return a **node with `children_count`** (or an explicit hint field) rather than `None`. -- Keep `**TreeOptions` vs CLI args** documented in `docs/phases.md` when adding root-like concepts so public architecture docs do not drift. +- Test full-snapshot replacement and rooted-subtree replacement separately. +- Test qualified and bare ref parsing, including mismatched snapshot IDs. +- Assert every truncation path exposes a boundary marker instead of silently + dropping descendants. +- Keep ref allocation in `crates/core/src/ref_alloc.rs`; full snapshots and + drill-downs must not grow separate allocators. -## Related Issues +## Related -- [Known pattern] DRY ref allocation — `[docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md](../best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md)` -- Plan: `docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md` +- [Single-owner ref allocation](../best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md) +- [Playwright-grade desktop reliability contract](../best-practices/playwright-grade-desktop-reliability-2026-06-02.md) diff --git a/npm/bin/agent-desktop.js b/npm/bin/agent-desktop.js index a05af6a..8928fc8 100755 --- a/npm/bin/agent-desktop.js +++ b/npm/bin/agent-desktop.js @@ -2,10 +2,12 @@ const { spawn } = require('child_process'); const { existsSync, accessSync, chmodSync, constants } = require('fs'); -const { dirname, join } = require('path'); +const { isAbsolute, join } = require('path'); const { platform, arch } = require('os'); const binDir = __dirname; +const MACOS_HELPER_NAME = 'agent-desktop-macos-helper'; +const MACOS_HELPER_PATH_ENV = 'AGENT_DESKTOP_MACOS_HELPER_PATH'; function getBinaryName() { const os = platform(); @@ -68,6 +70,30 @@ function main() { } } + if (platform() === 'darwin') { + const override = process.env[MACOS_HELPER_PATH_ENV]; + if (override && !isAbsolute(override)) { + console.error(`Error: ${MACOS_HELPER_PATH_ENV} must be an absolute path`); + process.exit(1); + } + const helperPath = override || join(binDir, MACOS_HELPER_NAME); + if (!existsSync(helperPath)) { + console.error(`Error: macOS helper not found: ${helperPath}`); + console.error('Reinstall agent-desktop so the CLI and helper come from the same release.'); + process.exit(1); + } + try { + accessSync(helperPath, constants.X_OK); + } catch { + try { + chmodSync(helperPath, 0o755); + } catch (err) { + console.error(`Error: Cannot make macOS helper executable: ${err.message}`); + process.exit(1); + } + } + } + const child = spawn(binaryPath, process.argv.slice(2), { stdio: 'inherit', windowsHide: false, diff --git a/npm/scripts/postinstall.js b/npm/scripts/postinstall.js index 7e2901b..dde791a 100644 --- a/npm/scripts/postinstall.js +++ b/npm/scripts/postinstall.js @@ -1,8 +1,19 @@ #!/usr/bin/env node -const { existsSync, mkdirSync, chmodSync, unlinkSync, renameSync, writeFileSync, symlinkSync, lstatSync } = require('fs'); -const { readFileSync } = require('fs'); -const { join } = require('path'); +const { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + renameSync, + symlinkSync, + unlinkSync, + writeFileSync, +} = require('fs'); +const { dirname, isAbsolute, join } = require('path'); const { platform, arch } = require('os'); const { execFileSync } = require('child_process'); const { createHash } = require('crypto'); @@ -13,6 +24,8 @@ const packageJson = JSON.parse(readFileSync(join(projectRoot, 'package.json'), ' const version = packageJson.version; const GITHUB_REPO = 'lahfir/agent-desktop'; +const MACOS_HELPER_NAME = 'agent-desktop-macos-helper'; +const MACOS_HELPER_PATH_ENV = 'AGENT_DESKTOP_MACOS_HELPER_PATH'; const TARGET_MAP = { 'darwin-arm64': 'aarch64-apple-darwin', @@ -36,6 +49,18 @@ function log(msg) { process.stderr.write(`agent-desktop: ${msg}\n`); } +function trashRecoverably(path, trashCommand = 'trash') { + try { + execFileSync(trashCommand, [path], { stdio: 'pipe', timeout: 30000 }); + } catch (err) { + if (!existsSync(path)) return; + const reason = err.code === 'ENOENT' + ? `trash command is unavailable: ${trashCommand}` + : `trash exited with status ${err.status ?? 'unknown'}`; + log(`Could not move cleanup artifact to Trash; retained at ${path}: ${reason}`); + } +} + function getPlatformKey() { return `${platform()}-${arch()}`; } @@ -55,9 +80,86 @@ function download(url, dest) { } function verifyChecksum(filePath, expectedHash) { - const fileBuffer = readFileSync(filePath); - const hash = createHash('sha256').update(fileBuffer).digest('hex'); - return hash === expectedHash; + return sha256(filePath) === expectedHash.toLowerCase(); +} + +function sha256(filePath) { + return createHash('sha256').update(readFileSync(filePath)).digest('hex'); +} + +function checksumFor(contents, fileName) { + for (const line of contents.split('\n')) { + const match = line.trim().match(/^([0-9a-fA-F]{64})\s+\*?(.+)$/); + if (match && match[2] === fileName) { + return match[1].toLowerCase(); + } + } + throw new Error(`Checksum entry missing for ${fileName}`); +} + +function installExecutable(source, destination) { + const temporary = `${destination}.install-${process.pid}`; + try { + writeFileSync(temporary, readFileSync(source), { mode: 0o755 }); + chmodSync(temporary, 0o755); + if (sha256(source) !== sha256(temporary)) { + throw new Error(`Executable copy verification failed for ${destination}`); + } + renameSync(temporary, destination); + } finally { + try { unlinkSync(temporary); } catch {} + } +} + +function customHelperPath(customBinaryPath) { + const override = process.env[MACOS_HELPER_PATH_ENV]; + if (override) { + if (!isAbsolute(override)) { + throw new Error(`${MACOS_HELPER_PATH_ENV} must be an absolute path`); + } + return override; + } + return join(dirname(customBinaryPath), MACOS_HELPER_NAME); +} + +function validateArchive(tarballPath) { + const listing = execFileSync('tar', ['-tzf', tarballPath], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30000, + }) + .split('\n') + .filter(Boolean) + .sort(); + const expected = ['agent-desktop', MACOS_HELPER_NAME].sort(); + if (JSON.stringify(listing) !== JSON.stringify(expected)) { + throw new Error(`Release archive has unexpected entries: ${listing.join(', ')}`); + } +} + +function installArchive(tarballPath, binaryPath, helperPath, trashCommand = 'trash') { + validateArchive(tarballPath); + const staging = mkdtempSync(join(binDir, '.extract-')); + try { + execFileSync('tar', ['-xzf', tarballPath, '-C', staging], { + stdio: 'pipe', + timeout: 30000, + }); + const entries = readdirSync(staging).sort(); + const expected = ['agent-desktop', MACOS_HELPER_NAME].sort(); + if (JSON.stringify(entries) !== JSON.stringify(expected)) { + throw new Error(`Extracted archive has unexpected entries: ${entries.join(', ')}`); + } + const extractedBinary = join(staging, 'agent-desktop'); + const extractedHelper = join(staging, MACOS_HELPER_NAME); + if (!lstatSync(extractedBinary).isFile() || !lstatSync(extractedHelper).isFile()) { + throw new Error('Release archive executables must be regular files'); + } + installExecutable(extractedHelper, helperPath); + installExecutable(extractedBinary, binaryPath); + } finally { + trashRecoverably(staging, trashCommand); + } } function fixGlobalInstallBin() { @@ -129,35 +231,43 @@ function main() { } const binaryPath = join(binDir, binaryName); - - if (process.env.AGENT_DESKTOP_BINARY_PATH) { - const customPath = process.env.AGENT_DESKTOP_BINARY_PATH; - if (existsSync(customPath)) { - try { - writeFileSync(binaryPath, readFileSync(customPath)); - chmodSync(binaryPath, 0o755); - log(`Using binary from AGENT_DESKTOP_BINARY_PATH: ${customPath}`); - fixGlobalInstallBin(); - promptSkillInstall(); - return; - } catch (err) { - log(`Failed to copy from AGENT_DESKTOP_BINARY_PATH: ${err.message}`); - } - } - } - - if (existsSync(binaryPath)) { - chmodSync(binaryPath, 0o755); - log(`Native binary ready: ${binaryName}`); - fixGlobalInstallBin(); - promptSkillInstall(); - return; - } + const helperPath = join(binDir, MACOS_HELPER_NAME); if (!existsSync(binDir)) { mkdirSync(binDir, { recursive: true }); } + if (process.env.AGENT_DESKTOP_BINARY_PATH) { + const customPath = process.env.AGENT_DESKTOP_BINARY_PATH; + try { + if (!existsSync(customPath) || !lstatSync(customPath).isFile()) { + throw new Error(`binary is not a regular file: ${customPath}`); + } + const sourceHelper = customHelperPath(customPath); + if (!existsSync(sourceHelper) || !lstatSync(sourceHelper).isFile()) { + throw new Error(`macOS helper not found at ${sourceHelper}`); + } + installExecutable(sourceHelper, helperPath); + installExecutable(customPath, binaryPath); + log(`Using binary from AGENT_DESKTOP_BINARY_PATH: ${customPath}`); + fixGlobalInstallBin(); + promptSkillInstall(); + } catch (err) { + log(`Failed to install from AGENT_DESKTOP_BINARY_PATH: ${err.message}`); + process.exitCode = 1; + } + return; + } + + if (existsSync(binaryPath) && existsSync(helperPath)) { + chmodSync(binaryPath, 0o755); + chmodSync(helperPath, 0o755); + log(`Native executables ready: ${binaryName}, ${MACOS_HELPER_NAME}`); + fixGlobalInstallBin(); + promptSkillInstall(); + return; + } + const tarball = `agent-desktop-v${version}-${target}.tar.gz`; const baseUrl = `https://github.com/${GITHUB_REPO}/releases/download/v${version}`; const tarballUrl = `${baseUrl}/${tarball}`; @@ -171,33 +281,23 @@ function main() { download(tarballUrl, tarballPath); download(checksumsUrl, checksumsPath); const checksums = readFileSync(checksumsPath, 'utf8'); - const expectedLine = checksums.split('\n').find((line) => line.includes(tarball)); - if (!expectedLine) { - throw new Error(`Checksum entry missing for ${tarball}`); - } - const expectedHash = expectedLine.split(/\s+/)[0]; + const expectedHash = checksumFor(checksums, tarball); if (!verifyChecksum(tarballPath, expectedHash)) { throw new Error('Checksum verification failed'); } unlinkSync(checksumsPath); log('Checksum verified'); - execFileSync('tar', ['-xzf', tarballPath, '-C', binDir], { stdio: 'pipe' }); - - const extractedBinary = join(binDir, 'agent-desktop'); - if (existsSync(extractedBinary) && extractedBinary !== binaryPath) { - renameSync(extractedBinary, binaryPath); - } - - chmodSync(binaryPath, 0o755); + installArchive(tarballPath, binaryPath, helperPath); unlinkSync(tarballPath); - log(`Installed native binary: ${binaryName}`); + log(`Installed native executables: ${binaryName}, ${MACOS_HELPER_NAME}`); } catch (err) { log(`Could not download native binary: ${err.message}`); log(''); log('Download manually from:'); log(` ${tarballUrl}`); - log(`Then place at: ${binaryPath}`); + log(`Then place agent-desktop at: ${binaryPath}`); + log(`And ${MACOS_HELPER_NAME} at: ${helperPath}`); try { if (existsSync(tarballPath)) unlinkSync(tarballPath); } catch {} try { if (existsSync(checksumsPath)) unlinkSync(checksumsPath); } catch {} @@ -210,4 +310,14 @@ function main() { promptSkillInstall(); } -main(); +if (require.main === module) { + main(); +} + +module.exports = { + checksumFor, + customHelperPath, + installArchive, + trashRecoverably, + validateArchive, +}; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 1a2d10c..6a5a047 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,3 @@ [toolchain] -channel = "stable" +channel = "1.89.0" profile = "minimal" -targets = ["aarch64-apple-darwin", "x86_64-apple-darwin"] diff --git a/scripts/cargo-test-isolated-home.sh b/scripts/cargo-test-isolated-home.sh new file mode 100755 index 0000000..6f8f84a --- /dev/null +++ b/scripts/cargo-test-isolated-home.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +original_home="${HOME:?HOME is required}" +cargo_home="${CARGO_HOME:-$original_home/.cargo}" +rustup_home="${RUSTUP_HOME:-$original_home/.rustup}" +test_home="$(mktemp -d "${TMPDIR:-/tmp}/agent-desktop-test-home.XXXXXX")" + +cleanup() { + if command -v trash >/dev/null 2>&1; then + trash "$test_home" + else + printf 'Retained isolated test home because trash is unavailable: %s\n' "$test_home" >&2 + fi +} +trap cleanup EXIT + +HOME="$test_home" CARGO_HOME="$cargo_home" RUSTUP_HOME="$rustup_home" cargo "$@" diff --git a/scripts/check-bash3-compat.sh b/scripts/check-bash3-compat.sh new file mode 100755 index 0000000..db63a3c --- /dev/null +++ b/scripts/check-bash3-compat.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" + +files=(.githooks/pre-commit scripts/*.sh tests/e2e/*.sh) +for file in "${files[@]}"; do + /bin/bash -n "$file" +done + +forbidden='declare[[:space:]]+-[a-zA-Z]*A|typeset[[:space:]]+-[a-zA-Z]*A|(^|[^[:alnum:]_])(mapfile|readarray|coproc)([^[:alnum:]_]|$)|\$\{[^}]+(,,|\^\^)' +if grep -En "$forbidden" "${files[@]}" | grep -v '^scripts/check-bash3-compat\.sh:'; then + echo "Bash 4+ syntax is not allowed in scripts that promise macOS Bash 3.2 support" >&2 + exit 1 +fi diff --git a/scripts/check-rust-file-size.sh b/scripts/check-rust-file-size.sh new file mode 100755 index 0000000..e1f2a6a --- /dev/null +++ b/scripts/check-rust-file-size.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +limit=400 +failed=0 + +while IFS= read -r file; do + [ -f "$file" ] || continue + if head -n 5 "$file" | grep -q '@generated'; then + continue + fi + lines="$(wc -l < "$file" | tr -d ' ')" + if [ "$lines" -gt "$limit" ]; then + printf '%s: %s lines (limit %s)\n' "$file" "$lines" "$limit" >&2 + failed=1 + fi +done < <(git ls-files --cached --others --exclude-standard -- '*.rs') + +if ! git ls-files -z --cached --others --exclude-standard -- '*.rs' \ + | python3 scripts/check_rust_comments.py; then + failed=1 +fi + +if [ "$failed" -ne 0 ]; then + exit 1 +fi diff --git a/scripts/check_rust_comments.py b/scripts/check_rust_comments.py new file mode 100644 index 0000000..82bad58 --- /dev/null +++ b/scripts/check_rust_comments.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +import sys +from pathlib import Path + + +def raw_string_end(source, start): + prefix_length = 2 if source.startswith("br", start) else 1 + if source[start : start + prefix_length] not in ("r", "br"): + return None + if start > 0 and (source[start - 1].isalnum() or source[start - 1] == "_"): + return None + cursor = start + prefix_length + hashes = 0 + while cursor < len(source) and source[cursor] == "#": + hashes += 1 + cursor += 1 + if cursor >= len(source) or source[cursor] != '"': + return None + marker = '"' + "#" * hashes + end = source.find(marker, cursor + 1) + return len(source) if end < 0 else end + len(marker) + + +def quoted_string_end(source, start): + prefix_length = 2 if source.startswith('b"', start) else 1 + if source[start : start + prefix_length] not in ('"', 'b"'): + return None + if prefix_length == 2 and start > 0 and ( + source[start - 1].isalnum() or source[start - 1] == "_" + ): + return None + cursor = start + prefix_length + escaped = False + while cursor < len(source): + character = source[cursor] + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + return cursor + 1 + cursor += 1 + return len(source) + + +def block_comment_end(source, start): + cursor = start + 2 + depth = 1 + while cursor < len(source) and depth: + if source.startswith("/*", cursor): + depth += 1 + cursor += 2 + elif source.startswith("*/", cursor): + depth -= 1 + cursor += 2 + else: + cursor += 1 + return cursor + + +def forbidden_comments(source): + findings = [] + cursor = 0 + line = 1 + code_on_line = False + while cursor < len(source): + raw_end = raw_string_end(source, cursor) + if raw_end is not None: + segment = source[cursor:raw_end] + line += segment.count("\n") + if "\n" in segment: + code_on_line = bool(segment.rsplit("\n", 1)[-1].strip()) + else: + code_on_line = True + cursor = raw_end + continue + string_end = quoted_string_end(source, cursor) + if string_end is not None: + segment = source[cursor:string_end] + line += segment.count("\n") + if "\n" in segment: + code_on_line = bool(segment.rsplit("\n", 1)[-1].strip()) + else: + code_on_line = True + cursor = string_end + continue + if source.startswith("//", cursor): + is_doc = source.startswith("///", cursor) or source.startswith("//!", cursor) + if not is_doc: + kind = "end-of-line comment" if code_on_line else "non-doc line comment" + findings.append((line, kind)) + end = source.find("\n", cursor + 2) + if end < 0: + break + cursor = end + continue + if source.startswith("/*", cursor): + is_doc = source.startswith("/**", cursor) or source.startswith("/*!", cursor) + if not is_doc: + findings.append((line, "block comment")) + end = block_comment_end(source, cursor) + line += source[cursor:end].count("\n") + code_on_line = False if "\n" in source[cursor:end] else code_on_line + cursor = end + continue + character = source[cursor] + if character == "\n": + line += 1 + code_on_line = False + elif not character.isspace(): + code_on_line = True + cursor += 1 + return findings + + +def check_path(path): + if not path.is_file(): + return [] + source = path.read_text(encoding="utf-8") + if "@generated" in "\n".join(source.splitlines()[:5]): + return [] + return forbidden_comments(source) + + +def main(): + failed = False + for raw_path in sys.stdin.buffer.read().split(b"\0"): + if not raw_path: + continue + path = Path(raw_path.decode()) + for line, kind in check_path(path): + print(f"{path}:{line}: {kind}s are forbidden", file=sys.stderr) + failed = True + raise SystemExit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_rust_comments_test.py b/scripts/check_rust_comments_test.py new file mode 100644 index 0000000..d742421 --- /dev/null +++ b/scripts/check_rust_comments_test.py @@ -0,0 +1,51 @@ +import importlib.util +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("check_rust_comments.py") +SPEC = importlib.util.spec_from_file_location("check_rust_comments", MODULE_PATH) +COMMENTS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(COMMENTS) + + +class RustCommentScannerTests(unittest.TestCase): + def test_ignores_doc_comments_and_comment_markers_in_strings(self): + source = ''' +/// Public documentation. +//! Module documentation. +let url = "https://example.test/path"; +let marker = "/* not a comment */"; +let raw = r###"// still text /* still text */"###; +''' + + self.assertEqual(COMMENTS.forbidden_comments(source), []) + + def test_reports_line_and_block_comments_only_in_rust_code(self): + source = ''' +// standalone +let value = 1; // trailing +/* block */ +let other = 2; /* trailing block */ +''' + + findings = COMMENTS.forbidden_comments(source) + + self.assertEqual( + findings, + [ + (2, "non-doc line comment"), + (3, "end-of-line comment"), + (4, "block comment"), + (5, "block comment"), + ], + ) + + def test_nested_doc_blocks_do_not_expose_inner_markers(self): + source = "/** docs with /* nested */ text */\nlet value = 1;\n" + + self.assertEqual(COMMENTS.forbidden_comments(source), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci-ffi-helper-discovery-smoke.sh b/scripts/ci-ffi-helper-discovery-smoke.sh new file mode 100755 index 0000000..0f502e7 --- /dev/null +++ b/scripts/ci-ffi-helper-discovery-smoke.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +STAGE=$(mktemp -d /tmp/agent-desktop-ffi-helper.XXXXXX) +cp target/release-ffi/libagent_desktop_ffi.dylib "$STAGE/" +cp target/release/agent-desktop-macos-helper "$STAGE/" +chmod +x "$STAGE/agent-desktop-macos-helper" +DYLIB_SHA=$(shasum -a 256 "$STAGE/libagent_desktop_ffi.dylib" | awk '{print $1}') +HELPER_SHA=$(shasum -a 256 "$STAGE/agent-desktop-macos-helper" | awk '{print $1}') +AD_DYLIB_PATH="$STAGE/libagent_desktop_ffi.dylib" \ + AD_HEADER_PATH=crates/ffi/include/agent_desktop.h \ + AD_EXPECT_MACOS_HELPER=1 \ + python3 tests/ffi-python/smoke.py +test "$DYLIB_SHA" = "$(shasum -a 256 "$STAGE/libagent_desktop_ffi.dylib" | awk '{print $1}')" +test "$HELPER_SHA" = "$(shasum -a 256 "$STAGE/agent-desktop-macos-helper" | awk '{print $1}')" diff --git a/scripts/ci-npm-wrapper-smoke.sh b/scripts/ci-npm-wrapper-smoke.sh new file mode 100755 index 0000000..fb2521d --- /dev/null +++ b/scripts/ci-npm-wrapper-smoke.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +case "$(uname -s)-$(uname -m)" in + Darwin-arm64) NAME=agent-desktop-darwin-arm64 ;; + Darwin-x86_64) NAME=agent-desktop-darwin-x64 ;; + *) + echo "Unsupported smoke-test platform: $(uname -s)-$(uname -m)" + exit 1 + ;; +esac +HELPER_NAME=agent-desktop-macos-helper +SOURCE=target/release/agent-desktop +HELPER_SOURCE=target/release/agent-desktop-macos-helper +SOURCE_SHA=$(shasum -a 256 "$SOURCE" | awk '{print $1}') +HELPER_SOURCE_SHA=$(shasum -a 256 "$HELPER_SOURCE" | awk '{print $1}') +cp "$SOURCE" "npm/bin/${NAME}" +cp "$HELPER_SOURCE" "npm/bin/${HELPER_NAME}" +chmod +x "npm/bin/${NAME}" +chmod +x "npm/bin/${HELPER_NAME}" +COPY_SHA=$(shasum -a 256 "npm/bin/${NAME}" | awk '{print $1}') +HELPER_COPY_SHA=$(shasum -a 256 "npm/bin/${HELPER_NAME}" | awk '{print $1}') +if [ "$SOURCE_SHA" != "$COPY_SHA" ] || [ "$HELPER_SOURCE_SHA" != "$HELPER_COPY_SHA" ]; then + echo "NPM smoke executable copy failed immutable identity verification" >&2 + exit 1 +fi +node npm/bin/agent-desktop.js version > /tmp/agent-desktop-version.json +node -e " + const out = require('fs').readFileSync('/tmp/agent-desktop-version.json', 'utf8'); + const json = JSON.parse(out); + if (json.ok !== true || !json.data || typeof json.data.version !== 'string') { + throw new Error('agent-desktop npm wrapper did not return version JSON'); + } +" +if [ "$SOURCE_SHA" != "$(shasum -a 256 "$SOURCE" | awk '{print $1}')" ] || \ + [ "$COPY_SHA" != "$(shasum -a 256 "npm/bin/${NAME}" | awk '{print $1}')" ] || \ + [ "$HELPER_SOURCE_SHA" != "$(shasum -a 256 "$HELPER_SOURCE" | awk '{print $1}')" ] || \ + [ "$HELPER_COPY_SHA" != "$(shasum -a 256 "npm/bin/${HELPER_NAME}" | awk '{print $1}')" ]; then + echo "NPM smoke executable changed while it was executed" >&2 + exit 1 +fi diff --git a/scripts/link-skills.sh b/scripts/link-skills.sh index a752d5d..dc66b2d 100755 --- a/scripts/link-skills.sh +++ b/scripts/link-skills.sh @@ -15,7 +15,11 @@ for skill_dir in "$REPO_ROOT"/skills/*/; do link="$CLAUDE_SKILLS/$name" if [ -L "$link" ]; then - rm "$link" + if ! command -v trash >/dev/null 2>&1; then + printf 'Cannot replace %s because trash is unavailable\n' "$link" >&2 + exit 1 + fi + trash "$link" fi ln -s "$target" "$link" diff --git a/scripts/perf-baseline-compare.sh b/scripts/perf-baseline-compare.sh new file mode 100755 index 0000000..04139f5 --- /dev/null +++ b/scripts/perf-baseline-compare.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# HEAD-vs-baseline performance comparison for agent-desktop. +# +# Builds the current checkout and the merge-base-with-main revision, then: +# 1. runs a fixture A/B (snapshots, reads, and fixture-only actions), +# 2. runs the deterministic synthetic locator benchmark on HEAD, +# 3. optionally probes real apps READ-ONLY (snapshot/find timing only), +# and writes a markdown report + raw JSON. +# +# Usage: +# scripts/perf-baseline-compare.sh [--base <ref>] [--apps "Slack,Google Chrome"] +# [--rounds N] [--out <dir>] [--skip-fixture] +# +# Real-app probes never dispatch actions. Apps this script launches itself are +# closed again afterwards; apps that were already running are left untouched. +set -euo pipefail + +base_ref="" +apps="" +rounds=10 +out="" +skip_fixture=0 +while [ $# -gt 0 ]; do + case "$1" in + --base) base_ref="$2"; shift 2 ;; + --apps) apps="$2"; shift 2 ;; + --rounds) rounds="$2"; shift 2 ;; + --out) out="$2"; shift 2 ;; + --skip-fixture) skip_fixture=1; shift ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +repo="$(git rev-parse --show-toplevel)" +cd "$repo" +[ -n "$base_ref" ] || base_ref="$(git merge-base HEAD "$(git rev-parse --verify --quiet origin/main >/dev/null 2>&1 && echo origin/main || echo main)")" +base_sha="$(git rev-parse --short "$base_ref")" +head_sha="$(git rev-parse --short HEAD)" +[ -n "$out" ] || out="$(mktemp -d "/tmp/agent-desktop-perf-${head_sha}-vs-${base_sha}.XXXXXX")" +mkdir -p "$out" +echo "== perf compare: HEAD ${head_sha} vs base ${base_sha} -> ${out}" + +echo "== building HEAD release binary" +cargo build --release -p agent-desktop >/dev/null +head_bin="$repo/target/release/agent-desktop" + +echo "== building base release binary (${base_sha}) in a temporary worktree" +wt="$out/base-worktree" +git worktree add --detach "$wt" "$base_ref" >/dev/null +trap 'git worktree remove --force "$wt" >/dev/null 2>&1 || true; git worktree prune >/dev/null 2>&1 || true' EXIT +(cd "$wt" && cargo build --release -p agent-desktop >/dev/null) +base_bin="$wt/target/release/agent-desktop" + +if [ "$skip_fixture" -ne 1 ]; then + echo "== fixture A/B (${rounds} rounds; actions run against the fixture only)" + bash tests/fixture-app/build.sh "$out/fixture" >/dev/null + open "$out/fixture/AgentDeskFixture.app" + sleep 3 + python3 scripts/perf_ab_probe.py --mode fixture --app AgentDeskFixture \ + --head-bin "$head_bin" --base-bin "$base_bin" --rounds "$rounds" \ + --json-out "$out/fixture-ab.json" >/dev/null + pkill -f AgentDeskFixture.app || true +fi + +echo "== synthetic locator benchmark (HEAD)" +cargo run -p agent-desktop-core --release --example locator_benchmark \ + > "$out/locator-synthetic.json" 2>/dev/null + +if [ -n "$apps" ]; then + IFS=',' read -ra app_list <<< "$apps" + for app in "${app_list[@]}"; do + app="$(echo "$app" | sed 's/^ *//;s/ *$//')" + slug="$(echo "$app" | tr ' ' '-' | tr '[:upper:]' '[:lower:]')" + was_running=1 + pgrep -f "/${app}.app/" >/dev/null 2>&1 || was_running=0 + if [ "$was_running" -eq 0 ]; then + echo "== launching ${app} for read-only observation" + open -a "$app" || { echo " skip: cannot open ${app}"; continue; } + sleep 12 + fi + echo "== read-only probe: ${app} (${rounds} rounds, snapshots + find only)" + python3 scripts/perf_ab_probe.py --mode app --app "$app" \ + --head-bin "$head_bin" --base-bin "$base_bin" --rounds "$rounds" \ + --json-out "$out/app-${slug}.json" >/dev/null || echo " probe failed for ${app}" + if [ "$was_running" -eq 0 ]; then + "$head_bin" close-app --app "$app" >/dev/null 2>&1 || true + fi + done +fi + +python3 scripts/perf_report_html.py "$out" --head "$head_sha" --base "$base_sha" +echo "== artifacts: $out" diff --git a/scripts/perf_ab_probe.py b/scripts/perf_ab_probe.py new file mode 100755 index 0000000..5ab312b --- /dev/null +++ b/scripts/perf_ab_probe.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""A/B latency probe: two agent-desktop binaries against one target app. + +Modes: + fixture -- full probe incl. mutating actions; ONLY safe against the + AgentDeskFixture app. + app -- strictly read-only observation (snapshot/find timing) for + arbitrary real apps (Slack, Chrome, ...). Never dispatches + actions. + +Each binary runs under an isolated HOME so cross-revision store formats +never collide. Rounds alternate between binaries to cancel machine drift. +""" +import argparse +import json +import os +import statistics +import subprocess +import sys +import tempfile +import time + +ENVS = {} + + +def env_for(binary): + if binary not in ENVS: + env = dict(os.environ) + env["HOME"] = tempfile.mkdtemp(prefix="ad-perf-home-") + ENVS[binary] = env + return ENVS[binary] + + +def run(binary, *args, timeout=90): + start = time.monotonic() + proc = subprocess.run( + [binary, *args], capture_output=True, text=True, timeout=timeout, env=env_for(binary) + ) + elapsed = (time.monotonic() - start) * 1000 + return elapsed, proc + + +def envelope(proc): + try: + return json.loads(proc.stdout) + except ValueError: + return {} + + +def walk(node): + yield node + for child in node.get("children", []): + yield from walk(child) + + +def tree_stats(env): + tree = env.get("data", {}).get("tree", {}) + nodes = sum(1 for _ in walk(tree)) + depth = 0 + + def measure(node, level): + nonlocal depth + depth = max(depth, level) + for child in node.get("children", []): + measure(child, level + 1) + + measure(tree, 1) + return nodes, depth, env.get("data", {}).get("ref_count") + + +def fixture_refs(binary, app): + _, proc = run(binary, "snapshot", "--app", app, "-i") + env = envelope(proc) + if "data" not in env: + raise SystemExit(f"{binary}: fixture snapshot failed: {proc.stdout[:300]}") + button = textfield = None + for node in walk(env["data"]["tree"]): + ref = node.get("ref") or node.get("ref_id") + if not ref: + continue + name = (node.get("name") or "").lower() + if node.get("role") == "button" and "primary" in name: + button = button or ref + if node.get("role") == "textfield": + textfield = textfield or ref + if not (button and textfield): + raise SystemExit(f"{binary}: fixture targets missing (button={button}, field={textfield})") + return button, textfield + + +def fixture_cases(binary, app): + button, textfield = fixture_refs(binary, app) + return [ + ("snapshot -i", ["snapshot", "--app", app, "-i"]), + ("snapshot d30", ["snapshot", "--app", app, "--max-depth", "30"]), + ("get", ["get", button]), + ("is visible", ["is", button, "--property", "visible"]), + ("click", ["click", button]), + ("set-value", ["set-value", textfield, "ab-probe"]), + ("type", ["type", textfield, "x"]), + ] + + +def app_cases(app): + return [ + ("snapshot -i", ["snapshot", "--app", app, "-i"]), + ("snapshot d30", ["snapshot", "--app", app, "--max-depth", "30"]), + ] + + +def collect(binaries, cases_for, rounds): + cases = {label: cases_for(label) for label in binaries} + results = {label: {} for label in binaries} + shapes = {label: {} for label in binaries} + for _ in range(rounds): + for label, binary in binaries.items(): + for name, args in cases[label]: + elapsed, proc = run(binary, *args) + env = envelope(proc) + ok = bool(env.get("ok")) + results[label].setdefault(name, []).append((elapsed, ok)) + if ok and name.startswith("snapshot") and name not in shapes[label]: + shapes[label][name] = tree_stats(env) + return results, shapes + + +def summarize(results, shapes): + report = {} + for label, cases in results.items(): + for name, samples in cases.items(): + times = sorted(t for t, _ in samples) + report.setdefault(name, {})[label] = { + "p50_ms": round(statistics.median(times), 1), + "p95_ms": round(times[max(0, int(len(times) * 0.95) - 1)], 1), + "ok_rate": sum(1 for _, ok in samples if ok) / len(samples), + "shape": shapes[label].get(name), + } + return report + + +def head_only_find(binary, app, rounds, report): + times, matches = [], None + for _ in range(rounds): + elapsed, proc = run(binary, "find", "--app", app, "--role", "button", "--first") + env = envelope(proc) + if env.get("ok"): + times.append(elapsed) + matches = env.get("data", {}).get("match", {}).get("role") + if times: + times.sort() + report["find --role button --first (HEAD-only)"] = { + "HEAD": { + "p50_ms": round(statistics.median(times), 1), + "p95_ms": round(times[max(0, int(len(times) * 0.95) - 1)], 1), + "ok_rate": len(times) / rounds, + "matched_role": matches, + } + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=["fixture", "app"], required=True) + parser.add_argument("--app", required=True) + parser.add_argument("--head-bin", required=True) + parser.add_argument("--base-bin", required=True) + parser.add_argument("--rounds", type=int, default=10) + parser.add_argument("--json-out", default="") + args = parser.parse_args() + + binaries = {"HEAD": args.head_bin, "BASE": args.base_bin} + if args.mode == "fixture": + results, shapes = collect(binaries, lambda label: fixture_cases(binaries[label], args.app), args.rounds) + else: + results, shapes = collect(binaries, lambda _label: app_cases(args.app), args.rounds) + report = summarize(results, shapes) + if args.mode == "app": + head_only_find(args.head_bin, args.app, args.rounds, report) + + payload = {"mode": args.mode, "app": args.app, "rounds": args.rounds, "cases": report} + if args.json_out: + with open(args.json_out, "w") as handle: + json.dump(payload, handle, indent=1) + json.dump(payload, sys.stdout, indent=1) + print() + + +if __name__ == "__main__": + main() diff --git a/scripts/perf_report_html.py b/scripts/perf_report_html.py new file mode 100644 index 0000000..513bcb0 --- /dev/null +++ b/scripts/perf_report_html.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Render perf-baseline-compare JSON artifacts into one self-contained HTML report. + +Usage: python3 scripts/perf_report_html.py <out_dir> --head <sha> --base <sha> +Reads fixture-ab.json, app-*.json, and locator-synthetic.json from <out_dir> (any may +be absent) and writes <out_dir>/report.html. Stdlib only. +""" +import argparse +import datetime +import glob +import json +import os +import sys + +import perf_report_svg as chart +from perf_report_svg import esc + +CSS = """ +:root { + --surface:#fcfcfb; --page:#f9f9f7; --ink:#0b0b0b; --secondary:#52514e; --muted:#898781; + --grid:#e1e0d9; --baseline:#c3c2b7; --border:rgba(11,11,11,0.10); + --head:#2a78d6; --base:#1baf7a; --good:#006300; --status-good:#0ca30c; + --good-tint:rgba(0,99,0,0.10); --status-good-tint:rgba(12,163,12,0.12); + --secondary-tint:rgba(82,81,78,0.09); --head-tint:rgba(42,120,214,0.10); +} +@media (prefers-color-scheme:dark) { + :root { + --surface:#1a1a19; --page:#0d0d0d; --ink:#ffffff; --secondary:#c3c2b7; --muted:#898781; + --grid:#2c2c2a; --baseline:#383835; --border:rgba(255,255,255,0.10); + --head:#3987e5; --base:#199e70; --good:#0ca30c; --status-good:#0ca30c; + --good-tint:rgba(12,163,12,0.16); --status-good-tint:rgba(12,163,12,0.16); + --secondary-tint:rgba(195,194,183,0.10); --head-tint:rgba(57,135,229,0.14); + } +} +* { box-sizing:border-box; } +body { margin:0; background:var(--page); color:var(--ink); line-height:1.5; + font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; } +.col { max-width:1040px; margin:0 auto; padding:0 24px 48px; display:flex; flex-direction:column; gap:24px; } +header.band, footer.band { max-width:1040px; margin:0 auto; padding:32px 24px 8px; } +footer.band { padding:8px 24px 40px; } +header.band h1 { font-size:22px; margin:0 0 6px; letter-spacing:-0.01em; } +header.band .subtitle, footer.band p { color:var(--secondary); font-size:13px; margin:4px 0; } +footer.band code, .card code { background:var(--secondary-tint); border-radius:4px; padding:1px 6px; font-size:12px; } +.tiles { max-width:1040px; margin:0 auto; padding:8px 24px 0; display:grid; + grid-template-columns:repeat(auto-fit,minmax(190px,1fr)); gap:16px; } +.tile { background:var(--surface); border:1px solid var(--border); border-radius:12px; padding:18px 20px; } +.tile-label { font-size:12px; color:var(--muted); margin-bottom:8px; } +.tile-value { font-size:26px; font-weight:600; } +.tile-delta { font-size:12px; margin-top:6px; } +.tile-delta.good { color:var(--good); } .tile-delta.secondary { color:var(--secondary); } .tile-delta.muted { color:var(--muted); } +.card { background:var(--surface); border:1px solid var(--border); border-radius:12px; padding:28px; } +.card-head { display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:12px; } +.card h2 { font-size:16px; margin:0; } +.hint, .shape-line { font-size:12.5px; color:var(--secondary); } +.hint { margin:6px 0 4px; } .shape-line { margin:12px 0 0; } +.legend, .legend-item { display:flex; align-items:center; } +.legend { gap:16px; font-size:12px; color:var(--secondary); } .legend-item { gap:6px; } +.swatch { width:10px; height:10px; border-radius:2px; display:inline-block; } +.swatch-head { background:var(--head); } .swatch-base { background:var(--base); } +.callout { border:1px solid var(--border); background:var(--head-tint); border-radius:10px; + padding:14px 16px; font-size:13px; color:var(--secondary); margin:16px 0 4px; } +.callout strong { color:var(--ink); } +svg.chart { display:block; width:100%; height:auto; margin:16px 0 4px; } +svg.chart text { font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; } +.gridline { stroke:var(--grid); stroke-width:1; } .tick { stroke:var(--ink); stroke-width:1.4; opacity:0.45; } +.parity { stroke:var(--baseline); stroke-width:1.5; stroke-dasharray:3 3; } +.bar-head { fill:var(--head); } .bar-base { fill:var(--base); } +.ink { fill:var(--ink); } .secondary { fill:var(--secondary); } .muted { fill:var(--muted); } +.tabular { font-variant-numeric:tabular-nums; } +.hit { fill:transparent; cursor:pointer; } +.value-label { font-weight:600; } +.chip-good { fill:var(--good-tint); } .chip-secondary, .chip-info { fill:var(--secondary-tint); } +.chip-status-good { fill:var(--status-good-tint); } +.chip-text-good { fill:var(--good); } .chip-text-secondary, .chip-text-info { fill:var(--secondary); } +.chip-text-status-good { fill:var(--status-good); } +details { margin-top:14px; } +summary { cursor:pointer; font-size:13px; color:var(--secondary); } +.table-wrap { overflow-x:auto; } +table { width:100%; border-collapse:collapse; margin-top:12px; font-size:12.5px; font-variant-numeric:tabular-nums; } +th, td { text-align:right; padding:7px 10px; border-bottom:1px solid var(--border); white-space:nowrap; } +th:first-child, td:first-child { text-align:left; } +th { color:var(--muted); font-weight:500; } +.tooltip { position:fixed; pointer-events:none; opacity:0; transition:opacity .08s ease; background:var(--ink); + color:var(--surface); border-radius:8px; padding:8px 10px; font-size:12px; z-index:20; max-width:240px; } +.tooltip .tt-title { font-weight:600; margin-bottom:4px; } +.tooltip .tt-row { display:flex; justify-content:space-between; gap:12px; opacity:.85; } +@media (max-width:640px) { .card { padding:20px; } .tile-value { font-size:22px; } } +""" + +JS = """ +(function () { + var tip = document.getElementById('tt'); + document.querySelectorAll('[data-tt-title]').forEach(function (el) { + el.addEventListener('mousemove', function (evt) { + var rows = (el.getAttribute('data-tt-rows') || '').split('|').filter(Boolean); + var html = '<div class="tt-title">' + el.getAttribute('data-tt-title') + '</div>'; + rows.forEach(function (row) { + var i = row.indexOf('='); + html += '<div class="tt-row"><span>' + row.slice(0, i) + '</span><b>' + row.slice(i + 1) + '</b></div>'; + }); + tip.innerHTML = html; + tip.style.left = Math.min(evt.clientX + 14, window.innerWidth - 250) + 'px'; + tip.style.top = (evt.clientY + 14) + 'px'; tip.style.opacity = '1'; + }); + el.addEventListener('mouseleave', function () { tip.style.opacity = '0'; }); + }); +})(); +""" + +PAGE = """<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"/> +<meta name="viewport" content="width=device-width, initial-scale=1"/> +<title>{title} + + + +{body} +
+ + + +""" + + +def load(path): + if not os.path.exists(path): + return None + with open(path) as handle: + return json.load(handle) + + +def fmt_ms(value): + return f"{value:.1f} ms" + + +def ok_pct(value): + return f"{value * 100:.0f}%" + + +def tip_rows(entry): + return [("p50", fmt_ms(entry["p50_ms"])), ("p95", fmt_ms(entry["p95_ms"])), + ("ok rate", ok_pct(entry.get("ok_rate", 1.0)))] + + +def build_ab_rows(payload): + rows = [] + for name, sides in payload.get("cases", {}).items(): + head, base = sides.get("HEAD"), sides.get("BASE") + if not head or not base: + continue + delta_text, delta_cls = chart.fmt_delta(head["p50_ms"], base["p50_ms"]) + rows.append({ + "label": name, + "head": {"value": head["p50_ms"], "tick": head["p95_ms"], + "tip_title": f"{name} — HEAD", "tip_rows": tip_rows(head)}, + "base": {"value": base["p50_ms"], "tick": base["p95_ms"], + "tip_title": f"{name} — BASE", "tip_rows": tip_rows(base)}, + "delta_text": delta_text, "delta_good": delta_cls == "good", + }) + return rows + + +def build_single_rows(payload): + rows = [] + for name, sides in payload.get("cases", {}).items(): + head, base = sides.get("HEAD"), sides.get("BASE") + if not head or base: + continue + label = name.replace(" (HEAD-only)", "") + rows.append({ + "label": label, "value": head["p50_ms"], "tick": head["p95_ms"], + "tip_title": f"{label} — HEAD", "tip_rows": tip_rows(head), + "badge_text": "new in HEAD - no baseline equivalent", + }) + return rows + + +def case_shapes(payload, case_name): + sides = payload.get("cases", {}).get(case_name, {}) + return (sides.get("HEAD") or {}).get("shape"), (sides.get("BASE") or {}).get("shape") + + +def shape_line(payload): + head_shape, base_shape = case_shapes(payload, "snapshot d30") + if not head_shape or not base_shape: + return None + hn, hd, hr = head_shape + bn, bd, br = base_shape + return f"HEAD d30: {hn} nodes / depth {hd} / {hr} refs — BASE d30: {bn} nodes / depth {bd} / {br} refs" + + +def needs_honesty_callout(payload): + head_shape, base_shape = case_shapes(payload, "snapshot -i") + if not head_shape or not base_shape: + return False + return base_shape[0] > 0 and head_shape[0] < 0.5 * base_shape[0] + + +def build_speedup_rows(report): + rows = [] + for sc in report.get("scenarios", []): + legacy, live, comp = sc["legacy_snapshot"], sc["live_find_selected_refs"], sc["comparison"] + attr_delta = chart.pct_delta(live["attributes_requested"], legacy["attributes_requested"]) + rows.append({ + "label": sc["name"], "speedup": comp["p50_find_speedup"], "tip_title": sc["name"], + "tip_rows": [ + ("legacy p50", f'{legacy["p50_us"] / 1000:.2f} ms'), ("live p50", f'{live["p50_us"] / 1000:.2f} ms'), + ("legacy p95", f'{legacy["p95_us"] / 1000:.2f} ms'), ("live p95", f'{live["p95_us"] / 1000:.2f} ms'), + ("attrs legacy/live", f'{legacy["attributes_requested"]} / {live["attributes_requested"]}'), + ], + "correctness_fix": comp.get("find_correctness_delta", 0) > 0, + "sub_label": f"{attr_delta:+.0f}% attribute reads" if attr_delta is not None else None, + }) + return rows + + +def tile(label, value, head, base): + delta_text, cls = chart.fmt_delta(head, base) + return {"label": label, "value": value, "delta_text": f"{delta_text} vs base" if delta_text else "n/a", "cls": cls} + + +def build_tiles(fixture, synthetic): + tiles = [] + if fixture: + click = fixture.get("cases", {}).get("click", {}) + if click.get("HEAD") and click.get("BASE"): + h, b = click["HEAD"]["p50_ms"], click["BASE"]["p50_ms"] + tiles.append(tile("Click p50 (fixture)", fmt_ms(h), h, b)) + d30 = fixture.get("cases", {}).get("snapshot d30", {}) + if d30.get("HEAD") and d30.get("BASE"): + h, b = d30["HEAD"]["p50_ms"], d30["BASE"]["p50_ms"] + tiles.append(tile("Snapshot d30 p50 (fixture)", fmt_ms(h), h, b)) + scenarios = (synthetic or {}).get("scenarios") or [] + if scenarios: + best = max(scenarios, key=lambda s: s["comparison"]["p50_find_speedup"]) + tiles.append({"label": "Best synthetic find speedup", "value": f'{best["comparison"]["p50_find_speedup"]:.1f}x', + "delta_text": best["name"].replace("_", " "), "cls": "muted"}) + fixes = sum(1 for s in scenarios if s["comparison"].get("find_correctness_delta", 0) > 0) + tiles.append({"label": "Correctness fixes", "value": f"{fixes} scenario{'' if fixes == 1 else 's'}", + "delta_text": "legacy path answered incorrectly" if fixes else "no legacy mismatches observed", + "cls": "good" if fixes else "muted"}) + return tiles + + +def tiles_html(tiles): + if not tiles: + return "" + cells = "".join( + f'
{esc(t["label"])}
' + f'
{esc(t["value"])}
' + f'
{esc(t["delta_text"])}
' + for t in tiles + ) + return f'
{cells}
' + + +LEGEND = ('
HEAD' + 'BASE
') + + +def table_html(headers, rows): + thead = "".join(f"{esc(h)}" for h in headers) + trs = "".join("" + "".join(f"{esc(c)}" for c in r) + "" for r in rows) + return f'
{thead}{trs}
' + + +def case_rows(payload, require_base): + rows = [] + for name, sides in payload.get("cases", {}).items(): + head, base = sides.get("HEAD"), sides.get("BASE") + if not head or (require_base and not base): + continue + delta_text = chart.fmt_delta(head["p50_ms"], base["p50_ms"])[0] if base else None + ok = (f'{ok_pct(base.get("ok_rate", 1.0))}/{ok_pct(head.get("ok_rate", 1.0))}' if base + else f'-/{ok_pct(head.get("ok_rate", 1.0))}') + rows.append([name, fmt_ms(base["p50_ms"]) if base else "-", fmt_ms(base["p95_ms"]) if base else "-", + fmt_ms(head["p50_ms"]), fmt_ms(head["p95_ms"]), delta_text or "n/a", ok]) + return rows + + +def fixture_section(payload): + if not payload: + return "" + rows = build_ab_rows(payload) + table = table_html(["Case", "BASE p50", "BASE p95", "HEAD p50", "HEAD p95", "Delta", "OK B/H"], + case_rows(payload, require_base=True)) + return ( + '
' + f'

Fixture A/B (live)

{LEGEND if rows else ""}
' + f'

Alternating rounds, isolated per-binary stores, wall-clock per process invocation ' + f'({payload.get("rounds", "?")} rounds).

{chart.grouped_bar_chart(rows)}' + f'
Table view{table}
' + ) + + +def app_section(payload): + if not payload: + return "" + ab_rows, single_rows = build_ab_rows(payload), build_single_rows(payload) + if needs_honesty_callout(payload): + for row in ab_rows: + if row["label"] == "snapshot -i": + row["delta_text"], row["delta_good"] = "not comparable", False + body = [f'

{esc(payload["app"])}

{LEGEND if ab_rows else ""}
'] + body.append(f'

Read-only observation ({payload.get("rounds", "?")} rounds).

') + if ab_rows: + body.append(chart.grouped_bar_chart(ab_rows)) + if needs_honesty_callout(payload): + body.append( + '
Reading note. HEAD’s default snapshot -i ' + 'reads a shallower tree than BASE under the progressive-traversal default-depth change, so the ' + 'snapshot -i row above is not like-for-like. The snapshot d30 rows are ' + 'the apples-to-apples comparison.
') + shape = shape_line(payload) + if shape: + body.append(f'

{esc(shape)}

') + if single_rows: + body.append(chart.single_series_bar_chart(single_rows)) + table = table_html(["Case", "BASE p50", "BASE p95", "HEAD p50", "HEAD p95", "Delta", "OK B/H"], + case_rows(payload, require_base=False)) + body.append(f'
Table view{table}
') + return f'
{"".join(body)}
' + + +def synthetic_section(report): + if not report: + return "" + rows = build_speedup_rows(report) + method = report.get("methodology", {}) + hint = (f'{method.get("measured_runs", "?")} measured runs, {method.get("warmup_runs", "?")} warmup. ' + f'Legacy: {method.get("legacy_path", "legacy snapshot path")}. ' + f'Live: {method.get("live_find_path", "live find path")}.') + table_rows = [] + for sc in report.get("scenarios", []): + legacy, live, comp = sc["legacy_snapshot"], sc["live_find_selected_refs"], sc["comparison"] + table_rows.append([sc["name"], f'{legacy["p50_us"] / 1000:.2f} ms', f'{legacy["p95_us"] / 1000:.2f} ms', + f'{live["p50_us"] / 1000:.2f} ms', f'{live["p95_us"] / 1000:.2f} ms', + legacy["attributes_requested"], live["attributes_requested"], + "yes" if comp.get("find_correctness_delta", 0) > 0 else "no"]) + table = table_html(["Scenario", "Legacy p50", "Legacy p95", "Live p50", "Live p95", "Legacy attrs", "Live attrs", "Fix"], + table_rows) + return ( + '

Synthetic locator benchmark

' + f'

{esc(hint)}

{chart.speedup_bar_chart(rows)}' + f'
Table view{table}
' + ) + + +def page_chrome(head, base, rounds, out_dir): + rounds_part = f" · {rounds} rounds" if rounds else "" + header = ( + '
' + f'

agent-desktop performance - HEAD {esc(head)} vs base {esc(base)}

' + f'

{datetime.date.today().isoformat()}{rounds_part} · alternating rounds, ' + 'isolated per-binary stores, wall-clock per process invocation

' + ) + footer = ( + '
' + '

Regenerate: bash scripts/perf-baseline-compare.sh --apps "Slack,Google Chrome"

' + f'

Artifacts: {esc(out_dir)} · ' + f'generated {esc(datetime.datetime.now().isoformat(timespec="seconds"))}

' + ) + return header, footer + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out_dir") + parser.add_argument("--head", required=True) + parser.add_argument("--base", required=True) + args = parser.parse_args() + + fixture = load(os.path.join(args.out_dir, "fixture-ab.json")) + synthetic = load(os.path.join(args.out_dir, "locator-synthetic.json")) + apps = [load(p) for p in sorted(glob.glob(os.path.join(args.out_dir, "app-*.json")))] + apps = [a for a in apps if a] + sections = [fixture_section(fixture)] + [app_section(a) for a in apps] + [synthetic_section(synthetic)] + tiles = build_tiles(fixture, synthetic) + rounds = fixture.get("rounds") if fixture else None + header, footer = page_chrome(args.head, args.base, rounds, os.path.abspath(args.out_dir)) + body = header + tiles_html(tiles) + '
' + "".join(sections) + "
" + footer + title = f"agent-desktop performance - HEAD {args.head} vs base {args.base}" + html_doc = PAGE.format(title=esc(title), css=CSS, body=body, js=JS) + + out_path = os.path.join(args.out_dir, "report.html") + with open(out_path, "w") as handle: + handle.write(html_doc) + + print(f"agent-desktop performance report: HEAD {args.head} vs base {args.base}") + for t in tiles: + print(f' {t["label"]}: {t["value"]} ({t["delta_text"]})') + print(f" apps covered: {', '.join(a['app'] for a in apps) if apps else 'none'}") + print(f" report: {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/perf_report_svg.py b/scripts/perf_report_svg.py new file mode 100644 index 0000000..1c6ac9d --- /dev/null +++ b/scripts/perf_report_svg.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Pure SVG chart builders for the agent-desktop performance report. + +No third-party deps, no file or network I/O. Every public function takes +plain-Python row dicts (already formatted by the caller) and returns a +self-contained ```` string. Colors are applied only via CSS classes +(``bar-head``, ``bar-base``, ``good``, ``secondary``, ``muted``, ``status-good``, +``chip-*``) so the same markup adapts to the host page's light/dark tokens. +""" +import math + +VB_W = 860 +BAR_H = 16 +BAR_GAP = 2 +ROW_GAP = 20 +TOP_PAD = 18 +AXIS_H = 30 +CHIP_GAP = 14 +LABEL_GAP = 14 +CHECK = "✓" + + +def esc(value): + text = str(value) + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def pct_delta(head, base): + if not base: + return None + return (head - base) / base * 100.0 + + +def fmt_delta(head, base): + delta = pct_delta(head, base) + if delta is None: + return None, "muted" + return f"{delta:+.0f}%", ("good" if head < base else "secondary") + + +def nice_ticks(max_value, target=4): + if max_value <= 0: + return [0, 1] + raw_step = max_value / target + magnitude = 10 ** math.floor(math.log10(raw_step)) + step = magnitude + for mult in (1, 2, 2.5, 5, 10): + step = mult * magnitude + if step >= raw_step: + break + ticks, value, limit = [], 0.0, max_value + step * 0.5 + while value <= limit: + ticks.append(round(value, 6)) + value += step + return ticks + + +def _clamp(value, lo, hi): + return max(lo, min(hi, value)) + + +def _measure(text, size): + return len(str(text)) * size * 0.54 + + +def _bar(x, y, w, h, cls): + w = max(w, 0.0) + if w <= 0: + return "" + r = min(4.0, w / 2, h / 2) + out = [f''] + if w > r: + out.append(f'') + return "".join(out) + + +def _tick(x, y, h, cls="tick"): + return f'' + + +def _hit(x, y, w, h, title, rows): + row_str = esc("|".join(f"{label}={value}" for label, value in rows)) + return ( + f'' + ) + + +def _chip(x_right, y_mid, text, cls, size=11): + w = _measure(text, size) + 18 + h = 20 + x, y = x_right - w, y_mid - h / 2 + return ( + f'' + f'{esc(text)}' + ) + + +def _axis(sx, ticks, top, bottom, unit, suffix_x=False): + out = [] + for tick in ticks: + tx = sx(tick) + out.append(f'') + label = f"{tick:g}x" if suffix_x else f"{tick:g}{unit}" + out.append( + f'{label}' + ) + return "".join(out) + + +def _grouped_summary(rows): + best = max(rows, key=lambda r: abs(pct_delta(r["head"]["value"], r["base"]["value"]) or 0)) + delta = pct_delta(best["head"]["value"], best["base"]["value"]) or 0 + return ( + f"Grouped horizontal bar chart comparing HEAD and BASE p50 latency across {len(rows)} cases. " + f"Largest change: {best['label']} at {delta:+.0f}%." + ) + + +def grouped_bar_chart(rows, unit=" ms"): + if not rows: + return '' + label_w = _clamp(max(_measure(r["label"], 12.5) for r in rows) + LABEL_GAP, 120, 300) + chip_w = _clamp( + max((_measure(r["delta_text"], 11) + 18 for r in rows if r.get("delta_text")), default=50), 50, 200 + ) + x0, x1 = label_w, VB_W - chip_w - CHIP_GAP + plot_w = max(x1 - x0, 40) + data_max = max(max(r["head"]["value"], r["head"]["tick"], r["base"]["value"], r["base"]["tick"]) for r in rows) + ticks = nice_ticks(data_max, 4) + domain_max = (ticks[-1] or 1) * 1.24 + group_h = BAR_H * 2 + BAR_GAP + row_pitch = group_h + ROW_GAP + top = TOP_PAD + bottom = top + row_pitch * len(rows) - ROW_GAP + height = bottom + AXIS_H + + def sx(v): + return x0 + (v / domain_max) * plot_w + + parts = [ + f'', + _axis(sx, ticks, top, bottom, unit), + ] + for i, row in enumerate(rows): + y = top + i * row_pitch + y_head, y_base, cy = y, y + BAR_H + BAR_GAP, y + group_h / 2 + parts.append( + f'{esc(row["label"])}' + ) + for series, sy, cls in (("head", y_head, "bar-head"), ("base", y_base, "bar-base")): + d = row[series] + w = max((d["value"] / domain_max) * plot_w, 0) + parts.append(_bar(x0, sy, w, BAR_H, cls)) + if d.get("tick") is not None: + parts.append(_tick(sx(d["tick"]), sy, BAR_H)) + parts.append( + f'{d["value"]:.1f}{unit}' + ) + parts.append(_hit(0, sy, VB_W, BAR_H, d["tip_title"], d["tip_rows"])) + if row.get("delta_text"): + parts.append(_chip(VB_W, cy, row["delta_text"], "good" if row.get("delta_good") else "secondary")) + parts.append("") + return "".join(parts) + + +def _single_summary(rows): + return f"Bar chart of {len(rows)} HEAD-only measurement(s) with no baseline equivalent." + + +def single_series_bar_chart(rows, unit=" ms"): + if not rows: + return '' + label_w = _clamp(max(_measure(r["label"], 12.5) for r in rows) + LABEL_GAP, 120, 300) + chip_w = _clamp( + max((_measure(r["badge_text"], 10.5) + 24 for r in rows if r.get("badge_text")), default=0), 0, 320 + ) + gap = CHIP_GAP if chip_w else 0 + x0, x1 = label_w, VB_W - chip_w - gap + plot_w = max(x1 - x0, 40) + data_max = max(max(r["value"], r.get("tick") or 0) for r in rows) + ticks = nice_ticks(data_max, 4) + domain_max = (ticks[-1] or 1) * 1.24 + row_pitch = BAR_H + ROW_GAP + top = TOP_PAD + bottom = top + row_pitch * len(rows) - ROW_GAP + height = bottom + AXIS_H + + def sx(v): + return x0 + (v / domain_max) * plot_w + + parts = [ + f'', + _axis(sx, ticks, top, bottom, unit), + ] + for i, row in enumerate(rows): + y = top + i * row_pitch + cy = y + BAR_H / 2 + parts.append( + f'{esc(row["label"])}' + ) + w = max((row["value"] / domain_max) * plot_w, 0) + parts.append(_bar(x0, y, w, BAR_H, "bar-head")) + if row.get("tick") is not None: + parts.append(_tick(sx(row["tick"]), y, BAR_H)) + parts.append( + f'{row["value"]:.1f}{unit}' + ) + if row.get("badge_text"): + parts.append(_chip(VB_W, cy, row["badge_text"], "info", size=10.5)) + parts.append(_hit(0, y, VB_W, BAR_H, row["tip_title"], row["tip_rows"])) + parts.append("") + return "".join(parts) + + +def _speedup_summary(rows): + best = max(rows, key=lambda r: r["speedup"]) + worst = min(rows, key=lambda r: r["speedup"]) + return ( + f"Horizontal speedup bars for {len(rows)} synthetic scenarios, live find path versus legacy snapshot path. " + f"Speedups range from {worst['speedup']:.1f}x to {best['speedup']:.1f}x against a 1.0x parity reference." + ) + + +def speedup_bar_chart(rows): + if not rows: + return '' + badge_text = f"{CHECK} legacy answered wrong - live path correct" + label_w = _clamp(max(_measure(r["label"], 11.5) for r in rows) + LABEL_GAP, 140, 340) + chip_w = _clamp(_measure(badge_text, 10.5) + 24 if any(r.get("correctness_fix") for r in rows) else 0, 0, 340) + gap = CHIP_GAP if chip_w else 0 + x0, x1 = label_w, VB_W - chip_w - gap + plot_w = max(x1 - x0, 40) + data_max = max([r["speedup"] for r in rows] + [1.0]) + ticks = nice_ticks(data_max, 4) + domain_max = (ticks[-1] or 1) * 1.2 + row_pitch = BAR_H + ROW_GAP + 14 + top = TOP_PAD + 6 + bottom = top + row_pitch * len(rows) - ROW_GAP - 14 + height = bottom + AXIS_H + + def sx(v): + return x0 + (v / domain_max) * plot_w + + parts = [ + f'', + _axis(sx, [t for t in ticks if abs(t - 1.0) > 1e-9], top, bottom, "", suffix_x=True), + ] + px = sx(1.0) + parts.append(f'') + parts.append( + f'parity' + ) + for i, row in enumerate(rows): + y = top + i * row_pitch + cy = y + BAR_H / 2 + parts.append( + f'{esc(row["label"])}' + ) + w = max((row["speedup"] / domain_max) * plot_w, 0) + parts.append(_bar(x0, y, w, BAR_H, "bar-head")) + end_x = x0 + w + 6 + parts.append( + f'{row["speedup"]:.1f}x' + ) + if row.get("sub_label"): + parts.append( + f'{esc(row["sub_label"])}' + ) + if row.get("correctness_fix"): + parts.append(_chip(VB_W, cy, badge_text, "status-good", size=10.5)) + parts.append(_hit(0, y - 7, VB_W, BAR_H + 14, row["tip_title"], row["tip_rows"])) + parts.append("") + return "".join(parts) diff --git a/scripts/run-native-e2e-ci.sh b/scripts/run-native-e2e-ci.sh new file mode 100755 index 0000000..32b8fc9 --- /dev/null +++ b/scripts/run-native-e2e-ci.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +if [ "$(uname -s)" != "Darwin" ]; then + echo "native E2E runner must be macOS" >&2 + exit 2 +fi +if [ "${AGENT_DESKTOP_NATIVE_E2E_RUNNER:-}" != "1" ]; then + echo "refusing native desktop control without AGENT_DESKTOP_NATIVE_E2E_RUNNER=1" >&2 + exit 2 +fi + +cargo build --locked --release -p agent-desktop +cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper +cargo build --locked --profile release-ffi -p agent-desktop-ffi + +AGENT_DESKTOP_E2E_EXCLUSIVE=1 \ +AGENT_DESKTOP_E2E_RELEASE_BIN="$ROOT/target/release/agent-desktop" \ +AGENT_DESKTOP_E2E_RELEASE_FFI="$ROOT/target/release-ffi/libagent_desktop_ffi.dylib" \ +AGENT_DESKTOP_E2E_RELEASE_FFI_HELPER="$ROOT/target/release/agent-desktop-macos-helper" \ + bash tests/e2e/run.sh diff --git a/skills/agent-desktop-ffi/SKILL.md b/skills/agent-desktop-ffi/SKILL.md index c6d6253..25469e7 100644 --- a/skills/agent-desktop-ffi/SKILL.md +++ b/skills/agent-desktop-ffi/SKILL.md @@ -38,9 +38,9 @@ Four reference topics, loaded as needed: for every `*mut T` the FFI hands back to the caller. - [error-handling.md](references/error-handling.md) — errno-style last-error contract, enum validation, panic boundary. -- [threading.md](references/threading.md) — macOS main-thread rule, - AXIsProcessTrusted inheritance when Python/Node dlopens the cdylib, - and the single-owner handle invariant. +- [threading.md](references/threading.md) — host-thread contract, + cross-process mutation serialization, AXIsProcessTrusted inheritance, + and adapter-bound native handles. - [build-and-link.md](references/build-and-link.md) — ABI handshake, struct size validation, minimal C and Python examples, observe-act workflow, and prebuilt archive locations. @@ -51,19 +51,19 @@ Four reference topics, loaded as needed: ad_init(AD_ABI_VERSION_MAJOR) // verify header ↔ dylib match adapter = ad_adapter_create_with_session("s1") // or ad_adapter_create() rc = ad_snapshot(adapter, "Finder", 0, 10, false, false, &json_out) -// parse json_out: locate @e refs in data.tree, note data.snapshot_id +// parse json_out: locate snapshot-qualified refs in data.tree ad_free_string(json_out) // build action: AdAction act = {0}; act.kind = AD_ACTION_KIND_CLICK; -rc = ad_execute_by_ref(adapter, "@e5", snapshot_id, &act, 0, &result_out) +rc = ad_execute_by_ref(adapter, "@s8f3k2p9:e5", NULL, &act, 0, &result_out) ad_free_string(result_out) ad_adapter_destroy(adapter) ``` `ad_snapshot` returns a `{version, ok, command, data}` JSON envelope -identical to the CLI output. The `data.tree` field contains `@e`-prefixed -ref IDs for interactive elements. Pass a ref ID and the `snapshot_id` -to `ad_execute_by_ref` to drive the CLI-semantics ref-action pipeline +identical to the CLI output. The `data.tree` field contains snapshot-qualified +ref IDs for interactive elements. Pass a qualified ref, or a legacy bare ref +plus its explicit `snapshot_id`, to `ad_execute_by_ref` to drive the pipeline (RefStore load → strict resolution → actionability preflight → dispatch). ## Core constraints @@ -77,9 +77,9 @@ to `ad_execute_by_ref` to drive the CLI-semantics ref-action pipeline - **Session adapters.** `ad_adapter_create_with_session("session-id")` associates the adapter with a session namespace for refmap persistence — the same as CLI - `--session `. Passing the same session ID across adapter lifetimes lets - `ad_execute_by_ref` with `snapshot_id=NULL` target the latest snapshot from - that session. Session IDs: 1–64 chars, ASCII alphanumeric / `-` / `_`. + `--session `. A null `snapshot_id` is valid only for a qualified ref; + legacy bare `@eN` refs require an explicit snapshot ID. Session IDs: 1–64 + chars, ASCII alphanumeric / `-` / `_`. Invalid IDs return null (check `ad_last_error_*`). - **Structured session trace (no ABI change).** File-based JSONL tracing activates @@ -91,16 +91,12 @@ to `ad_execute_by_ref` to drive the CLI-semantics ref-action pipeline reuses the same segment filename for all calls in that process. For unstructured diagnostics regardless of session manifest, use `ad_set_log_callback` (below). -- **Main thread only (macOS).** Call every adapter-touching entrypoint - (`ad_snapshot`, `ad_execute_by_ref`, `ad_wait`, `ad_get_tree`, `ad_find`, - `ad_get`, `ad_is`, `ad_resolve_element`, `ad_execute_action`, - `ad_execute_action_with_policy`, `ad_execute_ref_action_with_policy`, - `ad_screenshot`, clipboard get/set/clear, mouse, drag, launch, close, focus, - window-op, list-apps/windows/surfaces, notification list/dismiss/action) - from the process's main thread. macOS accessibility and Cocoa APIs require - this. The FFI enforces this at runtime in every build profile — a worker-thread - call returns `AD_RESULT_ERR_INTERNAL` with a diagnostic last-error. On - non-macOS platforms the check is a compile-time true; there is no runtime cost. +- **Threading and mutation leases.** Adapter entrypoints may be called from any + host thread. Native handles remain bound to their creating adapter and thread. + Desktop mutations acquire the same canonical cross-process interaction lease + as the CLI; reads carry finite deadlines without taking the mutation lock. See + [threading.md](references/threading.md) for the Apple documentation basis and + the read/read, read/mutation, and mutation/mutation ordering matrix. - **Release profile.** `cargo build --release` produces `panic = "abort"` — any Rust panic inside an `extern "C"` fn will `SIGABRT` the host. Use @@ -119,7 +115,7 @@ to `ad_execute_by_ref` to drive the CLI-semantics ref-action pipeline titles from the user's screen — treat as sensitive diagnostics and avoid routing to shared log surfaces. -- **Handle release.** Every `ad_resolve_element` / `ad_find` result must be +- **Handle release.** Every `ad_resolve_element_exact` / `ad_find_exact` result must be released with `ad_free_handle(adapter, &handle)` on the same adapter that produced it, before that adapter is destroyed. On macOS this balances the internal `CFRetain`; on Windows/Linux the call is a no-op but safe to issue. @@ -133,15 +129,26 @@ to `ad_execute_by_ref` to drive the CLI-semantics ref-action pipeline actions default to `headless`. Pass `AD_POLICY_KIND_HEADED` (2) to opt in to cursor-based fallbacks. +- **Generation-safe direct APIs.** ABI-v3 legacy `AdRefEntry` and + `AdWindowInfo` layouts remain available for binary compatibility, but they do + not carry process-generation evidence and direct targeting functions fail + closed. Use `AdExactRefEntry`, `AdExactWindowInfo`, + `ad_list_windows_exact`, and the `*_exact` targeting symbols. Likewise, + `ad_list_surfaces_exact` preserves `SurfaceInfo.id`; the legacy surface list + is an observation-only projection that omits it. + +- **Display discovery.** Call `ad_list_displays` before using + `AdScreenshotTarget.screen_index`. List order is the screenshot index order; + inspect each `AdDisplayInfo` for its stable display ID, bounds, primary flag, + and scale, then release the handle with `ad_display_list_free`. + - **Low-level action paths.** `ad_execute_action` (headless, no preflight) and `ad_execute_action_with_policy` are raw escape hatches for callers holding a live - `AdNativeHandle` from `ad_resolve_element` / `ad_find`. Use them when you need - to bypass the ref-action pipeline. `ad_execute_ref_action_with_policy` accepts a - pre-resolved `AdRefEntry` instead of a ref string — useful when you have - serialized an entry outside the RefStore pipeline. + `AdNativeHandle` from `ad_resolve_element_exact` / `ad_find_exact`. Use them when + you need to bypass the ref-action pipeline. -- **Ref-action preflight.** `ad_execute_by_ref` and `ad_execute_ref_action_with_policy` - both resolve the element strictly and run the live actionability preflight +- **Ref-action preflight.** `ad_execute_by_ref` and + `ad_execute_ref_action_exact_with_policy` both resolve the element strictly and run the live actionability preflight (visible, stable, enabled, supported action, policy, editable) before dispatching — a disabled or unsupported target fails before any platform call. On `AD_RESULT_ERR_ACTION_FAILED`, the structured check report is available as JSON @@ -192,7 +199,7 @@ to `ad_execute_by_ref` to drive the CLI-semantics ref-action pipeline symbols, new error codes) do not bump it. Before 1.0, pin the exact version of libagent_desktop_ffi you link against. -- **`ad_get_tree` vs `ad_snapshot`.** `ad_get_tree` returns a raw flat BFS tree +- **`ad_get_tree_exact` vs `ad_snapshot`.** `ad_get_tree_exact` returns a raw flat BFS tree without `@e` refs, no refmap persistence, and no JSON envelope — use it for custom traversal or UI inspection. For observe-act agents that drive actions via `ad_execute_by_ref`, always start with `ad_snapshot`. diff --git a/skills/agent-desktop-ffi/references/build-and-link.md b/skills/agent-desktop-ffi/references/build-and-link.md index a2bc4a3..ecfa4a1 100644 --- a/skills/agent-desktop-ffi/references/build-and-link.md +++ b/skills/agent-desktop-ffi/references/build-and-link.md @@ -185,7 +185,7 @@ install_name_tool -change \ ## Observe-act workflow in C -After parsing the snapshot JSON and extracting a ref ID (e.g. `"@e5"`): +After parsing the snapshot JSON and extracting a qualified ref ID: ```c AdAction act = {0}; // zero-init before setting any field @@ -194,8 +194,8 @@ act.kind = AD_ACTION_KIND_CLICK; char *result = NULL; AdResult rc = ad_execute_by_ref( adapter, - "@e5", // ref ID from snapshot data.tree - NULL, // snapshot_id — NULL = latest for this session + "@s8f3k2p9:e5", // qualified ref from snapshot data.tree + NULL, // the qualified ref embeds its snapshot ID &act, 0, // policy = Headless &result @@ -219,15 +219,16 @@ AdAction type_act = {0}; type_act.kind = AD_ACTION_KIND_TYPE_TEXT; type_act.text = "hello"; // TypeText defaults to focus_fallback via ad_execute_by_ref; explicit policy: -rc = ad_execute_by_ref(adapter, "@e3", NULL, &type_act, +rc = ad_execute_by_ref(adapter, "@s8f3k2p9:e3", NULL, &type_act, AD_POLICY_KIND_FOCUS_FALLBACK, &result); ``` -## Call graph reminder +## Threading reminder -All adapter-touching FFI calls must run on the **main thread** on macOS. -For Python that typically means the script's entry point, not a worker -spawned via `threading`. See [threading.md](threading.md). +Adapter entrypoints may be called from any host thread. Keep native handles on +the thread that resolved them and use snapshot-qualified refs across async or +worker boundaries. Mutations are serialized by a canonical cross-process +interaction lease. See [threading.md](threading.md). ## Minimal Python ctypes example @@ -240,7 +241,7 @@ lib = ctypes.CDLL("./target/release-ffi/libagent_desktop_ffi.dylib") # ABI handshake lib.ad_init.restype = c_int32 lib.ad_init.argtypes = [ctypes.c_uint32] -AD_ABI_VERSION_MAJOR = 1 # sync with header macro +AD_ABI_VERSION_MAJOR = 3 # sync with header macro rc = lib.ad_init(AD_ABI_VERSION_MAJOR) assert rc == 0, f"ABI mismatch: rc={rc}" diff --git a/skills/agent-desktop-ffi/references/error-handling.md b/skills/agent-desktop-ffi/references/error-handling.md index a548950..0a040b7 100644 --- a/skills/agent-desktop-ffi/references/error-handling.md +++ b/skills/agent-desktop-ffi/references/error-handling.md @@ -91,10 +91,40 @@ releases may add codes. | `AD_RESULT_ERR_TIMEOUT` | -9 | Wait exceeded deadline | | `AD_RESULT_ERR_INVALID_ARGS` | -10 | Null pointer, bad enum, invalid UTF-8 | | `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`| -11 | Notification index out of range or reordered | -| `AD_RESULT_ERR_INTERNAL` | -12 | Internal failure, off-main-thread, or foreign-subscriber conflict | +| `AD_RESULT_ERR_INTERNAL` | -12 | Internal failure or foreign-subscriber conflict | | `AD_RESULT_ERR_SNAPSHOT_NOT_FOUND` | -13 | Requested snapshot ref store is missing | | `AD_RESULT_ERR_POLICY_DENIED` | -14 | Current action policy blocks this fallback | | `AD_RESULT_ERR_AMBIGUOUS_TARGET` | -15 | Strict re-identification found multiple candidates; re-snapshot | +| `AD_RESULT_ERR_APP_UNRESPONSIVE` | -16 | Read-only liveness probe failed after an uncertain mutation | + +## Ref token validation + +Ref-taking entrypoints accept two canonical forms: + +- `@:e` is a qualified ref. `snapshot_id` may be null. If a + separate snapshot ID is supplied, it must match the embedded ID. +- `@e` is a legacy bare ref. It requires a non-null snapshot ID argument. + +`N` is a positive `u32` written without a sign and with at most 10 decimal +digits. Snapshot IDs are 3–64 ASCII alphanumeric, `-`, or `_` characters. +Malformed tokens, invalid UTF-8, a bare ref without a snapshot ID, or mismatched +embedded and explicit snapshot IDs return `AD_RESULT_ERR_INVALID_ARGS` before +dispatch. A well-formed token whose saved snapshot is absent returns +`AD_RESULT_ERR_SNAPSHOT_NOT_FOUND`; a saved snapshot with no matching local ref +returns `AD_RESULT_ERR_STALE_REF`. + +Snapshot lookup is confined to the adapter's namespace. An adapter created with +`ad_adapter_create_with_session` never searches the global namespace or another +session for a matching snapshot ID. + +## Off-main-thread migration + +ABI v3 removed the blanket macOS main-thread rejection. Adapter entrypoints may +now run on any host thread, so callers must stop treating +`AD_RESULT_ERR_INTERNAL` as an expected off-main-thread result. Native handles +remain thread-affine capabilities: cross-thread, cross-adapter, released, or +revoked handle use returns `AD_RESULT_ERR_INVALID_ARGS`. See +[threading.md](threading.md) for the concurrency and ownership contract. ## Enum validation @@ -108,15 +138,15 @@ enum slot. Affected fields: `AdAction.kind` (`AdActionKind`), `AdTreeOptions.surface` (`AdSnapshotSurface`), `AdScreenshotTarget.kind` (`AdScreenshotKind`), `AdWindowOp.kind` (`AdWindowOpKind`), and the `policy` parameter of `ad_execute_by_ref` / `ad_execute_action_with_policy` -/ `ad_execute_ref_action_with_policy` (`AdPolicyKind`). +/ `ad_execute_ref_action_exact_with_policy` (`AdPolicyKind`). ## Command-backed JSON entrypoints: dual-failure modes `ad_snapshot`, `ad_execute_by_ref`, `ad_wait`, `ad_status`, and `ad_version` have two distinct failure modes: -- **Argument / infrastructure failure** (null adapter, off-main-thread, - invalid UTF-8, bad discriminant, context error): `*out` is set to null, +- **Argument / infrastructure failure** (null adapter, invalid UTF-8, + bad discriminant, context error): `*out` is set to null, no allocation is made, and the last-error slot is the only failure indication. - **Command-level failure** (app not found, STALE_REF, TIMEOUT, etc.): diff --git a/skills/agent-desktop-ffi/references/ownership.md b/skills/agent-desktop-ffi/references/ownership.md index bb1588e..1b56fd6 100644 --- a/skills/agent-desktop-ffi/references/ownership.md +++ b/skills/agent-desktop-ffi/references/ownership.md @@ -34,8 +34,11 @@ dual-failure mode (command-level errors write `ok:false` JSON into | Allocates | Frees with | |--------------------------------------------------------------|-----------------------------------------| | `ad_list_apps(adapter, &list)` | `ad_app_list_free(list)` | +| `ad_list_displays(adapter, &list)` | `ad_display_list_free(list)` | | `ad_list_windows(adapter, app, focused, &list)` | `ad_window_list_free(list)` | +| `ad_list_windows_exact(adapter, app, focused, &list)` | `ad_exact_window_list_free(list)` | | `ad_list_surfaces(adapter, pid, &list)` | `ad_surface_list_free(list)` | +| `ad_list_surfaces_exact(adapter, pid, &list)` | `ad_exact_surface_list_free(list)` | | `ad_list_notifications(adapter, filter, &list)` | `ad_notification_list_free(list)` | | `ad_dismiss_all_notifications(adapter, f, &ok, &fail)` | `ad_notification_list_free` on each, or `ad_dismiss_all_notifications_free(ok, fail)` | @@ -44,14 +47,15 @@ dual-failure mode (command-level errors write `ok:false` JSON into | Allocates | Frees with | |-----------------------------------------------------------|---------------------------------------------------------------------| | `ad_launch_app(adapter, id, timeout, &out)` | `ad_release_window_fields(&out)` — frees interior strings only; the `AdWindowInfo` struct lives on the caller's stack | +| `ad_launch_app_exact(adapter, id, timeout, &out)` | `ad_release_exact_window_fields(&out)` | ### Raw tree and element access | Allocates | Frees with | |----------------------------------------------------------------------------------------|-----------------------------------------| -| `ad_get_tree(adapter, win, opts, &out)` | `ad_free_tree(&out)` | -| `ad_resolve_element(adapter, entry, &handle)` | `ad_free_handle(adapter, &handle)` — zeroes `handle.ptr` so a follow-up call is a no-op | -| `ad_find(adapter, win, query, &handle)` | same as `ad_resolve_element` | +| `ad_get_tree_exact(adapter, win, opts, &out)` | `ad_free_tree(&out)` | +| `ad_resolve_element_exact(adapter, entry, &handle)` | `ad_free_handle(adapter, &handle)` — zeroes `handle.ptr` so a follow-up call is a no-op | +| `ad_find_exact(adapter, win, query, &handle)` | same as `ad_resolve_element_exact` | ### Action results @@ -59,8 +63,8 @@ dual-failure mode (command-level errors write `ok:false` JSON into |----------------------------------------------------------------------------------------|------------------------------| | `ad_execute_action(adapter, handle, action, &out)` | `ad_free_action_result(&out)` | | `ad_execute_action_with_policy(adapter, handle, action, policy, &out)` | `ad_free_action_result(&out)` | -| `ad_execute_ref_action_with_policy(adapter, entry, action, policy, &out)` | `ad_free_action_result(&out)` | -| `ad_notification_action(adapter, idx, expected_app, expected_title, name, &out)` — pass the `app_name`/`title` from `ad_list_notifications` (either may be null) so NC reorder between list and press returns `ERR_NOTIFICATION_NOT_FOUND` instead of pressing a different notification | `ad_free_action_result(&out)` | +| `ad_execute_ref_action_exact_with_policy(adapter, entry, action, policy, &out)` | `ad_free_action_result(&out)` | +| `ad_notification_action(adapter, &request, &out)` — set `request.identity` from `ad_list_notifications` and choose an explicit non-headless policy; reorder mismatches fail closed | `ad_free_action_result(&out)` | ### Clipboard and image buffers @@ -84,9 +88,9 @@ dual-failure mode (command-level errors write `ok:false` JSON into after the platform release, so a follow-up call sees `NULL` and returns `AD_RESULT_OK` without re-entering `CFRelease`. - **Adapters must outlive their handles.** Free every handle with the - same adapter that produced it before calling `ad_adapter_destroy`. - Destroying the adapter first and later freeing its handles is - undefined behavior. + same adapter and on the same thread that produced it before calling + `ad_adapter_destroy`. Destroyed, wrong-adapter, cross-thread, forged, and + already-freed tokens are rejected without dereferencing foreign memory. - Pointers inside a struct (`.id`, `.title`, `.app_name`, each `AdNotificationInfo.body`, etc.) are freed by the struct's owning free function (list_free / release_fields) — do not call @@ -102,7 +106,7 @@ dual-failure mode (command-level errors write `ok:false` JSON into ## Out-param zeroing Every fallible FFI function zeroes its out-param **before** any guard -(pointer validation, main-thread check, UTF-8 validation). On error, +(pointer validation, UTF-8 validation, enum validation). On error, calling the paired free function is safe: all pointers inside are guaranteed null, all counts zero, so the free is a no-op rather than a double-free on a previous caller's allocation. @@ -120,5 +124,5 @@ In particular: `ad_version` write `*out = NULL` before any guard, so on infrastructure failures no allocation is made and `ad_free_string(NULL)` is a safe no-op. -- `ad_*_list` and `ad_resolve_element` / `ad_find` all apply the same +- `ad_*_list` and `ad_resolve_element_exact` / `ad_find_exact` all apply the same pattern to their handle / list out-params. diff --git a/skills/agent-desktop-ffi/references/threading.md b/skills/agent-desktop-ffi/references/threading.md index d28d33d..386699a 100644 --- a/skills/agent-desktop-ffi/references/threading.md +++ b/skills/agent-desktop-ffi/references/threading.md @@ -1,114 +1,137 @@ # Threading -## macOS: main-thread rule +## Host-thread contract -Every adapter-touching entrypoint **must be invoked on the process's -main thread**. macOS accessibility and Cocoa APIs require this. +FFI entrypoints may be called from any host thread. The library does not apply +a blanket macOS main-thread guard to Accessibility (`AXUIElement`) or Quartz +event (`CGEvent`) operations. -Entrypoints subject to the main-thread guard: +That contract follows Apple's published boundaries: -- `ad_snapshot`, `ad_execute_by_ref`, `ad_wait` -- `ad_get_tree`, `ad_find`, `ad_get`, `ad_is`, `ad_resolve_element` -- `ad_execute_action`, `ad_execute_action_with_policy`, - `ad_execute_ref_action_with_policy` -- `ad_screenshot` -- Clipboard: `ad_get_clipboard`, `ad_set_clipboard`, `ad_clear_clipboard` -- Input: `ad_mouse_event`, `ad_drag` -- App / window: `ad_launch_app`, `ad_close_app`, `ad_focus_window`, - `ad_window_op` -- Listing: `ad_list_apps`, `ad_list_windows`, `ad_list_surfaces` -- Notifications: `ad_list_notifications`, `ad_dismiss_notification`, - `ad_dismiss_all_notifications`, `ad_notification_action` +- Apple's [Thread Safety Summary](https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/Multithreading/ThreadSafetySummary/ThreadSafetySummary.html) + says objects restricted to the main thread are called out explicitly and + describes Core Foundation as thread-safe for common immutable query, retain, + release, and transfer operations. +- Apple documents [`AXUIElement`](https://developer.apple.com/documentation/applicationservices/axuielement) + as an accessibility object and its header as a CF type; it does not publish a + blanket main-thread requirement for AX calls. +- Apple documents [`CGEvent`](https://developer.apple.com/documentation/coregraphics/cgevent) + as a CF-derived low-level event type; it likewise does not publish a blanket + main-thread restriction. -The check runs at **runtime, in every build profile** — worker-thread -calls return `AD_RESULT_ERR_INTERNAL` with a `'static` diagnostic -`"agent_desktop FFI entry called off the main thread (macOS requires -main-thread AX/Cocoa calls)"`. No build-config difference; no silent -UB window in release builds. +AppKit view/event-loop rules do not automatically apply to an assistive +application calling AX or Quartz APIs. Apple explicitly says +[`NSWorkspace.shared`](https://developer.apple.com/documentation/appkit/nsworkspace/shared) +is safe to access from any thread, and does not publish a main-thread-only rule +for `NSPasteboard`. -On non-macOS targets the check is a compile-time `true` and has zero -runtime cost. +Code paths that use Cocoa objects create their required autorelease pools +internally. Apple's Thread Safety Summary requires a pool on secondary threads +that use Cocoa and identifies classes such as `NSView` as genuinely +main-thread-only. Because Rust and many foreign runtimes create POSIX threads, +FFI initialization also follows Apple's +[Using POSIX Threads in a Cocoa Application](https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/Multithreading/CreatingThreads/CreatingThreads.html) +guidance by starting one `NSThread` once, which causes Cocoa to install its +multithreading locks before worker-thread AppKit calls. -## Operations safe off the main thread +## Concurrency and mutation ordering -These functions carry no runtime main-thread guard: +Adapter registry handles can be acquired concurrently. Read operations carry a +finite deadline and do not take the interaction lease. -- `ad_adapter_create` / `ad_adapter_create_with_session` / `ad_adapter_destroy` -- `ad_init`, `ad_abi_version` -- `ad_version` (no adapter; pure serialization) -- `ad_status` (reads permission report and ref-store metadata only; no AX tree query) -- `ad_check_permissions` (pure process-wide AX trust query) -- `ad_set_log_callback` -- `ad_last_error_code`, `ad_last_error_message`, `ad_last_error_suggestion`, - `ad_last_error_platform_detail`, `ad_last_error_details` -- List accessors: `ad_app_list_count` / `_get` / `_free`, - `ad_window_list_count` / `_get` / `_free`, - `ad_surface_list_count` / `_get` / `_free`, - `ad_notification_list_count` / `_get` / `_free` -- `ad_image_buffer_data` / `_size` / `_width` / `_height` / `_format` / `_free` -- `ad_release_window_fields` -- `ad_free_handle` (invokes `CFRelease` which is thread-safe — but - still prefer calling from the thread that produced the handle) -- `ad_free_tree`, `ad_free_action_result`, `ad_free_string` +| Concurrent calls | Contract | +|------------------|----------| +| read + read | Both may run concurrently; neither waits for the interaction lease | +| read + mutation | The read does not wait for the lease and may overlap the mutation; its result is a point-in-time observation, not a transaction | +| mutation + mutation | One lease holder runs at a time for the same OS user; the waiter consumes its command deadline and returns a structured timeout if it cannot acquire the lease | +| native-handle call on another thread | Rejected with `AD_RESULT_ERR_INVALID_ARGS`; use a qualified ref instead | + +Snapshots, finds, gets, display/window/app listings, screenshots, clipboard +reads, permission reads, status, and trace reads are observations. Ref actions, +direct native-handle actions, input synthesis, clipboard writes/clear, app and +window changes, notification actions, and permission requests are mutations. +For example, `ad_snapshot` and `ad_execute_by_ref` may be called from different +threads, but the snapshot can overlap the action; callers that require +observe-then-act ordering must wait for the snapshot result before dispatching +the action. + +Desktop mutations take a process-independent advisory lock at +`/tmp/agent-desktop-/interaction.lock`. Current CLI and FFI builds therefore +serialize mutations across threads, processes, and different HOME values for +the same user. The lock is command-scoped, not transaction-scoped: another +actor may change the UI between an observation and a later action, or between +two independently invoked actions. + +This ordering is implemented by agent-desktop's in-process process guard plus a +Unix advisory file lock, not by `AXObserver`, an AppKit run loop, or a global +Apple accessibility mutex. On non-Unix platforms, each adapter must provide the +same `PlatformAdapter::acquire_interaction_lease` contract with its native +serialization primitive. + +The lease cannot coordinate: + +- older `agent-desktop` binaries that predate the lock; +- direct human input or unrelated automation tools; +- state changes initiated by the target application itself. + +Callers must still use strict refs/exact window identities and treat stale or +ambiguous targets as normal retryable automation outcomes. + +## Adapter destruction + +Adapter pointers are opaque registry tokens. Each call acquires a retained +adapter owner before platform work. `ad_adapter_destroy` revokes the token: +calls that already acquired it finish safely, while calls that begin afterward +return `AD_RESULT_ERR_INVALID_ARGS`. Concurrent destruction cannot free memory +still referenced by an in-flight call. + +Destroying an adapter also revokes native handles created by that adapter on the +calling thread. Other threads' handle registries reject subsequent use because +the owner adapter token no longer exists. + +## Native-handle thread ownership + +`ad_resolve_element_exact` and `ad_find_exact` return an opaque +`AdNativeHandle`. Native handles are adapter-bound and thread-affine: + +- resolve, use, and release a handle on the same thread; +- pass the same adapter that produced the handle; +- do not use a handle after destroying its adapter. + +Violations are rejected with `AD_RESULT_ERR_INVALID_ARGS`; the library does not +dereference forged, cross-adapter, cross-thread, released, or revoked tokens. +Prefer snapshot-qualified refs and `ad_execute_by_ref` when a handle would need +to cross an async task or thread boundary. ## Log callback threading -`ad_set_log_callback(cb)` is safe to call from any thread (no AX -involvement). However, the installed callback may be invoked from threads -other than the registering thread — tracing events can originate from any -thread that calls an `ad_*` function. +`ad_set_log_callback(cb)` may be called from any thread. The callback may be +invoked from any thread that calls an `ad_*` function. -A callback unregistered via `NULL` may still receive one or more -invocations from a concurrent thread for a brief window after this call -returns. The callback (and any data it captures) must remain valid for the -process lifetime, or the caller must quiesce all active adapter calls -before unregistering. +A callback unregistered via `NULL` may still receive an invocation already in +flight on another thread. Keep the callback and captured state alive for the +process lifetime, or quiesce active adapter calls before unregistering it. -## Python consumers +## Language runtimes -CPython's GIL serializes calls but does not pin them to the main thread. -If you're calling from anything other than the main interpreter thread -you will silently corrupt state on macOS. +Runtime serialization is not thread affinity. For example, CPython's GIL does +not guarantee that two FFI calls execute on the same OS thread. Store native +handles only in thread-confined objects, or avoid them and use +snapshot-qualified refs. Rust, Swift, Node, Go, and managed runtimes need the +same discipline when tasks can migrate between worker threads. -Two patterns work: +## Accessibility permission identity -1. **Restrict FFI calls to the main thread.** Use `asyncio` with the - default event loop pinned to main, or a synchronous entrypoint only. -2. **Marshal across threads yourself.** Use a queue; have a dedicated - main-thread worker that dequeues and invokes the FFI. +`ad_check_permissions` calls macOS `AXIsProcessTrusted()`, which reports trust +for the hosting executable (`python3`, `node`, a Swift app, and so on), not for +the dylib as a separate executable. Permission prompts and deployment guidance +must identify the host process that loads `libagent_desktop_ffi.dylib`. -## AXIsProcessTrusted inheritance +## Last error and blocking calls -`ad_check_permissions` calls macOS's `AXIsProcessTrusted()`, which -returns the trust status of the **hosting executable** — i.e., the -`python3` / `node` / `swift` process, not `agent-desktop` itself. +The last-error slot is thread-local. Thread A's failure does not change thread +B's error state. -Consequence: granting accessibility permission to one Python script's -Python interpreter grants it to every Python script that dlopens -`libagent_desktop_ffi.dylib`. Document this prominently for your -consumers; consider requiring opt-in permission prompts in host code -rather than relying on macOS's binary-level grant. - -## Thread-ownership of handles - -`ad_resolve_element` and `ad_find` return an opaque `AdNativeHandle` -that wraps a platform pointer. The handle is **single-owner, -single-thread** by FFI contract: - -- Create it on thread A → free it on thread A with `ad_free_handle`. - Transferring the handle to thread B is undefined behavior. -- Use it in FFI calls only from the same thread that produced it. - -## Last-error is thread-local - -Every thread has its own last-error slot. Thread A's failure does not -set thread B's last-error; `ad_last_error_*` accessors always see the -calling thread's state. - -## ad_wait lifecycle - -`ad_wait` blocks the calling thread for up to `timeout_ms` milliseconds -while it holds a live reference into the adapter's allocation. Do not -call `ad_adapter_destroy` on the same handle from another thread while -`ad_wait` is running — that is a use-after-free. Ensure the wait has -returned before destroying the adapter. +`ad_wait` blocks its calling thread for at most its finite deadline and retains +the adapter for that duration. Destroying the adapter token concurrently stops +new calls but does not invalidate an in-flight wait. diff --git a/skills/agent-desktop/SKILL.md b/skills/agent-desktop/SKILL.md index 2afd28e..ead5746 100644 --- a/skills/agent-desktop/SKILL.md +++ b/skills/agent-desktop/SKILL.md @@ -9,9 +9,10 @@ description: > Use when an AI agent needs to observe, interact with, or automate desktop applications (click buttons, fill forms, navigate menus, read UI state, toggle checkboxes, scroll, drag, type text, take screenshots, manage windows, use clipboard, manage notifications). - Covers 56 commands across observation, interaction, keyboard/mouse, app lifecycle, - notifications (macOS), clipboard, wait, session lifecycle, and a `skills` command - bundled docs straight from the binary. + Covers 58 command names (54 operational; four held-input names fail closed until + daemon ownership exists) across observation, interaction, keyboard/mouse, app + lifecycle, notifications (macOS), clipboard, wait, session lifecycle, and a + `skills` command that bundles docs straight from the binary. Triggers on: "click button", "fill form", "open app", "read UI", "automate desktop", "accessibility tree", "snapshot app", "type into field", "navigate menu", "toggle checkbox", "take screenshot", "desktop automation", "agent-desktop", or any desktop GUI interaction task. @@ -82,30 +83,32 @@ Use **progressive skeleton traversal** as the default approach. It reduces token ## Ref System -- Refs assigned depth-first: `@e1`, `@e2`, `@e3`... +- Refs are assigned depth-first and emitted with their snapshot, for example `@s8f3k2p9:e1`, `@s8f3k2p9:e2`, `@s8f3k2p9:e3`. Legacy bare refs require an explicit `--snapshot`. - An element gets a ref when it is addressable for an action: an interactive role (button, textfield, checkbox, link, menuitem, tab, slider, combobox, treeitem, cell, radiobutton, switch, ...) **or** any element advertising an action — so `scrollarea` (Scroll) and `disclosure` (Expand/Collapse) are ref-able and `scroll`/`expand`/`collapse` can target them - A `SetFocus`-only affordance does not earn a ref on its own - In skeleton mode, named/described containers at truncation boundary also get refs (drill-down targets with empty `available_actions`) - Static text and non-actionable groups/containers remain in tree for context but have no ref - Refs are deterministic within a snapshot but NOT stable across snapshots if UI changed -- Every snapshot returns `snapshot_id`; ref-consuming commands accept `--snapshot `, and explicit snapshot IDs do not require also passing `--session` +- Snapshot output uses qualified refs that embed `snapshot_id` and need no separate `--snapshot`; a session-owned ref still requires the same `--session` or `AGENT_DESKTOP_SESSION` scope because lookup never crosses namespaces - `last_refmap.json` is only a latest-snapshot inspection artifact. The command path uses snapshot-scoped storage. - After any action that changes UI, re-drill the affected region or re-snapshot -- **Scoped invalidation:** re-drilling `--root @e3` only replaces refs from @e3's previous drill — refs from other regions and the skeleton itself are preserved +- **Scoped invalidation:** re-drilling a qualified root ref only replaces refs from that root's previous drill — refs from other regions and the skeleton itself are preserved - **Strict resolution:** stale refs return `STALE_REF`; duplicate plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily. -- **Actionability:** ref actions check live visibility, stability, enabled state, supported action, policy, and editability before dispatch. -- **Headless vs headed:** ref actions are headless by default (AX-only, no cursor) and fail closed when only a physical gesture would work. `type` uses a focus-fallback base policy because typing needs focus but never moves the cursor. Pass the global `--headed` flag to permit cursor movement and focus stealing so physical fallbacks can complete; the AX path is still tried first, so `--headed` never regresses headless-capable elements. Raw cursor commands (`hover`, `drag`, `mouse-*`) are physical and require `--headed`; keyboard commands (`press`, `key-down`, `key-up`) are explicit low-level input. -- **Sessions and tracing:** run `session start` once per agent run to create a manifest with `trace: on` (default). Use `session start --screenshots` when you need replay artifacts (`artifacts: full`): pre/post-action PNGs and refmap copies under the session trace directory (sensitive — treat exports like screenshots). Subsequent commands record JSONL automatically to per-process segments under `~/.agent-desktop/sessions//trace/-.jsonl` — no `--trace` on every call. Read traces back with `trace show` (bounded JSON for agents) or `trace export` (single-file HTML for humans). A session owns both its trace and its latest-snapshot namespace; activating it (via pointer, env, or flag) relocates implicit "latest" to that session. Explicit `--snapshot ` still resolves cross-session. **`--session ` alone** (no manifest from `session start`) selects only the snapshot namespace — existing callers see no surprise trace files. **`--trace `** still overrides to one atomic file for CI or one-offs. Activation precedence: `--session` > `AGENT_DESKTOP_SESSION` > `~/.agent-desktop/current_session` (written only by `session start`). Concurrent independent agents set `AGENT_DESKTOP_SESSION` per process; the pointer is a single-active-session convenience. Multi-agent shared sessions: each agent acts on the `snapshot_id` from its own `snapshot` call — implicit latest is not a cross-agent guarantee. Run `status` to see `session_id` and `tracing`. Trace lines include `ts_ms`, monotonic per-process `seq`, and redacted sensitive fields (`text`, `value`, `expected`, `name`, `username`, `description`, `label`, `query`, `secret`, `token`, `password`, `title`, `url`, `help`, `placeholder` → `{ "redacted": true }`). `--trace-strict` fails on trace setup and pre-action writes; post-action success traces are best-effort. +- **Actionability:** every ref-addressed action checks its applicable live visibility, stability, enabled, editability, policy, supported-action, and hit-test requirements under one bounded budget before a single dispatch. Pointer actions focus before their final geometry read, re-resolve moving endpoints, and return `TIMEOUT` with `details.kind: "actionability_timeout"` instead of sending input after the deadline. +- **Headless vs headed:** ref actions are strictly headless by default: semantic accessibility APIs only, with no focus stealing, cursor movement, or synthesized keyboard input. In headed mode, core focuses the exact ref window before dispatch; pointer actions also require a verified target point, while the adapter owns OS delivery. On macOS, `click`, `right-click`, `type`, `clear`, and `scroll` are physical-first; double/triple-click, hover, and drag are physical-only; expand/collapse and other semantic actions remain semantic. Raw `--xy` input has no window identity and never steals focus. `press` is explicit physical keyboard input; held-input commands (`key-down`, `key-up`, `mouse-down`, `mouse-up`) are reserved and fail closed in the stateless CLI. +- **Sessions and tracing:** run `session start` once per agent run to create a manifest with `trace: on` (default), then pass its returned ID with `--session` or `AGENT_DESKTOP_SESSION`. Use `session start --screenshots` when you need replay artifacts (`artifacts: full`): pre/post-action PNGs and refmap copies under the session trace directory (sensitive — treat exports like screenshots). Commands in that explicit scope record JSONL automatically to per-process segments under `~/.agent-desktop/sessions//trace/-.jsonl` — no `--trace` on every call. Read traces back with `trace show` (bounded JSON for agents) or `trace export` (single-file HTML for humans). A session owns both its trace and its latest-snapshot namespace. Snapshot lookup never searches another namespace. **`--session ` alone** (no manifest from `session start`) selects only the snapshot namespace — existing callers see no surprise trace files. **`--trace `** still overrides to one atomic file for CI or one-offs. Activation precedence is `--session` > `AGENT_DESKTOP_SESSION` > no session; `session start` does not activate later processes. Multi-agent shared sessions: each agent acts on qualified refs from its own snapshot — implicit latest is not a cross-agent guarantee. Run `status` to see `session_id` and `tracing`. Trace lines include `ts_ms`, monotonic per-process `seq`, and redacted sensitive fields (`text`, `value`, `expected`, `name`, `username`, `description`, `label`, `query`, `secret`, `token`, `password`, `title`, `url`, `help`, `placeholder` → `{ "redacted": true }`). `--trace-strict` fails on trace setup and pre-action writes; post-action success traces are best-effort. ## JSON Output Contract Every command returns a JSON envelope on stdout: -**Success:** `{ "version": "2.0", "ok": true, "command": "snapshot", "data": { ... } }` -**Error:** `{ "version": "2.0", "ok": false, "command": "click", "error": { "code": "STALE_REF", "message": "...", "suggestion": "..." } }` +**Success:** `{ "version": "2.1", "ok": true, "command": "snapshot", "data": { ... } }` +**Error:** `{ "version": "2.1", "ok": false, "command": "click", "error": { "code": "STALE_REF", "message": "...", "suggestion": "..." } }` The `error` object may also carry an optional `details` object (e.g. the actionability report on an actionability failure, candidate summaries on `AMBIGUOUS_TARGET`, or the last observed state on a `wait` `TIMEOUT`). Parse errors leniently — `details` and future fields are additive, so do not reject responses with unknown keys. +An actionability failure on a hit-test action (`click`, `double-click`, `right-click`, `triple-click`, `hover`, `drag`) can carry a `receives_events` check with `reason: "occluded by "` plus a structured `occluder: { "role", "name", "bounds" }` — another element is on top of the target. Bring the target window/element to the front (or dismiss the occluder) rather than blind-retrying; see `references/commands-interaction.md` for the full check list. + Exit codes: `0` success, `1` structured error, `2` argument error. ### Error Codes @@ -121,21 +124,22 @@ Exit codes: `0` success, `1` structured error, `2` argument error. | `AMBIGUOUS_TARGET` | Multiple elements matched the old ref identity | Re-run snapshot and choose a more specific ref | | `SNAPSHOT_NOT_FOUND` | Snapshot ID is missing or expired | Run `snapshot` again and use the returned ID | | `POLICY_DENIED` | A physical/headed path was blocked | Use an explicit mouse/focus/keyboard command if physical interaction is intended | +| `APP_UNRESPONSIVE` | A read-only AX liveness probe also failed after an uncertain mutation response | Inspect with a fresh snapshot and wait for the app to recover before deciding whether to retry | | `WINDOW_NOT_FOUND` | No matching window | Check app name, use list-windows | | `PLATFORM_NOT_SUPPORTED` | Adapter method not implemented on this platform | Use a supported platform adapter | -| `TIMEOUT` | Wait condition not met | Increase `--timeout` for a predicate wait; for a chain deadline (`error.details.kind == "chain_deadline"`), increase `AGENT_DESKTOP_CHAIN_TIMEOUT_MS` — `--timeout` has no effect on chain deadlines | +| `TIMEOUT` | Wait or actionability condition not met | Inspect `error.details.kind`; increase the command budget only after checking the last report, and use `AGENT_DESKTOP_CHAIN_TIMEOUT_MS` only for `chain_deadline` | | `INVALID_ARGS` | Bad arguments | Check command syntax | | `NOTIFICATION_NOT_FOUND` | Notification index no longer exists | Re-run list-notifications | | `INTERNAL` | Unexpected platform/OS failure (e.g. event synthesis failed) | Read `message`/`suggestion` for cleanup state, then retry once; persistent failures indicate an environment problem | `TIMEOUT` errors carry a `details` object whose `kind` field selects the schema. `kind: "wait_timeout"` includes `predicate`, `timeout_ms`, and `last_observed` or `last_error`, plus `ref`/`title`/`text_chars` depending on the wait mode. `kind: "chain_deadline"` includes `value_before`, `value_at_timeout`, `target`, and `mutated` (increment waits) or `wanted_expanded`/`observed_expanded` (disclosure waits). `mutated: true` — or an unknown `observed_expanded` state — means re-read the element before retrying; `mutated: false` means the state did not change and retrying directly is safe. -## Command Quick Reference (56 commands) +## Command Quick Reference (58 names, 54 operational) ### Observation ``` agent-desktop snapshot --skeleton --app "App" -i --compact # Skeleton overview (preferred) -agent-desktop snapshot --root @e3 -i --compact # Drill into region +agent-desktop snapshot --root @s8f3k2p9:e3 -i --compact # Drill into region agent-desktop snapshot --app "App" -i # Full tree (simple apps) agent-desktop snapshot --app "App" --surface menu -i # Surface snapshot agent-desktop screenshot --app "App" out.png # PNG screenshot @@ -147,39 +151,37 @@ agent-desktop list-surfaces --app "App" # Available surfaces ### Interaction ``` -agent-desktop click @e5 --snapshot # AX-first click, no cursor move by default -agent-desktop double-click @e3 # AXOpen; physical double-click uses --headed mouse-click --count 2 -agent-desktop triple-click @e2 # Physical triple-click uses mouse-click --count 3 -agent-desktop right-click @e5 # Right-click; menu returned when verified +agent-desktop click @e5 --snapshot # Headless semantic click +agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click +agent-desktop --headed triple-click @s8f3k2p9:e2 # physical triple-click +agent-desktop right-click @s8f3k2p9:e5 # Right-click; inspect the resulting menu/effect separately agent-desktop type @e2 --snapshot "hello" # Headless AX text insertion when supported -agent-desktop set-value @e2 "new value" # Set value directly -agent-desktop clear @e2 # Clear element value -agent-desktop focus @e2 # Set keyboard focus -agent-desktop select @e4 "Option B" # Select dropdown/list option -agent-desktop toggle @e6 # Toggle checkbox/switch -agent-desktop check @e6 # Idempotent check -agent-desktop uncheck @e6 # Idempotent uncheck -agent-desktop expand @e7 # Expand disclosure -agent-desktop collapse @e7 # Collapse disclosure -agent-desktop scroll @e1 --direction down # Scroll element -agent-desktop scroll-to @e8 # Scroll into view +agent-desktop set-value @s8f3k2p9:e2 "new value" # Set value directly +agent-desktop clear @s8f3k2p9:e2 # Clear element value +agent-desktop focus @s8f3k2p9:e2 # Set keyboard focus +agent-desktop select @s8f3k2p9:e4 "Option B" # Select dropdown/list option +agent-desktop toggle @s8f3k2p9:e6 # Toggle checkbox/switch +agent-desktop check @s8f3k2p9:e6 # Idempotent check +agent-desktop uncheck @s8f3k2p9:e6 # Idempotent uncheck +agent-desktop expand @s8f3k2p9:e7 # Expand disclosure +agent-desktop collapse @s8f3k2p9:e7 # Collapse disclosure +agent-desktop scroll @s8f3k2p9:e1 --direction down # Scroll element +agent-desktop scroll-to @s8f3k2p9:e8 # Scroll into view ``` ### Keyboard & Mouse ``` agent-desktop press cmd+c # Key combo agent-desktop press return --app "App" # Targeted key press -agent-desktop key-down shift # Hold key -agent-desktop key-up shift # Release key -agent-desktop --headed hover @e5 # Explicit cursor movement +agent-desktop --headed hover @s8f3k2p9:e5 # Explicit cursor movement agent-desktop --headed hover --xy 500,300 # Cursor to coordinates -agent-desktop --headed drag --from @e1 --to @e5 # Drag between elements +agent-desktop --headed drag --from @s8f3k2p9:e1 --to @s8f3k2p9:e5 # Drag between elements agent-desktop --headed mouse-click --xy 500,300 # Click at coordinates agent-desktop --headed mouse-move --xy 100,200 # Move cursor -agent-desktop --headed mouse-down --xy 100,200 # Press mouse button -agent-desktop --headed mouse-up --xy 300,400 # Release mouse button ``` +`key-down`, `key-up`, `mouse-down`, and `mouse-up` return `ACTION_NOT_SUPPORTED` until a stateful daemon can own held-input lifetime. Use `press`, `mouse-click`, or `drag` instead. + ### App & Window ``` agent-desktop launch "System Settings" # Launch and wait @@ -195,17 +197,23 @@ agent-desktop maximize --app "App" agent-desktop restore --app "App" ``` +Use `--window-id ` from `list-windows` instead of `--app` when an app has multiple windows. + ### Notifications ``` -agent-desktop list-notifications # List all notifications -agent-desktop list-notifications --app "Slack" # Filter by app -agent-desktop list-notifications --text "deploy" --limit 5 # Filter by text -agent-desktop dismiss-notification 1 # Dismiss by index -agent-desktop dismiss-all-notifications # Dismiss all -agent-desktop dismiss-all-notifications --app "Slack" # Dismiss all from app -agent-desktop notification-action 1 "Reply" --expected-app Slack # Click action (with NC reorder guard) +agent-desktop --headed list-notifications # Open center if needed, then list +agent-desktop --headed list-notifications --app "Slack" # Filter by app +agent-desktop --headed list-notifications --text "deploy" --limit 5 # Filter by text +agent-desktop --headed dismiss-notification 1 --expected-app Slack --expected-title "Deploy complete" +agent-desktop --headed dismiss-all-notifications # Dismiss all +agent-desktop --headed dismiss-all-notifications --app "Slack" # Dismiss all from app +agent-desktop --headed notification-action 1 "Reply" --expected-app Slack --expected-title "Deploy complete" ``` +Every notification mutation requires global `--headed`; single-notification +mutations also require an app or title fingerprint from the same listing. +Headless listing works only while Notification Center is already open. + ### Clipboard ``` agent-desktop clipboard-get # Read clipboard @@ -217,8 +225,8 @@ agent-desktop clipboard-clear # Clear clipboard ``` agent-desktop wait 1000 # Pause 1 second agent-desktop wait --element @e5 --snapshot --timeout 5000 # Wait for element -agent-desktop wait --element @e5 --predicate actionable --timeout 5000 # Wait until actionable -agent-desktop wait --element @e5 --predicate value --value "Done" --timeout 5000 # Wait for value +agent-desktop wait --element @s8f3k2p9:e5 --predicate actionable --timeout 5000 # Wait until actionable +agent-desktop wait --element @s8f3k2p9:e5 --predicate value --value "Done" --timeout 5000 # Wait for value agent-desktop wait --window "Title" # Wait for window agent-desktop wait --text "Done" --app "App" # Wait for text agent-desktop wait --menu --app "App" # Wait for menu surface @@ -228,15 +236,15 @@ agent-desktop wait --notification --app "App" # Wait for new notification ### System ``` -agent-desktop session start [--name LABEL] [--screenshots] [--no-trace] [--force] # Trace-enabled session; --screenshots enables replay artifacts (sensitive) -agent-desktop session end [id] # Seal manifest, clear pointer +agent-desktop session start [--name LABEL] [--screenshots] [--no-trace] # Creates a session; pass the returned ID explicitly +agent-desktop session end [id] # Seal manifest agent-desktop session list # List session manifests agent-desktop session gc [--older-than SECS] [--ended] # Reclaim ended/stale sessions agent-desktop trace show [--limit N] [--event PREFIX] # Merge trace segments (default tail 500; 0 = all) agent-desktop trace export [--out path.html] [--limit N] # Self-contained HTML viewer (default tail 5000) agent-desktop status # Health, session_id, tracing, artifacts, permissions agent-desktop permissions # Check permission -agent-desktop permissions --request # Trigger permission dialog +agent-desktop permissions --request # Request missing permissions in an isolated helper agent-desktop version # Version info (always JSON envelope) agent-desktop batch '[...]' --stop-on-error # Batch uses the same typed command path as CLI agent-desktop skills # List bundled skill docs @@ -248,13 +256,13 @@ agent-desktop skills get desktop --full # Load this skill + all referenc 1. **Skeleton first, drill second.** Start with `--skeleton -i --compact` for dense apps. Drill into regions with `--root @ref`. Full snapshot only for simple apps. 2. **Use `-i --compact` flags.** Filters to interactive elements and collapses empty wrappers, minimizing tokens. 3. **Refs are snapshot-scoped.** Keep `snapshot_id` for deterministic multi-step use; re-drill the affected region after any UI-changing action. Scoped invalidation keeps other refs intact. -4. **Prefer refs over coordinates.** `click @e5` > `agent-desktop --headed mouse-click --xy 500,300`. +4. **Prefer refs over coordinates.** `click @s8f3k2p9:e5` > `agent-desktop --headed mouse-click --xy 500,300`. 5. **Use `wait` for async UI.** After launch/dialog triggers, wait for expected state. 6. **Check permissions first.** Run `permissions` on first use; screenshots also need Screen Recording. 7. **Handle errors.** Branch on `error.code` only — `error.message` and `error.suggestion` text is informational and may change between versions. 8. **Use `find` for targeted searches.** Faster than any snapshot when you know role/name. 9. **Use surfaces for overlays.** `snapshot --surface menu` for menus, `--surface sheet` for dialogs. Never `--skeleton` for surfaces — they're already focused. 10. **Batch for performance.** Multiple commands in one invocation. -11. **Headless by default.** Ref actions use semantic AX paths and block silent focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. Use explicit `focus`, `press`, `hover`, `drag`, or `mouse-*` commands only when physical/headed interaction is intended. -12. **Start a session once per run.** `session start` enables automatic tracing and relocates the latest-snapshot namespace. Use `AGENT_DESKTOP_SESSION` for concurrent independent agents; use `--session ` to override the active pointer for a single command. +11. **Headless by default.** Ref actions use semantic AX paths and block silent focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. Use `--headed` only when exact-window focus or physical delivery is intended; raw coordinates never imply focus. +12. **Start a session once per run.** `session start` creates the manifest; pass its returned ID through `AGENT_DESKTOP_SESSION` for the run or `--session ` for one command. It does not activate later processes implicitly. 13. **Trace hard failures.** With an active trace-enabled session, segments are written automatically. Add `--trace /tmp/agent-desktop.jsonl` only when you need a single override file (CI, one-offs). Check `status` when unsure whether tracing is active. diff --git a/skills/agent-desktop/references/commands-interaction.md b/skills/agent-desktop/references/commands-interaction.md index 410122a..98f0499 100644 --- a/skills/agent-desktop/references/commands-interaction.md +++ b/skills/agent-desktop/references/commands-interaction.md @@ -6,10 +6,10 @@ Commands for modifying UI state — clicking, typing, selecting, scrolling, and Ref-based actions run in two modes, Playwright-style: -- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the AX path cannot perform the action it fails closed rather than reaching for OS input synthesis. (`type` is the one exception: its base tier may focus the target field — required for reliable typing — but still never moves the cursor.) -- **`--headed`.** A global flag (`agent-desktop --headed click @e5`) that upgrades every ref action to permit focus stealing **and** cursor movement, unlocking the physical click/double-click/scroll/keypress fallbacks in the action chain. The AX path is still tried first, so `--headed` never regresses elements that work headlessly — it only adds fallbacks for elements that need a real gesture (e.g. a gesture-only button with no `AXOpen`). +- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the semantic path cannot perform the action it fails closed. +- **`--headed`.** A global flag (`agent-desktop --headed click @s8f3k2p9:e5`) that authorizes the action's core-owned preconditions. Ref actions that need keyboard delivery focus the exact source window; pointer actions focus that window and require a verified target point before the adapter runs. On macOS, `click`, `right-click`, `type`, `clear`, and `scroll` are physical-first; `double-click`, `triple-click`, `hover`, and `drag` are physical-only. `expand`, `collapse`, `set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, and `scroll-to` stay semantic. -Raw-input commands (`press`, `hover`, `drag`, `mouse-*`, `key-down`, `key-up`) are physical by nature. Cursor-moving commands (`hover`, `drag`, `mouse-*`) require `--headed`; keyboard commands are explicit low-level input. +`press` is explicit physical keyboard input. `hover`, `drag`, `mouse-move`, `mouse-click`, and `mouse-wheel` are explicit physical cursor input and require `--headed`. Raw coordinates carry no window identity, so they never focus an app. The held-input names (`key-down`, `key-up`, `mouse-down`, `mouse-up`) are reserved and return `ACTION_NOT_SUPPORTED` until a stateful daemon can own the hold lifetime. `--headed` is a global flag and also applies to every `batch` entry. @@ -19,8 +19,8 @@ Three global flags poll the accessibility tree until a compact selector matches ```bash agent-desktop snapshot --app Finder -w "button:OK" -agent-desktop click @e5 -w ":Saved!" -agent-desktop click @e5 --wait-for-gone "progressindicator" --wait-timeout 5000 +agent-desktop click @s8f3k2p9:e5 -w ":Saved!" +agent-desktop click @s8f3k2p9:e5 --wait-for-gone "progressindicator" --wait-timeout 5000 ``` | Flag | Short | Default | Meaning | @@ -31,7 +31,7 @@ agent-desktop click @e5 --wait-for-gone "progressindicator" --wait-timeout 5000 **Selector grammar:** one `role:text` string split on the first `:`. Examples: `"button:Submit"` (role + text), `"button"` (role only), `":Saved!"` (text only). Matching uses the same `find` matcher (`node_matches`); text searches name, value, and description. -**Supported commands:** `snapshot` plus the 16 ref-resolving actions (`click`, `type`, `set-value`, `scroll`, …) — 17 commands total. Other commands (`find`, `launch`, …) return `INVALID_ARGS`. Workaround: `snapshot --app Foo -w "button:Login"`. +**Supported commands:** `snapshot` plus all 18 ref-resolving actions (`click`, `type`, `set-value`, `scroll`, `hover`, `drag`, …) — 19 commands total. Other commands (`find`, `launch`, …) return `INVALID_ARGS`. Workaround: `snapshot --app Foo -w "button:Login"`. **Post-action waits** poll the **acted-on ref's own window** (`entry.source_window_id`, scoped to `entry.source_app`), not the frontmost window — critical in headless and multi-window apps where the terminal or a sibling window has focus. The action result is preserved under `after_action` in the returned envelope. @@ -47,75 +47,79 @@ The command surface is platform-agnostic: every ref action builds an `Action` an | Command | Headless path (macOS) | Notes | |---------|---------------|-------| -| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions; the default and most reliable surface | -| `type` | focus fallback | CLI `type` may focus the target field but never moves the cursor; use `set-value` for pure headless value mutation when supported | -| `double-click` | partial | `AXOpen` works headless on items that advertise it (Finder/list/outline rows, table cells). Falls back to `--headed` only for gesture-only targets with no `AXOpen`. | +| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions in strict headless mode | +| `type` | yes | uses `AXSelectedText` headlessly; `--headed` synthesizes keyboard input | +| `double-click` | no | a real two-click gesture; requires `--headed` | | `triple-click` | no | macOS exposes no triple-click action; it is purely 3 physical clicks → `--headed` only | | `hover` | no | hovering *is* moving the cursor over an element; no AX equivalent | | `drag` / drop | no | dragging *is* a cursor press-move-release; no general AX drag. Native cross-app drop needs the OS dragging-session/pasteboard protocol that synthetic events cannot start (works for same-view source-tracked gestures and web/Electron mouse-DnD) | | menu bar (`--surface menubar`) | enumerate/open | the app menu bar is readable and openable; SwiftUI `CommandMenu` items accept AXPress but do not route to their action closure (a SwiftUI limitation, like its Slider) — native AppKit menu items fire. `.contextMenu` item selection works. | -All ref-based interaction commands accept `--snapshot `. Omit it for the active session's latest saved snapshot, or pass the `snapshot_id` returned by `snapshot` to keep scripts pinned to the exact ref map they observed. Explicit snapshot IDs do not require also passing `--session`. After `session start`, implicit latest resolves inside the new session; snapshots taken before the boundary need explicit `--snapshot `. +All ref-based interaction commands accept `--snapshot `. Snapshot and find output already return qualified refs (`@:eN`), which embed the exact snapshot and need no separate flag. Legacy bare `@eN` input requires `--snapshot`; when a session owns that snapshot, the command also needs the same `--session` or `AGENT_DESKTOP_SESSION` scope. Lookup never searches another session namespace. Success responses for ref actions include a `steps` array when the activation chain recorded attempts: each entry is `{ "label": "AXPress", "outcome": "attempted" | "skipped" | "succeeded" }` in execution order, showing which activation path produced the result. -When the actionability preflight blocks an action, the error envelope carries the full report in `error.details`: `{ "actionable": false, "checks": [ { "name": "...", "status": "...", "reason": "..." } ] }`. Check names are `visible`, `stable`, `enabled`, `supported_action`, `policy`, and `editable`; statuses are `pass`, `fail`, and `unknown`. Use the failing check's `reason` to pick recovery: `wait --element --predicate actionable`, a fresh snapshot, or `--headed` when a `policy` check failed and a physical gesture is intended. +When the actionability preflight blocks an action, the error envelope carries the full report in `error.details`: `{ "actionable": false, "checks": [ { "check": "...", "status": "...", "reason": "..." } ] }`. The `check` identifiers are `visible`, `stable`, `enabled`, `supported_action`, `policy`, `editable`, and `receives_events`; statuses are `pass`, `fail`, and `unknown`. Failures split by whether waiting can help: the **transient** checks (`visible`, `stable`, `enabled`, `receives_events`) can change over time — scroll into view, settle, become enabled, occlusion clears — so they surface as `ACTION_FAILED` and are retried within `--timeout-ms`; the **terminal** checks (`supported_action`, `policy`, `editable`) cannot be healed by waiting (the element's role/action set or the interaction policy would have to change), so they fail fast with a precise code — `ACTION_NOT_SUPPORTED` (`supported_action`/`editable`) or `POLICY_DENIED` (`policy`) — instead of polling to `TIMEOUT`. The dispatch actions that activate an element (`click`, `double-click`, `right-click`, `triple-click`, `type`, `set-value`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, `clear`, `focus`, `scroll`, `scroll-to`) run the applicable `visible`/`stable`/`enabled`/`supported_action`/`policy`/`editable` battery; the four click variants additionally run `receives_events`. `hover` and each ref endpoint of `drag` use the pointer resolver instead: live visibility and bounds, one scroll-into-view attempt when needed, a second equal-bounds sample, then `receives_events`. They do not require enabled/editable or an element action capability because the gesture itself is raw pointer input. Use the failing check's `reason` to pick recovery: `wait --element --predicate actionable`, a fresh snapshot, or `--headed` when a `policy` check failed and a physical gesture is intended. + +**`receives_events` failures.** When a hit test at the target's center point lands on a different element, `receives_events` fails with `reason: "occluded by "` and a structured `occluder` object on that check: `{ "role", "name", "bounds" }` (the element that actually received the hit, when it can be identified). The target's own bounds have not changed — something else is now on top of them. Recovery is to bring the target's window or element to the front (or dismiss whatever is covering it), then retry; blind-retrying without changing z-order will fail the same way again. + +Every ref-resolving action accepts `--timeout-ms` (default `5000`), but it budgets different things. For the dispatch actions (`click`, `double-click`, `triple-click`, `right-click`, `clear`, `focus`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, `scroll-to`, `type`, `set-value`, `select`, `scroll`) it is the actionability-wait budget: they poll roughly every 100ms until the target becomes actionable, then fail with `TIMEOUT` once the budget is exhausted — unless the block is a terminal check (`supported_action`/`policy`/`editable`), which fails fast on the first attempt with `ACTION_NOT_SUPPORTED`/`POLICY_DENIED` rather than waiting out the budget. For `hover` and `drag`, the same budget covers ref resolution, live visibility/bounds, stability, and `receives_events`. Transient misses, app-unresponsive reads, and occlusion are polled until recovery or `TIMEOUT`; terminal errors are returned immediately with their original code. + +**Implicit scroll-into-view.** Standard ref actions whose `Action` declares a scroll precondition attempt `AXScrollToVisible` before dispatch. The pointer resolver for `hover` and `drag` independently makes one scroll attempt when a ref endpoint is not visibly bounded, then re-resolves and fails closed if it is still not visible. Use the standalone `scroll-to` command when you need an explicit, verifiable scroll result. ## Click Actions -Click commands use semantic AX activation first. In the default headless mode, coordinate click fallback is blocked; pass `--headed` to allow the physical click fallback, or use `agent-desktop --headed mouse-click` for a raw coordinate click. +Click commands use semantic AX activation in strict headless mode. Pass `--headed` to prefer a physical click, or use `agent-desktop --headed mouse-click` for a raw coordinate click. ### click ```bash -agent-desktop click @e5 +agent-desktop click @s8f3k2p9:e5 agent-desktop click @e5 --snapshot ``` -Primary activation. Tries verified AXPress > AXConfirm > AXOpen > AXPick > child activation > selection/value relays > custom actions > ancestor activation. Focus-stealing and coordinate fallback steps are not used by the default ref command path. +Primary activation. Headless uses `AXPress`; `--headed` performs a physical click first and reports `physical_synthetic` in `data.steps`. ### double-click ```bash -agent-desktop double-click @e3 +agent-desktop double-click @s8f3k2p9:e3 ``` -Tries AXOpen (headless). When the element advertises no `AXOpen`, the headless command fails closed with `POLICY_DENIED`; pass `--headed` to perform a real double-click (`agent-desktop --headed double-click @e3`), or use `agent-desktop --headed mouse-click --xy X,Y --count 2` for a raw coordinate double-click. +Double-click is a physical gesture and fails closed in headless mode. Pass `--headed` to perform it, or use `agent-desktop --headed mouse-click --xy X,Y --count 2` for raw coordinates. ### triple-click ```bash -agent-desktop triple-click @e2 +agent-desktop triple-click @s8f3k2p9:e2 ``` -Triple-click requires cursor/focus side effects and is blocked in headless mode; pass `--headed` (`agent-desktop --headed triple-click @e2`), or use `agent-desktop --headed mouse-click --xy X,Y --count 3` for a raw coordinate triple-click. +Triple-click requires cursor/focus side effects and is blocked in headless mode; pass `--headed` (`agent-desktop --headed triple-click @s8f3k2p9:e2`), or use `agent-desktop --headed mouse-click --xy X,Y --count 3` for a raw coordinate triple-click. ### right-click ```bash -agent-desktop right-click @e5 +agent-desktop right-click @s8f3k2p9:e5 ``` -Performs a semantic right-click/context-menu action and includes `menu` plus `menu_snapshot_id` when a menu surface can be verified. If the right-click action succeeds but menu probing fails, the command still returns the action result with `menu_probe.ok: false` so callers do not retry and double-open context menus. Combo boxes and menu buttons expose menu-opening actions for their primary dropdown; use `select` for those controls, not `right-click`. Focus-stealing and coordinate right-click fallback are blocked in headless mode; pass `--headed` to allow them. +Headless uses semantic context-menu actions. `--headed` performs a physical right-click first. On macOS, a semantic `AXShowMenu` can return `APP_UNRESPONSIVE` with uncertain delivery after opening a modal menu; inspect the effect and never retry blindly. Use `select` for combo boxes and menu buttons. ## Text Input ### type ```bash -agent-desktop type @e2 "hello@example.com" -agent-desktop type @e2 "multi line\ntext" +agent-desktop type @s8f3k2p9:e2 "hello@example.com" +agent-desktop type @s8f3k2p9:e2 "multi line\ntext" ``` -`type` uses the focus-fallback policy floor: it may focus the target field because typing requires focus, but it never moves the cursor. If the field cannot be updated and the focused-insert path is unavailable, it returns a structured error. Pass `--headed` to unlock physical keyboard synthesis and pasteboard-based insertion for fields that ignore AX value writes (common in web/Electron inputs). - -Under focus-fallback or `--headed`, non-ASCII text on macOS may be briefly placed on the clipboard to paste it. Do not use that path for secrets; prefer `set-value` when the target supports it. +Headless `type` uses `AXSelectedText` without focusing the app or synthesizing keys. Pass `--headed` to focus the target and synthesize keyboard input. Use `set-value` when direct semantic value assignment is the intended interaction. ### set-value ```bash -agent-desktop set-value @e2 "new value" +agent-desktop set-value @s8f3k2p9:e2 "new value" ``` Sets the value directly via the AX value attribute. Faster than `type` but may not trigger all UI callbacks. Use for text fields, text areas, and sliders. ### clear ```bash -agent-desktop clear @e2 +agent-desktop clear @s8f3k2p9:e2 ``` -Clears the element's value to an empty string. Equivalent to `set-value @e2 ""`. +Headless clears through `AXValue`. With `--headed`, it performs focus + Select All + Delete first. ### focus ```bash -agent-desktop focus @e2 +agent-desktop focus @s8f3k2p9:e2 ``` Sets keyboard focus on the element without clicking it. This is an explicit focus-changing command. It uses accessibility focus and does not move the cursor. @@ -124,25 +128,25 @@ This is an explicit focus-changing command. It uses accessibility focus and does ### select ```bash -agent-desktop select @e4 "Option B" +agent-desktop select @s8f3k2p9:e4 "Option B" ``` Selects an option in a list, dropdown, or combobox by display text. For menu-backed controls it opens the AX menu, presses the matching menu item, and verifies `AXValue` when the control exposes it. It returns a structured error when the matching item is missing or the exposed value does not change. ### toggle ```bash -agent-desktop toggle @e6 +agent-desktop toggle @s8f3k2p9:e6 ``` Toggles a checkbox or switch to the opposite state. ### check ```bash -agent-desktop check @e6 +agent-desktop check @s8f3k2p9:e6 ``` Sets a checkbox or switch to the checked/on state. Idempotent — does nothing if already checked. ### uncheck ```bash -agent-desktop uncheck @e6 +agent-desktop uncheck @s8f3k2p9:e6 ``` Sets a checkbox or switch to the unchecked/off state. Idempotent. @@ -150,13 +154,13 @@ Sets a checkbox or switch to the unchecked/off state. Idempotent. ### expand ```bash -agent-desktop expand @e7 +agent-desktop expand @s8f3k2p9:e7 ``` Expands a disclosure triangle, tree item, or accordion. ### collapse ```bash -agent-desktop collapse @e7 +agent-desktop collapse @s8f3k2p9:e7 ``` Collapses an expanded disclosure/tree item. @@ -164,22 +168,23 @@ Collapses an expanded disclosure/tree item. ### scroll ```bash -agent-desktop scroll @e1 --direction down --amount 3 -agent-desktop scroll @e1 --direction up --amount 5 -agent-desktop scroll @e1 --direction left --amount 2 -agent-desktop scroll @e1 --direction right --amount 2 +agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 +agent-desktop scroll @s8f3k2p9:e1 --direction up --amount 5 +agent-desktop scroll @s8f3k2p9:e1 --direction left --amount 2 +agent-desktop scroll @s8f3k2p9:e1 --direction right --amount 2 ``` | Flag | Default | Description | |------|---------|-------------| | `--direction` | down | `up`, `down`, `left`, `right` | | `--amount` | 3 | Number of scroll units | +| `--timeout-ms` | 5000 | Actionability wait budget in ms before failing with `TIMEOUT` | -Uses AX scroll actions, scroll bars, and state-setting paths. If those are unavailable, the command returns a structured error instead of stealing focus or sending wheel events. +Headless mode uses AX scroll actions, scroll bars, and state-setting paths. Headed mode focuses the exact ref window, resolves the target point, and sends a physical wheel gesture first. If the selected mode has no safe mechanism, the command returns a structured error. ### scroll-to ```bash -agent-desktop scroll-to @e8 +agent-desktop scroll-to @s8f3k2p9:e8 ``` Scrolls the element into the visible area of its scroll container. @@ -198,44 +203,35 @@ agent-desktop press cmd+a --app "TextEdit" | Flag | Description | |------|-------------| -| `--app` | Target application (focuses app before pressing) | +| `--app` | Target application; key delivery is PID-targeted, and `--headed` additionally focuses its exact window first | **Key names:** `return`, `escape`, `tab`, `space`, `delete`, `up`, `down`, `left`, `right`, `f1`-`f12` **Modifiers:** `cmd`, `ctrl`, `alt`, `shift` — combine with `+` -Dangerous shortcuts (e.g. `cmd+q`, `ctrl+cmd+q`, `cmd+alt+esc`, `cmd+shift+delete`) are refused with `POLICY_DENIED`. Normalization covers modifier order and key-name aliases (`escape`/`esc`, `backspace`/`delete`). The block is the **platform adapter's** decision, not core's — the calling agent stays in control: pass `--force` to send a flagged combo anyway (`agent-desktop press cmd+q --force`). `--force` is available on `press`, `key-down`, and `key-up`. +Dangerous shortcuts (e.g. `cmd+q`, `ctrl+cmd+q`, `cmd+alt+esc`, `cmd+shift+delete`) are refused with `POLICY_DENIED`. Normalization covers modifier order and key-name aliases (`escape`/`esc`, `backspace`/`delete`). The block is the **platform adapter's** decision, not core's — the calling agent stays in control: pass `--force` to send a flagged `press` combo anyway (`agent-desktop press cmd+q --force`). The reserved held-key names reject even when `--force` is present. -### key-down -```bash -agent-desktop key-down shift -``` -Holds a key or modifier down. Must be paired with `key-up`. The blocked-combo guard (same set as `press`) is enforced per invocation. **Known limitation:** because the tool is stateless per call, an agent could hold modifiers across separate `key-down` calls to assemble a blocked combo; that cross-invocation case is not guarded — a stateful guard arrives with the Phase-4 daemon. +### key-down / key-up -### key-up -```bash -agent-desktop key-up shift -``` -Releases a held key or modifier. The blocked-combo guard (same set as `press`) applies per invocation. +These names are reserved but fail closed with `ACTION_NOT_SUPPORTED` in the stateless CLI. Use the atomic `press` command. A future stateful daemon may own held-key lifetimes safely. ## Mouse ### hover ```bash -agent-desktop --headed hover @e5 +agent-desktop --headed hover @s8f3k2p9:e5 agent-desktop --headed hover --xy 500,300 -agent-desktop --headed hover @e5 --duration 2000 ``` -Moves cursor to element center or absolute coordinates. Optional `--duration` holds position for N ms. +Moves cursor to element center or absolute coordinates. A positive `--duration` is rejected because a stateless process cannot guarantee cursor ownership during a dwell; run hover without it, then use `wait ` for an explicit pause. This is an explicit cursor-moving command. -With `--headed`, a ref-addressed hover ensures the target app is frontmost before moving the cursor (raising it if needed, best-effort), and the response includes `"focused": true` when that frontmost state was confirmed. The field is only ever present as `true`: absence means focus was never attempted (headless default, or `--xy` input — the caller owns the target there) or the best-effort raise could not be confirmed. +With `--headed`, a ref-addressed hover must focus the target's exact window before moving the cursor and fails before delivery if focus cannot be confirmed. The response then includes `"focused": true`. Raw `--xy` hover never attempts focus because no target window identity exists. ### drag ```bash -agent-desktop --headed drag --from @e1 --to @e5 +agent-desktop --headed drag --from @s8f3k2p9:e1 --to @s8f3k2p9:e5 agent-desktop --headed drag --from-xy 100,200 --to-xy 400,500 -agent-desktop --headed drag --from @e1 --to-xy 400,500 --duration 500 -agent-desktop --headed drag --from @e1 --to @e5 --drop-delay 800 +agent-desktop --headed drag --from @s8f3k2p9:e1 --to-xy 400,500 --duration 500 +agent-desktop --headed drag --from @s8f3k2p9:e1 --to @s8f3k2p9:e5 --drop-delay 800 ``` | Flag | Description | @@ -246,10 +242,11 @@ agent-desktop --headed drag --from @e1 --to @e5 --drop-delay 800 | `--to-xy` | Destination coordinates as `x,y` | | `--duration` | Drag duration in milliseconds (movement from source to destination) | | `--drop-delay` | Milliseconds to hold over the destination before releasing; default 500 | +| `--timeout-ms` | Actionability wait budget in ms before failing with `TIMEOUT`; default 5000 | -Can mix ref and coordinate sources (e.g., `--from @e1 --to-xy 400,500`). +Can mix ref and coordinate sources (e.g., `--from @s8f3k2p9:e1 --to-xy 400,500`). -With `--headed`, a ref-addressed `--from` ensures the source app is frontmost before the mouse-down (the destination app is never pre-focused — raising it could cover the source point), and the response includes `"focused": true` when that frontmost state was confirmed. The field is only ever present as `true`: absence means focus was never attempted (headless default, or coordinate-only drags) or the best-effort raise could not be confirmed. For cross-app two-ref drags, ensure the destination window is visible (not fully occluded) before dragging — only the source app is raised. +With `--headed`, a ref-addressed `--from` must focus the source's exact window before mouse-down and fails before delivery if focus cannot be confirmed. The destination app is never pre-focused because raising it could cover the source point. Coordinate-only drags never attempt focus. For cross-app two-ref drags, keep the destination window visible; both endpoints still undergo live visibility, stability, and hit-test checks. macOS drop targets often need the dragged item to dwell over them before they register as the drop destination — too short and the gesture lands as a drag with no drop. The default 500ms dwell suits most targets; raise `--drop-delay` (e.g. 800–1200) for sluggish destinations like list reorders or cross-window drops. The dwell posts continuous drag events over the destination so it stays highlighted, rather than a dead pause. @@ -271,18 +268,28 @@ agent-desktop --headed mouse-click --xy 500,300 --count 2 | `--xy` | (required) | Coordinates as `x,y` | | `--button` | left | `left`, `right`, `middle` | | `--count` | 1 | Number of clicks | +| `--modifiers` | | Held modifiers: `shift`, `meta`, `ctrl`, `alt` (repeatable; `cmd`/`command` aliases are accepted); held during the click | ### mouse-down / mouse-up + +These names are reserved but fail closed with `ACTION_NOT_SUPPORTED` in the stateless CLI. Use the atomic `mouse-click` or `drag` command. A future stateful daemon may own held-button lifetimes safely. + +### mouse-wheel ```bash -agent-desktop --headed mouse-down --xy 100,200 -agent-desktop --headed mouse-up --xy 300,400 +agent-desktop --headed mouse-wheel --x 500 --y 300 +agent-desktop --headed mouse-wheel --x 500 --y 300 --dy -3 +agent-desktop --headed mouse-wheel --x 500 --y 300 --dx -2 --dy 0 +agent-desktop --headed mouse-wheel --x 500 --y 300 --modifiers shift ``` -Low-level press/release for custom drag or hold interactions. +Synthesizes a scroll-wheel event at absolute coordinates and requires `--headed`. This is distinct from `scroll `: `scroll` targets an element through AX scroll semantics, while `mouse-wheel` posts a raw wheel event at a screen point (for custom scroll surfaces or canvases with no AX scroll action). Held modifiers are applied to the event, so `--modifiers shift` produces the horizontal-scroll chord some apps expect. | Flag | Default | Description | |------|---------|-------------| -| `--xy` | (required) | Coordinates as `x,y` | -| `--button` | left | `left`, `right`, `middle` | +| `--x` | (required) | Absolute X coordinate | +| `--y` | (required) | Absolute Y coordinate | +| `--dy` | -3 | Vertical wheel lines; positive is up, negative is down | +| `--dx` | 0 | Horizontal wheel lines; positive is left, negative is right | +| `--modifiers` | | Held modifiers: `shift`, `meta`, `ctrl`, `alt` (repeatable; `cmd`/`command` aliases are accepted) | ## Choosing the Right Command diff --git a/skills/agent-desktop/references/commands-observation.md b/skills/agent-desktop/references/commands-observation.md index ab2b1e1..7ace05b 100644 --- a/skills/agent-desktop/references/commands-observation.md +++ b/skills/agent-desktop/references/commands-observation.md @@ -6,6 +6,10 @@ Commands for reading UI state without modifying it. Capture the accessibility tree as structured JSON with `@ref` IDs. +Output refs are qualified as `@:e`. Use that value directly on +later commands. Legacy bare `@eN` input remains valid only with the matching +explicit `--snapshot ` and inside the same session namespace. + ```bash agent-desktop snapshot --app "System Settings" -i agent-desktop snapshot --app "Finder" --max-depth 5 --include-bounds @@ -28,12 +32,12 @@ agent-desktop snapshot --root @e12 --snapshot -i | `--surface` | window | Target surface: `window`, `focused`, `menu`, `menubar`, `sheet`, `popover`, `alert` | | `--skeleton` | false | Clamp traversal to depth 3 and add `children_count` to truncated containers | | `--root ` | | Drill down from a ref discovered in a previous snapshot. Cannot be combined with `--surface` | -| `--snapshot ` | latest | Snapshot ID to use when resolving `--root` | +| `--snapshot ` | embedded in qualified root | Required only when `--root` is a legacy bare ref | **Output structure:** ```json { - "version": "2.0", + "version": "2.1", "ok": true, "command": "snapshot", "data": { @@ -46,7 +50,7 @@ agent-desktop snapshot --root @e12 --snapshot -i "name": "General", "children": [ { - "ref_id": "@e1", + "ref_id": "@s8f3k2p9:e1", "role": "button", "name": "About", "states": ["focused"] @@ -56,7 +60,7 @@ agent-desktop snapshot --root @e12 --snapshot -i "name": "Appearance", "children": [ { - "ref_id": "@e2", + "ref_id": "@s8f3k2p9:e2", "role": "checkbox", "name": "Dark Mode", "value": "0", @@ -114,6 +118,10 @@ agent-desktop find --app "Safari" --text "Sign In" --first agent-desktop find --app "App" --role checkbox --count agent-desktop find --app "App" --role button --nth 2 agent-desktop find --app "App" --role button --limit 20 +agent-desktop find --app "App" --role button --name "OK" --exact +agent-desktop find --app "App" --description "Closes the dialog" +agent-desktop find --app "App" --native-id "submitButton" +agent-desktop find --app "App" --state enabled --state focused=false ``` | Flag | Description | @@ -123,6 +131,10 @@ agent-desktop find --app "App" --role button --limit 20 | `--name` | Accessible name or label | | `--value` | Current value | | `--text` | Fuzzy match across name, value, title, and description | +| `--description` | Match by accessible description | +| `--native-id` | Match by native automation id (`AXIdentifier`) | +| `--exact` | Require exact (case-insensitive) matches for `--name`/`--description`/`--value` instead of fuzzy/substring matching | +| `--state TOKEN[=BOOL]` | Filter by state token; repeatable. Bare `TOKEN` requires the state present, `TOKEN=true`/`TOKEN=false` asserts its value (e.g. `--state enabled --state focused=false`) | | `--first` | Return first match only | | `--last` | Return last match only | | `--nth N` | Return Nth match (0-indexed) | @@ -133,14 +145,16 @@ agent-desktop find --app "App" --role button --limit 20 ```json { "data": { + "snapshot_id": "s8f3k2p9", "matches": [ - { "ref_id": "@e5", "role": "button", "name": "OK", "states": ["enabled"] } - ], - "count": 1 + { "ref_id": "@s8f3k2p9:e5", "role": "button", "name": "OK", "states": ["enabled"] } + ] } } ``` +Every non-count `find` response returns the `snapshot_id` that owns its refs. Pass that exact ID to later ref actions instead of relying on the mutable latest-snapshot pointer, especially when interleaving automation across apps or windows. Count-only responses create no ref namespace and omit `snapshot_id`. + **Output (no match — `roles_present` hint):** when a `--role` filter matches nothing, `roles_present` lists the roles actually in the searched tree so you can tell a wrong role name from "none on screen"; this applies to all non-count selection modes — an empty match list, or a `--first`/`--last`/`--nth` miss — whenever a role filter was active, making it a role-vocabulary hint for retries. ```json { @@ -157,13 +171,13 @@ agent-desktop find --app "App" --role button --limit 20 Read a specific property from an element. ```bash -agent-desktop get @e1 --property text +agent-desktop get @s8f3k2p9:e1 --property text agent-desktop get @e1 --snapshot --property text -agent-desktop get @e2 --property value -agent-desktop get @e3 --property bounds -agent-desktop get @e4 --property role -agent-desktop get @e5 --property states -agent-desktop get @e1 --property title +agent-desktop get @s8f3k2p9:e2 --property value +agent-desktop get @s8f3k2p9:e3 --property bounds +agent-desktop get @s8f3k2p9:e4 --property role +agent-desktop get @s8f3k2p9:e5 --property states +agent-desktop get @s8f3k2p9:e1 --property title ``` | Property | Returns | @@ -180,12 +194,12 @@ agent-desktop get @e1 --property title Check a boolean state on an element. ```bash -agent-desktop is @e1 --property visible +agent-desktop is @s8f3k2p9:e1 --property visible agent-desktop is @e1 --snapshot --property visible -agent-desktop is @e2 --property enabled -agent-desktop is @e3 --property checked -agent-desktop is @e4 --property focused -agent-desktop is @e5 --property expanded +agent-desktop is @s8f3k2p9:e2 --property enabled +agent-desktop is @s8f3k2p9:e3 --property checked +agent-desktop is @s8f3k2p9:e4 --property focused +agent-desktop is @s8f3k2p9:e5 --property expanded ``` | Property | Checks | @@ -198,7 +212,7 @@ agent-desktop is @e5 --property expanded **Output:** ```json -{ "data": { "ref": "@e3", "property": "checked", "result": true } } +{ "data": { "ref": "@s8f3k2p9:e3", "property": "checked", "result": true } } ``` ## screenshot @@ -209,18 +223,40 @@ Capture a PNG screenshot of an application window. agent-desktop screenshot --app "Finder" agent-desktop screenshot --app "Finder" output.png agent-desktop screenshot --window-id "w-1234" capture.png +agent-desktop screenshot --screen 0 display.png ``` | Flag | Description | |------|-------------| | `--app` | Application name | | `--window-id` | Specific window ID | +| `--screen` | Capture display by index instead of an app window (from `list-displays`; `0` = primary) | | (positional) | File path to save PNG (omit for base64 in JSON) | When no output path is given, the screenshot is returned as a base64-encoded string in the JSON `data` field. Screenshots require Screen Recording permission. Permission denial is reported as `PERM_DENIED`, not `INTERNAL`. +## list-displays + +List connected displays with bounds and scale factor. + +```bash +agent-desktop list-displays +``` + +Returns an array of `{ id, bounds: { x, y, width, height }, is_primary, scale }`, sorted primary-first. Use the array index (not `id`) with `screenshot --screen ` — `0` is always the primary display after sorting. + +**Output:** +```json +{ + "data": [ + { "id": "1", "bounds": { "x": 0, "y": 0, "width": 2560, "height": 1440 }, "is_primary": true, "scale": 2.0 }, + { "id": "2", "bounds": { "x": 2560, "y": 0, "width": 1920, "height": 1080 }, "is_primary": false, "scale": 1.0 } + ] +} +``` + ## list-surfaces List available accessibility surfaces for an application. diff --git a/skills/agent-desktop/references/commands-system.md b/skills/agent-desktop/references/commands-system.md index 56e7751..b7a3ef6 100644 --- a/skills/agent-desktop/references/commands-system.md +++ b/skills/agent-desktop/references/commands-system.md @@ -8,12 +8,21 @@ App lifecycle, window management, notifications, clipboard, wait, and system hea ```bash agent-desktop launch "System Settings" agent-desktop launch "com.apple.Safari" --timeout 10000 +agent-desktop launch "TextEdit" --arg /tmp/notes.txt +agent-desktop launch "MyTool" --arg --flag --arg value --env KEY=VALUE --cwd /tmp +agent-desktop launch "MyTool" --no-attach ``` Launches an application by name or bundle ID and waits until its window is visible. | Flag | Default | Description | |------|---------|-------------| | `--timeout` | 30000 | Max wait time in ms for window to appear | +| `--arg` | | Command-line argument passed to the launched app; repeatable, order preserved | +| `--env` | | `KEY=VALUE` environment variable for the launched process; repeatable | +| `--cwd` | | Working directory for the launched process | +| `--no-attach` | false | Require a fresh launch instead of the default attach-if-running behavior | + +By default, `launch` attaches to an already-running instance and returns its visible window. `--no-attach` rejects an already-running app with `ACTION_FAILED`; otherwise it starts a fresh instance and still waits for a real visible window. Windowless, menu-bar-only, or background apps return `WINDOW_NOT_FOUND` rather than a fabricated empty window response; use `list-apps` to observe those processes. ### close-app ```bash @@ -52,27 +61,32 @@ Brings a window to the front and confirms the OS reports that same window as foc ### resize-window ```bash agent-desktop resize-window --app "TextEdit" --width 800 --height 600 +agent-desktop resize-window --window-id w-4521 --width 800 --height 600 ``` ### move-window ```bash agent-desktop move-window --app "TextEdit" --x 0 --y 0 +agent-desktop move-window --window-id w-4521 --x 0 --y 0 ``` ### minimize ```bash agent-desktop minimize --app "TextEdit" +agent-desktop minimize --window-id w-4521 ``` ### maximize ```bash agent-desktop maximize --app "TextEdit" +agent-desktop maximize --window-id w-4521 ``` Zooms the window to fill the screen. ### restore ```bash agent-desktop restore --app "TextEdit" +agent-desktop restore --window-id w-4521 ``` Restores a minimized or maximized window to its previous size. @@ -82,11 +96,11 @@ If Notification Center fails to close after a successful list or dismiss operati ### list-notifications ```bash -agent-desktop list-notifications -agent-desktop list-notifications --app "Slack" -agent-desktop list-notifications --text "deploy" --limit 5 +agent-desktop --headed list-notifications +agent-desktop --headed list-notifications --app "Slack" +agent-desktop --headed list-notifications --text "deploy" --limit 5 ``` -Lists notifications in the Notification Center. Returns array of `{ index, app_name, title, body, actions }`. +Lists notifications in Notification Center. Headless mode can observe it only when it is already open; `--headed` may open it and restore the prior frontmost app afterward. Returns array of `{ index, app_name, title, body, actions }`. | Flag | Default | Description | |------|---------|-------------| @@ -96,40 +110,40 @@ Lists notifications in the Notification Center. Returns array of `{ index, app_n ### dismiss-notification ```bash -agent-desktop dismiss-notification 1 -agent-desktop dismiss-notification 3 --app "Slack" +agent-desktop --headed dismiss-notification 1 --expected-app "Slack" --expected-title "Deploy complete" +agent-desktop --headed dismiss-notification 3 --app "Slack" --expected-app "Slack" ``` -Dismisses a single notification by its 1-based index. Returns the dismissed notification info. +Dismisses a single notification by its 1-based index. Requires `--headed` and at least one fingerprint from the listing (`--expected-app` or `--expected-title`). Returns the dismissed notification info. | Flag | Default | Description | |------|---------|-------------| | (positional) | | 1-based notification index (required) | | `--app` | | Filter by app before indexing | +| `--expected-app` | | Fingerprint app name (at least one fingerprint required) | +| `--expected-title` | | Fingerprint title (at least one fingerprint required) | ### dismiss-all-notifications ```bash -agent-desktop dismiss-all-notifications -agent-desktop dismiss-all-notifications --app "Slack" +agent-desktop --headed dismiss-all-notifications +agent-desktop --headed dismiss-all-notifications --app "Slack" ``` -Dismisses all notifications, optionally filtered by app. Reports per-notification failures. +Dismisses all notifications, optionally filtered by app. Requires `--headed` because it mutates the focused system notification surface. Reports per-notification failures. Returns `{ "dismissed_count": N, "failures": [...], "failed_count": N }`. ### notification-action ```bash -agent-desktop notification-action 1 "Reply" -agent-desktop notification-action 2 "Mark as Read" --expected-app Slack --expected-title "#general" +agent-desktop --headed notification-action 1 "Reply" --expected-app Slack +agent-desktop --headed notification-action 2 "Mark as Read" --expected-app Slack --expected-title "#general" ``` -Clicks a named action button on a notification by its 1-based index. +Clicks a named action button on a notification by its 1-based index. Requires `--headed` and at least one listing fingerprint. `--expected-app` and `--expected-title` pin the call to the notification you observed in `list-notifications`. Notification Center reorders -entries between listings, so without a fingerprint an arriving or -dismissed notification can shift the target at `INDEX` and cause the -action to press the wrong row. When either flag is set and the row at +entries between listings, so an arriving or dismissed notification can shift +the target at `INDEX`. When the row at `INDEX` no longer matches, the call fails with `NOTIFICATION_NOT_FOUND` -instead of pressing. Both flags omitted preserves the legacy -index-only behavior for callers that reconcile themselves. +instead of pressing. Omitting both fingerprints is rejected with `INVALID_ARGS`. | Flag | Default | Description | |------|---------|-------------| @@ -145,18 +159,46 @@ agent-desktop wait --notification --text "build passed" --timeout 15000 ``` Blocks until a new notification appears (detects index-diff from a baseline captured at wait start). Supports `--app` and `--text` filters. Transient Notification Center errors (timeouts, element-not-found) are retried within the `--timeout` budget for both the baseline capture and polling; permanent errors (for example `PERM_DENIED`) fail immediately. Timeout errors include a `last_error` detail with the most recent transient failure. +Like listing, a headless wait can observe only an already-open Notification Center; use global `--headed` when the command may open and later restore it. + ## Clipboard ### clipboard-get ```bash agent-desktop clipboard-get +agent-desktop clipboard-get --format auto +agent-desktop clipboard-get --format image --out /tmp/clip.png +agent-desktop clipboard-get --format file-urls ``` -Returns `{ "data": { "text": "clipboard contents" } }`. +Reads a typed clipboard representation. + +| Flag | Default | Description | +|------|---------|-------------| +| `--format` | text | Representation to read: `text`, `auto` (richest available: file references, then image, then text), `image`, `file-urls` | +| `--out` | private temp file | Where to write image bytes when `--format image`/`auto` resolves to an image; defaults to a private file under the active session's directory, or `~/.agent-desktop/tmp` with no active session | + +**Output by format:** +```json +{ "data": { "type": "text", "text": "clipboard contents" } } +{ "data": { "type": "file_urls", "file_urls": ["/Users/me/Documents/report.pdf"] } } +{ "data": { "type": "image", "path": "/Users/me/.agent-desktop/sessions//clipboard/clipboard-...png", "width": 800, "height": 600 } } +``` +When the pasteboard has nothing in the requested representation, the response is `{ "data": { "type": "", "found": false } }` with no other payload fields. ### clipboard-set ```bash agent-desktop clipboard-set "Hello, world!" +agent-desktop clipboard-set --image /tmp/screenshot.png +agent-desktop clipboard-set --file-url /Users/me/Documents/report.pdf +agent-desktop clipboard-set --file-url /tmp/a.txt --file-url /tmp/b.txt ``` +Writes typed content to the clipboard. `--file-url` (repeatable) and `--image` each take priority over the positional text argument when present; only one representation is written per call. + +| Flag | Description | +|------|-------------| +| (positional) | Text to write (ignored if `--image` or `--file-url` is given) | +| `--image` | Path to a PNG file to write to the clipboard | +| `--file-url` | File path to write as a file reference; repeatable. Every path must exist on disk or the command returns `INVALID_ARGS` | ### clipboard-clear ```bash @@ -174,9 +216,9 @@ Pauses for N milliseconds. Use between actions that need time to settle. ### wait (element) ```bash agent-desktop wait --element @e5 --snapshot --timeout 5000 --app "App" -agent-desktop wait --element @e5 --predicate actionable --timeout 5000 -agent-desktop wait --element @e5 --predicate actionable --action type --timeout 5000 -agent-desktop wait --element @e5 --predicate value --value "Done" --timeout 5000 +agent-desktop wait --element @s8f3k2p9:e5 --predicate actionable --timeout 5000 +agent-desktop wait --element @s8f3k2p9:e5 --predicate actionable --action type --timeout 5000 +agent-desktop wait --element @s8f3k2p9:e5 --predicate value --value "Done" --timeout 5000 ``` Blocks until the element ref appears in the accessibility tree. Useful after triggering UI changes. When `--snapshot` is omitted, the command polls the caller's latest session refmap and refreshes it on the built-in debounce. When `--snapshot` is passed, it resolves that pinned refmap directly. Element resolution is capped by the remaining `--timeout`, and timeout errors include the last observed predicate/actionability state. @@ -207,6 +249,30 @@ agent-desktop wait --menu-closed --app "Finder" --timeout 3000 ``` Blocks until the menu surface is dismissed. +### wait (event) +```bash +agent-desktop wait --event window-opened --app "Finder" --timeout 10000 +agent-desktop wait --event window-closed --window-id "w-1234" --timeout 10000 +agent-desktop wait --event app-launched --app "Safari" --timeout 15000 +agent-desktop wait --event app-terminated --app "Safari" --timeout 15000 +agent-desktop wait --event focus-changed --timeout 10000 +agent-desktop wait --event surface-appeared --app "Finder" --timeout 5000 +agent-desktop wait --event window-opened --window "Untitled" --timeout 10000 +``` +Blocks until a desktop lifecycle signal is observed, detected by diffing a baseline captured at wait start against fresh reads — no need to know a new window's id or title up front. `--window-id`/`--window` are optional narrowing filters on top of `--event`, never a requirement by themselves (bare `--window` without `--event` instead selects the `wait (window)` mode above). + +| Token | Fires when | +|-------|------------| +| `window-opened` | A window not present in the baseline appears | +| `window-closed` | A baseline window disappears | +| `app-launched` | A process not present in the baseline starts | +| `app-terminated` | A baseline process exits | +| `focus-changed` | The OS-focused window differs from the baseline's | +| `surface-appeared` | A menu/sheet/popover/alert surface count increases | +| `surface-dismissed` | A menu/sheet/popover/alert surface count decreases | + +Transient errors (timeouts, element-not-found) are retried within the `--timeout` budget for both the baseline capture and polling; other errors fail immediately. Timeout errors include `baseline_counts` and, when a poll errored, `last_error`. + | Flag | Default | Description | |------|---------|-------------| | (positional) | | Milliseconds to pause | @@ -216,12 +282,14 @@ Blocks until the menu surface is dismissed. | `--value` | | Expected text for `--predicate value` | | `--action` | click | Action checked by `--predicate actionable`: `click`, `type`, `set-value`, `clear` | | `--count` | | Expected match count for `--text` waits | -| `--window` | | Window title to wait for | +| `--window` | | Window title to wait for; with `--event`, narrows the event to that window's title instead of selecting a mode | | `--text` | | Text to wait for; with `--notification`, filters notification title/body | | `--menu` | false | Wait for menu surface to open | | `--menu-closed` | false | Wait for menu surface to close | | `--notification` | false | Wait for a new notification | -| `--timeout` | 30000 | Timeout in ms (for element/window/text/menu waits) | +| `--event` | | Desktop lifecycle signal to wait for: `window-opened`, `window-closed`, `app-launched`, `app-terminated`, `focus-changed`, `surface-appeared`, `surface-dismissed` | +| `--window-id` | | Narrows `--event` to one window ID (window/focus events only) | +| `--timeout` | 30000 | Timeout in ms (for element/window/text/menu/event waits) | | `--app` | | Scope the wait to a specific application | ## Batch @@ -258,7 +326,7 @@ Each entry may include `"session": "id"` beside `command` and `args`. If omitted **Per-entry failure shape:** ```json { - "version": "2.0", + "version": "2.1", "ok": false, "command": "click", "error": { @@ -281,25 +349,22 @@ Each entry may include `"session": "id"` beside `command` and `args`. If omitted ## Session lifecycle -Sessions are on-disk containers under `~/.agent-desktop/sessions//` with a `session.json` manifest, snapshot refmaps, and (when tracing is on) a `trace/` directory. **`session start` is the only command that writes `~/.agent-desktop/current_session`.** +Sessions are on-disk containers under `~/.agent-desktop/sessions//` with a `session.json` manifest, snapshot refmaps, and (when tracing is on) a `trace/` directory. Session selection is explicit; `session start` returns an ID but does not activate it for later processes. ### session start ```bash agent-desktop session start agent-desktop session start --name "nightly-run" agent-desktop session start --no-trace # Namespace only — no automatic JSONL -agent-desktop session start --force # Override pointer even if it references a live session ``` -Creates the session directory, pre-creates `trace/` (when tracing is on), writes `session.json` (`trace: on` unless `--no-trace`), sets the current-session pointer, and prints `{ "session_id", "name", "trace", "created_at" }`. - -Refuses to clobber a pointer that still references a **live** session unless `--force`. Live means an active `refstore.lock` holder or recent writes under `trace/`. +Creates the session directory, pre-creates `trace/` (when tracing is on), writes `session.json` (`trace: on` unless `--no-trace`), and prints `{ "session_id", "name", "trace", "created_at" }`. Pass that ID through global `--session` or `AGENT_DESKTOP_SESSION` on later commands. ### session end ```bash -agent-desktop session end agent-desktop session end run-1719763200123-0 +agent-desktop --session run-1719763200123-0 session end ``` -Seals the manifest with `ended_at` and clears the pointer when it still points at this session. +Seals the manifest with `ended_at`. The ID is required either as the positional argument, global `--session`, or `AGENT_DESKTOP_SESSION`. ### session list ```bash @@ -313,15 +378,16 @@ agent-desktop session gc agent-desktop session gc --ended agent-desktop session gc --older-than 3600 ``` -Removes ended sessions that are not live and not pointer-referenced. Never reaps a session with a live lock holder or recent `trace/` activity. Refuses symlinked session directories. +Removes ended sessions that are not live. Never reaps a session with a live lock holder or recent `trace/` activity. Refuses symlinked session directories. ### Activation (all commands) | Source | Precedence | |--------|------------| | `--session ` | Highest | -| `AGENT_DESKTOP_SESSION` env var | Middle | -| `~/.agent-desktop/current_session` | Lowest (set only by `session start`) | +| `AGENT_DESKTOP_SESSION` env var | Fallback | + +With neither source, commands use the global, non-session namespace. There is no current-session pointer fallback. Trace-on requires a manifest with `trace: on` from `session start`. Bare `--session` or FFI `ad_adapter_create_with_session` without that manifest selects the snapshot namespace only. @@ -393,9 +459,9 @@ When `session_id` resolves to a session with a readable manifest, the response a agent-desktop permissions agent-desktop permissions --request ``` -Checks the cached per-process permission report: `accessibility`, `screen_recording`, and `automation`, each as `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`. The current macOS adapter reports concrete `granted` or `denied` states for Accessibility and Screen Recording, and `not_required` for Automation because shipped commands use Accessibility, Screen Recording for screenshots, and explicit keyboard/mouse input rather than Apple Events. Use `--request` to invoke the platform request path. +Checks the cached per-process permission report: `accessibility`, `screen_recording`, and `automation`, each as `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`. The current macOS adapter reports concrete `granted` or `denied` states for Accessibility and Screen Recording. Automation is probed against System Events without prompting; `{ "state": "unknown" }` means macOS would need to prompt or the target could not be probed. `--request` asks for all three permissions through a bounded isolated helper so a stalled native prompt cannot strand the command process. -`status`, `permissions`, command preflight, and `batch` share one permission probe per process. `permissions --request` is the only path that intentionally asks the platform to prompt again. +`status`, `permissions`, command preflight, and `batch` share one nonprompting permission probe per process. `permissions --request` is the only path that intentionally asks the platform to prompt again, and it does so in the isolated helper. ### version ```bash diff --git a/skills/agent-desktop/references/macos.md b/skills/agent-desktop/references/macos.md index 99e5eca..4ab398f 100644 --- a/skills/agent-desktop/references/macos.md +++ b/skills/agent-desktop/references/macos.md @@ -22,9 +22,9 @@ The report contains: |-------|---------| | `accessibility` | Required for accessibility tree reads and most commands | | `screen_recording` | Required for screenshots | -| `automation` | Reserved for future Apple Event automation paths; `{ "state": "not_required" }` for the current macOS command set | +| `automation` | Apple Event Automation permission for System Events; plain `permissions` checks without prompting, `permissions --request` may prompt in the bounded isolated helper | -Each field is an object: `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`. The current macOS adapter reports concrete `granted` or `denied` states for Accessibility and Screen Recording, and `not_required` for Automation. +Each field is an object: `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`. The current macOS adapter reports concrete `granted` or `denied` states for Accessibility and Screen Recording. Automation may report `unknown` when macOS would need to prompt or System Events could not be probed. **To grant manually:** 1. Open System Settings > Privacy & Security > Accessibility @@ -52,38 +52,32 @@ agent-desktop uses the macOS Accessibility API (`AXUIElement`) to read and manip 3. Attributes like `kAXRoleAttribute`, `kAXNameAttribute`, `kAXValueAttribute` provide element details 4. Actions like `kAXPressAction`, `kAXConfirmAction` trigger element behavior -### Smart Activation Chain +### Action Chains -When you run `click @ref`, agent-desktop doesn't just do a simple click. It runs a multi-step activation chain: +Core resolves and validates a ref, applies its headed focus/cursor requirement, and then asks the macOS adapter to execute an action-specific chain. The chain records each attempted mechanism and never invents a generic fallback ladder: -1. **AXScrollToVisible** — ensure element is on screen -2. **AXPress** — standard press action -3. **AXConfirm** — confirmation action -4. **AXOpen** — open action (for links, files) -5. **AXPick** — picker action -6. **AXShowAlternateUI** — reveal hidden UI, then press child -7. **Child activation** — try pressing child elements -8. **AXSelected** — set selected attribute -9. **Select via parent** — set parent's selected rows (for tables/lists) -10. **Custom actions** — AXPerformCustomAction -11. **Ancestor activation** — try pressing ancestor elements -12. **Explicit physical path** — coordinate click only when the caller selected a policy that allows focus stealing and cursor movement +- Headless `click` uses `AXPress`; headed `click` uses a verified-point `CGClick` first, then `AXPress` only if physical delivery was not attempted. +- Headed `right-click` uses physical right-click first; headless uses the bounded `AXShowMenu` family. +- Headed `type`, `clear`, and `scroll` use PID-targeted keyboard, keyboard clear, and wheel delivery respectively; their headless paths use AX semantics. +- `expand` and `collapse` use a verified semantic disclosure mutation in both modes. +- `set-value`, `select`, toggle/check/uncheck, focus, and `scroll-to` are semantic-only. +- Double/triple-click, hover, and drag are physical gestures and require headed mode. -For `right-click`, AXShowMenu and related semantic menu paths include a `menu` tree only after a real menu surface appears. If the action succeeds but the menu probe cannot verify a surface, the command still returns success with `menu_probe.ok: false`; callers should inspect that field instead of retrying blindly. Combo boxes and menu buttons use the same AX menu mechanism for their primary dropdown; use `select` for those controls. Coordinate right-click is blocked by the default headless policy. +For `right-click`, `AXShowMenu` may enter modal menu tracking and return `kAXErrorCannotComplete` after the menu opened. The command reports this as `APP_UNRESPONSIVE` with `delivery: delivery_uncertain` and `retry: unsafe`; inspect the resulting menu or target effect instead of retrying blindly. Combo boxes and menu buttons use the same AX menu mechanism for their primary dropdown; use `select` for those controls. Coordinate right-click is blocked by the default headless policy. -Menu verification intentionally requires a closed-to-open transition. If a menu is already open for the target app, `right-click` and `select` refuse to treat that existing menu as proof of success; dismiss the old menu and retry. This avoids acting on a stale sibling menu opened by a prior command. +Menu verification for `select` intentionally requires a closed-to-open transition. If a menu is already open for the target app, dismiss it before selecting so an existing sibling menu is not treated as proof of success. The default activation-chain deadline is 10 seconds. Set `AGENT_DESKTOP_CHAIN_TIMEOUT_MS` to a positive millisecond value when diagnosing unusually slow AX targets; values are capped at 300000 ms. Menu verification waits use `AGENT_DESKTOP_MENU_TIMEOUT_MS` with a default of 750 ms and a 10000 ms cap. Toggle verification uses `AGENT_DESKTOP_TOGGLE_TIMEOUT_MS` with a default of 600 ms and a 10000 ms cap; changed toggle values must remain stable for `AGENT_DESKTOP_TOGGLE_STABLE_MS`, default 200 ms and cap 2000 ms. ### Headless Interaction Policy -Ref commands use `ActionRequest { action, policy }`. The default policy forbids focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. macOS actions split semantic AX steps from explicit physical/headed paths: +Ref commands use `ActionRequest { action, policy }`. The default policy forbids focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. Core maps headed actions to `FocusedWindow` or `FocusedWindowAndCursor`, focuses the exact ref window, and resolves pointer coordinates before calling macOS. The adapter owns AppKit/AX/CGEvent delivery: -- `click`, `right-click`, `scroll`, `set-value`, `clear`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, and `scroll-to` try AX-first semantics and fail clearly when the headless path is unavailable. -- `type` uses focus fallback in the CLI/ref-action path. It may focus the target field, never moves the cursor, and can use the pasteboard for non-ASCII insertion. Use `set-value` for pure headless value mutation when supported. -- `focus`, `press`, `hover`, `drag`, and `mouse-*` are explicit physical/focus/cursor commands. -- FFI ref-action callers should use focus fallback for `type` to match CLI behavior; direct-handle `ad_execute_action` is lower-level and defaults to headless. -- Explicit focus/physical policy can use the clipboard briefly for non-ASCII text insertion. Use `set-value` for sensitive text when possible. +- `click`, `right-click`, `type`, `clear`, and `scroll` use semantic AX delivery headlessly and physical-first delivery with `--headed`. +- `expand`, `collapse`, `set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, and `scroll-to` remain semantic under `--headed` after any core focus precondition. +- `press`, `hover`, `drag`, `mouse-move`, `mouse-click`, and `mouse-wheel` are explicit physical input; cursor-moving commands require `--headed`. +- Raw coordinates never trigger focus because they carry no window identity. Held-input commands are reserved and fail closed until a daemon owns their lifetime. +- FFI ref-action callers get the same strict headless `type` default; direct-handle `ad_execute_action` is lower-level and applies the supplied policy verbatim. - If a command would need a forbidden physical path, it returns a structured error with a recovery hint. ### Surfaces @@ -94,7 +88,7 @@ macOS apps can have multiple accessibility surfaces: |---------|-------------|-------------| | `window` | Main application window (default) | General UI interaction | | `focused` | Currently focused element's context | Inspecting active element | -| `menu` | Open dropdown or context menu | After `select`, verified `right-click`, or explicit menu trigger | +| `menu` | Open dropdown or context menu | After `select`, `right-click`, or an explicit menu trigger | | `menubar` | Application menu bar | Navigating File/Edit/View menus | | `sheet` | Modal sheet (Save dialog, etc.) | After triggering sheet dialogs | | `popover` | Popover/popup content | Inspecting tooltips, popovers | @@ -127,18 +121,20 @@ agent-desktop interacts with macOS Notification Center via the accessibility API ### How It Works -1. **NcSession** opens Notification Center by clicking the clock in ControlCenter (if not already open) +1. **NcSession** reuses an already-open Notification Center headlessly, or opens it only when headed policy allows focus/cursor side effects 2. Notifications are read from the AX tree under the NotificationCenter process -3. After operations, NcSession closes NC and restores the previously focused app +3. When NcSession opened the center, it closes it and restores the previously focused app afterward 4. The `Drop` impl ensures cleanup even on errors +All notification mutations require `--headed`. Listing and notification waits can remain headless only when Notification Center is already open. + ### Dismiss Strategy -Headless approach (no cursor movement from the default ref command path): +Mutation chain under explicit headed policy: 1. **AXDismiss** / **AXRemoveFromParent** — native accessibility actions 2. **Close button** — find and press AXButton named "close", "clear", or "dismiss" -3. **Hover + close button** — move cursor to reveal hidden close button, then press it +3. **Hover + close button** — use the authorized pointer path to reveal a hidden close button, then press it 4. If all fail, returns `ACTION_FAILED` **Important:** `AXPress` is intentionally excluded from dismiss — it "clicks" the notification body (opening the source app) without actually dismissing it. @@ -237,7 +233,7 @@ Large apps (Xcode, Safari with many tabs) can have deep trees. ### Context Menu Doesn't Appear -After `right-click @ref`, inspect `menu` first. If it is absent and `menu_probe.ok` is `false`: +After `right-click @ref`, inspect the open menu or the target effect. If macOS returns `APP_UNRESPONSIVE` for `AXShowMenu` with unsafe retry semantics: 1. The element may not support context menus 2. If the target is a combo box or menu button, use `select @ref "Option"` instead 3. Run `list-surfaces --app "App"` to confirm whether a menu surface exists diff --git a/skills/agent-desktop/references/workflows.md b/skills/agent-desktop/references/workflows.md index d2ccadd..8cc536f 100644 --- a/skills/agent-desktop/references/workflows.md +++ b/skills/agent-desktop/references/workflows.md @@ -2,6 +2,10 @@ Patterns for using agent-desktop effectively in multi-step desktop automation tasks. +Snapshot output uses qualified refs such as `@s8f3k2p9:e3`. Examples that pair +a legacy bare ref with `--snapshot ` intentionally demonstrate the +still-supported compatibility form; never use a bare ref without that flag. + ## First-Time Setup Before any automation, verify permissions: @@ -17,12 +21,13 @@ For screenshots, also grant Screen Recording. `permissions` reports `accessibili ## Pattern: Session-Scoped Tracing (Default for Multi-Step Runs) -Start one session per agent run so tracing and the latest-snapshot namespace follow automatically — no `--trace` on every command. +Start one session per agent run, then explicitly select its ID so tracing and the latest-snapshot namespace follow consistently — no `--trace` on every command. ```bash -# 1. Start once — creates manifest (trace: on), pointer, and trace/ directory +# 1. Start once — creates manifest (trace: on) and trace/ directory agent-desktop session start --name "invoice-bot" -# Note session_id from data.session_id (also written to ~/.agent-desktop/current_session) +# Note session_id from data.session_id, then export it for later processes +export AGENT_DESKTOP_SESSION= # 2. Observe-act loop — segments land under sessions//trace/-*.jsonl agent-desktop snapshot --app "Preview" -i --compact @@ -30,11 +35,11 @@ agent-desktop click @e3 --snapshot agent-desktop status # confirms session_id + tracing: true # 3. End and reclaim when finished -agent-desktop session end +agent-desktop session end "$AGENT_DESKTOP_SESSION" agent-desktop session gc ``` -**Concurrent independent agents:** set `AGENT_DESKTOP_SESSION=` in each process instead of sharing the global pointer. Each agent still uses the `snapshot_id` from its own `snapshot` call when sharing a session id. +**Concurrent independent agents:** set `AGENT_DESKTOP_SESSION=` in each process. Each agent should use qualified refs from its own snapshot when sharing a session ID. **Namespace without tracing:** `session start --no-trace` or bare `--session legacy-id` (no manifest) — snapshots namespaced, no JSONL files. @@ -126,8 +131,8 @@ agent-desktop snapshot --app "TextEdit" --surface sheet -i ## Pattern: Right-Click Context Menu ```bash -# 1. Right-click the target element. Success means a menu surface was verified. -agent-desktop right-click @e3 +# 1. Right-click the target. On macOS, APP_UNRESPONSIVE can mean AXShowMenu entered modal tracking after delivery; inspect the effect before retrying. +agent-desktop right-click @s8f3k2p9:e3 # 2. Use the returned menu tree, or snapshot the menu surface if you need a fresh read. agent-desktop snapshot --app "Finder" --surface menu -i @@ -186,22 +191,22 @@ agent-desktop click @e14 --snapshot ```bash # For sequential form filling without needing refs for each field: -agent-desktop focus @e1 # Explicit focus change -agent-desktop type @e1 "value1" +agent-desktop focus @s8f3k2p9:e1 # Explicit focus change +agent-desktop type @s8f3k2p9:e1 "value1" agent-desktop press tab # Now in next field — type directly since focus moved agent-desktop press tab # Skip a field -agent-desktop type @e3 "value3" # Or snapshot again to get new refs +agent-desktop type @s8f3k2p9:e3 "value3" # Or snapshot again to get new refs ``` ## Pattern: Copy Text from Element ```bash # Option A: Read directly via accessibility -agent-desktop get @e5 --property value +agent-desktop get @s8f3k2p9:e5 --property value # Option B: Copy via keyboard -agent-desktop focus @e5 +agent-desktop focus @s8f3k2p9:e5 agent-desktop press cmd+a agent-desktop press cmd+c agent-desktop clipboard-get @@ -211,13 +216,13 @@ agent-desktop clipboard-get ```bash # Between elements (by ref) -agent-desktop --headed drag --from @e3 --to @e8 +agent-desktop --headed drag --from @s8f3k2p9:e3 --to @s8f3k2p9:e8 # Between coordinates agent-desktop --headed drag --from-xy 100,200 --to-xy 500,400 # Mixed: element to coordinates -agent-desktop --headed drag --from @e3 --to-xy 500,400 --duration 500 +agent-desktop --headed drag --from @s8f3k2p9:e3 --to-xy 500,400 --duration 500 ``` ## Pattern: Wait for Async UI @@ -269,19 +274,19 @@ agent-desktop snapshot --app "Finder" --window-id "w-5678" -i ```bash # Check if already in desired state -agent-desktop is @e6 --property checked +agent-desktop is @s8f3k2p9:e6 --property checked # If result is false, then check it -agent-desktop check @e6 +agent-desktop check @s8f3k2p9:e6 # Or use check/uncheck directly (they're idempotent) -agent-desktop check @e6 # No-op if already checked -agent-desktop uncheck @e6 # No-op if already unchecked +agent-desktop check @s8f3k2p9:e6 # No-op if already checked +agent-desktop uncheck @s8f3k2p9:e6 # No-op if already unchecked ``` ## Pattern: Batch Operations ```bash -# Run multiple commands atomically +# Run multiple commands sequentially in one process; this is not a transaction agent-desktop batch '[ {"command":"click","args":{"ref_id":"@e1","snapshot":""}}, {"command":"wait","args":{"ms":200}}, @@ -301,4 +306,4 @@ agent-desktop batch '[ 7. **Assuming UI stability.** Re-drill the affected region after every action that could change the UI. 8. **Snapshotting the full window when an overlay is open.** Use `--surface sheet/alert/popover/menu` instead. Never `--skeleton` for surfaces — they're already focused. 9. **Re-snapshotting everything after one action.** Use scoped re-drill (`--root @ref`) to refresh only the affected region. Other refs stay valid. -10. **Relying on implicit focus or cursor movement.** Non-mouse ref commands use semantic paths and block silent physical/headed paths. +10. **Assuming headed and headless have the same side effects.** Headless ref actions block implicit focus/cursor input. Headed ref actions intentionally focus the exact source window when required, and headed pointer actions may move the cursor; raw coordinates never infer focus. diff --git a/src/Cargo.toml b/src/Cargo.toml index 6487fe5..6456a52 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -1,8 +1,9 @@ [package] -name = "agent-desktop" +name = "agent-desktop" version.workspace = true edition.workspace = true license.workspace = true +autotests = false [dependencies] agent-desktop-core.workspace = true @@ -34,5 +35,9 @@ path = "tests/snapshot_test.rs" name = "conformance" path = "tests/conformance.rs" +[[test]] +name = "cli_process" +path = "tests/cli_process.rs" + [lints] workspace = true diff --git a/src/batch/baseline_tests.rs b/src/batch/baseline_tests.rs new file mode 100644 index 0000000..dfe6c52 --- /dev/null +++ b/src/batch/baseline_tests.rs @@ -0,0 +1,108 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use agent_desktop_core::{ + PermissionReport, SignalBaseline, SignalCompleteness, SignalFilter, WindowState, +}; + +use super::execute; +use crate::cli_args::batch::BatchArgs; + +struct AtomicEventAdapter { + opened: AtomicBool, +} + +impl agent_desktop_core::ObservationOps for AtomicEventAdapter { + fn list_apps( + &self, + _deadline: agent_desktop_core::Deadline, + ) -> Result, agent_desktop_core::AdapterError> { + Ok(vec![agent_desktop_core::AppInfo { + name: "Fixture".into(), + pid: agent_desktop_core::ProcessId::new(42), + bundle_id: Some("com.example.fixture".into()), + process_instance: Some("fixture-42".into()), + }]) + } +} + +impl agent_desktop_core::ActionOps for AtomicEventAdapter {} + +impl agent_desktop_core::InputOps for AtomicEventAdapter { + fn clear_clipboard( + &self, + _lease: &agent_desktop_core::InteractionLease, + ) -> Result<(), agent_desktop_core::AdapterError> { + self.opened.store(true, Ordering::SeqCst); + Ok(()) + } +} + +impl agent_desktop_core::SystemOps for AtomicEventAdapter { + fn acquire_interaction_lease( + &self, + deadline: agent_desktop_core::Deadline, + ) -> Result { + agent_desktop_core::InteractionLease::guarded(deadline, ()) + } + + fn capture_signal_baseline( + &self, + _filter: &SignalFilter, + _deadline: agent_desktop_core::Deadline, + ) -> Result { + let windows = self + .opened + .load(Ordering::SeqCst) + .then(|| agent_desktop_core::WindowInfo { + id: "w-sync".into(), + title: "Synchronous".into(), + app: "Fixture".into(), + pid: agent_desktop_core::ProcessId::new(42), + process_instance: Some("fixture-42".into()), + bounds: None, + state: WindowState { + is_focused: true, + ..WindowState::default() + }, + }) + .into_iter() + .collect(); + Ok(SignalBaseline { + windows, + apps: Vec::new(), + surfaces: Vec::new(), + completeness: SignalCompleteness::complete(), + }) + } +} + +#[test] +fn action_then_event_wait_uses_a_pre_action_baseline() { + let args = BatchArgs { + commands_json: serde_json::json!([ + {"command": "clipboard-clear", "args": {}}, + { + "command": "wait", + "args": {"event": "window-opened", "app": "Fixture", "timeout": 100} + } + ]) + .to_string(), + stop_on_error: true, + timeout_ms: 60_000, + }; + let adapter = AtomicEventAdapter { + opened: AtomicBool::new(false), + }; + + let value = execute( + args, + &adapter, + &PermissionReport::default(), + &agent_desktop_core::CommandContext::default(), + ) + .unwrap(); + + assert_eq!(value["results"][0]["ok"], true, "{value}"); + assert_eq!(value["results"][1]["ok"], true, "{value}"); + assert_eq!(value["results"][1]["data"]["event"]["window_id"], "w-sync"); +} diff --git a/src/batch/bounded_json.rs b/src/batch/bounded_json.rs new file mode 100644 index 0000000..b473474 --- /dev/null +++ b/src/batch/bounded_json.rs @@ -0,0 +1,38 @@ +use std::io::{Error, ErrorKind, Write}; + +use serde_json::Value; + +pub(super) fn serialized_size(value: &Value) -> usize { + let mut writer = CountingWriter::new(usize::MAX); + let _ = serde_json::to_writer(&mut writer, value); + writer.written +} + +pub(super) fn serialized_fits(value: &Value, limit: usize) -> bool { + serde_json::to_writer(CountingWriter::new(limit), value).is_ok() +} + +struct CountingWriter { + limit: usize, + written: usize, +} + +impl CountingWriter { + fn new(limit: usize) -> Self { + Self { limit, written: 0 } + } +} + +impl Write for CountingWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.limit.saturating_sub(self.written) { + return Err(Error::new(ErrorKind::FileTooLarge, "JSON output limit")); + } + self.written += bytes.len(); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} diff --git a/src/batch/execution.rs b/src/batch/execution.rs new file mode 100644 index 0000000..901c755 --- /dev/null +++ b/src/batch/execution.rs @@ -0,0 +1,208 @@ +use agent_desktop_core::{ + AdapterError, AppError, Deadline, DeliverySemantics, ErrorCode, PermissionReport, + PlatformAdapter, SignalBaseline, SignalFilter, context::CommandContext, +}; +use serde_json::{Value, json}; + +use crate::{cli::Commands, cli_args::batch::BatchArgs}; + +use super::{ + bounded_json::serialized_size, + preparation::{MAX_BATCH_ENTRIES, MAX_BATCH_JSON_BYTES, PreparedCommand, prepare}, + result_entry::{MAX_BATCH_OUTPUT_BYTES, bounded_entry, not_started_entry}, +}; + +pub(super) fn execute( + args: BatchArgs, + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, + context: &CommandContext, +) -> Result { + let deadline = Deadline::after(args.timeout_ms) + .map_err(|error| error.with_disposition(DeliverySemantics::not_delivered()))?; + let batch_context = context.clone().with_inherited_deadline(deadline); + let mut commands = prepare(&args.commands_json, permission_report, &batch_context)?; + let total = commands.len(); + let mut results = Vec::with_capacity(total); + let mut results_bytes = 0; + let mut completed = 0; + let mut pending_baseline: Option> = None; + let mut stopped = None; + + for index in 0..total { + if deadline.is_expired() { + let error = batch_timeout(index, &commands[index].name, args.timeout_ms); + push_small_entry( + &mut results, + &mut results_bytes, + not_started_entry(index, &commands[index].name, "deadline", error), + ); + stopped = Some(json!({ "reason": "deadline", "index": index })); + break; + } + + let current_baseline = pending_baseline.take(); + pending_baseline = match commands.get(index + 1).and_then(event_filter) { + Some(filter) => match adapter.capture_signal_baseline(&filter, deadline) { + Ok(baseline) => Some(Ok(baseline)), + Err(error) => { + let wait_index = index + 1; + let wait_command = &commands[wait_index].name; + let error = baseline_error( + index, + &commands[index].name, + wait_index, + wait_command, + error, + ); + push_small_entry( + &mut results, + &mut results_bytes, + not_started_entry( + index, + &commands[index].name, + "pre_action_baseline_failed", + error, + ), + ); + stopped = Some(json!({ + "reason": "pre_action_baseline_failed", + "blocked_index": index, + "blocked_command": commands[index].name, + "wait_index": wait_index, + "wait_command": wait_command, + })); + break; + } + }, + None => None, + }; + + let item_context = commands[index] + .context + .clone() + .with_inherited_deadline(deadline) + .with_event_baseline(current_baseline); + let command = std::mem::replace(&mut commands[index].command, Commands::Version); + let result = crate::dispatch::dispatch(command, adapter, permission_report, &item_context); + let failed = result.is_err(); + let (entry, oversized) = bounded_entry(index, &commands[index].name, result, results_bytes); + completed += 1; + push_small_entry(&mut results, &mut results_bytes, entry); + + if oversized { + stopped = Some(json!({ "reason": "output_limit", "index": index })); + break; + } + if failed && args.stop_on_error { + stopped = Some(json!({ "reason": "stop_on_error", "index": index })); + break; + } + if deadline.is_expired() && index + 1 < total { + stopped = Some(json!({ "reason": "deadline", "after_index": index })); + break; + } + } + + let mut body = json!({ + "results": results, + "semantics": { + "atomic": false, + "order": "sequential", + "batch_retries": false, + "command_retry_contracts": "preserved", + "successful_action_disposition": "data.disposition", + "error_disposition": "error.disposition", + }, + "total_entries": total, + "completed_entries": completed, + "not_started_entries": total.saturating_sub(completed), + "timeout_ms": args.timeout_ms, + "elapsed_ms": deadline.elapsed().as_millis(), + "limits": { + "max_entries": MAX_BATCH_ENTRIES, + "max_input_bytes": MAX_BATCH_JSON_BYTES, + "max_output_bytes": MAX_BATCH_OUTPUT_BYTES, + } + }); + if let Some(stopped) = stopped { + body["stopped"] = stopped; + } + if serialized_size(&body) > MAX_BATCH_OUTPUT_BYTES { + let disposition = if completed == 0 { + DeliverySemantics::not_delivered() + } else { + DeliverySemantics::uncertain() + }; + return Err(AdapterError::new( + ErrorCode::Internal, + "Batch response exceeded its output contract after final serialization", + ) + .with_details(json!({ + "kind": "batch_output_limit", + "completed_entries": completed, + "max_output_bytes": MAX_BATCH_OUTPUT_BYTES, + })) + .with_disposition(disposition) + .into()); + } + Ok(body) +} + +fn event_filter(command: &PreparedCommand) -> Option { + match &command.command { + Commands::Wait(args) if args.event.event.is_some() => Some(SignalFilter { + app: args.app.clone(), + process: None, + }), + _ => None, + } +} + +fn baseline_error( + blocked_index: usize, + blocked_command: &str, + wait_index: usize, + wait_command: &str, + mut source: AdapterError, +) -> AppError { + let cause_details = source.details.take(); + source.message = format!( + "Batch entry {blocked_index} ('{blocked_command}') was not started because the baseline for following wait entry {wait_index} ('{wait_command}') failed: {}", + source.message + ); + let mut details = json!({ + "kind": "pre_action_baseline_failed", + "blocked_index": blocked_index, + "blocked_command": blocked_command, + "wait_index": wait_index, + "wait_command": wait_command, + }); + if let Some(cause_details) = cause_details { + details["cause_details"] = cause_details; + } + source.details = Some(details); + source.disposition = DeliverySemantics::not_delivered(); + source.into() +} + +fn batch_timeout(index: usize, command: &str, timeout_ms: u64) -> AppError { + AdapterError::timeout("Batch deadline elapsed before the entry started") + .with_details(json!({ + "kind": "batch_deadline", + "batch_index": index, + "batch_command": command, + "timeout_ms": timeout_ms, + })) + .with_disposition(DeliverySemantics::not_delivered()) + .into() +} + +fn push_small_entry(results: &mut Vec, used: &mut usize, entry: Value) { + *used = used.saturating_add(serialized_size(&entry).saturating_add(1)); + results.push(entry); +} + +#[cfg(test)] +#[path = "execution_tests.rs"] +mod tests; diff --git a/src/batch/execution_tests.rs b/src/batch/execution_tests.rs new file mode 100644 index 0000000..f592042 --- /dev/null +++ b/src/batch/execution_tests.rs @@ -0,0 +1,255 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use agent_desktop_core::{CommandContext, InteractionLease, PermissionReport, PermissionState}; +use serde_json::json; + +use super::*; + +struct CountingAdapter { + clears: AtomicUsize, +} + +impl agent_desktop_core::ObservationOps for CountingAdapter {} +impl agent_desktop_core::ActionOps for CountingAdapter {} + +impl agent_desktop_core::InputOps for CountingAdapter { + fn clear_clipboard(&self, _lease: &InteractionLease) -> Result<(), AdapterError> { + self.clears.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +impl agent_desktop_core::SystemOps for CountingAdapter { + fn acquire_interaction_lease( + &self, + deadline: agent_desktop_core::Deadline, + ) -> Result { + agent_desktop_core::InteractionLease::guarded(deadline, ()) + } +} + +fn args(commands: Value, timeout_ms: u64) -> BatchArgs { + BatchArgs { + commands_json: commands.to_string(), + stop_on_error: false, + timeout_ms, + } +} + +fn adapter() -> CountingAdapter { + CountingAdapter { + clears: AtomicUsize::new(0), + } +} + +#[test] +fn validates_every_entry_before_the_first_side_effect() { + let adapter = adapter(); + for commands in [ + json!([ + {"command": "clipboard-clear", "args": {}}, + {"command": "missing", "args": {}} + ]), + json!([ + {"command": "clipboard-clear", "args": {}}, + {"command": "click", "args": {"ref_id": "not-a-ref"}} + ]), + ] { + let error = execute( + args(commands, 60_000), + &adapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect_err("a malformed or policy-invalid later entry rejects the whole batch"); + assert_eq!(error.code(), "INVALID_ARGS"); + } + let denied = PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "grant accessibility".into(), + }, + screen_recording: PermissionState::Granted, + automation: PermissionState::NotRequired, + }; + let error = execute( + args( + json!([ + {"command": "clipboard-clear", "args": {}}, + {"command": "snapshot", "args": {}} + ]), + 60_000, + ), + &adapter, + &denied, + &CommandContext::default(), + ) + .expect_err("a later permission failure rejects the whole batch"); + assert_eq!(error.code(), "PERM_DENIED"); + assert_eq!(adapter.clears.load(Ordering::SeqCst), 0); +} + +#[test] +fn dispatches_each_entry_at_most_once() { + let adapter = adapter(); + let output = execute( + args(json!([{"command": "clipboard-clear", "args": {}}]), 60_000), + &adapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect("valid batch executes"); + + assert_eq!(adapter.clears.load(Ordering::SeqCst), 1); + assert_eq!(output["semantics"]["batch_retries"], false); + assert_eq!(output["semantics"]["atomic"], false); + assert_eq!(output["results"][0]["execution"], "completed"); +} + +#[test] +fn expired_batch_never_starts_the_entry() { + let adapter = adapter(); + let output = execute( + args(json!([{"command": "clipboard-clear", "args": {}}]), 0), + &adapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect("deadline is reported in the batch result"); + + assert_eq!(adapter.clears.load(Ordering::SeqCst), 0); + assert_eq!(output["results"][0]["execution"], "not_started"); + assert_eq!(output["results"][0]["not_started_reason"], "deadline"); + assert_eq!( + output["results"][0]["error"]["disposition"]["delivery"], + "not_delivered" + ); + assert_eq!( + output["results"][0]["error"]["disposition"]["retry"], + "safe" + ); + assert_eq!(output["stopped"]["reason"], "deadline"); +} + +#[test] +fn baseline_failure_prevents_the_producer_action() { + let adapter = adapter(); + let output = execute( + args( + json!([ + {"command": "clipboard-clear", "args": {}}, + {"command": "wait", "args": {"event": "window-opened", "timeout": 100}} + ]), + 60_000, + ), + &adapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect("runtime baseline failure remains a structured batch result"); + + assert_eq!(adapter.clears.load(Ordering::SeqCst), 0); + assert_eq!(output["results"][0]["execution"], "not_started"); + assert_eq!( + output["results"][0]["not_started_reason"], + "pre_action_baseline_failed" + ); + assert_eq!(output["stopped"]["blocked_index"], 0); + assert_eq!(output["stopped"]["wait_index"], 1); + assert_eq!(output["stopped"]["reason"], "pre_action_baseline_failed"); +} + +#[test] +fn entry_count_and_output_are_bounded() { + let adapter = adapter(); + let input_error = execute( + BatchArgs { + commands_json: " ".repeat(MAX_BATCH_JSON_BYTES + 1), + stop_on_error: false, + timeout_ms: 60_000, + }, + &adapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect_err("oversized JSON is rejected before parsing"); + assert_eq!(input_error.code(), "INVALID_ARGS"); + + let commands = (0..=MAX_BATCH_ENTRIES) + .map(|_| json!({"command": "version", "args": {}})) + .collect::>(); + let error = execute( + args(Value::Array(commands), 60_000), + &adapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect_err("oversized entry count is rejected"); + assert_eq!(error.code(), "INVALID_ARGS"); + + let huge = Ok(json!({ + "payload": "x".repeat(MAX_BATCH_OUTPUT_BYTES), + "disposition": { + "delivery": "delivered_verified", + "retry": "unsafe" + } + })); + let (entry, oversized) = bounded_entry(0, "snapshot", huge, 0); + assert!(oversized); + assert!(serialized_size(&entry) < super::super::result_entry::OUTPUT_METADATA_RESERVE); + assert_eq!( + entry["error"]["disposition"]["delivery"], + "delivered_verified" + ); + assert_eq!(entry["error"]["disposition"]["retry"], "unsafe"); +} + +struct SlowReadAdapter; + +impl agent_desktop_core::ObservationOps for SlowReadAdapter {} +impl agent_desktop_core::ActionOps for SlowReadAdapter {} +impl agent_desktop_core::InputOps for SlowReadAdapter {} + +impl agent_desktop_core::SystemOps for SlowReadAdapter { + fn list_displays( + &self, + deadline: agent_desktop_core::Deadline, + ) -> Result, AdapterError> { + std::thread::sleep( + deadline + .remaining() + .saturating_add(Duration::from_millis(2)), + ); + Err(deadline.timeout_error()) + } +} + +#[test] +fn inherited_batch_deadline_bounds_slow_non_action_command() { + let started = Instant::now(); + let output = execute( + args(json!([{"command": "list-displays", "args": {}}]), 25), + &SlowReadAdapter, + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect("entry timeout remains a structured batch result"); + + assert!(started.elapsed() < Duration::from_millis(500)); + assert_eq!(output["results"][0]["error"]["code"], "TIMEOUT"); +} + +#[test] +fn inherited_batch_deadline_caps_sleep_without_variant_rewriting() { + let started = Instant::now(); + let output = execute( + args(json!([{"command": "wait", "args": {"ms": 5_000}}]), 25), + &adapter(), + &PermissionReport::default(), + &CommandContext::default(), + ) + .expect("entry timeout remains a structured batch result"); + + assert!(started.elapsed() < Duration::from_millis(500)); + assert_eq!(output["results"][0]["error"]["code"], "TIMEOUT"); +} diff --git a/src/batch/mod.rs b/src/batch/mod.rs index 83bf78c..1629f86 100644 --- a/src/batch/mod.rs +++ b/src/batch/mod.rs @@ -1,51 +1,37 @@ use agent_desktop_core::{ - PermissionReport, - adapter::PlatformAdapter, - commands::batch::BatchCommand, + AppError, PermissionReport, PlatformAdapter, commands::batch::BatchCommand, context::CommandContext, - error::AppError, - output::{ENVELOPE_VERSION, ErrorPayload}, }; use serde::Deserialize; use serde::de::DeserializeOwned; -use serde_json::{Map, Value, json}; +use serde_json::{Map, Value}; use crate::{ cli::Commands, cli_args::{ + batch::BatchArgs, session::{SessionAction, SessionArgs, SessionEndArgs, SessionGcArgs, SessionStartArgs}, skills::{SkillsAction, SkillsArgs, SkillsGetArgs}, - system::BatchArgs, trace::{TraceAction, TraceArgs, TraceExportArgs, TraceShowArgs}, }, }; +mod bounded_json; +mod execution; +mod preparation; +mod result_entry; + +#[cfg(test)] +#[path = "baseline_tests.rs"] +mod baseline_tests; + pub(crate) fn execute( args: BatchArgs, adapter: &dyn PlatformAdapter, permission_report: &PermissionReport, context: &CommandContext, ) -> Result { - let commands = agent_desktop_core::commands::batch::parse_commands(&args.commands_json)?; - let mut results = Vec::new(); - - for item in commands { - let command = item.command.clone(); - let result = match context.for_batch_item(item.session.clone()) { - Ok(item_context) => parse_command(item).and_then(|typed| { - crate::command_policy::preflight(&typed, permission_report)?; - crate::dispatch::dispatch(typed, adapter, permission_report, &item_context) - }), - Err(err) => Err(err), - }; - let ok = result.is_ok(); - results.push(batch_entry(&command, result)); - if !ok && args.stop_on_error { - break; - } - } - - Ok(json!({ "results": results })) + execution::execute(args, adapter, permission_report, context) } pub(crate) fn parse_command(item: BatchCommand) -> Result { @@ -81,9 +67,11 @@ pub(crate) fn parse_command(item: BatchCommand) -> Result { "mouse-click" => decode(command, item.args).map(Commands::MouseClick), "mouse-down" => decode(command, item.args).map(Commands::MouseDown), "mouse-up" => decode(command, item.args).map(Commands::MouseUp), + "mouse-wheel" => decode(command, item.args).map(Commands::MouseWheel), "launch" => decode(command, item.args).map(Commands::Launch), "close-app" => decode(command, item.args).map(Commands::CloseApp), "list-windows" => decode(command, item.args).map(Commands::ListWindows), + "list-displays" => no_args(command, item.args).map(|()| Commands::ListDisplays), "list-apps" => decode(command, item.args).map(Commands::ListApps), "focus-window" => decode(command, item.args).map(Commands::FocusWindow), "resize-window" => decode(command, item.args).map(Commands::ResizeWindow), @@ -98,7 +86,7 @@ pub(crate) fn parse_command(item: BatchCommand) -> Result { decode(command, item.args).map(Commands::DismissAllNotifications) } "notification-action" => decode(command, item.args).map(Commands::NotificationAction), - "clipboard-get" => no_args(command, item.args).map(|()| Commands::ClipboardGet), + "clipboard-get" => decode(command, item.args).map(Commands::ClipboardGet), "clipboard-set" => decode(command, item.args).map(Commands::ClipboardSet), "clipboard-clear" => no_args(command, item.args).map(|()| Commands::ClipboardClear), "wait" => decode(command, item.args).map(Commands::Wait), @@ -113,30 +101,20 @@ pub(crate) fn parse_command(item: BatchCommand) -> Result { "Flatten nested batches into one top-level batch array", )), other => Err(AppError::invalid_input(format!( - "Unknown batch command '{other}'" + "Unknown batch command {}", + crate::diagnostic::token_label(other) ))), } } -fn batch_entry(command: &str, result: Result) -> Value { - match result { - Ok(data) => { - json!({ "version": ENVELOPE_VERSION, "ok": true, "command": command, "data": data }) - } - Err(err) => { - let error = ErrorPayload::from_app_error(&err); - json!({ "version": ENVELOPE_VERSION, "ok": false, "command": command, "error": error }) - } - } -} - fn decode(command: &str, args: Value) -> Result where T: DeserializeOwned, { - serde_json::from_value(args_or_empty(args)).map_err(|e| { + serde_json::from_value(args_or_empty(args)).map_err(|error| { + let diagnostic = crate::diagnostic::bounded_text(&error.to_string(), 512); AppError::invalid_input_with_suggestion( - format!("Invalid batch args for '{command}': {e}"), + format!("Invalid batch args for '{command}': {diagnostic}"), "Use the same argument names and value types as the matching CLI command", ) }) @@ -165,32 +143,49 @@ struct BatchSkillsArgs { action: Option, name: Option, reference: Option, - #[serde(default)] - full: bool, + full: Option, } fn parse_skills(args: Value) -> Result { let args: BatchSkillsArgs = decode("skills", args)?; let action = match args.action.as_deref() { - None if args.name.is_none() => Some(SkillsAction::List), + None if args.name.is_none() && args.reference.is_none() && args.full.is_none() => { + Some(SkillsAction::List) + } None | Some("get") => Some(SkillsAction::Get(SkillsGetArgs { name: args .name .ok_or_else(|| AppError::invalid_input("Batch skills get requires 'name'"))?, reference: args.reference, - full: args.full, + full: args.full.unwrap_or(false), })), - Some("list") => Some(SkillsAction::List), - Some("path") => Some(SkillsAction::Path), + Some("list") => { + reject_skills_extras(&args, "list")?; + Some(SkillsAction::List) + } + Some("path") => { + reject_skills_extras(&args, "path")?; + Some(SkillsAction::Path) + } Some(other) => { return Err(AppError::invalid_input(format!( - "Unknown skills action '{other}'" + "Unknown skills action {}", + crate::diagnostic::token_label(other) ))); } }; Ok(SkillsArgs { action }) } +fn reject_skills_extras(args: &BatchSkillsArgs, action: &str) -> Result<(), AppError> { + if args.name.is_some() || args.reference.is_some() || args.full.is_some() { + return Err(AppError::invalid_input(format!( + "Batch skills {action} does not accept name, reference, or full" + ))); + } + Ok(()) +} + #[derive(Deserialize)] struct BatchSessionActionArgs { #[serde(default)] @@ -207,8 +202,6 @@ struct BatchSessionStartArgs { no_trace: bool, #[serde(default)] screenshots: bool, - #[serde(default)] - force: bool, } #[derive(Deserialize)] @@ -239,7 +232,6 @@ fn parse_session(args: Value) -> Result { name: args.name, no_trace: args.no_trace, screenshots: args.screenshots, - force: args.force, }) } Some("end") => { @@ -255,7 +247,8 @@ fn parse_session(args: Value) -> Result { } Some(other) => { return Err(AppError::invalid_input(format!( - "Unknown session action '{other}'" + "Unknown session action {}", + crate::diagnostic::token_label(other) ))); } }; @@ -275,21 +268,36 @@ struct BatchTraceArgs { fn parse_trace(args: Value) -> Result { let args: BatchTraceArgs = decode("trace", args)?; let action = match args.action.as_str() { - "show" => TraceAction::Show(TraceShowArgs { - limit: args - .limit - .unwrap_or(agent_desktop_core::commands::trace::TRACE_SHOW_DEFAULT_LIMIT), - event: args.event, - }), - "export" => TraceAction::Export(TraceExportArgs { - limit: args - .limit - .unwrap_or(agent_desktop_core::trace_read::TRACE_EXPORT_DEFAULT_LIMIT), - out: args.out, - }), + "show" => { + if args.out.is_some() { + return Err(AppError::invalid_input( + "Batch trace show does not accept out", + )); + } + TraceAction::Show(TraceShowArgs { + limit: args + .limit + .unwrap_or(agent_desktop_core::commands::trace::TRACE_SHOW_DEFAULT_LIMIT), + event: args.event, + }) + } + "export" => { + if args.event.is_some() { + return Err(AppError::invalid_input( + "Batch trace export does not accept event", + )); + } + TraceAction::Export(TraceExportArgs { + limit: args + .limit + .unwrap_or(agent_desktop_core::trace_read::TRACE_EXPORT_DEFAULT_LIMIT), + out: args.out, + }) + } other => { return Err(AppError::invalid_input(format!( - "Unknown trace action '{other}'" + "Unknown trace action {}", + crate::diagnostic::token_label(other) ))); } }; diff --git a/src/batch/preparation.rs b/src/batch/preparation.rs new file mode 100644 index 0000000..675d0cd --- /dev/null +++ b/src/batch/preparation.rs @@ -0,0 +1,103 @@ +use agent_desktop_core::{ + AdapterError, AppError, DeliverySemantics, ErrorCode, PermissionReport, + commands::batch::BatchCommand, context::CommandContext, +}; +use serde_json::{Value, json}; + +use crate::cli::Commands; + +pub(super) const MAX_BATCH_JSON_BYTES: usize = 1024 * 1024; +pub(super) const MAX_BATCH_ENTRIES: usize = 64; + +pub(super) struct PreparedCommand { + pub name: String, + pub command: Commands, + pub context: CommandContext, +} + +pub(super) fn prepare( + input: &str, + permission_report: &PermissionReport, + context: &CommandContext, +) -> Result, AppError> { + if input.len() > MAX_BATCH_JSON_BYTES { + return Err(limit_error( + "Batch JSON exceeds the input limit", + json!({ "actual_bytes": input.len(), "max_bytes": MAX_BATCH_JSON_BYTES }), + )); + } + let items = agent_desktop_core::commands::batch::parse_commands(input)?; + if items.len() > MAX_BATCH_ENTRIES { + return Err(limit_error( + "Batch contains too many entries", + json!({ "actual_entries": items.len(), "max_entries": MAX_BATCH_ENTRIES }), + )); + } + + let parsed = items + .into_iter() + .enumerate() + .map(|(index, item)| parse_one(index, item, permission_report)) + .collect::, _>>()?; + parsed + .into_iter() + .map(|(index, name, command, session)| { + let item_context = context + .for_batch_item(session) + .map_err(|error| located_error(index, &name, error))?; + Ok(PreparedCommand { + name, + command, + context: item_context, + }) + }) + .collect() +} + +fn parse_one( + index: usize, + item: BatchCommand, + permission_report: &PermissionReport, +) -> Result<(usize, String, Commands, Option), AppError> { + let name = item.command.clone(); + let session = item.session.clone(); + if let Some(session) = session.as_deref() { + agent_desktop_core::context::validate_session_id(session) + .map_err(|error| located_error(index, &name, error))?; + } + let command = super::parse_command(item).map_err(|error| located_error(index, &name, error))?; + crate::command_policy::preflight(&command, permission_report) + .map_err(|error| located_error(index, &name, error))?; + Ok((index, name, command, session)) +} + +fn located_error(index: usize, command: &str, error: AppError) -> AppError { + match error { + AppError::Adapter(mut source) => { + let cause_details = source.details.take(); + let mut details = json!({ "batch_index": index, "batch_command": command }); + if let Some(cause_details) = cause_details { + details["cause_details"] = cause_details; + } + source.message = format!( + "Batch entry {index} ('{command}') failed validation: {}", + source.message + ); + source.details = Some(details); + source.disposition = DeliverySemantics::not_delivered(); + source.into() + } + other => AdapterError::new(ErrorCode::Internal, other.to_string()) + .with_details(json!({ "batch_index": index, "batch_command": command })) + .with_disposition(DeliverySemantics::not_delivered()) + .into(), + } +} + +fn limit_error(message: &str, details: Value) -> AppError { + AdapterError::new(ErrorCode::InvalidArgs, message) + .with_suggestion("Split the batch or narrow commands that return large payloads") + .with_details(details) + .with_disposition(DeliverySemantics::not_delivered()) + .into() +} diff --git a/src/batch/result_entry.rs b/src/batch/result_entry.rs new file mode 100644 index 0000000..2ebbeb5 --- /dev/null +++ b/src/batch/result_entry.rs @@ -0,0 +1,96 @@ +use agent_desktop_core::{ + AdapterError, AppError, DeliverySemantics, ErrorCode, + output::{ENVELOPE_VERSION, ErrorPayload}, +}; +use serde_json::{Value, json}; + +use super::bounded_json::serialized_fits; + +pub(super) const MAX_BATCH_OUTPUT_BYTES: usize = 4 * 1024 * 1024; +pub(super) const OUTPUT_METADATA_RESERVE: usize = 8 * 1024; + +pub(super) fn bounded_entry( + index: usize, + command: &str, + result: Result, + used: usize, +) -> (Value, bool) { + let entry = completed_entry(index, command, result); + let available = MAX_BATCH_OUTPUT_BYTES + .saturating_sub(OUTPUT_METADATA_RESERVE) + .saturating_sub(used); + if serialized_fits(&entry, available) { + return (entry, false); + } + let disposition = entry_disposition(&entry); + let error = AdapterError::new( + ErrorCode::InvalidArgs, + "Batch entry completed but its result exceeded the response limit", + ) + .with_suggestion("Split the batch or narrow commands that return large payloads") + .with_details(json!({ + "batch_index": index, + "max_output_bytes": MAX_BATCH_OUTPUT_BYTES, + "result_omitted": true, + })) + .with_disposition(disposition) + .into(); + (completed_entry(index, command, Err(error)), true) +} + +pub(super) fn completed_entry( + index: usize, + command: &str, + result: Result, +) -> Value { + match result { + Ok(data) => json!({ + "version": ENVELOPE_VERSION, + "ok": true, + "command": command, + "index": index, + "execution": "completed", + "data": data, + }), + Err(error) => json!({ + "version": ENVELOPE_VERSION, + "ok": false, + "command": command, + "index": index, + "execution": "completed", + "error": ErrorPayload::from_app_error(&error), + }), + } +} + +pub(super) fn not_started_entry( + index: usize, + command: &str, + reason: &str, + error: AppError, +) -> Value { + json!({ + "version": ENVELOPE_VERSION, + "ok": false, + "command": command, + "index": index, + "execution": "not_started", + "not_started_reason": reason, + "error": ErrorPayload::from_app_error(&error), + }) +} + +fn entry_disposition(entry: &Value) -> DeliverySemantics { + let disposition = entry + .get("data") + .and_then(|data| data.get("disposition")) + .or_else(|| { + entry + .get("error") + .and_then(|error| error.get("disposition")) + }); + disposition + .cloned() + .and_then(|value| serde_json::from_value(value).ok()) + .unwrap_or_else(DeliverySemantics::uncertain) +} diff --git a/src/batch/tests.rs b/src/batch/tests.rs index b573414..146a968 100644 --- a/src/batch/tests.rs +++ b/src/batch/tests.rs @@ -1,11 +1,9 @@ use super::*; use crate::cli_args::Surface; -use agent_desktop_core::{PermissionReport, adapter::PlatformAdapter}; +use crate::test_noop_ops::NoopAdapter; +use agent_desktop_core::{PermissionReport, output::ENVELOPE_VERSION}; use clap::CommandFactory; -struct NoopAdapter; -impl PlatformAdapter for NoopAdapter {} - fn item(command: &str, args: Value) -> BatchCommand { BatchCommand { command: command.to_string(), @@ -58,8 +56,8 @@ fn applies_cli_defaults_during_batch_decode() { match command { Commands::Snapshot(args) => { - assert_eq!(args.max_depth, 10); - assert!(!args.interactive_only); + assert_eq!(args.tree.max_depth, 10); + assert!(!args.tree.interactive_only); assert!(matches!(args.surface, Surface::Window)); } other => panic!("unexpected command: {other:?}"), @@ -100,11 +98,12 @@ fn rejects_version_args_after_json_flag_removal() { fn stop_on_error_halts_after_first_failure() { let args = BatchArgs { commands_json: serde_json::json!([ - {"command": "missing", "args": {}}, + {"command": "snapshot", "args": {}}, {"command": "version", "args": {}} ]) .to_string(), stop_on_error: true, + timeout_ms: 60_000, }; let value = execute( @@ -119,14 +118,8 @@ fn stop_on_error_halts_after_first_failure() { assert_eq!(results.len(), 1); assert_eq!(results[0]["ok"], false); assert_eq!(results[0]["version"], ENVELOPE_VERSION); - assert_eq!(results[0]["command"], "missing"); - assert_eq!(results[0]["error"]["code"], "INVALID_ARGS"); - assert!( - results[0]["error"]["message"] - .as_str() - .unwrap() - .contains("Unknown batch command") - ); + assert_eq!(results[0]["command"], "snapshot"); + assert_eq!(results[0]["error"]["code"], "PLATFORM_NOT_SUPPORTED"); } #[test] @@ -138,6 +131,22 @@ fn no_args_rejection_has_suggestion() { assert!(err.suggestion().is_some()); } +#[test] +fn list_displays_accepts_only_empty_args() { + assert!(matches!( + parse_command(item("list-displays", serde_json::Value::Null)), + Ok(Commands::ListDisplays) + )); + assert!(matches!( + parse_command(item("list-displays", serde_json::json!({}))), + Ok(Commands::ListDisplays) + )); + + let err = parse_command(item("list-displays", serde_json::json!({ "display": 1 }))) + .expect_err("list-displays rejects unknown args"); + assert_eq!(err.code(), "INVALID_ARGS"); +} + #[test] fn nested_batch_rejection_has_suggestion() { let err = parse_command(item("batch", serde_json::json!({}))).expect_err("batch rejected"); @@ -196,6 +205,26 @@ fn trace_batch_rejects_unknown_field() { assert_eq!(err.code(), "INVALID_ARGS"); } +#[test] +fn batch_variants_reject_known_but_inapplicable_fields() { + for (command, args) in [ + ("skills", serde_json::json!({"action": "list", "name": "x"})), + ( + "skills", + serde_json::json!({"action": "path", "full": false}), + ), + ("trace", serde_json::json!({"action": "show", "out": "x"})), + ( + "trace", + serde_json::json!({"action": "export", "event": "x"}), + ), + ] { + let error = parse_command(item(command, args)) + .expect_err("fields for another action must be rejected"); + assert_eq!(error.code(), "INVALID_ARGS"); + } +} + #[test] fn session_batch_accepts_screenshots_flag() { let command = parse_command(item( diff --git a/src/cli/contract_tests.rs b/src/cli/contract_tests.rs index 7088f4e..b0a6dc2 100644 --- a/src/cli/contract_tests.rs +++ b/src/cli/contract_tests.rs @@ -1,24 +1,34 @@ use crate::cli::Cli; -use clap::CommandFactory; +use clap::{CommandFactory, Parser, error::ErrorKind}; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; const NON_COMMAND_MODULES: &[&str] = &[ "combo", "execute_by_ref", + "find_live", "helpers", "helpers_test_support", + "input_hold_policy", "mod", + "notification_identity", + "notification_policy", "point_resolve", + "pointer_action", "query", + "stale_retry_test_support", "wait_element", "wait_latest_ref_cache", "wait_mode", "wait_predicate", + "wait_surface", "wait_test_support", "wait_text_match", "wait_timeout", + "wait_event", + "wait_event_input", "wait_selector", + "window_target", ]; const COMMAND_SPECIFIC_TESTS: &[&str] = &[ @@ -56,6 +66,28 @@ const SHARED_REF_ACTION_TESTS: &[&str] = &[ const BINARY_CONTRACT_TESTS: &[&str] = &["batch", "permissions", "version"]; +#[test] +fn standard_version_flag_reports_the_package_version() { + let error = Cli::try_parse_from(["agent-desktop", "--version"]) + .expect_err("--version should use clap's display-version path"); + + assert_eq!(error.kind(), ErrorKind::DisplayVersion); + assert_eq!( + error.to_string().trim(), + format!("agent-desktop {}", env!("CARGO_PKG_VERSION")) + ); +} + +#[test] +fn ci_compares_the_release_binary_to_the_workspace_version() { + let workflow = include_str!("../../.github/workflows/ci.yml"); + + assert!(workflow.contains("PACKAGE_ID=$(cargo pkgid -p agent-desktop)")); + assert!(workflow.contains("EXPECTED_VERSION=${PACKAGE_ID##*@}")); + assert!(workflow.contains("EXPECTED_OUTPUT=\"agent-desktop $EXPECTED_VERSION\"")); + assert!(workflow.contains("[ \"$ACTUAL_OUTPUT\" != \"$EXPECTED_OUTPUT\" ]")); +} + const ADAPTER_PASSTHROUGH_COMMANDS: &[&str] = &[ "clipboard-clear", "clipboard-get", @@ -72,12 +104,14 @@ const ADAPTER_PASSTHROUGH_COMMANDS: &[&str] = &[ "list-notifications", "list-surfaces", "list-windows", + "list-displays", "maximize", "minimize", "mouse-click", "mouse-down", "mouse-move", "mouse-up", + "mouse-wheel", "move-window", "notification-action", "press", @@ -128,6 +162,87 @@ fn every_cli_subcommand_has_explicit_test_coverage_classification() { } } +#[test] +fn macos_capability_count_includes_restored_notifications() { + let commands = cli_command_names(); + assert_eq!( + commands.len(), + 58, + "the published CLI command count changed" + ); + assert_eq!( + commands.len(), + 58, + "macOS operational command count changed; update capability documentation" + ); +} + +#[test] +fn every_visible_cli_argument_has_help_text() { + fn inspect(command: &clap::Command) { + for argument in command + .get_arguments() + .filter(|argument| !argument.is_hide_set()) + { + assert!( + argument + .get_help() + .is_some_and(|help| !help.to_string().is_empty()), + "{} argument {:?} has no help text", + command.get_name(), + argument.get_id() + ); + } + for child in command.get_subcommands() { + inspect(child); + } + } + + inspect(&Cli::command()); +} + +#[test] +fn curated_help_exposes_new_surfaces_and_current_ref_contract() { + let help = Cli::command().render_long_help().to_string(); + for expected in [ + "mouse-wheel", + "list-displays", + "--file-url", + "wait --event ", + "@s8f3k2p9:e1", + "does not activate", + "session-owned refs require the same scope", + ] { + assert!( + help.contains(expected), + "missing curated help text: {expected}" + ); + } + assert!(!help.contains("current-session pointer")); + assert!(!help.contains("explicit --snapshot IDs do not require it")); +} + +#[test] +fn permissions_help_names_the_isolated_request_boundary() { + let mut command = Cli::command(); + let help = command + .find_subcommand_mut("permissions") + .expect("permissions command") + .render_long_help() + .to_string(); + + assert!(help.contains("Request missing permissions in the bounded isolated helper")); +} + +#[test] +fn faq_uses_the_unwind_safe_ffi_build_profile() { + let faq = include_str!("../../docs/faq.md"); + + assert!(faq.contains("cargo build --profile release-ffi -p agent-desktop-ffi")); + assert!(faq.contains("target/release-ffi/")); + assert!(!faq.contains("cargo build --release\n# Outputs: libagent_desktop_ffi")); +} + #[test] fn core_command_modules_keep_tests_in_sibling_files() { for path in command_module_paths() { diff --git a/src/cli/help_after.txt b/src/cli/help_after.txt index 641ec1b..7d67891 100644 --- a/src/cli/help_after.txt +++ b/src/cli/help_after.txt @@ -8,7 +8,7 @@ OBSERVATION INTERACTION click Click element (kAXPress) - double-click Open via AXOpen; physical double-click uses mouse-click + double-click Physical double-click; requires --headed triple-click Triple-click element; POLICY_DENIED if physical input is disabled right-click Right-click; includes menu when verified type Insert text; may use focus fallback only for explicit policy paths @@ -36,11 +36,13 @@ MOUSE mouse-click --xy x,y Click at coordinates (--button, --count; requires --headed) mouse-down --xy x,y Press mouse button at coordinates (requires --headed) mouse-up --xy x,y Release mouse button at coordinates (requires --headed) + mouse-wheel --x X --y Y Post wheel deltas at coordinates (--dx, --dy; requires --headed) APP & WINDOW launch Launch app and wait until window is visible close-app Quit app gracefully (--force to terminate) list-windows All visible windows (--app to filter) + list-displays Displays and screenshot indices list-apps All running GUI applications (--app substring filter) focus-window Bring window to front and confirm focus resize-window Resize window (--width, --height) @@ -56,8 +58,8 @@ NOTIFICATIONS notification-action Click action button on notification CLIPBOARD - clipboard-get Read plain-text clipboard - clipboard-set Write text to clipboard + clipboard-get Read text, image, or file URLs (--format; --out for binary data) + clipboard-set [text] Write text, --image PATH, or repeatable --file-url URL clipboard-clear Clear the clipboard WAIT @@ -71,15 +73,16 @@ WAIT wait --menu --app Block until a menu surface is open wait --menu-closed --app Block until a menu surface is dismissed wait --notification Block until a new notification arrives (--text filters it) + wait --event Block on a baseline diff: window/app/focus/surface lifecycle SYSTEM status Adapter health, platform, permission, session, and tracing state permissions Check nested permission states: {state,...} version Show version, target architecture, and OS skills Bundled skill docs for AI agents (list, get, path) - session start Create a trace-enabled session and set the active pointer + session start Create a trace-enabled session; pass its returned ID explicitly session start --screenshots Opt in to screenshot/refmap replay artifacts (requires tracing) - session end Seal the session manifest and clear the active pointer + session end Seal a manifest (ID from argument, --session, or environment) session list List session manifests session gc Remove ended or provably-stale sessions @@ -88,20 +91,18 @@ TRACE trace export Export a self-contained HTML trace viewer (--limit defaults to 5000; 0 = all) BATCH - batch Run commands from a JSON array (--stop-on-error) + batch Run a bounded, sequential, non-atomic JSON command batch batch items may set "session": "id" to override the inherited --session REF IDs - snapshot assigns @e1, @e2, ... to interactive elements in depth-first order. - Use a ref wherever appears. Refs are snapshot-scoped. Pass - --snapshot to pin a command to a known snapshot; explicit - snapshot IDs do not require --session. When --snapshot is omitted, ref - commands use the active session's latest saved snapshot. Run snapshot again - after UI changes. Use `session start` once per run for automatic JSONL tracing - under ~/.agent-desktop/sessions//trace/, or pass --session for - snapshot namespace only (no trace without a session manifest). Set - AGENT_DESKTOP_SESSION for concurrent independent sessions. Explicit --session - overrides the env var and current-session pointer. + snapshot emits qualified refs such as @s8f3k2p9:e1 in depth-first order. + Qualified refs embed their snapshot ID and need no --snapshot flag. Legacy + bare @eN input requires an explicit --snapshot . Snapshot lookup + stays inside the selected namespace: pass the same --session or set + AGENT_DESKTOP_SESSION when the snapshot belongs to a session. Explicit + --session overrides the environment; with neither, commands use the global + non-session namespace. `session start` returns an ID but does not activate + later processes. Run snapshot again after UI changes. Ref actions use strict resolution: stale targets return STALE_REF; duplicate plausible targets return AMBIGUOUS_TARGET instead of choosing arbitrarily. Ref actions run actionability checks before dispatch. Use --trace to @@ -123,23 +124,24 @@ EXAMPLES agent-desktop check @e3 --snapshot agent-desktop type @e2 --snapshot "hello@example.com" agent-desktop press cmd+z - agent-desktop --headed drag --from @e1 --to @e5 - agent-desktop --headed hover @e5 + agent-desktop --headed drag --from @s8f3k2p9:e1 --to @s8f3k2p9:e5 + agent-desktop --headed hover @s8f3k2p9:e5 agent-desktop minimize --app TextEdit agent-desktop resize-window --app TextEdit --width 800 --height 600 + agent-desktop move-window --window-id w-4521 --x 100 --y 100 agent-desktop --headed mouse-click --xy 500,300 agent-desktop wait --text "Loading complete" --app Safari --timeout 5000 - agent-desktop wait --element @e5 --predicate actionable --timeout 5000 + agent-desktop wait --element @s8f3k2p9:e5 --predicate actionable --timeout 5000 agent-desktop snapshot --app Finder -w "button:OK" - agent-desktop click @e5 -w ":Saved!" - agent-desktop session start --name run-a - agent-desktop click @e5 + agent-desktop click @s8f3k2p9:e5 -w ":Saved!" + agent-desktop session start --name run-a # note data.session_id + agent-desktop --session click @s8f3k2p9:e5 agent-desktop batch '[{"command":"click","args":{"ref_id":"@e1","snapshot":""}}]' agent-desktop --session run-b batch '[{"command":"status","session":"run-b","args":{}}]' AGENT LOOP 1. Use `skills get desktop --full` when you need detailed guidance. - 2. Run `session start` once per agent run (or set AGENT_DESKTOP_SESSION). + 2. Run `session start`, then pass its returned ID via --session or AGENT_DESKTOP_SESSION. 3. Observe with `snapshot -i` or `snapshot --skeleton -i --compact`. 4. Keep `snapshot_id`; pass it to every ref-consuming action. 5. Use `wait --predicate actionable`, `wait --text`, or `wait --window` diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 5377040..feb0a43 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,12 +1,19 @@ -use clap::{Parser, Subcommand}; -use std::path::PathBuf; +use clap::Subcommand; + +mod post_action_wait; +mod root; + +pub(crate) use root::Cli; use crate::cli_args::{ FindArgs, GetArgs, IsArgs, ListSurfacesArgs, RefArgs, ScreenshotArgs, SnapshotArgs, actions::{ - DragCliArgs, HoverArgs, KeyComboArgs, MouseClickArgs, MouseMoveArgs, MousePointArgs, - PressArgs, ScrollArgs, SelectArgs, SetValueArgs, TypeArgs, + HoverArgs, KeyComboArgs, MouseClickArgs, MouseMoveArgs, MousePointArgs, PressArgs, + ScrollArgs, SelectArgs, SetValueArgs, TypeArgs, }, + batch::BatchArgs, + drag::DragCliArgs, + mouse_wheel::MouseWheelArgs, notifications::{ DismissAllNotificationsCliArgs, DismissNotificationCliArgs, ListNotificationsCliArgs, NotificationActionCliArgs, @@ -14,90 +21,13 @@ use crate::cli_args::{ session::SessionArgs, skills::SkillsArgs, system::{ - AppRefArgs, BatchArgs, ClipboardSetArgs, CloseAppArgs, FocusWindowArgs, LaunchArgs, + AppRefArgs, ClipboardGetArgs, ClipboardSetArgs, CloseAppArgs, FocusWindowArgs, LaunchArgs, ListAppsArgs, ListWindowsArgs, MoveWindowCliArgs, PermissionsArgs, ResizeWindowCliArgs, WaitArgs, }, trace::TraceArgs, }; -const BEFORE_HELP: &str = include_str!("help_before.txt"); -const AFTER_HELP: &str = include_str!("help_after.txt"); - -#[derive(Parser, Debug)] -#[command( - name = "agent-desktop", - about = "Desktop automation CLI for AI agents", - long_about = None, - before_help = BEFORE_HELP, - after_help = AFTER_HELP, -)] -pub(crate) struct Cli { - #[arg( - long, - short = 'v', - global = true, - help = "Enable debug logging to stderr" - )] - pub verbose: bool, - - #[arg( - long, - global = true, - help = "Select the latest-snapshot namespace; explicit --snapshot IDs do not require it" - )] - pub session: Option, - - #[arg( - long, - global = true, - help = "Append reliability trace JSONL to this path" - )] - pub trace: Option, - - #[arg( - long, - global = true, - help = "Fail on trace setup/pre-action write errors" - )] - pub trace_strict: bool, - - #[arg( - long, - global = true, - help = "Permit cursor movement and focus stealing for physical input commands and fallbacks. Default is headless (AX-only, no cursor)." - )] - pub headed: bool, - - #[arg( - long, - short = 'w', - global = true, - conflicts_with = "wait_for_gone", - help = "Poll until an element matching the role:text selector appears, then return the snapshot" - )] - pub wait_for: Option, - - #[arg( - long, - global = true, - conflicts_with = "wait_for", - help = "Poll until no element matches the role:text selector, then return the snapshot" - )] - pub wait_for_gone: Option, - - #[arg( - long, - global = true, - default_value_t = 30_000, - help = "Maximum wait time in milliseconds for --wait-for / --wait-for-gone (default: 30000)" - )] - pub wait_timeout: u64, - - #[command(subcommand)] - pub command: Option, -} - #[derive(Subcommand, Debug)] pub(crate) enum Commands { #[command(about = "Capture accessibility tree as structured JSON with @ref IDs")] @@ -112,13 +42,13 @@ pub(crate) enum Commands { Is(IsArgs), #[command(about = "Click element via accessibility press action")] Click(RefArgs), - #[command(about = "Open element via AXOpen; physical double-click uses mouse-click")] + #[command(about = "Physically double-click element; requires --headed")] DoubleClick(RefArgs), #[command( about = "Triple-click element; returns POLICY_DENIED when physical input is disabled" )] TripleClick(RefArgs), - #[command(about = "Right-click and include menu/menu_probe details when available")] + #[command(about = "Open a context menu semantically, or physically with --headed")] RightClick(RefArgs), #[command(about = "Insert text into a text target")] Type(TypeArgs), @@ -162,12 +92,16 @@ pub(crate) enum Commands { MouseDown(MousePointArgs), #[command(about = "Release mouse button at coordinates (requires --headed)")] MouseUp(MousePointArgs), + #[command(about = "Scroll the mouse wheel at absolute coordinates (requires --headed)")] + MouseWheel(MouseWheelArgs), #[command(about = "Launch application and wait until its window is visible")] Launch(LaunchArgs), #[command(about = "Quit an application gracefully (--force to terminate)")] CloseApp(CloseAppArgs), #[command(about = "List all visible windows (--app to filter by application)")] ListWindows(ListWindowsArgs), + #[command(about = "List connected displays with bounds and scale factor")] + ListDisplays, #[command(about = "List all running GUI applications (--app to filter)")] ListApps(ListAppsArgs), #[command(about = "Bring a window to front and confirm OS focus")] @@ -192,8 +126,8 @@ pub(crate) enum Commands { DismissAllNotifications(DismissAllNotificationsCliArgs), #[command(about = "Click an action button on a notification")] NotificationAction(NotificationActionCliArgs), - #[command(about = "Read plain-text clipboard contents")] - ClipboardGet, + #[command(about = "Read plain-text or typed clipboard contents")] + ClipboardGet(ClipboardGetArgs), #[command(about = "Write text to the clipboard")] ClipboardSet(ClipboardSetArgs), #[command(about = "Clear the clipboard")] @@ -208,7 +142,7 @@ pub(crate) enum Commands { Permissions(PermissionsArgs), #[command(about = "Show version, target architecture, and OS")] Version, - #[command(about = "Execute multiple commands from a JSON array (--stop-on-error)")] + #[command(about = "Execute a bounded, sequential, non-atomic JSON command batch")] Batch(BatchArgs), #[command(about = "Bundled skill docs for AI agents (list, get, path)")] Skills(SkillsArgs), @@ -218,65 +152,159 @@ pub(crate) enum Commands { Trace(TraceArgs), } +#[derive(Clone, Copy)] +struct CommandMetadata { + name: &'static str, + post_action_wait: bool, +} + +impl CommandMetadata { + const fn new(name: &'static str, post_action_wait: bool) -> Self { + Self { + name, + post_action_wait, + } + } +} + impl Commands { - pub(crate) fn name(&self) -> &'static str { + fn metadata(&self) -> CommandMetadata { match self { - Self::Snapshot(_) => "snapshot", - Self::Find(_) => "find", - Self::Screenshot(_) => "screenshot", - Self::Get(_) => "get", - Self::Is(_) => "is", - Self::Click(_) => "click", - Self::DoubleClick(_) => "double-click", - Self::TripleClick(_) => "triple-click", - Self::RightClick(_) => "right-click", - Self::Type(_) => "type", - Self::SetValue(_) => "set-value", - Self::Clear(_) => "clear", - Self::Focus(_) => "focus", - Self::Select(_) => "select", - Self::Toggle(_) => "toggle", - Self::Check(_) => "check", - Self::Uncheck(_) => "uncheck", - Self::Expand(_) => "expand", - Self::Collapse(_) => "collapse", - Self::Scroll(_) => "scroll", - Self::ScrollTo(_) => "scroll-to", - Self::Press(_) => "press", - Self::KeyDown(_) => "key-down", - Self::KeyUp(_) => "key-up", - Self::Hover(_) => "hover", - Self::Drag(_) => "drag", - Self::MouseMove(_) => "mouse-move", - Self::MouseClick(_) => "mouse-click", - Self::MouseDown(_) => "mouse-down", - Self::MouseUp(_) => "mouse-up", - Self::Launch(_) => "launch", - Self::CloseApp(_) => "close-app", - Self::ListWindows(_) => "list-windows", - Self::ListApps(_) => "list-apps", - Self::FocusWindow(_) => "focus-window", - Self::ResizeWindow(_) => "resize-window", - Self::MoveWindow(_) => "move-window", - Self::Minimize(_) => "minimize", - Self::Maximize(_) => "maximize", - Self::Restore(_) => "restore", - Self::ListSurfaces(_) => "list-surfaces", - Self::ListNotifications(_) => "list-notifications", - Self::DismissNotification(_) => "dismiss-notification", - Self::DismissAllNotifications(_) => "dismiss-all-notifications", - Self::NotificationAction(_) => "notification-action", - Self::ClipboardGet => "clipboard-get", - Self::ClipboardSet(_) => "clipboard-set", - Self::ClipboardClear => "clipboard-clear", - Self::Wait(_) => "wait", - Self::Status => "status", - Self::Permissions(_) => "permissions", - Self::Version => "version", - Self::Batch(_) => "batch", - Self::Skills(_) => "skills", - Self::Session(_) => "session", - Self::Trace(_) => "trace", + Self::Snapshot(_) => CommandMetadata::new("snapshot", true), + Self::Find(_) => CommandMetadata::new("find", false), + Self::Screenshot(_) => CommandMetadata::new("screenshot", false), + Self::Get(_) => CommandMetadata::new("get", false), + Self::Is(_) => CommandMetadata::new("is", false), + Self::Click(_) => CommandMetadata::new("click", true), + Self::DoubleClick(_) => CommandMetadata::new("double-click", true), + Self::TripleClick(_) => CommandMetadata::new("triple-click", true), + Self::RightClick(_) => CommandMetadata::new("right-click", true), + Self::Type(_) => CommandMetadata::new("type", true), + Self::SetValue(_) => CommandMetadata::new("set-value", true), + Self::Clear(_) => CommandMetadata::new("clear", true), + Self::Focus(_) => CommandMetadata::new("focus", true), + Self::Select(_) => CommandMetadata::new("select", true), + Self::Toggle(_) => CommandMetadata::new("toggle", true), + Self::Check(_) => CommandMetadata::new("check", true), + Self::Uncheck(_) => CommandMetadata::new("uncheck", true), + Self::Expand(_) => CommandMetadata::new("expand", true), + Self::Collapse(_) => CommandMetadata::new("collapse", true), + Self::Scroll(_) => CommandMetadata::new("scroll", true), + Self::ScrollTo(_) => CommandMetadata::new("scroll-to", true), + Self::Press(_) => CommandMetadata::new("press", false), + Self::KeyDown(_) => CommandMetadata::new("key-down", false), + Self::KeyUp(_) => CommandMetadata::new("key-up", false), + Self::Hover(_) => CommandMetadata::new("hover", true), + Self::Drag(_) => CommandMetadata::new("drag", true), + Self::MouseMove(_) => CommandMetadata::new("mouse-move", false), + Self::MouseClick(_) => CommandMetadata::new("mouse-click", false), + Self::MouseDown(_) => CommandMetadata::new("mouse-down", false), + Self::MouseUp(_) => CommandMetadata::new("mouse-up", false), + Self::MouseWheel(_) => CommandMetadata::new("mouse-wheel", false), + Self::Launch(_) => CommandMetadata::new("launch", false), + Self::CloseApp(_) => CommandMetadata::new("close-app", false), + Self::ListWindows(_) => CommandMetadata::new("list-windows", false), + Self::ListDisplays => CommandMetadata::new("list-displays", false), + Self::ListApps(_) => CommandMetadata::new("list-apps", false), + Self::FocusWindow(_) => CommandMetadata::new("focus-window", false), + Self::ResizeWindow(_) => CommandMetadata::new("resize-window", false), + Self::MoveWindow(_) => CommandMetadata::new("move-window", false), + Self::Minimize(_) => CommandMetadata::new("minimize", false), + Self::Maximize(_) => CommandMetadata::new("maximize", false), + Self::Restore(_) => CommandMetadata::new("restore", false), + Self::ListSurfaces(_) => CommandMetadata::new("list-surfaces", false), + Self::ListNotifications(_) => CommandMetadata::new("list-notifications", false), + Self::DismissNotification(_) => CommandMetadata::new("dismiss-notification", false), + Self::DismissAllNotifications(_) => { + CommandMetadata::new("dismiss-all-notifications", false) + } + Self::NotificationAction(_) => CommandMetadata::new("notification-action", false), + Self::ClipboardGet(_) => CommandMetadata::new("clipboard-get", false), + Self::ClipboardSet(_) => CommandMetadata::new("clipboard-set", false), + Self::ClipboardClear => CommandMetadata::new("clipboard-clear", false), + Self::Wait(_) => CommandMetadata::new("wait", false), + Self::Status => CommandMetadata::new("status", false), + Self::Permissions(_) => CommandMetadata::new("permissions", false), + Self::Version => CommandMetadata::new("version", false), + Self::Batch(_) => CommandMetadata::new("batch", false), + Self::Skills(_) => CommandMetadata::new("skills", false), + Self::Session(_) => CommandMetadata::new("session", false), + Self::Trace(_) => CommandMetadata::new("trace", false), + } + } + + pub(crate) fn name(&self) -> &'static str { + self.metadata().name + } + + pub(crate) fn supports_post_action_wait(&self) -> bool { + self.metadata().post_action_wait + } + + pub(crate) fn is_mutating(&self) -> bool { + match self { + Self::Click(_) + | Self::DoubleClick(_) + | Self::TripleClick(_) + | Self::RightClick(_) + | Self::Type(_) + | Self::SetValue(_) + | Self::Clear(_) + | Self::Focus(_) + | Self::Select(_) + | Self::Toggle(_) + | Self::Check(_) + | Self::Uncheck(_) + | Self::Expand(_) + | Self::Collapse(_) + | Self::Scroll(_) + | Self::ScrollTo(_) + | Self::Press(_) + | Self::KeyDown(_) + | Self::KeyUp(_) + | Self::Hover(_) + | Self::Drag(_) + | Self::MouseMove(_) + | Self::MouseClick(_) + | Self::MouseDown(_) + | Self::MouseUp(_) + | Self::MouseWheel(_) + | Self::Launch(_) + | Self::CloseApp(_) + | Self::FocusWindow(_) + | Self::ResizeWindow(_) + | Self::MoveWindow(_) + | Self::Minimize(_) + | Self::Maximize(_) + | Self::Restore(_) + | Self::DismissNotification(_) + | Self::DismissAllNotifications(_) + | Self::NotificationAction(_) + | Self::ClipboardSet(_) + | Self::ClipboardClear + | Self::Batch(_) => true, + Self::Screenshot(args) => args.output_path.is_some(), + Self::ClipboardGet(args) => args.out.is_some(), + Self::Permissions(args) => args.request, + Self::Session(args) => { + !matches!(&args.action, crate::cli_args::session::SessionAction::List) + } + Self::Trace(args) => { + matches!(&args.action, crate::cli_args::trace::TraceAction::Export(_)) + } + Self::Snapshot(_) + | Self::Find(_) + | Self::Get(_) + | Self::Is(_) + | Self::ListWindows(_) + | Self::ListDisplays + | Self::ListApps(_) + | Self::ListSurfaces(_) + | Self::ListNotifications(_) + | Self::Wait(_) + | Self::Status + | Self::Version + | Self::Skills(_) => false, } } } diff --git a/src/cli/post_action_wait.rs b/src/cli/post_action_wait.rs new file mode 100644 index 0000000..fdaefdf --- /dev/null +++ b/src/cli/post_action_wait.rs @@ -0,0 +1,26 @@ +use clap::Args; + +#[derive(Args, Debug)] +pub(crate) struct PostActionWaitArgs { + #[arg( + long, + short = 'w', + global = true, + conflicts_with = "wait_for_gone", + help = "Poll until an element matching the role:text selector appears, then return the snapshot" + )] + pub wait_for: Option, + #[arg( + long, + global = true, + conflicts_with = "wait_for", + help = "Poll until no element matches the role:text selector, then return the snapshot" + )] + pub wait_for_gone: Option, + #[arg( + long, + global = true, + help = "Maximum wait time in milliseconds for --wait-for / --wait-for-gone (default: 30000)" + )] + pub wait_timeout: Option, +} diff --git a/src/cli/root.rs b/src/cli/root.rs new file mode 100644 index 0000000..5dc6c9a --- /dev/null +++ b/src/cli/root.rs @@ -0,0 +1,55 @@ +use std::path::PathBuf; + +use clap::Parser; + +use super::{Commands, post_action_wait::PostActionWaitArgs}; + +const BEFORE_HELP: &str = include_str!("help_before.txt"); +const AFTER_HELP: &str = include_str!("help_after.txt"); + +#[derive(Parser, Debug)] +#[command( + name = "agent-desktop", + version, + about = "Desktop automation CLI for AI agents", + long_about = None, + before_help = BEFORE_HELP, + after_help = AFTER_HELP, +)] +pub(crate) struct Cli { + #[arg( + long, + short = 'v', + global = true, + help = "Enable debug logging to stderr" + )] + pub verbose: bool, + #[arg( + long, + global = true, + help = "Select the snapshot namespace; session-owned refs require the same scope" + )] + pub session: Option, + #[arg( + long, + global = true, + help = "Append reliability trace JSONL to this path" + )] + pub trace: Option, + #[arg( + long, + global = true, + help = "Fail on trace setup/pre-action write errors" + )] + pub trace_strict: bool, + #[arg( + long, + global = true, + help = "Prefer physical delivery for natural input commands and permit focus/cursor side effects. Default is strict headless semantic delivery." + )] + pub headed: bool, + #[command(flatten)] + pub post_action_wait: PostActionWaitArgs, + #[command(subcommand)] + pub command: Option, +} diff --git a/src/cli/wait_for_cli_tests.rs b/src/cli/wait_for_cli_tests.rs index 16b8049..55358c7 100644 --- a/src/cli/wait_for_cli_tests.rs +++ b/src/cli/wait_for_cli_tests.rs @@ -1,4 +1,4 @@ -use super::Cli; +use super::{Cli, Commands}; use agent_desktop_core::context::WaitSelector; use clap::{CommandFactory, Parser}; @@ -10,6 +10,15 @@ fn selector(query_raw: &str) -> WaitSelector { } } +fn command(arguments: &[&str]) -> Commands { + let mut argv = vec!["agent-desktop"]; + argv.extend_from_slice(arguments); + Cli::try_parse_from(argv) + .expect("command parses") + .command + .expect("subcommand is present") +} + #[test] fn help_lists_global_wait_for_flags() { let help = Cli::command().render_help(); @@ -46,8 +55,18 @@ fn short_w_flag_maps_to_wait_for() { "Finder", ]) .expect("short -w should parse"); - assert_eq!(cli.wait_for.as_deref(), Some("button:Submit")); - assert_eq!(cli.wait_timeout, 30_000); + assert_eq!( + cli.post_action_wait.wait_for.as_deref(), + Some("button:Submit") + ); + assert_eq!(cli.post_action_wait.wait_timeout, None); + assert_eq!( + crate::build_wait_selector(&cli) + .expect("selector is valid") + .expect("selector is present") + .timeout_ms, + 30_000 + ); } #[test] @@ -61,41 +80,76 @@ fn wait_timeout_parses_custom_value() { "snapshot", ]) .expect("custom wait timeout should parse"); - assert_eq!(cli.wait_timeout, 5_000); + assert_eq!(cli.post_action_wait.wait_timeout, Some(5_000)); +} + +#[test] +fn wait_timeout_without_selector_is_rejected_before_dispatch() { + let cli = Cli::try_parse_from(["agent-desktop", "--wait-timeout", "5000", "snapshot"]) + .expect("syntax parses before semantic validation"); + + let error = crate::build_wait_selector(&cli) + .expect_err("an unused wait timeout must not be silently ignored"); + assert_eq!(error.code(), "INVALID_ARGS"); } #[test] fn validate_rejects_unsupported_command() { - let err = crate::validate_wait_for_command("find", &selector("button:OK")) - .expect_err("find must not accept --wait-for"); + let err = crate::validate_wait_for_command( + &command(&["find", "--role", "button"]), + &selector("button:OK"), + ) + .expect_err("find must not accept --wait-for"); assert_eq!(err.code(), "INVALID_ARGS"); } #[test] fn validate_rejects_match_everything_selector_before_dispatch() { - let err = crate::validate_wait_for_command("click", &selector("")) + let click = command(&["click", "@e1"]); + let err = crate::validate_wait_for_command(&click, &selector("")) .expect_err("empty selector must be rejected before the action runs"); assert_eq!(err.code(), "INVALID_ARGS"); - assert!(crate::validate_wait_for_command("click", &selector(":")).is_err()); + assert!(crate::validate_wait_for_command(&click, &selector(":")).is_err()); } #[test] fn validate_accepts_supported_command_with_constraining_selector() { - assert!(crate::validate_wait_for_command("click", &selector(":Saved!")).is_ok()); - assert!(crate::validate_wait_for_command("snapshot", &selector("button")).is_ok()); + for supported in [ + command(&["snapshot"]), + command(&["click", "@e1"]), + command(&["hover", "@e1"]), + command(&["drag", "--from", "@e1", "--to", "@e2"]), + ] { + assert!(crate::validate_wait_for_command(&supported, &selector(":Saved!")).is_ok()); + } } #[test] -fn wait_supported_names_are_real_subcommands() { - let cmd = Cli::command(); - let subcommands: Vec = cmd - .get_subcommands() - .map(|sub| sub.get_name().to_string()) - .collect(); - for name in crate::WAIT_SUPPORTED { - assert!( - subcommands.iter().any(|sub| sub == name), - "WAIT_SUPPORTED entry '{name}' is not a real subcommand" - ); - } +fn typed_metadata_couples_name_and_wait_support_to_each_variant() { + let hover = command(&["hover", "@e1"]); + let drag = command(&["drag", "--from", "@e1", "--to", "@e2"]); + let find = command(&["find", "--role", "button"]); + + assert_eq!(hover.name(), "hover"); + assert!(hover.supports_post_action_wait()); + assert_eq!(drag.name(), "drag"); + assert!(drag.supports_post_action_wait()); + assert_eq!(find.name(), "find"); + assert!(!find.supports_post_action_wait()); +} + +#[test] +fn mutation_metadata_distinguishes_read_only_and_file_writing_forms() { + assert!(command(&["click", "@e1"]).is_mutating()); + assert!(!command(&["find", "--role", "button"]).is_mutating()); + assert!(!command(&["screenshot"]).is_mutating()); + assert!(command(&["screenshot", "/tmp/capture.png"]).is_mutating()); + assert!(!command(&["clipboard-get"]).is_mutating()); + assert!(command(&["clipboard-get", "--out", "/tmp/clipboard.png"]).is_mutating()); + assert!(!command(&["permissions"]).is_mutating()); + assert!(command(&["permissions", "--request"]).is_mutating()); + assert!(!command(&["session", "list"]).is_mutating()); + assert!(command(&["session", "start"]).is_mutating()); + assert!(!command(&["trace", "show"]).is_mutating()); + assert!(command(&["trace", "export"]).is_mutating()); } diff --git a/src/cli_args/actions.rs b/src/cli_args/actions.rs index 056b147..473690b 100644 --- a/src/cli_args/actions.rs +++ b/src/cli_args/actions.rs @@ -17,30 +17,47 @@ fn default_scroll_direction() -> String { "down".to_string() } +fn default_ref_timeout_ms() -> u64 { + 5000 +} + #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct TypeArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg(value_name = "TEXT", allow_hyphen_values = true, help = "Text to type")] pub text: String, + #[arg( + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" + )] + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, } #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct SetValueArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg( @@ -49,32 +66,52 @@ pub(crate) struct SetValueArgs { help = "Value to set" )] pub value: String, + #[arg( + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" + )] + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, } #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct SelectArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg(value_name = "VALUE", help = "Option to select")] pub value: String, + #[arg( + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" + )] + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, } #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct ScrollArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg( @@ -87,6 +124,13 @@ pub(crate) struct ScrollArgs { #[arg(long, default_value = "3", help = "Number of scroll units")] #[serde(default = "default_scroll_amount")] pub amount: u32, + #[arg( + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" + )] + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, } #[derive(Parser, Debug, Deserialize)] @@ -112,7 +156,7 @@ pub(crate) struct PressArgs { pub(crate) struct KeyComboArgs { #[arg( value_name = "COMBO", - help = "Key or modifier to hold/release: shift, cmd, ctrl ..." + help = "Key or modifier to hold/release: shift, meta, ctrl ... (cmd is accepted)" )] pub combo: String, #[arg( @@ -134,48 +178,23 @@ pub(crate) struct HoverArgs { #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg(long, help = "Absolute coordinates as x,y; requires --headed")] pub xy: Option, - #[arg(long, help = "Hold hover position for N milliseconds")] - pub duration: Option, -} - -#[derive(Parser, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct DragCliArgs { - #[arg(long, help = "Source element ref; requires --headed")] - pub from: Option, #[arg( long, - name = "from-xy", - help = "Source coordinates as x,y; requires --headed" + help = "Reserved for compatibility; positive values are rejected in stateless mode (run hover, then `wait ` instead)" )] - pub from_xy: Option, - #[arg(long, help = "Destination element ref; requires --headed")] - pub to: Option, - #[arg( - long, - name = "to-xy", - help = "Destination coordinates as x,y; requires --headed" - )] - pub to_xy: Option, - #[arg( - long, - value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" - )] - pub snapshot: Option, - #[arg(long, help = "Drag duration in milliseconds")] pub duration: Option, #[arg( - long = "drop-delay", - value_name = "MS", - help = "Hold over the destination this many ms before releasing, so the drop target activates (macOS); default 500" + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" )] - pub drop_delay: Option, + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, } #[derive(Parser, Debug, Deserialize)] @@ -200,6 +219,13 @@ pub(crate) struct MouseClickArgs { #[arg(long, default_value = "1", help = "Number of clicks")] #[serde(default = "default_mouse_click_count")] pub count: u32, + #[arg( + long, + value_name = "MODIFIER", + help = "Held modifiers: shift, meta, ctrl, alt (repeatable; cmd is accepted)" + )] + #[serde(default)] + pub modifiers: Vec, } #[derive(Parser, Debug, Deserialize)] @@ -214,4 +240,15 @@ pub(crate) struct MousePointArgs { )] #[serde(default = "default_mouse_button")] pub button: String, + #[arg( + long, + value_name = "MODIFIER", + help = "Held modifiers: shift, meta, ctrl, alt (repeatable; cmd is accepted)" + )] + #[serde(default)] + pub modifiers: Vec, } + +#[cfg(test)] +#[path = "actions_tests.rs"] +mod tests; diff --git a/src/cli_args/actions_tests.rs b/src/cli_args/actions_tests.rs new file mode 100644 index 0000000..06d26a4 --- /dev/null +++ b/src/cli_args/actions_tests.rs @@ -0,0 +1,157 @@ +use super::*; +use crate::cli_args::drag::DragCliArgs; +use clap::CommandFactory; + +/// F2 regression: `TypeArgs`, `SetValueArgs`, `SelectArgs`, `ScrollArgs`, +/// `HoverArgs`, and `DragCliArgs` previously had no `--timeout-ms` field at +/// all, so the auto-wait budget these commands' CLI docs promise (R7) was +/// unreachable for them. Each pair below proves the paired clap/serde default +/// (5000) actually fires from both entry points: clap's `default_value_t` +/// covers the CLI, and `#[serde(default = "default_ref_timeout_ms")]` covers +/// batch JSON, since clap defaults never fire during `serde_json::from_value`. +#[test] +fn type_args_cli_omitted_timeout_defaults_to_5000() { + let args = TypeArgs::try_parse_from(["type", "@e1", "hello"]).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn type_args_cli_explicit_timeout_is_honored() { + let args = TypeArgs::try_parse_from(["type", "@e1", "hello", "--timeout-ms", "1500"]).unwrap(); + assert_eq!(args.timeout_ms, 1500); +} + +#[test] +fn type_args_batch_json_omitted_timeout_defaults_to_5000() { + let args: TypeArgs = + serde_json::from_value(serde_json::json!({ "ref_id": "@e1", "text": "hello" })).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn set_value_args_cli_omitted_timeout_defaults_to_5000() { + let args = SetValueArgs::try_parse_from(["set-value", "@e1", "value"]).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn set_value_args_batch_json_omitted_timeout_defaults_to_5000() { + let args: SetValueArgs = + serde_json::from_value(serde_json::json!({ "ref_id": "@e1", "value": "v" })).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn select_args_cli_omitted_timeout_defaults_to_5000() { + let args = SelectArgs::try_parse_from(["select", "@e1", "choice"]).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn select_args_batch_json_omitted_timeout_defaults_to_5000() { + let args: SelectArgs = + serde_json::from_value(serde_json::json!({ "ref_id": "@e1", "value": "choice" })).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn scroll_args_cli_omitted_timeout_defaults_to_5000() { + let args = ScrollArgs::try_parse_from(["scroll", "@e1"]).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn scroll_args_batch_json_omitted_timeout_defaults_to_5000() { + let args: ScrollArgs = serde_json::from_value(serde_json::json!({ "ref_id": "@e1" })).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn hover_args_cli_omitted_timeout_defaults_to_5000() { + let args = HoverArgs::try_parse_from(["hover", "@e1"]).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn hover_args_batch_json_omitted_timeout_defaults_to_5000() { + let args: HoverArgs = serde_json::from_value(serde_json::json!({ "ref_id": "@e1" })).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +#[test] +fn hover_help_discloses_stateless_duration_rejection() { + let help = HoverArgs::command().render_long_help().to_string(); + assert!(help.contains("positive values are rejected in stateless mode")); + assert!(help.contains("wait ")); +} + +#[test] +fn drag_cli_args_cli_omitted_timeout_defaults_to_5000() { + let args = DragCliArgs::try_parse_from(["drag", "--from", "@e1", "--to", "@e2"]).unwrap(); + assert_eq!(args.timeout_ms, 5000); + assert_eq!(args.target.from.as_deref(), Some("@e1")); + assert_eq!(args.target.to.as_deref(), Some("@e2")); +} + +#[test] +fn drag_cli_args_batch_json_omitted_timeout_defaults_to_5000() { + let args: DragCliArgs = + serde_json::from_value(serde_json::json!({ "from": "@e1", "to": "@e2" })).unwrap(); + assert_eq!(args.timeout_ms, 5000); +} + +/// F10 regression: `MouseClickArgs`/`MousePointArgs` (mouse-click, mouse-down, +/// mouse-up) previously had no `--modifiers` flag at all — only mouse-wheel +/// carried one — so a chorded click was unreachable from the CLI regardless +/// of what the macOS layer supported. Each pair proves the repeatable flag +/// parses on the CLI and that legacy batch JSON without the key still +/// deserializes (serde default empty), matching `MouseWheelArgs`'s existing +/// contract. +#[test] +fn mouse_click_args_cli_modifiers_repeatable_flag_parses() { + let args = MouseClickArgs::try_parse_from([ + "mouse-click", + "--xy", + "10,20", + "--modifiers", + "cmd", + "--modifiers", + "shift", + ]) + .unwrap(); + assert_eq!(args.modifiers, vec!["cmd".to_string(), "shift".to_string()]); +} + +#[test] +fn mouse_click_args_cli_omitted_modifiers_defaults_to_empty() { + let args = MouseClickArgs::try_parse_from(["mouse-click", "--xy", "10,20"]).unwrap(); + assert!(args.modifiers.is_empty()); +} + +#[test] +fn mouse_click_args_batch_json_without_modifiers_key_still_deserializes() { + let args: MouseClickArgs = + serde_json::from_value(serde_json::json!({ "xy": "10,20" })).unwrap(); + assert!(args.modifiers.is_empty()); +} + +#[test] +fn mouse_point_args_cli_modifiers_repeatable_flag_parses() { + let args = + MousePointArgs::try_parse_from(["mouse-down", "--xy", "10,20", "--modifiers", "ctrl"]) + .unwrap(); + assert_eq!(args.modifiers, vec!["ctrl".to_string()]); +} + +#[test] +fn mouse_point_args_cli_omitted_modifiers_defaults_to_empty() { + let args = MousePointArgs::try_parse_from(["mouse-up", "--xy", "10,20"]).unwrap(); + assert!(args.modifiers.is_empty()); +} + +#[test] +fn mouse_point_args_batch_json_without_modifiers_key_still_deserializes() { + let args: MousePointArgs = + serde_json::from_value(serde_json::json!({ "xy": "10,20" })).unwrap(); + assert!(args.modifiers.is_empty()); +} diff --git a/src/cli_args/batch.rs b/src/cli_args/batch.rs new file mode 100644 index 0000000..36c6015 --- /dev/null +++ b/src/cli_args/batch.rs @@ -0,0 +1,15 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +pub(crate) struct BatchArgs { + #[arg(value_name = "JSON", help = "JSON array of {command, args} objects")] + pub commands_json: String, + #[arg(long, help = "Halt the batch on the first failed command")] + pub stop_on_error: bool, + #[arg( + long, + default_value = "60000", + help = "Absolute wall-clock budget for the whole non-atomic batch" + )] + pub timeout_ms: u64, +} diff --git a/src/cli_args/drag.rs b/src/cli_args/drag.rs new file mode 100644 index 0000000..cd8cfde --- /dev/null +++ b/src/cli_args/drag.rs @@ -0,0 +1,37 @@ +use clap::Parser; +use serde::Deserialize; + +use super::drag_target::DragTargetArgs; + +fn default_ref_timeout_ms() -> u64 { + 5000 +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct DragCliArgs { + #[command(flatten)] + #[serde(flatten)] + pub target: DragTargetArgs, + #[arg( + long, + value_name = "SNAPSHOT_ID", + help = "Snapshot ID required for legacy bare @eN endpoints; omit for qualified refs" + )] + pub snapshot: Option, + #[arg(long, help = "Drag duration in milliseconds")] + pub duration: Option, + #[arg( + long = "drop-delay", + value_name = "MS", + help = "Hold over the destination this many ms before releasing, so the drop target activates (macOS); default 500" + )] + pub drop_delay: Option, + #[arg( + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" + )] + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, +} diff --git a/src/cli_args/drag_target.rs b/src/cli_args/drag_target.rs new file mode 100644 index 0000000..d2c5373 --- /dev/null +++ b/src/cli_args/drag_target.rs @@ -0,0 +1,23 @@ +use clap::Args; +use serde::Deserialize; + +#[derive(Args, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct DragTargetArgs { + #[arg(long, help = "Source element ref; requires --headed")] + pub from: Option, + #[arg( + long, + name = "from-xy", + help = "Source coordinates as x,y; requires --headed" + )] + pub from_xy: Option, + #[arg(long, help = "Destination element ref; requires --headed")] + pub to: Option, + #[arg( + long, + name = "to-xy", + help = "Destination coordinates as x,y; requires --headed" + )] + pub to_xy: Option, +} diff --git a/src/cli_args/mod.rs b/src/cli_args/mod.rs index 6999705..3a9f419 100644 --- a/src/cli_args/mod.rs +++ b/src/cli_args/mod.rs @@ -1,16 +1,20 @@ -use clap::{Parser, ValueEnum}; +use clap::{Args, Parser, ValueEnum}; use serde::Deserialize; pub(crate) mod actions; +pub(crate) mod batch; +pub(crate) mod drag; +pub(crate) mod drag_target; +pub(crate) mod mouse_wheel; pub(crate) mod notifications; pub(crate) mod session; pub(crate) mod skills; +mod snapshot; +pub(crate) mod snapshot_tree; pub(crate) mod system; pub(crate) mod trace; -fn default_max_depth() -> u8 { - 10 -} +pub(crate) use snapshot::SnapshotArgs; fn default_get_property() -> String { "text".to_string() @@ -31,11 +35,23 @@ pub(crate) enum Surface { Sheet, Popover, Alert, + Desktop, + Taskbar, + SystemTray, + QuickSettings, + NotificationCenter, + Toolbar, + Dock, + Spotlight, + MenuBarExtras, + SystemTrayOverflow, + StartMenu, + ActionCenter, } impl Surface { - pub(crate) fn to_core(&self) -> agent_desktop_core::adapter::SnapshotSurface { - use agent_desktop_core::adapter::SnapshotSurface; + pub(crate) fn to_core(&self) -> agent_desktop_core::SnapshotSurface { + use agent_desktop_core::SnapshotSurface; match self { Self::Window => SnapshotSurface::Window, Self::Focused => SnapshotSurface::Focused, @@ -44,65 +60,47 @@ impl Surface { Self::Sheet => SnapshotSurface::Sheet, Self::Popover => SnapshotSurface::Popover, Self::Alert => SnapshotSurface::Alert, + Self::Desktop => SnapshotSurface::Desktop, + Self::Taskbar => SnapshotSurface::Taskbar, + Self::SystemTray => SnapshotSurface::SystemTray, + Self::QuickSettings => SnapshotSurface::QuickSettings, + Self::NotificationCenter => SnapshotSurface::NotificationCenter, + Self::Toolbar => SnapshotSurface::Toolbar, + Self::Dock => SnapshotSurface::Dock, + Self::Spotlight => SnapshotSurface::Spotlight, + Self::MenuBarExtras => SnapshotSurface::MenuBarExtras, + Self::SystemTrayOverflow => SnapshotSurface::SystemTrayOverflow, + Self::StartMenu => SnapshotSurface::StartMenu, + Self::ActionCenter => SnapshotSurface::ActionCenter, } } } -#[derive(Parser, Debug, Deserialize)] +/// Window-targeting scope shared by the read/capture commands that choose which +/// window to operate on (`snapshot`, `find`, `screenshot`). Ref-based commands +/// (`click`/`type`/`get`/`is`) deliberately omit it — a ref already carries its +/// source window through its `RefEntry` — and keyboard input targets the focused +/// window, so neither needs an explicit window selector. +#[derive(Args, Debug, Deserialize, Default)] #[serde(deny_unknown_fields)] -pub(crate) struct SnapshotArgs { +pub(crate) struct WindowScope { #[arg(long, help = "Filter to application by name")] + #[serde(default)] pub app: Option, #[arg( long, name = "window-id", - help = "Filter to window ID (from list-windows)" + help = "Scope to a single window ID (from list-windows)" )] + #[serde(default)] pub window_id: Option, - #[arg(long, default_value = "10", help = "Maximum tree depth")] - #[serde(default = "default_max_depth")] - pub max_depth: u8, - #[arg(long, help = "Include element bounds (x, y, width, height)")] - #[serde(default)] - pub include_bounds: bool, - #[arg(long, short = 'i', help = "Include interactive elements only")] - #[serde(default)] - pub interactive_only: bool, - #[arg( - long, - help = "Collapse single-child unnamed nodes to reduce tree depth" - )] - #[serde(default)] - pub compact: bool, - #[arg( - long, - value_enum, - default_value_t = Surface::Window, - help = "Surface to snapshot" - )] - #[serde(default)] - pub surface: Surface, - #[arg( - long, - help = "Shallow overview with children_count on truncated containers" - )] - #[serde(default)] - pub skeleton: bool, - #[arg(long, help = "Start traversal from this ref instead of window root")] - pub root: Option, - #[arg( - long, - value_name = "SNAPSHOT_ID", - help = "Snapshot ID to use when resolving --root" - )] - pub snapshot: Option, } -#[derive(Parser, Debug, Deserialize)] +/// Match-criteria fields, grouped out of [`FindArgs`] to keep it under the +/// repo's field-count limit. Mirrors `core::commands::find::FindFilterArgs`. +#[derive(Args, Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct FindArgs { - #[arg(long, help = "Filter to application by name")] - pub app: Option, +pub(crate) struct FindFilterArgs { #[arg( long, help = "Match by accessibility role (button, textfield, checkbox ...)" @@ -114,6 +112,25 @@ pub(crate) struct FindArgs { pub value: Option, #[arg(long, help = "Match by text in name, value, title, or description")] pub text: Option, + #[arg(long, help = "Match by accessible description")] + pub description: Option, + #[arg(long, help = "Match by native automation id (AXIdentifier)")] + pub native_id: Option, + #[arg( + long, + help = "Require exact (case-insensitive) name/description/value matches" + )] + #[serde(default)] + pub exact: bool, +} + +/// Result-shaping fields, grouped out of [`FindArgs`] to keep it under the +/// repo's field-count limit. Mutually exclusive at the CLI layer (enforced +/// via `conflicts_with_all` on each field, unaffected by the grouping since +/// clap arg ids stay global across `#[command(flatten)]` boundaries). +#[derive(Args, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct FindSelectionArgs { #[arg( long, help = "Return match count only", @@ -149,17 +166,38 @@ pub(crate) struct FindArgs { pub limit: Option, } +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct FindArgs { + #[command(flatten)] + #[serde(flatten)] + pub scope: WindowScope, + #[command(flatten)] + #[serde(flatten)] + pub filter: FindFilterArgs, + #[arg( + long = "state", + value_name = "TOKEN[=BOOL]", + help = "Filter by state token (repeatable); append =true or =false" + )] + #[serde(default)] + pub states: Vec, + #[command(flatten)] + #[serde(flatten)] + pub selection: FindSelectionArgs, +} + #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct ScreenshotArgs { - #[arg(long, help = "Filter to application by name")] - pub app: Option, + #[command(flatten)] + #[serde(flatten)] + pub scope: WindowScope, #[arg( long, - name = "window-id", - help = "Filter to window ID (from list-windows)" + help = "Capture display by index (from list-displays; 0 = primary)" )] - pub window_id: Option, + pub screen: Option, #[arg(value_name = "PATH", help = "Save to file instead of returning base64")] pub output_path: Option, } @@ -167,12 +205,15 @@ pub(crate) struct ScreenshotArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct GetArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg( @@ -187,12 +228,15 @@ pub(crate) struct GetArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct IsArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg( @@ -207,15 +251,29 @@ pub(crate) struct IsArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RefArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + #[arg( + value_name = "REF", + help = "Qualified ref from snapshot (@:eN), or legacy @eN with --snapshot" + )] pub ref_id: String, #[arg( long = "snapshot", value_name = "SNAPSHOT_ID", - help = "Snapshot ID returned by snapshot; omit to use active session latest" + help = "Snapshot ID required for a legacy bare @eN ref; omit for a qualified ref" )] #[serde(rename = "snapshot", alias = "snapshot_id")] pub snapshot_id: Option, + #[arg( + long = "timeout-ms", + default_value_t = 5000, + help = "Maximum ref-resolution and transient-actionability wait in milliseconds; terminal failures return immediately" + )] + #[serde(default = "default_ref_timeout_ms")] + pub timeout_ms: u64, +} + +fn default_ref_timeout_ms() -> u64 { + 5000 } #[derive(Parser, Debug, Deserialize)] @@ -224,3 +282,7 @@ pub(crate) struct ListSurfacesArgs { #[arg(long, help = "Filter to application by name")] pub app: Option, } + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod mod_tests; diff --git a/src/cli_args/mod_tests.rs b/src/cli_args/mod_tests.rs new file mode 100644 index 0000000..a0e589d --- /dev/null +++ b/src/cli_args/mod_tests.rs @@ -0,0 +1,240 @@ +use super::*; +use agent_desktop_core::SnapshotSurface; +use clap::ValueEnum; + +/// The exhaustive set of `SnapshotSurface` values the CLI's `Surface` enum is +/// meant to expose. `SnapshotSurface` is `#[non_exhaustive]`, so a new variant +/// added to it will not force `Surface::to_core`'s match arms (which match on +/// the local `Surface`, not on `SnapshotSurface`) to change — this list is the +/// independent tripwire that must be updated by hand alongside any new +/// `Surface`/`to_core` arm. +const EXPECTED_CORE_SURFACES: &[SnapshotSurface] = &[ + SnapshotSurface::Window, + SnapshotSurface::Focused, + SnapshotSurface::Menu, + SnapshotSurface::Menubar, + SnapshotSurface::Sheet, + SnapshotSurface::Popover, + SnapshotSurface::Alert, + SnapshotSurface::Desktop, + SnapshotSurface::Taskbar, + SnapshotSurface::SystemTray, + SnapshotSurface::QuickSettings, + SnapshotSurface::NotificationCenter, + SnapshotSurface::Toolbar, + SnapshotSurface::Dock, + SnapshotSurface::Spotlight, + SnapshotSurface::MenuBarExtras, + SnapshotSurface::SystemTrayOverflow, + SnapshotSurface::StartMenu, + SnapshotSurface::ActionCenter, +]; + +/// Keeps the CLI `Surface` enum and the hand-maintained `EXPECTED_CORE_SURFACES` +/// tripwire in one-to-one correspondence: every `Surface` variant (enumerated +/// via its `ValueEnum` derive, so this side stays exhaustive automatically) maps +/// through `.to_core()` to a distinct `SnapshotSurface`, the tripwire lists each +/// of those exactly once, and the two sets match. An inconsistent hand-edit — a +/// `to_core` arm that collides with another, or the tripwire updated without a +/// matching `Surface`/`to_core` arm (or the reverse) — fails here. The one gap it +/// cannot close: because `SnapshotSurface` is `#[non_exhaustive]` with no +/// reflection, a new core variant added with no CLI-side edit keeps both sides +/// their current size and passes — the by-hand tripwire update is the only +/// checkpoint for that case (see `EXPECTED_CORE_SURFACES`). +#[test] +fn surface_to_core_maps_one_to_one_onto_expected_core_surfaces() { + let cli_variants = Surface::value_variants(); + assert_eq!( + cli_variants.len(), + EXPECTED_CORE_SURFACES.len(), + "Surface has {} variants but EXPECTED_CORE_SURFACES has {} — \ + update EXPECTED_CORE_SURFACES (and Surface::to_core) to match", + cli_variants.len(), + EXPECTED_CORE_SURFACES.len() + ); + + for (i, a) in EXPECTED_CORE_SURFACES.iter().enumerate() { + for (j, b) in EXPECTED_CORE_SURFACES.iter().enumerate() { + assert!( + i == j || a != b, + "EXPECTED_CORE_SURFACES lists SnapshotSurface::{a:?} more than once — \ + each expected core surface must appear exactly once" + ); + } + } + + let mapped: Vec = cli_variants.iter().map(Surface::to_core).collect(); + for (i, a) in mapped.iter().enumerate() { + for (j, b) in mapped.iter().enumerate() { + assert!( + i == j || a != b, + "Surface::{:?} and Surface::{:?} both map to the same \ + SnapshotSurface::{a:?} via to_core — mapping must be distinct", + cli_variants[i], + cli_variants[j] + ); + } + } + + for expected in EXPECTED_CORE_SURFACES { + assert!( + mapped.contains(expected), + "no Surface variant maps to SnapshotSurface::{expected:?} via to_core — \ + a Surface arm is missing for this core surface" + ); + } +} + +/// God-object regression: `FindArgs` used to carry 14 flat fields; the +/// match-criteria and result-shaping groups now live in +/// `FindFilterArgs`/`FindSelectionArgs`, flattened back on. Proves the CLI +/// surface (flag names, `conflicts_with_all`) is unchanged by the +/// regrouping — a missing `#[command(flatten)]` would make clap reject these +/// flags as unrecognized. +#[test] +fn find_args_cli_flags_still_resolve_through_flattened_groups() { + let args = FindArgs::try_parse_from([ + "find", "--role", "button", "--name", "Save", "--exact", "--first", + ]) + .unwrap(); + + assert_eq!(args.filter.role.as_deref(), Some("button")); + assert_eq!(args.filter.name.as_deref(), Some("Save")); + assert!(args.filter.exact); + assert!(args.selection.first); +} + +#[test] +fn find_args_selection_conflicts_still_enforced_across_the_flatten_boundary() { + let err = FindArgs::try_parse_from(["find", "--first", "--last"]).unwrap_err(); + + assert_eq!(err.kind(), clap::error::ErrorKind::ArgumentConflict); +} + +/// Batch JSON is decoded straight into `FindArgs` via serde; a flat payload +/// (the shape every existing caller sends) must still deserialize now that +/// the Rust type nests `filter`/`selection`. +#[test] +fn find_args_batch_json_flat_payload_deserializes_into_nested_groups() { + let args: FindArgs = serde_json::from_value(serde_json::json!({ + "role": "checkbox", + "native_id": "agree", + "count": true + })) + .unwrap(); + + assert_eq!(args.filter.role.as_deref(), Some("checkbox")); + assert_eq!(args.filter.native_id.as_deref(), Some("agree")); + assert!(args.selection.count); + assert!(args.states.is_empty()); +} + +/// `WindowScope` (`app`/`window_id`) is shared across `FindArgs`, +/// `SnapshotArgs`, and `ScreenshotArgs` via `#[serde(flatten)]`, each also +/// carrying `#[serde(deny_unknown_fields)]` (`F6`). A flat batch payload +/// (the shape every existing caller sends) must still deserialize into +/// `.scope.app`/`.scope.window_id` now that the selector lives on its own +/// struct instead of directly on `FindArgs`. +#[test] +fn find_args_batch_json_flat_scope_deserializes_into_scope_group() { + let args: FindArgs = serde_json::from_value(serde_json::json!({ + "app": "Finder", + "window_id": "w-2" + })) + .unwrap(); + + assert_eq!(args.scope.app.as_deref(), Some("Finder")); + assert_eq!(args.scope.window_id.as_deref(), Some("w-2")); + assert!(args.filter.role.is_none()); +} + +/// `WindowScope`'s `deny_unknown_fields` must still reject a typo'd scope +/// field on `FindArgs` instead of silently dropping the window filter — a +/// mistyped `app` (`ap`) or `window_id` (`windo_id`) should fail loudly +/// rather than deserialize into a scopeless, all-windows match. +#[test] +fn find_args_batch_json_rejects_scope_field_typo() { + let err = serde_json::from_value::(serde_json::json!({ + "ap": "Finder", + "window_id": "w-2" + })) + .unwrap_err(); + assert!(err.to_string().contains("`ap`")); + + let err = serde_json::from_value::(serde_json::json!({ + "app": "Finder", + "windo_id": "w-2" + })) + .unwrap_err(); + assert!(err.to_string().contains("`windo_id`")); +} + +/// Same `WindowScope` flatten, pinned for `SnapshotArgs` (`F6`): a flat +/// `app`/`window_id` payload must still populate `.scope.*` alongside +/// `SnapshotArgs`'s own defaulted fields. +#[test] +fn snapshot_args_batch_json_flat_scope_deserializes_into_scope_group() { + let args: SnapshotArgs = serde_json::from_value(serde_json::json!({ + "app": "Finder", + "window_id": "w-2" + })) + .unwrap(); + + assert_eq!(args.scope.app.as_deref(), Some("Finder")); + assert_eq!(args.scope.window_id.as_deref(), Some("w-2")); + assert_eq!(args.tree.max_depth, 10); +} + +/// `SnapshotArgs` inherits `WindowScope`'s `deny_unknown_fields`; a typo'd +/// scope field must still be rejected rather than silently ignored. +#[test] +fn snapshot_args_batch_json_rejects_scope_field_typo() { + let err = serde_json::from_value::(serde_json::json!({ + "ap": "Finder", + "window_id": "w-2" + })) + .unwrap_err(); + assert!(err.to_string().contains("`ap`")); + + let err = serde_json::from_value::(serde_json::json!({ + "app": "Finder", + "windo_id": "w-2" + })) + .unwrap_err(); + assert!(err.to_string().contains("`windo_id`")); +} + +/// Same `WindowScope` flatten, pinned for `ScreenshotArgs` (`F6`): a flat +/// `app`/`window_id` payload must still populate `.scope.*` alongside +/// `ScreenshotArgs`'s own optional fields. +#[test] +fn screenshot_args_batch_json_flat_scope_deserializes_into_scope_group() { + let args: ScreenshotArgs = serde_json::from_value(serde_json::json!({ + "app": "Finder", + "window_id": "w-2" + })) + .unwrap(); + + assert_eq!(args.scope.app.as_deref(), Some("Finder")); + assert_eq!(args.scope.window_id.as_deref(), Some("w-2")); + assert!(args.screen.is_none()); +} + +/// `ScreenshotArgs` inherits `WindowScope`'s `deny_unknown_fields`; a typo'd +/// scope field must still be rejected rather than silently ignored. +#[test] +fn screenshot_args_batch_json_rejects_scope_field_typo() { + let err = serde_json::from_value::(serde_json::json!({ + "ap": "Finder", + "window_id": "w-2" + })) + .unwrap_err(); + assert!(err.to_string().contains("`ap`")); + + let err = serde_json::from_value::(serde_json::json!({ + "app": "Finder", + "windo_id": "w-2" + })) + .unwrap_err(); + assert!(err.to_string().contains("`windo_id`")); +} diff --git a/src/cli_args/mouse_wheel.rs b/src/cli_args/mouse_wheel.rs new file mode 100644 index 0000000..9492195 --- /dev/null +++ b/src/cli_args/mouse_wheel.rs @@ -0,0 +1,36 @@ +use clap::Parser; +use serde::Deserialize; + +fn default_wheel_delta() -> f64 { + -3.0 +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct MouseWheelArgs { + #[arg(long, help = "Absolute X coordinate for wheel event")] + pub x: f64, + #[arg(long, help = "Absolute Y coordinate for wheel event")] + pub y: f64, + #[arg( + long, + default_value = "-3", + help = "Vertical wheel lines; positive scrolls up, negative scrolls down" + )] + #[serde(default = "default_wheel_delta")] + pub dy: f64, + #[arg( + long, + default_value = "0", + help = "Horizontal wheel lines; positive scrolls left, negative scrolls right" + )] + #[serde(default)] + pub dx: f64, + #[arg( + long, + value_name = "MODIFIER", + help = "Held modifiers: shift, meta, ctrl, alt (repeatable; cmd is accepted)" + )] + #[serde(default)] + pub modifiers: Vec, +} diff --git a/src/cli_args/notifications.rs b/src/cli_args/notifications.rs index 06d1503..759cc5c 100644 --- a/src/cli_args/notifications.rs +++ b/src/cli_args/notifications.rs @@ -20,6 +20,18 @@ pub(crate) struct DismissNotificationCliArgs { pub index: u64, #[arg(long, help = "Filter notifications by app before selecting index")] pub app: Option, + #[arg( + long, + help = "Expected app name at INDEX; one expected identity field is required", + value_name = "APP" + )] + pub expected_app: Option, + #[arg( + long, + help = "Expected title at INDEX; one expected identity field is required", + value_name = "TITLE" + )] + pub expected_title: Option, } #[derive(Parser, Debug, Deserialize)] @@ -42,13 +54,13 @@ pub(crate) struct NotificationActionCliArgs { pub action: String, #[arg( long, - help = "Expected app name of the notification at INDEX. If set and NC reordered, the action is refused with NOTIFICATION_NOT_FOUND instead of pressing the wrong notification.", + help = "Expected app name at INDEX; one expected identity field is required", value_name = "APP" )] pub expected_app: Option, #[arg( long, - help = "Expected title of the notification at INDEX. Same reorder-safety behavior as --expected-app.", + help = "Expected title at INDEX; one expected identity field is required", value_name = "TITLE" )] pub expected_title: Option, diff --git a/src/cli_args/session.rs b/src/cli_args/session.rs index e6f157c..ae9aeed 100644 --- a/src/cli_args/session.rs +++ b/src/cli_args/session.rs @@ -8,9 +8,9 @@ pub(crate) struct SessionArgs { #[derive(Subcommand, Debug)] pub(crate) enum SessionAction { - #[command(about = "Create a session directory, manifest, and current-session pointer")] + #[command(about = "Create a session directory and manifest")] Start(SessionStartArgs), - #[command(about = "Seal the session manifest and clear the current-session pointer")] + #[command(about = "Seal a session manifest")] End(SessionEndArgs), #[command(about = "List session manifests")] List, @@ -29,16 +29,11 @@ pub(crate) struct SessionStartArgs { help = "Capture pre/post-action screenshots and refmap copies (requires tracing; sensitive)" )] pub screenshots: bool, - #[arg( - long, - help = "Override the current-session pointer even when it references a live session" - )] - pub force: bool, } #[derive(Args, Debug)] pub(crate) struct SessionEndArgs { - #[arg(help = "Session id to end; defaults to the current-session pointer")] + #[arg(help = "Session id to end; defaults to global --session or AGENT_DESKTOP_SESSION")] pub id: Option, } diff --git a/src/cli_args/snapshot.rs b/src/cli_args/snapshot.rs new file mode 100644 index 0000000..ce8dbcc --- /dev/null +++ b/src/cli_args/snapshot.rs @@ -0,0 +1,31 @@ +use clap::Parser; +use serde::Deserialize; + +use super::{Surface, WindowScope, snapshot_tree::SnapshotTreeArgs}; + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SnapshotArgs { + #[command(flatten)] + #[serde(flatten)] + pub scope: WindowScope, + #[command(flatten)] + #[serde(flatten)] + pub tree: SnapshotTreeArgs, + #[arg( + long, + value_enum, + default_value_t = Surface::Window, + help = "Surface to snapshot" + )] + #[serde(default)] + pub surface: Surface, + #[arg(long, help = "Start traversal from this ref instead of window root")] + pub root: Option, + #[arg( + long, + value_name = "SNAPSHOT_ID", + help = "Snapshot ID to use when resolving --root" + )] + pub snapshot: Option, +} diff --git a/src/cli_args/snapshot_tree.rs b/src/cli_args/snapshot_tree.rs new file mode 100644 index 0000000..85cb914 --- /dev/null +++ b/src/cli_args/snapshot_tree.rs @@ -0,0 +1,32 @@ +use clap::Args; +use serde::Deserialize; + +fn default_max_depth() -> u8 { + 10 +} + +#[derive(Args, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SnapshotTreeArgs { + #[arg(long, default_value = "10", help = "Maximum tree depth")] + #[serde(default = "default_max_depth")] + pub max_depth: u8, + #[arg(long, help = "Include element bounds (x, y, width, height)")] + #[serde(default)] + pub include_bounds: bool, + #[arg(long, short = 'i', help = "Include interactive elements only")] + #[serde(default)] + pub interactive_only: bool, + #[arg( + long, + help = "Collapse single-child unnamed nodes to reduce tree depth" + )] + #[serde(default)] + pub compact: bool, + #[arg( + long, + help = "Shallow overview with children_count on truncated containers" + )] + #[serde(default)] + pub skeleton: bool, +} diff --git a/src/cli_args/system.rs b/src/cli_args/system.rs index d6e738b..dd512b9 100644 --- a/src/cli_args/system.rs +++ b/src/cli_args/system.rs @@ -1,6 +1,8 @@ use clap::{Args, Parser}; use serde::Deserialize; +use super::WindowScope; + fn default_launch_timeout() -> u64 { 30000 } @@ -21,6 +23,27 @@ pub(crate) struct LaunchArgs { )] #[serde(default = "default_launch_timeout")] pub timeout: u64, + #[arg( + long = "arg", + help = "Command-line argument passed to the launched app" + )] + #[serde(default)] + pub args: Vec, + #[arg( + long = "env", + value_name = "KEY=VALUE", + help = "Environment variable for the launched process" + )] + #[serde(default)] + pub env: Vec, + #[arg(long, help = "Working directory for the launched process")] + pub cwd: Option, + #[arg( + long = "no-attach", + help = "Require a fresh instance; fail if the app is already running" + )] + #[serde(default)] + pub no_attach: bool, } #[derive(Parser, Debug, Deserialize)] @@ -64,8 +87,9 @@ pub(crate) struct FocusWindowArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct ResizeWindowCliArgs { - #[arg(long, help = "Application name")] - pub app: Option, + #[command(flatten)] + #[serde(flatten)] + pub scope: WindowScope, #[arg(long, help = "New window width in pixels")] pub width: f64, #[arg(long, help = "New window height in pixels")] @@ -75,8 +99,9 @@ pub(crate) struct ResizeWindowCliArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct MoveWindowCliArgs { - #[arg(long, help = "Application name")] - pub app: Option, + #[command(flatten)] + #[serde(flatten)] + pub scope: WindowScope, #[arg(long, help = "New window X position")] pub x: f64, #[arg(long, help = "New window Y position")] @@ -86,17 +111,61 @@ pub(crate) struct MoveWindowCliArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct AppRefArgs { - #[arg(long, help = "Application name")] - pub app: Option, + #[command(flatten)] + #[serde(flatten)] + pub scope: WindowScope, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ClipboardGetArgs { + #[arg( + long, + value_name = "FORMAT", + help = "Clipboard format to read: text (default), auto, image, file-urls" + )] + pub format: Option, + #[arg( + long, + value_name = "PATH", + help = "Where to write image content; defaults to a private temp file under the session dir" + )] + pub out: Option, } #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct ClipboardSetArgs { - #[arg(value_name = "TEXT", help = "Text to write to the clipboard")] - pub text: String, + #[arg( + value_name = "TEXT", + help = "Text to write to the clipboard (ignored if --image or --file-url is given)" + )] + pub text: Option, + #[arg( + long, + value_name = "PATH", + help = "Path to a PNG file to write to the clipboard" + )] + pub image: Option, + #[arg( + long = "file-url", + value_name = "PATH", + help = "File path to write to the clipboard as a file reference; repeatable" + )] + #[serde(default)] + pub file_url: Vec, } +/// `event`/`window_id` flatten in as a sibling of `mode`/`predicate` here +/// (not nested inside [`WaitModeArgs`]) even though they are conceptually +/// part of the wait mode: serde does not support `#[serde(deny_unknown_fields)]` +/// on a struct that is both a flatten *target* and a flatten *owner*, and +/// every struct in this file needs `deny_unknown_fields` to keep rejecting +/// typoed batch-JSON keys. Keeping the flatten nesting at exactly one level +/// (three flatten fields side by side, mirroring the existing `mode`/ +/// `predicate` split) sidesteps that limitation; the CLI surface is +/// unaffected since `#[command(flatten)]` merges flags regardless of Rust +/// struct nesting depth. #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct WaitArgs { @@ -105,6 +174,9 @@ pub(crate) struct WaitArgs { pub mode: WaitModeArgs, #[command(flatten)] #[serde(flatten)] + pub event: WaitEventArgs, + #[command(flatten)] + #[serde(flatten)] pub predicate: WaitPredicateArgs, #[arg( long, @@ -117,6 +189,25 @@ pub(crate) struct WaitArgs { pub app: Option, } +/// The `--event`/`--window-id` pair, grouped out of [`WaitModeArgs`] to keep +/// it under the repo's field-count limit. `window_id` only ever narrows an +/// `--event` wait, so the two travel together. +#[derive(Args, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct WaitEventArgs { + #[arg( + long, + help = "Block until a desktop lifecycle signal, detected by baseline diff without needing to know the id/title up front: window-opened, window-closed, app-launched, app-terminated, focus-changed, surface-appeared, surface-dismissed" + )] + pub event: Option, + #[arg( + long, + name = "window-id", + help = "Optional: narrow --event window-opened/window-closed/focus-changed to one window ID" + )] + pub window_id: Option, +} + #[derive(Args, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct WaitModeArgs { @@ -124,7 +215,10 @@ pub(crate) struct WaitModeArgs { pub ms: Option, #[arg(long, help = "Block until this element ref appears in the tree")] pub element: Option, - #[arg(long, help = "Block until a window with this title appears")] + #[arg( + long, + help = "Block until a window with this title appears; with --event, narrows the event wait to that window title instead of selecting a mode" + )] pub window: Option, #[arg( long, @@ -148,7 +242,7 @@ pub(crate) struct WaitPredicateArgs { #[arg( long, value_name = "SNAPSHOT_ID", - help = "Snapshot ID for --element waits; omit to use active session latest" + help = "Snapshot ID required when --element is a legacy bare @eN ref; omit for a qualified ref" )] pub snapshot: Option, #[arg( @@ -180,15 +274,14 @@ pub(crate) struct WaitPredicateArgs { #[derive(Parser, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct PermissionsArgs { - #[arg(long, help = "Trigger the system accessibility permission dialog")] + #[arg( + long, + help = "Request missing permissions in the bounded isolated helper" + )] #[serde(default)] pub request: bool, } -#[derive(Parser, Debug)] -pub(crate) struct BatchArgs { - #[arg(value_name = "JSON", help = "JSON array of {command, args} objects")] - pub commands_json: String, - #[arg(long, help = "Halt the batch on the first failed command")] - pub stop_on_error: bool, -} +#[cfg(test)] +#[path = "system_tests.rs"] +mod tests; diff --git a/src/cli_args/system_tests.rs b/src/cli_args/system_tests.rs new file mode 100644 index 0000000..0652b86 --- /dev/null +++ b/src/cli_args/system_tests.rs @@ -0,0 +1,80 @@ +use super::*; + +/// God-object regression: `WaitModeArgs` used to carry 9 flat fields; +/// `event`/`window_id` now live in `WaitEventArgs`, flattened onto +/// `WaitArgs` as a sibling of `mode`/`predicate` (nesting it inside +/// `WaitModeArgs` instead hits a real serde limitation — see the doc +/// comment on `WaitArgs`). Proves the CLI surface is unchanged by the +/// regrouping — a missing `#[command(flatten)]` would make clap reject +/// `--event`/`--window-id` as unrecognized. +#[test] +fn wait_event_and_window_id_still_resolve_through_the_flattened_group() { + let args = + WaitArgs::try_parse_from(["wait", "--event", "window-opened", "--window-id", "w-42"]) + .unwrap(); + + assert_eq!(args.event.event.as_deref(), Some("window-opened")); + assert_eq!(args.event.window_id.as_deref(), Some("w-42")); +} + +/// Batch JSON is decoded straight into `WaitArgs` via serde; a flat payload +/// (the shape every existing caller sends) must still deserialize now that +/// `event`/`window_id` live on the separate `WaitEventArgs` group. +#[test] +fn wait_args_batch_json_flat_event_payload_deserializes_into_flattened_group() { + let args: WaitArgs = serde_json::from_value(serde_json::json!({ + "event": "app-launched", + "window_id": "w-7" + })) + .unwrap(); + + assert_eq!(args.event.event.as_deref(), Some("app-launched")); + assert_eq!(args.event.window_id.as_deref(), Some("w-7")); + assert!(args.mode.element.is_none()); + assert_eq!(args.timeout, 30000); +} + +/// The three-way sibling flatten (`mode`/`event`/`predicate`) must still +/// reject a genuinely unrecognized batch-JSON key — the exact property +/// `rejects_unknown_wait_batch_args_after_flattening` in `batch::tests` +/// already guards for the two-flatten-field shape; this extends it to the +/// three-field shape introduced by this split. +#[test] +fn wait_args_batch_json_rejects_unknown_field_across_three_flattened_groups() { + let err = serde_json::from_value::(serde_json::json!({ + "ms": 1, + "totally_bogus": true + })) + .unwrap_err(); + + assert!(err.to_string().contains("totally_bogus")); +} + +#[test] +fn window_mutation_args_accept_window_ids() { + let resize = ResizeWindowCliArgs::try_parse_from([ + "resize-window", + "--window-id", + "w-42", + "--width", + "800", + "--height", + "600", + ]) + .unwrap(); + let move_window = MoveWindowCliArgs::try_parse_from([ + "move-window", + "--window-id", + "w-42", + "--x", + "10", + "--y", + "20", + ]) + .unwrap(); + let minimize = AppRefArgs::try_parse_from(["minimize", "--window-id", "w-42"]).unwrap(); + + assert_eq!(resize.scope.window_id.as_deref(), Some("w-42")); + assert_eq!(move_window.scope.window_id.as_deref(), Some("w-42")); + assert_eq!(minimize.scope.window_id.as_deref(), Some("w-42")); +} diff --git a/src/command_policy/mod.rs b/src/command_policy/mod.rs index 9f5ea62..0aa22c2 100644 --- a/src/command_policy/mod.rs +++ b/src/command_policy/mod.rs @@ -1,7 +1,5 @@ use agent_desktop_core::{ - PermissionReport, - error::{AdapterError, AppError, ErrorCode}, - refs::validate_ref_id, + AdapterError, AppError, DeliverySemantics, ErrorCode, PermissionReport, refs::validate_ref_id, }; use crate::cli::Commands; @@ -19,8 +17,8 @@ pub(crate) fn policy_for(cmd: &Commands) -> PermissionNeed { match cmd { Commands::Version | Commands::Skills(_) | Commands::Session(_) | Commands::Trace(_) => None, Commands::Status | Commands::Permissions(_) => None, - Commands::ListWindows(_) | Commands::ListApps(_) => None, - Commands::ClipboardGet | Commands::ClipboardSet(_) | Commands::ClipboardClear => None, + Commands::ListWindows(_) | Commands::ListDisplays | Commands::ListApps(_) => None, + Commands::ClipboardGet(_) | Commands::ClipboardSet(_) | Commands::ClipboardClear => None, Commands::Batch(_) => None, Commands::Snapshot(_) @@ -29,7 +27,7 @@ pub(crate) fn policy_for(cmd: &Commands) -> PermissionNeed { | Commands::Wait(_) | Commands::ListNotifications(_) => Accessibility, - Commands::Screenshot(a) if a.app.is_some() || a.window_id.is_some() => { + Commands::Screenshot(a) if a.scope.app.is_some() || a.scope.window_id.is_some() => { AccessibilityAndScreenRecording } Commands::Screenshot(_) => ScreenRecording, @@ -59,7 +57,8 @@ pub(crate) fn policy_for(cmd: &Commands) -> PermissionNeed { | Commands::MouseMove(_) | Commands::MouseClick(_) | Commands::MouseDown(_) - | Commands::MouseUp(_) => Accessibility, + | Commands::MouseUp(_) + | Commands::MouseWheel(_) => Accessibility, Commands::Launch(_) | Commands::CloseApp(_) @@ -87,7 +86,8 @@ pub(crate) fn preflight(cmd: &Commands, report: &PermissionReport) -> Result<(), report .accessibility_suggestion() .unwrap_or("Grant Accessibility permission and retry"), - ); + ) + .with_disposition(DeliverySemantics::not_delivered()); return Err(AppError::Adapter(err)); } if requires_screen_recording(permission) && report.screen_recording_denied() { @@ -99,12 +99,21 @@ pub(crate) fn preflight(cmd: &Commands, report: &PermissionReport) -> Result<(), report .screen_recording_suggestion() .unwrap_or("Grant Screen Recording permission and retry"), - ); + ) + .with_disposition(DeliverySemantics::not_delivered()); return Err(AppError::Adapter(err)); } Ok(()) } +pub(crate) fn requires_permission_report(cmd: &Commands) -> bool { + policy_for(cmd) != PermissionNeed::None + || matches!( + cmd, + Commands::Status | Commands::Permissions(_) | Commands::Batch(_) + ) +} + fn validate_args(cmd: &Commands) -> Result<(), AppError> { match cmd { Commands::Snapshot(args) => { @@ -149,10 +158,10 @@ fn validate_args(cmd: &Commands) -> Result<(), AppError> { } } Commands::Drag(args) => { - if let Some(ref_id) = &args.from { + if let Some(ref_id) = &args.target.from { validate_ref_id(ref_id)?; } - if let Some(ref_id) = &args.to { + if let Some(ref_id) = &args.target.to { validate_ref_id(ref_id)?; } } @@ -161,6 +170,22 @@ fn validate_args(cmd: &Commands) -> Result<(), AppError> { validate_ref_id(ref_id)?; } } + Commands::DismissNotification(args) + if args.expected_app.as_deref().is_none_or(str::is_empty) + && args.expected_title.as_deref().is_none_or(str::is_empty) => + { + return Err(AppError::invalid_input( + "dismiss-notification requires --expected-app or --expected-title", + )); + } + Commands::NotificationAction(args) + if args.expected_app.as_deref().is_none_or(str::is_empty) + && args.expected_title.as_deref().is_none_or(str::is_empty) => + { + return Err(AppError::invalid_input( + "notification-action requires --expected-app or --expected-title", + )); + } Commands::Find(_) | Commands::Screenshot(_) | Commands::Press(_) @@ -170,9 +195,11 @@ fn validate_args(cmd: &Commands) -> Result<(), AppError> { | Commands::MouseClick(_) | Commands::MouseDown(_) | Commands::MouseUp(_) + | Commands::MouseWheel(_) | Commands::Launch(_) | Commands::CloseApp(_) | Commands::ListWindows(_) + | Commands::ListDisplays | Commands::ListApps(_) | Commands::FocusWindow(_) | Commands::ResizeWindow(_) @@ -185,7 +212,7 @@ fn validate_args(cmd: &Commands) -> Result<(), AppError> { | Commands::DismissNotification(_) | Commands::DismissAllNotifications(_) | Commands::NotificationAction(_) - | Commands::ClipboardGet + | Commands::ClipboardGet(_) | Commands::ClipboardSet(_) | Commands::ClipboardClear | Commands::Status diff --git a/src/command_policy/tests.rs b/src/command_policy/tests.rs index 7c7204a..29a5a4f 100644 --- a/src/command_policy/tests.rs +++ b/src/command_policy/tests.rs @@ -1,90 +1,36 @@ use super::*; -use crate::cli::{Cli, Commands}; -use crate::cli_args::{RefArgs, ScreenshotArgs, SnapshotArgs}; +use crate::cli::Commands; +use crate::cli_args::{ + RefArgs, ScreenshotArgs, SnapshotArgs, WindowScope, snapshot_tree::SnapshotTreeArgs, +}; use agent_desktop_core::{PermissionReport, PermissionState}; -use clap::CommandFactory; const VALID_REF_ID: &str = "@e1"; #[test] -fn every_cli_subcommand_has_policy() { - for subcommand in Cli::command().get_subcommands() { - let name = subcommand.get_name(); - assert!( - command_name_is_covered(name), - "missing permission policy coverage for {name}" - ); - } -} +fn permission_report_is_collected_only_for_permission_consumers() { + let status = Commands::Status; + let list_displays = Commands::ListDisplays; + let batch = Commands::Batch(crate::cli_args::batch::BatchArgs { + commands_json: "[]".into(), + stop_on_error: false, + timeout_ms: 1, + }); -fn command_name_is_covered(name: &str) -> bool { - matches!( - name, - "snapshot" - | "find" - | "screenshot" - | "get" - | "is" - | "click" - | "double-click" - | "triple-click" - | "right-click" - | "type" - | "set-value" - | "clear" - | "focus" - | "select" - | "toggle" - | "check" - | "uncheck" - | "expand" - | "collapse" - | "scroll" - | "scroll-to" - | "press" - | "key-down" - | "key-up" - | "hover" - | "drag" - | "mouse-move" - | "mouse-click" - | "mouse-down" - | "mouse-up" - | "launch" - | "close-app" - | "list-windows" - | "list-apps" - | "focus-window" - | "resize-window" - | "move-window" - | "minimize" - | "maximize" - | "restore" - | "list-surfaces" - | "list-notifications" - | "dismiss-notification" - | "dismiss-all-notifications" - | "notification-action" - | "clipboard-get" - | "clipboard-set" - | "clipboard-clear" - | "wait" - | "status" - | "permissions" - | "version" - | "batch" - | "skills" - | "session" - | "trace" - ) + assert!(requires_permission_report(&status)); + assert!(requires_permission_report(&batch)); + assert!(!requires_permission_report(&list_displays)); } #[test] fn unknown_permission_does_not_mask_platform_errors() { let report = PermissionReport::default(); let command = Commands::Screenshot(ScreenshotArgs { - app: None, - window_id: None, + scope: WindowScope { + app: None, + window_id: None, + }, + screen: None, output_path: None, }); @@ -101,8 +47,11 @@ fn screen_recording_denial_is_preflighted() { automation: PermissionState::NotRequired, }; let command = Commands::Screenshot(ScreenshotArgs { - app: None, - window_id: None, + scope: WindowScope { + app: None, + window_id: None, + }, + screen: None, output_path: None, }); @@ -123,6 +72,7 @@ fn accessibility_denial_is_preflighted_for_ax_commands() { let command = Commands::Click(crate::cli_args::RefArgs { ref_id: VALID_REF_ID.into(), snapshot_id: None, + timeout_ms: 5000, }); let err = preflight(&command, &report).expect_err("denied accessibility fails"); @@ -143,6 +93,7 @@ fn invalid_ref_args_are_rejected_before_permission_preflight() { let command = Commands::Click(RefArgs { ref_id: "bad-ref".into(), snapshot_id: None, + timeout_ms: 5000, }); let err = preflight(&command, &report).expect_err("invalid ref fails first"); @@ -160,14 +111,18 @@ fn invalid_snapshot_root_is_rejected_before_permission_preflight() { automation: PermissionState::NotRequired, }; let command = Commands::Snapshot(SnapshotArgs { - app: None, - window_id: None, - max_depth: 10, - include_bounds: false, - interactive_only: false, - compact: false, + scope: WindowScope { + app: None, + window_id: None, + }, + tree: SnapshotTreeArgs { + max_depth: 10, + include_bounds: false, + interactive_only: false, + compact: false, + skeleton: false, + }, surface: crate::cli_args::Surface::Window, - skeleton: false, root: Some("bad-root".into()), snapshot: None, }); @@ -177,6 +132,54 @@ fn invalid_snapshot_root_is_rejected_before_permission_preflight() { assert_eq!(err.code(), "INVALID_ARGS"); } +#[test] +fn notification_identity_is_required_before_permission_preflight() { + let report = PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "grant accessibility".into(), + }, + screen_recording: PermissionState::Granted, + automation: PermissionState::NotRequired, + }; + let command = + Commands::NotificationAction(crate::cli_args::notifications::NotificationActionCliArgs { + index: 1, + action: "Reply".into(), + expected_app: None, + expected_title: None, + }); + + let error = preflight(&command, &report).unwrap_err(); + + assert_eq!(error.code(), "INVALID_ARGS"); +} + +#[test] +fn dismiss_notification_identity_is_required_before_permission_preflight() { + let report = PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "grant accessibility".into(), + }, + screen_recording: PermissionState::Granted, + automation: PermissionState::NotRequired, + }; + for (expected_app, expected_title) in [(None, None), (Some(String::new()), None)] { + let command = Commands::DismissNotification( + crate::cli_args::notifications::DismissNotificationCliArgs { + index: 1, + app: None, + expected_app, + expected_title, + }, + ); + + let error = preflight(&command, &report).expect_err("identity fails before permission"); + + assert_eq!(error.code(), "INVALID_ARGS"); + assert!(error.to_string().contains("--expected-app")); + } +} + #[test] fn trace_show_passes_preflight_with_permissions_denied() { let report = PermissionReport { diff --git a/src/diagnostic.rs b/src/diagnostic.rs new file mode 100644 index 0000000..91a9163 --- /dev/null +++ b/src/diagnostic.rs @@ -0,0 +1,24 @@ +pub(crate) fn bounded_text(text: &str, max_chars: usize) -> String { + let mut chars = text.chars(); + let prefix = chars.by_ref().take(max_chars).collect::(); + if chars.next().is_none() { + return prefix; + } + format!("{prefix}… ", text.len()) +} + +pub(crate) fn token_label(token: &str) -> String { + if token.len() <= 64 + && token + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + format!("'{token}'") + } else { + format!("token of {} bytes", token.len()) + } +} + +#[cfg(test)] +#[path = "diagnostic_tests.rs"] +mod tests; diff --git a/src/diagnostic_tests.rs b/src/diagnostic_tests.rs new file mode 100644 index 0000000..01ad85e --- /dev/null +++ b/src/diagnostic_tests.rs @@ -0,0 +1,16 @@ +use super::*; + +#[test] +fn bounded_text_truncates_on_character_boundaries() { + let output = bounded_text("aé日z", 3); + + assert!(output.starts_with("aé日…")); + assert!(output.contains("7 bytes total")); +} + +#[test] +fn token_labels_show_only_small_identifier_shaped_values() { + assert_eq!(token_label("set-value"), "'set-value'"); + assert_eq!(token_label("secret value"), "token of 12 bytes"); + assert_eq!(token_label(&"x".repeat(65)), "token of 65 bytes"); +} diff --git a/src/dispatch/app_window.rs b/src/dispatch/app_window.rs new file mode 100644 index 0000000..f52a48d --- /dev/null +++ b/src/dispatch/app_window.rs @@ -0,0 +1,155 @@ +use agent_desktop_core::{ + AppError, PlatformAdapter, + commands::{ + close_app as close_app_command, focus_window as focus_window_command, helpers, + launch as launch_command, list_apps as list_apps_command, + list_displays as list_displays_command, list_surfaces as list_surfaces_command, + list_windows as list_windows_command, maximize as maximize_command, + minimize as minimize_command, move_window as move_window_command, + resize_window as resize_window_command, restore as restore_command, + }, +}; +use serde_json::Value; + +use crate::cli_args::{ + ListSurfacesArgs, + system::{ + AppRefArgs, CloseAppArgs, FocusWindowArgs, LaunchArgs, ListAppsArgs, ListWindowsArgs, + MoveWindowCliArgs, ResizeWindowCliArgs, + }, +}; +use crate::dispatch::parse::build_launch_options; + +pub(super) fn launch(args: LaunchArgs, adapter: &dyn PlatformAdapter) -> Result { + launch_command::execute( + launch_command::LaunchArgs { + app: args.app, + options: build_launch_options( + &args.args, + &args.env, + args.cwd, + args.timeout, + args.no_attach, + )?, + }, + adapter, + ) +} + +pub(super) fn close_app( + args: CloseAppArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + close_app_command::execute( + close_app_command::CloseAppArgs { + app: args.app, + force: args.force, + }, + adapter, + ) +} + +pub(super) fn list_windows( + args: ListWindowsArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + list_windows_command::execute( + list_windows_command::ListWindowsArgs { app: args.app }, + adapter, + ) +} + +pub(super) fn list_displays(adapter: &dyn PlatformAdapter) -> Result { + list_displays_command::execute(adapter) +} + +pub(super) fn list_apps( + args: ListAppsArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + list_apps_command::execute(list_apps_command::ListAppsArgs { app: args.app }, adapter) +} + +pub(super) fn list_surfaces( + args: ListSurfacesArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + list_surfaces_command::execute( + list_surfaces_command::ListSurfacesArgs { app: args.app }, + adapter, + ) +} + +pub(super) fn focus_window( + args: FocusWindowArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + focus_window_command::execute( + focus_window_command::FocusWindowArgs { + window_id: args.window_id, + app: args.app, + title: args.title, + }, + adapter, + ) +} + +pub(super) fn resize_window( + args: ResizeWindowCliArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + resize_window_command::execute( + resize_window_command::ResizeWindowArgs { + app: args.scope.app, + window_id: args.scope.window_id, + width: args.width, + height: args.height, + }, + adapter, + ) +} + +pub(super) fn move_window( + args: MoveWindowCliArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + move_window_command::execute( + move_window_command::MoveWindowArgs { + app: args.scope.app, + window_id: args.scope.window_id, + x: args.x, + y: args.y, + }, + adapter, + ) +} + +pub(super) fn minimize(args: AppRefArgs, adapter: &dyn PlatformAdapter) -> Result { + minimize_command::execute( + helpers::AppArgs { + app: args.scope.app, + window_id: args.scope.window_id, + }, + adapter, + ) +} + +pub(super) fn maximize(args: AppRefArgs, adapter: &dyn PlatformAdapter) -> Result { + maximize_command::execute( + helpers::AppArgs { + app: args.scope.app, + window_id: args.scope.window_id, + }, + adapter, + ) +} + +pub(super) fn restore(args: AppRefArgs, adapter: &dyn PlatformAdapter) -> Result { + restore_command::execute( + helpers::AppArgs { + app: args.scope.app, + window_id: args.scope.window_id, + }, + adapter, + ) +} diff --git a/src/dispatch/clipboard.rs b/src/dispatch/clipboard.rs new file mode 100644 index 0000000..5291f05 --- /dev/null +++ b/src/dispatch/clipboard.rs @@ -0,0 +1,46 @@ +use agent_desktop_core::{ + AppError, PlatformAdapter, + commands::{clipboard_clear, clipboard_get, clipboard_set}, + context::CommandContext, +}; +use serde_json::Value; + +use crate::cli_args::system::{ClipboardGetArgs, ClipboardSetArgs}; +use crate::dispatch::parse::parse_clipboard_format; + +pub(super) fn get( + args: ClipboardGetArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + clipboard_get::execute( + clipboard_get::ClipboardGetArgs { + format: args + .format + .as_deref() + .map(parse_clipboard_format) + .transpose()?, + out: args.out, + }, + adapter, + context, + ) +} + +pub(super) fn set( + args: ClipboardSetArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + clipboard_set::execute( + clipboard_set::ClipboardSetArgs { + text: args.text, + image: args.image, + file_urls: args.file_url, + }, + adapter, + ) +} + +pub(super) fn clear(adapter: &dyn PlatformAdapter) -> Result { + clipboard_clear::execute(adapter) +} diff --git a/src/dispatch/interaction.rs b/src/dispatch/interaction.rs new file mode 100644 index 0000000..8522b62 --- /dev/null +++ b/src/dispatch/interaction.rs @@ -0,0 +1,193 @@ +use agent_desktop_core::{ + AppError, PlatformAdapter, + commands::{ + check as check_command, clear as clear_command, click as click_command, + collapse as collapse_command, double_click as double_click_command, + expand as expand_command, focus as focus_command, helpers, + right_click as right_click_command, scroll as scroll_command, + scroll_to as scroll_to_command, select as select_command, set_value as set_value_command, + toggle as toggle_command, triple_click as triple_click_command, + type_text as type_text_command, uncheck as uncheck_command, + }, + context::CommandContext, +}; +use serde_json::Value; + +use crate::cli_args::{ + RefArgs, + actions::{ScrollArgs, SelectArgs, SetValueArgs, TypeArgs}, +}; +use crate::dispatch::parse::parse_direction; + +pub(super) fn click( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + click_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn double_click( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + double_click_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn triple_click( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + triple_click_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn right_click( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + right_click_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn type_text( + args: TypeArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + type_text_command::execute( + type_text_command::TypeArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + text: args.text, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + }, + adapter, + context, + ) +} + +pub(super) fn set_value( + args: SetValueArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + set_value_command::execute( + set_value_command::SetValueArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + value: args.value, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + }, + adapter, + context, + ) +} + +pub(super) fn clear( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + clear_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn focus( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + focus_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn toggle( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + toggle_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn check( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + check_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn uncheck( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + uncheck_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn expand( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + expand_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn collapse( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + collapse_command::execute(ref_args(args), adapter, context) +} + +pub(super) fn select( + args: SelectArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + select_command::execute( + select_command::SelectArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + value: args.value, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + }, + adapter, + context, + ) +} + +pub(super) fn scroll( + args: ScrollArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + scroll_command::execute( + scroll_command::ScrollArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + direction: parse_direction(&args.direction)?, + amount: args.amount, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + }, + adapter, + context, + ) +} + +pub(super) fn scroll_to( + args: RefArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + scroll_to_command::execute(ref_args(args), adapter, context) +} + +fn ref_args(args: RefArgs) -> helpers::RefArgs { + helpers::RefArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot_id, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + } +} diff --git a/src/dispatch/keyboard_mouse.rs b/src/dispatch/keyboard_mouse.rs new file mode 100644 index 0000000..8918753 --- /dev/null +++ b/src/dispatch/keyboard_mouse.rs @@ -0,0 +1,182 @@ +use agent_desktop_core::{ + AppError, PlatformAdapter, + commands::{ + drag as drag_command, helpers, hover as hover_command, key_down as key_down_command, + key_up as key_up_command, mouse_click as mouse_click_command, + mouse_down as mouse_down_command, mouse_move as mouse_move_command, + mouse_up as mouse_up_command, mouse_wheel as mouse_wheel_command, press as press_command, + }, + context::CommandContext, +}; +use serde_json::Value; + +use crate::cli_args::{ + actions::{HoverArgs, KeyComboArgs, MouseClickArgs, MouseMoveArgs, MousePointArgs, PressArgs}, + drag::DragCliArgs, + mouse_wheel::MouseWheelArgs, +}; +use crate::dispatch::parse::{parse_modifiers, parse_mouse_button, parse_xy, parse_xy_opt}; + +pub(super) fn press( + args: PressArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + press_command::execute( + press_command::PressArgs { + combo: args.combo, + app: args.app, + force: args.force, + }, + adapter, + context, + ) +} + +pub(super) fn key_down( + args: KeyComboArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + key_down_command::execute( + key_down_command::KeyDownArgs { + combo: args.combo, + force: args.force, + }, + adapter, + ) +} + +pub(super) fn key_up(args: KeyComboArgs, adapter: &dyn PlatformAdapter) -> Result { + key_up_command::execute( + key_up_command::KeyUpArgs { + combo: args.combo, + force: args.force, + }, + adapter, + ) +} + +pub(super) fn hover( + args: HoverArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + hover_command::execute( + hover_command::HoverArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + xy: parse_xy_opt(args.xy.as_deref())?, + duration_ms: args.duration, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + }, + adapter, + context, + ) +} + +pub(super) fn drag( + args: DragCliArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + drag_command::execute( + drag_command::DragArgs { + from: drag_command::DragEndpoint { + ref_id: args.target.from, + xy: parse_xy_opt(args.target.from_xy.as_deref())?, + }, + to: drag_command::DragEndpoint { + ref_id: args.target.to, + xy: parse_xy_opt(args.target.to_xy.as_deref())?, + }, + snapshot_id: args.snapshot, + duration_ms: args.duration, + drop_delay_ms: args.drop_delay, + timeout_ms: helpers::normalize_action_timeout_ms(args.timeout_ms), + }, + adapter, + context, + ) +} + +pub(super) fn mouse_move( + args: MouseMoveArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let (x, y) = parse_xy(&args.xy)?; + mouse_move_command::execute(mouse_move_command::MouseMoveArgs { x, y }, adapter, context) +} + +pub(super) fn mouse_click( + args: MouseClickArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let (x, y) = parse_xy(&args.xy)?; + mouse_click_command::execute( + mouse_click_command::MouseClickArgs { + x, + y, + button: parse_mouse_button(&args.button)?, + count: args.count, + modifiers: parse_modifiers(&args.modifiers)?, + }, + adapter, + context, + ) +} + +pub(super) fn mouse_down( + args: MousePointArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let (x, y) = parse_xy(&args.xy)?; + mouse_down_command::execute( + mouse_down_command::MouseDownArgs { + x, + y, + button: parse_mouse_button(&args.button)?, + modifiers: parse_modifiers(&args.modifiers)?, + }, + adapter, + context, + ) +} + +pub(super) fn mouse_up( + args: MousePointArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let (x, y) = parse_xy(&args.xy)?; + mouse_up_command::execute( + mouse_up_command::MouseUpArgs { + x, + y, + button: parse_mouse_button(&args.button)?, + modifiers: parse_modifiers(&args.modifiers)?, + }, + adapter, + context, + ) +} + +pub(super) fn mouse_wheel( + args: MouseWheelArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + mouse_wheel_command::execute( + mouse_wheel_command::MouseWheelArgs { + x: args.x, + y: args.y, + dy: args.dy, + dx: args.dx, + modifiers: parse_modifiers(&args.modifiers)?, + }, + adapter, + context, + ) +} diff --git a/src/dispatch/mod.rs b/src/dispatch/mod.rs index 3af4690..935df32 100644 --- a/src/dispatch/mod.rs +++ b/src/dispatch/mod.rs @@ -1,30 +1,18 @@ +mod app_window; +mod clipboard; +mod interaction; +mod keyboard_mouse; mod notifications; +mod observation; mod parse; mod session; +mod system; mod trace; -use agent_desktop_core::{ - PermissionReport, - adapter::PlatformAdapter, - commands::{ - check, clear, click, clipboard_clear, clipboard_get, clipboard_set, close_app, collapse, - double_click, drag, expand, find, focus, focus_window, get, helpers, hover, is_check, - key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize, minimize, - mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press, - resize_window, restore, right_click, screenshot, scroll, scroll_to, select, set_value, - skills, snapshot, status, toggle, triple_click, type_text, uncheck, version, wait, - }, - context::CommandContext, - error::AppError, -}; +use agent_desktop_core::{AppError, PermissionReport, PlatformAdapter, context::CommandContext}; use serde_json::Value; use crate::cli::Commands; -use crate::cli_args::skills::SkillsAction; -use parse::{ - parse_direction, parse_get_property, parse_is_property, parse_mouse_button, parse_xy, - parse_xy_opt, -}; pub(crate) fn dispatch( cmd: Commands, @@ -33,362 +21,73 @@ pub(crate) fn dispatch( context: &CommandContext, ) -> Result { tracing::debug!("dispatch: {}", cmd.name()); - let scope = context.command_scope(cmd.name()); - let result = dispatch_inner(cmd, adapter, permission_report, context); - scope.complete(&result); + let scope = if cmd.is_mutating() { + context.mutating_command_scope(cmd.name())? + } else { + context.command_scope(cmd.name())? + }; + let result = match cmd { + Commands::Snapshot(args) => observation::snapshot(args, adapter, context), + Commands::Find(args) => observation::find(args, adapter, context), + Commands::Screenshot(args) => observation::screenshot(args, adapter), + Commands::Get(args) => observation::get(args, adapter, context), + Commands::Is(args) => observation::is(args, adapter, context), + Commands::Click(args) => interaction::click(args, adapter, context), + Commands::DoubleClick(args) => interaction::double_click(args, adapter, context), + Commands::TripleClick(args) => interaction::triple_click(args, adapter, context), + Commands::RightClick(args) => interaction::right_click(args, adapter, context), + Commands::Type(args) => interaction::type_text(args, adapter, context), + Commands::SetValue(args) => interaction::set_value(args, adapter, context), + Commands::Clear(args) => interaction::clear(args, adapter, context), + Commands::Focus(args) => interaction::focus(args, adapter, context), + Commands::Select(args) => interaction::select(args, adapter, context), + Commands::Toggle(args) => interaction::toggle(args, adapter, context), + Commands::Check(args) => interaction::check(args, adapter, context), + Commands::Uncheck(args) => interaction::uncheck(args, adapter, context), + Commands::Expand(args) => interaction::expand(args, adapter, context), + Commands::Collapse(args) => interaction::collapse(args, adapter, context), + Commands::Scroll(args) => interaction::scroll(args, adapter, context), + Commands::ScrollTo(args) => interaction::scroll_to(args, adapter, context), + Commands::Press(args) => keyboard_mouse::press(args, adapter, context), + Commands::KeyDown(args) => keyboard_mouse::key_down(args, adapter), + Commands::KeyUp(args) => keyboard_mouse::key_up(args, adapter), + Commands::Hover(args) => keyboard_mouse::hover(args, adapter, context), + Commands::Drag(args) => keyboard_mouse::drag(args, adapter, context), + Commands::MouseMove(args) => keyboard_mouse::mouse_move(args, adapter, context), + Commands::MouseClick(args) => keyboard_mouse::mouse_click(args, adapter, context), + Commands::MouseDown(args) => keyboard_mouse::mouse_down(args, adapter, context), + Commands::MouseUp(args) => keyboard_mouse::mouse_up(args, adapter, context), + Commands::MouseWheel(args) => keyboard_mouse::mouse_wheel(args, adapter, context), + Commands::Launch(args) => app_window::launch(args, adapter), + Commands::CloseApp(args) => app_window::close_app(args, adapter), + Commands::ListWindows(args) => app_window::list_windows(args, adapter), + Commands::ListDisplays => app_window::list_displays(adapter), + Commands::ListApps(args) => app_window::list_apps(args, adapter), + Commands::FocusWindow(args) => app_window::focus_window(args, adapter), + Commands::ResizeWindow(args) => app_window::resize_window(args, adapter), + Commands::MoveWindow(args) => app_window::move_window(args, adapter), + Commands::Minimize(args) => app_window::minimize(args, adapter), + Commands::Maximize(args) => app_window::maximize(args, adapter), + Commands::Restore(args) => app_window::restore(args, adapter), + Commands::ListSurfaces(args) => app_window::list_surfaces(args, adapter), + Commands::ListNotifications(args) => notifications::list(args, adapter, context), + Commands::DismissNotification(args) => notifications::dismiss(args, adapter, context), + Commands::DismissAllNotifications(args) => { + notifications::dismiss_all(args, adapter, context) + } + Commands::NotificationAction(args) => notifications::action(args, adapter, context), + Commands::ClipboardGet(args) => clipboard::get(args, adapter, context), + Commands::ClipboardSet(args) => clipboard::set(args, adapter), + Commands::ClipboardClear => clipboard::clear(adapter), + Commands::Wait(args) => system::wait(args, adapter, context), + Commands::Status => system::status(adapter, permission_report, context), + Commands::Permissions(args) => system::permissions(args, adapter, permission_report), + Commands::Version => system::version(), + Commands::Batch(args) => system::batch(args, adapter, permission_report, context), + Commands::Skills(args) => system::skills(args), + Commands::Session(args) => system::session(args, context), + Commands::Trace(args) => system::trace(args, context), + }; + scope.complete(&result)?; result } - -fn dispatch_inner( - cmd: Commands, - adapter: &dyn PlatformAdapter, - permission_report: &PermissionReport, - context: &CommandContext, -) -> Result { - match cmd { - Commands::Snapshot(a) => snapshot::execute( - snapshot::SnapshotArgs { - app: a.app, - window_id: a.window_id, - max_depth: a.max_depth, - include_bounds: a.include_bounds, - interactive_only: a.interactive_only, - compact: a.compact, - surface: a.surface.to_core(), - skeleton: a.skeleton, - root_ref: a.root, - snapshot_id: a.snapshot, - }, - adapter, - context, - ), - - Commands::Find(a) => find::execute( - find::FindArgs { - app: a.app, - role: a.role, - name: a.name, - value: a.value, - text: a.text, - count: a.count, - first: a.first, - last: a.last, - nth: a.nth, - limit: a.limit, - }, - adapter, - context, - ), - - Commands::Screenshot(a) => screenshot::execute( - screenshot::ScreenshotArgs { - app: a.app, - window_id: a.window_id, - output_path: a.output_path, - }, - adapter, - ), - - Commands::Get(a) => get::execute( - get::GetArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - property: parse_get_property(&a.property)?, - }, - adapter, - context, - ), - - Commands::Is(a) => is_check::execute( - is_check::IsArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - property: parse_is_property(&a.property)?, - }, - adapter, - context, - ), - - Commands::Click(a) => click::execute(ref_args(a), adapter, context), - Commands::DoubleClick(a) => double_click::execute(ref_args(a), adapter, context), - Commands::TripleClick(a) => triple_click::execute(ref_args(a), adapter, context), - Commands::RightClick(a) => right_click::execute(ref_args(a), adapter, context), - - Commands::Type(a) => type_text::execute( - type_text::TypeArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - text: a.text, - }, - adapter, - context, - ), - - Commands::SetValue(a) => set_value::execute( - set_value::SetValueArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - value: a.value, - }, - adapter, - context, - ), - - Commands::Clear(a) => clear::execute(ref_args(a), adapter, context), - - Commands::Focus(a) => focus::execute(ref_args(a), adapter, context), - Commands::Toggle(a) => toggle::execute(ref_args(a), adapter, context), - Commands::Check(a) => check::execute(ref_args(a), adapter, context), - Commands::Uncheck(a) => uncheck::execute(ref_args(a), adapter, context), - Commands::Expand(a) => expand::execute(ref_args(a), adapter, context), - Commands::Collapse(a) => collapse::execute(ref_args(a), adapter, context), - - Commands::Select(a) => select::execute( - select::SelectArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - value: a.value, - }, - adapter, - context, - ), - - Commands::Scroll(a) => scroll::execute( - scroll::ScrollArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - direction: parse_direction(&a.direction)?, - amount: a.amount, - }, - adapter, - context, - ), - - Commands::ScrollTo(a) => scroll_to::execute(ref_args(a), adapter, context), - - Commands::Press(a) => press::execute( - press::PressArgs { - combo: a.combo, - app: a.app, - force: a.force, - }, - adapter, - ), - - Commands::KeyDown(a) => key_down::execute( - key_down::KeyDownArgs { - combo: a.combo, - force: a.force, - }, - adapter, - ), - - Commands::KeyUp(a) => key_up::execute( - key_up::KeyUpArgs { - combo: a.combo, - force: a.force, - }, - adapter, - ), - - Commands::Hover(a) => hover::execute( - hover::HoverArgs { - ref_id: a.ref_id, - snapshot_id: a.snapshot, - xy: parse_xy_opt(a.xy.as_deref())?, - duration_ms: a.duration, - }, - adapter, - context, - ), - - Commands::Drag(a) => drag::execute( - drag::DragArgs { - from_ref: a.from, - from_xy: parse_xy_opt(a.from_xy.as_deref())?, - to_ref: a.to, - to_xy: parse_xy_opt(a.to_xy.as_deref())?, - snapshot_id: a.snapshot, - duration_ms: a.duration, - drop_delay_ms: a.drop_delay, - }, - adapter, - context, - ), - - Commands::MouseMove(a) => { - let (x, y) = parse_xy(&a.xy)?; - mouse_move::execute(mouse_move::MouseMoveArgs { x, y }, adapter, context) - } - - Commands::MouseClick(a) => { - let (x, y) = parse_xy(&a.xy)?; - mouse_click::execute( - mouse_click::MouseClickArgs { - x, - y, - button: parse_mouse_button(&a.button)?, - count: a.count, - }, - adapter, - context, - ) - } - - Commands::MouseDown(a) => { - let (x, y) = parse_xy(&a.xy)?; - mouse_down::execute( - mouse_down::MouseDownArgs { - x, - y, - button: parse_mouse_button(&a.button)?, - }, - adapter, - context, - ) - } - - Commands::MouseUp(a) => { - let (x, y) = parse_xy(&a.xy)?; - mouse_up::execute( - mouse_up::MouseUpArgs { - x, - y, - button: parse_mouse_button(&a.button)?, - }, - adapter, - context, - ) - } - - Commands::Launch(a) => launch::execute( - launch::LaunchArgs { - app: a.app, - timeout_ms: a.timeout, - }, - adapter, - ), - - Commands::CloseApp(a) => close_app::execute( - close_app::CloseAppArgs { - app: a.app, - force: a.force, - }, - adapter, - ), - - Commands::ListWindows(a) => { - list_windows::execute(list_windows::ListWindowsArgs { app: a.app }, adapter) - } - - Commands::ListApps(a) => { - list_apps::execute(list_apps::ListAppsArgs { app: a.app }, adapter) - } - - Commands::ListSurfaces(a) => { - list_surfaces::execute(list_surfaces::ListSurfacesArgs { app: a.app }, adapter) - } - - Commands::FocusWindow(a) => focus_window::execute( - focus_window::FocusWindowArgs { - window_id: a.window_id, - app: a.app, - title: a.title, - }, - adapter, - ), - - Commands::ResizeWindow(a) => resize_window::execute( - resize_window::ResizeWindowArgs { - app: a.app, - width: a.width, - height: a.height, - }, - adapter, - ), - - Commands::MoveWindow(a) => move_window::execute( - move_window::MoveWindowArgs { - app: a.app, - x: a.x, - y: a.y, - }, - adapter, - ), - - Commands::Minimize(a) => minimize::execute(helpers::AppArgs { app: a.app }, adapter), - - Commands::Maximize(a) => maximize::execute(helpers::AppArgs { app: a.app }, adapter), - - Commands::Restore(a) => restore::execute(helpers::AppArgs { app: a.app }, adapter), - - Commands::ListNotifications(_) - | Commands::DismissNotification(_) - | Commands::DismissAllNotifications(_) - | Commands::NotificationAction(_) => notifications::dispatch_notification(cmd, adapter), - - Commands::ClipboardGet => clipboard_get::execute(adapter), - Commands::ClipboardSet(a) => clipboard_set::execute(a.text, adapter), - Commands::ClipboardClear => clipboard_clear::execute(adapter), - - Commands::Wait(a) => wait::execute( - wait::WaitArgs { - mode: wait::WaitModeArgs { - ms: a.mode.ms, - element: a.mode.element, - window: a.mode.window, - text: a.mode.text, - menu: a.mode.menu, - menu_closed: a.mode.menu_closed, - notification: a.mode.notification, - }, - predicate: wait::WaitPredicateArgs { - snapshot_id: a.predicate.snapshot, - predicate: a.predicate.predicate, - value: a.predicate.value, - action: a.predicate.action, - count: a.predicate.count, - }, - timeout_ms: a.timeout, - app: a.app, - }, - adapter, - context, - ), - - Commands::Status => { - status::execute_with_report_with_context(adapter, permission_report, context) - } - - Commands::Permissions(a) => permissions::execute_with_report( - permissions::PermissionsArgs { request: a.request }, - adapter, - permission_report, - ), - - Commands::Version => version::execute(), - - Commands::Skills(a) => match a.action.unwrap_or(SkillsAction::List) { - SkillsAction::List => skills::list(), - SkillsAction::Path => skills::path(), - SkillsAction::Get(g) => skills::get(skills::GetArgs { - name: g.name, - full: g.full, - reference: g.reference, - }), - }, - - Commands::Session(a) => session::dispatch(a, context), - - Commands::Trace(a) => trace::dispatch(a, context), - - Commands::Batch(a) => crate::batch::execute(a, adapter, permission_report, context), - } -} - -fn ref_args(args: crate::cli_args::RefArgs) -> helpers::RefArgs { - helpers::RefArgs { - ref_id: args.ref_id, - snapshot_id: args.snapshot_id, - } -} diff --git a/src/dispatch/notifications.rs b/src/dispatch/notifications.rs index a7b913c..3f1279e 100644 --- a/src/dispatch/notifications.rs +++ b/src/dispatch/notifications.rs @@ -1,54 +1,77 @@ use agent_desktop_core::{ - adapter::PlatformAdapter, + AppError, PlatformAdapter, commands::{ dismiss_all_notifications, dismiss_notification, list_notifications, notification_action, }, - error::{AppError, ErrorCode}, + context::CommandContext, }; use serde_json::Value; -use crate::cli::Commands; +use crate::cli_args::notifications::{ + DismissAllNotificationsCliArgs, DismissNotificationCliArgs, ListNotificationsCliArgs, + NotificationActionCliArgs, +}; -pub(crate) fn dispatch_notification( - cmd: Commands, +pub(super) fn list( + args: ListNotificationsCliArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { - match cmd { - Commands::ListNotifications(a) => list_notifications::execute( - list_notifications::ListNotificationsArgs { - app: a.app, - text: a.text, - limit: a.limit, - }, - adapter, - ), - Commands::DismissNotification(a) => dismiss_notification::execute( - dismiss_notification::DismissNotificationArgs { - index: notification_index(a.index)?, - app: a.app, - }, - adapter, - ), - Commands::DismissAllNotifications(a) => dismiss_all_notifications::execute( - dismiss_all_notifications::DismissAllNotificationsArgs { app: a.app }, - adapter, - ), - Commands::NotificationAction(a) => notification_action::execute( - notification_action::NotificationActionArgs { - index: notification_index(a.index)?, - action: a.action, - expected_app: a.expected_app, - expected_title: a.expected_title, - }, - adapter, - ), - _ => Err(AppError::Adapter( - agent_desktop_core::error::AdapterError::new( - ErrorCode::InvalidArgs, - "dispatch_notification received a non-notification command", - ), - )), - } + list_notifications::execute( + list_notifications::ListNotificationsArgs { + app: args.app, + text: args.text, + limit: args.limit, + }, + adapter, + context, + ) +} + +pub(super) fn dismiss( + args: DismissNotificationCliArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + dismiss_notification::execute( + dismiss_notification::DismissNotificationArgs { + index: notification_index(args.index)?, + app: args.app, + expected_app: args.expected_app, + expected_title: args.expected_title, + }, + adapter, + context, + ) +} + +pub(super) fn dismiss_all( + args: DismissAllNotificationsCliArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + dismiss_all_notifications::execute( + dismiss_all_notifications::DismissAllNotificationsArgs { app: args.app }, + adapter, + context, + ) +} + +pub(super) fn action( + args: NotificationActionCliArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + notification_action::execute( + notification_action::NotificationActionArgs { + index: notification_index(args.index)?, + action: args.action, + expected_app: args.expected_app, + expected_title: args.expected_title, + }, + adapter, + context, + ) } fn notification_index(index: u64) -> Result { @@ -63,20 +86,19 @@ fn notification_index(index: u64) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::cli_args::notifications::{DismissNotificationCliArgs, NotificationActionCliArgs}; - - struct NoopAdapter; - - impl PlatformAdapter for NoopAdapter {} + use crate::test_noop_ops::NoopAdapter; #[test] fn dismiss_notification_rejects_zero_index_before_adapter() { - let err = dispatch_notification( - Commands::DismissNotification(DismissNotificationCliArgs { + let err = dismiss( + DismissNotificationCliArgs { index: 0, app: None, - }), + expected_app: None, + expected_title: None, + }, &NoopAdapter, + &CommandContext::default(), ) .unwrap_err(); @@ -85,14 +107,15 @@ mod tests { #[test] fn notification_action_rejects_zero_index_before_adapter() { - let err = dispatch_notification( - Commands::NotificationAction(NotificationActionCliArgs { + let err = action( + NotificationActionCliArgs { index: 0, action: "Reply".into(), expected_app: None, expected_title: None, - }), + }, &NoopAdapter, + &CommandContext::default(), ) .unwrap_err(); diff --git a/src/dispatch/observation.rs b/src/dispatch/observation.rs new file mode 100644 index 0000000..e02b40c --- /dev/null +++ b/src/dispatch/observation.rs @@ -0,0 +1,119 @@ +use agent_desktop_core::{ + AppError, PlatformAdapter, + commands::{ + find as find_command, get as get_command, is_check as is_command, + screenshot as screenshot_command, snapshot as snapshot_command, + }, + context::CommandContext, +}; +use serde_json::Value; + +use crate::cli_args::{FindArgs, GetArgs, IsArgs, ScreenshotArgs, SnapshotArgs}; +use crate::dispatch::parse::{parse_get_property, parse_is_property}; + +pub(super) fn snapshot( + args: SnapshotArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + snapshot_command::execute( + snapshot_command::SnapshotArgs { + app: args.scope.app, + window_id: args.scope.window_id, + max_depth: args.tree.max_depth, + include_bounds: args.tree.include_bounds, + interactive_only: args.tree.interactive_only, + compact: args.tree.compact, + surface: args.surface.to_core(), + skeleton: args.tree.skeleton, + root_ref: args.root, + snapshot_id: args.snapshot, + }, + adapter, + context, + ) +} + +pub(super) fn find( + args: FindArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + let states = args + .states + .iter() + .map(|raw| find_command::parse_state_flag(raw)) + .collect::, _>>()?; + find_command::execute( + find_command::FindArgs { + app: args.scope.app, + window_id: args.scope.window_id, + filter: find_command::FindFilterArgs { + role: args.filter.role, + name: args.filter.name, + description: args.filter.description, + native_id: args.filter.native_id, + value: args.filter.value, + text: args.filter.text, + exact: args.filter.exact, + }, + states, + selection: find_command::FindSelectionArgs { + count: args.selection.count, + first: args.selection.first, + last: args.selection.last, + nth: args.selection.nth, + limit: args.selection.limit, + }, + }, + adapter, + context, + ) +} + +pub(super) fn screenshot( + args: ScreenshotArgs, + adapter: &dyn PlatformAdapter, +) -> Result { + screenshot_command::execute( + screenshot_command::ScreenshotArgs { + app: args.scope.app, + window_id: args.scope.window_id, + screen: args.screen, + output_path: args.output_path, + }, + adapter, + ) +} + +pub(super) fn get( + args: GetArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + get_command::execute( + get_command::GetArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + property: parse_get_property(&args.property)?, + }, + adapter, + context, + ) +} + +pub(super) fn is( + args: IsArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + is_command::execute( + is_command::IsArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot, + property: parse_is_property(&args.property)?, + }, + adapter, + context, + ) +} diff --git a/src/dispatch/parse.rs b/src/dispatch/parse.rs index 2d17e6a..4042d27 100644 --- a/src/dispatch/parse.rs +++ b/src/dispatch/parse.rs @@ -1,8 +1,9 @@ use agent_desktop_core::{ - action::{Direction, MouseButton}, + AppError, ClipboardFormat, Direction, Modifier, MouseButton, commands::{get, is_check}, - error::AppError, + launch_options::LaunchOptions, }; +use std::collections::BTreeMap; pub(crate) fn parse_get_property(s: &str) -> Result { match s { @@ -12,9 +13,9 @@ pub(crate) fn parse_get_property(s: &str) -> Result "bounds" => Ok(get::GetProperty::Bounds), "role" => Ok(get::GetProperty::Role), "states" => Ok(get::GetProperty::States), - other => Err(AppError::invalid_input(format!( - "Unknown property '{other}'. Valid: text, value, title, bounds, role, states" - ))), + _ => Err(AppError::invalid_input( + "Unknown property. Valid: text, value, title, bounds, role, states", + )), } } @@ -25,9 +26,9 @@ pub(crate) fn parse_is_property(s: &str) -> Result Ok(is_check::IsProperty::Checked), "focused" => Ok(is_check::IsProperty::Focused), "expanded" => Ok(is_check::IsProperty::Expanded), - other => Err(AppError::invalid_input(format!( - "Unknown property '{other}'. Valid: visible, enabled, checked, focused, expanded" - ))), + _ => Err(AppError::invalid_input( + "Unknown property. Valid: visible, enabled, checked, focused, expanded", + )), } } @@ -37,9 +38,9 @@ pub(crate) fn parse_direction(s: &str) -> Result { "down" => Ok(Direction::Down), "left" => Ok(Direction::Left), "right" => Ok(Direction::Right), - other => Err(AppError::invalid_input(format!( - "Unknown direction '{other}'. Valid: up, down, left, right" - ))), + _ => Err(AppError::invalid_input( + "Unknown direction. Valid: up, down, left, right", + )), } } @@ -48,27 +49,34 @@ pub(crate) fn parse_mouse_button(s: &str) -> Result { "left" => Ok(MouseButton::Left), "right" => Ok(MouseButton::Right), "middle" => Ok(MouseButton::Middle), - other => Err(AppError::invalid_input(format!( - "Unknown button '{other}'. Valid: left, right, middle" - ))), + _ => Err(AppError::invalid_input( + "Unknown button. Valid: left, right, middle", + )), } } pub(crate) fn parse_xy(s: &str) -> Result<(f64, f64), AppError> { - let parts: Vec<&str> = s.split(',').collect(); - if parts.len() != 2 { - return Err(AppError::invalid_input(format!( - "Invalid coordinates '{s}'. Expected format: x,y (e.g., 500,300)" - ))); + let (x_raw, y_raw) = s.split_once(',').ok_or_else(|| { + AppError::invalid_input("Invalid coordinates. Expected format: x,y (e.g., 500,300)") + })?; + if y_raw.contains(',') { + return Err(AppError::invalid_input( + "Invalid coordinates. Expected exactly one comma", + )); } - let x: f64 = parts[0] + let x: f64 = x_raw .trim() .parse() - .map_err(|_| AppError::invalid_input(format!("Invalid x coordinate: '{}'", parts[0])))?; - let y: f64 = parts[1] + .map_err(|_| AppError::invalid_input("Invalid x coordinate"))?; + let y: f64 = y_raw .trim() .parse() - .map_err(|_| AppError::invalid_input(format!("Invalid y coordinate: '{}'", parts[1])))?; + .map_err(|_| AppError::invalid_input("Invalid y coordinate"))?; + if !x.is_finite() || !y.is_finite() { + return Err(AppError::invalid_input( + "Coordinates must be finite numbers", + )); + } Ok((x, y)) } @@ -79,6 +87,96 @@ pub(crate) fn parse_xy_opt(s: Option<&str>) -> Result, AppErr } } +pub(crate) fn parse_clipboard_format(s: &str) -> Result { + match s { + "auto" => Ok(ClipboardFormat::Auto), + "text" | "plain_text" | "plaintext" => Ok(ClipboardFormat::Text), + "image" | "png" => Ok(ClipboardFormat::Image), + "file-urls" | "file_urls" | "fileurls" => Ok(ClipboardFormat::FileUrls), + _ => Err(AppError::invalid_input( + "Unknown clipboard format. Valid: auto, text, image, file-urls", + )), + } +} + +pub(crate) fn parse_modifiers(values: &[String]) -> Result, AppError> { + let mut parsed = Vec::with_capacity(values.len()); + for value in values { + let modifier = parse_modifier(value)?; + if parsed.contains(&modifier) { + return Err(AppError::invalid_input( + "Each mouse modifier may be supplied only once", + )); + } + parsed.push(modifier); + } + Ok(parsed) +} + +pub(crate) fn parse_modifier(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "shift" => Ok(Modifier::Shift), + "meta" | "cmd" | "command" => Ok(Modifier::Meta), + "ctrl" | "control" => Ok(Modifier::Ctrl), + "alt" | "option" => Ok(Modifier::Alt), + _ => Err(AppError::invalid_input( + "Unknown modifier. Valid: shift, meta, ctrl, alt (cmd/command aliases are accepted)", + )), + } +} + +pub(crate) fn build_launch_options( + args: &[String], + env: &[String], + cwd: Option, + timeout_ms: u64, + no_attach: bool, +) -> Result { + if let Some(index) = args.iter().position(|argument| argument.contains('\0')) { + return Err(AppError::invalid_input(format!( + "Invalid --arg entry #{index}: argument contains a NUL byte" + ))); + } + let mut env_map = BTreeMap::new(); + for (idx, pair) in env.iter().enumerate() { + let (key, value) = parse_env_pair(pair, idx)?; + if env_map.insert(key, value).is_some() { + return Err(AppError::invalid_input(format!( + "Duplicate --env key at entry #{idx}" + ))); + } + } + Ok(LaunchOptions { + args: args.to_vec(), + env: env_map, + cwd, + timeout_ms, + attach_if_running: !no_attach, + }) +} + +fn parse_env_pair(pair: &str, idx: usize) -> Result<(String, String), AppError> { + let (key, value) = pair.split_once('=').ok_or_else(|| { + AppError::invalid_input(format!("Invalid --env entry #{idx}: expected KEY=VALUE")) + })?; + if key.is_empty() + || !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + || key.as_bytes()[0].is_ascii_digit() + { + return Err(AppError::invalid_input(format!( + "Invalid --env entry #{idx}: KEY must be a portable environment identifier" + ))); + } + if value.contains('\0') { + return Err(AppError::invalid_input(format!( + "Invalid --env entry #{idx}: VALUE contains a NUL byte" + ))); + } + Ok((key.to_string(), value.to_string())) +} + #[cfg(test)] #[path = "parse_tests.rs"] mod tests; diff --git a/src/dispatch/parse_tests.rs b/src/dispatch/parse_tests.rs index 1be8589..4ae9070 100644 --- a/src/dispatch/parse_tests.rs +++ b/src/dispatch/parse_tests.rs @@ -8,6 +8,31 @@ fn rejects_unknown_direction() { ); } +#[test] +fn env_parse_errors_never_echo_the_value() { + let secret = "sk_live_supersecret_token_value"; + let no_equals = parse_env_pair(secret, 0).unwrap_err(); + let no_equals_msg = no_equals.to_string(); + assert_eq!(no_equals.code(), "INVALID_ARGS"); + assert!( + !no_equals_msg.contains(secret), + "malformed --env message leaked the raw value: {no_equals_msg}" + ); + + let empty_key = format!("={secret}"); + let err = parse_env_pair(&empty_key, 3).unwrap_err(); + let err_msg = err.to_string(); + assert_eq!(err.code(), "INVALID_ARGS"); + assert!( + !err_msg.contains(secret), + "empty-key --env message leaked the raw value: {err_msg}" + ); + assert!( + err_msg.contains("#3"), + "message should carry the entry index" + ); +} + #[test] fn rejects_unknown_get_property() { match parse_get_property("placeholder") { @@ -42,4 +67,35 @@ fn rejects_bad_xy_shape_and_numbers() { assert_eq!(parse_xy("10").unwrap_err().code(), "INVALID_ARGS"); assert_eq!(parse_xy("x,20").unwrap_err().code(), "INVALID_ARGS"); assert_eq!(parse_xy("10,y").unwrap_err().code(), "INVALID_ARGS"); + assert_eq!(parse_xy("1,2,3").unwrap_err().code(), "INVALID_ARGS"); + assert_eq!(parse_xy("NaN,20").unwrap_err().code(), "INVALID_ARGS"); + assert_eq!(parse_xy("10,inf").unwrap_err().code(), "INVALID_ARGS"); +} + +#[test] +fn rejects_duplicate_modifiers() { + let error = parse_modifiers(&["shift".into(), "shift".into()]) + .expect_err("duplicate modifier must not be silently applied twice"); + assert_eq!(error.code(), "INVALID_ARGS"); +} + +#[test] +fn modifier_aliases_map_to_portable_meta() { + for alias in ["meta", "cmd", "command", "META"] { + assert_eq!(parse_modifier(alias).unwrap(), Modifier::Meta); + } +} + +#[test] +fn launch_options_reject_ambiguous_or_nonportable_environment() { + let duplicate = build_launch_options(&[], &["A=1".into(), "A=2".into()], None, 100, false) + .expect_err("duplicate keys must not silently overwrite"); + assert_eq!(duplicate.code(), "INVALID_ARGS"); + + for entry in ["9KEY=value", "BAD-KEY=value", "KEY=bad\0value"] { + assert_eq!(parse_env_pair(entry, 0).unwrap_err().code(), "INVALID_ARGS"); + } + let nul_arg = build_launch_options(&["bad\0arg".into()], &[], None, 100, false) + .expect_err("NUL arguments must fail before platform spawn"); + assert_eq!(nul_arg.code(), "INVALID_ARGS"); } diff --git a/src/dispatch/session.rs b/src/dispatch/session.rs index c14cb5f..d053ee0 100644 --- a/src/dispatch/session.rs +++ b/src/dispatch/session.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::{commands::session, context::CommandContext, error::AppError}; +use agent_desktop_core::{AppError, commands::session, context::CommandContext}; use serde_json::Value; use crate::cli_args::session::{SessionAction, SessionArgs}; @@ -9,11 +9,11 @@ pub(crate) fn dispatch(args: SessionArgs, context: &CommandContext) -> Result session::execute(session::SessionAction::End { - id: e.id.or_else(|| context.session_id().map(str::to_string)), }), + SessionAction::End(e) => { + let id = resolve_end_session_id(e.id, context.session_id())?; + session::execute(session::SessionAction::End { id }) + } SessionAction::List => session::execute(session::SessionAction::List), SessionAction::Gc(g) => session::execute(session::SessionAction::Gc { older_than_secs: g.older_than, @@ -21,3 +21,21 @@ pub(crate) fn dispatch(args: SessionArgs, context: &CommandContext) -> Result, + active: Option<&str>, +) -> Result { + explicit + .or_else(|| active.map(str::to_string)) + .ok_or_else(|| { + AppError::invalid_input_with_suggestion( + "No session id was supplied and no active session scope is configured", + "Pass `session end `, global `--session `, or AGENT_DESKTOP_SESSION", + ) + }) +} + +#[cfg(test)] +#[path = "session_tests.rs"] +mod tests; diff --git a/src/dispatch/session_tests.rs b/src/dispatch/session_tests.rs new file mode 100644 index 0000000..9de54c2 --- /dev/null +++ b/src/dispatch/session_tests.rs @@ -0,0 +1,25 @@ +use super::resolve_end_session_id; + +#[test] +fn explicit_session_end_id_precedes_active_scope() { + assert_eq!( + resolve_end_session_id(Some("explicit".into()), Some("active")).unwrap(), + "explicit" + ); +} + +#[test] +fn session_end_falls_back_to_active_scope() { + assert_eq!( + resolve_end_session_id(None, Some("active")).unwrap(), + "active" + ); +} + +#[test] +fn session_end_without_any_scope_is_invalid() { + let error = resolve_end_session_id(None, None).expect_err("missing scope must fail"); + + assert_eq!(error.code(), "INVALID_ARGS"); + assert!(error.to_string().contains("No session id")); +} diff --git a/src/dispatch/system.rs b/src/dispatch/system.rs new file mode 100644 index 0000000..d3d187d --- /dev/null +++ b/src/dispatch/system.rs @@ -0,0 +1,108 @@ +use agent_desktop_core::{ + AppError, PermissionReport, PlatformAdapter, + commands::{ + permissions as permissions_command, skills as skills_command, status as status_command, + version as version_command, wait as wait_command, wait_surface::SurfaceWait, + }, + context::CommandContext, +}; +use serde_json::Value; + +use crate::cli_args::{ + batch::BatchArgs, + session::SessionArgs, + skills::{SkillsAction, SkillsArgs}, + system::{PermissionsArgs, WaitArgs}, + trace::TraceArgs, +}; +use crate::dispatch::{session as session_dispatch, trace as trace_dispatch}; + +pub(super) fn wait( + args: WaitArgs, + adapter: &dyn PlatformAdapter, + context: &CommandContext, +) -> Result { + wait_command::execute( + wait_command::WaitArgs { + mode: wait_command::WaitModeArgs { + ms: args.mode.ms, + element: args.mode.element, + window: args.mode.window, + text: args.mode.text, + surface: SurfaceWait::from_flags( + args.mode.menu, + args.mode.menu_closed, + args.mode.notification, + )?, + event: args.event.event, + window_id: args.event.window_id, + }, + predicate: wait_command::WaitPredicateArgs { + snapshot_id: args.predicate.snapshot, + predicate: args.predicate.predicate, + value: args.predicate.value, + action: args.predicate.action, + count: args.predicate.count, + }, + timeout_ms: args.timeout, + app: args.app, + }, + adapter, + context, + ) +} + +pub(super) fn status( + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, + context: &CommandContext, +) -> Result { + status_command::execute_with_report_with_context(adapter, permission_report, context) +} + +pub(super) fn permissions( + args: PermissionsArgs, + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, +) -> Result { + permissions_command::execute_with_report( + permissions_command::PermissionsArgs { + request: args.request, + }, + adapter, + permission_report, + ) +} + +pub(super) fn version() -> Result { + version_command::execute() +} + +pub(super) fn skills(args: SkillsArgs) -> Result { + match args.action.unwrap_or(SkillsAction::List) { + SkillsAction::List => skills_command::list(), + SkillsAction::Path => skills_command::path(), + SkillsAction::Get(get) => skills_command::get(skills_command::GetArgs { + name: get.name, + full: get.full, + reference: get.reference, + }), + } +} + +pub(super) fn session(args: SessionArgs, context: &CommandContext) -> Result { + session_dispatch::dispatch(args, context) +} + +pub(super) fn trace(args: TraceArgs, context: &CommandContext) -> Result { + trace_dispatch::dispatch(args, context) +} + +pub(super) fn batch( + args: BatchArgs, + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, + context: &CommandContext, +) -> Result { + crate::batch::execute(args, adapter, permission_report, context) +} diff --git a/src/dispatch/trace.rs b/src/dispatch/trace.rs index a1dff5a..eb7d0fe 100644 --- a/src/dispatch/trace.rs +++ b/src/dispatch/trace.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::{commands::trace, context::CommandContext, error::AppError}; +use agent_desktop_core::{AppError, commands::trace, context::CommandContext}; use serde_json::Value; use crate::cli_args::trace::{TraceAction, TraceArgs}; diff --git a/src/main.rs b/src/main.rs index b1ba2f0..89c4c87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,42 +1,58 @@ +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))] + mod batch; mod cli; mod cli_args; mod command_policy; +mod diagnostic; mod dispatch; +/// Shared blanket-default `PlatformAdapter` test double, sourced once from +/// `tests/support/noop_ops.rs` (also consumed by the standalone +/// `conformance` integration-test crate) and reused by any in-crate unit +/// test that needs "some adapter" without exercising a live capability. +#[cfg(test)] +#[path = "../tests/support/noop_ops.rs"] +mod test_noop_ops; + use agent_desktop_core::{ - adapter::PlatformAdapter, + AdapterError, AppError, DeliverySemantics, ErrorCode, context::{CommandContext, WaitSelector}, - error::AppError, - output::{ENVELOPE_VERSION, ErrorPayload, Response}, + output::{ErrorPayload, Response}, session::resolve_active_session, }; use clap::{CommandFactory, Parser}; use cli::{Cli, Commands}; use cli_args::skills::SkillsAction; use std::io::{BufWriter, Write}; +use std::process::ExitCode; -const WAIT_SUPPORTED: &[&str] = &[ - "snapshot", - "click", - "double-click", - "triple-click", - "right-click", - "clear", - "focus", - "toggle", - "check", - "uncheck", - "expand", - "collapse", - "scroll-to", - "type", - "set-value", - "select", - "scroll", -]; +const EXIT_INVALID_ARGS: u8 = 2; -fn main() { +fn main() -> ExitCode { + #[cfg(target_os = "macos")] + if let Some(exit_code) = run_permission_prompt_helper() { + return exit_code; + } + run() +} + +#[cfg(target_os = "macos")] +fn run_permission_prompt_helper() -> Option { + let (exit_code, response) = agent_desktop_macos::permission_prompt_helper_from_env()?; + let stdout = std::io::stdout(); + let mut writer = BufWriter::new(stdout.lock()); + let result = writer + .write_all(response.as_bytes()) + .and_then(|()| writer.write_all(b"\n")) + .and_then(|()| writer.flush()); + Some(match result { + Ok(()) => ExitCode::from(exit_code), + Err(error) => report_output_failure(error), + }) +} + +fn run() -> ExitCode { let mut cli = match Cli::try_parse() { Ok(c) => c, Err(e) => { @@ -44,15 +60,22 @@ fn main() { e.kind(), clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion ) { - e.exit(); + return match e.print() { + Ok(()) => ExitCode::SUCCESS, + Err(write_err) => report_output_failure(write_err), + }; } let msg = e.to_string(); let first_line = msg.lines().next().unwrap_or("parse error"); - emit_response(&Response::err( + let error = AppError::invalid_input(diagnostic::bounded_text(first_line, 512)); + return if let Err(write_err) = emit_response(&Response::err( "unknown", - ErrorPayload::new("INVALID_ARGS", first_line), - )); - std::process::exit(2); + ErrorPayload::from_app_error(&error), + )) { + report_output_failure(write_err) + } else { + ExitCode::from(EXIT_INVALID_ARGS) + }; } }; @@ -61,17 +84,20 @@ fn main() { let cmd = match cli.command.take() { Some(c) => c, None => { - Cli::command().print_help().unwrap_or(()); - std::process::exit(0); + return match Cli::command().print_help() { + Ok(()) => ExitCode::SUCCESS, + Err(write_err) => report_output_failure(write_err), + }; } }; let cmd_name = cmd.name(); match cmd { - Commands::Version => { - finish(cmd_name, agent_desktop_core::commands::version::execute()); - } + Commands::Version => finish( + cmd_name, + agent_desktop_core::commands::version::execute().map_err(pre_dispatch_error), + ), Commands::Skills(a) => { let result = match a.action.unwrap_or(SkillsAction::List) { SkillsAction::List => agent_desktop_core::commands::skills::list(), @@ -84,18 +110,20 @@ fn main() { }, ), }; - finish(cmd_name, result); + finish(cmd_name, result.map_err(pre_dispatch_error)) } cmd => { - let wait_selector = build_wait_selector(&cli); + let wait_selector = match build_wait_selector(&cli) { + Ok(wait_selector) => wait_selector, + Err(error) => return finish(cmd_name, Err(error)), + }; let session_id = match resolve_active_session( cli.session.as_deref(), std::env::var("AGENT_DESKTOP_SESSION").ok().as_deref(), ) { Ok(session_id) => session_id, Err(err) => { - finish(cmd_name, Err(err)); - return; + return finish(cmd_name, Err(pre_dispatch_error(err))); } }; let context = match CommandContext::new(session_id, cli.trace, cli.trace_strict) { @@ -103,38 +131,54 @@ fn main() { .with_headed(cli.headed) .with_wait_selector(wait_selector.clone()), Err(err) => { - finish(cmd_name, Err(err)); - return; + return finish(cmd_name, Err(pre_dispatch_error(err))); } }; if let Some(wait) = wait_selector.as_ref() { - if let Err(err) = validate_wait_for_command(cmd_name, wait) { - finish(cmd_name, Err(err)); - return; + if let Err(err) = validate_wait_for_command(&cmd, wait) { + return finish(cmd_name, Err(err)); } } - run_with_adapter(cmd, cmd_name, &context); + run_with_adapter(cmd, cmd_name, &context) } } } -fn build_wait_selector(cli: &Cli) -> Option { - let (query_raw, gone) = cli +fn build_wait_selector(cli: &Cli) -> Result, AppError> { + let query = cli + .post_action_wait .wait_for .as_ref() .map(|raw| (raw, false)) - .or_else(|| cli.wait_for_gone.as_ref().map(|raw| (raw, true)))?; - Some(WaitSelector { + .or_else(|| { + cli.post_action_wait + .wait_for_gone + .as_ref() + .map(|raw| (raw, true)) + }); + let Some((query_raw, gone)) = query else { + if cli.post_action_wait.wait_timeout.is_some() { + return Err(AppError::invalid_input_with_suggestion( + "--wait-timeout requires --wait-for or --wait-for-gone", + "Add a selector wait flag or remove --wait-timeout", + )); + } + return Ok(None); + }; + Ok(Some(WaitSelector { query_raw: query_raw.clone(), gone, - timeout_ms: cli.wait_timeout, - }) + timeout_ms: cli.post_action_wait.wait_timeout.unwrap_or(30_000), + })) } -fn validate_wait_for_command(cmd_name: &str, wait: &WaitSelector) -> Result<(), AppError> { - if !WAIT_SUPPORTED.contains(&cmd_name) { +fn validate_wait_for_command(cmd: &Commands, wait: &WaitSelector) -> Result<(), AppError> { + if !cmd.supports_post_action_wait() { return Err(AppError::invalid_input_with_suggestion( - format!("Command '{cmd_name}' does not support --wait-for or --wait-for-gone"), + format!( + "Command '{}' does not support --wait-for or --wait-for-gone", + cmd.name() + ), "Use snapshot --wait-for \"\" or a supported ref action (click, type, …).", )); } @@ -142,60 +186,74 @@ fn validate_wait_for_command(cmd_name: &str, wait: &WaitSelector) -> Result<(), Ok(()) } -fn run_with_adapter(cmd: Commands, cmd_name: &str, context: &CommandContext) { +fn run_with_adapter(cmd: Commands, cmd_name: &str, context: &CommandContext) -> ExitCode { let adapter = build_adapter(); - let report = adapter.permission_report(); + let adapter: &dyn agent_desktop_core::PlatformAdapter = &adapter; + let report = if command_policy::requires_permission_report(&cmd) { + match agent_desktop_core::Deadline::standard() + .map_err(AppError::from) + .and_then(|deadline| adapter.permission_report(deadline).map_err(AppError::from)) + { + Ok(report) => report, + Err(err) => return finish(cmd_name, Err(pre_dispatch_error(err))), + } + } else { + agent_desktop_core::PermissionReport::default() + }; if let Err(err) = command_policy::preflight(&cmd, &report) { - finish(cmd_name, Err(err)); - return; + return finish(cmd_name, Err(err)); } - let result = dispatch::dispatch(cmd, &adapter, &report, context); - finish(cmd_name, result); + let result = dispatch::dispatch(cmd, adapter, &report, context); + finish(cmd_name, result) } -fn finish(cmd_name: &str, result: Result) { +fn pre_dispatch_error(error: AppError) -> AppError { + match error { + AppError::Adapter(mut source) => { + source.disposition = DeliverySemantics::not_delivered(); + source.into() + } + other => AdapterError::new(ErrorCode::Internal, other.to_string()) + .with_disposition(DeliverySemantics::not_delivered()) + .into(), + } +} + +fn finish(cmd_name: &str, result: Result) -> ExitCode { match result { Ok(data) => { - emit_response(&Response::ok(cmd_name, data)); - std::process::exit(0); + if let Err(write_err) = emit_response(&Response::ok(cmd_name, data)) { + return report_output_failure(write_err); + } + ExitCode::SUCCESS } Err(e) => { - emit_response(&Response::err( + if let Err(write_err) = emit_response(&Response::err( cmd_name, agent_desktop_core::ErrorPayload::from_app_error(&e), - )); - std::process::exit(1); + )) { + return report_output_failure(write_err); + } + ExitCode::FAILURE } } } -fn emit_response(response: &Response) { - match serde_json::to_value(response) { - Ok(value) => emit_json(&value), - Err(err) => emit_json(&serde_json::json!({ - "version": ENVELOPE_VERSION, - "ok": false, - "command": "internal", - "error": { - "code": "INTERNAL", - "message": format!("Failed to serialize response: {err}") - } - })), - } +fn report_output_failure(write_err: std::io::Error) -> ExitCode { + eprintln!("agent-desktop: failed to write response to stdout: {write_err}"); + ExitCode::FAILURE } -fn emit_json(value: &serde_json::Value) { +fn emit_response(response: &Response) -> std::io::Result<()> { let stdout = std::io::stdout(); let mut writer = BufWriter::new(stdout.lock()); - if serde_json::to_writer(&mut writer, value).is_err() { - return; - } - let _ = writer.write_all(b"\n"); - let _ = writer.flush(); + serde_json::to_writer(&mut writer, response).map_err(std::io::Error::other)?; + writer.write_all(b"\n")?; + writer.flush() } -fn build_adapter() -> impl agent_desktop_core::adapter::PlatformAdapter { +fn build_adapter() -> impl agent_desktop_core::PlatformAdapter { #[cfg(target_os = "macos")] { agent_desktop_macos::MacOSAdapter::new() @@ -226,3 +284,7 @@ fn init_tracing(verbose: bool) { .with_ansi(false) .init(); } + +#[cfg(test)] +#[path = "tests/main_tests.rs"] +mod main_tests; diff --git a/src/tests/cli_process.rs b/src/tests/cli_process.rs new file mode 100644 index 0000000..62aa968 --- /dev/null +++ b/src/tests/cli_process.rs @@ -0,0 +1,82 @@ +use std::process::Command; + +fn binary() -> Command { + Command::new(env!("CARGO_BIN_EXE_agent-desktop")) +} + +#[test] +fn clap_help_returns_success_without_structured_error_noise() { + let output = binary().arg("--help").output().expect("binary starts"); + + assert!(output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).contains("Usage:")); + assert!(output.stderr.is_empty()); +} + +#[test] +fn clap_parse_failure_is_structured_on_stdout_with_exit_two() { + let output = binary() + .arg("--definitely-not-a-real-flag") + .output() + .expect("binary starts"); + let envelope: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("stdout is one JSON envelope"); + + assert_eq!(output.status.code(), Some(2)); + assert_eq!(envelope["ok"], false); + assert_eq!(envelope["command"], "unknown"); + assert_eq!(envelope["error"]["code"], "INVALID_ARGS"); + assert_eq!( + envelope["error"]["disposition"]["delivery"], + "not_delivered" + ); + assert_eq!(envelope["error"]["disposition"]["retry"], "safe"); +} + +#[test] +fn version_has_exact_package_identity() { + let output = binary().arg("version").output().expect("binary starts"); + let envelope: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("stdout is one JSON envelope"); + + assert!(output.status.success()); + assert_eq!(envelope["ok"], true); + assert_eq!(envelope["data"]["version"], env!("CARGO_PKG_VERSION")); +} + +#[cfg(target_os = "macos")] +#[test] +fn malformed_permission_helper_invocation_bypasses_clap_and_tracing() { + const HELPER_ENV: [&str; 6] = [ + "AGENT_DESKTOP_PERMISSION_HELPER", + "AGENT_DESKTOP_PERMISSION_OPERATION", + "AGENT_DESKTOP_PERMISSION_TOKEN", + "AGENT_DESKTOP_PERMISSION_PARENT_PID", + "AGENT_DESKTOP_PERMISSION_PARENT_INSTANCE", + "AGENT_DESKTOP_PERMISSION_EXECUTABLE", + ]; + + let mut command = binary(); + command.arg("--definitely-not-a-real-flag"); + for name in HELPER_ENV { + command.env_remove(name); + } + let output = command + .env("AGENT_DESKTOP_PERMISSION_HELPER", "invalid") + .env("RUST_LOG", "trace") + .output() + .expect("binary starts"); + + assert_eq!(output.status.code(), Some(2)); + assert!(output.stderr.is_empty()); + assert_eq!( + output.stdout.iter().filter(|byte| **byte == b'\n').count(), + 1 + ); + assert_eq!(output.stdout.last(), Some(&b'\n')); + let response: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("stdout is one JSON line"); + assert_eq!(response["version"], 1); + assert_eq!(response["ok"], false); + assert_eq!(response["error"], "invalid_helper_invocation"); +} diff --git a/src/tests/conformance.rs b/src/tests/conformance.rs index 33aa7ef..15af140 100644 --- a/src/tests/conformance.rs +++ b/src/tests/conformance.rs @@ -1,19 +1,22 @@ +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, PlatformAdapter, SystemOps}; use agent_desktop_core::{ - action_request::ActionRequest, - action_result::ActionResult, - adapter::{LiveElement, NativeHandle, PlatformAdapter, SnapshotSurface}, - capability, - element_state::ElementState, - error::{AdapterError, ErrorCode}, - node::Rect, - refs::RefEntry, + ActionRequest, ActionResult, AdapterError, Deadline, ElementState, ErrorCode, + IdentifierEvidence, InteractionLease, LiveElement, LiveIdentity, LocatorField, NativeHandle, + Rect, RefCapabilities, RefEntry, RefEntryIdentity, RefGeometry, RefProcess, RefScope, + RefSource, SnapshotSurface, capability, }; use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Duration; #[path = "../../tests/conformance/ref_action_contract.rs"] mod ref_action_contract; +#[path = "../../tests/conformance/window_identity_contract.rs"] +mod window_identity_contract; + +#[cfg(target_os = "macos")] +#[path = "../../tests/conformance/macos_notification_contract.rs"] +mod macos_notification_contract; + struct ContractAdapter { resolve: ResolveMode, live_bounds: Option, @@ -28,50 +31,109 @@ enum ResolveMode { Ambiguous, } -impl PlatformAdapter for ContractAdapter { - fn resolve_element_strict(&self, _entry: &RefEntry) -> Result { - self.resolve() - } - - fn resolve_element_strict_with_timeout( +impl ObservationOps for ContractAdapter { + fn resolve_element_strict( &self, _entry: &RefEntry, - _timeout: Duration, + _deadline: Deadline, ) -> Result { self.resolve() } - fn get_live_element(&self, _handle: &NativeHandle) -> Result { + fn get_live_element( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result { Ok(LiveElement { - state: Some(ElementState { + identity: LiveIdentity { + name: LocatorField::Known("OK".into()), + description: LocatorField::Absent, + identifiers: IdentifierEvidence::absent(), + }, + state: ElementState { role: "button".into(), states: vec![], value: self.live_value.clone(), - }), + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), + }, + states_complete: true, bounds: self.live_bounds, - available_actions: Some(vec![capability::CLICK.into()]), + available_actions: vec![capability::CLICK.into()], }) } - fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_live_state( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result, AdapterError> { Ok(Some(ElementState { role: "button".into(), states: vec![], value: self.live_value.clone(), + enabled: Some(true), + hidden: Some(false), + offscreen: Some(false), })) } - fn get_live_value(&self, _handle: &NativeHandle) -> Result, AdapterError> { + fn get_live_value( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result, AdapterError> { Ok(self.live_value.clone()) } + fn get_element_bounds( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result, AdapterError> { + Ok(self.live_bounds) + } + + fn get_live_actions( + &self, + _handle: &NativeHandle, + _deadline: Deadline, + ) -> Result>, AdapterError> { + Ok(Some(vec![capability::CLICK.into()])) + } + + fn hit_test( + &self, + _handle: &NativeHandle, + _point: agent_desktop_core::Point, + _deadline: Deadline, + ) -> Result { + Ok(agent_desktop_core::HitTestResult::ReachesTarget) + } +} + +impl ActionOps for ContractAdapter { fn execute_action( &self, _handle: &NativeHandle, _request: ActionRequest, + _lease: &InteractionLease, ) -> Result { self.dispatches.fetch_add(1, Ordering::SeqCst); - Ok(ActionResult::new("click")) + Ok(ActionResult::delivered_unverified("click")) + } +} + +impl InputOps for ContractAdapter {} + +impl SystemOps for ContractAdapter { + fn acquire_interaction_lease( + &self, + deadline: Deadline, + ) -> Result { + InteractionLease::guarded(deadline, ()) } } @@ -101,25 +163,80 @@ impl ContractAdapter { fn entry(bounds: Rect) -> RefEntry { RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - description: None, - states: vec![], - bounds: Some(bounds), - bounds_hash: Some(bounds.bounds_hash()), - available_actions: vec![capability::CLICK.into()], - source_app: None, - source_window_id: None, - source_window_title: None, - source_surface: SnapshotSurface::Window, - root_ref: None, - path_is_absolute: true, - path: Default::default(), + process: RefProcess { + pid: agent_desktop_core::ProcessId::new(1), + process_instance: Some("contract-process".into()), + }, + identity: RefEntryIdentity { + role: "button".into(), + name: Some("OK".into()), + value: None, + description: None, + native_id: None, + }, + geometry: RefGeometry { + bounds: Some(bounds), + bounds_hash: bounds.bounds_hash(), + }, + capabilities: RefCapabilities { + states: vec![], + available_actions: vec![capability::CLICK.into()], + }, + source: RefSource { + source_app: None, + source_window_id: None, + source_window_title: None, + source_window_bounds_hash: None, + source_surface: SnapshotSurface::Window, + }, + scope: RefScope { + root_ref: None, + path_is_absolute: true, + path: Default::default(), + }, } } +#[test] +fn platform_adapter_exposes_all_capability_methods() { + fn exercise(adapter: &dyn PlatformAdapter) { + let deadline = Deadline::standard().unwrap(); + let _ = adapter.list_windows( + &agent_desktop_core::WindowFilter { + focused_only: false, + app: None, + }, + deadline, + ); + let _ = adapter.list_apps(deadline); + let _ = adapter.permission_report(deadline); + let _ = adapter.get_clipboard_content(agent_desktop_core::ClipboardFormat::Text, deadline); + let handle = adapter + .resolve_element_strict(&entry(Rect { + x: 0.0, + y: 0.0, + width: 1.0, + height: 1.0, + }), deadline) + .expect("ObservationOps::resolve_element_strict must be reachable through &dyn PlatformAdapter"); + let lease = adapter.acquire_interaction_lease(deadline).unwrap(); + let _ = adapter.execute_action( + &handle, + ActionRequest::headless(agent_desktop_core::Action::Click), + &lease, + ); + } + exercise(&ContractAdapter::new( + ResolveMode::Ok, + Some(Rect { + x: 0.0, + y: 0.0, + width: 1.0, + height: 1.0, + }), + )); +} + #[test] fn adapter_contract_dispatches_when_live_identity_moved() { let snapshot_bounds = Rect { diff --git a/src/tests/main_tests.rs b/src/tests/main_tests.rs new file mode 100644 index 0000000..a4c2b27 --- /dev/null +++ b/src/tests/main_tests.rs @@ -0,0 +1,18 @@ +use agent_desktop_core::{AdapterError, AppError, DeliveryDisposition, RetryDisposition}; + +#[test] +fn pre_dispatch_failures_are_always_safe_to_retry() { + for error in [ + AppError::from(AdapterError::internal("trace setup failed")), + AppError::from(std::io::Error::other("read failed")), + ] { + let AppError::Adapter(error) = crate::pre_dispatch_error(error) else { + panic!("pre-dispatch error must be normalized to AdapterError"); + }; + assert_eq!(error.disposition.retry(), RetryDisposition::Safe); + assert_eq!( + error.disposition.delivery(), + DeliveryDisposition::NotDelivered + ); + } +} diff --git a/src/tests/snapshot_test.rs b/src/tests/snapshot_test.rs index d074dfd..ddb42db 100644 --- a/src/tests/snapshot_test.rs +++ b/src/tests/snapshot_test.rs @@ -1,227 +1,148 @@ -/// Integration tests for the snapshot command. -/// -/// These tests require macOS with Accessibility permissions granted to the -/// terminal running the tests. They are skipped automatically on other -/// platforms or when the binary is not built. #[cfg(test)] mod tests { + use std::path::PathBuf; use std::process::Command; - fn agent_desktop_bin() -> std::path::PathBuf { - let mut p = std::env::current_exe().unwrap(); - p.pop(); - p.pop(); - p.push("agent-desktop"); - p + fn agent_desktop_bin() -> PathBuf { + let mut path = std::env::current_exe().expect("test executable path must be available"); + path.pop(); + path.pop(); + path.push("agent-desktop"); + assert!( + path.is_file(), + "agent-desktop test binary is missing at {}; build the binary before running tests", + path.display() + ); + path } + fn run(args: &[&str]) -> serde_json::Value { + let output = Command::new(agent_desktop_bin()) + .args(args) + .output() + .expect("failed to run agent-desktop"); + serde_json::from_slice(&output.stdout).unwrap_or_else(|error| { + panic!( + "agent-desktop output is not JSON: {error}; stderr={}", + String::from_utf8_lossy(&output.stderr) + ) + }) + } + + /// Guards the native AX-to-CG window identity bridge used by snapshots. #[test] #[cfg(target_os = "macos")] #[ignore = "requires Accessibility permissions and running macOS apps"] - fn snapshot_finder_returns_non_empty_tree() { - let bin = agent_desktop_bin(); - let output = Command::new(&bin) - .args(["snapshot", "--app", "Finder"]) - .output() - .expect("failed to run agent-desktop"); + fn snapshot_resolves_a_window_id_reported_by_list_windows() { + let list = run(&["list-windows", "--app", "Finder"]); + let ids: Vec<&str> = list["data"] + .as_array() + .into_iter() + .flatten() + .filter_map(|window| window["id"].as_str()) + .collect(); + if ids.is_empty() { + eprintln!( + "SKIP snapshot_resolves_a_window_id_reported_by_list_windows: Finder has no windows" + ); + return; + } - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = - serde_json::from_str(&stdout).expect("output is not valid JSON"); - - assert_eq!(json["ok"], true); - assert!(json["data"]["ref_count"].as_u64().unwrap_or(0) > 0); + let snapshot = run(&["snapshot", "--app", "Finder"]); + assert_eq!( + snapshot["ok"], true, + "snapshot must resolve a live window, got error {}", + snapshot["error"]["code"] + ); + let resolved = snapshot["data"]["window"]["id"].as_str().unwrap_or(""); + assert!( + ids.contains(&resolved), + "snapshot window id {resolved:?} must be reported by list-windows {ids:?}" + ); } + /// Guards consistent accessible-name computation across snapshot and find. #[test] #[cfg(target_os = "macos")] #[ignore = "requires Accessibility permissions and running macOS apps"] - fn snapshot_textedit_returns_refs() { - let bin = agent_desktop_bin(); - let output = Command::new(&bin) - .args(["snapshot", "--app", "TextEdit"]) - .output() - .expect("failed to run agent-desktop"); + fn find_by_name_matches_the_element_that_reports_that_name() { + let by_role = run(&["find", "--app", "Finder", "--role", "button", "--first"]); + let matched = &by_role["data"]["match"]; + let (Some(ref_id), Some(name)) = (matched["ref_id"].as_str(), matched["name"].as_str()) + else { + eprintln!( + "SKIP find_by_name_matches_the_element_that_reports_that_name: no named button" + ); + return; + }; + if name.is_empty() || name.starts_with("(unnamed") { + eprintln!( + "SKIP find_by_name_matches_the_element_that_reports_that_name: unusable name" + ); + return; + } - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = - serde_json::from_str(&stdout).expect("output is not valid JSON"); + let by_name = run(&[ + "find", "--app", "Finder", "--role", "button", "--name", name, "--first", + ]); + assert_eq!( + by_name["data"]["match"]["ref_id"].as_str(), + Some(ref_id), + "find --name {name:?} must resolve the element that reported that name" + ); + } - assert_eq!(json["ok"], true); + /// Guards strict identity re-resolution for a freshly produced ref. + #[test] + #[cfg(target_os = "macos")] + #[ignore = "requires Accessibility permissions and running macOS apps"] + fn a_ref_from_find_reresolves_through_get() { + let found = run(&["find", "--app", "Finder", "--role", "button", "--first"]); + let Some(ref_id) = found["data"]["match"]["ref_id"].as_str() else { + eprintln!("SKIP a_ref_from_find_reresolves_through_get: no button found"); + return; + }; + + let got = run(&["get", ref_id, "--property", "role"]); + assert_eq!( + got["ok"], true, + "a ref from find must re-resolve through get, got error {}", + got["error"]["code"] + ); + } + + #[test] + fn real_app_regression_gate_stays_registered() { + let source = include_str!("snapshot_test.rs"); + assert_eq!(source.matches(concat!("#[", "ignore =")).count(), 3); + for name in [ + "snapshot_resolves_a_window_id_reported_by_list_windows", + "find_by_name_matches_the_element_that_reports_that_name", + "a_ref_from_find_reresolves_through_get", + ] { + assert!(source.contains(&format!("fn {name}()"))); + } } #[test] fn version_command_outputs_json() { - let bin = agent_desktop_bin(); - if !bin.exists() { - return; - } - let output = Command::new(&bin) - .args(["version"]) - .output() - .expect("failed to run agent-desktop"); - - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = - serde_json::from_str(&stdout).expect("output is not valid JSON"); - + let json = run(&["version"]); assert_eq!(json["ok"], true); assert!(json["data"]["version"].is_string()); } - #[test] - #[cfg(target_os = "macos")] - #[ignore = "requires Accessibility permissions and running macOS apps"] - fn snapshot_skeleton_returns_shallow_tree_with_children_count() { - let bin = agent_desktop_bin(); - let output = Command::new(&bin) - .args(["snapshot", "--app", "Finder", "--skeleton", "-i"]) - .output() - .expect("failed to run agent-desktop"); - - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = - serde_json::from_str(&stdout).expect("output is not valid JSON"); - - assert_eq!(json["ok"], true); - let tree = &json["data"]["tree"]; - let max_depth = find_max_depth(tree, 0); - assert!( - max_depth <= 4, - "skeleton must clamp to depth ~3, got depth {max_depth}" - ); - } - - #[test] - #[cfg(target_os = "macos")] - #[ignore = "requires Accessibility permissions and running macOS apps"] - fn snapshot_skeleton_refresh_does_not_accumulate_stale_refs() { - let bin = agent_desktop_bin(); - let run = |extra: &[&str]| { - let mut args = vec!["snapshot", "--app", "Finder", "--skeleton", "-i"]; - args.extend_from_slice(extra); - Command::new(&bin) - .args(&args) - .output() - .expect("failed to run agent-desktop") - }; - - let first = run(&[]); - let first_json: serde_json::Value = - serde_json::from_str(&String::from_utf8_lossy(&first.stdout)).unwrap(); - let first_count = first_json["data"]["ref_count"].as_u64().unwrap_or(0); - - let second = run(&[]); - let second_json: serde_json::Value = - serde_json::from_str(&String::from_utf8_lossy(&second.stdout)).unwrap(); - let second_count = second_json["data"]["ref_count"].as_u64().unwrap_or(0); - - assert_eq!( - first_count, second_count, - "repeated skeleton refresh must produce identical ref_count (no accumulation)" - ); - } - #[test] fn snapshot_invalid_root_ref_format_returns_invalid_args() { - let bin = agent_desktop_bin(); - if !bin.exists() { - return; - } - let output = Command::new(&bin) - .args(["snapshot", "--app", "Finder", "--root", "bad-ref"]) - .output() - .expect("failed to run agent-desktop"); - - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = - serde_json::from_str(&stdout).expect("output is not valid JSON"); - + let json = run(&["snapshot", "--app", "Finder", "--root", "bad-ref"]); assert_eq!(json["ok"], false); - assert_eq!( - json["error"]["code"], "INVALID_ARGS", - "malformed --root must return INVALID_ARGS, got: {}", - json["error"]["code"] - ); - } - - #[test] - #[cfg(target_os = "macos")] - #[ignore = "requires Accessibility permissions and running macOS apps"] - fn snapshot_root_drill_returns_non_empty_subtree() { - let bin = agent_desktop_bin(); - let skeleton_out = Command::new(&bin) - .args(["snapshot", "--app", "Finder", "--skeleton", "-i"]) - .output() - .expect("failed to run agent-desktop"); - - let skeleton_json: serde_json::Value = - serde_json::from_str(&String::from_utf8_lossy(&skeleton_out.stdout)).unwrap(); - assert_eq!(skeleton_json["ok"], true); - - let first_ref = first_ref_id(&skeleton_json["data"]["tree"]); - let Some(ref_id) = first_ref else { - return; - }; - - let drill_out = Command::new(&bin) - .args(["snapshot", "--app", "Finder", "--root", &ref_id, "-i"]) - .output() - .expect("failed to run agent-desktop"); - - let drill_json: serde_json::Value = - serde_json::from_str(&String::from_utf8_lossy(&drill_out.stdout)).unwrap(); - - assert_eq!(drill_json["ok"], true); - assert!( - drill_json["data"]["ref_count"].as_u64().unwrap_or(0) > 0, - "drill-down must return refs" - ); - } - - fn find_max_depth(node: &serde_json::Value, depth: usize) -> usize { - let children = match node.get("children").and_then(|c| c.as_array()) { - Some(c) if !c.is_empty() => c, - _ => return depth, - }; - children - .iter() - .map(|c| find_max_depth(c, depth + 1)) - .max() - .unwrap_or(depth) - } - - fn first_ref_id(node: &serde_json::Value) -> Option { - if let Some(r) = node.get("ref_id").and_then(|v| v.as_str()) { - return Some(r.to_string()); - } - if let Some(children) = node.get("children").and_then(|c| c.as_array()) { - for child in children { - if let Some(r) = first_ref_id(child) { - return Some(r); - } - } - } - None + assert_eq!(json["error"]["code"], "INVALID_ARGS"); } #[test] fn list_apps_on_non_macos_errors_gracefully() { #[cfg(not(target_os = "macos"))] { - let bin = agent_desktop_bin(); - if !bin.exists() { - return; - } - let output = Command::new(&bin) - .args(["list-apps"]) - .output() - .expect("failed to run agent-desktop"); - - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = - serde_json::from_str(&stdout).expect("output is not valid JSON"); - + let json = run(&["list-apps"]); assert_eq!(json["ok"], false); assert_eq!(json["error"]["code"], "PLATFORM_NOT_SUPPORTED"); } diff --git a/tests/conformance/README.md b/tests/conformance/README.md index 6a03098..4e14c09 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -43,3 +43,18 @@ Add adapter-specific integration fixtures, but keep expected errors and JSON shapes identical. Prefer semantic platform actions first (`AXPress`, UIA Invoke/Value/Selection patterns, AT-SPI actions). Coordinate input is a lower confidence fallback and must remain explicit in policy or command choice. + +## Windows Private-Storage Runtime Gate + +Windows support must not be enabled from cross-compilation evidence alone. A +native Windows integration suite must continuously rename or replace every +ancestor of the private state directory while `open`, bounded read, append, +lock, and atomic replace operations run. Each operation must either access the +intended handle-bound file or fail closed; it must never follow a reparse point +or escape the guarded ancestor chain. + +The same gate must verify owner-only protected DACL creation and rejection, +hardlink rejection, local-versus-remote storage detection, drive and UNC +verbatim path encoding beyond `MAX_PATH`, and atomic replacement durability. +Static Windows target checks are required before this suite, but are not a +substitute for the native race test. diff --git a/tests/conformance/macos_notification_contract.rs b/tests/conformance/macos_notification_contract.rs new file mode 100644 index 0000000..248400a --- /dev/null +++ b/tests/conformance/macos_notification_contract.rs @@ -0,0 +1,56 @@ +use agent_desktop_core::SystemOps; +use agent_desktop_core::{ + Deadline, DeliverySemantics, DismissAllNotificationsRequest, DismissNotificationRequest, + ErrorCode, InteractionLease, InteractionPolicy, NotificationActionRequest, + NotificationIdentity, +}; + +#[test] +fn notification_mutations_fail_before_foreground_access_under_headless_policy() { + let adapter = agent_desktop_macos::MacOSAdapter::new(); + let deadline = Deadline::standard().unwrap(); + let lease = InteractionLease::guarded(deadline, ()).unwrap(); + let identity = NotificationIdentity { + expected_app: Some("Messages".into()), + expected_title: Some("New message".into()), + }; + let policy = InteractionPolicy::headless(); + let errors = [ + SystemOps::dismiss_notification( + &adapter, + DismissNotificationRequest { + index: 0, + app_filter: None, + identity: &identity, + policy, + }, + &lease, + ) + .unwrap_err(), + SystemOps::dismiss_all_notifications( + &adapter, + DismissAllNotificationsRequest { + app_filter: None, + policy, + }, + &lease, + ) + .unwrap_err(), + SystemOps::notification_action( + &adapter, + NotificationActionRequest { + index: 0, + identity: &identity, + action_name: "Reply", + policy, + }, + &lease, + ) + .unwrap_err(), + ]; + + for error in errors { + assert_eq!(error.code, ErrorCode::PolicyDenied); + assert_eq!(error.disposition, DeliverySemantics::not_delivered()); + } +} diff --git a/tests/conformance/ref_action_contract.rs b/tests/conformance/ref_action_contract.rs index ce13fbe..d1cf9c1 100644 --- a/tests/conformance/ref_action_contract.rs +++ b/tests/conformance/ref_action_contract.rs @@ -1,9 +1,7 @@ use agent_desktop_core::{ - adapter::PlatformAdapter, + PlatformAdapter, RefEntry, RefMap, RefStore, commands::{click, helpers::RefArgs, wait}, context::CommandContext, - refs::{RefEntry, RefMap}, - refs_store::RefStore, }; use std::sync::{Mutex, MutexGuard}; @@ -19,6 +17,7 @@ pub fn run_click_command( RefArgs { ref_id: "@e1".into(), snapshot_id: Some(snapshot_id), + timeout_ms: None, }, adapter, &context, @@ -40,7 +39,7 @@ pub fn run_wait_element_command_with_predicate( context: &CommandContext, predicate: WaitPredicate<'_>, ) -> Result { - with_saved_entry(entry, context, |_| { + with_saved_entry(entry, context, |snapshot_id| { wait::execute( wait::WaitArgs { mode: wait::WaitModeArgs { @@ -48,18 +47,18 @@ pub fn run_wait_element_command_with_predicate( element: Some("@e1".into()), window: None, text: None, - menu: false, - menu_closed: false, - notification: false, + surface: None, + event: None, + window_id: None, }, predicate: wait::WaitPredicateArgs { - snapshot_id: None, + snapshot_id: Some(snapshot_id), predicate: Some(predicate.name.into()), value: predicate.value.map(String::from), action: predicate.action.map(String::from), count: None, }, - timeout_ms: 100, + timeout_ms: 250, app: None, }, adapter, @@ -101,7 +100,7 @@ fn with_saved_entry( ) -> Result { let _home = TestHome::new(); let mut refmap = RefMap::new(); - refmap.allocate(entry); + refmap.try_allocate(entry)?; let snapshot_id = RefStore::for_session(context.session_id())?.save_new_snapshot(&refmap)?; run(snapshot_id) } diff --git a/tests/conformance/window_identity_contract.rs b/tests/conformance/window_identity_contract.rs new file mode 100644 index 0000000..4361531 --- /dev/null +++ b/tests/conformance/window_identity_contract.rs @@ -0,0 +1,156 @@ +use agent_desktop_core::{ + ActionOps, AdapterError, Deadline, ErrorCode, InputOps, InteractionLease, ObservationOps, + SystemOps, WindowInfo, WindowOp, WindowState, +}; +use std::sync::Mutex; + +#[path = "../support/noop_ops.rs"] +mod noop_ops; + +struct WindowIdentityAdapter { + windows: Vec, + last_window_op_id: Mutex>, +} + +impl ObservationOps for WindowIdentityAdapter {} + +impl ActionOps for WindowIdentityAdapter {} + +impl InputOps for WindowIdentityAdapter {} + +impl SystemOps for WindowIdentityAdapter { + fn acquire_interaction_lease( + &self, + deadline: Deadline, + ) -> Result { + InteractionLease::guarded(deadline, ()) + } + + fn resolve_window_strict( + &self, + win: &WindowInfo, + _deadline: Deadline, + ) -> Result { + let live = self + .windows + .iter() + .find(|candidate| candidate.id == win.id) + .cloned() + .ok_or_else(|| { + AdapterError::new( + ErrorCode::WindowNotFound, + format!("Window '{}' not found", win.id), + ) + })?; + if live.pid != win.pid || live.process_instance != win.process_instance { + return Err(AdapterError::new( + ErrorCode::WindowNotFound, + format!("Window '{}' identity mismatch", win.id), + )); + } + if !win.title.is_empty() && live.title != win.title { + return Err(AdapterError::new( + ErrorCode::WindowNotFound, + format!("Window '{}' identity mismatch", win.id), + )); + } + Ok(live) + } + + fn window_op( + &self, + win: &WindowInfo, + _op: WindowOp, + lease: &InteractionLease, + ) -> Result<(), AdapterError> { + let resolved = self.resolve_window_strict(win, lease.deadline())?; + *self.last_window_op_id.lock().unwrap() = Some(resolved.id); + Ok(()) + } +} + +fn untitled(id: &str, pid: u32) -> WindowInfo { + WindowInfo { + id: id.into(), + title: "Untitled".into(), + app: "TextEdit".into(), + pid: agent_desktop_core::ProcessId::new(pid), + process_instance: Some(format!("contract-process-{pid}")), + bounds: None, + state: WindowState::default(), + } +} + +fn lease() -> InteractionLease { + InteractionLease::guarded(Deadline::standard().unwrap(), ()).unwrap() +} + +#[test] +fn id_addressed_window_op_targets_matching_id_not_first_title_match() { + let adapter = WindowIdentityAdapter { + windows: vec![untitled("w-1", 10), untitled("w-2", 10)], + last_window_op_id: Mutex::new(None), + }; + let target = untitled("w-2", 10); + + SystemOps::window_op(&adapter, &target, WindowOp::Minimize, &lease()).unwrap(); + + assert_eq!( + *adapter.last_window_op_id.lock().unwrap(), + Some("w-2".into()) + ); +} + +#[test] +fn missing_id_returns_window_not_found() { + let adapter = WindowIdentityAdapter { + windows: vec![untitled("w-1", 10)], + last_window_op_id: Mutex::new(None), + }; + let target = untitled("w-999", 10); + + let err = SystemOps::window_op(&adapter, &target, WindowOp::Minimize, &lease()).unwrap_err(); + + assert_eq!(err.code, ErrorCode::WindowNotFound); +} + +#[test] +fn recycled_id_with_wrong_pid_fails_closed() { + let adapter = WindowIdentityAdapter { + windows: vec![untitled("w-100", 99)], + last_window_op_id: Mutex::new(None), + }; + let target = untitled("w-100", 10); + + let err = adapter + .resolve_window_strict(&target, Deadline::standard().unwrap()) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::WindowNotFound); +} + +#[test] +fn recycled_id_with_same_pid_and_new_process_instance_fails_closed() { + let mut live = untitled("w-100", 10); + live.process_instance = Some("replacement-process".into()); + let adapter = WindowIdentityAdapter { + windows: vec![live], + last_window_op_id: Mutex::new(None), + }; + let target = untitled("w-100", 10); + + let err = adapter + .resolve_window_strict(&target, Deadline::standard().unwrap()) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::WindowNotFound); +} + +#[test] +fn resolve_window_strict_default_is_not_supported() { + let err = noop_ops::NoopAdapter + .resolve_window_strict(&untitled("w-1", 10), Deadline::standard().unwrap()) + .unwrap_err(); + + assert_eq!(err.code, ErrorCode::PlatformNotSupported); +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 0d1949b..812329d 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,87 +1,144 @@ -# End-to-End Tests (real binary vs. real app) +# Native macOS end-to-end tests -These tests drive the **release `agent-desktop` binary** against a **real macOS -application** and verify every effect by **independent observation** — never the -command's own `ok: true`. A command that returns success without producing the -effect is caught, because each check re-reads the UI (status label, element -value, scroll offset, `list-apps`, …) and asserts the observed `before`/`after`. +This suite drives the release `agent-desktop` binary against a real macOS app +and verifies effects through independent accessibility observations. A command +returning `ok: true` is never sufficient when the fixture exposes an observable +status, value, process, window, or surface transition. -This is the layer that mock-adapter unit tests cannot cover: it exercises the -contract against the actual macOS Accessibility API. +## Run the fixture suite + +```bash +AGENT_DESKTOP_E2E_EXCLUSIVE=1 bash tests/e2e/run.sh +``` + +The single prerequisite gate requires: + +- macOS; +- Accessibility permission granted to the terminal or runner; and +- a buildable Swift fixture app. + +`AGENT_DESKTOP_E2E_EXCLUSIVE=1` is a safety acknowledgement that the caller has +made the desktop exclusive, including stopping user input and other automation. +The harness lock prevents a second native harness from starting, but it cannot +stop an unrelated `agent-desktop` process from starting between commands. Do +not set the acknowledgement while the desktop is in use. + +Before taking the desktop lock, `run.sh` builds the CLI, macOS helper, and FFI +library from the current checkout with `--locked` into the canonical repository +`target` directory. It then hashes and copies those exact artifacts to read-only +paths under a private suite directory. The suite uses only those copies plus an +isolated `HOME`, ref/session stores, fixture build, and `TMPDIR`. Every CLI child +runs in its own process group with an absolute timeout and bounded stdout/stderr +capture. Artifact hashes and exact JSON/Clap version identity are checked again +before success is reported. + +The harness then builds and launches `AgentDeskFixture.app`. Once the fixture is +running, a missing control, window, status readout, bounds record, or surface is +a test failure. Scenarios never turn a missing prerequisite into a successful +`SKIP`. Exit `0` means every assertion passed from an uncontaminated run, exit +`1` means a behavioral failure, and exit `2` means the global prerequisite gate +failed. + +Headed cases move the real cursor. Run them on a machine where that is safe. +The fixture process is force-closed by the cleanup trap even when a test fails. + +## Exact ref namespaces + +Every non-count `find` returns two inseparable values: + +```json +{"ref_id":"@e12","snapshot_id":"..."} +``` + +`lib.sh` carries that pair as one target and every ref action, `get`, `is`, and +element `wait` passes the exact `--snapshot` value. The harness deliberately +runs unrelated observations between `find` and `get` to prove that a ref does +not depend on the mutable latest-snapshot pointer. Do not add a helper that +returns a bare `@eN`. ## Layout -| Path | Role | -|------|------| -| `tests/fixture-app/AgentDeskFixture.swift` | SwiftUI + AppKit fixture exposing a fixed, diverse AX surface | -| `tests/fixture-app/build.sh` | Compiles the fixture into `build/AgentDeskFixture.app` | -| `tests/e2e/run.sh` | The harness: launches the fixture, runs the binary, asserts by observation | +| Path | Responsibility | +|---|---| +| `run.sh` | Global gate, fixture lifecycle, semantic suite orchestration | +| `lib.sh` | Fail-closed assertions and exact `{ref_id,snapshot_id}` target helpers | +| `scenarios/observation.sh` | Snapshots, locators, namespace pinning, strict twins | +| `scenarios/interaction.sh` | Headless/headed actions and exact-once effects | +| `scenarios/acceptance.sh` | Named AE1-AE7 acceptance contract | +| `scenarios/reliability.sh` | Stale refs, waits, drill-down, sessions | +| `scenarios/surfaces.sh` | Sheets, menus, drag, disclosure | +| `scenarios/trace_performance.sh` | Trace artifacts, redaction, timings, cleanup | +| `permission-contract.sh` | Deterministic AE5 mapping tests without mutating TCC | +| `electron-live.sh` | Opt-in installed Electron/Chromium app measurement | -The fixture is a **principal-engineer-grade slice of real UI**, not a target -tuned to make the CLI pass. It deliberately mixes AX-actionable native AppKit -controls (`NSSlider`, `NSStepper`) with gesture-only and ambiguous patterns. **A -failure here is a finding about the CLI or the harness — never a reason to edit -the fixture so the CLI passes.** +Every shell file stays below the repository's 400-line limit. -## Running +## AE1-AE7 acceptance map + +`scenarios/acceptance.sh` names and fails closed on every plan example: + +| Example | Native fixture assertion | +|---|---| +| AE1 | An addressable button whose live AX frame is zero reports `visible=false`. | +| AE2 | CLI and native release-FFI consumers wait for a button enabled after 800 ms, dispatch exactly once, and perform one immediate check under timeout zero without a late effect. | +| AE3 | A permanently disabled button with `--timeout-ms 2000` returns `TIMEOUT`, `details.kind=actionability_timeout`, and `details.last_report` near 2 s. | +| AE4 | Two windows with the same title receive distinct ids; focusing the second id focuses that exact window. | +| AE5 | Deterministic tests prove prompt isolation and nonprompting Automation probes; when the runner already has denied Automation TCC, a native headed Notification Center operation must return `PERM_DENIED`. Other TCC states are explicitly logged as unavailable rather than claimed as exercised. | +| AE6 | One batch captures a pre-action baseline, clicks open a sheet, and reports `surface_appeared` without naming the surface title. | +| AE7 | The same disabled action with no timeout flag returns the structured timeout near the untouched 5 s default. | + +AE2, AE3, and AE7 run in both headless and headed policy modes. Elapsed-time +tolerances account for process launch and scheduler noise but are narrow enough +to detect a missing wait, the wrong default, or an unbounded overrun. Delayed +action status is observed after a settle interval so a duplicate late dispatch +cannot pass. + +`permission-contract.sh` is intentionally deterministic. It tests the expired +deadline, helper result, and Automation-not-required contracts without calling +native prompt APIs, resetting TCC, or assuming the developer's consent state. + +## Installed Electron/Chromium measurement + +The live harness is opt-in and is never called by `run.sh` or unprivileged CI: ```bash -cargo build --release # the harness uses target/release/agent-desktop -bash tests/e2e/run.sh +AGENT_DESKTOP_E2E_EXCLUSIVE=1 bash tests/e2e/electron-live.sh --app Slack +AGENT_DESKTOP_E2E_EXCLUSIVE=1 bash tests/e2e/electron-live.sh \ + --app "Visual Studio Code" \ + --baseline-binary /path/to/previous/agent-desktop \ + --out /tmp/vscode-ax.json ``` -Requirements: +The app must already be installed, running, and exposing at least one window. +The harness does not launch, focus, click, type into, or close the user's app. +If any prerequisite is unavailable, invocation fails instead of emitting a +successful partial benchmark. -- macOS with **Accessibility permission** granted to the terminal running the - harness (System Settings → Privacy & Security → Accessibility). -- The harness builds the fixture `.app` automatically if it is missing. -- `--headed` checks move the real cursor; run on a machine where that is OK. +It performs exactly five warmup pairs and 31 measured release-binary pairs of +`find --role button --first`. When `--baseline-binary` is supplied, immutable +baseline (A) and current (B) binaries run in balanced AB/BA order against one +exact process generation and window inventory, with separate `HOME` +directories. State is checked before and after every pair. The v3 JSON schema +includes nearest-rank wall and CPU p50/p95, paired current-minus-baseline +deltas, success and correctness rates, AX traversal/read statistics when a +binary emits them, renderer activation counts, SHA-256/version identities, raw +samples, and exact-snapshot re-resolution checks. It deliberately omits RSS: +`RUSAGE_CHILDREN.ru_maxrss` is a cumulative harness maximum, not a per-command +measurement. -Exit code is `0` when every scenario passes, `1` on any failure (with the -observed values printed inline for each failing check), `2` on setup error. - -## What it verifies - -- **Observation:** snapshot role diversity, `find` vocabulary + `roles_present` - hint, textarea→textfield alias. -- **Interaction in BOTH modes (the headless/headed contract):** every ref-action - command (`click`, `type`, `set-value`, `clear`, `check`, `uncheck`, `select`, - `set-value` on slider/stepper, `scroll`) is driven **twice** — default - **headless**, then **`--headed`** — with mode-specific target values so a - regression in either mode is caught by an independent `before`/`after` - observation. Headed must not regress the AX path (it is tried first); it only - adds cursor/physical fallbacks. -- **The headless/headed discriminator:** `double-click` on a gesture-only button - (no `AXOpen`) **fails closed with `POLICY_DENIED` headless**, and **completes - with `--headed`** (physical double-click). This is the crisp proof the two - modes differ and that `--headed` unlocks the physical fallback. -- **Strict resolution:** ambiguous twins do not silently act; a removed element - fails closed with `STALE_REF`. -- **Reliability core:** `wait` predicates (`enabled`, `actionable`, `visible`, - `value`), `wait --text` async appearance, skeleton traversal + scoped - drill-down, session isolation + session-independent explicit snapshots, trace - JSONL with secret redaction. -- **Surfaces / drag / expand / close:** open a sheet and act inside it, - source-tracked drag gesture verified by a tracking canvas, expand a - press-toggled disclosure, `close-app --force` verified via `list-apps`. - -## Documented limitations (tracked, not failures) - -- **Cross-target native drag-and-drop (`onDrop`)** needs the OS dragging-session - / pasteboard protocol. Synthetic mouse events route mouse-up to the drag - origin, so they cannot drop onto a separate native target. Source-tracked - gestures (and web/Electron mouse-DnD) work; the harness verifies the gesture - via a tracking canvas. -- **SwiftUI `Slider`/`Stepper`/`DisclosureGroup` are not AX-actionable.** The - fixture uses native AppKit `NSSlider`/`NSStepper` (which are), so `set-value` - and `expand` are proven on controls that actually expose the capability. +The report explicitly labels its limits: it describes one machine, app state, +and TCC state; it measures observation and ref re-resolution rather than action +delivery; and it never invents a baseline when none was explicitly supplied. +The default output is a timestamped file under `/tmp`; pass `--out` to retain it +elsewhere. ## Adding a scenario -1. Expose the surface in `AgentDeskFixture.swift` with a stable - `accessibilityLabel` and a `statictext` status readout that reflects the - real state (so the harness can observe the effect, not the command's claim). -2. Add a check in `run.sh` using `verify .*?)\n\}", source, re.DOTALL) + self.assertIsNotNone(abort) + body = abort.group("body") + self.assertIn('badmsg "$1"', body) + self.assertIn("finish", body) + self.assertNotIn("cleanup", body) + self.assertRegex(body, r"(?m)^\s*exit 1\s*$") + finish = re.search(r"finish\(\) \{(?P.*?)\n\}", source, re.DOTALL) + self.assertIsNotNone(finish) + self.assertNotIn("abort_suite", finish.group("body")) + + def test_shell_harness_never_permanently_removes_artifacts(self): + permanent_delete = re.compile(r"(?m)(?:^|[;&|({])\s*rm(?:\s|$)") + offenders = [] + shell_paths = [ + *E2E_ROOT.glob("*.sh"), + *(E2E_ROOT / "scenarios").glob("*.sh"), + *(E2E_ROOT.parent / "fixture-app").glob("*.sh"), + *(E2E_ROOT.parent.parent / "scripts").glob("*.sh"), + ] + for path in sorted(shell_paths): + if permanent_delete.search(path.read_text()): + offenders.append(path.name) + + self.assertEqual(offenders, []) + + def test_wait_scenarios_use_independent_oracles(self): + source = (E2E_ROOT / "scenarios" / "reliability.sh").read_text() + + self.assertIn('is_target "$delayed" enabled', source) + self.assertIn('is_target "$primary" visible', source) + self.assertIn('get_target "$text_input" --property value', source) + self.assertIn("require_value actionable_before click-status", source) + self.assertIn("require_value actionable_after click-status", source) + + def test_acceptance_visibility_uses_a_qualified_zero_bounds_ref(self): + source = (E2E_ROOT / "scenarios" / "acceptance.sh").read_text() + + self.assertIn("require_target zero_target button zero-bounds-button", source) + self.assertIn('is_target "$zero_target" visible', source) + self.assertIn('data.applicable)" = "True"', source) + self.assertIn('data.result)" = "False"', source) + + def test_hover_uses_the_headed_qualified_ref_pipeline(self): + source = (E2E_ROOT / "scenarios" / "interaction.sh").read_text() + + self.assertIn("require_target hover_target button hover-target", source) + self.assertIn('act_target "$hover_target" hover', source) + self.assertIn("require_value hover_after hover-status", source) + self.assertNotIn("hover --xy", source) + + def test_denied_automation_skips_when_notification_center_cannot_be_closed(self): + source = (E2E_ROOT / "scenarios" / "acceptance.sh").read_text() + + unavailable = source.index( + "Notification Center could not be proven closed after best-effort preparation" + ) + permission_gate = source.index( + 'if [ "$(json_field "$nc_probe" error.code)" != "POLICY_DENIED" ]' + ) + denial_call = source.index('denied_output="$("$bin" --headed list-notifications') + self.assertLess(permission_gate, unavailable) + self.assertLess(unavailable, denial_call) + self.assertIn("skipmsg", source[permission_gate:denial_call]) + self.assertNotIn("badmsg", source[permission_gate:denial_call]) + + def test_native_runner_gates_on_strict_headless_non_interference(self): + source = (E2E_ROOT / "run.sh").read_text() + + safe_gate = source.index('bash "$here/safe-semantic.sh"') + focused_fixture = source.index("prepare_native_harness") + self.assertLess(safe_gate, focused_fixture) + + def test_native_runner_builds_locked_canonical_artifacts_before_desktop_lock(self): + source = (E2E_ROOT / "run.sh").read_text() + + cli_build = source.index("cargo build --locked --release -p agent-desktop") + ffi_build = source.index("cargo build --locked --profile release-ffi") + desktop_lock = source.index('exec python3 "$here/interaction_lock.py"') + self.assertLess(cli_build, desktop_lock) + self.assertLess(ffi_build, desktop_lock) + self.assertIn( + 'AGENT_DESKTOP_E2E_RELEASE_BIN="$repo/target/release/agent-desktop"', + source, + ) + self.assertIn( + 'AGENT_DESKTOP_E2E_RELEASE_FFI="$repo/target/release-ffi/libagent_desktop_ffi.dylib"', + source, + ) + + def test_post_launch_readiness_failure_still_closes_owned_fixture(self): + run_source = (E2E_ROOT / "run.sh").read_text() + opened = run_source.index('guard_exec 10 1048576 open "$fixture_app"') + owned = run_source.index("fixture_owned=1", opened) + readiness = run_source.index('if [ -z "$ready" ]', owned) + self.assertLess(opened, owned) + self.assertLess(owned, readiness) + + with tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "calls" + mock_bin = Path(directory) / "agent-desktop" + mock_bin.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$MOCK_CALLS"\n') + mock_bin.chmod(0o700) + script = r''' +source "$1" +bin="$2" +fixture_owned=1 +fixture_started=0 +cleanup_isolated_environment() { :; } +release_exclusive_lock() { :; } +cleanup +''' + environment = os.environ.copy() + environment["MOCK_CALLS"] = str(log) + subprocess.run( + [ + "bash", + "-c", + script, + "cleanup-test", + str(E2E_ROOT / "lib.sh"), + str(mock_bin), + ], + check=True, + env=environment, + ) + + self.assertEqual(log.read_text().strip(), "close-app AgentDeskFixture --force") + + def test_success_checks_every_staged_native_artifact_before_returning(self): + source = (E2E_ROOT / "lib.sh").read_text() + finish = re.search(r"finish\(\) \{(?P.*?)\n\}", source, re.DOTALL) + + self.assertIsNotNone(finish) + body = finish.group("body") + self.assertIn('verify_immutable_binary "$raw_bin" "$raw_bin_sha"', body) + self.assertIn('verify_immutable_binary "$ffi_dylib" "$ffi_dylib_sha"', body) + self.assertIn('verify_immutable_binary "$ffi_helper" "$ffi_helper_sha"', body) + decision = body.index('if [ "$fail" -gt 0 ]') + self.assertLess(body.index('verify_immutable_binary "$ffi_dylib"'), decision) + self.assertLess(body.index('verify_immutable_binary "$ffi_helper"'), decision) + + def test_fixture_status_oracles_use_stable_native_identifiers(self): + source = (E2E_ROOT / "lib.sh").read_text() + + self.assertIn('--role statictext --native-id "$name" --first', source) + + def test_trace_artifact_actions_use_the_session_that_created_their_refs(self): + source = (E2E_ROOT / "scenarios" / "trace_performance.sh").read_text() + + self.assertIn('"$bin" --session "$trace_session" find', source) + self.assertIn('trace_click="$("$bin" --session "$trace_session" click', source) + self.assertIn( + 'trace_session_type="$("$bin" --headed --session "$trace_session" type', + source, + ) + + def test_sheet_scenarios_scroll_the_button_before_clicking(self): + fixtures = [ + ( + "scenarios/surfaces.sh", + 'act_target "$open_sheet" scroll-to', + 'act_target "$open_sheet" click', + ), + ( + "scenarios/acceptance.sh", + 'act_target "$open_sheet" scroll-to', + 'batch_payload="$(python3', + ), + ] + for filename, scroll_marker, click_marker in fixtures: + with self.subTest(filename=filename): + source = (E2E_ROOT / filename).read_text() + scroll = source.index(scroll_marker) + click = source.index(click_marker) + self.assertLess(scroll, click) + + def test_recoverable_trash_moves_artifact_with_available_backend(self): + with tempfile.TemporaryDirectory() as root: + root_path = Path(root) + artifact = root_path / "artifact" + artifact.mkdir() + trash_dir = root_path / "trash" + trash_dir.mkdir() + bin_dir = root_path / "bin" + bin_dir.mkdir() + trash = bin_dir / "trash" + trash.write_text('#!/bin/sh\n/bin/mv "$1" "$FAKE_TRASH/"\n') + trash.chmod(0o700) + + result = self.run_trash_helper(artifact, bin_dir, trash_dir) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(artifact.exists()) + self.assertTrue((trash_dir / artifact.name).is_dir()) + + def test_recoverable_trash_retains_artifact_without_backend(self): + with tempfile.TemporaryDirectory() as root: + artifact = Path(root) / "artifact" + artifact.mkdir() + + result = self.run_trash_helper(artifact, Path(root) / "missing", Path(root)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(artifact.is_dir()) + self.assertIn("retained artifact", result.stderr) + + def run_trash_helper(self, artifact: Path, path: Path, trash_dir: Path): + script = f''' +source "{E2E_ROOT / "harness.sh"}" +host_home="{artifact.parent}" +host_tmp="{artifact.parent}" +PATH="{path}" +if trash_recoverably "{artifact}"; then + test ! -e "{artifact}" +else + test -e "{artifact}" +fi +''' + env = os.environ.copy() + env["FAKE_TRASH"] = str(trash_dir) + return subprocess.run( + ["/bin/bash", "-c", script], + capture_output=True, + check=False, + text=True, + env=env, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/test_interaction_lock.py b/tests/e2e/test_interaction_lock.py new file mode 100644 index 0000000..737661f --- /dev/null +++ b/tests/e2e/test_interaction_lock.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +import tempfile +import unittest + +import interaction_lock + + +PROBE = """ +import fcntl, os, sys +fd = os.open(sys.argv[1], os.O_RDWR) +try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) +except BlockingIOError: + raise SystemExit(23) +raise SystemExit(0) +""" + + +class InteractionLockTests(unittest.TestCase): + def probe(self, path): + return subprocess.run( + [sys.executable, "-c", PROBE, path], + check=False, + ).returncode + + def test_outsider_contention_and_inherited_child_crash(self): + with tempfile.TemporaryDirectory() as root: + fd = interaction_lock.acquire(root) + self.assertIsNotNone(fd) + _, path = interaction_lock.canonical_lock_path(root) + self.assertEqual(self.probe(path), 23) + + subprocess.run( + [sys.executable, "-c", "import os; os._exit(71)"], + pass_fds=(fd,), + check=False, + ) + self.assertEqual(self.probe(path), 23) + + os.close(fd) + self.assertEqual(self.probe(path), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/test_json_tool.py b/tests/e2e/test_json_tool.py new file mode 100644 index 0000000..77835e8 --- /dev/null +++ b/tests/e2e/test_json_tool.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +import os +import sys +import tempfile +import time +import unittest +from unittest import mock + +from json_tool import command_delivered_mechanism, run_bounded + + +class RunBoundedTests(unittest.TestCase): + def test_delivered_mechanism_reports_the_first_successful_step(self): + payload = { + "ok": True, + "data": { + "steps": [ + {"outcome": "skipped", "mechanism": "physical_synthetic"}, + {"outcome": "succeeded", "mechanism": "semantic_api"}, + ] + }, + } + with mock.patch("json_tool.read_json", return_value=payload), mock.patch( + "builtins.print" + ) as output: + command_delivered_mechanism() + + output.assert_called_once_with("semantic_api") + + def test_marked_child_receives_the_canonical_interaction_lease_fd(self): + with tempfile.TemporaryFile() as lease: + os.set_inheritable(lease.fileno(), True) + environment = os.environ.copy() + environment["AGENT_DESKTOP_INTERACTION_LEASE_FD"] = str(lease.fileno()) + result = run_bounded( + [ + sys.executable, + "-c", + "import os; os.fstat(int(os.environ['AGENT_DESKTOP_INTERACTION_LEASE_FD'])); print('inherited')", + ], + timeout_seconds=2, + max_capture_bytes=1024, + env=environment, + inherit_interaction_lease=True, + ) + + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "inherited\n") + + def test_generic_shell_child_does_not_receive_the_interaction_lease(self): + with tempfile.TemporaryFile() as lease: + os.set_inheritable(lease.fileno(), True) + environment = os.environ.copy() + environment["AGENT_DESKTOP_INTERACTION_LEASE_FD"] = str(lease.fileno()) + result = run_bounded( + [ + "/bin/sh", + "-c", + 'test -z "${AGENT_DESKTOP_INTERACTION_LEASE_FD+x}" && ' + 'test ! -e "/dev/fd/$1" && printf "not inherited\\n"', + "lease-check", + str(lease.fileno()), + ], + timeout_seconds=2, + max_capture_bytes=1024, + env=environment, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "not inherited\n") + + def test_captures_a_successful_child(self): + result = run_bounded( + [sys.executable, "-c", "print('ok')"], + timeout_seconds=2, + max_capture_bytes=1024, + ) + + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "ok\n") + self.assertFalse(result.timed_out) + self.assertFalse(result.output_limited) + self.assertIsNone(result.termination_error) + + def test_kills_the_process_group_at_the_absolute_timeout(self): + result = run_bounded( + [ + sys.executable, + "-c", + "import subprocess,sys,time; " + "subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)']); " + "time.sleep(30)", + ], + timeout_seconds=0.2, + max_capture_bytes=1024, + ) + + self.assertEqual(result.returncode, 124) + self.assertTrue(result.timed_out) + self.assertLess(result.wall_ms, 2000) + + def test_descendant_holding_output_pipe_cannot_extend_the_deadline(self): + result = run_bounded( + [ + sys.executable, + "-c", + "import subprocess,sys,time; " + "subprocess.Popen([sys.executable,'-c','import time; time.sleep(30)']); " + "print('ready', flush=True); time.sleep(30)", + ], + timeout_seconds=0.2, + max_capture_bytes=1024, + ) + + self.assertTrue(result.timed_out) + self.assertIn("ready", result.stdout) + self.assertLess(result.wall_ms, 2000) + + def test_timeout_kills_term_ignoring_descendant_after_leader_exits(self): + child_pid = None + child = ( + "import os,signal,time; " + "signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print(f'child={os.getpid()}', flush=True); time.sleep(30)" + ) + leader = ( + "import signal,subprocess,sys,time; " + "signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)); " + "subprocess.Popen([sys.executable, '-c', sys.argv[1]]); " + "time.sleep(30)" + ) + try: + result = run_bounded( + [sys.executable, "-c", leader, child], + timeout_seconds=0.3, + max_capture_bytes=1024, + ) + child_pid = int(result.stdout.split("child=", 1)[1].splitlines()[0]) + deadline = time.monotonic() + 1 + while time.monotonic() < deadline: + try: + os.kill(child_pid, 0) + except ProcessLookupError: + break + time.sleep(0.01) + else: + self.fail(f"TERM-ignoring descendant {child_pid} survived timeout cleanup") + finally: + if child_pid is not None: + try: + os.kill(child_pid, 9) + except ProcessLookupError: + pass + + def test_group_signal_failure_is_reported_without_blocking(self): + with mock.patch("json_tool.os.killpg", side_effect=PermissionError("denied")): + result = run_bounded( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout_seconds=0.05, + max_capture_bytes=1024, + ) + + self.assertTrue(result.timed_out) + self.assertIn("SIGTERM failed", result.termination_error) + self.assertLess(result.wall_ms, 2000) + + def test_stops_unbounded_output(self): + result = run_bounded( + [sys.executable, "-c", "import sys; sys.stdout.write('x' * 100000)"], + timeout_seconds=2, + max_capture_bytes=4096, + ) + + self.assertEqual(result.returncode, 125) + self.assertTrue(result.output_limited) + self.assertLessEqual(len(result.stdout.encode()), 4096) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/test_safe_semantic_guard.py b/tests/e2e/test_safe_semantic_guard.py new file mode 100644 index 0000000..89c25bd --- /dev/null +++ b/tests/e2e/test_safe_semantic_guard.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +import unittest + +from safe_semantic_guard import ( + SafetyError, + fixture_window_identity, + frontmost_identity, + non_fixture_frontmost_identity, + unique_target, + unique_value, + validate_command, + validate_semantic_action, +) + + +APP = "AgentDeskFixture" +WINDOW = "w-fixture" +PID = 4242 +INSTANCE = "4242:fixture-generation" + + +def fixture_windows(**overrides): + window = { + "id": WINDOW, + "title": "AgentDesk Fixture", + "app_name": APP, + "pid": PID, + "process_instance": INSTANCE, + "is_focused": False, + "visible": True, + "minimized": False, + } + window.update(overrides) + return [window] + + +def action_data(action="click", delivery="delivered_verified", mechanism="semantic_api"): + return { + "action": action, + "disposition": {"delivery": delivery, "retry": "unsafe"}, + "steps": [ + { + "label": "AXPress", + "outcome": "succeeded", + "mechanism": mechanism, + "verified": True, + } + ], + } + + +class CommandPolicyTests(unittest.TestCase): + def test_allows_only_exact_fixture_scoped_observations(self): + validate_command(["version"], APP, "", False) + validate_command(["permissions"], APP, "", False) + validate_command(["list-windows"], APP, "", False) + validate_command(["list-windows", "--app", APP], APP, "", False) + validate_command( + [ + "find", "--app", APP, "--window-id", WINDOW, + "--role", "button", "--name", "primary-button", + "--exact", "--limit", "2", + ], + APP, + WINDOW, + False, + ) + validate_command( + [ + "find", "--app", APP, "--window-id", WINDOW, + "--role", "statictext", "--native-id", "click-status", + "--exact", "--limit", "2", + ], + APP, + WINDOW, + False, + ) + + def test_rejects_unscoped_or_non_allowlisted_find(self): + with self.assertRaises(SafetyError): + validate_command( + [ + "find", "--app", APP, "--role", "button", + "--name", "primary-button", "--exact", "--limit", "2", + ], + APP, + WINDOW, + False, + ) + with self.assertRaises(SafetyError): + validate_command( + [ + "find", "--app", APP, "--window-id", WINDOW, + "--role", "button", "--name", "delete-button", + "--exact", "--limit", "2", + ], + APP, + WINDOW, + False, + ) + + def test_rejects_every_banned_command_class(self): + banned = [ + ["list-apps", "--app", APP], + ["focus-window", "--app", APP], + ["launch", APP], + ["close-app", APP], + ["press", "cmd+a"], + ["mouse-click", "--xy", "1,1"], + ["hover", "@snap:e1"], + ["drag", "@snap:e1", "@snap:e2"], + ["clipboard-get"], + ["clipboard-set", "secret"], + ["screenshot", "--app", APP], + ["resize-window", WINDOW, "800", "600"], + ["move-window", WINDOW, "0", "0"], + ["batch", "commands.json"], + ] + for argv in banned: + with self.subTest(argv=argv), self.assertRaises(SafetyError): + validate_command(argv, APP, WINDOW, True) + + def test_rejects_headed_trace_wait_and_unknown_flags(self): + variants = [ + ["list-windows", "--headed"], + ["list-windows", "--trace", "/tmp/trace"], + ["list-windows", "--wait-for", "button:x"], + ["list-windows", "--verbose"], + ] + for argv in variants: + with self.subTest(argv=argv), self.assertRaises(SafetyError): + validate_command(argv, APP, WINDOW, False) + + def test_mutations_require_arming_ref_snapshot_and_timeout(self): + click = ["click", "@snap-1:e1", "--snapshot", "snap-1", "--timeout-ms", "1500"] + validate_command(click, APP, WINDOW, True) + for armed, window, argv in [ + (False, WINDOW, click), + (True, "", click), + (True, WINDOW, ["click", "@snap-1:e1"]), + (True, WINDOW, ["click", "@snap-1:e1", "--snapshot", "snap-1", "--timeout-ms", "9000"]), + ]: + with self.assertRaises(SafetyError): + validate_command(argv, APP, window, armed) + + def test_text_payloads_are_narrowly_bounded(self): + for command in ["type", "set-value"]: + validate_command( + [ + command, "@snap-1:e2", "safe-semantic-4242", + "--snapshot", "snap-1", "--timeout-ms", "1500", + ], + APP, + WINDOW, + True, + ) + unsafe = [[ + "set-value", "@snap-1:e2", "arbitrary user text", + "--snapshot", "snap-1", "--timeout-ms", "1500", + ]] + for argv in unsafe: + with self.subTest(argv=argv), self.assertRaises(SafetyError): + validate_command(argv, APP, WINDOW, True) + + +class IdentityGuardTests(unittest.TestCase): + def test_frontmost_identity_returns_opaque_process_and_window_tokens_only(self): + identity = frontmost_identity( + [ + { + "id": "private-window", + "title": "Private Document Title", + "app_name": "Private App Name", + "pid": 99, + "process_instance": "99:generation", + "is_focused": True, + "visible": True, + "minimized": False, + } + ] + ) + self.assertEqual( + identity, + {"pid": 99, "process_instance": "99:generation", "window_id": "private-window"}, + ) + self.assertNotIn("Private", str(identity)) + + def test_frontmost_identity_fails_closed_on_ambiguity_or_missing_generation(self): + base = fixture_windows(is_focused=True) + with self.assertRaises(SafetyError): + frontmost_identity(base + fixture_windows(id="another", is_focused=True)) + base[0].pop("process_instance") + with self.assertRaises(SafetyError): + frontmost_identity(base) + + def test_frontmost_may_change_between_user_apps_but_never_to_fixture(self): + other = fixture_windows( + pid=99, + process_instance="99:generation", + is_focused=True, + ) + + identity = non_fixture_frontmost_identity(other, PID) + + self.assertEqual(identity["pid"], 99) + with self.assertRaises(SafetyError): + non_fixture_frontmost_identity(fixture_windows(is_focused=True), PID) + + def test_fixture_identity_binds_pid_generation_and_one_unfocused_window(self): + identity = fixture_window_identity(fixture_windows(), APP, PID) + self.assertEqual(identity["pid"], PID) + self.assertEqual(identity["process_instance"], INSTANCE) + self.assertEqual(identity["window_id"], WINDOW) + + def test_fixture_identity_ignores_owned_hidden_companion_window(self): + hidden = fixture_windows( + id="hidden-companion", + title="Hidden Companion", + visible=False, + minimized=False, + ) + + identity = fixture_window_identity(fixture_windows() + hidden, APP, PID) + + self.assertEqual(identity["window_id"], WINDOW) + + def test_fixture_identity_rejects_focus_wrong_pid_and_surface_ambiguity(self): + cases = [ + (fixture_windows(is_focused=True), PID), + (fixture_windows(), PID + 1), + (fixture_windows() + fixture_windows(id="second", visible=True), PID), + ] + for windows, pid in cases: + with self.subTest(pid=pid, windows=len(windows)), self.assertRaises(SafetyError): + fixture_window_identity(windows, APP, pid) + + +class ResultGuardTests(unittest.TestCase): + def test_unique_target_and_status_reject_ambiguous_results(self): + target = { + "snapshot_id": "snap-1", + "matches": [ + { + "role": "button", + "name": "primary-button", + "interactive": True, + "ref_id": "@snap-1:e1", + } + ], + } + self.assertEqual(unique_target(target, "button", "primary-button"), ("@snap-1:e1", "snap-1")) + with self.assertRaises(SafetyError): + unique_target({**target, "matches": target["matches"] * 2}, "button", "primary-button") + value = { + "matches": [ + { + "role": "statictext", + "name": "dynamic-value", + "interactive": False, + "value": None, + } + ] + } + self.assertEqual(unique_value(value, "text-echo"), "") + with self.assertRaises(SafetyError): + unique_value({"matches": [{**value["matches"][0], "interactive": True}]}, "text-echo") + + def test_changed_action_requires_a_semantic_successful_delivery(self): + validate_semantic_action(action_data(), "click", "changed") + for bad in [ + action_data(mechanism="physical_synthetic"), + action_data(delivery="not_delivered"), + {**action_data(), "steps": []}, + ]: + with self.assertRaises(SafetyError): + validate_semantic_action(bad, "click", "changed") + + def test_noop_action_requires_explicit_not_delivered_evidence(self): + noop = { + "action": "check", + "disposition": {"delivery": "not_delivered", "retry": "safe"}, + "steps": [{"label": "AlreadyInState", "outcome": "skipped", "verified": True}], + } + validate_semantic_action(noop, "check", "noop") + with self.assertRaises(SafetyError): + validate_semantic_action(action_data(action="check"), "check", "noop") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ffi-python/smoke.py b/tests/ffi-python/smoke.py index da67c30..37290da 100644 --- a/tests/ffi-python/smoke.py +++ b/tests/ffi-python/smoke.py @@ -127,6 +127,8 @@ def main() -> None: # Adapter lifecycle ad_adapter_create = bind(lib, "ad_adapter_create", c_void_p, []) ad_adapter_destroy = bind(lib, "ad_adapter_destroy", None, [c_void_p]) + ad_get_clipboard = bind(lib, "ad_get_clipboard", c_int, [c_void_p, POINTER(c_char_p)]) + ad_last_error_message = bind(lib, "ad_last_error_message", c_char_p, []) # Snapshot (adapter leg) ad_snapshot = bind( @@ -247,7 +249,7 @@ def main() -> None: # Expected stub-adapter path: ok:false + PLATFORM_NOT_SUPPORTED. err = snap_env.get("error", {}) code = err.get("code", "") - if code != "PLATFORM_NOT_SUPPORTED": + if expect_stub and code != "PLATFORM_NOT_SUPPORTED": ad_adapter_destroy(adapter) fail( f"ad_snapshot() envelope error.code = {code!r}, " @@ -265,6 +267,34 @@ def main() -> None: ad_adapter_destroy(adapter) ok("ad_adapter_destroy() — no crash, no leak") + if os.environ.get("AD_EXPECT_MACOS_HELPER") == "1": + print("\nLeg 5 — non-colocated Python host discovers the dylib-adjacent macOS helper") + adapter = ad_adapter_create() + if not adapter: + fail("ad_adapter_create() returned null for helper discovery leg") + clipboard = c_char_p() + rc = ad_get_clipboard(adapter, byref(clipboard)) + if rc != 0: + message = ad_last_error_message() + helper_runtime_errors = { + b"System clipboard is unavailable", + b"System clipboard read access is denied", + } + if message in helper_runtime_errors: + ad_adapter_destroy(adapter) + ok( + "ctypes host reached the authenticated packaged helper; " + f"pasteboard access was unavailable in this host ({message!r})" + ) + print("\nAll legs passed.") + return + ad_adapter_destroy(adapter) + fail(f"dylib-adjacent helper discovery failed rc={rc}: {message!r}") + if clipboard.value is not None: + ad_free_string(clipboard) + ad_adapter_destroy(adapter) + ok("ctypes host read clipboard through the packaged helper") + print("\nAll legs passed.") diff --git a/tests/fixture-app/AgentDeskFixture.swift b/tests/fixture-app/AgentDeskFixture.swift index b530fc1..ed6aeb6 100644 --- a/tests/fixture-app/AgentDeskFixture.swift +++ b/tests/fixture-app/AgentDeskFixture.swift @@ -38,6 +38,7 @@ struct ContentView: View { // async / dynamic @State private var delayedEnabled = false @State private var delayedText = "waiting" + @State private var delayedActionCount = 0 @State private var removableVisible = true @State private var appearedText = "" // drag @@ -55,7 +56,7 @@ struct ContentView: View { var body: some View { ScrollView([.vertical, .horizontal]) { - VStack(alignment: .leading, spacing: 16) { + VStack(alignment: .leading, spacing: 8) { Text("AgentDesk Fixture") .font(.title2) .accessibilityLabel("fixture-title") @@ -63,14 +64,14 @@ struct ContentView: View { row2 row3 } - .padding(20) + .padding(10) } .frame(minWidth: 980, minHeight: 720) .sheet(isPresented: $showSheet) { sheetContent } } private var row1: some View { - HStack(alignment: .top, spacing: 16) { + HStack(alignment: .top, spacing: 12) { clicksCard textCard stateCard @@ -78,7 +79,7 @@ struct ContentView: View { } private var row2: some View { - HStack(alignment: .top, spacing: 16) { + HStack(alignment: .top, spacing: 12) { choiceCard collectionsCard asyncCard @@ -86,7 +87,7 @@ struct ContentView: View { } private var row3: some View { - HStack(alignment: .top, spacing: 16) { + HStack(alignment: .top, spacing: 12) { dragCard surfacesCard ScrollCard() @@ -119,11 +120,11 @@ struct ContentView: View { } StatusReadout(name: "right-status", value: rightStatus) - Text("Hover Target") + Button("Hover Target") { } .padding(6) .background(hoverStatus == "hovered" ? Color.yellow.opacity(0.4) : Color.clear) .accessibilityLabel("hover-target") - .onHover { inside in if inside { hoverStatus = "hovered" } } + .onHover { inside in hoverStatus = inside ? "hovered" : "idle" } StatusReadout(name: "hover-status", value: hoverStatus) /// Two controls sharing role and name. Each records a distinct @@ -190,6 +191,7 @@ struct ContentView: View { Text("Gamma").tag("Gamma") } .accessibilityLabel("option-picker") + .accessibilityIdentifier("option-picker") StatusReadout(name: "picker-status", value: pickerChoice) Picker("Radio Group", selection: $radioChoice) { @@ -235,32 +237,13 @@ struct ContentView: View { // MARK: async / dynamic private var asyncCard: some View { - Card(title: "Async & Dynamic") { - Button("Enable Later") { - DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { - delayedEnabled = true - delayedText = "ready" - } - } - .accessibilityLabel("enable-later") - Button("Delayed Button") { } - .disabled(!delayedEnabled) - .accessibilityLabel("delayed-button") - StatusReadout(name: "delayed-text", value: delayedText) - - Button("Appear Later") { - DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { appearedText = "appeared-text" } - } - .accessibilityLabel("appear-later") - if !appearedText.isEmpty { - Text(appearedText).accessibilityLabel("appeared-text") - } - - if removableVisible { - Button("Removable Row") { }.accessibilityLabel("removable-row") - } - Button("Remove Row") { removableVisible = false }.accessibilityLabel("remove-row") - } + AsyncFixtureCard( + delayedEnabled: $delayedEnabled, + delayedText: $delayedText, + delayedActionCount: $delayedActionCount, + appearedText: $appearedText, + removableVisible: $removableVisible + ) } // MARK: drag (interactive row + non-interactive image) @@ -364,11 +347,21 @@ struct AgentDeskFixtureApp: App { } final class AppDelegate: NSObject, NSApplicationDelegate { + private var isBackgroundFixture: Bool { + ProcessInfo.processInfo.environment["AGENT_DESKTOP_FIXTURE_NO_ACTIVATE"] == "1" + } + + func applicationWillFinishLaunching(_ notification: Notification) { + if isBackgroundFixture { + NSApp.setActivationPolicy(.accessory) + } + } + func applicationDidFinishLaunching(_ notification: Notification) { + if isBackgroundFixture { + return + } NSApp.setActivationPolicy(.regular) - // Bring the window up without forcibly stealing focus from other apps: - // the E2E harness drives focus explicitly via focus-window, and an - // unconditional steal could mask headless-policy focus violations. NSApp.activate(ignoringOtherApps: false) } func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true } diff --git a/tests/fixture-app/FixtureAsync.swift b/tests/fixture-app/FixtureAsync.swift new file mode 100644 index 0000000..6f75791 --- /dev/null +++ b/tests/fixture-app/FixtureAsync.swift @@ -0,0 +1,70 @@ +import SwiftUI + +struct AsyncFixtureCard: View { + @Binding var delayedEnabled: Bool + @Binding var delayedText: String + @Binding var delayedActionCount: Int + @Binding var appearedText: String + @Binding var removableVisible: Bool + + var body: some View { + Card(title: "Async & Dynamic") { + Button("Enable Later") { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { + delayedEnabled = true + delayedText = "ready" + } + } + .accessibilityLabel("enable-later") + .accessibilityIdentifier("enable-later") + Button("Delayed Button") { delayedActionCount += 1 } + .disabled(!delayedEnabled) + .accessibilityLabel("delayed-button") + .accessibilityIdentifier("delayed-button") + StatusReadout(name: "delayed-text", value: delayedText) + StatusReadout(name: "delayed-action-status", value: String(delayedActionCount)) + + Button("Reset Delayed Button") { + delayedEnabled = false + delayedText = "waiting" + delayedActionCount = 0 + } + .accessibilityLabel("reset-delayed-button") + .accessibilityIdentifier("reset-delayed-button") + + Button("Permanently Disabled") { delayedActionCount += 100 } + .disabled(true) + .accessibilityLabel("permanently-disabled") + + ZeroBoundsButton().frame(width: 1, height: 1) + + Button("Open Duplicate Windows") { + DuplicateWindowController.shared.openWindows() + } + .accessibilityLabel("open-duplicate-windows") + Button("Close Duplicate Windows") { + DuplicateWindowController.shared.closeWindows() + } + .accessibilityLabel("close-duplicate-windows") + + Button("Appear Later") { + DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { + appearedText = "appeared-text" + } + } + .accessibilityLabel("appear-later") + Button("Reset Appeared Text") { appearedText = "" } + .accessibilityLabel("reset-appeared-text") + if !appearedText.isEmpty { + Text(appearedText).accessibilityLabel("appeared-text") + } + + if removableVisible { + Button("Removable Row") {}.accessibilityLabel("removable-row") + } + Button("Remove Row") { removableVisible = false }.accessibilityLabel("remove-row") + Button("Reset Removable Row") { removableVisible = true } + .accessibilityLabel("reset-removable") + } + } +} diff --git a/tests/fixture-app/FixtureComponents.swift b/tests/fixture-app/FixtureComponents.swift index 4089745..af9779b 100644 --- a/tests/fixture-app/FixtureComponents.swift +++ b/tests/fixture-app/FixtureComponents.swift @@ -13,6 +13,7 @@ struct StatusReadout: View { Text("\(name): \(value)") .accessibilityLabel(name) .accessibilityValue(value) + .accessibilityIdentifier(name) } } @@ -66,6 +67,82 @@ final class NativeControlCoordinator: NSObject { @objc func changed(_ sender: NSControl) { onChange(sender.doubleValue) } } +struct ZeroBoundsButton: NSViewRepresentable { + func makeNSView(context: Context) -> ZeroBoundsAXHost { ZeroBoundsAXHost() } + + func updateNSView(_ view: ZeroBoundsAXHost, context: Context) {} +} + +final class ZeroBoundsAXHost: NSView { + private let visual = NSButton(title: "Zero Bounds", target: nil, action: nil) + private let zeroBoundsElement = NSAccessibilityElement() + + override init(frame frameRect: NSRect) { + super.init(frame: frameRect) + visual.setAccessibilityElement(false) + addSubview(visual) + setAccessibilityElement(false) + zeroBoundsElement.setAccessibilityRole(.button) + zeroBoundsElement.setAccessibilityLabel("zero-bounds-button") + zeroBoundsElement.setAccessibilityParent(self) + zeroBoundsElement.setAccessibilityFrame(.zero) + } + + required init?(coder: NSCoder) { nil } + + override var intrinsicContentSize: NSSize { visual.intrinsicContentSize } + + override func layout() { + super.layout() + visual.frame = bounds + zeroBoundsElement.setAccessibilityFrameInParentSpace(.zero) + } + + override func accessibilityChildren() -> [Any]? { [zeroBoundsElement] } +} + +final class DuplicateWindowController { + static let shared = DuplicateWindowController() + + private var windows: [NSWindow] = [] + + func openWindows() { + closeWindows() + windows = (0..<2).map { index in + let window = NSWindow( + contentRect: NSRect(x: 160 + index * 380, y: 180, width: 320, height: 180), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, + defer: false + ) + window.isReleasedWhenClosed = false + window.title = "Duplicate Window" + window.identifier = NSUserInterfaceItemIdentifier("duplicate-window-\(index)") + window.contentView = duplicateWindowBody(index: index) + window.orderFront(nil) + return window + } + } + + func closeWindows() { + windows.forEach { $0.close() } + windows = [] + } + + private func duplicateWindowBody(index: Int) -> NSView { + let label = NSTextField(labelWithString: "duplicate-window-\(index)") + label.setAccessibilityLabel("duplicate-window-\(index)") + let container = NSView(frame: NSRect(x: 0, y: 0, width: 320, height: 180)) + label.translatesAutoresizingMaskIntoConstraints = false + container.addSubview(label) + NSLayoutConstraint.activate([ + label.centerXAnchor.constraint(equalTo: container.centerXAnchor), + label.centerYAnchor.constraint(equalTo: container.centerYAnchor), + ]) + return container + } +} + // A single view that tracks a mouse-drag gesture end to end (down, dragged, // up). Synthetic mouse events route to whoever received the mouse-down, so a // drag is observable when source and target are the same view — which is what diff --git a/tests/fixture-app/build.sh b/tests/fixture-app/build.sh index 66c5667..62d253e 100755 --- a/tests/fixture-app/build.sh +++ b/tests/fixture-app/build.sh @@ -10,7 +10,10 @@ app="$out_dir/AgentDeskFixture.app" macos_dir="$app/Contents/MacOS" bin="$macos_dir/AgentDeskFixture" -rm -rf "$app" +if [ -e "$app" ]; then + previous="$(mktemp -d "$out_dir/.fixture-previous.XXXXXX")" + mv "$app" "$previous/" +fi mkdir -p "$macos_dir" # Pin the SDK and deployment target so the fixture builds reproducibly instead @@ -43,4 +46,8 @@ cat > "$app/Contents/Info.plist" <<'PLIST' PLIST +if [ "${AGENT_DESKTOP_FIXTURE_BACKGROUND_BUNDLE:-}" = "1" ]; then + /usr/bin/plutil -insert LSUIElement -bool true "$app/Contents/Info.plist" +fi + echo "Built: $app" diff --git a/tests/npm/postinstall.test.js b/tests/npm/postinstall.test.js new file mode 100644 index 0000000..b0db9e5 --- /dev/null +++ b/tests/npm/postinstall.test.js @@ -0,0 +1,173 @@ +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const { + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + writeFileSync, +} = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); +const { afterEach, test } = require('node:test'); + +const postinstall = require('../../npm/scripts/postinstall.js'); + +const roots = []; + +afterEach(() => { + delete process.env.AGENT_DESKTOP_MACOS_HELPER_PATH; + for (const root of roots.splice(0)) { + postinstall.trashRecoverably(root); + } +}); + +function temporaryDirectory() { + const root = mkdtempSync(join(tmpdir(), 'agent-desktop-npm-test-')); + roots.push(root); + return root; +} + +function archive(entries) { + const root = temporaryDirectory(); + const source = join(root, 'source'); + mkdirSync(source); + for (const [name, contents] of Object.entries(entries)) { + const path = join(source, name); + writeFileSync(path, contents, { mode: 0o755 }); + chmodSync(path, 0o755); + } + const tarball = join(root, 'release.tar.gz'); + execFileSync('tar', ['-czf', tarball, '-C', source, ...Object.keys(entries)]); + return { root, tarball }; +} + +function executable(contents) { + const root = temporaryDirectory(); + const path = join(root, 'trash'); + writeFileSync(path, contents, { mode: 0o755 }); + return path; +} + +function captureWarnings(run) { + const warnings = []; + const write = process.stderr.write; + process.stderr.write = (chunk) => { + warnings.push(String(chunk)); + return true; + }; + try { + run(); + return warnings.join(''); + } finally { + process.stderr.write = write; + } +} + +test('checksum lookup requires an exact archive name', () => { + const hash = 'a'.repeat(64); + assert.equal(postinstall.checksumFor(`${hash} release.tar.gz\n`, 'release.tar.gz'), hash); + assert.throws( + () => postinstall.checksumFor(`${hash} old-release.tar.gz\n`, 'release.tar.gz'), + /Checksum entry missing/, + ); +}); + +test('archive validation rejects missing and additional payloads', () => { + const valid = archive({ + 'agent-desktop': 'cli', + 'agent-desktop-macos-helper': 'helper', + }); + postinstall.validateArchive(valid.tarball); + + const extra = archive({ + 'agent-desktop': 'cli', + 'agent-desktop-macos-helper': 'helper', + 'unexpected': 'payload', + }); + assert.throws(() => postinstall.validateArchive(extra.tarball), /unexpected entries/); +}); + +test('archive installation preserves the exact paired executables', () => { + const payload = archive({ + 'agent-desktop': 'cli-build-v1', + 'agent-desktop-macos-helper': 'helper-build-v1', + }); + const destination = temporaryDirectory(); + const binary = join(destination, 'agent-desktop-darwin-arm64'); + const helper = join(destination, 'agent-desktop-macos-helper'); + + postinstall.installArchive(payload.tarball, binary, helper); + + assert.equal(readFileSync(binary, 'utf8'), 'cli-build-v1'); + assert.equal(readFileSync(helper, 'utf8'), 'helper-build-v1'); +}); + +test('recoverable cleanup invokes trash and removes the original path', () => { + const target = temporaryDirectory(); + const recovered = `${target}.recovered`; + roots.push(recovered); + const fakeTrash = executable('#!/bin/sh\nmv "$1" "$1.recovered"\n'); + + postinstall.trashRecoverably(target, fakeTrash); + assert.equal(existsSync(target), false); + assert.equal(existsSync(recovered), true); +}); + +test('recoverable cleanup retains artifacts and warns when trash is unavailable or fails', () => { + const unavailable = temporaryDirectory(); + const failing = temporaryDirectory(); + const fakeTrash = executable('#!/bin/sh\nexit 9\n'); + + for (const [target, command, reason] of [ + [ + unavailable, + '/definitely-missing-agent-desktop-trash', + 'trash command is unavailable: /definitely-missing-agent-desktop-trash', + ], + [failing, fakeTrash, 'trash exited with status 9'], + ]) { + const warnings = captureWarnings(() => + postinstall.trashRecoverably(target, command), + ); + assert.equal(existsSync(target), true); + assert.ok(warnings.includes(`retained at ${target}:`)); + assert.ok(warnings.includes(reason)); + } +}); + +test('cleanup failure does not mask a successful archive install', () => { + const payload = archive({ + 'agent-desktop': 'cli-build-v2', + 'agent-desktop-macos-helper': 'helper-build-v2', + }); + const destination = temporaryDirectory(); + const binary = join(destination, 'agent-desktop-darwin-arm64'); + const helper = join(destination, 'agent-desktop-macos-helper'); + + const warnings = captureWarnings(() => + postinstall.installArchive( + payload.tarball, + binary, + helper, + '/definitely-missing-agent-desktop-trash', + ), + ); + + assert.equal(readFileSync(binary, 'utf8'), 'cli-build-v2'); + assert.equal(readFileSync(helper, 'utf8'), 'helper-build-v2'); + assert.match(warnings, /Could not move cleanup artifact to Trash; retained at .*\.extract-/); + const retained = warnings.match(/retained at (.*\.extract-[^:]+):/)?.[1]; + assert.ok(retained); + assert.equal(existsSync(retained), true); + roots.push(retained); +}); + +test('custom helper override must be absolute', () => { + process.env.AGENT_DESKTOP_MACOS_HELPER_PATH = 'relative-helper'; + assert.throws( + () => postinstall.customHelperPath('/tmp/agent-desktop'), + /must be an absolute path/, + ); +}); diff --git a/tests/support/noop_ops.rs b/tests/support/noop_ops.rs new file mode 100644 index 0000000..fed7b74 --- /dev/null +++ b/tests/support/noop_ops.rs @@ -0,0 +1,16 @@ +use agent_desktop_core::{ActionOps, InputOps, ObservationOps, SystemOps}; + +/// Blanket-default `PlatformAdapter` test double: implements the four +/// capability supertraits with zero overrides, so every call surfaces that +/// trait's own `not_supported()` default. Shared by any test that needs +/// "some adapter" without exercising a live capability. Single source of +/// truth included via `#[path]` from every call site — the different +/// crates it's compiled into (the `agent-desktop` binary's own unit tests +/// and the standalone `conformance` integration-test crate) cannot share a +/// Rust module across a crate boundary, but they can share this file. +pub(crate) struct NoopAdapter; + +impl ObservationOps for NoopAdapter {} +impl ActionOps for NoopAdapter {} +impl InputOps for NoopAdapter {} +impl SystemOps for NoopAdapter {}