agent-desktop/tests/e2e/run.sh
Lahfir 3f322728b4
feat!: implement Playwright-grade foundation contract
Settle the Playwright-grade reliability contract in agent-desktop-core
before the Windows/Linux adapters are built, so they inherit it instead
of redesigning it. Every command now observes, waits, verifies, and
reports honestly instead of firing blindly.

Highlights: capability-supertrait split of PlatformAdapter with
not_supported() defaults; canonical role/state vocabulary with live
`is --property visible`; display enumeration (`list-displays`) and honest
`--screen` with scale factor; truthful Automation permission; `native_id`
identity spine; window-id-first resolution; serializable `LocatorQuery`
with live `find`; default-on auto-wait before every ref action; three-way
`hit_test` occlusion gate; `scroll_into_view` in core; core accessible-name
precedence; typed `ActionStep` delivery tier; `ProcessState` and
`APP_UNRESPONSIVE`; `LaunchOptions`; baseline-diff desktop signals
(`wait --event`); typed clipboard (`Text`/`Image`/`FileUrls`); mouse
modifier chords and `mouse-wheel`. Hardened through a 35-reviewer pass with
independent validation and a green live e2e gate (109/0), plus a
head-vs-main performance comparison harness.

BREAKING CHANGE: default-on auto-wait changes the timing of every
previously-untouched ref-action call (bounded 5000 ms default; `--timeout-ms 0`
restores single-shot). `ENVELOPE_VERSION` is now `2.1` (adds the
`APP_UNRESPONSIVE` code and process state in error details). FFI ABI major
is `3` (append-only struct evolution; `wait --event` is intentionally not
exposed over FFI). The legacy string clipboard API is removed in favor of
typed content. `key-down`/`key-up` fail closed until daemon-owned held input
exists. `close-app` verifies termination and the osascript fallback path is
removed. `--text` matching is subtree containment: `find --text X --first`
returns the outermost matching container.
2026-07-20 00:21:38 -07:00

139 lines
5 KiB
Bash
Executable file

#!/usr/bin/env bash
set -uo pipefail
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo="$(cd "$here/../.." && pwd)"
if [ -z "${AGENT_DESKTOP_INTERACTION_LEASE_FD:-}" ]; then
if [ "$(uname -s)" != "Darwin" ]; then
echo "macOS is required for the native accessibility E2E suite" >&2
exit 2
fi
if [ "${AGENT_DESKTOP_E2E_EXCLUSIVE:-}" != "1" ]; then
echo "SKIP (blocked): native E2E requires an exclusive desktop" >&2
exit 2
fi
CARGO_TARGET_DIR="$repo/target" cargo build --locked --release -p agent-desktop || exit 2
CARGO_TARGET_DIR="$repo/target" cargo build --locked --release \
-p agent-desktop-macos --bin agent-desktop-macos-helper || exit 2
CARGO_TARGET_DIR="$repo/target" cargo build --locked --profile release-ffi \
-p agent-desktop-ffi || exit 2
export AGENT_DESKTOP_E2E_RELEASE_BIN="$repo/target/release/agent-desktop"
export AGENT_DESKTOP_E2E_RELEASE_FFI="$repo/target/release-ffi/libagent_desktop_ffi.dylib"
export AGENT_DESKTOP_E2E_RELEASE_FFI_HELPER="$repo/target/release/agent-desktop-macos-helper"
exec python3 "$here/interaction_lock.py" run "$0" "$@"
fi
# shellcheck source=tests/e2e/lib.sh
source "$here/lib.sh"
trap cleanup EXIT
note "Global prerequisite gate"
if [ "$(uname -s)" != "Darwin" ]; then
echo "macOS is required for the native accessibility E2E suite" >&2
exit 2
fi
if [ ! -x "$release_bin" ]; then
echo "release binary missing at $release_bin; run 'cargo build --release'" >&2
exit 2
fi
if [ ! -f "$release_ffi" ] || [ ! -x "$release_ffi_helper" ]; then
echo "release FFI bundle missing; build agent-desktop-ffi with release-ffi and the agent-desktop-macos-helper binary with release" >&2
exit 2
fi
note "Strict headless non-interference gate"
AGENT_DESKTOP_E2E_RELEASE_BIN="$release_bin" bash "$here/safe-semantic.sh"
safe_semantic_status=$?
if [ "$safe_semantic_status" -ne 0 ]; then
exit "$safe_semantic_status"
fi
if ! prepare_native_harness; then
exit 2
fi
permission_json="$("$bin" permissions 2>/dev/null)"
ax_state="$(json_field "$permission_json" data.accessibility.state)"
if [ "$ax_state" != "granted" ]; then
echo "accessibility permission not granted (state='${ax_state:-unknown}')." >&2
echo "Grant trust to this terminal/runner in System Settings > Privacy & Security > Accessibility." >&2
exit 2
fi
guard_exec 120 4194304 "$repo/tests/fixture-app/build.sh" "$suite_root/fixture" >/dev/null || {
echo "fixture build failed; cannot run E2E" >&2
exit 2
}
"$bin" --headed mouse-move --xy 20,20 >/dev/null 2>&1 || {
echo "fixture cursor precondition could not be established" >&2
exit 2
}
guard_exec 10 1048576 open "$fixture_app" || {
echo "fixture could not be opened" >&2
exit 2
}
fixture_owned=1
ready=""
tries=0
for _ in $(seq 1 20); do
tries=$((tries + 1))
"$bin" focus-window --app "$app" >/dev/null 2>&1 || true
if find_target button primary-button >/dev/null; then
ready=1
break
fi
sleep 0.5
done
if [ -z "$ready" ]; then
badmsg "fixture launched but primary-button was not exposed after $tries attempts"
finish
exit 1
fi
if ! record_fixture_process; then
finish
exit 1
fi
okmsg "fixture launched and primary-button exposed with exact snapshot namespace"
frame="$("$bin" list-displays 2>/dev/null | python3 -c '
import json, sys
d = json.load(sys.stdin)["data"]
best = max(d, key=lambda x: x["bounds"]["width"] * x["bounds"]["height"])
b = best["bounds"]
w = min(int(b["width"]) - 40, 1900)
h = min(int(b["height"]) - 34, 1080)
print(int(b["x"]) + 20, int(b["y"]) + 28, w, h)
')"
if [ -n "$frame" ]; then
read -r fx fy fw fh <<EOF_FRAME
$frame
EOF_FRAME
osascript -e "tell application \"System Events\" to tell process \"$app\"" \
-e "set position of front window to {$fx, $fy}" \
-e "set size of front window to {$fw, $fh}" \
-e "end tell" >/dev/null 2>&1 || true
sleep 0.5
okmsg "fixture window normalized to {$fx,$fy} ${fw}x${fh} on the largest display"
else
badmsg "fixture window normalization skipped: list-displays returned nothing"
fi
check_fixture_contention || { finish; exit 1; }
# shellcheck source=tests/e2e/scenarios/observation.sh
source "$here/scenarios/observation.sh"
check_fixture_contention || { finish; exit 1; }
# shellcheck source=tests/e2e/scenarios/interaction.sh
source "$here/scenarios/interaction.sh"
check_fixture_contention || { finish; exit 1; }
# shellcheck source=tests/e2e/scenarios/acceptance.sh
source "$here/scenarios/acceptance.sh"
check_fixture_contention || { finish; exit 1; }
# shellcheck source=tests/e2e/scenarios/reliability.sh
source "$here/scenarios/reliability.sh"
check_fixture_contention || { finish; exit 1; }
# shellcheck source=tests/e2e/scenarios/surfaces.sh
source "$here/scenarios/surfaces.sh"
check_fixture_contention || { finish; exit 1; }
# shellcheck source=tests/e2e/scenarios/trace_performance.sh
source "$here/scenarios/trace_performance.sh"
# shellcheck source=tests/e2e/scenarios/notifications.sh
source "$here/scenarios/notifications.sh"
finish