mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-29 02:12:52 +00:00
Settle the Playwright-grade reliability contract in agent-desktop-core before the Windows/Linux adapters are built, so they inherit it instead of redesigning it. Every command now observes, waits, verifies, and reports honestly instead of firing blindly. Highlights: capability-supertrait split of PlatformAdapter with not_supported() defaults; canonical role/state vocabulary with live `is --property visible`; display enumeration (`list-displays`) and honest `--screen` with scale factor; truthful Automation permission; `native_id` identity spine; window-id-first resolution; serializable `LocatorQuery` with live `find`; default-on auto-wait before every ref action; three-way `hit_test` occlusion gate; `scroll_into_view` in core; core accessible-name precedence; typed `ActionStep` delivery tier; `ProcessState` and `APP_UNRESPONSIVE`; `LaunchOptions`; baseline-diff desktop signals (`wait --event`); typed clipboard (`Text`/`Image`/`FileUrls`); mouse modifier chords and `mouse-wheel`. Hardened through a 35-reviewer pass with independent validation and a green live e2e gate (109/0), plus a head-vs-main performance comparison harness. BREAKING CHANGE: default-on auto-wait changes the timing of every previously-untouched ref-action call (bounded 5000 ms default; `--timeout-ms 0` restores single-shot). `ENVELOPE_VERSION` is now `2.1` (adds the `APP_UNRESPONSIVE` code and process state in error details). FFI ABI major is `3` (append-only struct evolution; `wait --event` is intentionally not exposed over FFI). The legacy string clipboard API is removed in favor of typed content. `key-down`/`key-up` fail closed until daemon-owned held input exists. `close-app` verifies termination and the osascript fallback path is removed. `--text` matching is subtree containment: `find --text X --first` returns the outermost matching container.
315 lines
9.9 KiB
Bash
315 lines
9.9 KiB
Bash
e2e_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=tests/e2e/harness.sh
|
|
source "$e2e_dir/harness.sh"
|
|
|
|
bin=""
|
|
raw_bin=""
|
|
raw_bin_sha=""
|
|
fixture_app=""
|
|
app="AgentDeskFixture"
|
|
fixture_pid=""
|
|
fixture_owned=0
|
|
fixture_started=0
|
|
contaminated=0
|
|
ffi_dylib=""
|
|
ffi_dylib_sha=""
|
|
ffi_helper=""
|
|
ffi_helper_sha=""
|
|
|
|
pass=0
|
|
fail=0
|
|
skip=0
|
|
declare -a failures=()
|
|
declare -a skips=()
|
|
declare -a cleanup_files=()
|
|
MODE_FLAG=""
|
|
|
|
prepare_native_harness() {
|
|
require_exclusive_acknowledgement || return 1
|
|
acquire_exclusive_lock || return 1
|
|
setup_isolated_environment fixture || return 1
|
|
copy_immutable_binary "$release_bin" agent-desktop || return 1
|
|
raw_bin="$prepared_binary"
|
|
raw_bin_sha="$prepared_binary_sha"
|
|
export AGENT_DESKTOP_E2E_BINARY="$raw_bin"
|
|
bin="$command_guard"
|
|
copy_native_ffi_bundle || return 1
|
|
ffi_dylib="$AGENT_DESKTOP_E2E_FFI_DYLIB"
|
|
ffi_dylib_sha="$AGENT_DESKTOP_E2E_FFI_DYLIB_SHA"
|
|
ffi_helper="$(dirname "$ffi_dylib")/agent-desktop-macos-helper"
|
|
ffi_helper_sha="$AGENT_DESKTOP_E2E_FFI_HELPER_SHA"
|
|
fixture_app="$suite_root/fixture/AgentDeskFixture.app"
|
|
|
|
local expected_version version_json actual_version clap_version existing
|
|
expected_version="$(awk -F'"' '/^version[[:space:]]*=/{print $2; exit}' "$repo/Cargo.toml")"
|
|
version_json="$("$bin" version 2>/dev/null)" || return 1
|
|
actual_version="$(json_field "$version_json" data.version)"
|
|
clap_version="$("$bin" --version 2>/dev/null)" || return 1
|
|
if [ -z "$expected_version" ] || [ "$actual_version" != "$expected_version" ] || \
|
|
[ "$clap_version" != "agent-desktop $expected_version" ]; then
|
|
echo "binary version identity mismatch: cargo='$expected_version' JSON='$actual_version' clap='$clap_version'" >&2
|
|
return 1
|
|
fi
|
|
|
|
existing="$(pgrep -x "$app" 2>/dev/null || true)"
|
|
if [ -n "$existing" ]; then
|
|
echo "SKIP (blocked): native E2E contention; $app already has process(es): $existing" >&2
|
|
contaminated=1
|
|
return 1
|
|
fi
|
|
printf ' immutable binary: %s sha256=%s version=%s\n' "$raw_bin" "$raw_bin_sha" "$actual_version"
|
|
}
|
|
|
|
record_fixture_process() {
|
|
fixture_pid="$(pgrep -x "$app" 2>/dev/null || true)"
|
|
case "$fixture_pid" in
|
|
''|*' '*)
|
|
echo "fixture process identity was missing or ambiguous: '${fixture_pid:-none}'" >&2
|
|
contaminated=1
|
|
return 1
|
|
;;
|
|
esac
|
|
if printf '%s' "$fixture_pid" | grep -q '[[:space:]]'; then
|
|
echo "multiple fixture processes detected: $fixture_pid" >&2
|
|
contaminated=1
|
|
return 1
|
|
fi
|
|
fixture_started=1
|
|
}
|
|
|
|
check_fixture_contention() {
|
|
local current
|
|
current="$(pgrep -x "$app" 2>/dev/null || true)"
|
|
if [ "$fixture_started" -ne 1 ] || [ "$current" != "$fixture_pid" ]; then
|
|
badmsg "native E2E fixture ownership changed; expected pid '$fixture_pid', found '${current:-none}'"
|
|
contaminated=1
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
note() { printf '\n\033[1;34m== %s ==\033[0m\n' "$1"; }
|
|
okmsg() { printf ' \033[0;32mPASS\033[0m %s\n' "$1"; pass=$((pass + 1)); }
|
|
badmsg() {
|
|
printf ' \033[0;31mFAIL\033[0m %s\n' "$1"
|
|
fail=$((fail + 1))
|
|
failures+=("$1")
|
|
}
|
|
skipmsg() {
|
|
printf ' \033[0;33mSKIP\033[0m %s\n' "$1"
|
|
skip=$((skip + 1))
|
|
skips+=("$1")
|
|
}
|
|
|
|
abort_suite() {
|
|
badmsg "$1"
|
|
finish || true
|
|
exit 1
|
|
}
|
|
assert() {
|
|
if [ "$2" = "1" ]; then
|
|
okmsg "$1 [$3]"
|
|
else
|
|
badmsg "$1 [$3]"
|
|
fi
|
|
}
|
|
|
|
json_field() { printf '%s' "$1" | python3 "$json_tool" get "$2" 2>/dev/null; }
|
|
target_ref() { printf '%s' "${1%%$'\t'*}"; }
|
|
target_snapshot() { printf '%s' "${1#*$'\t'}"; }
|
|
|
|
find_target() {
|
|
local role="$1" name="$2" output value
|
|
for _ in $(seq 1 25); do
|
|
output="$("$bin" find --app "$app" --role "$role" --name "$name" --first 2>/dev/null)"
|
|
if [ -n "$output" ]; then
|
|
value="$(printf '%s' "$output" | python3 "$json_tool" target 2>/dev/null)"
|
|
if [ -n "$value" ]; then
|
|
printf '%s' "$value"
|
|
return 0
|
|
fi
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
printf 'find_target %s %s last output: %s\n' "$role" "$name" "$output" >&2
|
|
return 1
|
|
}
|
|
|
|
find_target_by_id() {
|
|
local role="$1" native_id="$2" output value
|
|
for _ in $(seq 1 25); do
|
|
if output="$("$bin" find --app "$app" --role "$role" --native-id "$native_id" --first 2>/dev/null)"; then
|
|
value="$(printf '%s' "$output" | python3 "$json_tool" target 2>/dev/null)"
|
|
if [ -n "$value" ]; then
|
|
printf '%s' "$value"
|
|
return 0
|
|
fi
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
require_target() {
|
|
local variable="$1" role="$2" name="$3" value
|
|
value="$(find_target "$role" "$name")" || {
|
|
abort_suite "required fixture target '$name' ($role) is missing or lacks ref_id + snapshot_id"
|
|
}
|
|
printf -v "$variable" '%s' "$value"
|
|
}
|
|
|
|
require_target_by_id() {
|
|
local variable="$1" role="$2" native_id="$3" value
|
|
value="$(find_target_by_id "$role" "$native_id")" || {
|
|
abort_suite "required fixture target '$native_id' ($role) is missing or lacks ref_id + snapshot_id"
|
|
}
|
|
printf -v "$variable" '%s' "$value"
|
|
}
|
|
|
|
find_value() {
|
|
local name="$1" output value
|
|
for _ in $(seq 1 25); do
|
|
if output="$("$bin" find --app "$app" --role statictext --native-id "$name" --first 2>/dev/null)" \
|
|
&& value="$(printf '%s' "$output" | python3 "$json_tool" match-value 2>/dev/null)"; then
|
|
printf '%s' "$value"
|
|
return 0
|
|
fi
|
|
sleep 0.2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
require_value() {
|
|
local variable="$1" name="$2" value
|
|
value="$(find_value "$name")" || {
|
|
abort_suite "required fixture status '$name' is missing"
|
|
}
|
|
printf -v "$variable" '%s' "$value"
|
|
}
|
|
|
|
act() {
|
|
if [ -n "$MODE_FLAG" ]; then
|
|
"$bin" "$MODE_FLAG" "$@"
|
|
else
|
|
"$bin" "$@"
|
|
fi
|
|
}
|
|
|
|
act_target() {
|
|
local target="$1" command="$2"
|
|
shift 2
|
|
act "$command" "$(target_ref "$target")" --snapshot "$(target_snapshot "$target")" "$@"
|
|
}
|
|
|
|
get_target() {
|
|
local target="$1"
|
|
shift
|
|
"$bin" get "$(target_ref "$target")" --snapshot "$(target_snapshot "$target")" "$@"
|
|
}
|
|
|
|
is_target() {
|
|
local target="$1" property="$2"
|
|
"$bin" is "$(target_ref "$target")" --snapshot "$(target_snapshot "$target")" --property "$property"
|
|
}
|
|
|
|
wait_target() {
|
|
local target="$1" predicate="$2" timeout="$3"
|
|
shift 3
|
|
"$bin" wait --element "$(target_ref "$target")" --snapshot "$(target_snapshot "$target")" \
|
|
--predicate "$predicate" --timeout "$timeout" "$@"
|
|
}
|
|
|
|
verify() {
|
|
local label="$1" status="$2" expected="$3" expected_mechanism="$4" target="$5" command="$6"
|
|
shift 6
|
|
local before after output command_ok error mechanism
|
|
require_value before "$status"
|
|
output="$(act_target "$target" "$command" "$@" 2>&1)"
|
|
sleep 0.35
|
|
require_value after "$status"
|
|
command_ok="$(json_field "$output" ok)"
|
|
error="$(json_field "$output" error.code)"
|
|
mechanism="$(printf '%s' "$output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)"
|
|
assert "$label" "$([ "$after" = "$expected" ] && [ "$command_ok" = "True" ] && [ "$mechanism" = "$expected_mechanism" ] && echo 1 || echo 0)" \
|
|
"before='$before' after='$after' expected='$expected' ok=$command_ok mechanism=$mechanism${error:+ error=$error}"
|
|
}
|
|
|
|
run_timed() {
|
|
local output_file metadata
|
|
output_file="$(mktemp -t agentdesk-e2e-command.XXXXXX)"
|
|
cleanup_files+=("$output_file")
|
|
metadata="$(AGENT_DESKTOP_E2E_INHERIT_LEASE=1 \
|
|
python3 "$json_tool" run "$output_file" "$@")"
|
|
RUN_EXIT="${metadata%%$'\t'*}"
|
|
RUN_MS="${metadata#*$'\t'}"
|
|
RUN_JSON="$(<"$output_file")"
|
|
}
|
|
|
|
elapsed_in_range() {
|
|
python3 - "$1" "$2" "$3" <<'PY'
|
|
import sys
|
|
value, lower, upper = map(float, sys.argv[1:])
|
|
print(1 if lower <= value <= upper else 0)
|
|
PY
|
|
}
|
|
|
|
running() {
|
|
"$bin" list-apps --app "$app" 2>/dev/null | python3 -c \
|
|
'import json,sys; d=json.load(sys.stdin); print(any(a.get("name") == "AgentDeskFixture" for a in d.get("data",{}).get("apps",[])))' \
|
|
2>/dev/null
|
|
}
|
|
|
|
cleanup() {
|
|
if [ "$fixture_owned" -eq 1 ] && [ -x "$bin" ]; then
|
|
"$bin" close-app "$app" --force >/dev/null 2>&1 || true
|
|
fi
|
|
for file in "${cleanup_files[@]}"; do
|
|
trash_recoverably "$file" || true
|
|
done
|
|
if [ -n "$raw_bin" ] && ! verify_immutable_binary "$raw_bin" "$raw_bin_sha"; then
|
|
contaminated=1
|
|
fi
|
|
if [ -n "$ffi_dylib" ] && ! verify_immutable_binary "$ffi_dylib" "$ffi_dylib_sha"; then
|
|
contaminated=1
|
|
fi
|
|
if [ -n "$ffi_helper" ] && ! verify_immutable_binary "$ffi_helper" "$ffi_helper_sha"; then
|
|
contaminated=1
|
|
fi
|
|
cleanup_isolated_environment
|
|
release_exclusive_lock
|
|
}
|
|
|
|
finish() {
|
|
note "Summary"
|
|
if [ -n "$raw_bin" ] && ! verify_immutable_binary "$raw_bin" "$raw_bin_sha"; then
|
|
contaminated=1
|
|
badmsg "immutable release binary identity changed during the suite"
|
|
fi
|
|
if [ -n "$ffi_dylib" ] && ! verify_immutable_binary "$ffi_dylib" "$ffi_dylib_sha"; then
|
|
contaminated=1
|
|
badmsg "immutable release FFI library identity changed during the suite"
|
|
fi
|
|
if [ -n "$ffi_helper" ] && ! verify_immutable_binary "$ffi_helper" "$ffi_helper_sha"; then
|
|
contaminated=1
|
|
badmsg "immutable release FFI helper identity changed during the suite"
|
|
fi
|
|
printf ' passed: %d failed: %d skipped: %d\n' "$pass" "$fail" "$skip"
|
|
if [ "$fail" -gt 0 ] || [ "$contaminated" -ne 0 ]; then
|
|
printf '\n failures:\n'
|
|
local failure
|
|
for failure in "${failures[@]}"; do
|
|
printf ' - %s\n' "$failure"
|
|
done
|
|
if [ "$contaminated" -ne 0 ]; then
|
|
printf ' - run was contaminated; no success claim is valid\n'
|
|
fi
|
|
return 1
|
|
fi
|
|
if [ "$skip" -gt 0 ]; then
|
|
printf '\n capability skips:\n'
|
|
local skipped
|
|
for skipped in "${skips[@]}"; do
|
|
printf ' - %s\n' "$skipped"
|
|
done
|
|
fi
|
|
echo " all asserted E2E scenarios passed"
|
|
}
|