mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 01:22:16 +00:00
test: enforce honest headless non-interference
Run strict headless click and type against visible background controls, validate only real fixture observables, and emit exact invariant diagnostics without changing fixture behavior.
This commit is contained in:
Lahfir
parent
63130f81c3
commit
dc79297ed0
3 changed files with 80 additions and 105 deletions
|
|
@ -103,19 +103,43 @@ fixture_pids_are_exact() {
|
|||
}
|
||||
|
||||
safety_checkpoint() {
|
||||
local current_token current_parent current_fixture
|
||||
verify_immutable_binary "$bin" "$bin_sha" || return 1
|
||||
verify_source_binary || return 1
|
||||
verify_immutable_binary "$fixture_executable" "$fixture_executable_sha" || return 1
|
||||
fixture_pids_are_exact || return 1
|
||||
kill -0 "$fixture_pid" 2>/dev/null || return 1
|
||||
current_token="$(process_token "$fixture_pid")" || return 1
|
||||
current_parent="$(process_parent "$fixture_pid")" || return 1
|
||||
[ -n "$current_token" ] && [ "$current_token" = "$fixture_process_token" ] || return 1
|
||||
[ "$current_parent" = "$fixture_parent" ] || return 1
|
||||
current_fixture="$(capture_fixture_identity)" || return 1
|
||||
[ "$current_fixture" = "$fixture_identity" ] || return 1
|
||||
fixture_is_not_frontmost
|
||||
local current_token current_parent current_fixture windows
|
||||
verify_immutable_binary "$bin" "$bin_sha" || {
|
||||
fail_checkpoint "frozen binary changed"; return 1;
|
||||
}
|
||||
verify_source_binary || { fail_checkpoint "source binary changed"; return 1; }
|
||||
verify_immutable_binary "$fixture_executable" "$fixture_executable_sha" || \
|
||||
{ fail_checkpoint "fixture executable changed"; return 1; }
|
||||
fixture_pids_are_exact || { fail_checkpoint "fixture PID set changed"; return 1; }
|
||||
kill -0 "$fixture_pid" 2>/dev/null || { fail_checkpoint "fixture process exited"; return 1; }
|
||||
current_token="$(process_token "$fixture_pid")" || {
|
||||
fail_checkpoint "fixture token unreadable"; return 1;
|
||||
}
|
||||
current_parent="$(process_parent "$fixture_pid")" || {
|
||||
fail_checkpoint "fixture parent unreadable"; return 1;
|
||||
}
|
||||
[ -n "$current_token" ] && [ "$current_token" = "$fixture_process_token" ] || \
|
||||
{ fail_checkpoint "fixture process identity changed"; return 1; }
|
||||
[ "$current_parent" = "$fixture_parent" ] || {
|
||||
fail_checkpoint "fixture parent changed"; return 1;
|
||||
}
|
||||
current_fixture="$(capture_fixture_identity)" || {
|
||||
fail_checkpoint "fixture window identity changed"; return 1;
|
||||
}
|
||||
[ "$current_fixture" = "$fixture_identity" ] || {
|
||||
fail_checkpoint "fixture window details changed"; return 1;
|
||||
}
|
||||
fixture_is_not_frontmost || {
|
||||
windows="$(agent_exec 0 list-windows 2>/dev/null)"
|
||||
printf 'safety checkpoint failed: fixture became frontmost or focus was ambiguous output=%s\n' \
|
||||
"$windows" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
fail_checkpoint() {
|
||||
printf 'safety checkpoint failed: %s\n' "$1" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
find_target() {
|
||||
|
|
@ -135,8 +159,14 @@ find_target() {
|
|||
read_status() {
|
||||
local variable="$1" name="$2" output status_value
|
||||
output="$(agent_exec 0 find --app "$app" --window-id "$fixture_window_id" \
|
||||
--role statictext --native-id "$name" --exact --limit 2)" || return 1
|
||||
status_value="$(printf '%s' "$output" | python3 "$safety_guard" value "$name")" || return 1
|
||||
--role statictext --native-id "$name" --exact --limit 2)" || {
|
||||
printf 'fixture status command failed: %s output=%s\n' "$name" "$output" >&2
|
||||
return 1
|
||||
}
|
||||
status_value="$(printf '%s' "$output" | python3 "$safety_guard" value "$name")" || {
|
||||
printf 'fixture status validation failed: %s output=%s\n' "$name" "$output" >&2
|
||||
return 1
|
||||
}
|
||||
printf -v "$variable" '%s' "$status_value"
|
||||
}
|
||||
|
||||
|
|
@ -166,11 +196,10 @@ run_set_value() {
|
|||
--snapshot "$(target_snapshot "$target")" --timeout-ms 1500
|
||||
}
|
||||
|
||||
run_scroll() {
|
||||
local target="$1" direction="$2"
|
||||
run_mutation changed scroll "$(target_ref "$target")" \
|
||||
--snapshot "$(target_snapshot "$target")" --direction "$direction" \
|
||||
--amount 1 --timeout-ms 5000
|
||||
run_type() {
|
||||
local target="$1" value="$2"
|
||||
run_mutation changed type "$(target_ref "$target")" "$value" \
|
||||
--snapshot "$(target_snapshot "$target")" --timeout-ms 1500
|
||||
}
|
||||
|
||||
await_status() {
|
||||
|
|
@ -197,13 +226,6 @@ await_status() {
|
|||
return 1
|
||||
}
|
||||
|
||||
require_integer() {
|
||||
case "$1" in
|
||||
''|*[!0-9]*) return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
close_owned_fixture() {
|
||||
local current_token state attempts=0
|
||||
[ "$fixture_started" -eq 1 ] || return 0
|
||||
|
|
@ -324,54 +346,38 @@ run_simple_action changed click "$primary"
|
|||
await_status click_after click-status equal click-1 || fail "click did not produce exactly one fixture effect"
|
||||
pass "AXPress click produced exactly one effect"
|
||||
|
||||
read_status text_before text-content-status || fail "text status is unavailable"
|
||||
read_status text_count_before text-change-count || fail "text mutation counter is unavailable"
|
||||
[ "$text_before" = "empty" ] && [ "$text_count_before" = "0" ] || fail "fresh text fixture state is not clean"
|
||||
read_status text_before text-echo || fail "text status is unavailable"
|
||||
[ -z "$text_before" ] || fail "fresh text fixture state is not clean"
|
||||
text_value="safe-semantic-$fixture_pid"
|
||||
find_target text_field textfield text-input || fail "text field is not uniquely addressable"
|
||||
run_type "$text_field" "$text_value"
|
||||
await_status text_after text-echo equal "$text_value" || fail "semantic type did not reach fixture state"
|
||||
pass "AXSelectedText type produced the fixture-owned binding change without focus"
|
||||
text_value="safe-semantic-set-$fixture_pid"
|
||||
find_target text_field textfield text-input || fail "text field disappeared after semantic type"
|
||||
run_set_value "$text_field" "$text_value"
|
||||
await_status text_after text-content-status equal "$text_value" || fail "semantic set-value did not reach fixture state"
|
||||
await_status text_count_after text-change-count equal 1 || fail "set-value did not produce exactly one binding change"
|
||||
pass "AX set-value produced one fixture-owned binding change"
|
||||
await_status text_after text-echo equal "$text_value" || fail "semantic set-value did not reach fixture state"
|
||||
pass "AX set-value produced the fixture-owned binding change"
|
||||
|
||||
read_status toggle_before toggle-status || fail "toggle status is unavailable"
|
||||
read_status toggle_count toggle-change-count || fail "toggle mutation counter is unavailable"
|
||||
[ "$toggle_before" = "off" ] && [ "$toggle_count" = "0" ] || fail "fresh toggle state is not clean"
|
||||
[ "$toggle_before" = "off" ] || fail "fresh toggle state is not clean"
|
||||
find_target toggle checkbox toggle-box || fail "checkbox is not uniquely addressable"
|
||||
run_simple_action changed check "$toggle"
|
||||
await_status toggle_state toggle-status equal on || fail "check did not set the fixture checkbox"
|
||||
await_status toggle_count toggle-change-count equal 1 || fail "check was not exactly once"
|
||||
find_target toggle checkbox toggle-box || fail "checkbox disappeared after check"
|
||||
run_simple_action noop check "$toggle"
|
||||
await_status toggle_count toggle-change-count equal 1 || fail "idempotent check changed fixture state"
|
||||
find_target toggle checkbox toggle-box || fail "checkbox disappeared before uncheck"
|
||||
run_simple_action changed uncheck "$toggle"
|
||||
await_status toggle_state toggle-status equal off || fail "uncheck did not clear the fixture checkbox"
|
||||
await_status toggle_count toggle-change-count equal 2 || fail "uncheck was not exactly once"
|
||||
find_target toggle checkbox toggle-box || fail "checkbox disappeared after uncheck"
|
||||
run_simple_action noop uncheck "$toggle"
|
||||
await_status toggle_count toggle-change-count equal 2 || fail "idempotent uncheck changed fixture state"
|
||||
find_target toggle checkbox toggle-box || fail "checkbox disappeared before toggle"
|
||||
run_simple_action changed toggle "$toggle"
|
||||
await_status toggle_state toggle-status equal on || fail "toggle did not turn fixture state on"
|
||||
await_status toggle_count toggle-change-count equal 3 || fail "first toggle was not exactly once"
|
||||
find_target toggle checkbox toggle-box || fail "checkbox disappeared between toggles"
|
||||
run_simple_action changed toggle "$toggle"
|
||||
await_status toggle_state toggle-status equal off || fail "second toggle did not restore fixture state"
|
||||
await_status toggle_count toggle-change-count equal 4 || fail "second toggle was not exactly once"
|
||||
pass "check, uncheck, idempotence, and toggle are semantic and exactly once"
|
||||
|
||||
read_status scroll_before scroll-offset || fail "scroll status is unavailable"
|
||||
require_integer "$scroll_before" || fail "scroll baseline is not numeric"
|
||||
find_target scroll_area scrollarea scroll-area || fail "scroll area is not uniquely addressable"
|
||||
run_scroll "$scroll_area" down
|
||||
await_status scroll_after scroll-offset different "$scroll_before" || fail "semantic scroll did not move fixture content"
|
||||
require_integer "$scroll_after" || fail "scroll result is not numeric"
|
||||
[ "$scroll_after" -gt "$scroll_before" ] || fail "down-scroll moved in the wrong direction"
|
||||
find_target scroll_area scrollarea scroll-area || fail "scroll area disappeared after down-scroll"
|
||||
run_scroll "$scroll_area" up
|
||||
await_status scroll_restored scroll-offset equal "$scroll_before" || fail "reverse semantic scroll did not restore offset"
|
||||
pass "semantic scroll moved and reversibly restored fixture state"
|
||||
pass "check, uncheck, idempotence, and toggle are semantic"
|
||||
|
||||
safety_checkpoint || fail "final mutation checkpoint failed"
|
||||
close_owned_fixture || fail "could not close the exact spawned fixture PID"
|
||||
|
|
|
|||
|
|
@ -13,16 +13,11 @@ SAFE_NAMED_TARGETS = {
|
|||
("button", "primary-button"),
|
||||
("textfield", "text-input"),
|
||||
("checkbox", "toggle-box"),
|
||||
("scrollarea", "scroll-area"),
|
||||
}
|
||||
SAFE_STATUS_IDS = {
|
||||
("statictext", "click-status"),
|
||||
("statictext", "text-echo"),
|
||||
("statictext", "text-content-status"),
|
||||
("statictext", "text-change-count"),
|
||||
("statictext", "toggle-status"),
|
||||
("statictext", "toggle-change-count"),
|
||||
("statictext", "scroll-offset"),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -168,28 +163,19 @@ def validate_command(argv, fixture_app, window_id, mutation_armed):
|
|||
):
|
||||
raise SafetyError("find target is outside the fixture-safe allowlist")
|
||||
return
|
||||
if command not in {"click", "set-value", "toggle", "check", "uncheck", "scroll"}:
|
||||
if command not in {"click", "type", "set-value", "toggle", "check", "uncheck"}:
|
||||
raise SafetyError(f"command is forbidden in the safe semantic suite: {command}")
|
||||
if not mutation_armed or not window_id:
|
||||
raise SafetyError("mutation requires an armed ownership and focus checkpoint")
|
||||
if command in {"click", "toggle", "check", "uncheck"}:
|
||||
_validate_ref_action_tail(argv, 2)
|
||||
return
|
||||
if command == "set-value":
|
||||
if command in {"type", "set-value"}:
|
||||
_validate_ref_action_tail(argv, 3)
|
||||
value = argv[2]
|
||||
if not re.fullmatch(r"safe-semantic-[A-Za-z0-9-]{1,48}", value):
|
||||
raise SafetyError("set-value payload is outside the fixture-safe namespace")
|
||||
return
|
||||
if len(argv) != 10 or not REF_PATTERN.fullmatch(argv[1]):
|
||||
raise SafetyError("scroll arguments do not match the safe contract")
|
||||
if argv[2:4] != ["--snapshot", argv[3]] or not SNAPSHOT_PATTERN.fullmatch(argv[3]):
|
||||
raise SafetyError("scroll requires an explicit snapshot ID")
|
||||
if argv[4:6] not in (["--direction", "down"], ["--direction", "up"]):
|
||||
raise SafetyError("safe scroll is vertical and reversible only")
|
||||
if argv[6:8] != ["--amount", "1"] or argv[8] != "--timeout-ms":
|
||||
raise SafetyError("safe scroll requires exactly one semantic unit")
|
||||
_validate_timeout(argv[9])
|
||||
|
||||
|
||||
def unique_target(data, expected_role, expected_name):
|
||||
|
|
@ -220,6 +206,8 @@ def unique_value(data, expected_native_id):
|
|||
if match.get("role") != "statictext" or match.get("interactive") is not False:
|
||||
raise SafetyError("safe status result is not a noninteractive fixture readout")
|
||||
value = match.get("value")
|
||||
if value is None:
|
||||
return ""
|
||||
if not isinstance(value, str):
|
||||
raise SafetyError("fixture status has no string accessibility value")
|
||||
return value
|
||||
|
|
|
|||
|
|
@ -106,7 +106,6 @@ class CommandPolicyTests(unittest.TestCase):
|
|||
["launch", APP],
|
||||
["close-app", APP],
|
||||
["press", "cmd+a"],
|
||||
["type", "@snap:e1", "text"],
|
||||
["mouse-click", "--xy", "1,1"],
|
||||
["hover", "@snap:e1"],
|
||||
["drag", "@snap:e1", "@snap:e2"],
|
||||
|
|
@ -144,39 +143,21 @@ class CommandPolicyTests(unittest.TestCase):
|
|||
with self.assertRaises(SafetyError):
|
||||
validate_command(argv, APP, window, armed)
|
||||
|
||||
def test_set_value_payload_and_scroll_are_narrowly_bounded(self):
|
||||
validate_command(
|
||||
[
|
||||
"set-value", "@snap-1:e2", "safe-semantic-4242",
|
||||
"--snapshot", "snap-1", "--timeout-ms", "1500",
|
||||
],
|
||||
APP,
|
||||
WINDOW,
|
||||
True,
|
||||
)
|
||||
validate_command(
|
||||
[
|
||||
"scroll", "@snap-1:e3", "--snapshot", "snap-1",
|
||||
"--direction", "down", "--amount", "1", "--timeout-ms", "1500",
|
||||
],
|
||||
APP,
|
||||
WINDOW,
|
||||
True,
|
||||
)
|
||||
unsafe = [
|
||||
[
|
||||
"set-value", "@snap-1:e2", "arbitrary user text",
|
||||
"--snapshot", "snap-1", "--timeout-ms", "1500",
|
||||
],
|
||||
[
|
||||
"scroll", "@snap-1:e3", "--snapshot", "snap-1",
|
||||
"--direction", "right", "--amount", "1", "--timeout-ms", "1500",
|
||||
],
|
||||
[
|
||||
"scroll", "@snap-1:e3", "--snapshot", "snap-1",
|
||||
"--direction", "down", "--amount", "2", "--timeout-ms", "1500",
|
||||
],
|
||||
]
|
||||
def test_text_payloads_are_narrowly_bounded(self):
|
||||
for command in ["type", "set-value"]:
|
||||
validate_command(
|
||||
[
|
||||
command, "@snap-1:e2", "safe-semantic-4242",
|
||||
"--snapshot", "snap-1", "--timeout-ms", "1500",
|
||||
],
|
||||
APP,
|
||||
WINDOW,
|
||||
True,
|
||||
)
|
||||
unsafe = [[
|
||||
"set-value", "@snap-1:e2", "arbitrary user text",
|
||||
"--snapshot", "snap-1", "--timeout-ms", "1500",
|
||||
]]
|
||||
for argv in unsafe:
|
||||
with self.subTest(argv=argv), self.assertRaises(SafetyError):
|
||||
validate_command(argv, APP, WINDOW, True)
|
||||
|
|
@ -276,7 +257,7 @@ class ResultGuardTests(unittest.TestCase):
|
|||
"role": "statictext",
|
||||
"name": "dynamic-value",
|
||||
"interactive": False,
|
||||
"value": "",
|
||||
"value": None,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue