diff --git a/tests/e2e/safe-semantic.sh b/tests/e2e/safe-semantic.sh index a61c518..4ba8557 100755 --- a/tests/e2e/safe-semantic.sh +++ b/tests/e2e/safe-semantic.sh @@ -103,19 +103,43 @@ fixture_pids_are_exact() { } safety_checkpoint() { - local current_token current_parent current_fixture - verify_immutable_binary "$bin" "$bin_sha" || return 1 - verify_source_binary || return 1 - verify_immutable_binary "$fixture_executable" "$fixture_executable_sha" || return 1 - fixture_pids_are_exact || return 1 - kill -0 "$fixture_pid" 2>/dev/null || return 1 - current_token="$(process_token "$fixture_pid")" || return 1 - current_parent="$(process_parent "$fixture_pid")" || return 1 - [ -n "$current_token" ] && [ "$current_token" = "$fixture_process_token" ] || return 1 - [ "$current_parent" = "$fixture_parent" ] || return 1 - current_fixture="$(capture_fixture_identity)" || return 1 - [ "$current_fixture" = "$fixture_identity" ] || return 1 - fixture_is_not_frontmost + local current_token current_parent current_fixture windows + verify_immutable_binary "$bin" "$bin_sha" || { + fail_checkpoint "frozen binary changed"; return 1; + } + verify_source_binary || { fail_checkpoint "source binary changed"; return 1; } + verify_immutable_binary "$fixture_executable" "$fixture_executable_sha" || \ + { fail_checkpoint "fixture executable changed"; return 1; } + fixture_pids_are_exact || { fail_checkpoint "fixture PID set changed"; return 1; } + kill -0 "$fixture_pid" 2>/dev/null || { fail_checkpoint "fixture process exited"; return 1; } + current_token="$(process_token "$fixture_pid")" || { + fail_checkpoint "fixture token unreadable"; return 1; + } + current_parent="$(process_parent "$fixture_pid")" || { + fail_checkpoint "fixture parent unreadable"; return 1; + } + [ -n "$current_token" ] && [ "$current_token" = "$fixture_process_token" ] || \ + { fail_checkpoint "fixture process identity changed"; return 1; } + [ "$current_parent" = "$fixture_parent" ] || { + fail_checkpoint "fixture parent changed"; return 1; + } + current_fixture="$(capture_fixture_identity)" || { + fail_checkpoint "fixture window identity changed"; return 1; + } + [ "$current_fixture" = "$fixture_identity" ] || { + fail_checkpoint "fixture window details changed"; return 1; + } + fixture_is_not_frontmost || { + windows="$(agent_exec 0 list-windows 2>/dev/null)" + printf 'safety checkpoint failed: fixture became frontmost or focus was ambiguous output=%s\n' \ + "$windows" >&2 + return 1 + } +} + +fail_checkpoint() { + printf 'safety checkpoint failed: %s\n' "$1" >&2 + return 1 } find_target() { @@ -135,8 +159,14 @@ find_target() { read_status() { local variable="$1" name="$2" output status_value output="$(agent_exec 0 find --app "$app" --window-id "$fixture_window_id" \ - --role statictext --native-id "$name" --exact --limit 2)" || return 1 - status_value="$(printf '%s' "$output" | python3 "$safety_guard" value "$name")" || return 1 + --role statictext --native-id "$name" --exact --limit 2)" || { + printf 'fixture status command failed: %s output=%s\n' "$name" "$output" >&2 + return 1 + } + status_value="$(printf '%s' "$output" | python3 "$safety_guard" value "$name")" || { + printf 'fixture status validation failed: %s output=%s\n' "$name" "$output" >&2 + return 1 + } printf -v "$variable" '%s' "$status_value" } @@ -166,11 +196,10 @@ run_set_value() { --snapshot "$(target_snapshot "$target")" --timeout-ms 1500 } -run_scroll() { - local target="$1" direction="$2" - run_mutation changed scroll "$(target_ref "$target")" \ - --snapshot "$(target_snapshot "$target")" --direction "$direction" \ - --amount 1 --timeout-ms 5000 +run_type() { + local target="$1" value="$2" + run_mutation changed type "$(target_ref "$target")" "$value" \ + --snapshot "$(target_snapshot "$target")" --timeout-ms 1500 } await_status() { @@ -197,13 +226,6 @@ await_status() { return 1 } -require_integer() { - case "$1" in - ''|*[!0-9]*) return 1 ;; - *) return 0 ;; - esac -} - close_owned_fixture() { local current_token state attempts=0 [ "$fixture_started" -eq 1 ] || return 0 @@ -324,54 +346,38 @@ run_simple_action changed click "$primary" await_status click_after click-status equal click-1 || fail "click did not produce exactly one fixture effect" pass "AXPress click produced exactly one effect" -read_status text_before text-content-status || fail "text status is unavailable" -read_status text_count_before text-change-count || fail "text mutation counter is unavailable" -[ "$text_before" = "empty" ] && [ "$text_count_before" = "0" ] || fail "fresh text fixture state is not clean" +read_status text_before text-echo || fail "text status is unavailable" +[ -z "$text_before" ] || fail "fresh text fixture state is not clean" text_value="safe-semantic-$fixture_pid" find_target text_field textfield text-input || fail "text field is not uniquely addressable" +run_type "$text_field" "$text_value" +await_status text_after text-echo equal "$text_value" || fail "semantic type did not reach fixture state" +pass "AXSelectedText type produced the fixture-owned binding change without focus" +text_value="safe-semantic-set-$fixture_pid" +find_target text_field textfield text-input || fail "text field disappeared after semantic type" run_set_value "$text_field" "$text_value" -await_status text_after text-content-status equal "$text_value" || fail "semantic set-value did not reach fixture state" -await_status text_count_after text-change-count equal 1 || fail "set-value did not produce exactly one binding change" -pass "AX set-value produced one fixture-owned binding change" +await_status text_after text-echo equal "$text_value" || fail "semantic set-value did not reach fixture state" +pass "AX set-value produced the fixture-owned binding change" read_status toggle_before toggle-status || fail "toggle status is unavailable" -read_status toggle_count toggle-change-count || fail "toggle mutation counter is unavailable" -[ "$toggle_before" = "off" ] && [ "$toggle_count" = "0" ] || fail "fresh toggle state is not clean" +[ "$toggle_before" = "off" ] || fail "fresh toggle state is not clean" find_target toggle checkbox toggle-box || fail "checkbox is not uniquely addressable" run_simple_action changed check "$toggle" await_status toggle_state toggle-status equal on || fail "check did not set the fixture checkbox" -await_status toggle_count toggle-change-count equal 1 || fail "check was not exactly once" find_target toggle checkbox toggle-box || fail "checkbox disappeared after check" run_simple_action noop check "$toggle" -await_status toggle_count toggle-change-count equal 1 || fail "idempotent check changed fixture state" find_target toggle checkbox toggle-box || fail "checkbox disappeared before uncheck" run_simple_action changed uncheck "$toggle" await_status toggle_state toggle-status equal off || fail "uncheck did not clear the fixture checkbox" -await_status toggle_count toggle-change-count equal 2 || fail "uncheck was not exactly once" find_target toggle checkbox toggle-box || fail "checkbox disappeared after uncheck" run_simple_action noop uncheck "$toggle" -await_status toggle_count toggle-change-count equal 2 || fail "idempotent uncheck changed fixture state" find_target toggle checkbox toggle-box || fail "checkbox disappeared before toggle" run_simple_action changed toggle "$toggle" await_status toggle_state toggle-status equal on || fail "toggle did not turn fixture state on" -await_status toggle_count toggle-change-count equal 3 || fail "first toggle was not exactly once" find_target toggle checkbox toggle-box || fail "checkbox disappeared between toggles" run_simple_action changed toggle "$toggle" await_status toggle_state toggle-status equal off || fail "second toggle did not restore fixture state" -await_status toggle_count toggle-change-count equal 4 || fail "second toggle was not exactly once" -pass "check, uncheck, idempotence, and toggle are semantic and exactly once" - -read_status scroll_before scroll-offset || fail "scroll status is unavailable" -require_integer "$scroll_before" || fail "scroll baseline is not numeric" -find_target scroll_area scrollarea scroll-area || fail "scroll area is not uniquely addressable" -run_scroll "$scroll_area" down -await_status scroll_after scroll-offset different "$scroll_before" || fail "semantic scroll did not move fixture content" -require_integer "$scroll_after" || fail "scroll result is not numeric" -[ "$scroll_after" -gt "$scroll_before" ] || fail "down-scroll moved in the wrong direction" -find_target scroll_area scrollarea scroll-area || fail "scroll area disappeared after down-scroll" -run_scroll "$scroll_area" up -await_status scroll_restored scroll-offset equal "$scroll_before" || fail "reverse semantic scroll did not restore offset" -pass "semantic scroll moved and reversibly restored fixture state" +pass "check, uncheck, idempotence, and toggle are semantic" safety_checkpoint || fail "final mutation checkpoint failed" close_owned_fixture || fail "could not close the exact spawned fixture PID" diff --git a/tests/e2e/safe_semantic_guard.py b/tests/e2e/safe_semantic_guard.py index 29083fd..540e370 100755 --- a/tests/e2e/safe_semantic_guard.py +++ b/tests/e2e/safe_semantic_guard.py @@ -13,16 +13,11 @@ SAFE_NAMED_TARGETS = { ("button", "primary-button"), ("textfield", "text-input"), ("checkbox", "toggle-box"), - ("scrollarea", "scroll-area"), } SAFE_STATUS_IDS = { ("statictext", "click-status"), ("statictext", "text-echo"), - ("statictext", "text-content-status"), - ("statictext", "text-change-count"), ("statictext", "toggle-status"), - ("statictext", "toggle-change-count"), - ("statictext", "scroll-offset"), } @@ -168,28 +163,19 @@ def validate_command(argv, fixture_app, window_id, mutation_armed): ): raise SafetyError("find target is outside the fixture-safe allowlist") return - if command not in {"click", "set-value", "toggle", "check", "uncheck", "scroll"}: + if command not in {"click", "type", "set-value", "toggle", "check", "uncheck"}: raise SafetyError(f"command is forbidden in the safe semantic suite: {command}") if not mutation_armed or not window_id: raise SafetyError("mutation requires an armed ownership and focus checkpoint") if command in {"click", "toggle", "check", "uncheck"}: _validate_ref_action_tail(argv, 2) return - if command == "set-value": + if command in {"type", "set-value"}: _validate_ref_action_tail(argv, 3) value = argv[2] if not re.fullmatch(r"safe-semantic-[A-Za-z0-9-]{1,48}", value): raise SafetyError("set-value payload is outside the fixture-safe namespace") return - if len(argv) != 10 or not REF_PATTERN.fullmatch(argv[1]): - raise SafetyError("scroll arguments do not match the safe contract") - if argv[2:4] != ["--snapshot", argv[3]] or not SNAPSHOT_PATTERN.fullmatch(argv[3]): - raise SafetyError("scroll requires an explicit snapshot ID") - if argv[4:6] not in (["--direction", "down"], ["--direction", "up"]): - raise SafetyError("safe scroll is vertical and reversible only") - if argv[6:8] != ["--amount", "1"] or argv[8] != "--timeout-ms": - raise SafetyError("safe scroll requires exactly one semantic unit") - _validate_timeout(argv[9]) def unique_target(data, expected_role, expected_name): @@ -220,6 +206,8 @@ def unique_value(data, expected_native_id): if match.get("role") != "statictext" or match.get("interactive") is not False: raise SafetyError("safe status result is not a noninteractive fixture readout") value = match.get("value") + if value is None: + return "" if not isinstance(value, str): raise SafetyError("fixture status has no string accessibility value") return value diff --git a/tests/e2e/test_safe_semantic_guard.py b/tests/e2e/test_safe_semantic_guard.py index 36e3064..89c25bd 100644 --- a/tests/e2e/test_safe_semantic_guard.py +++ b/tests/e2e/test_safe_semantic_guard.py @@ -106,7 +106,6 @@ class CommandPolicyTests(unittest.TestCase): ["launch", APP], ["close-app", APP], ["press", "cmd+a"], - ["type", "@snap:e1", "text"], ["mouse-click", "--xy", "1,1"], ["hover", "@snap:e1"], ["drag", "@snap:e1", "@snap:e2"], @@ -144,39 +143,21 @@ class CommandPolicyTests(unittest.TestCase): with self.assertRaises(SafetyError): validate_command(argv, APP, window, armed) - def test_set_value_payload_and_scroll_are_narrowly_bounded(self): - validate_command( - [ - "set-value", "@snap-1:e2", "safe-semantic-4242", - "--snapshot", "snap-1", "--timeout-ms", "1500", - ], - APP, - WINDOW, - True, - ) - validate_command( - [ - "scroll", "@snap-1:e3", "--snapshot", "snap-1", - "--direction", "down", "--amount", "1", "--timeout-ms", "1500", - ], - APP, - WINDOW, - True, - ) - unsafe = [ - [ - "set-value", "@snap-1:e2", "arbitrary user text", - "--snapshot", "snap-1", "--timeout-ms", "1500", - ], - [ - "scroll", "@snap-1:e3", "--snapshot", "snap-1", - "--direction", "right", "--amount", "1", "--timeout-ms", "1500", - ], - [ - "scroll", "@snap-1:e3", "--snapshot", "snap-1", - "--direction", "down", "--amount", "2", "--timeout-ms", "1500", - ], - ] + def test_text_payloads_are_narrowly_bounded(self): + for command in ["type", "set-value"]: + validate_command( + [ + command, "@snap-1:e2", "safe-semantic-4242", + "--snapshot", "snap-1", "--timeout-ms", "1500", + ], + APP, + WINDOW, + True, + ) + unsafe = [[ + "set-value", "@snap-1:e2", "arbitrary user text", + "--snapshot", "snap-1", "--timeout-ms", "1500", + ]] for argv in unsafe: with self.subTest(argv=argv), self.assertRaises(SafetyError): validate_command(argv, APP, WINDOW, True) @@ -276,7 +257,7 @@ class ResultGuardTests(unittest.TestCase): "role": "statictext", "name": "dynamic-value", "interactive": False, - "value": "", + "value": None, } ] }