mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-08-01 03:40:58 +00:00
Settle the Playwright-grade reliability contract in agent-desktop-core before the Windows/Linux adapters are built, so they inherit it instead of redesigning it. Every command now observes, waits, verifies, and reports honestly instead of firing blindly. Highlights: capability-supertrait split of PlatformAdapter with not_supported() defaults; canonical role/state vocabulary with live `is --property visible`; display enumeration (`list-displays`) and honest `--screen` with scale factor; truthful Automation permission; `native_id` identity spine; window-id-first resolution; serializable `LocatorQuery` with live `find`; default-on auto-wait before every ref action; three-way `hit_test` occlusion gate; `scroll_into_view` in core; core accessible-name precedence; typed `ActionStep` delivery tier; `ProcessState` and `APP_UNRESPONSIVE`; `LaunchOptions`; baseline-diff desktop signals (`wait --event`); typed clipboard (`Text`/`Image`/`FileUrls`); mouse modifier chords and `mouse-wheel`. Hardened through a 35-reviewer pass with independent validation and a green live e2e gate (109/0), plus a head-vs-main performance comparison harness. BREAKING CHANGE: default-on auto-wait changes the timing of every previously-untouched ref-action call (bounded 5000 ms default; `--timeout-ms 0` restores single-shot). `ENVELOPE_VERSION` is now `2.1` (adds the `APP_UNRESPONSIVE` code and process state in error details). FFI ABI major is `3` (append-only struct evolution; `wait --event` is intentionally not exposed over FFI). The legacy string clipboard API is removed in favor of typed content. `key-down`/`key-up` fail closed until daemon-owned held input exists. `close-app` verifies termination and the osascript fallback path is removed. `--text` matching is subtree containment: `find --text X --first` returns the outermost matching container.
151 lines
5.9 KiB
Python
Executable file
151 lines
5.9 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
import ctypes
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from ctypes import POINTER, Structure, byref, c_bool, c_char_p, c_double, c_int32
|
|
from ctypes import c_int64, c_uint32, c_uint64, c_uint8, c_void_p
|
|
|
|
|
|
class Scroll(Structure):
|
|
_fields_ = [("direction", c_int32), ("amount", c_uint32)]
|
|
|
|
|
|
class Key(Structure):
|
|
_fields_ = [("key", c_char_p), ("modifiers", POINTER(c_int32)), ("count", c_uint32)]
|
|
|
|
|
|
class Point(Structure):
|
|
_fields_ = [("x", c_double), ("y", c_double)]
|
|
|
|
|
|
class Drag(Structure):
|
|
_fields_ = [("from_point", Point), ("to_point", Point),
|
|
("duration_ms", c_uint64), ("drop_delay_ms", c_uint64)]
|
|
|
|
|
|
class Action(Structure):
|
|
_fields_ = [("kind", c_int32), ("text", c_char_p), ("scroll", Scroll),
|
|
("key", Key), ("drag", Drag)]
|
|
|
|
|
|
def bind(lib, name, result, args):
|
|
function = getattr(lib, name)
|
|
function.restype = result
|
|
function.argtypes = args
|
|
return function
|
|
|
|
|
|
def take_json(pointer, free_string):
|
|
if not pointer.value:
|
|
return None
|
|
raw = pointer.value
|
|
free_string(pointer)
|
|
return json.loads(raw)
|
|
|
|
|
|
def walk(node):
|
|
yield node
|
|
for child in node.get("children", []):
|
|
yield from walk(child)
|
|
|
|
|
|
def ref_for(envelope, native_id):
|
|
for node in walk(envelope["data"]["tree"]):
|
|
identifier = node.get("native_id")
|
|
identifier_value = identifier.get("value") if isinstance(identifier, dict) else identifier
|
|
if identifier_value == native_id:
|
|
ref_id = node.get("ref") or node.get("ref_id")
|
|
if ref_id:
|
|
return ref_id
|
|
raise RuntimeError(f"snapshot omitted ref for {native_id}")
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) != 2 or sys.argv[1] not in {"default", "zero"}:
|
|
raise RuntimeError("usage: ffi_live.py default|zero")
|
|
mode = sys.argv[1]
|
|
dylib = os.environ.get("AGENT_DESKTOP_E2E_FFI_DYLIB", "")
|
|
if not dylib or not os.path.isfile(dylib):
|
|
raise RuntimeError("immutable release FFI dylib is unavailable")
|
|
lib = ctypes.CDLL(dylib)
|
|
create = bind(lib, "ad_adapter_create_with_session", c_void_p, [c_char_p])
|
|
destroy = bind(lib, "ad_adapter_destroy", None, [c_void_p])
|
|
free_string = bind(lib, "ad_free_string", None, [c_char_p])
|
|
action_size = bind(lib, "ad_action_size", ctypes.c_size_t, [])
|
|
snapshot = bind(lib, "ad_snapshot", c_int32,
|
|
[c_void_p, c_char_p, c_int32, c_uint8, c_bool, c_bool, POINTER(c_char_p)])
|
|
execute = bind(lib, "ad_execute_by_ref", c_int32,
|
|
[c_void_p, c_char_p, c_char_p, POINTER(Action), c_int32, POINTER(c_char_p)])
|
|
execute_timeout = bind(lib, "ad_execute_by_ref_timeout", c_int32,
|
|
[c_void_p, c_char_p, c_char_p, POINTER(Action), c_int32,
|
|
c_int64, POINTER(c_char_p)])
|
|
if action_size() != ctypes.sizeof(Action):
|
|
raise RuntimeError("AdAction ctypes layout does not match the loaded release dylib")
|
|
|
|
adapter = create(f"ffi-live-{os.getpid()}".encode())
|
|
if not adapter:
|
|
raise RuntimeError("release dylib refused to create a native adapter")
|
|
click = Action()
|
|
click.kind = 0
|
|
|
|
def snap():
|
|
out = c_char_p()
|
|
rc = snapshot(adapter, b"AgentDeskFixture", 0, 10, False, False, byref(out))
|
|
envelope = take_json(out, free_string)
|
|
if rc == -8 or (envelope and envelope.get("error", {}).get("code") == "PLATFORM_NOT_SUPPORTED"):
|
|
raise RuntimeError("stub FFI dylib rejected: native macOS adapter is required")
|
|
if rc != 0 or not envelope or not envelope.get("ok"):
|
|
raise RuntimeError(f"native FFI snapshot failed: rc={rc} envelope={envelope}")
|
|
return envelope
|
|
|
|
def act(ref_id, timeout=None):
|
|
out = c_char_p()
|
|
started = time.monotonic()
|
|
if timeout is None:
|
|
rc = execute(adapter, ref_id.encode(), None, byref(click), 0, byref(out))
|
|
else:
|
|
rc = execute_timeout(adapter, ref_id.encode(), None, byref(click), 0,
|
|
timeout, byref(out))
|
|
elapsed = round((time.monotonic() - started) * 1000)
|
|
return rc, take_json(out, free_string), elapsed
|
|
|
|
try:
|
|
observed = snap()
|
|
if act(ref_for(observed, "reset-delayed-button"))[0] != 0:
|
|
raise RuntimeError("FFI fixture reset failed")
|
|
observed = snap()
|
|
if act(ref_for(observed, "enable-later"))[0] != 0:
|
|
raise RuntimeError("FFI delayed-enable trigger failed")
|
|
delayed = ref_for(observed, "delayed-button")
|
|
if mode == "default":
|
|
default_rc, default_env, default_ms = act(delayed)
|
|
if default_rc != 0 or not default_env or not default_env.get("ok"):
|
|
raise RuntimeError(f"default FFI auto-wait failed: {default_env}")
|
|
if not 500 <= default_ms <= 2800:
|
|
raise RuntimeError(f"default FFI auto-wait timing out of range: {default_ms}ms")
|
|
print(json.dumps({"ok": True, "mode": mode, "elapsed_ms": default_ms}))
|
|
return
|
|
|
|
zero_rc, zero_env, zero_ms = act(delayed, 0)
|
|
if zero_rc == 0 or not zero_env or zero_env.get("ok") is not False:
|
|
raise RuntimeError(f"timeout-zero FFI action unexpectedly dispatched: {zero_env}")
|
|
details = zero_env.get("error", {}).get("details", {})
|
|
if details.get("actionable") is not False or not details.get("checks"):
|
|
raise RuntimeError(f"timeout-zero FFI failure was not an actionability rejection: {zero_env}")
|
|
if zero_ms > 900:
|
|
raise RuntimeError(f"timeout-zero FFI action was not single-shot: {zero_ms}ms")
|
|
time.sleep(1.2)
|
|
print(json.dumps({"ok": True, "mode": mode, "elapsed_ms": zero_ms,
|
|
"zero_code": zero_env.get("error", {}).get("code")}))
|
|
finally:
|
|
destroy(adapter)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print(json.dumps({"ok": False, "error": str(error)}))
|
|
sys.exit(1)
|