Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-07-27 01:22:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix: close native e2e contract gaps

Browse source
This commit is contained in:
Lahfir 2026-07-12 18:01:23 -07:00
parent d0ce41fd84
commit 88e42f02a0
15 changed files with 286 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/core/src/ref_action.rs
View file

@ -10,7 +10,7 @@ use crate::{
};
use serde_json::json;
const TRACE_CAPTURE_BUDGET_MS: u64 = 100;
const TRACE_CAPTURE_BUDGET_MS: u64 = 1_000;
/// A strictly resolved ref-action target plus the tracing identity for it.
pub(crate) struct ResolvedRefAction<'a> {

19
crates/core/src/trace_artifacts_more_tests.rs
View file

@ -1,4 +1,7 @@
use super::tests::{artifacts_session, entry, png_adapter, run_ref_action, setup_artifacts_test};
use super::tests::{
artifacts_session, deadline_png_adapter, entry, png_adapter, run_ref_action,
setup_artifacts_test,
};
use super::*;
use crate::context::CommandContext;
use crate::refs::RefMap;
@ -43,6 +46,20 @@ fn artifacts_full_captures_pre_and_post_pngs() {
assert!(body.contains("screens/"));
}
#[test]
fn artifact_capture_budget_supports_deadline_bound_screenshot_backends() {
let (_home, _lock) = setup_artifacts_test();
let manifest = artifacts_session();
let context = CommandContext::new(Some(manifest.id.clone()), None, false).unwrap();
run_ref_action(&context, &deadline_png_adapter(250), 42).unwrap();
let screens = RefStore::for_session(Some(&manifest.id))
.unwrap()
.trace_dir()
.join("screens");
assert_eq!(std::fs::read_dir(screens).unwrap().count(), 2);
}
#[test]
fn byte_budget_exhaustion_skips_with_budget_reason() {
let (_home, _lock) = setup_artifacts_test();

16
crates/core/src/trace_artifacts_tests.rs
View file

@ -17,7 +17,7 @@ use crate::{capability, refs::RefEntry};
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
const MINI_PNG: &[u8] = &[
pub(super) const MINI_PNG: &[u8] = &[
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4,
0x89, 0x00, 0x00, 0x00, 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00,
@ -78,6 +78,7 @@ pub(super) fn artifacts_session() -> crate::session::SessionManifest {
pub(super) struct PngAdapter {
target: Mutex<Option<ScreenshotTarget>>,
minimum_budget_ms: u64,
}
impl ObservationOps for PngAdapter {
@ -111,9 +112,12 @@ impl SystemOps for PngAdapter {
fn screenshot(
&self,
target: ScreenshotTarget,
_deadline: crate::Deadline,
deadline: crate::Deadline,
) -> Result<ImageBuffer, AdapterError> {
*self.target.lock().unwrap() = Some(target);
if deadline.remaining_ms() < self.minimum_budget_ms {
return Err(deadline.timeout_error());
}
Ok(ImageBuffer {
data: MINI_PNG.to_vec(),
format: ImageFormat::Png,
@ -127,6 +131,14 @@ impl SystemOps for PngAdapter {
pub(super) fn png_adapter() -> PngAdapter {
PngAdapter {
target: Mutex::new(None),
minimum_budget_ms: 0,
}
}
pub(super) fn deadline_png_adapter(minimum_budget_ms: u64) -> PngAdapter {
PngAdapter {
target: Mutex::new(None),
minimum_budget_ms,
}
}

13
crates/macos/src/actions/chain.rs
View file

@ -32,7 +32,11 @@ mod imp {
for (i, step) in def.steps.iter().enumerate() {
ctx.ensure_budget()?;
if matches!(step, ChainStep::CGClick { .. }) && !physical_click_permitted(policy) {
if matches!(
step,
ChainStep::CGClick { .. } | ChainStep::CGDisclosureClick { .. }
) && !physical_click_permitted(policy)
{
return Err(AdapterError::policy_denied_for_policy(
"Physical click fallback is disabled by the current interaction policy",
policy,
@ -62,9 +66,9 @@ mod imp {
pub(crate) fn step_mechanism(step: &ChainStep) -> StepMechanism {
match step {
ChainStep::CGClick { .. } | ChainStep::FocusThenClearByKeyboard => {
StepMechanism::PhysicalSynthetic
}
ChainStep::CGClick { .. }
| ChainStep::CGDisclosureClick { .. }
| ChainStep::FocusThenClearByKeyboard => StepMechanism::PhysicalSynthetic,
_ => StepMechanism::SemanticApi,
}
}
@ -106,6 +110,7 @@ mod imp {
ChainStep::FocusThenClearByKeyboard => "FocusThenClearByKeyboard",
ChainStep::CustomWithDeadline { label, .. } => label,
ChainStep::CGClick { .. } => "CGClick",
ChainStep::CGDisclosureClick { .. } => "CGDisclosureClick",
}
}

43
crates/macos/src/actions/chain_defs.rs
View file

@ -12,10 +12,7 @@ mod imp {
pub(crate) static CLICK_CHAIN: ChainDef = ChainDef {
steps: &[
ChainStep::Action("AXPress"),
ChainStep::CGClick {
button: MouseButton::Left,
count: 1,
},
ChainStep::CGDisclosureClick { expanded: true },
],
suggestion: "Target an element that advertises Click or use an explicit point click.",
continue_after_unverified_delivery: false,
@ -53,19 +50,25 @@ mod imp {
};
pub(crate) static EXPAND_CHAIN: ChainDef = ChainDef {
steps: &[ChainStep::CustomWithDeadline {
label: "expand_verified",
func: chain_disclosure_steps::press_to_expand,
}],
steps: &[
ChainStep::CustomWithDeadline {
label: "expand_verified",
func: chain_disclosure_steps::press_to_expand,
},
ChainStep::CGDisclosureClick { expanded: true },
],
suggestion: "Target a control with a readable expandable state.",
continue_after_unverified_delivery: false,
};
pub(crate) static COLLAPSE_CHAIN: ChainDef = ChainDef {
steps: &[ChainStep::CustomWithDeadline {
label: "collapse_verified",
func: chain_disclosure_steps::press_to_collapse,
}],
steps: &[
ChainStep::CustomWithDeadline {
label: "collapse_verified",
func: chain_disclosure_steps::press_to_collapse,
},
ChainStep::CGDisclosureClick { expanded: false },
],
suggestion: "Target a control with a readable expandable state.",
continue_after_unverified_delivery: false,
};
@ -144,6 +147,22 @@ mod imp {
.with_verified(false),
])
}
#[cfg(test)]
mod tests {
use super::{COLLAPSE_CHAIN, EXPAND_CHAIN};
use crate::actions::chain_step::ChainStep;
#[test]
fn disclosure_chains_end_with_a_headed_physical_fallback() {
for chain in [&EXPAND_CHAIN, &COLLAPSE_CHAIN] {
assert!(matches!(
chain.steps.last(),
Some(ChainStep::CGDisclosureClick { .. })
));
}
}
}
}
#[cfg(not(target_os = "macos"))]

114
crates/macos/src/actions/chain_disclosure_steps.rs
View file

@ -32,7 +32,11 @@ mod imp {
let action = if expanded { "AXExpand" } else { "AXCollapse" };
prepare(element, deadline)?;
if crate::actions::ax_helpers::try_ax_action_or_err(element, action, deadline)? {
return verify_disclosure(element, expanded, deadline).map_err(after_delivery);
if verify_disclosure(element, expanded, deadline).map_err(after_delivery)?
== DeliveryOutcome::DeliveredVerified
{
return Ok(DeliveryOutcome::DeliveredVerified);
}
}
prepare(element, deadline)?;
if crate::actions::ax_helpers::is_attr_settable(element, "AXExpanded", deadline)? {
@ -43,13 +47,21 @@ mod imp {
expanded,
deadline,
)? {
return verify_disclosure(element, expanded, deadline).map_err(after_delivery);
if verify_disclosure(element, expanded, deadline).map_err(after_delivery)?
== DeliveryOutcome::DeliveredVerified
{
return Ok(DeliveryOutcome::DeliveredVerified);
}
}
}
if allow_toggle {
if allow_toggle && disclosed_state(element, deadline)? == Some(!expanded) {
prepare(element, deadline)?;
if crate::actions::ax_helpers::try_ax_action_or_err(element, "AXPress", deadline)? {
return verify_disclosure(element, expanded, deadline).map_err(after_delivery);
if verify_disclosure(element, expanded, deadline).map_err(after_delivery)?
== DeliveryOutcome::DeliveredVerified
{
return Ok(DeliveryOutcome::DeliveredVerified);
}
}
}
Ok(DeliveryOutcome::NotDelivered)
@ -66,8 +78,12 @@ mod imp {
) -> Result<DeliveryOutcome, AdapterError> {
let local_end = Instant::now() + Duration::from_millis(200);
loop {
if disclosed_state(element, deadline)? == Some(expanded) {
return Ok(DeliveryOutcome::DeliveredVerified);
let current_state = disclosed_state(element, deadline)?;
match current_state {
Some(current) if current == expanded => {
return Ok(DeliveryOutcome::DeliveredVerified);
}
_ => {}
}
if deadline.is_expired() {
return Err(deadline.timeout_error().with_details(serde_json::json!({
@ -75,13 +91,64 @@ mod imp {
})));
}
if Instant::now() >= local_end {
return Ok(DeliveryOutcome::DeliveredUnverified);
return unobserved_delivery(current_state, expanded);
}
let pause = deadline.remaining_slice(Duration::from_millis(20))?;
std::thread::sleep(pause.min(Duration::from_millis(20)));
}
}
fn unobserved_delivery(
last_state: Option<bool>,
expanded: bool,
) -> Result<DeliveryOutcome, AdapterError> {
if last_state == Some(!expanded) {
Ok(DeliveryOutcome::NotDelivered)
} else {
Err(AdapterError::new(
agent_desktop_core::ErrorCode::ActionFailed,
"Disclosure state became unreadable after native delivery",
)
.with_details(serde_json::json!({
"verification": "expanded_state_unknown_after_delivery",
})))
}
}
pub(crate) fn physical_fallback_allowed(
element: &AXElement,
expanded: bool,
deadline: Deadline,
) -> Result<bool, AdapterError> {
Ok(disclosed_state(element, deadline)? == Some(!expanded))
}
pub(crate) fn verify_physical_delivery(
element: &AXElement,
expanded: bool,
deadline: Deadline,
) -> Result<DeliveryOutcome, AdapterError> {
let outcome = verify_disclosure(element, expanded, deadline).map_err(after_delivery)?;
require_verified_physical_delivery(outcome)
}
fn require_verified_physical_delivery(
outcome: DeliveryOutcome,
) -> Result<DeliveryOutcome, AdapterError> {
if outcome == DeliveryOutcome::DeliveredVerified {
return Ok(outcome);
}
Err(after_delivery(
AdapterError::new(
agent_desktop_core::ErrorCode::ActionFailed,
"Physical disclosure click did not reach the requested state",
)
.with_details(serde_json::json!({
"verification": "expanded_state_not_observed_after_physical_click",
})),
))
}
fn disclosed_state(
element: &AXElement,
deadline: Deadline,
@ -105,7 +172,10 @@ mod imp {
#[cfg(test)]
mod tests {
use super::disclosure_plan;
use super::{
DeliveryOutcome, disclosure_plan, require_verified_physical_delivery,
unobserved_delivery,
};
#[test]
fn disclosure_plan_never_blindly_toggles_unknown_state() {
@ -116,8 +186,34 @@ mod imp {
assert_eq!(disclosure_plan(Some(true), false), (false, true));
assert_eq!(disclosure_plan(None, false), (false, false));
}
#[test]
fn unobserved_delivery_distinguishes_no_effect_from_unknown_state() {
assert_eq!(
unobserved_delivery(Some(false), true).expect("known opposite state"),
DeliveryOutcome::NotDelivered
);
assert!(unobserved_delivery(None, true).is_err());
}
#[test]
fn physical_fallback_requires_verified_requested_state() {
assert_eq!(
require_verified_physical_delivery(DeliveryOutcome::DeliveredVerified)
.expect("verified delivery"),
DeliveryOutcome::DeliveredVerified
);
let error = require_verified_physical_delivery(DeliveryOutcome::NotDelivered)
.expect_err("unverified physical click must fail closed");
assert_eq!(
error.disposition,
agent_desktop_core::DeliverySemantics::delivered_unverified()
);
}
}
}
#[cfg(target_os = "macos")]
pub(crate) use imp::{press_to_collapse, press_to_expand};
pub(crate) use imp::{
physical_fallback_allowed, press_to_collapse, press_to_expand, verify_physical_delivery,
};

3
crates/macos/src/actions/chain_step.rs
View file

@ -22,4 +22,7 @@ pub(crate) enum ChainStep {
button: MouseButton,
count: u32,
},
CGDisclosureClick {
expanded: bool,
},
}

47
crates/macos/src/actions/chain_step_exec.rs
View file

@ -80,22 +80,47 @@ mod imp {
ChainStep::CustomWithDeadline { label: _, func } => func(el, ctx.deadline),
ChainStep::CGClick { button, count } => {
ctx.ensure_budget()?;
crate::actions::physical_click::click_via_bounds(
el,
crate::actions::physical_click::PhysicalClick {
button: button.clone(),
count: *count,
verified_point: ctx.verified_point.cloned(),
},
policy,
ctx.deadline,
)?;
physical_click(el, button.clone(), *count, ctx, policy)?;
Ok(DeliveryOutcome::DeliveredUnverified)
}
ChainStep::CGDisclosureClick { expanded } => {
if !crate::actions::chain_disclosure_steps::physical_fallback_allowed(
el,
*expanded,
ctx.deadline,
)? {
return Ok(DeliveryOutcome::NotDelivered);
}
physical_click(el, agent_desktop_core::MouseButton::Left, 1, ctx, policy)?;
crate::actions::chain_disclosure_steps::verify_physical_delivery(
el,
*expanded,
ctx.deadline,
)
}
}
}
fn physical_click(
element: &AXElement,
button: agent_desktop_core::MouseButton,
count: u32,
context: &ChainContext<'_>,
policy: InteractionPolicy,
) -> Result<(), AdapterError> {
context.ensure_budget()?;
crate::actions::physical_click::click_via_bounds(
element,
crate::actions::physical_click::PhysicalClick {
button,
count,
verified_point: context.verified_point.cloned(),
},
policy,
context.deadline,
)
}
fn prepare(
element: &AXElement,
deadline: agent_desktop_core::Deadline,

4
crates/macos/src/actions/chain_tests.rs
View file

@ -38,6 +38,10 @@ fn step_mechanism_tags_physical_for_cgclick_and_keyboard_clear() {
}),
StepMechanism::PhysicalSynthetic
);
assert_eq!(
step_mechanism(&ChainStep::CGDisclosureClick { expanded: true }),
StepMechanism::PhysicalSynthetic
);
assert_eq!(
step_mechanism(&ChainStep::FocusThenClearByKeyboard),
StepMechanism::PhysicalSynthetic

17
crates/macos/src/notifications/actions.rs
View file

@ -346,16 +346,15 @@ fn strategy_verified(
pid: i32,
deadline: Deadline,
) -> Result<bool, AdapterError> {
if !strategy_succeeded(outcome, deadline)? {
return Ok(false);
}
match super::dismiss_verify::disappeared(entry, filter, matching_before, pid, deadline) {
Ok(disappeared) => Ok(disappeared),
Err(error) => {
crate::notifications::read::tolerate_ax_strategy_error(error, deadline)?;
Ok(false)
super::dismiss_verify::verified_after_strategy(outcome, deadline, || {
match super::dismiss_verify::disappeared(entry, filter, matching_before, pid, deadline) {
Ok(disappeared) => Ok(disappeared),
Err(error) => {
crate::notifications::read::tolerate_ax_strategy_error(error, deadline)?;
Ok(false)
}
}
}
})
}
fn all_dismiss_strategies_failed() -> AdapterError {

11
crates/macos/src/notifications/dismiss_verify.rs
View file

@ -32,6 +32,17 @@ pub(super) fn disappeared(
}
}
pub(super) fn verified_after_strategy(
outcome: Result<bool, AdapterError>,
deadline: Deadline,
verify: impl FnOnce() -> Result<bool, AdapterError>,
) -> Result<bool, AdapterError> {
if let Err(error) = outcome {
super::read::tolerate_ax_strategy_error(error, deadline)?;
}
verify()
}
fn wait_with(
mut is_present: impl FnMut() -> Result<bool, AdapterError>,
deadline: Deadline,

14
crates/macos/src/notifications/dismiss_verify_tests.rs
View file

@ -21,6 +21,20 @@ fn acknowledged_strategy_succeeds_only_after_reobservation_finds_no_row() {
assert!(result.expect("verification succeeds"));
}
#[test]
fn tolerated_native_error_succeeds_when_reobservation_proves_removal() {
let result = super::verified_after_strategy(
Err(AdapterError::new(
ErrorCode::ActionFailed,
"target disappeared during AXDismiss",
)),
Deadline::standard().expect("deadline"),
|| Ok(true),
);
assert!(result.expect("verified removal must dominate native acknowledgement"));
}
#[test]
fn fallback_identity_includes_the_body() {
let original = NotificationInfo {

6
tests/e2e/scenarios-interaction.sh
View file

@ -116,6 +116,12 @@ assert "headed triple-click delivers a three-tap gesture" \
require_value hover_before hover-status
assert "hover baseline is clean" "$([ "$hover_before" != "hovered" ] && echo 1 || echo 0)" \
"status=$hover_before"
hover_preposition="$("$bin" --headed mouse-move --xy 20,20 2>&1)"
"$bin" focus-window --app "$app" >/dev/null 2>&1
sleep 0.2
assert "hover precondition positions the pointer outside the target" \
"$([ "$(json_field "$hover_preposition" ok)" = "True" ] && echo 1 || echo 0)" \
"ok=$(json_field "$hover_preposition" ok) error=$(json_field "$hover_preposition" error.code)"
hover_snapshot="$("$bin" snapshot --app "$app" --include-bounds --max-depth 30 2>/dev/null)"
hover_xy="$(printf '%s' "$hover_snapshot" | python3 "$json_tool" tree hover-target center 2>/dev/null)" || hover_xy=""
if [ -z "$hover_xy" ]; then

17
tests/e2e/scenarios-surfaces.sh
View file

@ -83,18 +83,27 @@ disclosed_present() {
[ -n "$(json_field "$out" data.match)" ] && echo 1 || echo 0
}
require_target disclosure disclosure disclosure-section
precondition_output="$(act_target "$disclosure" click 2>&1)"
initially_hidden="$(disclosed_present)"
assert "disclosure starts with its child hidden" "$([ "$initially_hidden" = "0" ] && echo 1 || echo 0)" \
"content_present=$initially_hidden"
MODE_FLAG="--headed"
first_expand_output="$(act_target "$disclosure" expand 2>&1)"
MODE_FLAG=""
sleep 0.4
expanded_before="$(disclosed_present)"
assert "disclosure precondition is independently expanded" \
"$([ "$(json_field "$precondition_output" ok)" = "True" ] && [ "$expanded_before" = "1" ] && echo 1 || echo 0)" \
"content_present=$expanded_before error=$(json_field "$precondition_output" error.code)"
assert "expand reveals the initially hidden disclosure content" \
"$([ "$(json_field "$first_expand_output" ok)" = "True" ] && [ "$expanded_before" = "1" ] && echo 1 || echo 0)" \
"content_present=$expanded_before error=$(json_field "$first_expand_output" error.code)"
require_target disclosure disclosure disclosure-section
MODE_FLAG="--headed"
collapse_output="$(act_target "$disclosure" collapse 2>&1)"
MODE_FLAG=""
sleep 0.4
collapsed_hidden="$(disclosed_present)"
require_target disclosure disclosure disclosure-section
MODE_FLAG="--headed"
expand_output="$(act_target "$disclosure" expand 2>&1)"
MODE_FLAG=""
sleep 0.4
expanded_shown="$(disclosed_present)"
assert "collapse hides the disclosed content" \

15
tests/e2e/scenarios-trace-performance.sh
View file

@ -44,7 +44,18 @@ trace_events="$(printf '%s' "$trace_show" | python3 -c '
import json,sys
data=json.load(sys.stdin)
events=[entry.get("event") for entry in data.get("data",{}).get("events",[])]
print(1 if {"command.start","command.end","snapshot.saved"}.issubset(events) else 0)
required={"command.start","command.end","snapshot.saved","action.artifacts"}
print(1 if required.issubset(events) else 0)
' 2>/dev/null)"
trace_artifact_skips="$(printf '%s' "$trace_show" | python3 -c '
import json,sys
data=json.load(sys.stdin)
keys=("skipped","skipped_pre","skipped_post")
items=[]
for event in data.get("data",{}).get("events",[]):
if event.get("event") == "action.artifacts":
items.extend(f"{key}={event[key]}" for key in keys if key in event)
print(",".join(items) or "none")
' 2>/dev/null)"
trace_dir="${HOME}/.agent-desktop/sessions/${trace_session}/trace"
png_magic=0
@ -59,7 +70,7 @@ assert "trace show includes command and snapshot events" "$trace_events" \
trace_artifacts_ok="$([ "$(json_field "$trace_click" ok)" = "True" ] && \
[ "$(json_field "$trace_session_type" ok)" = "True" ] && [ "$png_magic" = "1" ] && echo 1 || echo 0)"
assert "trace screenshot artifacts have PNG magic" "$trace_artifacts_ok" \
"click_ok=$(json_field "$trace_click" ok) type_ok=$(json_field "$trace_session_type" ok) directory=$trace_dir/screens exists=$([ -d "$trace_dir/screens" ] && echo yes || echo no) pngs=$(find "$trace_dir/screens" -name '*.png' 2>/dev/null | wc -l | tr -d ' ')"
"click_ok=$(json_field "$trace_click" ok) type_ok=$(json_field "$trace_session_type" ok) directory=$trace_dir/screens exists=$([ -d "$trace_dir/screens" ] && echo yes || echo no) pngs=$(find "$trace_dir/screens" -name '*.png' 2>/dev/null | wc -l | tr -d ' ') skips=$trace_artifact_skips"
trace_html="$(mktemp -t agentdesk-e2e-trace-export.XXXXXX.html)"
cleanup_files+=("$trace_html")