agent-desktop/crates/core/examples/locator_benchmark.rs

#[path = "locator_benchmark/adapter.rs"]
mod adapter;
#[path = "locator_benchmark/fixture.rs"]
mod fixture;
#[path = "locator_benchmark/fixture_builder.rs"]
mod fixture_builder;
#[path = "locator_benchmark/fixture_node.rs"]
mod fixture_node;
#[path = "locator_benchmark/legacy.rs"]
mod legacy;
#[path = "locator_benchmark/live.rs"]
mod live;
#[path = "locator_benchmark/scenario.rs"]
mod scenario;
#[path = "locator_benchmark/scenarios.rs"]
mod scenarios;

use crate::{
    legacy::run_legacy,
    live::{run_live_count, run_live_direct, run_live_find},
    scenario::Scenario,
};
use serde_json::{Value, json};
use std::{error::Error, hint::black_box};

const WARMUP_RUNS: usize = 5;
const MEASURED_RUNS: usize = 31;

fn main() -> Result<(), Box<dyn Error>> {
    let reports = scenarios::all()
        .iter()
        .map(benchmark_scenario)
        .collect::<Result<Vec<_>, _>>()?;
    let report = json!({
        "schema_version": "1.3",
        "benchmark": "locator-resolution-electron-synthetic",
        "methodology": {
            "warmup_runs": WARMUP_RUNS,
            "measured_runs": MEASURED_RUNS,
            "timing": "wall clock around fixture-to-result resolution; fixture generation excluded",
            "legacy_path": "full AccessibilityNode snapshot, ref allocation, recursive matcher",
            "live_direct_path": "handle-free observed tree, strict direct target selection without ref-map materialization, memoized evaluator",
            "live_find_path": "CLI-compatible default find selection with selected-match-only ref materialization from the same observed tree",
            "live_count_path": "CLI-compatible count selection without ref evidence or ref-map materialization",
            "read_accounting": "per-path requested AX attribute slots plus action, child-label, promotion, and settable probe classes from locator stats",
            "native_ipc": "not measured; native action and settable probe counts vary by role",
            "scope": "deterministic synthetic Chromium/Electron accessibility fixtures; CPU and requested synthetic attributes only",
        },
        "scenarios": reports,
    });
    println!("{}", serde_json::to_string_pretty(&report)?);
    Ok(())
}

fn benchmark_scenario(scenario: &Scenario) -> Result<Value, Box<dyn Error>> {
    for run in 0..WARMUP_RUNS {
        let fixture = scenario.frame(run);
        black_box(run_legacy(fixture, &scenario.query)?);
        black_box(run_live_direct(fixture, &scenario.query)?);
        black_box(run_live_find(fixture, &scenario.query)?);
        black_box(run_live_count(fixture, &scenario.query)?);
    }

    let mut legacy_ns = Vec::with_capacity(MEASURED_RUNS);
    let mut direct_ns = Vec::with_capacity(MEASURED_RUNS);
    let mut find_ns = Vec::with_capacity(MEASURED_RUNS);
    let mut count_ns = Vec::with_capacity(MEASURED_RUNS);
    let mut legacy_counts = Vec::with_capacity(MEASURED_RUNS);
    let mut direct_counts = Vec::with_capacity(MEASURED_RUNS);
    let mut find_counts = Vec::with_capacity(MEASURED_RUNS);
    let mut count_counts = Vec::with_capacity(MEASURED_RUNS);
    let mut legacy_scanned = 0_u64;
    let mut direct_scanned = 0_u64;
    let mut find_scanned = 0_u64;
    let mut legacy_predicate_visits = 0_u64;
    let mut direct_predicate_cells = 0_u64;
    let mut find_predicate_cells = 0_u64;
    let mut direct_dom_matches = 0_u64;
    let mut find_dom_matches = 0_u64;
    let mut find_ref_count = 0_usize;
    let mut count_action_reads = 0_u64;
    let mut direct_action_reads = 0_u64;
    let mut find_action_reads = 0_u64;
    let mut direct_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64);
    let mut find_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64);
    let mut count_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64);
    let mut find_refs_reresolvable = true;

    for run in 0..MEASURED_RUNS {
        let fixture = scenario.frame(run);
        let (legacy, direct, find, count) = match run % 4 {
            0 => (
                run_legacy(fixture, &scenario.query)?,
                run_live_direct(fixture, &scenario.query)?,
                run_live_find(fixture, &scenario.query)?,
                run_live_count(fixture, &scenario.query)?,
            ),
            1 => {
                let direct = run_live_direct(fixture, &scenario.query)?;
                let find = run_live_find(fixture, &scenario.query)?;
                let count = run_live_count(fixture, &scenario.query)?;
                let legacy = run_legacy(fixture, &scenario.query)?;
                (legacy, direct, find, count)
            }
            2 => {
                let find = run_live_find(fixture, &scenario.query)?;
                let count = run_live_count(fixture, &scenario.query)?;
                let legacy = run_legacy(fixture, &scenario.query)?;
                let direct = run_live_direct(fixture, &scenario.query)?;
                (legacy, direct, find, count)
            }
            _ => {
                let count = run_live_count(fixture, &scenario.query)?;
                let legacy = run_legacy(fixture, &scenario.query)?;
                let direct = run_live_direct(fixture, &scenario.query)?;
                let find = run_live_find(fixture, &scenario.query)?;
                (legacy, direct, find, count)
            }
        };
        legacy_ns.push(legacy.0);
        direct_ns.push(direct.elapsed_ns);
        find_ns.push(find.elapsed_ns);
        count_ns.push(count.elapsed_ns);
        legacy_counts.push(legacy.1);
        direct_counts.push(direct.correctness.matches);
        find_counts.push(find.correctness.matches);
        count_counts.push(count.correctness.matches);
        legacy_scanned = legacy.2;
        legacy_predicate_visits = legacy.3;
        direct_scanned = direct.visited;
        direct_predicate_cells = direct.memo_cells;
        direct_dom_matches = direct.dom_matches;
        find_scanned = find.visited;
        find_predicate_cells = find.memo_cells;
        find_dom_matches = find.dom_matches;
        find_ref_count = find.ref_count;
        count_action_reads = count.action_reads;
        direct_action_reads = direct.action_reads;
        find_action_reads = find.action_reads;
        direct_reads = read_counts(&direct);
        find_reads = read_counts(&find);
        count_reads = read_counts(&count);
        find_refs_reresolvable &= find.correctness.selected_refs_reresolvable;
    }

    let expected = scenario.expected_matches;
    let legacy_correct = legacy_counts.iter().all(|count| *count == expected);
    let direct_correct = direct_counts.iter().all(|count| *count == expected);
    let find_correct = find_counts.iter().all(|count| *count == expected);
    let count_correct = count_counts.iter().all(|count| *count == expected);
    let legacy_p50 = percentile(&mut legacy_ns, 50);
    let legacy_p95 = percentile(&mut legacy_ns, 95);
    let direct_p50 = percentile(&mut direct_ns, 50);
    let direct_p95 = percentile(&mut direct_ns, 95);
    let find_p50 = percentile(&mut find_ns, 50);
    let find_p95 = percentile(&mut find_ns, 95);
    let count_p50 = percentile(&mut count_ns, 50);
    let count_p95 = percentile(&mut count_ns, 95);
    let fixture = scenario.frame(0);
    let legacy_attributes_requested = fixture.nodes.len() as u64 * 22;

    Ok(json!({
        "name": scenario.name,
        "fixture": {
            "frames": scenario.frames.len(),
            "nodes": fixture.nodes.len(),
            "roots": fixture.roots.len(),
            "moving_bounds": scenario.moving_bounds_verified(),
            "legacy_locator_result_retained_handles": fixture.nodes.len(),
            "observed_tree_retained_handles": 0,
        },
        "expectation": {
            "matches": expected,
            "cardinality": cardinality(expected),
        },
        "legacy_snapshot": {
            "p50_us": ns_to_us(legacy_p50),
            "p95_us": ns_to_us(legacy_p95),
            "observed_matches": legacy_counts[0],
            "correct_all_runs": legacy_correct,
            "candidate_nodes_scanned": legacy_scanned,
            "predicate_node_visits": legacy_predicate_visits,
            "attributes_requested": legacy_attributes_requested,
            "attribute_batches": fixture.nodes.len(),
            "action_reads": fixture.nodes.len(),
        },
        "live_arena_direct": {
            "p50_us": ns_to_us(direct_p50),
            "p95_us": ns_to_us(direct_p95),
            "observed_matches": direct_counts[0],
            "correct_all_runs": direct_correct,
            "candidate_nodes_scanned": direct_scanned,
            "memo_cells_evaluated": direct_predicate_cells,
            "native_id_dom_matches": direct_dom_matches,
            "attributes_requested": direct_reads.0,
            "attribute_batches": direct_reads.1,
            "child_label_reads": direct_reads.2,
            "promotion_reads": direct_reads.3,
            "settable_reads": direct_reads.4,
            "action_reads": direct_action_reads,
            "peak_handles_owned": direct_reads.5,
        },
        "live_find_selected_refs": {
            "p50_us": ns_to_us(find_p50),
            "p95_us": ns_to_us(find_p95),
            "observed_matches": find_counts[0],
            "correct_all_runs": find_correct,
            "candidate_nodes_scanned": find_scanned,
            "memo_cells_evaluated": find_predicate_cells,
            "native_id_dom_matches": find_dom_matches,
            "ref_count": find_ref_count,
            "selected_refs_reresolvable": find_refs_reresolvable,
            "attributes_requested": find_reads.0,
            "attribute_batches": find_reads.1,
            "child_label_reads": find_reads.2,
            "promotion_reads": find_reads.3,
            "settable_reads": find_reads.4,
            "action_reads": find_action_reads,
            "peak_handles_owned": find_reads.5,
        },
        "live_count_no_refmap": {
            "p50_us": ns_to_us(count_p50),
            "p95_us": ns_to_us(count_p95),
            "observed_matches": count_counts[0],
            "correct_all_runs": count_correct,
            "action_reads": count_action_reads,
            "ref_count": 0,
            "attributes_requested": count_reads.0,
            "attribute_batches": count_reads.1,
            "child_label_reads": count_reads.2,
            "promotion_reads": count_reads.3,
            "settable_reads": count_reads.4,
            "peak_handles_owned": count_reads.5,
        },
        "comparison": {
            "p50_find_speedup": ratio(legacy_p50, find_p50),
            "p95_find_speedup": ratio(legacy_p95, find_p95),
            "p50_direct_target_speedup": ratio(legacy_p50, direct_p50),
            "p95_direct_target_speedup": ratio(legacy_p95, direct_p95),
            "p50_count_speedup": ratio(legacy_p50, count_p50),
            "p95_count_speedup": ratio(legacy_p95, count_p95),
            "find_correctness_delta": i8::from(find_correct) - i8::from(legacy_correct),
            "direct_target_correctness_delta": i8::from(direct_correct) - i8::from(legacy_correct),
            "count_correctness_delta": i8::from(count_correct) - i8::from(legacy_correct),
        },
    }))
}

fn read_counts(run: &live::LiveRun) -> (u64, u64, u64, u64, u64, u64) {
    (
        run.attributes_requested,
        run.attribute_batches,
        run.child_label_reads,
        run.promotion_reads,
        run.settable_reads,
        run.peak_handles_owned,
    )
}

fn percentile(values: &mut [u128], percentile: usize) -> u128 {
    values.sort_unstable();
    let index = (values.len() * percentile).div_ceil(100).saturating_sub(1);
    values[index]
}

fn ns_to_us(nanoseconds: u128) -> f64 {
    nanoseconds as f64 / 1_000.0
}

fn ratio(before: u128, after: u128) -> f64 {
    if after == 0 {
        return 0.0;
    }
    before as f64 / after as f64
}

fn cardinality(matches: usize) -> &'static str {
    match matches {
        0 => "zero",
        1 => "one",
        _ => "many",
    }
}