diff --git a/benchmarks/locator-resolution/README.md b/benchmarks/locator-resolution/README.md index 75607e8..445784b 100644 --- a/benchmarks/locator-resolution/README.md +++ b/benchmarks/locator-resolution/README.md @@ -24,17 +24,10 @@ role-dependent action and settable probes make that model inaccurate. Native Slack/Electron IPC measurements require Accessibility permission and belong in the privileged macOS integration suite. -Generate the standalone sanitized report from the synthetic and paired live -evidence: - -```bash -python3 benchmarks/locator-resolution/generate_performance_report.py \ - --synthetic /private/tmp/agent-desktop-locator-synthetic.json \ - --live /private/tmp/agent-desktop-electron-slack-paired-final.json \ - --output /private/tmp/agent-desktop-locator-performance-report.html -``` - -Benchmark outputs are generated evidence, not source. Keep them outside the repository. +This synthetic benchmark JSON is consumed by the repo-wide performance report: +run `bash scripts/perf-baseline-compare.sh` (optionally with `--apps "Slack,Google +Chrome"`) to produce `report.html`, covering the HEAD-vs-main fixture A/B, +optional real-app read-only probes, and this synthetic benchmark. For privileged Electron/Chromium runs, the macOS adapter first asks whether the application root exposes `AXManualAccessibility` as settable, reads its current diff --git a/benchmarks/locator-resolution/generate_performance_report.py b/benchmarks/locator-resolution/generate_performance_report.py deleted file mode 100644 index db9a353..0000000 --- a/benchmarks/locator-resolution/generate_performance_report.py +++ /dev/null @@ -1,336 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import html -import json -from pathlib import Path - - -DEFAULT_BASELINE_BYTES = 1_793_312 -DEFAULT_CURRENT_BYTES = 2_162_496 -SCENARIO_LABELS = { - "deep_anonymous_has_text": "Deep anonymous text", - "duplicate_button_role_and_name": "Duplicate role + name", - "electron_dual_identifier_moving_bounds": "Moving Electron bounds", - "large_nested_channel_has_unread": "Large nested tree", -} - - -def require(mapping, *keys): - value = mapping - for key in keys: - if not isinstance(value, dict) or key not in value: - raise ValueError(f"missing required field: {'.'.join(keys)}") - value = value[key] - return value - - -def load_json(path): - with path.open(encoding="utf-8") as source: - return json.load(source) - - -def fmt(value, digits=1): - return f"{float(value):,.{digits}f}" - - -def rate(value): - return float(value) * 100.0 - - -def directional_change(value, lower_label, higher_label, unchanged_label): - value = float(value) - if value < 0: - return abs(value), lower_label - if value > 0: - return value, higher_label - return 0.0, unchanged_label - - -def size_change(baseline_bytes, current_bytes): - delta = current_bytes - baseline_bytes - percent = abs(delta) / baseline_bytes * 100.0 - if delta < 0: - return percent, "smaller", f"{delta:,}" - if delta > 0: - return percent, "larger", f"+{delta:,}" - return 0.0, "the same size", "0" - - -def paired_wall_deltas(live): - baseline = { - int(sample["pair_index"]): float(sample["wall_ms"]) - for sample in require(live, "runs", "baseline", "samples") - if sample.get("command_success") and isinstance(sample.get("wall_ms"), (int, float)) - } - current = { - int(sample["pair_index"]): float(sample["wall_ms"]) - for sample in require(live, "runs", "current", "samples") - if sample.get("command_success") and isinstance(sample.get("wall_ms"), (int, float)) - } - indexes = sorted(baseline.keys() & current.keys()) - return [current[index] - baseline[index] for index in indexes] - - -def svg_grouped_bars(title, baseline, current, unit): - labels = ("p50", "p95") - values = [baseline[0], current[0], baseline[1], current[1]] - maximum = max(values) * 1.18 - bars = [] - for group, label in enumerate(labels): - group_x = 86 + group * 200 - for offset, (series, value, css_class) in enumerate( - (("Baseline", baseline[group], "baseline"), ("Current", current[group], "current")) - ): - height = 155 * value / maximum - x = group_x + offset * 66 - y = 193 - height - bars.append( - f'' - f'{fmt(value)} {unit}' - f'{series}' - ) - bars.append(f'{label}') - summary = ( - f"{title}. Baseline p50 {fmt(baseline[0])} and p95 {fmt(baseline[1])} {unit}; " - f"current p50 {fmt(current[0])} and p95 {fmt(current[1])} {unit}." - ) - return ( - f'' - f"{html.escape(title)}{html.escape(summary)}" - '' - + "".join(bars) - + "" - ) - - -def svg_delta_plot(deltas): - if not deltas: - raise ValueError("live report has no comparable successful wall-time pairs") - ordered = sorted(deltas) - limit = max(abs(min(ordered)), abs(max(ordered)), 1.0) * 1.1 - zero_y = 132 - scale = 102 / limit - plot_width = 610 - bar_width = max(3.0, plot_width / len(ordered) - 3.0) - bars = [] - for index, value in enumerate(ordered): - x = 49 + index * plot_width / len(ordered) - height = abs(value) * scale - y = zero_y if value >= 0 else zero_y - height - css_class = "slower" if value > 0 else "faster" - bars.append( - f'' - f"Pair {index + 1}: {fmt(value)} ms" - ) - faster = sum(value < 0 for value in ordered) - summary = ( - f"Ordered current minus baseline wall-time deltas for {len(ordered)} pairs. " - f"Current was faster in {faster} pairs. Negative values are faster." - ) - return ( - 'Paired wall-time deltas' - f"{html.escape(summary)}" - 'fasterslower' - '' - + "".join(bars) - + f'{fmt(min(ordered))} ms' - + f'{fmt(max(ordered))} ms' - + "" - ) - - -def svg_rate_chart(live): - metrics = ( - ("Correct", "correct_result_rate"), - ("Addressable", "addressable_result_rate"), - ("Exact re-resolution", "exact_reresolution_rate"), - ) - rows = [] - summaries = [] - for index, (label, key) in enumerate(metrics): - baseline = rate(require(live, "runs", "baseline", "reliability", key)) - current = rate(require(live, "runs", "current", "reliability", key)) - summaries.append(f"{label}: baseline {fmt(baseline, 0)} percent, current {fmt(current, 0)} percent") - y = 42 + index * 67 - rows.append(f'{html.escape(label)}') - for offset, (value, css_class, series) in enumerate( - ((baseline, "baseline", "Baseline"), (current, "current", "Current")) - ): - bar_y = y + offset * 22 - rows.append( - f'' - f'' - f'{series} {fmt(value, 0)}%' - ) - summary = "; ".join(summaries) - return ( - 'Live reliability' - f"{html.escape(summary)}" - + "".join(rows) - + "" - ) - - -def svg_speedups(synthetic): - speedups = [ - ( - SCENARIO_LABELS.get(item["name"], item["name"].replace("_", " ").title()), - float(require(item, "comparison", "p50_find_speedup")), - float(require(item, "comparison", "p95_find_speedup")), - ) - for item in require(synthetic, "scenarios") - ] - maximum = max(max(row[1:]) for row in speedups) * 1.15 - rows = [] - for index, (label, p50, p95) in enumerate(speedups): - y = 34 + index * 64 - rows.append(f'{html.escape(label)}') - for offset, (value, css_class, percentile) in enumerate( - ((p50, "current", "p50"), (p95, "synthetic-p95", "p95")) - ): - bar_y = y + offset * 22 - width = value / maximum * 365 - rows.append( - f'' - f'{percentile} {fmt(value, 2)}×' - ) - summary = "; ".join(f"{label}: p50 {fmt(p50, 2)} times, p95 {fmt(p95, 2)} times" for label, p50, p95 in speedups) - return ( - '' - "Synthetic find speedup" - f"{html.escape(summary)}" - + "".join(rows) - + "" - ) - - -def svg_binary_size(baseline_bytes, current_bytes): - maximum = max(baseline_bytes, current_bytes) * 1.15 - bars = [] - for index, (label, value, css_class) in enumerate( - (("Baseline", baseline_bytes, "baseline"), ("Current", current_bytes, "current")) - ): - width = value / maximum * 470 - y = 45 + index * 62 - bars.append( - f'{label}' - f'' - f'{value / 1_000_000:.3f} MB' - ) - return ( - '' - "Release binary sizeDecimal megabytes. The current binary remains below the 15 megabyte project ceiling." - + "".join(bars) - + "" - ) - - -def render_report(synthetic, live, baseline_bytes, current_bytes, current_sha256=None): - wall = tuple( - require(live, "runs", name, "metrics", "end_to_end_wall_all_attempts", percentile) - for name in ("baseline", "current") - for percentile in ("p50", "p95") - ) - cpu = tuple( - require(live, "runs", name, "metrics", "process_cpu_all_attempts", percentile) - for name in ("baseline", "current") - for percentile in ("p50", "p95") - ) - pairs = int(require(live, "paired_comparison", "comparable_successful_pairs")) - faster_rate = rate(require(live, "paired_comparison", "current_minus_baseline", "current_faster_wall_rate")) - delta_p50 = require(live, "paired_comparison", "current_minus_baseline", "wall", "p50") - wall_change, wall_direction = directional_change( - delta_p50, "reduction", "increase", "change" - ) - reliability_keys = ( - "correct_result_rate", - "addressable_result_rate", - "exact_reresolution_rate", - ) - current_reliability = min( - rate(require(live, "runs", "current", "reliability", key)) - for key in reliability_keys - ) - synthetic_scenarios = require(synthetic, "scenarios") - synthetic_total = len(synthetic_scenarios) - legacy_synthetic_correct = sum( - bool(require(item, "legacy_snapshot", "correct_all_runs")) - for item in synthetic_scenarios - ) - current_synthetic_correct = sum( - bool(require(item, "live_find_selected_refs", "correct_all_runs")) - and bool(require(item, "live_find_selected_refs", "selected_refs_reresolvable")) - for item in synthetic_scenarios - ) - size_percent, size_direction, size_delta = size_change(baseline_bytes, current_bytes) - captured_at = html.escape(str(live.get("generated_at", "unknown time"))) - measured_sha = str(require(live, "runs", "current", "binary", "sha256")) - if current_sha256 and measured_sha != current_sha256: - live_evidence_note = ( - f'

Live evidence was captured at {captured_at} ' - f'against build {html.escape(measured_sha[:12])}. The final build ' - f'{html.escape(current_sha256[:12])} differs and could not be re-probed; ' - 'treat live latency as pre-final evidence.

' - ) - else: - live_evidence_note = ( - f'

Live evidence captured at {captured_at} against build ' - f'{html.escape(measured_sha[:12])}.

' - ) - return f''' - -agent-desktop performance evidence -
-

agent-desktop performance evidence

Baseline versus current branch, with live read-only Electron evidence separated from deterministic synthetic locator evidence.

-{live_evidence_note} -
{fmt(wall_change)} mslive paired p50 wall-time {wall_direction}
{fmt(faster_rate, 1)}%live pairs where current was faster
{fmt(current_reliability, 1)}%lowest current correctness reliability rate
-
-

Live wall time

Slack (read-only), {pairs} successful paired runs. Lower is better.

BaselineCurrent
{svg_grouped_bars("Live end-to-end wall time", (wall[0], wall[1]), (wall[2], wall[3]), "ms")}
-

Live process CPU

Per-command user + system CPU. Lower is better.

BaselineCurrent
{svg_grouped_bars("Live process CPU", (cpu[0], cpu[1]), (cpu[2], cpu[3]), "ms")}
-

Paired live wall-time deltas

All {pairs} current-minus-baseline deltas, ordered by value. Negative means current is faster.

{svg_delta_plot(paired_wall_deltas(live))}
-

Live locator reliability

Command exit success alone was 100% for both binaries; correctness, addressability, and exact re-resolution distinguish usable results.

{svg_rate_chart(live)}
-

Synthetic locator speedup

31 measured runs after 5 warmups per deterministic Chromium/Electron fixture. Current find was correct and re-resolvable in {current_synthetic_correct}/{synthetic_total} scenarios; legacy was correct in {legacy_synthetic_correct}/{synthetic_total}. Speed ratios are timing evidence, not a substitute for correctness. These timings exclude native accessibility IPC.

p50 find speedupp95 find speedup
{svg_speedups(synthetic)}
-

Release binary size

Current is {fmt(size_percent, 2)}% {size_direction} ({size_delta} bytes), while remaining far below the 15 MB ceiling.

{svg_binary_size(baseline_bytes, current_bytes)}
-
-

Scope and limitations

  • Live data is one machine, one app state, and one macOS accessibility-permission state; it is not a cross-machine claim.
  • The live operation was read-only find plus exact ref re-resolution. No click, typing, navigation, or message mutation occurred.
  • Synthetic data measures platform-neutral locator work on deterministic Electron-shaped fixtures; it does not measure native IPC or whole-command latency.
  • Binary size is {fmt(size_percent, 2)}% {size_direction}; performance evidence does not erase packaging changes.
-
''' - - -def parse_args(argv=None): - parser = argparse.ArgumentParser(description="Generate the sanitized agent-desktop performance report") - parser.add_argument("--synthetic", type=Path, required=True, help="synthetic locator benchmark JSON") - parser.add_argument("--live", type=Path, required=True, help="paired live benchmark JSON") - parser.add_argument("--output", type=Path, required=True, help="standalone HTML destination") - parser.add_argument("--baseline-bytes", type=int, default=DEFAULT_BASELINE_BYTES) - parser.add_argument("--current-bytes", type=int, default=DEFAULT_CURRENT_BYTES) - parser.add_argument("--current-sha256") - return parser.parse_args(argv) - - -def main(argv=None): - args = parse_args(argv) - if args.baseline_bytes <= 0 or args.current_bytes <= 0: - raise ValueError("binary sizes must be positive") - report = render_report( - load_json(args.synthetic), - load_json(args.live), - args.baseline_bytes, - args.current_bytes, - args.current_sha256, - ) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(report, encoding="utf-8") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/locator-resolution/test_performance_report.py b/benchmarks/locator-resolution/test_performance_report.py deleted file mode 100644 index 7cd4d59..0000000 --- a/benchmarks/locator-resolution/test_performance_report.py +++ /dev/null @@ -1,158 +0,0 @@ -import importlib.util -import json -import tempfile -import unittest -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("generate_performance_report.py") -SPEC = importlib.util.spec_from_file_location("performance_report", MODULE_PATH) -REPORT = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(REPORT) - - -def synthetic_fixture(): - return { - "scenarios": [ - { - "name": name, - "comparison": { - "p50_find_speedup": 1.25 + index / 10, - "p95_find_speedup": 1.35 + index / 10, - }, - "legacy_snapshot": {"correct_all_runs": True}, - "live_find_selected_refs": { - "correct_all_runs": True, - "selected_refs_reresolvable": True, - }, - } - for index, name in enumerate(REPORT.SCENARIO_LABELS) - ] - } - - -def live_fixture(): - samples = [ - { - "pair_index": index, - "wall_ms": wall, - "command_success": True, - "stats": {"private_ui_text": "SECRET-SAMPLE-SENTINEL"}, - } - for index, wall in enumerate((300.0, 320.0, 340.0)) - ] - current_samples = [dict(sample, wall_ms=sample["wall_ms"] - 70.0) for sample in samples] - reliability = { - "correct_result_rate": 0.0, - "addressable_result_rate": 0.0, - "exact_reresolution_rate": 0.0, - } - return { - "app": "SECRET-APP-SENTINEL", - "state_reference": { - "target_window_id": "SECRET-WINDOW-SENTINEL", - "windows": [{"title": "SECRET-TITLE-SENTINEL", "pid": 987654}], - }, - "paired_comparison": { - "comparable_successful_pairs": 3, - "current_minus_baseline": { - "current_faster_wall_rate": 1.0, - "wall": {"p50": -70.0}, - }, - }, - "runs": { - "baseline": { - "binary": {"sha256": "baseline-build-sha"}, - "samples": samples, - "metrics": { - "end_to_end_wall_all_attempts": {"p50": 300.0, "p95": 340.0}, - "process_cpu_all_attempts": {"p50": 50.0, "p95": 60.0}, - }, - "reliability": reliability, - }, - "current": { - "binary": {"sha256": "measured-build-sha"}, - "samples": current_samples, - "metrics": { - "end_to_end_wall_all_attempts": {"p50": 230.0, "p95": 270.0}, - "process_cpu_all_attempts": {"p50": 31.0, "p95": 40.0}, - }, - "reliability": {key: 1.0 for key in reliability}, - }, - }, - } - - -class PerformanceReportTests(unittest.TestCase): - def test_report_contains_metrics_and_excludes_sensitive_input(self): - output = REPORT.render_report(synthetic_fixture(), live_fixture(), 1_793_312, 2_162_480) - self.assertIn("Slack (read-only)", output) - self.assertIn("230.0", output) - self.assertIn("20.59%", output) - self.assertIn("Paired live wall-time deltas", output) - self.assertIn("aria-label=", output) - self.assertIn(" + + +""" + + +def load(path): + if not os.path.exists(path): + return None + with open(path) as handle: + return json.load(handle) + + +def fmt_ms(value): + return f"{value:.1f} ms" + + +def ok_pct(value): + return f"{value * 100:.0f}%" + + +def tip_rows(entry): + return [("p50", fmt_ms(entry["p50_ms"])), ("p95", fmt_ms(entry["p95_ms"])), + ("ok rate", ok_pct(entry.get("ok_rate", 1.0)))] + + +def build_ab_rows(payload): + rows = [] + for name, sides in payload.get("cases", {}).items(): + head, base = sides.get("HEAD"), sides.get("BASE") + if not head or not base: + continue + delta_text, delta_cls = chart.fmt_delta(head["p50_ms"], base["p50_ms"]) + rows.append({ + "label": name, + "head": {"value": head["p50_ms"], "tick": head["p95_ms"], + "tip_title": f"{name} — HEAD", "tip_rows": tip_rows(head)}, + "base": {"value": base["p50_ms"], "tick": base["p95_ms"], + "tip_title": f"{name} — BASE", "tip_rows": tip_rows(base)}, + "delta_text": delta_text, "delta_good": delta_cls == "good", + }) + return rows + + +def build_single_rows(payload): + rows = [] + for name, sides in payload.get("cases", {}).items(): + head, base = sides.get("HEAD"), sides.get("BASE") + if not head or base: + continue + label = name.replace(" (HEAD-only)", "") + rows.append({ + "label": label, "value": head["p50_ms"], "tick": head["p95_ms"], + "tip_title": f"{label} — HEAD", "tip_rows": tip_rows(head), + "badge_text": "new in HEAD - no baseline equivalent", + }) + return rows + + +def case_shapes(payload, case_name): + sides = payload.get("cases", {}).get(case_name, {}) + return (sides.get("HEAD") or {}).get("shape"), (sides.get("BASE") or {}).get("shape") + + +def shape_line(payload): + head_shape, base_shape = case_shapes(payload, "snapshot d30") + if not head_shape or not base_shape: + return None + hn, hd, hr = head_shape + bn, bd, br = base_shape + return f"HEAD d30: {hn} nodes / depth {hd} / {hr} refs — BASE d30: {bn} nodes / depth {bd} / {br} refs" + + +def needs_honesty_callout(payload): + head_shape, base_shape = case_shapes(payload, "snapshot -i") + if not head_shape or not base_shape: + return False + return base_shape[0] > 0 and head_shape[0] < 0.5 * base_shape[0] + + +def build_speedup_rows(report): + rows = [] + for sc in report.get("scenarios", []): + legacy, live, comp = sc["legacy_snapshot"], sc["live_find_selected_refs"], sc["comparison"] + attr_delta = chart.pct_delta(live["attributes_requested"], legacy["attributes_requested"]) + rows.append({ + "label": sc["name"], "speedup": comp["p50_find_speedup"], "tip_title": sc["name"], + "tip_rows": [ + ("legacy p50", f'{legacy["p50_us"] / 1000:.2f} ms'), ("live p50", f'{live["p50_us"] / 1000:.2f} ms'), + ("legacy p95", f'{legacy["p95_us"] / 1000:.2f} ms'), ("live p95", f'{live["p95_us"] / 1000:.2f} ms'), + ("attrs legacy/live", f'{legacy["attributes_requested"]} / {live["attributes_requested"]}'), + ], + "correctness_fix": comp.get("find_correctness_delta", 0) > 0, + "sub_label": f"{attr_delta:+.0f}% attribute reads" if attr_delta is not None else None, + }) + return rows + + +def tile(label, value, head, base): + delta_text, cls = chart.fmt_delta(head, base) + return {"label": label, "value": value, "delta_text": f"{delta_text} vs base" if delta_text else "n/a", "cls": cls} + + +def build_tiles(fixture, synthetic): + tiles = [] + if fixture: + click = fixture.get("cases", {}).get("click", {}) + if click.get("HEAD") and click.get("BASE"): + h, b = click["HEAD"]["p50_ms"], click["BASE"]["p50_ms"] + tiles.append(tile("Click p50 (fixture)", fmt_ms(h), h, b)) + d30 = fixture.get("cases", {}).get("snapshot d30", {}) + if d30.get("HEAD") and d30.get("BASE"): + h, b = d30["HEAD"]["p50_ms"], d30["BASE"]["p50_ms"] + tiles.append(tile("Snapshot d30 p50 (fixture)", fmt_ms(h), h, b)) + scenarios = (synthetic or {}).get("scenarios") or [] + if scenarios: + best = max(scenarios, key=lambda s: s["comparison"]["p50_find_speedup"]) + tiles.append({"label": "Best synthetic find speedup", "value": f'{best["comparison"]["p50_find_speedup"]:.1f}x', + "delta_text": best["name"].replace("_", " "), "cls": "muted"}) + fixes = sum(1 for s in scenarios if s["comparison"].get("find_correctness_delta", 0) > 0) + tiles.append({"label": "Correctness fixes", "value": f"{fixes} scenario{'' if fixes == 1 else 's'}", + "delta_text": "legacy path answered incorrectly" if fixes else "no legacy mismatches observed", + "cls": "good" if fixes else "muted"}) + return tiles + + +def tiles_html(tiles): + if not tiles: + return "" + cells = "".join( + f'
{esc(t["label"])}
' + f'
{esc(t["value"])}
' + f'
{esc(t["delta_text"])}
' + for t in tiles + ) + return f'
{cells}
' + + +LEGEND = ('
HEAD' + 'BASE
') + + +def table_html(headers, rows): + thead = "".join(f"{esc(h)}" for h in headers) + trs = "".join("" + "".join(f"{esc(c)}" for c in r) + "" for r in rows) + return f'
{thead}{trs}
' + + +def case_rows(payload, require_base): + rows = [] + for name, sides in payload.get("cases", {}).items(): + head, base = sides.get("HEAD"), sides.get("BASE") + if not head or (require_base and not base): + continue + delta_text = chart.fmt_delta(head["p50_ms"], base["p50_ms"])[0] if base else None + ok = (f'{ok_pct(base.get("ok_rate", 1.0))}/{ok_pct(head.get("ok_rate", 1.0))}' if base + else f'-/{ok_pct(head.get("ok_rate", 1.0))}') + rows.append([name, fmt_ms(base["p50_ms"]) if base else "-", fmt_ms(base["p95_ms"]) if base else "-", + fmt_ms(head["p50_ms"]), fmt_ms(head["p95_ms"]), delta_text or "n/a", ok]) + return rows + + +def fixture_section(payload): + if not payload: + return "" + rows = build_ab_rows(payload) + table = table_html(["Case", "BASE p50", "BASE p95", "HEAD p50", "HEAD p95", "Delta", "OK B/H"], + case_rows(payload, require_base=True)) + return ( + '
' + f'

Fixture A/B (live)

{LEGEND if rows else ""}
' + f'

Alternating rounds, isolated per-binary stores, wall-clock per process invocation ' + f'({payload.get("rounds", "?")} rounds).

{chart.grouped_bar_chart(rows)}' + f'
Table view{table}
' + ) + + +def app_section(payload): + if not payload: + return "" + ab_rows, single_rows = build_ab_rows(payload), build_single_rows(payload) + if needs_honesty_callout(payload): + for row in ab_rows: + if row["label"] == "snapshot -i": + row["delta_text"], row["delta_good"] = "not comparable", False + body = [f'

{esc(payload["app"])}

{LEGEND if ab_rows else ""}
'] + body.append(f'

Read-only observation ({payload.get("rounds", "?")} rounds).

') + if ab_rows: + body.append(chart.grouped_bar_chart(ab_rows)) + if needs_honesty_callout(payload): + body.append( + '
Reading note. HEAD’s default snapshot -i ' + 'reads a shallower tree than BASE under the progressive-traversal default-depth change, so the ' + 'snapshot -i row above is not like-for-like. The snapshot d30 rows are ' + 'the apples-to-apples comparison.
') + shape = shape_line(payload) + if shape: + body.append(f'

{esc(shape)}

') + if single_rows: + body.append(chart.single_series_bar_chart(single_rows)) + table = table_html(["Case", "BASE p50", "BASE p95", "HEAD p50", "HEAD p95", "Delta", "OK B/H"], + case_rows(payload, require_base=False)) + body.append(f'
Table view{table}
') + return f'
{"".join(body)}
' + + +def synthetic_section(report): + if not report: + return "" + rows = build_speedup_rows(report) + method = report.get("methodology", {}) + hint = (f'{method.get("measured_runs", "?")} measured runs, {method.get("warmup_runs", "?")} warmup. ' + f'Legacy: {method.get("legacy_path", "legacy snapshot path")}. ' + f'Live: {method.get("live_find_path", "live find path")}.') + table_rows = [] + for sc in report.get("scenarios", []): + legacy, live, comp = sc["legacy_snapshot"], sc["live_find_selected_refs"], sc["comparison"] + table_rows.append([sc["name"], f'{legacy["p50_us"] / 1000:.2f} ms', f'{legacy["p95_us"] / 1000:.2f} ms', + f'{live["p50_us"] / 1000:.2f} ms', f'{live["p95_us"] / 1000:.2f} ms', + legacy["attributes_requested"], live["attributes_requested"], + "yes" if comp.get("find_correctness_delta", 0) > 0 else "no"]) + table = table_html(["Scenario", "Legacy p50", "Legacy p95", "Live p50", "Live p95", "Legacy attrs", "Live attrs", "Fix"], + table_rows) + return ( + '

Synthetic locator benchmark

' + f'

{esc(hint)}

{chart.speedup_bar_chart(rows)}' + f'
Table view{table}
' + ) + + +def page_chrome(head, base, rounds, out_dir): + rounds_part = f" · {rounds} rounds" if rounds else "" + header = ( + '
' + f'

agent-desktop performance - HEAD {esc(head)} vs base {esc(base)}

' + f'

{datetime.date.today().isoformat()}{rounds_part} · alternating rounds, ' + 'isolated per-binary stores, wall-clock per process invocation

' + ) + footer = ( + '
' + '

Regenerate: bash scripts/perf-baseline-compare.sh --apps "Slack,Google Chrome"

' + f'

Artifacts: {esc(out_dir)} · ' + f'generated {esc(datetime.datetime.now().isoformat(timespec="seconds"))}

' + ) + return header, footer + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out_dir") + parser.add_argument("--head", required=True) + parser.add_argument("--base", required=True) + args = parser.parse_args() + + fixture = load(os.path.join(args.out_dir, "fixture-ab.json")) + synthetic = load(os.path.join(args.out_dir, "locator-synthetic.json")) + apps = [load(p) for p in sorted(glob.glob(os.path.join(args.out_dir, "app-*.json")))] + apps = [a for a in apps if a] + sections = [fixture_section(fixture)] + [app_section(a) for a in apps] + [synthetic_section(synthetic)] + tiles = build_tiles(fixture, synthetic) + rounds = fixture.get("rounds") if fixture else None + header, footer = page_chrome(args.head, args.base, rounds, os.path.abspath(args.out_dir)) + body = header + tiles_html(tiles) + '
' + "".join(sections) + "
" + footer + title = f"agent-desktop performance - HEAD {args.head} vs base {args.base}" + html_doc = PAGE.format(title=esc(title), css=CSS, body=body, js=JS) + + out_path = os.path.join(args.out_dir, "report.html") + with open(out_path, "w") as handle: + handle.write(html_doc) + + print(f"agent-desktop performance report: HEAD {args.head} vs base {args.base}") + for t in tiles: + print(f' {t["label"]}: {t["value"]} ({t["delta_text"]})') + print(f" apps covered: {', '.join(a['app'] for a in apps) if apps else 'none'}") + print(f" report: {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/perf_report_svg.py b/scripts/perf_report_svg.py new file mode 100644 index 0000000..1c6ac9d --- /dev/null +++ b/scripts/perf_report_svg.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Pure SVG chart builders for the agent-desktop performance report. + +No third-party deps, no file or network I/O. Every public function takes +plain-Python row dicts (already formatted by the caller) and returns a +self-contained ```` string. Colors are applied only via CSS classes +(``bar-head``, ``bar-base``, ``good``, ``secondary``, ``muted``, ``status-good``, +``chip-*``) so the same markup adapts to the host page's light/dark tokens. +""" +import math + +VB_W = 860 +BAR_H = 16 +BAR_GAP = 2 +ROW_GAP = 20 +TOP_PAD = 18 +AXIS_H = 30 +CHIP_GAP = 14 +LABEL_GAP = 14 +CHECK = "✓" + + +def esc(value): + text = str(value) + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def pct_delta(head, base): + if not base: + return None + return (head - base) / base * 100.0 + + +def fmt_delta(head, base): + delta = pct_delta(head, base) + if delta is None: + return None, "muted" + return f"{delta:+.0f}%", ("good" if head < base else "secondary") + + +def nice_ticks(max_value, target=4): + if max_value <= 0: + return [0, 1] + raw_step = max_value / target + magnitude = 10 ** math.floor(math.log10(raw_step)) + step = magnitude + for mult in (1, 2, 2.5, 5, 10): + step = mult * magnitude + if step >= raw_step: + break + ticks, value, limit = [], 0.0, max_value + step * 0.5 + while value <= limit: + ticks.append(round(value, 6)) + value += step + return ticks + + +def _clamp(value, lo, hi): + return max(lo, min(hi, value)) + + +def _measure(text, size): + return len(str(text)) * size * 0.54 + + +def _bar(x, y, w, h, cls): + w = max(w, 0.0) + if w <= 0: + return "" + r = min(4.0, w / 2, h / 2) + out = [f''] + if w > r: + out.append(f'') + return "".join(out) + + +def _tick(x, y, h, cls="tick"): + return f'' + + +def _hit(x, y, w, h, title, rows): + row_str = esc("|".join(f"{label}={value}" for label, value in rows)) + return ( + f'' + ) + + +def _chip(x_right, y_mid, text, cls, size=11): + w = _measure(text, size) + 18 + h = 20 + x, y = x_right - w, y_mid - h / 2 + return ( + f'' + f'{esc(text)}' + ) + + +def _axis(sx, ticks, top, bottom, unit, suffix_x=False): + out = [] + for tick in ticks: + tx = sx(tick) + out.append(f'') + label = f"{tick:g}x" if suffix_x else f"{tick:g}{unit}" + out.append( + f'{label}' + ) + return "".join(out) + + +def _grouped_summary(rows): + best = max(rows, key=lambda r: abs(pct_delta(r["head"]["value"], r["base"]["value"]) or 0)) + delta = pct_delta(best["head"]["value"], best["base"]["value"]) or 0 + return ( + f"Grouped horizontal bar chart comparing HEAD and BASE p50 latency across {len(rows)} cases. " + f"Largest change: {best['label']} at {delta:+.0f}%." + ) + + +def grouped_bar_chart(rows, unit=" ms"): + if not rows: + return '' + label_w = _clamp(max(_measure(r["label"], 12.5) for r in rows) + LABEL_GAP, 120, 300) + chip_w = _clamp( + max((_measure(r["delta_text"], 11) + 18 for r in rows if r.get("delta_text")), default=50), 50, 200 + ) + x0, x1 = label_w, VB_W - chip_w - CHIP_GAP + plot_w = max(x1 - x0, 40) + data_max = max(max(r["head"]["value"], r["head"]["tick"], r["base"]["value"], r["base"]["tick"]) for r in rows) + ticks = nice_ticks(data_max, 4) + domain_max = (ticks[-1] or 1) * 1.24 + group_h = BAR_H * 2 + BAR_GAP + row_pitch = group_h + ROW_GAP + top = TOP_PAD + bottom = top + row_pitch * len(rows) - ROW_GAP + height = bottom + AXIS_H + + def sx(v): + return x0 + (v / domain_max) * plot_w + + parts = [ + f'', + _axis(sx, ticks, top, bottom, unit), + ] + for i, row in enumerate(rows): + y = top + i * row_pitch + y_head, y_base, cy = y, y + BAR_H + BAR_GAP, y + group_h / 2 + parts.append( + f'{esc(row["label"])}' + ) + for series, sy, cls in (("head", y_head, "bar-head"), ("base", y_base, "bar-base")): + d = row[series] + w = max((d["value"] / domain_max) * plot_w, 0) + parts.append(_bar(x0, sy, w, BAR_H, cls)) + if d.get("tick") is not None: + parts.append(_tick(sx(d["tick"]), sy, BAR_H)) + parts.append( + f'{d["value"]:.1f}{unit}' + ) + parts.append(_hit(0, sy, VB_W, BAR_H, d["tip_title"], d["tip_rows"])) + if row.get("delta_text"): + parts.append(_chip(VB_W, cy, row["delta_text"], "good" if row.get("delta_good") else "secondary")) + parts.append("") + return "".join(parts) + + +def _single_summary(rows): + return f"Bar chart of {len(rows)} HEAD-only measurement(s) with no baseline equivalent." + + +def single_series_bar_chart(rows, unit=" ms"): + if not rows: + return '' + label_w = _clamp(max(_measure(r["label"], 12.5) for r in rows) + LABEL_GAP, 120, 300) + chip_w = _clamp( + max((_measure(r["badge_text"], 10.5) + 24 for r in rows if r.get("badge_text")), default=0), 0, 320 + ) + gap = CHIP_GAP if chip_w else 0 + x0, x1 = label_w, VB_W - chip_w - gap + plot_w = max(x1 - x0, 40) + data_max = max(max(r["value"], r.get("tick") or 0) for r in rows) + ticks = nice_ticks(data_max, 4) + domain_max = (ticks[-1] or 1) * 1.24 + row_pitch = BAR_H + ROW_GAP + top = TOP_PAD + bottom = top + row_pitch * len(rows) - ROW_GAP + height = bottom + AXIS_H + + def sx(v): + return x0 + (v / domain_max) * plot_w + + parts = [ + f'', + _axis(sx, ticks, top, bottom, unit), + ] + for i, row in enumerate(rows): + y = top + i * row_pitch + cy = y + BAR_H / 2 + parts.append( + f'{esc(row["label"])}' + ) + w = max((row["value"] / domain_max) * plot_w, 0) + parts.append(_bar(x0, y, w, BAR_H, "bar-head")) + if row.get("tick") is not None: + parts.append(_tick(sx(row["tick"]), y, BAR_H)) + parts.append( + f'{row["value"]:.1f}{unit}' + ) + if row.get("badge_text"): + parts.append(_chip(VB_W, cy, row["badge_text"], "info", size=10.5)) + parts.append(_hit(0, y, VB_W, BAR_H, row["tip_title"], row["tip_rows"])) + parts.append("") + return "".join(parts) + + +def _speedup_summary(rows): + best = max(rows, key=lambda r: r["speedup"]) + worst = min(rows, key=lambda r: r["speedup"]) + return ( + f"Horizontal speedup bars for {len(rows)} synthetic scenarios, live find path versus legacy snapshot path. " + f"Speedups range from {worst['speedup']:.1f}x to {best['speedup']:.1f}x against a 1.0x parity reference." + ) + + +def speedup_bar_chart(rows): + if not rows: + return '' + badge_text = f"{CHECK} legacy answered wrong - live path correct" + label_w = _clamp(max(_measure(r["label"], 11.5) for r in rows) + LABEL_GAP, 140, 340) + chip_w = _clamp(_measure(badge_text, 10.5) + 24 if any(r.get("correctness_fix") for r in rows) else 0, 0, 340) + gap = CHIP_GAP if chip_w else 0 + x0, x1 = label_w, VB_W - chip_w - gap + plot_w = max(x1 - x0, 40) + data_max = max([r["speedup"] for r in rows] + [1.0]) + ticks = nice_ticks(data_max, 4) + domain_max = (ticks[-1] or 1) * 1.2 + row_pitch = BAR_H + ROW_GAP + 14 + top = TOP_PAD + 6 + bottom = top + row_pitch * len(rows) - ROW_GAP - 14 + height = bottom + AXIS_H + + def sx(v): + return x0 + (v / domain_max) * plot_w + + parts = [ + f'', + _axis(sx, [t for t in ticks if abs(t - 1.0) > 1e-9], top, bottom, "", suffix_x=True), + ] + px = sx(1.0) + parts.append(f'') + parts.append( + f'parity' + ) + for i, row in enumerate(rows): + y = top + i * row_pitch + cy = y + BAR_H / 2 + parts.append( + f'{esc(row["label"])}' + ) + w = max((row["speedup"] / domain_max) * plot_w, 0) + parts.append(_bar(x0, y, w, BAR_H, "bar-head")) + end_x = x0 + w + 6 + parts.append( + f'{row["speedup"]:.1f}x' + ) + if row.get("sub_label"): + parts.append( + f'{esc(row["sub_label"])}' + ) + if row.get("correctness_fix"): + parts.append(_chip(VB_W, cy, badge_text, "status-good", size=10.5)) + parts.append(_hit(0, y - 7, VB_W, BAR_H + 14, row["tip_title"], row["tip_rows"])) + parts.append("") + return "".join(parts)