mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 01:22:16 +00:00
feat: add head-vs-main performance comparison harness with html report
One command now measures the branch against its merge-base: builds both revisions, runs a fixture A/B (isolated per-binary stores, alternating rounds), optional strictly-read-only probes of real apps, and the synthetic locator benchmark, then renders a self-contained report.html with validated light/dark palettes, grouped p50/p95 bars, honesty callouts for non-comparable defaults, and table fallbacks. Replaces the markdown summary and the standalone locator-report generator, whose sanitized-report flow required manual invocation and hardcoded sizes.
This commit is contained in:
Lahfir
parent
fc3e08a667
commit
ee88ee8323
7 changed files with 976 additions and 505 deletions
|
|
@ -24,17 +24,10 @@ role-dependent action and settable probes make that model inaccurate. Native
|
|||
Slack/Electron IPC measurements require Accessibility permission and belong in
|
||||
the privileged macOS integration suite.
|
||||
|
||||
Generate the standalone sanitized report from the synthetic and paired live
|
||||
evidence:
|
||||
|
||||
```bash
|
||||
python3 benchmarks/locator-resolution/generate_performance_report.py \
|
||||
--synthetic /private/tmp/agent-desktop-locator-synthetic.json \
|
||||
--live /private/tmp/agent-desktop-electron-slack-paired-final.json \
|
||||
--output /private/tmp/agent-desktop-locator-performance-report.html
|
||||
```
|
||||
|
||||
Benchmark outputs are generated evidence, not source. Keep them outside the repository.
|
||||
This synthetic benchmark JSON is consumed by the repo-wide performance report:
|
||||
run `bash scripts/perf-baseline-compare.sh` (optionally with `--apps "Slack,Google
|
||||
Chrome"`) to produce `report.html`, covering the HEAD-vs-main fixture A/B,
|
||||
optional real-app read-only probes, and this synthetic benchmark.
|
||||
|
||||
For privileged Electron/Chromium runs, the macOS adapter first asks whether the
|
||||
application root exposes `AXManualAccessibility` as settable, reads its current
|
||||
|
|
|
|||
|
|
@ -1,336 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DEFAULT_BASELINE_BYTES = 1_793_312
|
||||
DEFAULT_CURRENT_BYTES = 2_162_496
|
||||
SCENARIO_LABELS = {
|
||||
"deep_anonymous_has_text": "Deep anonymous text",
|
||||
"duplicate_button_role_and_name": "Duplicate role + name",
|
||||
"electron_dual_identifier_moving_bounds": "Moving Electron bounds",
|
||||
"large_nested_channel_has_unread": "Large nested tree",
|
||||
}
|
||||
|
||||
|
||||
def require(mapping, *keys):
|
||||
value = mapping
|
||||
for key in keys:
|
||||
if not isinstance(value, dict) or key not in value:
|
||||
raise ValueError(f"missing required field: {'.'.join(keys)}")
|
||||
value = value[key]
|
||||
return value
|
||||
|
||||
|
||||
def load_json(path):
|
||||
with path.open(encoding="utf-8") as source:
|
||||
return json.load(source)
|
||||
|
||||
|
||||
def fmt(value, digits=1):
|
||||
return f"{float(value):,.{digits}f}"
|
||||
|
||||
|
||||
def rate(value):
|
||||
return float(value) * 100.0
|
||||
|
||||
|
||||
def directional_change(value, lower_label, higher_label, unchanged_label):
|
||||
value = float(value)
|
||||
if value < 0:
|
||||
return abs(value), lower_label
|
||||
if value > 0:
|
||||
return value, higher_label
|
||||
return 0.0, unchanged_label
|
||||
|
||||
|
||||
def size_change(baseline_bytes, current_bytes):
|
||||
delta = current_bytes - baseline_bytes
|
||||
percent = abs(delta) / baseline_bytes * 100.0
|
||||
if delta < 0:
|
||||
return percent, "smaller", f"{delta:,}"
|
||||
if delta > 0:
|
||||
return percent, "larger", f"+{delta:,}"
|
||||
return 0.0, "the same size", "0"
|
||||
|
||||
|
||||
def paired_wall_deltas(live):
|
||||
baseline = {
|
||||
int(sample["pair_index"]): float(sample["wall_ms"])
|
||||
for sample in require(live, "runs", "baseline", "samples")
|
||||
if sample.get("command_success") and isinstance(sample.get("wall_ms"), (int, float))
|
||||
}
|
||||
current = {
|
||||
int(sample["pair_index"]): float(sample["wall_ms"])
|
||||
for sample in require(live, "runs", "current", "samples")
|
||||
if sample.get("command_success") and isinstance(sample.get("wall_ms"), (int, float))
|
||||
}
|
||||
indexes = sorted(baseline.keys() & current.keys())
|
||||
return [current[index] - baseline[index] for index in indexes]
|
||||
|
||||
|
||||
def svg_grouped_bars(title, baseline, current, unit):
|
||||
labels = ("p50", "p95")
|
||||
values = [baseline[0], current[0], baseline[1], current[1]]
|
||||
maximum = max(values) * 1.18
|
||||
bars = []
|
||||
for group, label in enumerate(labels):
|
||||
group_x = 86 + group * 200
|
||||
for offset, (series, value, css_class) in enumerate(
|
||||
(("Baseline", baseline[group], "baseline"), ("Current", current[group], "current"))
|
||||
):
|
||||
height = 155 * value / maximum
|
||||
x = group_x + offset * 66
|
||||
y = 193 - height
|
||||
bars.append(
|
||||
f'<rect class="{css_class}" x="{x}" y="{y:.2f}" width="48" height="{height:.2f}" rx="4" />'
|
||||
f'<text x="{x + 24}" y="{max(20, y - 7):.2f}" text-anchor="middle">{fmt(value)} {unit}</text>'
|
||||
f'<text class="muted" x="{x + 24}" y="216" text-anchor="middle">{series}</text>'
|
||||
)
|
||||
bars.append(f'<text x="{group_x + 57}" y="240" text-anchor="middle">{label}</text>')
|
||||
summary = (
|
||||
f"{title}. Baseline p50 {fmt(baseline[0])} and p95 {fmt(baseline[1])} {unit}; "
|
||||
f"current p50 {fmt(current[0])} and p95 {fmt(current[1])} {unit}."
|
||||
)
|
||||
return (
|
||||
f'<svg class="chart" viewBox="0 0 480 255" role="img" aria-label="{html.escape(summary)}">'
|
||||
f"<title>{html.escape(title)}</title><desc>{html.escape(summary)}</desc>"
|
||||
'<line class="axis" x1="54" y1="193" x2="450" y2="193" />'
|
||||
+ "".join(bars)
|
||||
+ "</svg>"
|
||||
)
|
||||
|
||||
|
||||
def svg_delta_plot(deltas):
|
||||
if not deltas:
|
||||
raise ValueError("live report has no comparable successful wall-time pairs")
|
||||
ordered = sorted(deltas)
|
||||
limit = max(abs(min(ordered)), abs(max(ordered)), 1.0) * 1.1
|
||||
zero_y = 132
|
||||
scale = 102 / limit
|
||||
plot_width = 610
|
||||
bar_width = max(3.0, plot_width / len(ordered) - 3.0)
|
||||
bars = []
|
||||
for index, value in enumerate(ordered):
|
||||
x = 49 + index * plot_width / len(ordered)
|
||||
height = abs(value) * scale
|
||||
y = zero_y if value >= 0 else zero_y - height
|
||||
css_class = "slower" if value > 0 else "faster"
|
||||
bars.append(
|
||||
f'<rect class="{css_class}" x="{x:.2f}" y="{y:.2f}" width="{bar_width:.2f}" height="{height:.2f}">'
|
||||
f"<title>Pair {index + 1}: {fmt(value)} ms</title></rect>"
|
||||
)
|
||||
faster = sum(value < 0 for value in ordered)
|
||||
summary = (
|
||||
f"Ordered current minus baseline wall-time deltas for {len(ordered)} pairs. "
|
||||
f"Current was faster in {faster} pairs. Negative values are faster."
|
||||
)
|
||||
return (
|
||||
'<svg class="chart wide" viewBox="0 0 680 280" role="img" '
|
||||
f'aria-label="{html.escape(summary)}"><title>Paired wall-time deltas</title>'
|
||||
f"<desc>{html.escape(summary)}</desc>"
|
||||
'<text class="muted" x="10" y="30">faster</text><text class="muted" x="10" y="252">slower</text>'
|
||||
'<line class="axis" x1="47" y1="132" x2="665" y2="132" />'
|
||||
+ "".join(bars)
|
||||
+ f'<text x="49" y="270">{fmt(min(ordered))} ms</text>'
|
||||
+ f'<text x="665" y="270" text-anchor="end">{fmt(max(ordered))} ms</text>'
|
||||
+ "</svg>"
|
||||
)
|
||||
|
||||
|
||||
def svg_rate_chart(live):
|
||||
metrics = (
|
||||
("Correct", "correct_result_rate"),
|
||||
("Addressable", "addressable_result_rate"),
|
||||
("Exact re-resolution", "exact_reresolution_rate"),
|
||||
)
|
||||
rows = []
|
||||
summaries = []
|
||||
for index, (label, key) in enumerate(metrics):
|
||||
baseline = rate(require(live, "runs", "baseline", "reliability", key))
|
||||
current = rate(require(live, "runs", "current", "reliability", key))
|
||||
summaries.append(f"{label}: baseline {fmt(baseline, 0)} percent, current {fmt(current, 0)} percent")
|
||||
y = 42 + index * 67
|
||||
rows.append(f'<text x="8" y="{y + 14}">{html.escape(label)}</text>')
|
||||
for offset, (value, css_class, series) in enumerate(
|
||||
((baseline, "baseline", "Baseline"), (current, "current", "Current"))
|
||||
):
|
||||
bar_y = y + offset * 22
|
||||
rows.append(
|
||||
f'<rect class="track" x="148" y="{bar_y}" width="390" height="16" rx="3" />'
|
||||
f'<rect class="{css_class}" x="148" y="{bar_y}" width="{3.9 * value:.2f}" height="16" rx="3" />'
|
||||
f'<text x="548" y="{bar_y + 13}">{series} {fmt(value, 0)}%</text>'
|
||||
)
|
||||
summary = "; ".join(summaries)
|
||||
return (
|
||||
'<svg class="chart wide" viewBox="0 0 680 250" role="img" '
|
||||
f'aria-label="{html.escape(summary)}"><title>Live reliability</title>'
|
||||
f"<desc>{html.escape(summary)}</desc>"
|
||||
+ "".join(rows)
|
||||
+ "</svg>"
|
||||
)
|
||||
|
||||
|
||||
def svg_speedups(synthetic):
|
||||
speedups = [
|
||||
(
|
||||
SCENARIO_LABELS.get(item["name"], item["name"].replace("_", " ").title()),
|
||||
float(require(item, "comparison", "p50_find_speedup")),
|
||||
float(require(item, "comparison", "p95_find_speedup")),
|
||||
)
|
||||
for item in require(synthetic, "scenarios")
|
||||
]
|
||||
maximum = max(max(row[1:]) for row in speedups) * 1.15
|
||||
rows = []
|
||||
for index, (label, p50, p95) in enumerate(speedups):
|
||||
y = 34 + index * 64
|
||||
rows.append(f'<text x="8" y="{y + 13}">{html.escape(label)}</text>')
|
||||
for offset, (value, css_class, percentile) in enumerate(
|
||||
((p50, "current", "p50"), (p95, "synthetic-p95", "p95"))
|
||||
):
|
||||
bar_y = y + offset * 22
|
||||
width = value / maximum * 365
|
||||
rows.append(
|
||||
f'<rect class="{css_class}" x="220" y="{bar_y}" width="{width:.2f}" height="16" rx="3" />'
|
||||
f'<text x="{min(650, 228 + width):.2f}" y="{bar_y + 13}">{percentile} {fmt(value, 2)}×</text>'
|
||||
)
|
||||
summary = "; ".join(f"{label}: p50 {fmt(p50, 2)} times, p95 {fmt(p95, 2)} times" for label, p50, p95 in speedups)
|
||||
return (
|
||||
'<svg class="chart wide" viewBox="0 0 680 310" role="img" '
|
||||
f'aria-label="Synthetic find speedup. {html.escape(summary)}">'
|
||||
"<title>Synthetic find speedup</title>"
|
||||
f"<desc>{html.escape(summary)}</desc>"
|
||||
+ "".join(rows)
|
||||
+ "</svg>"
|
||||
)
|
||||
|
||||
|
||||
def svg_binary_size(baseline_bytes, current_bytes):
|
||||
maximum = max(baseline_bytes, current_bytes) * 1.15
|
||||
bars = []
|
||||
for index, (label, value, css_class) in enumerate(
|
||||
(("Baseline", baseline_bytes, "baseline"), ("Current", current_bytes, "current"))
|
||||
):
|
||||
width = value / maximum * 470
|
||||
y = 45 + index * 62
|
||||
bars.append(
|
||||
f'<text x="8" y="{y + 17}">{label}</text>'
|
||||
f'<rect class="{css_class}" x="96" y="{y}" width="{width:.2f}" height="24" rx="4" />'
|
||||
f'<text x="{min(650, 106 + width):.2f}" y="{y + 17}">{value / 1_000_000:.3f} MB</text>'
|
||||
)
|
||||
return (
|
||||
'<svg class="chart wide" viewBox="0 0 680 180" role="img" '
|
||||
f'aria-label="Baseline binary {baseline_bytes} bytes; current binary {current_bytes} bytes.">'
|
||||
"<title>Release binary size</title><desc>Decimal megabytes. The current binary remains below the 15 megabyte project ceiling.</desc>"
|
||||
+ "".join(bars)
|
||||
+ "</svg>"
|
||||
)
|
||||
|
||||
|
||||
def render_report(synthetic, live, baseline_bytes, current_bytes, current_sha256=None):
|
||||
wall = tuple(
|
||||
require(live, "runs", name, "metrics", "end_to_end_wall_all_attempts", percentile)
|
||||
for name in ("baseline", "current")
|
||||
for percentile in ("p50", "p95")
|
||||
)
|
||||
cpu = tuple(
|
||||
require(live, "runs", name, "metrics", "process_cpu_all_attempts", percentile)
|
||||
for name in ("baseline", "current")
|
||||
for percentile in ("p50", "p95")
|
||||
)
|
||||
pairs = int(require(live, "paired_comparison", "comparable_successful_pairs"))
|
||||
faster_rate = rate(require(live, "paired_comparison", "current_minus_baseline", "current_faster_wall_rate"))
|
||||
delta_p50 = require(live, "paired_comparison", "current_minus_baseline", "wall", "p50")
|
||||
wall_change, wall_direction = directional_change(
|
||||
delta_p50, "reduction", "increase", "change"
|
||||
)
|
||||
reliability_keys = (
|
||||
"correct_result_rate",
|
||||
"addressable_result_rate",
|
||||
"exact_reresolution_rate",
|
||||
)
|
||||
current_reliability = min(
|
||||
rate(require(live, "runs", "current", "reliability", key))
|
||||
for key in reliability_keys
|
||||
)
|
||||
synthetic_scenarios = require(synthetic, "scenarios")
|
||||
synthetic_total = len(synthetic_scenarios)
|
||||
legacy_synthetic_correct = sum(
|
||||
bool(require(item, "legacy_snapshot", "correct_all_runs"))
|
||||
for item in synthetic_scenarios
|
||||
)
|
||||
current_synthetic_correct = sum(
|
||||
bool(require(item, "live_find_selected_refs", "correct_all_runs"))
|
||||
and bool(require(item, "live_find_selected_refs", "selected_refs_reresolvable"))
|
||||
for item in synthetic_scenarios
|
||||
)
|
||||
size_percent, size_direction, size_delta = size_change(baseline_bytes, current_bytes)
|
||||
captured_at = html.escape(str(live.get("generated_at", "unknown time")))
|
||||
measured_sha = str(require(live, "runs", "current", "binary", "sha256"))
|
||||
if current_sha256 and measured_sha != current_sha256:
|
||||
live_evidence_note = (
|
||||
f'<p class="evidence-warning">Live evidence was captured at {captured_at} '
|
||||
f'against build <code>{html.escape(measured_sha[:12])}</code>. The final build '
|
||||
f'<code>{html.escape(current_sha256[:12])}</code> differs and could not be re-probed; '
|
||||
'treat live latency as pre-final evidence.</p>'
|
||||
)
|
||||
else:
|
||||
live_evidence_note = (
|
||||
f'<p class="note">Live evidence captured at {captured_at} against build '
|
||||
f'<code>{html.escape(measured_sha[:12])}</code>.</p>'
|
||||
)
|
||||
return f'''<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>agent-desktop performance evidence</title>
|
||||
<style>
|
||||
:root{{--bg:#f6f7f9;--surface:#fff;--text:#172033;--muted:#566176;--border:#d9deea;--baseline:#667085;--current:#167a5b;--p95:#2463a3;--faster:#167a5b;--slower:#b54708;--track:#e8ebf1}}
|
||||
@media(prefers-color-scheme:dark){{:root{{--bg:#10131a;--surface:#181d27;--text:#eef1f7;--muted:#aeb7c8;--border:#343c4b;--baseline:#98a2b3;--current:#55c69a;--p95:#6da9e3;--faster:#55c69a;--slower:#f0a35b;--track:#2a3140}}}}
|
||||
*{{box-sizing:border-box}} body{{margin:0;background:var(--bg);color:var(--text);font:15px/1.55 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}} main{{max-width:1120px;margin:auto;padding:32px 20px 64px}} header{{display:flex;justify-content:space-between;gap:20px;align-items:flex-start;margin-bottom:28px}} h1{{font-size:clamp(1.7rem,4vw,2.7rem);line-height:1.1;margin:0 0 10px}} h2{{font-size:1.35rem;margin:0 0 8px}} p{{margin:0 0 12px}} .lede,.note{{color:var(--muted)}} .evidence-warning{{color:var(--slower);font-weight:600}} button{{font:inherit;color:var(--text);background:var(--surface);border:1px solid var(--border);border-radius:7px;padding:8px 12px;cursor:pointer}} button:focus-visible{{outline:3px solid var(--p95);outline-offset:2px}} .summary{{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:14px;margin:24px 0 34px}} .stat,.panel{{background:var(--surface);border:1px solid var(--border);border-radius:10px}} .stat{{padding:16px}} .stat strong{{display:block;font-size:1.55rem;font-weight:500}} .stat span{{color:var(--muted)}} .grid{{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:18px}} .panel{{padding:18px;min-width:0}} .panel.wide{{grid-column:1/-1}} .chart{{display:block;width:100%;height:auto;margin-top:8px}} .chart text{{fill:var(--text);font:12px system-ui,sans-serif}} .chart .muted{{fill:var(--muted)}} .axis{{stroke:var(--border);stroke-width:2}} .baseline{{fill:var(--baseline)}} .current{{fill:var(--current)}} .synthetic-p95{{fill:var(--p95)}} .track{{fill:var(--track)}} .faster{{fill:var(--faster)}} .slower{{fill:var(--slower)}} .legend{{display:flex;gap:18px;flex-wrap:wrap;color:var(--muted);font-size:.9rem}} .swatch{{display:inline-block;width:11px;height:11px;border-radius:2px;margin-right:6px}} .limitations{{margin-top:28px;padding-top:24px;border-top:1px solid var(--border)}} li{{margin:6px 0}} code{{font-size:.9em}} @media(max-width:720px){{header{{display:block}} header button{{margin-top:12px}} .summary,.grid{{grid-template-columns:1fr}} .panel.wide{{grid-column:auto}} main{{padding:24px 12px 48px}}}}
|
||||
@media print{{button{{display:none}} body{{background:#fff}} .panel,.stat{{break-inside:avoid}}}}
|
||||
</style></head><body><main>
|
||||
<header><div><h1>agent-desktop performance evidence</h1><p class="lede">Baseline versus current branch, with live read-only Electron evidence separated from deterministic synthetic locator evidence.</p></div><button id="print-report" type="button">Print report</button></header>
|
||||
{live_evidence_note}
|
||||
<section class="summary" aria-label="Key results"><div class="stat"><strong>{fmt(wall_change)} ms</strong><span>live paired p50 wall-time {wall_direction}</span></div><div class="stat"><strong>{fmt(faster_rate, 1)}%</strong><span>live pairs where current was faster</span></div><div class="stat"><strong>{fmt(current_reliability, 1)}%</strong><span>lowest current correctness reliability rate</span></div></section>
|
||||
<div class="grid">
|
||||
<section class="panel"><h2>Live wall time</h2><p class="note">Slack (read-only), {pairs} successful paired runs. Lower is better.</p><div class="legend"><span><i class="swatch baseline"></i>Baseline</span><span><i class="swatch current"></i>Current</span></div>{svg_grouped_bars("Live end-to-end wall time", (wall[0], wall[1]), (wall[2], wall[3]), "ms")}</section>
|
||||
<section class="panel"><h2>Live process CPU</h2><p class="note">Per-command user + system CPU. Lower is better.</p><div class="legend"><span><i class="swatch baseline"></i>Baseline</span><span><i class="swatch current"></i>Current</span></div>{svg_grouped_bars("Live process CPU", (cpu[0], cpu[1]), (cpu[2], cpu[3]), "ms")}</section>
|
||||
<section class="panel wide"><h2>Paired live wall-time deltas</h2><p class="note">All {pairs} current-minus-baseline deltas, ordered by value. Negative means current is faster.</p>{svg_delta_plot(paired_wall_deltas(live))}</section>
|
||||
<section class="panel wide"><h2>Live locator reliability</h2><p class="note">Command exit success alone was 100% for both binaries; correctness, addressability, and exact re-resolution distinguish usable results.</p>{svg_rate_chart(live)}</section>
|
||||
<section class="panel wide"><h2>Synthetic locator speedup</h2><p class="note">31 measured runs after 5 warmups per deterministic Chromium/Electron fixture. Current find was correct and re-resolvable in {current_synthetic_correct}/{synthetic_total} scenarios; legacy was correct in {legacy_synthetic_correct}/{synthetic_total}. Speed ratios are timing evidence, not a substitute for correctness. These timings exclude native accessibility IPC.</p><div class="legend"><span><i class="swatch current"></i>p50 find speedup</span><span><i class="swatch synthetic-p95"></i>p95 find speedup</span></div>{svg_speedups(synthetic)}</section>
|
||||
<section class="panel wide"><h2>Release binary size</h2><p class="note">Current is {fmt(size_percent, 2)}% {size_direction} ({size_delta} bytes), while remaining far below the 15 MB ceiling.</p>{svg_binary_size(baseline_bytes, current_bytes)}</section>
|
||||
</div>
|
||||
<section class="limitations"><h2>Scope and limitations</h2><ul><li>Live data is one machine, one app state, and one macOS accessibility-permission state; it is not a cross-machine claim.</li><li>The live operation was read-only <code>find</code> plus exact ref re-resolution. No click, typing, navigation, or message mutation occurred.</li><li>Synthetic data measures platform-neutral locator work on deterministic Electron-shaped fixtures; it does not measure native IPC or whole-command latency.</li><li>Binary size is {fmt(size_percent, 2)}% {size_direction}; performance evidence does not erase packaging changes.</li></ul></section>
|
||||
</main><script>document.getElementById("print-report").addEventListener("click",function(){{window.print()}});</script></body></html>'''
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
parser = argparse.ArgumentParser(description="Generate the sanitized agent-desktop performance report")
|
||||
parser.add_argument("--synthetic", type=Path, required=True, help="synthetic locator benchmark JSON")
|
||||
parser.add_argument("--live", type=Path, required=True, help="paired live benchmark JSON")
|
||||
parser.add_argument("--output", type=Path, required=True, help="standalone HTML destination")
|
||||
parser.add_argument("--baseline-bytes", type=int, default=DEFAULT_BASELINE_BYTES)
|
||||
parser.add_argument("--current-bytes", type=int, default=DEFAULT_CURRENT_BYTES)
|
||||
parser.add_argument("--current-sha256")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = parse_args(argv)
|
||||
if args.baseline_bytes <= 0 or args.current_bytes <= 0:
|
||||
raise ValueError("binary sizes must be positive")
|
||||
report = render_report(
|
||||
load_json(args.synthetic),
|
||||
load_json(args.live),
|
||||
args.baseline_bytes,
|
||||
args.current_bytes,
|
||||
args.current_sha256,
|
||||
)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(report, encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
import importlib.util
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("generate_performance_report.py")
|
||||
SPEC = importlib.util.spec_from_file_location("performance_report", MODULE_PATH)
|
||||
REPORT = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(REPORT)
|
||||
|
||||
|
||||
def synthetic_fixture():
|
||||
return {
|
||||
"scenarios": [
|
||||
{
|
||||
"name": name,
|
||||
"comparison": {
|
||||
"p50_find_speedup": 1.25 + index / 10,
|
||||
"p95_find_speedup": 1.35 + index / 10,
|
||||
},
|
||||
"legacy_snapshot": {"correct_all_runs": True},
|
||||
"live_find_selected_refs": {
|
||||
"correct_all_runs": True,
|
||||
"selected_refs_reresolvable": True,
|
||||
},
|
||||
}
|
||||
for index, name in enumerate(REPORT.SCENARIO_LABELS)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def live_fixture():
|
||||
samples = [
|
||||
{
|
||||
"pair_index": index,
|
||||
"wall_ms": wall,
|
||||
"command_success": True,
|
||||
"stats": {"private_ui_text": "SECRET-SAMPLE-SENTINEL"},
|
||||
}
|
||||
for index, wall in enumerate((300.0, 320.0, 340.0))
|
||||
]
|
||||
current_samples = [dict(sample, wall_ms=sample["wall_ms"] - 70.0) for sample in samples]
|
||||
reliability = {
|
||||
"correct_result_rate": 0.0,
|
||||
"addressable_result_rate": 0.0,
|
||||
"exact_reresolution_rate": 0.0,
|
||||
}
|
||||
return {
|
||||
"app": "SECRET-APP-SENTINEL",
|
||||
"state_reference": {
|
||||
"target_window_id": "SECRET-WINDOW-SENTINEL",
|
||||
"windows": [{"title": "SECRET-TITLE-SENTINEL", "pid": 987654}],
|
||||
},
|
||||
"paired_comparison": {
|
||||
"comparable_successful_pairs": 3,
|
||||
"current_minus_baseline": {
|
||||
"current_faster_wall_rate": 1.0,
|
||||
"wall": {"p50": -70.0},
|
||||
},
|
||||
},
|
||||
"runs": {
|
||||
"baseline": {
|
||||
"binary": {"sha256": "baseline-build-sha"},
|
||||
"samples": samples,
|
||||
"metrics": {
|
||||
"end_to_end_wall_all_attempts": {"p50": 300.0, "p95": 340.0},
|
||||
"process_cpu_all_attempts": {"p50": 50.0, "p95": 60.0},
|
||||
},
|
||||
"reliability": reliability,
|
||||
},
|
||||
"current": {
|
||||
"binary": {"sha256": "measured-build-sha"},
|
||||
"samples": current_samples,
|
||||
"metrics": {
|
||||
"end_to_end_wall_all_attempts": {"p50": 230.0, "p95": 270.0},
|
||||
"process_cpu_all_attempts": {"p50": 31.0, "p95": 40.0},
|
||||
},
|
||||
"reliability": {key: 1.0 for key in reliability},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class PerformanceReportTests(unittest.TestCase):
|
||||
def test_report_contains_metrics_and_excludes_sensitive_input(self):
|
||||
output = REPORT.render_report(synthetic_fixture(), live_fixture(), 1_793_312, 2_162_480)
|
||||
self.assertIn("Slack (read-only)", output)
|
||||
self.assertIn("230.0", output)
|
||||
self.assertIn("20.59%", output)
|
||||
self.assertIn("Paired live wall-time deltas", output)
|
||||
self.assertIn("aria-label=", output)
|
||||
self.assertIn("<script>", output)
|
||||
for sentinel in (
|
||||
"SECRET-APP-SENTINEL",
|
||||
"SECRET-WINDOW-SENTINEL",
|
||||
"SECRET-TITLE-SENTINEL",
|
||||
"SECRET-SAMPLE-SENTINEL",
|
||||
"987654",
|
||||
"state_reference",
|
||||
):
|
||||
self.assertNotIn(sentinel, output)
|
||||
|
||||
def test_cli_writes_standalone_report(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
synthetic_path = root / "synthetic.json"
|
||||
live_path = root / "live.json"
|
||||
output_path = root / "report.html"
|
||||
synthetic_path.write_text(json.dumps(synthetic_fixture()), encoding="utf-8")
|
||||
live_path.write_text(json.dumps(live_fixture()), encoding="utf-8")
|
||||
REPORT.main(
|
||||
[
|
||||
"--synthetic",
|
||||
str(synthetic_path),
|
||||
"--live",
|
||||
str(live_path),
|
||||
"--output",
|
||||
str(output_path),
|
||||
]
|
||||
)
|
||||
self.assertTrue(output_path.read_text(encoding="utf-8").startswith("<!doctype html>"))
|
||||
|
||||
def test_summary_uses_measured_reliability_instead_of_a_literal_claim(self):
|
||||
live = live_fixture()
|
||||
live["runs"]["current"]["reliability"]["addressable_result_rate"] = 0.8
|
||||
|
||||
output = REPORT.render_report(synthetic_fixture(), live, 1_793_312, 2_162_480)
|
||||
|
||||
self.assertIn("<strong>80.0%</strong>", output)
|
||||
self.assertNotIn("<strong>100%</strong><span>current correct", output)
|
||||
|
||||
def test_summary_describes_regressions_without_malformed_signs(self):
|
||||
live = live_fixture()
|
||||
live["paired_comparison"]["current_minus_baseline"]["wall"]["p50"] = 12.5
|
||||
|
||||
output = REPORT.render_report(synthetic_fixture(), live, 2_200_000, 2_000_000)
|
||||
|
||||
self.assertIn("12.5 ms</strong><span>live paired p50 wall-time increase", output)
|
||||
self.assertIn("Current is 9.09% smaller (-200,000 bytes)", output)
|
||||
self.assertNotIn("larger (+-", output)
|
||||
|
||||
def test_mismatched_final_build_is_labeled_as_pre_final_live_evidence(self):
|
||||
output = REPORT.render_report(
|
||||
synthetic_fixture(),
|
||||
live_fixture(),
|
||||
1_793_312,
|
||||
2_162_480,
|
||||
current_sha256="final-build-sha",
|
||||
)
|
||||
|
||||
self.assertIn("could not be re-probed", output)
|
||||
self.assertIn("pre-final evidence", output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
92
scripts/perf-baseline-compare.sh
Executable file
92
scripts/perf-baseline-compare.sh
Executable file
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env bash
|
||||
# HEAD-vs-baseline performance comparison for agent-desktop.
|
||||
#
|
||||
# Builds the current checkout and the merge-base-with-main revision, then:
|
||||
# 1. runs a fixture A/B (snapshots, reads, and fixture-only actions),
|
||||
# 2. runs the deterministic synthetic locator benchmark on HEAD,
|
||||
# 3. optionally probes real apps READ-ONLY (snapshot/find timing only),
|
||||
# and writes a markdown report + raw JSON.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/perf-baseline-compare.sh [--base <ref>] [--apps "Slack,Google Chrome"]
|
||||
# [--rounds N] [--out <dir>] [--skip-fixture]
|
||||
#
|
||||
# Real-app probes never dispatch actions. Apps this script launches itself are
|
||||
# closed again afterwards; apps that were already running are left untouched.
|
||||
set -euo pipefail
|
||||
|
||||
base_ref=""
|
||||
apps=""
|
||||
rounds=10
|
||||
out=""
|
||||
skip_fixture=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--base) base_ref="$2"; shift 2 ;;
|
||||
--apps) apps="$2"; shift 2 ;;
|
||||
--rounds) rounds="$2"; shift 2 ;;
|
||||
--out) out="$2"; shift 2 ;;
|
||||
--skip-fixture) skip_fixture=1; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
repo="$(git rev-parse --show-toplevel)"
|
||||
cd "$repo"
|
||||
[ -n "$base_ref" ] || base_ref="$(git merge-base HEAD "$(git rev-parse --verify --quiet origin/main >/dev/null 2>&1 && echo origin/main || echo main)")"
|
||||
base_sha="$(git rev-parse --short "$base_ref")"
|
||||
head_sha="$(git rev-parse --short HEAD)"
|
||||
[ -n "$out" ] || out="$(mktemp -d "/tmp/agent-desktop-perf-${head_sha}-vs-${base_sha}.XXXXXX")"
|
||||
mkdir -p "$out"
|
||||
echo "== perf compare: HEAD ${head_sha} vs base ${base_sha} -> ${out}"
|
||||
|
||||
echo "== building HEAD release binary"
|
||||
cargo build --release -p agent-desktop >/dev/null
|
||||
head_bin="$repo/target/release/agent-desktop"
|
||||
|
||||
echo "== building base release binary (${base_sha}) in a temporary worktree"
|
||||
wt="$out/base-worktree"
|
||||
git worktree add --detach "$wt" "$base_ref" >/dev/null
|
||||
trap 'git worktree remove --force "$wt" >/dev/null 2>&1 || true; git worktree prune >/dev/null 2>&1 || true' EXIT
|
||||
(cd "$wt" && cargo build --release -p agent-desktop >/dev/null)
|
||||
base_bin="$wt/target/release/agent-desktop"
|
||||
|
||||
if [ "$skip_fixture" -ne 1 ]; then
|
||||
echo "== fixture A/B (${rounds} rounds; actions run against the fixture only)"
|
||||
bash tests/fixture-app/build.sh "$out/fixture" >/dev/null
|
||||
open "$out/fixture/AgentDeskFixture.app"
|
||||
sleep 3
|
||||
python3 scripts/perf_ab_probe.py --mode fixture --app AgentDeskFixture \
|
||||
--head-bin "$head_bin" --base-bin "$base_bin" --rounds "$rounds" \
|
||||
--json-out "$out/fixture-ab.json" >/dev/null
|
||||
pkill -f AgentDeskFixture.app || true
|
||||
fi
|
||||
|
||||
echo "== synthetic locator benchmark (HEAD)"
|
||||
cargo run -p agent-desktop-core --release --example locator_benchmark \
|
||||
> "$out/locator-synthetic.json" 2>/dev/null
|
||||
|
||||
if [ -n "$apps" ]; then
|
||||
IFS=',' read -ra app_list <<< "$apps"
|
||||
for app in "${app_list[@]}"; do
|
||||
app="$(echo "$app" | sed 's/^ *//;s/ *$//')"
|
||||
slug="$(echo "$app" | tr ' ' '-' | tr '[:upper:]' '[:lower:]')"
|
||||
was_running=1
|
||||
pgrep -f "/${app}.app/" >/dev/null 2>&1 || was_running=0
|
||||
if [ "$was_running" -eq 0 ]; then
|
||||
echo "== launching ${app} for read-only observation"
|
||||
open -a "$app" || { echo " skip: cannot open ${app}"; continue; }
|
||||
sleep 12
|
||||
fi
|
||||
echo "== read-only probe: ${app} (${rounds} rounds, snapshots + find only)"
|
||||
python3 scripts/perf_ab_probe.py --mode app --app "$app" \
|
||||
--head-bin "$head_bin" --base-bin "$base_bin" --rounds "$rounds" \
|
||||
--json-out "$out/app-${slug}.json" >/dev/null || echo " probe failed for ${app}"
|
||||
if [ "$was_running" -eq 0 ]; then
|
||||
"$head_bin" close-app --app "$app" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
python3 scripts/perf_report_html.py "$out" --head "$head_sha" --base "$base_sha"
|
||||
echo "== artifacts: $out"
|
||||
189
scripts/perf_ab_probe.py
Executable file
189
scripts/perf_ab_probe.py
Executable file
|
|
@ -0,0 +1,189 @@
|
|||
#!/usr/bin/env python3
|
||||
"""A/B latency probe: two agent-desktop binaries against one target app.
|
||||
|
||||
Modes:
|
||||
fixture -- full probe incl. mutating actions; ONLY safe against the
|
||||
AgentDeskFixture app.
|
||||
app -- strictly read-only observation (snapshot/find timing) for
|
||||
arbitrary real apps (Slack, Chrome, ...). Never dispatches
|
||||
actions.
|
||||
|
||||
Each binary runs under an isolated HOME so cross-revision store formats
|
||||
never collide. Rounds alternate between binaries to cancel machine drift.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ENVS = {}
|
||||
|
||||
|
||||
def env_for(binary):
|
||||
if binary not in ENVS:
|
||||
env = dict(os.environ)
|
||||
env["HOME"] = tempfile.mkdtemp(prefix="ad-perf-home-")
|
||||
ENVS[binary] = env
|
||||
return ENVS[binary]
|
||||
|
||||
|
||||
def run(binary, *args, timeout=90):
|
||||
start = time.monotonic()
|
||||
proc = subprocess.run(
|
||||
[binary, *args], capture_output=True, text=True, timeout=timeout, env=env_for(binary)
|
||||
)
|
||||
elapsed = (time.monotonic() - start) * 1000
|
||||
return elapsed, proc
|
||||
|
||||
|
||||
def envelope(proc):
|
||||
try:
|
||||
return json.loads(proc.stdout)
|
||||
except ValueError:
|
||||
return {}
|
||||
|
||||
|
||||
def walk(node):
|
||||
yield node
|
||||
for child in node.get("children", []):
|
||||
yield from walk(child)
|
||||
|
||||
|
||||
def tree_stats(env):
|
||||
tree = env.get("data", {}).get("tree", {})
|
||||
nodes = sum(1 for _ in walk(tree))
|
||||
depth = 0
|
||||
|
||||
def measure(node, level):
|
||||
nonlocal depth
|
||||
depth = max(depth, level)
|
||||
for child in node.get("children", []):
|
||||
measure(child, level + 1)
|
||||
|
||||
measure(tree, 1)
|
||||
return nodes, depth, env.get("data", {}).get("ref_count")
|
||||
|
||||
|
||||
def fixture_refs(binary, app):
|
||||
_, proc = run(binary, "snapshot", "--app", app, "-i")
|
||||
env = envelope(proc)
|
||||
if "data" not in env:
|
||||
raise SystemExit(f"{binary}: fixture snapshot failed: {proc.stdout[:300]}")
|
||||
button = textfield = None
|
||||
for node in walk(env["data"]["tree"]):
|
||||
ref = node.get("ref") or node.get("ref_id")
|
||||
if not ref:
|
||||
continue
|
||||
name = (node.get("name") or "").lower()
|
||||
if node.get("role") == "button" and "primary" in name:
|
||||
button = button or ref
|
||||
if node.get("role") == "textfield":
|
||||
textfield = textfield or ref
|
||||
if not (button and textfield):
|
||||
raise SystemExit(f"{binary}: fixture targets missing (button={button}, field={textfield})")
|
||||
return button, textfield
|
||||
|
||||
|
||||
def fixture_cases(binary, app):
|
||||
button, textfield = fixture_refs(binary, app)
|
||||
return [
|
||||
("snapshot -i", ["snapshot", "--app", app, "-i"]),
|
||||
("snapshot d30", ["snapshot", "--app", app, "--max-depth", "30"]),
|
||||
("get", ["get", button]),
|
||||
("is visible", ["is", button, "--property", "visible"]),
|
||||
("click", ["click", button]),
|
||||
("set-value", ["set-value", textfield, "ab-probe"]),
|
||||
("type", ["type", textfield, "x"]),
|
||||
]
|
||||
|
||||
|
||||
def app_cases(app):
|
||||
return [
|
||||
("snapshot -i", ["snapshot", "--app", app, "-i"]),
|
||||
("snapshot d30", ["snapshot", "--app", app, "--max-depth", "30"]),
|
||||
]
|
||||
|
||||
|
||||
def collect(binaries, cases_for, rounds):
|
||||
cases = {label: cases_for(label) for label in binaries}
|
||||
results = {label: {} for label in binaries}
|
||||
shapes = {label: {} for label in binaries}
|
||||
for _ in range(rounds):
|
||||
for label, binary in binaries.items():
|
||||
for name, args in cases[label]:
|
||||
elapsed, proc = run(binary, *args)
|
||||
env = envelope(proc)
|
||||
ok = bool(env.get("ok"))
|
||||
results[label].setdefault(name, []).append((elapsed, ok))
|
||||
if ok and name.startswith("snapshot") and name not in shapes[label]:
|
||||
shapes[label][name] = tree_stats(env)
|
||||
return results, shapes
|
||||
|
||||
|
||||
def summarize(results, shapes):
|
||||
report = {}
|
||||
for label, cases in results.items():
|
||||
for name, samples in cases.items():
|
||||
times = sorted(t for t, _ in samples)
|
||||
report.setdefault(name, {})[label] = {
|
||||
"p50_ms": round(statistics.median(times), 1),
|
||||
"p95_ms": round(times[max(0, int(len(times) * 0.95) - 1)], 1),
|
||||
"ok_rate": sum(1 for _, ok in samples if ok) / len(samples),
|
||||
"shape": shapes[label].get(name),
|
||||
}
|
||||
return report
|
||||
|
||||
|
||||
def head_only_find(binary, app, rounds, report):
|
||||
times, matches = [], None
|
||||
for _ in range(rounds):
|
||||
elapsed, proc = run(binary, "find", "--app", app, "--role", "button", "--first")
|
||||
env = envelope(proc)
|
||||
if env.get("ok"):
|
||||
times.append(elapsed)
|
||||
matches = env.get("data", {}).get("match", {}).get("role")
|
||||
if times:
|
||||
times.sort()
|
||||
report["find --role button --first (HEAD-only)"] = {
|
||||
"HEAD": {
|
||||
"p50_ms": round(statistics.median(times), 1),
|
||||
"p95_ms": round(times[max(0, int(len(times) * 0.95) - 1)], 1),
|
||||
"ok_rate": len(times) / rounds,
|
||||
"matched_role": matches,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=["fixture", "app"], required=True)
|
||||
parser.add_argument("--app", required=True)
|
||||
parser.add_argument("--head-bin", required=True)
|
||||
parser.add_argument("--base-bin", required=True)
|
||||
parser.add_argument("--rounds", type=int, default=10)
|
||||
parser.add_argument("--json-out", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
binaries = {"HEAD": args.head_bin, "BASE": args.base_bin}
|
||||
if args.mode == "fixture":
|
||||
results, shapes = collect(binaries, lambda label: fixture_cases(binaries[label], args.app), args.rounds)
|
||||
else:
|
||||
results, shapes = collect(binaries, lambda _label: app_cases(args.app), args.rounds)
|
||||
report = summarize(results, shapes)
|
||||
if args.mode == "app":
|
||||
head_only_find(args.head_bin, args.app, args.rounds, report)
|
||||
|
||||
payload = {"mode": args.mode, "app": args.app, "rounds": args.rounds, "cases": report}
|
||||
if args.json_out:
|
||||
with open(args.json_out, "w") as handle:
|
||||
json.dump(payload, handle, indent=1)
|
||||
json.dump(payload, sys.stdout, indent=1)
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
400
scripts/perf_report_html.py
Normal file
400
scripts/perf_report_html.py
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Render perf-baseline-compare JSON artifacts into one self-contained HTML report.
|
||||
|
||||
Usage: python3 scripts/perf_report_html.py <out_dir> --head <sha> --base <sha>
|
||||
Reads fixture-ab.json, app-*.json, and locator-synthetic.json from <out_dir> (any may
|
||||
be absent) and writes <out_dir>/report.html. Stdlib only.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import perf_report_svg as chart
|
||||
from perf_report_svg import esc
|
||||
|
||||
CSS = """
|
||||
:root {
|
||||
--surface:#fcfcfb; --page:#f9f9f7; --ink:#0b0b0b; --secondary:#52514e; --muted:#898781;
|
||||
--grid:#e1e0d9; --baseline:#c3c2b7; --border:rgba(11,11,11,0.10);
|
||||
--head:#2a78d6; --base:#1baf7a; --good:#006300; --status-good:#0ca30c;
|
||||
--good-tint:rgba(0,99,0,0.10); --status-good-tint:rgba(12,163,12,0.12);
|
||||
--secondary-tint:rgba(82,81,78,0.09); --head-tint:rgba(42,120,214,0.10);
|
||||
}
|
||||
@media (prefers-color-scheme:dark) {
|
||||
:root {
|
||||
--surface:#1a1a19; --page:#0d0d0d; --ink:#ffffff; --secondary:#c3c2b7; --muted:#898781;
|
||||
--grid:#2c2c2a; --baseline:#383835; --border:rgba(255,255,255,0.10);
|
||||
--head:#3987e5; --base:#199e70; --good:#0ca30c; --status-good:#0ca30c;
|
||||
--good-tint:rgba(12,163,12,0.16); --status-good-tint:rgba(12,163,12,0.16);
|
||||
--secondary-tint:rgba(195,194,183,0.10); --head-tint:rgba(57,135,229,0.14);
|
||||
}
|
||||
}
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; background:var(--page); color:var(--ink); line-height:1.5;
|
||||
font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; }
|
||||
.col { max-width:1040px; margin:0 auto; padding:0 24px 48px; display:flex; flex-direction:column; gap:24px; }
|
||||
header.band, footer.band { max-width:1040px; margin:0 auto; padding:32px 24px 8px; }
|
||||
footer.band { padding:8px 24px 40px; }
|
||||
header.band h1 { font-size:22px; margin:0 0 6px; letter-spacing:-0.01em; }
|
||||
header.band .subtitle, footer.band p { color:var(--secondary); font-size:13px; margin:4px 0; }
|
||||
footer.band code, .card code { background:var(--secondary-tint); border-radius:4px; padding:1px 6px; font-size:12px; }
|
||||
.tiles { max-width:1040px; margin:0 auto; padding:8px 24px 0; display:grid;
|
||||
grid-template-columns:repeat(auto-fit,minmax(190px,1fr)); gap:16px; }
|
||||
.tile { background:var(--surface); border:1px solid var(--border); border-radius:12px; padding:18px 20px; }
|
||||
.tile-label { font-size:12px; color:var(--muted); margin-bottom:8px; }
|
||||
.tile-value { font-size:26px; font-weight:600; }
|
||||
.tile-delta { font-size:12px; margin-top:6px; }
|
||||
.tile-delta.good { color:var(--good); } .tile-delta.secondary { color:var(--secondary); } .tile-delta.muted { color:var(--muted); }
|
||||
.card { background:var(--surface); border:1px solid var(--border); border-radius:12px; padding:28px; }
|
||||
.card-head { display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:12px; }
|
||||
.card h2 { font-size:16px; margin:0; }
|
||||
.hint, .shape-line { font-size:12.5px; color:var(--secondary); }
|
||||
.hint { margin:6px 0 4px; } .shape-line { margin:12px 0 0; }
|
||||
.legend, .legend-item { display:flex; align-items:center; }
|
||||
.legend { gap:16px; font-size:12px; color:var(--secondary); } .legend-item { gap:6px; }
|
||||
.swatch { width:10px; height:10px; border-radius:2px; display:inline-block; }
|
||||
.swatch-head { background:var(--head); } .swatch-base { background:var(--base); }
|
||||
.callout { border:1px solid var(--border); background:var(--head-tint); border-radius:10px;
|
||||
padding:14px 16px; font-size:13px; color:var(--secondary); margin:16px 0 4px; }
|
||||
.callout strong { color:var(--ink); }
|
||||
svg.chart { display:block; width:100%; height:auto; margin:16px 0 4px; }
|
||||
svg.chart text { font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif; }
|
||||
.gridline { stroke:var(--grid); stroke-width:1; } .tick { stroke:var(--ink); stroke-width:1.4; opacity:0.45; }
|
||||
.parity { stroke:var(--baseline); stroke-width:1.5; stroke-dasharray:3 3; }
|
||||
.bar-head { fill:var(--head); } .bar-base { fill:var(--base); }
|
||||
.ink { fill:var(--ink); } .secondary { fill:var(--secondary); } .muted { fill:var(--muted); }
|
||||
.tabular { font-variant-numeric:tabular-nums; }
|
||||
.hit { fill:transparent; cursor:pointer; }
|
||||
.value-label { font-weight:600; }
|
||||
.chip-good { fill:var(--good-tint); } .chip-secondary, .chip-info { fill:var(--secondary-tint); }
|
||||
.chip-status-good { fill:var(--status-good-tint); }
|
||||
.chip-text-good { fill:var(--good); } .chip-text-secondary, .chip-text-info { fill:var(--secondary); }
|
||||
.chip-text-status-good { fill:var(--status-good); }
|
||||
details { margin-top:14px; }
|
||||
summary { cursor:pointer; font-size:13px; color:var(--secondary); }
|
||||
.table-wrap { overflow-x:auto; }
|
||||
table { width:100%; border-collapse:collapse; margin-top:12px; font-size:12.5px; font-variant-numeric:tabular-nums; }
|
||||
th, td { text-align:right; padding:7px 10px; border-bottom:1px solid var(--border); white-space:nowrap; }
|
||||
th:first-child, td:first-child { text-align:left; }
|
||||
th { color:var(--muted); font-weight:500; }
|
||||
.tooltip { position:fixed; pointer-events:none; opacity:0; transition:opacity .08s ease; background:var(--ink);
|
||||
color:var(--surface); border-radius:8px; padding:8px 10px; font-size:12px; z-index:20; max-width:240px; }
|
||||
.tooltip .tt-title { font-weight:600; margin-bottom:4px; }
|
||||
.tooltip .tt-row { display:flex; justify-content:space-between; gap:12px; opacity:.85; }
|
||||
@media (max-width:640px) { .card { padding:20px; } .tile-value { font-size:22px; } }
|
||||
"""
|
||||
|
||||
JS = """
|
||||
(function () {
|
||||
var tip = document.getElementById('tt');
|
||||
document.querySelectorAll('[data-tt-title]').forEach(function (el) {
|
||||
el.addEventListener('mousemove', function (evt) {
|
||||
var rows = (el.getAttribute('data-tt-rows') || '').split('|').filter(Boolean);
|
||||
var html = '<div class="tt-title">' + el.getAttribute('data-tt-title') + '</div>';
|
||||
rows.forEach(function (row) {
|
||||
var i = row.indexOf('=');
|
||||
html += '<div class="tt-row"><span>' + row.slice(0, i) + '</span><b>' + row.slice(i + 1) + '</b></div>';
|
||||
});
|
||||
tip.innerHTML = html;
|
||||
tip.style.left = Math.min(evt.clientX + 14, window.innerWidth - 250) + 'px';
|
||||
tip.style.top = (evt.clientY + 14) + 'px'; tip.style.opacity = '1';
|
||||
});
|
||||
el.addEventListener('mouseleave', function () { tip.style.opacity = '0'; });
|
||||
});
|
||||
})();
|
||||
"""
|
||||
|
||||
PAGE = """<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>{title}</title>
|
||||
<style>{css}</style>
|
||||
</head>
|
||||
<body>
|
||||
{body}
|
||||
<div id="tt" class="tooltip" role="status"></div>
|
||||
<script>{js}</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def load(path):
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
with open(path) as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def fmt_ms(value):
|
||||
return f"{value:.1f} ms"
|
||||
|
||||
|
||||
def ok_pct(value):
|
||||
return f"{value * 100:.0f}%"
|
||||
|
||||
|
||||
def tip_rows(entry):
|
||||
return [("p50", fmt_ms(entry["p50_ms"])), ("p95", fmt_ms(entry["p95_ms"])),
|
||||
("ok rate", ok_pct(entry.get("ok_rate", 1.0)))]
|
||||
|
||||
|
||||
def build_ab_rows(payload):
|
||||
rows = []
|
||||
for name, sides in payload.get("cases", {}).items():
|
||||
head, base = sides.get("HEAD"), sides.get("BASE")
|
||||
if not head or not base:
|
||||
continue
|
||||
delta_text, delta_cls = chart.fmt_delta(head["p50_ms"], base["p50_ms"])
|
||||
rows.append({
|
||||
"label": name,
|
||||
"head": {"value": head["p50_ms"], "tick": head["p95_ms"],
|
||||
"tip_title": f"{name} — HEAD", "tip_rows": tip_rows(head)},
|
||||
"base": {"value": base["p50_ms"], "tick": base["p95_ms"],
|
||||
"tip_title": f"{name} — BASE", "tip_rows": tip_rows(base)},
|
||||
"delta_text": delta_text, "delta_good": delta_cls == "good",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def build_single_rows(payload):
|
||||
rows = []
|
||||
for name, sides in payload.get("cases", {}).items():
|
||||
head, base = sides.get("HEAD"), sides.get("BASE")
|
||||
if not head or base:
|
||||
continue
|
||||
label = name.replace(" (HEAD-only)", "")
|
||||
rows.append({
|
||||
"label": label, "value": head["p50_ms"], "tick": head["p95_ms"],
|
||||
"tip_title": f"{label} — HEAD", "tip_rows": tip_rows(head),
|
||||
"badge_text": "new in HEAD - no baseline equivalent",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def case_shapes(payload, case_name):
|
||||
sides = payload.get("cases", {}).get(case_name, {})
|
||||
return (sides.get("HEAD") or {}).get("shape"), (sides.get("BASE") or {}).get("shape")
|
||||
|
||||
|
||||
def shape_line(payload):
|
||||
head_shape, base_shape = case_shapes(payload, "snapshot d30")
|
||||
if not head_shape or not base_shape:
|
||||
return None
|
||||
hn, hd, hr = head_shape
|
||||
bn, bd, br = base_shape
|
||||
return f"HEAD d30: {hn} nodes / depth {hd} / {hr} refs — BASE d30: {bn} nodes / depth {bd} / {br} refs"
|
||||
|
||||
|
||||
def needs_honesty_callout(payload):
|
||||
head_shape, base_shape = case_shapes(payload, "snapshot -i")
|
||||
if not head_shape or not base_shape:
|
||||
return False
|
||||
return base_shape[0] > 0 and head_shape[0] < 0.5 * base_shape[0]
|
||||
|
||||
|
||||
def build_speedup_rows(report):
|
||||
rows = []
|
||||
for sc in report.get("scenarios", []):
|
||||
legacy, live, comp = sc["legacy_snapshot"], sc["live_find_selected_refs"], sc["comparison"]
|
||||
attr_delta = chart.pct_delta(live["attributes_requested"], legacy["attributes_requested"])
|
||||
rows.append({
|
||||
"label": sc["name"], "speedup": comp["p50_find_speedup"], "tip_title": sc["name"],
|
||||
"tip_rows": [
|
||||
("legacy p50", f'{legacy["p50_us"] / 1000:.2f} ms'), ("live p50", f'{live["p50_us"] / 1000:.2f} ms'),
|
||||
("legacy p95", f'{legacy["p95_us"] / 1000:.2f} ms'), ("live p95", f'{live["p95_us"] / 1000:.2f} ms'),
|
||||
("attrs legacy/live", f'{legacy["attributes_requested"]} / {live["attributes_requested"]}'),
|
||||
],
|
||||
"correctness_fix": comp.get("find_correctness_delta", 0) > 0,
|
||||
"sub_label": f"{attr_delta:+.0f}% attribute reads" if attr_delta is not None else None,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def tile(label, value, head, base):
|
||||
delta_text, cls = chart.fmt_delta(head, base)
|
||||
return {"label": label, "value": value, "delta_text": f"{delta_text} vs base" if delta_text else "n/a", "cls": cls}
|
||||
|
||||
|
||||
def build_tiles(fixture, synthetic):
|
||||
tiles = []
|
||||
if fixture:
|
||||
click = fixture.get("cases", {}).get("click", {})
|
||||
if click.get("HEAD") and click.get("BASE"):
|
||||
h, b = click["HEAD"]["p50_ms"], click["BASE"]["p50_ms"]
|
||||
tiles.append(tile("Click p50 (fixture)", fmt_ms(h), h, b))
|
||||
d30 = fixture.get("cases", {}).get("snapshot d30", {})
|
||||
if d30.get("HEAD") and d30.get("BASE"):
|
||||
h, b = d30["HEAD"]["p50_ms"], d30["BASE"]["p50_ms"]
|
||||
tiles.append(tile("Snapshot d30 p50 (fixture)", fmt_ms(h), h, b))
|
||||
scenarios = (synthetic or {}).get("scenarios") or []
|
||||
if scenarios:
|
||||
best = max(scenarios, key=lambda s: s["comparison"]["p50_find_speedup"])
|
||||
tiles.append({"label": "Best synthetic find speedup", "value": f'{best["comparison"]["p50_find_speedup"]:.1f}x',
|
||||
"delta_text": best["name"].replace("_", " "), "cls": "muted"})
|
||||
fixes = sum(1 for s in scenarios if s["comparison"].get("find_correctness_delta", 0) > 0)
|
||||
tiles.append({"label": "Correctness fixes", "value": f"{fixes} scenario{'' if fixes == 1 else 's'}",
|
||||
"delta_text": "legacy path answered incorrectly" if fixes else "no legacy mismatches observed",
|
||||
"cls": "good" if fixes else "muted"})
|
||||
return tiles
|
||||
|
||||
|
||||
def tiles_html(tiles):
|
||||
if not tiles:
|
||||
return ""
|
||||
cells = "".join(
|
||||
f'<div class="tile"><div class="tile-label">{esc(t["label"])}</div>'
|
||||
f'<div class="tile-value">{esc(t["value"])}</div>'
|
||||
f'<div class="tile-delta {t["cls"]}">{esc(t["delta_text"])}</div></div>'
|
||||
for t in tiles
|
||||
)
|
||||
return f'<div class="tiles">{cells}</div>'
|
||||
|
||||
|
||||
LEGEND = ('<div class="legend"><span class="legend-item"><span class="swatch swatch-head"></span>HEAD</span>'
|
||||
'<span class="legend-item"><span class="swatch swatch-base"></span>BASE</span></div>')
|
||||
|
||||
|
||||
def table_html(headers, rows):
|
||||
thead = "".join(f"<th>{esc(h)}</th>" for h in headers)
|
||||
trs = "".join("<tr>" + "".join(f"<td>{esc(c)}</td>" for c in r) + "</tr>" for r in rows)
|
||||
return f'<div class="table-wrap"><table><thead><tr>{thead}</tr></thead><tbody>{trs}</tbody></table></div>'
|
||||
|
||||
|
||||
def case_rows(payload, require_base):
|
||||
rows = []
|
||||
for name, sides in payload.get("cases", {}).items():
|
||||
head, base = sides.get("HEAD"), sides.get("BASE")
|
||||
if not head or (require_base and not base):
|
||||
continue
|
||||
delta_text = chart.fmt_delta(head["p50_ms"], base["p50_ms"])[0] if base else None
|
||||
ok = (f'{ok_pct(base.get("ok_rate", 1.0))}/{ok_pct(head.get("ok_rate", 1.0))}' if base
|
||||
else f'-/{ok_pct(head.get("ok_rate", 1.0))}')
|
||||
rows.append([name, fmt_ms(base["p50_ms"]) if base else "-", fmt_ms(base["p95_ms"]) if base else "-",
|
||||
fmt_ms(head["p50_ms"]), fmt_ms(head["p95_ms"]), delta_text or "n/a", ok])
|
||||
return rows
|
||||
|
||||
|
||||
def fixture_section(payload):
|
||||
if not payload:
|
||||
return ""
|
||||
rows = build_ab_rows(payload)
|
||||
table = table_html(["Case", "BASE p50", "BASE p95", "HEAD p50", "HEAD p95", "Delta", "OK B/H"],
|
||||
case_rows(payload, require_base=True))
|
||||
return (
|
||||
'<section class="card">'
|
||||
f'<div class="card-head"><h2>Fixture A/B (live)</h2>{LEGEND if rows else ""}</div>'
|
||||
f'<p class="hint">Alternating rounds, isolated per-binary stores, wall-clock per process invocation '
|
||||
f'({payload.get("rounds", "?")} rounds).</p>{chart.grouped_bar_chart(rows)}'
|
||||
f'<details><summary>Table view</summary>{table}</details></section>'
|
||||
)
|
||||
|
||||
|
||||
def app_section(payload):
|
||||
if not payload:
|
||||
return ""
|
||||
ab_rows, single_rows = build_ab_rows(payload), build_single_rows(payload)
|
||||
if needs_honesty_callout(payload):
|
||||
for row in ab_rows:
|
||||
if row["label"] == "snapshot -i":
|
||||
row["delta_text"], row["delta_good"] = "not comparable", False
|
||||
body = [f'<div class="card-head"><h2>{esc(payload["app"])}</h2>{LEGEND if ab_rows else ""}</div>']
|
||||
body.append(f'<p class="hint">Read-only observation ({payload.get("rounds", "?")} rounds).</p>')
|
||||
if ab_rows:
|
||||
body.append(chart.grouped_bar_chart(ab_rows))
|
||||
if needs_honesty_callout(payload):
|
||||
body.append(
|
||||
'<div class="callout"><strong>Reading note.</strong> HEAD’s default <code>snapshot -i</code> '
|
||||
'reads a shallower tree than BASE under the progressive-traversal default-depth change, so the '
|
||||
'<code>snapshot -i</code> row above is not like-for-like. The <code>snapshot d30</code> rows are '
|
||||
'the apples-to-apples comparison.</div>')
|
||||
shape = shape_line(payload)
|
||||
if shape:
|
||||
body.append(f'<p class="shape-line tabular">{esc(shape)}</p>')
|
||||
if single_rows:
|
||||
body.append(chart.single_series_bar_chart(single_rows))
|
||||
table = table_html(["Case", "BASE p50", "BASE p95", "HEAD p50", "HEAD p95", "Delta", "OK B/H"],
|
||||
case_rows(payload, require_base=False))
|
||||
body.append(f'<details><summary>Table view</summary>{table}</details>')
|
||||
return f'<section class="card">{"".join(body)}</section>'
|
||||
|
||||
|
||||
def synthetic_section(report):
|
||||
if not report:
|
||||
return ""
|
||||
rows = build_speedup_rows(report)
|
||||
method = report.get("methodology", {})
|
||||
hint = (f'{method.get("measured_runs", "?")} measured runs, {method.get("warmup_runs", "?")} warmup. '
|
||||
f'Legacy: {method.get("legacy_path", "legacy snapshot path")}. '
|
||||
f'Live: {method.get("live_find_path", "live find path")}.')
|
||||
table_rows = []
|
||||
for sc in report.get("scenarios", []):
|
||||
legacy, live, comp = sc["legacy_snapshot"], sc["live_find_selected_refs"], sc["comparison"]
|
||||
table_rows.append([sc["name"], f'{legacy["p50_us"] / 1000:.2f} ms', f'{legacy["p95_us"] / 1000:.2f} ms',
|
||||
f'{live["p50_us"] / 1000:.2f} ms', f'{live["p95_us"] / 1000:.2f} ms',
|
||||
legacy["attributes_requested"], live["attributes_requested"],
|
||||
"yes" if comp.get("find_correctness_delta", 0) > 0 else "no"])
|
||||
table = table_html(["Scenario", "Legacy p50", "Legacy p95", "Live p50", "Live p95", "Legacy attrs", "Live attrs", "Fix"],
|
||||
table_rows)
|
||||
return (
|
||||
'<section class="card"><div class="card-head"><h2>Synthetic locator benchmark</h2></div>'
|
||||
f'<p class="hint">{esc(hint)}</p>{chart.speedup_bar_chart(rows)}'
|
||||
f'<details><summary>Table view</summary>{table}</details></section>'
|
||||
)
|
||||
|
||||
|
||||
def page_chrome(head, base, rounds, out_dir):
|
||||
rounds_part = f" · {rounds} rounds" if rounds else ""
|
||||
header = (
|
||||
'<header class="band">'
|
||||
f'<h1>agent-desktop performance - HEAD {esc(head)} vs base {esc(base)}</h1>'
|
||||
f'<p class="subtitle">{datetime.date.today().isoformat()}{rounds_part} · alternating rounds, '
|
||||
'isolated per-binary stores, wall-clock per process invocation</p></header>'
|
||||
)
|
||||
footer = (
|
||||
'<footer class="band">'
|
||||
'<p>Regenerate: <code>bash scripts/perf-baseline-compare.sh --apps "Slack,Google Chrome"</code></p>'
|
||||
f'<p>Artifacts: <code>{esc(out_dir)}</code> · '
|
||||
f'generated {esc(datetime.datetime.now().isoformat(timespec="seconds"))}</p></footer>'
|
||||
)
|
||||
return header, footer
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("out_dir")
|
||||
parser.add_argument("--head", required=True)
|
||||
parser.add_argument("--base", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
fixture = load(os.path.join(args.out_dir, "fixture-ab.json"))
|
||||
synthetic = load(os.path.join(args.out_dir, "locator-synthetic.json"))
|
||||
apps = [load(p) for p in sorted(glob.glob(os.path.join(args.out_dir, "app-*.json")))]
|
||||
apps = [a for a in apps if a]
|
||||
sections = [fixture_section(fixture)] + [app_section(a) for a in apps] + [synthetic_section(synthetic)]
|
||||
tiles = build_tiles(fixture, synthetic)
|
||||
rounds = fixture.get("rounds") if fixture else None
|
||||
header, footer = page_chrome(args.head, args.base, rounds, os.path.abspath(args.out_dir))
|
||||
body = header + tiles_html(tiles) + '<main class="col">' + "".join(sections) + "</main>" + footer
|
||||
title = f"agent-desktop performance - HEAD {args.head} vs base {args.base}"
|
||||
html_doc = PAGE.format(title=esc(title), css=CSS, body=body, js=JS)
|
||||
|
||||
out_path = os.path.join(args.out_dir, "report.html")
|
||||
with open(out_path, "w") as handle:
|
||||
handle.write(html_doc)
|
||||
|
||||
print(f"agent-desktop performance report: HEAD {args.head} vs base {args.base}")
|
||||
for t in tiles:
|
||||
print(f' {t["label"]}: {t["value"]} ({t["delta_text"]})')
|
||||
print(f" apps covered: {', '.join(a['app'] for a in apps) if apps else 'none'}")
|
||||
print(f" report: {out_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
291
scripts/perf_report_svg.py
Normal file
291
scripts/perf_report_svg.py
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Pure SVG chart builders for the agent-desktop performance report.
|
||||
|
||||
No third-party deps, no file or network I/O. Every public function takes
|
||||
plain-Python row dicts (already formatted by the caller) and returns a
|
||||
self-contained ``<svg>`` string. Colors are applied only via CSS classes
|
||||
(``bar-head``, ``bar-base``, ``good``, ``secondary``, ``muted``, ``status-good``,
|
||||
``chip-*``) so the same markup adapts to the host page's light/dark tokens.
|
||||
"""
|
||||
import math
|
||||
|
||||
VB_W = 860
|
||||
BAR_H = 16
|
||||
BAR_GAP = 2
|
||||
ROW_GAP = 20
|
||||
TOP_PAD = 18
|
||||
AXIS_H = 30
|
||||
CHIP_GAP = 14
|
||||
LABEL_GAP = 14
|
||||
CHECK = "✓"
|
||||
|
||||
|
||||
def esc(value):
|
||||
text = str(value)
|
||||
return (
|
||||
text.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace('"', """)
|
||||
.replace("'", "'")
|
||||
)
|
||||
|
||||
|
||||
def pct_delta(head, base):
|
||||
if not base:
|
||||
return None
|
||||
return (head - base) / base * 100.0
|
||||
|
||||
|
||||
def fmt_delta(head, base):
|
||||
delta = pct_delta(head, base)
|
||||
if delta is None:
|
||||
return None, "muted"
|
||||
return f"{delta:+.0f}%", ("good" if head < base else "secondary")
|
||||
|
||||
|
||||
def nice_ticks(max_value, target=4):
|
||||
if max_value <= 0:
|
||||
return [0, 1]
|
||||
raw_step = max_value / target
|
||||
magnitude = 10 ** math.floor(math.log10(raw_step))
|
||||
step = magnitude
|
||||
for mult in (1, 2, 2.5, 5, 10):
|
||||
step = mult * magnitude
|
||||
if step >= raw_step:
|
||||
break
|
||||
ticks, value, limit = [], 0.0, max_value + step * 0.5
|
||||
while value <= limit:
|
||||
ticks.append(round(value, 6))
|
||||
value += step
|
||||
return ticks
|
||||
|
||||
|
||||
def _clamp(value, lo, hi):
|
||||
return max(lo, min(hi, value))
|
||||
|
||||
|
||||
def _measure(text, size):
|
||||
return len(str(text)) * size * 0.54
|
||||
|
||||
|
||||
def _bar(x, y, w, h, cls):
|
||||
w = max(w, 0.0)
|
||||
if w <= 0:
|
||||
return ""
|
||||
r = min(4.0, w / 2, h / 2)
|
||||
out = [f'<rect class="{cls}" x="{x:.1f}" y="{y:.1f}" width="{w:.1f}" height="{h:.1f}" rx="{r:.1f}" ry="{r:.1f}"/>']
|
||||
if w > r:
|
||||
out.append(f'<rect class="{cls}" x="{x:.1f}" y="{y:.1f}" width="{r:.1f}" height="{h:.1f}"/>')
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _tick(x, y, h, cls="tick"):
|
||||
return f'<line class="{cls}" x1="{x:.1f}" y1="{y - 3:.1f}" x2="{x:.1f}" y2="{y + h + 3:.1f}"/>'
|
||||
|
||||
|
||||
def _hit(x, y, w, h, title, rows):
|
||||
row_str = esc("|".join(f"{label}={value}" for label, value in rows))
|
||||
return (
|
||||
f'<rect class="hit" x="{x:.1f}" y="{y:.1f}" width="{w:.1f}" height="{h:.1f}" '
|
||||
f'data-tt-title="{esc(title)}" data-tt-rows="{row_str}"/>'
|
||||
)
|
||||
|
||||
|
||||
def _chip(x_right, y_mid, text, cls, size=11):
|
||||
w = _measure(text, size) + 18
|
||||
h = 20
|
||||
x, y = x_right - w, y_mid - h / 2
|
||||
return (
|
||||
f'<rect class="chip chip-{cls}" x="{x:.1f}" y="{y:.1f}" width="{w:.1f}" height="{h}" rx="10" ry="10"/>'
|
||||
f'<text class="chip-text chip-text-{cls}" x="{x_right - 9:.1f}" y="{y_mid + 4:.1f}" '
|
||||
f'text-anchor="end" font-size="{size}">{esc(text)}</text>'
|
||||
)
|
||||
|
||||
|
||||
def _axis(sx, ticks, top, bottom, unit, suffix_x=False):
|
||||
out = []
|
||||
for tick in ticks:
|
||||
tx = sx(tick)
|
||||
out.append(f'<line class="gridline" x1="{tx:.1f}" y1="{top - 6:.1f}" x2="{tx:.1f}" y2="{bottom + 6:.1f}"/>')
|
||||
label = f"{tick:g}x" if suffix_x else f"{tick:g}{unit}"
|
||||
out.append(
|
||||
f'<text class="axis-tick muted tabular" x="{tx:.1f}" y="{bottom + 22:.1f}" '
|
||||
f'text-anchor="middle" font-size="10.5">{label}</text>'
|
||||
)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _grouped_summary(rows):
|
||||
best = max(rows, key=lambda r: abs(pct_delta(r["head"]["value"], r["base"]["value"]) or 0))
|
||||
delta = pct_delta(best["head"]["value"], best["base"]["value"]) or 0
|
||||
return (
|
||||
f"Grouped horizontal bar chart comparing HEAD and BASE p50 latency across {len(rows)} cases. "
|
||||
f"Largest change: {best['label']} at {delta:+.0f}%."
|
||||
)
|
||||
|
||||
|
||||
def grouped_bar_chart(rows, unit=" ms"):
|
||||
if not rows:
|
||||
return '<svg class="chart" viewBox="0 0 860 40" role="img" aria-label="No data available"></svg>'
|
||||
label_w = _clamp(max(_measure(r["label"], 12.5) for r in rows) + LABEL_GAP, 120, 300)
|
||||
chip_w = _clamp(
|
||||
max((_measure(r["delta_text"], 11) + 18 for r in rows if r.get("delta_text")), default=50), 50, 200
|
||||
)
|
||||
x0, x1 = label_w, VB_W - chip_w - CHIP_GAP
|
||||
plot_w = max(x1 - x0, 40)
|
||||
data_max = max(max(r["head"]["value"], r["head"]["tick"], r["base"]["value"], r["base"]["tick"]) for r in rows)
|
||||
ticks = nice_ticks(data_max, 4)
|
||||
domain_max = (ticks[-1] or 1) * 1.24
|
||||
group_h = BAR_H * 2 + BAR_GAP
|
||||
row_pitch = group_h + ROW_GAP
|
||||
top = TOP_PAD
|
||||
bottom = top + row_pitch * len(rows) - ROW_GAP
|
||||
height = bottom + AXIS_H
|
||||
|
||||
def sx(v):
|
||||
return x0 + (v / domain_max) * plot_w
|
||||
|
||||
parts = [
|
||||
f'<svg class="chart" viewBox="0 0 {VB_W} {height:.0f}" width="100%" role="img" '
|
||||
f'aria-label="{esc(_grouped_summary(rows))}">',
|
||||
_axis(sx, ticks, top, bottom, unit),
|
||||
]
|
||||
for i, row in enumerate(rows):
|
||||
y = top + i * row_pitch
|
||||
y_head, y_base, cy = y, y + BAR_H + BAR_GAP, y + group_h / 2
|
||||
parts.append(
|
||||
f'<text class="row-label ink" x="{x0 - LABEL_GAP:.1f}" y="{cy + 4:.1f}" '
|
||||
f'text-anchor="end" font-size="12.5">{esc(row["label"])}</text>'
|
||||
)
|
||||
for series, sy, cls in (("head", y_head, "bar-head"), ("base", y_base, "bar-base")):
|
||||
d = row[series]
|
||||
w = max((d["value"] / domain_max) * plot_w, 0)
|
||||
parts.append(_bar(x0, sy, w, BAR_H, cls))
|
||||
if d.get("tick") is not None:
|
||||
parts.append(_tick(sx(d["tick"]), sy, BAR_H))
|
||||
parts.append(
|
||||
f'<text class="value-label ink tabular" x="{x0 + w + 6:.1f}" y="{sy + BAR_H / 1.4:.1f}" '
|
||||
f'font-size="12">{d["value"]:.1f}{unit}</text>'
|
||||
)
|
||||
parts.append(_hit(0, sy, VB_W, BAR_H, d["tip_title"], d["tip_rows"]))
|
||||
if row.get("delta_text"):
|
||||
parts.append(_chip(VB_W, cy, row["delta_text"], "good" if row.get("delta_good") else "secondary"))
|
||||
parts.append("</svg>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _single_summary(rows):
|
||||
return f"Bar chart of {len(rows)} HEAD-only measurement(s) with no baseline equivalent."
|
||||
|
||||
|
||||
def single_series_bar_chart(rows, unit=" ms"):
|
||||
if not rows:
|
||||
return '<svg class="chart" viewBox="0 0 860 40" role="img" aria-label="No data available"></svg>'
|
||||
label_w = _clamp(max(_measure(r["label"], 12.5) for r in rows) + LABEL_GAP, 120, 300)
|
||||
chip_w = _clamp(
|
||||
max((_measure(r["badge_text"], 10.5) + 24 for r in rows if r.get("badge_text")), default=0), 0, 320
|
||||
)
|
||||
gap = CHIP_GAP if chip_w else 0
|
||||
x0, x1 = label_w, VB_W - chip_w - gap
|
||||
plot_w = max(x1 - x0, 40)
|
||||
data_max = max(max(r["value"], r.get("tick") or 0) for r in rows)
|
||||
ticks = nice_ticks(data_max, 4)
|
||||
domain_max = (ticks[-1] or 1) * 1.24
|
||||
row_pitch = BAR_H + ROW_GAP
|
||||
top = TOP_PAD
|
||||
bottom = top + row_pitch * len(rows) - ROW_GAP
|
||||
height = bottom + AXIS_H
|
||||
|
||||
def sx(v):
|
||||
return x0 + (v / domain_max) * plot_w
|
||||
|
||||
parts = [
|
||||
f'<svg class="chart" viewBox="0 0 {VB_W} {height:.0f}" width="100%" role="img" '
|
||||
f'aria-label="{esc(_single_summary(rows))}">',
|
||||
_axis(sx, ticks, top, bottom, unit),
|
||||
]
|
||||
for i, row in enumerate(rows):
|
||||
y = top + i * row_pitch
|
||||
cy = y + BAR_H / 2
|
||||
parts.append(
|
||||
f'<text class="row-label ink" x="{x0 - LABEL_GAP:.1f}" y="{cy + 4:.1f}" '
|
||||
f'text-anchor="end" font-size="12.5">{esc(row["label"])}</text>'
|
||||
)
|
||||
w = max((row["value"] / domain_max) * plot_w, 0)
|
||||
parts.append(_bar(x0, y, w, BAR_H, "bar-head"))
|
||||
if row.get("tick") is not None:
|
||||
parts.append(_tick(sx(row["tick"]), y, BAR_H))
|
||||
parts.append(
|
||||
f'<text class="value-label ink tabular" x="{x0 + w + 6:.1f}" y="{cy + 4:.1f}" '
|
||||
f'font-size="12">{row["value"]:.1f}{unit}</text>'
|
||||
)
|
||||
if row.get("badge_text"):
|
||||
parts.append(_chip(VB_W, cy, row["badge_text"], "info", size=10.5))
|
||||
parts.append(_hit(0, y, VB_W, BAR_H, row["tip_title"], row["tip_rows"]))
|
||||
parts.append("</svg>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _speedup_summary(rows):
|
||||
best = max(rows, key=lambda r: r["speedup"])
|
||||
worst = min(rows, key=lambda r: r["speedup"])
|
||||
return (
|
||||
f"Horizontal speedup bars for {len(rows)} synthetic scenarios, live find path versus legacy snapshot path. "
|
||||
f"Speedups range from {worst['speedup']:.1f}x to {best['speedup']:.1f}x against a 1.0x parity reference."
|
||||
)
|
||||
|
||||
|
||||
def speedup_bar_chart(rows):
|
||||
if not rows:
|
||||
return '<svg class="chart" viewBox="0 0 860 40" role="img" aria-label="No data available"></svg>'
|
||||
badge_text = f"{CHECK} legacy answered wrong - live path correct"
|
||||
label_w = _clamp(max(_measure(r["label"], 11.5) for r in rows) + LABEL_GAP, 140, 340)
|
||||
chip_w = _clamp(_measure(badge_text, 10.5) + 24 if any(r.get("correctness_fix") for r in rows) else 0, 0, 340)
|
||||
gap = CHIP_GAP if chip_w else 0
|
||||
x0, x1 = label_w, VB_W - chip_w - gap
|
||||
plot_w = max(x1 - x0, 40)
|
||||
data_max = max([r["speedup"] for r in rows] + [1.0])
|
||||
ticks = nice_ticks(data_max, 4)
|
||||
domain_max = (ticks[-1] or 1) * 1.2
|
||||
row_pitch = BAR_H + ROW_GAP + 14
|
||||
top = TOP_PAD + 6
|
||||
bottom = top + row_pitch * len(rows) - ROW_GAP - 14
|
||||
height = bottom + AXIS_H
|
||||
|
||||
def sx(v):
|
||||
return x0 + (v / domain_max) * plot_w
|
||||
|
||||
parts = [
|
||||
f'<svg class="chart" viewBox="0 0 {VB_W} {height:.0f}" width="100%" role="img" '
|
||||
f'aria-label="{esc(_speedup_summary(rows))}">',
|
||||
_axis(sx, [t for t in ticks if abs(t - 1.0) > 1e-9], top, bottom, "", suffix_x=True),
|
||||
]
|
||||
px = sx(1.0)
|
||||
parts.append(f'<line class="parity" x1="{px:.1f}" y1="{top - 6:.1f}" x2="{px:.1f}" y2="{bottom + 6:.1f}"/>')
|
||||
parts.append(
|
||||
f'<text class="muted" x="{px:.1f}" y="{top - 9:.1f}" text-anchor="middle" font-size="10.5">parity</text>'
|
||||
)
|
||||
for i, row in enumerate(rows):
|
||||
y = top + i * row_pitch
|
||||
cy = y + BAR_H / 2
|
||||
parts.append(
|
||||
f'<text class="row-label ink" x="{x0 - LABEL_GAP:.1f}" y="{cy + 4:.1f}" '
|
||||
f'text-anchor="end" font-size="11.5">{esc(row["label"])}</text>'
|
||||
)
|
||||
w = max((row["speedup"] / domain_max) * plot_w, 0)
|
||||
parts.append(_bar(x0, y, w, BAR_H, "bar-head"))
|
||||
end_x = x0 + w + 6
|
||||
parts.append(
|
||||
f'<text class="value-label ink tabular" x="{end_x:.1f}" y="{cy + 4:.1f}" '
|
||||
f'font-size="12">{row["speedup"]:.1f}x</text>'
|
||||
)
|
||||
if row.get("sub_label"):
|
||||
parts.append(
|
||||
f'<text class="muted" x="{end_x:.1f}" y="{cy + 17:.1f}" font-size="10">{esc(row["sub_label"])}</text>'
|
||||
)
|
||||
if row.get("correctness_fix"):
|
||||
parts.append(_chip(VB_W, cy, badge_text, "status-good", size=10.5))
|
||||
parts.append(_hit(0, y - 7, VB_W, BAR_H + 14, row["tip_title"], row["tip_rows"]))
|
||||
parts.append("</svg>")
|
||||
return "".join(parts)
|
||||
Loading…
Reference in a new issue