diff --git a/CLAUDE.md b/CLAUDE.md index 042088a..c1a8b0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -429,9 +429,9 @@ Target binary size: <15MB per platform. - `cargo test --workspace` - Binary size check: fail if release binary exceeds 15MB -## Implemented Commands (53) +## Implemented Commands (54) -> **Platform note:** All 53 commands are implemented on macOS (Phase 1). Windows and Linux adapters are planned (Phase 2/3) and will support the same command surface; notification commands depend on platform-specific notification APIs. +> **Platform note:** All 54 commands are implemented on macOS (Phase 1). Windows and Linux adapters are planned (Phase 2/3) and will support the same command surface; notification commands depend on platform-specific notification APIs. | Category | Commands | |----------|----------| @@ -444,7 +444,7 @@ Target binary size: <15MB per platform. | Notifications (4) *(macOS)* | `list-notifications`, `dismiss-notification`, `dismiss-all-notifications`, `notification-action` | | Clipboard (3) | `clipboard-get`, `clipboard-set`, `clipboard-clear` | | Wait (1) | `wait` (with `--element`, `--window`, `--text`, `--menu`, `--notification` flags) | -| System (3) | `status`, `permissions`, `version` | +| System (4) | `status`, `permissions`, `version`, `skills` | | Batch (1) | `batch` | ## Non-Goals diff --git a/README.md b/README.md index 058ab6e..1fcbdbf 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ - **Native Rust CLI**: Fast, single binary, no runtime dependencies - **C-ABI cdylib** (`libagent_desktop_ffi`): Load once from Python / Swift / Go / Ruby / Node / C instead of forking the CLI per call -- **53 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management +- **54 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, plus a bundled `skills` doc loader - **Progressive skeleton traversal**: 78–96% token reduction on dense apps via shallow overview + targeted drill-down - **Snapshot & refs**: AI-optimized workflow using deterministic element references (`@e1`, `@e2`) - **AX-first interactions**: Every action exhausts pure accessibility API strategies before falling back to mouse events diff --git a/crates/core/src/commands/mod.rs b/crates/core/src/commands/mod.rs index 8a9f578..239e33a 100644 --- a/crates/core/src/commands/mod.rs +++ b/crates/core/src/commands/mod.rs @@ -44,6 +44,7 @@ pub mod scroll; pub mod scroll_to; pub mod select; pub mod set_value; +pub mod skills; pub mod snapshot; pub mod status; pub mod toggle; diff --git a/crates/core/src/commands/skills.rs b/crates/core/src/commands/skills.rs new file mode 100644 index 0000000..e0f30fb --- /dev/null +++ b/crates/core/src/commands/skills.rs @@ -0,0 +1,244 @@ +use crate::error::AppError; +use serde_json::{json, Value}; + +const SKILL_DESKTOP_MAIN: &str = include_str!("../../../../skills/agent-desktop/SKILL.md"); +const SKILL_DESKTOP_REF_OBSERVATION: &str = + include_str!("../../../../skills/agent-desktop/references/commands-observation.md"); +const SKILL_DESKTOP_REF_INTERACTION: &str = + include_str!("../../../../skills/agent-desktop/references/commands-interaction.md"); +const SKILL_DESKTOP_REF_SYSTEM: &str = + include_str!("../../../../skills/agent-desktop/references/commands-system.md"); +const SKILL_DESKTOP_REF_WORKFLOWS: &str = + include_str!("../../../../skills/agent-desktop/references/workflows.md"); +const SKILL_DESKTOP_REF_MACOS: &str = + include_str!("../../../../skills/agent-desktop/references/macos.md"); + +const SKILL_FFI_MAIN: &str = include_str!("../../../../skills/agent-desktop-ffi/SKILL.md"); +const SKILL_FFI_REF_BUILD: &str = + include_str!("../../../../skills/agent-desktop-ffi/references/build-and-link.md"); +const SKILL_FFI_REF_ERRORS: &str = + include_str!("../../../../skills/agent-desktop-ffi/references/error-handling.md"); +const SKILL_FFI_REF_OWNERSHIP: &str = + include_str!("../../../../skills/agent-desktop-ffi/references/ownership.md"); +const SKILL_FFI_REF_THREADING: &str = + include_str!("../../../../skills/agent-desktop-ffi/references/threading.md"); + +struct SkillRef { + rel_path: &'static str, + body: &'static str, +} + +struct Skill { + canonical: &'static str, + aliases: &'static [&'static str], + summary: &'static str, + main: &'static str, + refs: &'static [SkillRef], +} + +const SKILLS: &[Skill] = &[ + Skill { + canonical: "agent-desktop", + aliases: &["desktop", "agent-desktop"], + summary: "Primary guide. Snapshot/ref loop, JSON envelope, 53 commands across observation, interaction, keyboard/mouse, app lifecycle, notifications, clipboard, wait.", + main: SKILL_DESKTOP_MAIN, + refs: &[ + SkillRef { rel_path: "references/commands-observation.md", body: SKILL_DESKTOP_REF_OBSERVATION }, + SkillRef { rel_path: "references/commands-interaction.md", body: SKILL_DESKTOP_REF_INTERACTION }, + SkillRef { rel_path: "references/commands-system.md", body: SKILL_DESKTOP_REF_SYSTEM }, + SkillRef { rel_path: "references/workflows.md", body: SKILL_DESKTOP_REF_WORKFLOWS }, + SkillRef { rel_path: "references/macos.md", body: SKILL_DESKTOP_REF_MACOS }, + ], + }, + Skill { + canonical: "agent-desktop-ffi", + aliases: &["ffi", "agent-desktop-ffi"], + summary: "Embedding agent-desktop in another process via the C ABI. Build/link, error propagation, handle ownership, threading rules.", + main: SKILL_FFI_MAIN, + refs: &[ + SkillRef { rel_path: "references/build-and-link.md", body: SKILL_FFI_REF_BUILD }, + SkillRef { rel_path: "references/error-handling.md", body: SKILL_FFI_REF_ERRORS }, + SkillRef { rel_path: "references/ownership.md", body: SKILL_FFI_REF_OWNERSHIP }, + SkillRef { rel_path: "references/threading.md", body: SKILL_FFI_REF_THREADING }, + ], + }, +]; + +pub struct GetArgs { + pub name: String, + pub full: bool, + pub reference: Option, +} + +pub fn list() -> Result { + let entries: Vec = SKILLS + .iter() + .map(|s| { + json!({ + "name": s.canonical, + "aliases": s.aliases, + "summary": s.summary, + "references": s.refs.iter().map(|r| r.rel_path).collect::>(), + }) + }) + .collect(); + Ok(json!({ "skills": entries })) +} + +pub fn get(args: GetArgs) -> Result { + let skill = find_skill(&args.name)?; + + if let Some(rel) = args.reference { + let r = skill + .refs + .iter() + .find(|r| matches_ref(r.rel_path, &rel)) + .ok_or_else(|| { + let available: Vec<&str> = skill.refs.iter().map(|r| r.rel_path).collect(); + AppError::invalid_input(format!( + "Unknown reference '{rel}' for skill '{}'. Available: {}", + skill.canonical, + available.join(", ") + )) + })?; + return Ok(json!({ + "skill": skill.canonical, + "reference": r.rel_path, + "content": r.body, + })); + } + + let content = if args.full { + render_full(skill) + } else { + skill.main.to_string() + }; + + Ok(json!({ + "skill": skill.canonical, + "full": args.full, + "content": content, + })) +} + +pub fn path() -> Result { + Ok(json!({ + "location": "embedded", + "note": "Skills are compiled into this binary and are always version-matched. Run `agent-desktop skills get ` to print a skill, or redirect into a file to extract a copy.", + "available": SKILLS.iter().map(|s| s.canonical).collect::>(), + })) +} + +fn find_skill(name: &str) -> Result<&'static Skill, AppError> { + let needle = name.trim(); + SKILLS + .iter() + .find(|s| s.aliases.iter().any(|a| a.eq_ignore_ascii_case(needle))) + .ok_or_else(|| { + let known: Vec<&str> = SKILLS + .iter() + .flat_map(|s| s.aliases.iter().copied()) + .collect(); + AppError::invalid_input(format!( + "Unknown skill '{name}'. Known: {}", + known.join(", ") + )) + }) +} + +fn matches_ref(rel_path: &str, query: &str) -> bool { + if rel_path.eq_ignore_ascii_case(query) { + return true; + } + let stem = rel_path + .rsplit('/') + .next() + .and_then(|f| f.strip_suffix(".md").or(Some(f))) + .unwrap_or(rel_path); + stem.eq_ignore_ascii_case(query) +} + +fn render_full(skill: &Skill) -> String { + let mut out = String::with_capacity( + skill.main.len() + skill.refs.iter().map(|r| r.body.len() + 64).sum::(), + ); + out.push_str(skill.main); + for r in skill.refs { + if !out.ends_with('\n') { + out.push('\n'); + } + out.push_str("\n--- "); + out.push_str(r.rel_path); + out.push_str(" ---\n\n"); + out.push_str(r.body); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn list_returns_known_skills() { + let v = list().expect("list"); + let arr = v["skills"].as_array().expect("array"); + assert!(arr.iter().any(|s| s["name"] == "agent-desktop")); + assert!(arr.iter().any(|s| s["name"] == "agent-desktop-ffi")); + } + + #[test] + fn get_resolves_alias() { + let v = get(GetArgs { + name: "desktop".into(), + full: false, + reference: None, + }) + .expect("get"); + assert_eq!(v["skill"], "agent-desktop"); + assert!(v["content"].as_str().unwrap().contains("agent-desktop")); + } + + #[test] + fn get_full_inlines_references() { + let v = get(GetArgs { + name: "desktop".into(), + full: true, + reference: None, + }) + .expect("get full"); + let content = v["content"].as_str().expect("string"); + assert!(content.contains("--- references/workflows.md ---")); + assert!(content.contains("--- references/macos.md ---")); + } + + #[test] + fn get_specific_reference() { + let v = get(GetArgs { + name: "desktop".into(), + full: false, + reference: Some("workflows".into()), + }) + .expect("get ref"); + assert_eq!(v["reference"], "references/workflows.md"); + } + + #[test] + fn unknown_skill_errors() { + let err = get(GetArgs { + name: "nope".into(), + full: false, + reference: None, + }) + .expect_err("should error"); + assert!(format!("{err}").contains("Unknown skill")); + } + + #[test] + fn path_lists_canonical_names() { + let v = path().expect("path"); + assert_eq!(v["location"], "embedded"); + let avail = v["available"].as_array().expect("arr"); + assert!(avail.iter().any(|s| s == "agent-desktop")); + } +} diff --git a/skills/agent-desktop/SKILL.md b/skills/agent-desktop/SKILL.md index eee960b..d555790 100644 --- a/skills/agent-desktop/SKILL.md +++ b/skills/agent-desktop/SKILL.md @@ -9,8 +9,9 @@ description: > Use when an AI agent needs to observe, interact with, or automate desktop applications (click buttons, fill forms, navigate menus, read UI state, toggle checkboxes, scroll, drag, type text, take screenshots, manage windows, use clipboard, manage notifications). - Covers 53 commands across observation, interaction, keyboard/mouse, app lifecycle, - notifications (macOS), clipboard, and wait. + Covers 54 commands across observation, interaction, keyboard/mouse, app lifecycle, + notifications (macOS), clipboard, wait, and a `skills` command that prints these + bundled docs straight from the binary. Triggers on: "click button", "fill form", "open app", "read UI", "automate desktop", "accessibility tree", "snapshot app", "type into field", "navigate menu", "toggle checkbox", "take screenshot", "desktop automation", "agent-desktop", or any desktop GUI interaction task. @@ -110,7 +111,7 @@ Exit codes: `0` success, `1` structured error, `2` argument error. | `TIMEOUT` | Wait condition not met | Increase --timeout | | `INVALID_ARGS` | Bad arguments | Check command syntax | -## Command Quick Reference (53 commands) +## Command Quick Reference (54 commands) ### Observation ``` @@ -211,6 +212,8 @@ agent-desktop permissions # Check permission agent-desktop permissions --request # Trigger permission dialog agent-desktop version --json # Version info agent-desktop batch '[...]' --stop-on-error # Batch commands +agent-desktop skills # List bundled skill docs +agent-desktop skills get desktop --full # Load this skill + all references ``` ## Key Principles for Agents diff --git a/skills/agent-desktop/references/commands-system.md b/skills/agent-desktop/references/commands-system.md index efb55cf..0083cde 100644 --- a/skills/agent-desktop/references/commands-system.md +++ b/skills/agent-desktop/references/commands-system.md @@ -259,3 +259,35 @@ agent-desktop version agent-desktop version --json ``` Returns version string. Use `--json` for `{ "version": "0.1.3", "platform": "macos", "arch": "aarch64" }`. + +## Skills (bundled docs) + +Skill markdown ships compiled into the binary. Use these to load up-to-date guidance without hitting the network. + +### skills (or `skills list`) +```bash +agent-desktop skills +``` +Lists every bundled skill with aliases, summaries, and reference filenames. + +### skills get +```bash +agent-desktop skills get desktop # Primary guide (this skill's main file) +agent-desktop skills get desktop --full # Main + every reference inlined with `--- references/ ---` separators +agent-desktop skills get desktop workflows # Single reference; bare stem or `references/workflows.md` both work +agent-desktop skills get ffi # Specialized: embedding via the C ABI +``` + +| Arg / Flag | Description | +|------------|-------------| +| `` | Skill name or alias. `desktop` ↔ `agent-desktop`, `ffi` ↔ `agent-desktop-ffi`. | +| `` (positional) | Reference filename (stem or full `references/.md`). Omit for the main guide. | +| `--full` | Inline every reference after the main file. Ignored when a specific reference is requested. | + +JSON envelope contains the markdown under `data.content`. Pipe to `jq -r .data.content` (or extract with `python3 -c`) to print just the markdown. + +### skills path +```bash +agent-desktop skills path +``` +Reports `{ "location": "embedded", ... }` — skills are baked into this binary via `include_str!`. To extract a copy on disk, redirect `skills get ` output into a file. diff --git a/src/cli.rs b/src/cli.rs index 5b6f179..94402ab 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,115 +2,18 @@ use clap::{Parser, Subcommand}; pub use crate::cli_args::*; pub use crate::cli_args_notifications::*; +pub use crate::cli_args_skills::*; + +const BEFORE_HELP: &str = include_str!("help_before.txt"); +const AFTER_HELP: &str = include_str!("help_after.txt"); #[derive(Parser, Debug)] #[command( name = "agent-desktop", about = "Desktop automation CLI for AI agents", long_about = None, - after_help = "\ -OBSERVATION - snapshot Accessibility tree as JSON with @ref IDs - screenshot PNG screenshot of an application window - find Search elements by role, name, value, or text - get Read element property: text, value, title, bounds, role, states - is Check state: visible, enabled, checked, focused, expanded - list-surfaces Available surfaces for an app - -INTERACTION - click Click element (kAXPress) - double-click Double-click element - triple-click Triple-click element (select line/paragraph) - right-click Right-click and open context menu - type Focus element and type text - set-value Set value attribute directly - clear Clear element value to empty string - focus Set keyboard focus - select Select option in list or dropdown - toggle Toggle checkbox or switch - check Set checkbox/switch to checked (idempotent) - uncheck Set checkbox/switch to unchecked (idempotent) - expand Expand disclosure triangle or tree item - collapse Collapse disclosure triangle or tree item - scroll Scroll element (--direction, --amount) - scroll-to Scroll element into visible area - -KEYBOARD - press Key combo: return, escape, cmd+c, shift+tab ... - key-down Hold a key or modifier down - key-up Release a held key or modifier - -MOUSE - hover Move cursor to element or coordinates - drag Drag from one element/point to another - mouse-move --xy x,y Move cursor to absolute coordinates - mouse-click --xy x,y Click at coordinates (--button, --count) - mouse-down --xy x,y Press mouse button at coordinates - mouse-up --xy x,y Release mouse button at coordinates - -APP & WINDOW - launch Launch app and wait until window is visible - close-app Quit app gracefully (--force to kill) - list-windows All visible windows (--app to filter) - list-apps All running GUI applications - focus-window Bring window to front - resize-window Resize window (--width, --height) - move-window Move window (--x, --y) - minimize Minimize window - maximize Maximize/zoom window - restore Restore minimized/maximized window - -NOTIFICATIONS - list-notifications List notifications from Notification Center - dismiss-notification Dismiss notification by index - dismiss-all-notifications Dismiss all notifications - notification-action Click action button on notification - -CLIPBOARD - clipboard-get Read plain-text clipboard - clipboard-set Write text to clipboard - clipboard-clear Clear the clipboard - -WAIT - wait [ms] Pause for N milliseconds - wait --element Block until element appears (--timeout ms) - wait --window Block until window appears - wait --text <text> Block until text appears in app - wait --notification Block until a new notification arrives - -SYSTEM - status Adapter health, platform, and permission state - permissions Check accessibility permission (--request to prompt) - version Version string (--json for machine-readable) - -BATCH - batch <json> Run commands from a JSON array (--stop-on-error) - -REF IDs - snapshot assigns @e1, @e2, ... to interactive elements in depth-first order. - Use a ref wherever <ref> appears. Refs are snapshot-scoped; run snapshot - again after UI changes. - -KEY COMBOS - Single keys: return, escape, tab, space, delete, up, down, left, right - Function keys: f1 - f12 - With modifiers: cmd+c, cmd+v, cmd+z, cmd+shift+z, ctrl+a, shift+tab - Modifiers: cmd, ctrl, alt, shift - -EXAMPLES - agent-desktop snapshot --app \"System Settings\" -i - agent-desktop find --role button --name \"OK\" - agent-desktop click @e5 - agent-desktop check @e3 - agent-desktop type @e2 \"hello@example.com\" - agent-desktop press cmd+z - agent-desktop drag --from @e1 --to @e5 - agent-desktop hover @e5 - agent-desktop minimize --app TextEdit - agent-desktop resize-window --app TextEdit --width 800 --height 600 - agent-desktop mouse-click --xy 500,300 - agent-desktop wait --text \"Loading complete\" --app Safari --timeout 5000 - agent-desktop batch '[{\"command\":\"click\",\"args\":{\"ref_id\":\"@e1\"}}]'" + before_help = BEFORE_HELP, + after_help = AFTER_HELP, )] pub struct Cli { #[arg( @@ -233,6 +136,8 @@ pub enum Commands { Version(VersionArgs), #[command(about = "Execute multiple commands from a JSON array (--stop-on-error)")] Batch(BatchArgs), + #[command(about = "Bundled skill docs for AI agents (list, get, path)")] + Skills(SkillsArgs), } impl Commands { @@ -291,6 +196,7 @@ impl Commands { Self::Permissions(_) => "permissions", Self::Version(_) => "version", Self::Batch(_) => "batch", + Self::Skills(_) => "skills", } } } diff --git a/src/cli_args_skills.rs b/src/cli_args_skills.rs new file mode 100644 index 0000000..a209027 --- /dev/null +++ b/src/cli_args_skills.rs @@ -0,0 +1,40 @@ +use clap::{Args, Subcommand}; + +#[derive(Args, Debug)] +#[command(after_help = "\ +Skills travel inside the binary so they always match this exact +agent-desktop version. Output is raw markdown on stdout — parse it +directly, or redirect into a file for storage. + +Examples: + agent-desktop skills # List skills + agent-desktop skills get desktop # Primary guide + agent-desktop skills get desktop --full # Plus every reference + agent-desktop skills get desktop workflows # Single reference + agent-desktop skills path # Where skills live")] +pub struct SkillsArgs { + #[command(subcommand)] + pub action: Option<SkillsAction>, +} + +#[derive(Subcommand, Debug)] +pub enum SkillsAction { + #[command(about = "List bundled skills with summaries (default)")] + List, + #[command(about = "Print a skill's markdown to stdout")] + Get(SkillsGetArgs), + #[command(about = "Print where bundled skills live")] + Path, +} + +#[derive(Args, Debug)] +pub struct SkillsGetArgs { + #[arg(help = "Skill name or alias (desktop, ffi, ...)")] + pub name: String, + #[arg( + help = "Reference filename (e.g. workflows or references/workflows.md). Omit for the main guide." + )] + pub reference: Option<String>, + #[arg(long, help = "Append every reference file to the output")] + pub full: bool, +} diff --git a/src/dispatch.rs b/src/dispatch.rs index 289972d..97e39bc 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -7,13 +7,13 @@ use agent_desktop_core::{ is_check, key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize, minimize, mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press, resize_window, restore, right_click, screenshot, scroll, scroll_to, select, set_value, - snapshot, status, toggle, triple_click, type_text, uncheck, version, wait, + skills, snapshot, status, toggle, triple_click, type_text, uncheck, version, wait, }, error::AppError, }; use serde_json::Value; -use crate::cli::Commands; +use crate::cli::{Commands, SkillsAction}; pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> { tracing::debug!("dispatch: {}", cmd.name()); @@ -301,6 +301,16 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A Commands::Version(a) => version::execute(version::VersionArgs { json: a.json }), + Commands::Skills(a) => match a.action.unwrap_or(SkillsAction::List) { + SkillsAction::List => skills::list(), + SkillsAction::Path => skills::path(), + SkillsAction::Get(g) => skills::get(skills::GetArgs { + name: g.name, + full: g.full, + reference: g.reference, + }), + }, + Commands::Batch(a) => { let commands = batch::parse_commands(&a.commands_json)?; let mut results = Vec::new(); diff --git a/src/help_after.txt b/src/help_after.txt new file mode 100644 index 0000000..9ddbd88 --- /dev/null +++ b/src/help_after.txt @@ -0,0 +1,104 @@ +OBSERVATION + snapshot Accessibility tree as JSON with @ref IDs + screenshot PNG screenshot of an application window + find Search elements by role, name, value, or text + get <ref> <property> Read element property: text, value, title, bounds, role, states + is <ref> <property> Check state: visible, enabled, checked, focused, expanded + list-surfaces Available surfaces for an app + +INTERACTION + click <ref> Click element (kAXPress) + double-click <ref> Double-click element + triple-click <ref> Triple-click element (select line/paragraph) + right-click <ref> Right-click and open context menu + type <ref> <text> Focus element and type text + set-value <ref> <value> Set value attribute directly + clear <ref> Clear element value to empty string + focus <ref> Set keyboard focus + select <ref> <value> Select option in list or dropdown + toggle <ref> Toggle checkbox or switch + check <ref> Set checkbox/switch to checked (idempotent) + uncheck <ref> Set checkbox/switch to unchecked (idempotent) + expand <ref> Expand disclosure triangle or tree item + collapse <ref> Collapse disclosure triangle or tree item + scroll <ref> Scroll element (--direction, --amount) + scroll-to <ref> Scroll element into visible area + +KEYBOARD + press <combo> Key combo: return, escape, cmd+c, shift+tab ... + key-down <combo> Hold a key or modifier down + key-up <combo> Release a held key or modifier + +MOUSE + hover <ref|--xy> Move cursor to element or coordinates + drag Drag from one element/point to another + mouse-move --xy x,y Move cursor to absolute coordinates + mouse-click --xy x,y Click at coordinates (--button, --count) + mouse-down --xy x,y Press mouse button at coordinates + mouse-up --xy x,y Release mouse button at coordinates + +APP & WINDOW + launch <app> Launch app and wait until window is visible + close-app <app> Quit app gracefully (--force to kill) + list-windows All visible windows (--app to filter) + list-apps All running GUI applications + focus-window Bring window to front + resize-window Resize window (--width, --height) + move-window Move window (--x, --y) + minimize Minimize window + maximize Maximize/zoom window + restore Restore minimized/maximized window + +NOTIFICATIONS + list-notifications List notifications from Notification Center + dismiss-notification <n> Dismiss notification by index + dismiss-all-notifications Dismiss all notifications + notification-action <n> <action> Click action button on notification + +CLIPBOARD + clipboard-get Read plain-text clipboard + clipboard-set <text> Write text to clipboard + clipboard-clear Clear the clipboard + +WAIT + wait [ms] Pause for N milliseconds + wait --element <ref> Block until element appears (--timeout ms) + wait --window <title> Block until window appears + wait --text <text> Block until text appears in app + wait --notification Block until a new notification arrives + +SYSTEM + status Adapter health, platform, and permission state + permissions Check accessibility permission (--request to prompt) + version Version string (--json for machine-readable) + skills Bundled skill docs for AI agents (list, get, path) + +BATCH + batch <json> Run commands from a JSON array (--stop-on-error) + +REF IDs + snapshot assigns @e1, @e2, ... to interactive elements in depth-first order. + Use a ref wherever <ref> appears. Refs are snapshot-scoped; run snapshot + again after UI changes. + +KEY COMBOS + Single keys: return, escape, tab, space, delete, up, down, left, right + Function keys: f1 - f12 + With modifiers: cmd+c, cmd+v, cmd+z, cmd+shift+z, ctrl+a, shift+tab + Modifiers: cmd, ctrl, alt, shift + +EXAMPLES + agent-desktop skills get desktop --full Load the primary skill (start here) + agent-desktop snapshot --app "System Settings" -i + agent-desktop find --role button --name "OK" + agent-desktop click @e5 + agent-desktop check @e3 + agent-desktop type @e2 "hello@example.com" + agent-desktop press cmd+z + agent-desktop drag --from @e1 --to @e5 + agent-desktop hover @e5 + agent-desktop minimize --app TextEdit + agent-desktop resize-window --app TextEdit --width 800 --height 600 + agent-desktop mouse-click --xy 500,300 + agent-desktop wait --text "Loading complete" --app Safari --timeout 5000 + agent-desktop batch '[{"command":"click","args":{"ref_id":"@e1"}}]' diff --git a/src/help_before.txt b/src/help_before.txt new file mode 100644 index 0000000..e7b0f49 --- /dev/null +++ b/src/help_before.txt @@ -0,0 +1,15 @@ +For AI agents — read this first: + agent-desktop skills get desktop --full + + Bundled skill docs travel with the binary, so they always match this + exact version. They cover the snapshot/ref loop, JSON envelope contract, + error codes with recovery hints, and copy-paste patterns for forms, + menus, dialogs, and async waits. Specialized skills go deeper on macOS + internals (TCC, AX API, surfaces, Notification Center) and on embedding + agent-desktop through its C ABI. + + skills List bundled skills with summaries + skills get desktop Primary guide — overview + the observe-act loop + skills get desktop --full Same, plus every reference file inlined + skills get <name> Open a focused skill (e.g. macos, ffi) + skills path Where skills live (compiled into this binary) diff --git a/src/main.rs b/src/main.rs index d380610..0a2a45d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod batch_dispatch_ext; mod cli; mod cli_args; mod cli_args_notifications; +mod cli_args_skills; mod dispatch; mod dispatch_notifications; @@ -46,23 +47,37 @@ fn main() { let cmd_name = cmd.name(); - match &cmd { + match cmd { Commands::Version(a) => { let result = agent_desktop_core::commands::version::execute( agent_desktop_core::commands::version::VersionArgs { json: a.json }, ); finish(cmd_name, result); - return; } Commands::Status => { let adapter = build_adapter(); let result = agent_desktop_core::commands::status::execute(&adapter); finish(cmd_name, result); - return; } - _ => {} + Commands::Skills(a) => { + let result = match a.action.unwrap_or(cli::SkillsAction::List) { + cli::SkillsAction::List => agent_desktop_core::commands::skills::list(), + cli::SkillsAction::Path => agent_desktop_core::commands::skills::path(), + cli::SkillsAction::Get(g) => agent_desktop_core::commands::skills::get( + agent_desktop_core::commands::skills::GetArgs { + name: g.name, + full: g.full, + reference: g.reference, + }, + ), + }; + finish(cmd_name, result); + } + cmd => run_with_adapter(cmd, cmd_name), } +} +fn run_with_adapter(cmd: Commands, cmd_name: &str) { let adapter = build_adapter(); if let agent_desktop_core::adapter::PermissionStatus::Denied { suggestion } =