mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 01:22:16 +00:00
feat: bundle skill docs and refactor --help for AI agents
Add `agent-desktop skills` (list/get/path) so agents can load version-matched usage guidance straight from the binary, and rework `--help` with a top-of-help "For AI agents — read this first" section that points there. Skills are embedded via `include_str!`, with `desktop` and `ffi` aliases resolving to canonical names. Also extract the long after_help text out of cli.rs into help_after.txt and a new help_before.txt to keep the file under the 400 LOC limit, and sweep the 53→54 command count across CLAUDE.md, README.md, and SKILL.md.
This commit is contained in:
Lahfir
parent
b6b0445baf
commit
0b8dd7be64
12 changed files with 486 additions and 116 deletions
|
|
@ -429,9 +429,9 @@ Target binary size: <15MB per platform.
|
|||
- `cargo test --workspace`
|
||||
- Binary size check: fail if release binary exceeds 15MB
|
||||
|
||||
## Implemented Commands (53)
|
||||
## Implemented Commands (54)
|
||||
|
||||
> **Platform note:** All 53 commands are implemented on macOS (Phase 1). Windows and Linux adapters are planned (Phase 2/3) and will support the same command surface; notification commands depend on platform-specific notification APIs.
|
||||
> **Platform note:** All 54 commands are implemented on macOS (Phase 1). Windows and Linux adapters are planned (Phase 2/3) and will support the same command surface; notification commands depend on platform-specific notification APIs.
|
||||
|
||||
| Category | Commands |
|
||||
|----------|----------|
|
||||
|
|
@ -444,7 +444,7 @@ Target binary size: <15MB per platform.
|
|||
| Notifications (4) *(macOS)* | `list-notifications`, `dismiss-notification`, `dismiss-all-notifications`, `notification-action` |
|
||||
| Clipboard (3) | `clipboard-get`, `clipboard-set`, `clipboard-clear` |
|
||||
| Wait (1) | `wait` (with `--element`, `--window`, `--text`, `--menu`, `--notification` flags) |
|
||||
| System (3) | `status`, `permissions`, `version` |
|
||||
| System (4) | `status`, `permissions`, `version`, `skills` |
|
||||
| Batch (1) | `batch` |
|
||||
|
||||
## Non-Goals
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@
|
|||
|
||||
- **Native Rust CLI**: Fast, single binary, no runtime dependencies
|
||||
- **C-ABI cdylib** (`libagent_desktop_ffi`): Load once from Python / Swift / Go / Ruby / Node / C instead of forking the CLI per call
|
||||
- **53 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management
|
||||
- **54 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, plus a bundled `skills` doc loader
|
||||
- **Progressive skeleton traversal**: 78–96% token reduction on dense apps via shallow overview + targeted drill-down
|
||||
- **Snapshot & refs**: AI-optimized workflow using deterministic element references (`@e1`, `@e2`)
|
||||
- **AX-first interactions**: Every action exhausts pure accessibility API strategies before falling back to mouse events
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ pub mod scroll;
|
|||
pub mod scroll_to;
|
||||
pub mod select;
|
||||
pub mod set_value;
|
||||
pub mod skills;
|
||||
pub mod snapshot;
|
||||
pub mod status;
|
||||
pub mod toggle;
|
||||
|
|
|
|||
244
crates/core/src/commands/skills.rs
Normal file
244
crates/core/src/commands/skills.rs
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
use crate::error::AppError;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
const SKILL_DESKTOP_MAIN: &str = include_str!("../../../../skills/agent-desktop/SKILL.md");
|
||||
const SKILL_DESKTOP_REF_OBSERVATION: &str =
|
||||
include_str!("../../../../skills/agent-desktop/references/commands-observation.md");
|
||||
const SKILL_DESKTOP_REF_INTERACTION: &str =
|
||||
include_str!("../../../../skills/agent-desktop/references/commands-interaction.md");
|
||||
const SKILL_DESKTOP_REF_SYSTEM: &str =
|
||||
include_str!("../../../../skills/agent-desktop/references/commands-system.md");
|
||||
const SKILL_DESKTOP_REF_WORKFLOWS: &str =
|
||||
include_str!("../../../../skills/agent-desktop/references/workflows.md");
|
||||
const SKILL_DESKTOP_REF_MACOS: &str =
|
||||
include_str!("../../../../skills/agent-desktop/references/macos.md");
|
||||
|
||||
const SKILL_FFI_MAIN: &str = include_str!("../../../../skills/agent-desktop-ffi/SKILL.md");
|
||||
const SKILL_FFI_REF_BUILD: &str =
|
||||
include_str!("../../../../skills/agent-desktop-ffi/references/build-and-link.md");
|
||||
const SKILL_FFI_REF_ERRORS: &str =
|
||||
include_str!("../../../../skills/agent-desktop-ffi/references/error-handling.md");
|
||||
const SKILL_FFI_REF_OWNERSHIP: &str =
|
||||
include_str!("../../../../skills/agent-desktop-ffi/references/ownership.md");
|
||||
const SKILL_FFI_REF_THREADING: &str =
|
||||
include_str!("../../../../skills/agent-desktop-ffi/references/threading.md");
|
||||
|
||||
struct SkillRef {
|
||||
rel_path: &'static str,
|
||||
body: &'static str,
|
||||
}
|
||||
|
||||
struct Skill {
|
||||
canonical: &'static str,
|
||||
aliases: &'static [&'static str],
|
||||
summary: &'static str,
|
||||
main: &'static str,
|
||||
refs: &'static [SkillRef],
|
||||
}
|
||||
|
||||
const SKILLS: &[Skill] = &[
|
||||
Skill {
|
||||
canonical: "agent-desktop",
|
||||
aliases: &["desktop", "agent-desktop"],
|
||||
summary: "Primary guide. Snapshot/ref loop, JSON envelope, 53 commands across observation, interaction, keyboard/mouse, app lifecycle, notifications, clipboard, wait.",
|
||||
main: SKILL_DESKTOP_MAIN,
|
||||
refs: &[
|
||||
SkillRef { rel_path: "references/commands-observation.md", body: SKILL_DESKTOP_REF_OBSERVATION },
|
||||
SkillRef { rel_path: "references/commands-interaction.md", body: SKILL_DESKTOP_REF_INTERACTION },
|
||||
SkillRef { rel_path: "references/commands-system.md", body: SKILL_DESKTOP_REF_SYSTEM },
|
||||
SkillRef { rel_path: "references/workflows.md", body: SKILL_DESKTOP_REF_WORKFLOWS },
|
||||
SkillRef { rel_path: "references/macos.md", body: SKILL_DESKTOP_REF_MACOS },
|
||||
],
|
||||
},
|
||||
Skill {
|
||||
canonical: "agent-desktop-ffi",
|
||||
aliases: &["ffi", "agent-desktop-ffi"],
|
||||
summary: "Embedding agent-desktop in another process via the C ABI. Build/link, error propagation, handle ownership, threading rules.",
|
||||
main: SKILL_FFI_MAIN,
|
||||
refs: &[
|
||||
SkillRef { rel_path: "references/build-and-link.md", body: SKILL_FFI_REF_BUILD },
|
||||
SkillRef { rel_path: "references/error-handling.md", body: SKILL_FFI_REF_ERRORS },
|
||||
SkillRef { rel_path: "references/ownership.md", body: SKILL_FFI_REF_OWNERSHIP },
|
||||
SkillRef { rel_path: "references/threading.md", body: SKILL_FFI_REF_THREADING },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
pub struct GetArgs {
|
||||
pub name: String,
|
||||
pub full: bool,
|
||||
pub reference: Option<String>,
|
||||
}
|
||||
|
||||
pub fn list() -> Result<Value, AppError> {
|
||||
let entries: Vec<Value> = SKILLS
|
||||
.iter()
|
||||
.map(|s| {
|
||||
json!({
|
||||
"name": s.canonical,
|
||||
"aliases": s.aliases,
|
||||
"summary": s.summary,
|
||||
"references": s.refs.iter().map(|r| r.rel_path).collect::<Vec<_>>(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "skills": entries }))
|
||||
}
|
||||
|
||||
pub fn get(args: GetArgs) -> Result<Value, AppError> {
|
||||
let skill = find_skill(&args.name)?;
|
||||
|
||||
if let Some(rel) = args.reference {
|
||||
let r = skill
|
||||
.refs
|
||||
.iter()
|
||||
.find(|r| matches_ref(r.rel_path, &rel))
|
||||
.ok_or_else(|| {
|
||||
let available: Vec<&str> = skill.refs.iter().map(|r| r.rel_path).collect();
|
||||
AppError::invalid_input(format!(
|
||||
"Unknown reference '{rel}' for skill '{}'. Available: {}",
|
||||
skill.canonical,
|
||||
available.join(", ")
|
||||
))
|
||||
})?;
|
||||
return Ok(json!({
|
||||
"skill": skill.canonical,
|
||||
"reference": r.rel_path,
|
||||
"content": r.body,
|
||||
}));
|
||||
}
|
||||
|
||||
let content = if args.full {
|
||||
render_full(skill)
|
||||
} else {
|
||||
skill.main.to_string()
|
||||
};
|
||||
|
||||
Ok(json!({
|
||||
"skill": skill.canonical,
|
||||
"full": args.full,
|
||||
"content": content,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn path() -> Result<Value, AppError> {
|
||||
Ok(json!({
|
||||
"location": "embedded",
|
||||
"note": "Skills are compiled into this binary and are always version-matched. Run `agent-desktop skills get <name>` to print a skill, or redirect into a file to extract a copy.",
|
||||
"available": SKILLS.iter().map(|s| s.canonical).collect::<Vec<_>>(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn find_skill(name: &str) -> Result<&'static Skill, AppError> {
|
||||
let needle = name.trim();
|
||||
SKILLS
|
||||
.iter()
|
||||
.find(|s| s.aliases.iter().any(|a| a.eq_ignore_ascii_case(needle)))
|
||||
.ok_or_else(|| {
|
||||
let known: Vec<&str> = SKILLS
|
||||
.iter()
|
||||
.flat_map(|s| s.aliases.iter().copied())
|
||||
.collect();
|
||||
AppError::invalid_input(format!(
|
||||
"Unknown skill '{name}'. Known: {}",
|
||||
known.join(", ")
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn matches_ref(rel_path: &str, query: &str) -> bool {
|
||||
if rel_path.eq_ignore_ascii_case(query) {
|
||||
return true;
|
||||
}
|
||||
let stem = rel_path
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.and_then(|f| f.strip_suffix(".md").or(Some(f)))
|
||||
.unwrap_or(rel_path);
|
||||
stem.eq_ignore_ascii_case(query)
|
||||
}
|
||||
|
||||
fn render_full(skill: &Skill) -> String {
|
||||
let mut out = String::with_capacity(
|
||||
skill.main.len() + skill.refs.iter().map(|r| r.body.len() + 64).sum::<usize>(),
|
||||
);
|
||||
out.push_str(skill.main);
|
||||
for r in skill.refs {
|
||||
if !out.ends_with('\n') {
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str("\n--- ");
|
||||
out.push_str(r.rel_path);
|
||||
out.push_str(" ---\n\n");
|
||||
out.push_str(r.body);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn list_returns_known_skills() {
|
||||
let v = list().expect("list");
|
||||
let arr = v["skills"].as_array().expect("array");
|
||||
assert!(arr.iter().any(|s| s["name"] == "agent-desktop"));
|
||||
assert!(arr.iter().any(|s| s["name"] == "agent-desktop-ffi"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_resolves_alias() {
|
||||
let v = get(GetArgs {
|
||||
name: "desktop".into(),
|
||||
full: false,
|
||||
reference: None,
|
||||
})
|
||||
.expect("get");
|
||||
assert_eq!(v["skill"], "agent-desktop");
|
||||
assert!(v["content"].as_str().unwrap().contains("agent-desktop"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_full_inlines_references() {
|
||||
let v = get(GetArgs {
|
||||
name: "desktop".into(),
|
||||
full: true,
|
||||
reference: None,
|
||||
})
|
||||
.expect("get full");
|
||||
let content = v["content"].as_str().expect("string");
|
||||
assert!(content.contains("--- references/workflows.md ---"));
|
||||
assert!(content.contains("--- references/macos.md ---"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_specific_reference() {
|
||||
let v = get(GetArgs {
|
||||
name: "desktop".into(),
|
||||
full: false,
|
||||
reference: Some("workflows".into()),
|
||||
})
|
||||
.expect("get ref");
|
||||
assert_eq!(v["reference"], "references/workflows.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_skill_errors() {
|
||||
let err = get(GetArgs {
|
||||
name: "nope".into(),
|
||||
full: false,
|
||||
reference: None,
|
||||
})
|
||||
.expect_err("should error");
|
||||
assert!(format!("{err}").contains("Unknown skill"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_lists_canonical_names() {
|
||||
let v = path().expect("path");
|
||||
assert_eq!(v["location"], "embedded");
|
||||
let avail = v["available"].as_array().expect("arr");
|
||||
assert!(avail.iter().any(|s| s == "agent-desktop"));
|
||||
}
|
||||
}
|
||||
|
|
@ -9,8 +9,9 @@ description: >
|
|||
Use when an AI agent needs to observe, interact with, or automate desktop applications
|
||||
(click buttons, fill forms, navigate menus, read UI state, toggle checkboxes, scroll,
|
||||
drag, type text, take screenshots, manage windows, use clipboard, manage notifications).
|
||||
Covers 53 commands across observation, interaction, keyboard/mouse, app lifecycle,
|
||||
notifications (macOS), clipboard, and wait.
|
||||
Covers 54 commands across observation, interaction, keyboard/mouse, app lifecycle,
|
||||
notifications (macOS), clipboard, wait, and a `skills` command that prints these
|
||||
bundled docs straight from the binary.
|
||||
Triggers on: "click button", "fill form", "open app", "read UI", "automate desktop",
|
||||
"accessibility tree", "snapshot app", "type into field", "navigate menu", "toggle checkbox",
|
||||
"take screenshot", "desktop automation", "agent-desktop", or any desktop GUI interaction task.
|
||||
|
|
@ -110,7 +111,7 @@ Exit codes: `0` success, `1` structured error, `2` argument error.
|
|||
| `TIMEOUT` | Wait condition not met | Increase --timeout |
|
||||
| `INVALID_ARGS` | Bad arguments | Check command syntax |
|
||||
|
||||
## Command Quick Reference (53 commands)
|
||||
## Command Quick Reference (54 commands)
|
||||
|
||||
### Observation
|
||||
```
|
||||
|
|
@ -211,6 +212,8 @@ agent-desktop permissions # Check permission
|
|||
agent-desktop permissions --request # Trigger permission dialog
|
||||
agent-desktop version --json # Version info
|
||||
agent-desktop batch '[...]' --stop-on-error # Batch commands
|
||||
agent-desktop skills # List bundled skill docs
|
||||
agent-desktop skills get desktop --full # Load this skill + all references
|
||||
```
|
||||
|
||||
## Key Principles for Agents
|
||||
|
|
|
|||
|
|
@ -259,3 +259,35 @@ agent-desktop version
|
|||
agent-desktop version --json
|
||||
```
|
||||
Returns version string. Use `--json` for `{ "version": "0.1.3", "platform": "macos", "arch": "aarch64" }`.
|
||||
|
||||
## Skills (bundled docs)
|
||||
|
||||
Skill markdown ships compiled into the binary. Use these to load up-to-date guidance without hitting the network.
|
||||
|
||||
### skills (or `skills list`)
|
||||
```bash
|
||||
agent-desktop skills
|
||||
```
|
||||
Lists every bundled skill with aliases, summaries, and reference filenames.
|
||||
|
||||
### skills get
|
||||
```bash
|
||||
agent-desktop skills get desktop # Primary guide (this skill's main file)
|
||||
agent-desktop skills get desktop --full # Main + every reference inlined with `--- references/<file> ---` separators
|
||||
agent-desktop skills get desktop workflows # Single reference; bare stem or `references/workflows.md` both work
|
||||
agent-desktop skills get ffi # Specialized: embedding via the C ABI
|
||||
```
|
||||
|
||||
| Arg / Flag | Description |
|
||||
|------------|-------------|
|
||||
| `<name>` | Skill name or alias. `desktop` ↔ `agent-desktop`, `ffi` ↔ `agent-desktop-ffi`. |
|
||||
| `<reference>` (positional) | Reference filename (stem or full `references/<file>.md`). Omit for the main guide. |
|
||||
| `--full` | Inline every reference after the main file. Ignored when a specific reference is requested. |
|
||||
|
||||
JSON envelope contains the markdown under `data.content`. Pipe to `jq -r .data.content` (or extract with `python3 -c`) to print just the markdown.
|
||||
|
||||
### skills path
|
||||
```bash
|
||||
agent-desktop skills path
|
||||
```
|
||||
Reports `{ "location": "embedded", ... }` — skills are baked into this binary via `include_str!`. To extract a copy on disk, redirect `skills get <name>` output into a file.
|
||||
|
|
|
|||
112
src/cli.rs
112
src/cli.rs
|
|
@ -2,115 +2,18 @@ use clap::{Parser, Subcommand};
|
|||
|
||||
pub use crate::cli_args::*;
|
||||
pub use crate::cli_args_notifications::*;
|
||||
pub use crate::cli_args_skills::*;
|
||||
|
||||
const BEFORE_HELP: &str = include_str!("help_before.txt");
|
||||
const AFTER_HELP: &str = include_str!("help_after.txt");
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "agent-desktop",
|
||||
about = "Desktop automation CLI for AI agents",
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
OBSERVATION
|
||||
snapshot Accessibility tree as JSON with @ref IDs
|
||||
screenshot PNG screenshot of an application window
|
||||
find Search elements by role, name, value, or text
|
||||
get <ref> <property> Read element property: text, value, title, bounds, role, states
|
||||
is <ref> <property> Check state: visible, enabled, checked, focused, expanded
|
||||
list-surfaces Available surfaces for an app
|
||||
|
||||
INTERACTION
|
||||
click <ref> Click element (kAXPress)
|
||||
double-click <ref> Double-click element
|
||||
triple-click <ref> Triple-click element (select line/paragraph)
|
||||
right-click <ref> Right-click and open context menu
|
||||
type <ref> <text> Focus element and type text
|
||||
set-value <ref> <value> Set value attribute directly
|
||||
clear <ref> Clear element value to empty string
|
||||
focus <ref> Set keyboard focus
|
||||
select <ref> <value> Select option in list or dropdown
|
||||
toggle <ref> Toggle checkbox or switch
|
||||
check <ref> Set checkbox/switch to checked (idempotent)
|
||||
uncheck <ref> Set checkbox/switch to unchecked (idempotent)
|
||||
expand <ref> Expand disclosure triangle or tree item
|
||||
collapse <ref> Collapse disclosure triangle or tree item
|
||||
scroll <ref> Scroll element (--direction, --amount)
|
||||
scroll-to <ref> Scroll element into visible area
|
||||
|
||||
KEYBOARD
|
||||
press <combo> Key combo: return, escape, cmd+c, shift+tab ...
|
||||
key-down <combo> Hold a key or modifier down
|
||||
key-up <combo> Release a held key or modifier
|
||||
|
||||
MOUSE
|
||||
hover <ref|--xy> Move cursor to element or coordinates
|
||||
drag Drag from one element/point to another
|
||||
mouse-move --xy x,y Move cursor to absolute coordinates
|
||||
mouse-click --xy x,y Click at coordinates (--button, --count)
|
||||
mouse-down --xy x,y Press mouse button at coordinates
|
||||
mouse-up --xy x,y Release mouse button at coordinates
|
||||
|
||||
APP & WINDOW
|
||||
launch <app> Launch app and wait until window is visible
|
||||
close-app <app> Quit app gracefully (--force to kill)
|
||||
list-windows All visible windows (--app to filter)
|
||||
list-apps All running GUI applications
|
||||
focus-window Bring window to front
|
||||
resize-window Resize window (--width, --height)
|
||||
move-window Move window (--x, --y)
|
||||
minimize Minimize window
|
||||
maximize Maximize/zoom window
|
||||
restore Restore minimized/maximized window
|
||||
|
||||
NOTIFICATIONS
|
||||
list-notifications List notifications from Notification Center
|
||||
dismiss-notification <n> Dismiss notification by index
|
||||
dismiss-all-notifications Dismiss all notifications
|
||||
notification-action <n> <action> Click action button on notification
|
||||
|
||||
CLIPBOARD
|
||||
clipboard-get Read plain-text clipboard
|
||||
clipboard-set <text> Write text to clipboard
|
||||
clipboard-clear Clear the clipboard
|
||||
|
||||
WAIT
|
||||
wait [ms] Pause for N milliseconds
|
||||
wait --element <ref> Block until element appears (--timeout ms)
|
||||
wait --window <title> Block until window appears
|
||||
wait --text <text> Block until text appears in app
|
||||
wait --notification Block until a new notification arrives
|
||||
|
||||
SYSTEM
|
||||
status Adapter health, platform, and permission state
|
||||
permissions Check accessibility permission (--request to prompt)
|
||||
version Version string (--json for machine-readable)
|
||||
|
||||
BATCH
|
||||
batch <json> Run commands from a JSON array (--stop-on-error)
|
||||
|
||||
REF IDs
|
||||
snapshot assigns @e1, @e2, ... to interactive elements in depth-first order.
|
||||
Use a ref wherever <ref> appears. Refs are snapshot-scoped; run snapshot
|
||||
again after UI changes.
|
||||
|
||||
KEY COMBOS
|
||||
Single keys: return, escape, tab, space, delete, up, down, left, right
|
||||
Function keys: f1 - f12
|
||||
With modifiers: cmd+c, cmd+v, cmd+z, cmd+shift+z, ctrl+a, shift+tab
|
||||
Modifiers: cmd, ctrl, alt, shift
|
||||
|
||||
EXAMPLES
|
||||
agent-desktop snapshot --app \"System Settings\" -i
|
||||
agent-desktop find --role button --name \"OK\"
|
||||
agent-desktop click @e5
|
||||
agent-desktop check @e3
|
||||
agent-desktop type @e2 \"hello@example.com\"
|
||||
agent-desktop press cmd+z
|
||||
agent-desktop drag --from @e1 --to @e5
|
||||
agent-desktop hover @e5
|
||||
agent-desktop minimize --app TextEdit
|
||||
agent-desktop resize-window --app TextEdit --width 800 --height 600
|
||||
agent-desktop mouse-click --xy 500,300
|
||||
agent-desktop wait --text \"Loading complete\" --app Safari --timeout 5000
|
||||
agent-desktop batch '[{\"command\":\"click\",\"args\":{\"ref_id\":\"@e1\"}}]'"
|
||||
before_help = BEFORE_HELP,
|
||||
after_help = AFTER_HELP,
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[arg(
|
||||
|
|
@ -233,6 +136,8 @@ pub enum Commands {
|
|||
Version(VersionArgs),
|
||||
#[command(about = "Execute multiple commands from a JSON array (--stop-on-error)")]
|
||||
Batch(BatchArgs),
|
||||
#[command(about = "Bundled skill docs for AI agents (list, get, path)")]
|
||||
Skills(SkillsArgs),
|
||||
}
|
||||
|
||||
impl Commands {
|
||||
|
|
@ -291,6 +196,7 @@ impl Commands {
|
|||
Self::Permissions(_) => "permissions",
|
||||
Self::Version(_) => "version",
|
||||
Self::Batch(_) => "batch",
|
||||
Self::Skills(_) => "skills",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
40
src/cli_args_skills.rs
Normal file
40
src/cli_args_skills.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use clap::{Args, Subcommand};
|
||||
|
||||
#[derive(Args, Debug)]
|
||||
#[command(after_help = "\
|
||||
Skills travel inside the binary so they always match this exact
|
||||
agent-desktop version. Output is raw markdown on stdout — parse it
|
||||
directly, or redirect into a file for storage.
|
||||
|
||||
Examples:
|
||||
agent-desktop skills # List skills
|
||||
agent-desktop skills get desktop # Primary guide
|
||||
agent-desktop skills get desktop --full # Plus every reference
|
||||
agent-desktop skills get desktop workflows # Single reference
|
||||
agent-desktop skills path # Where skills live")]
|
||||
pub struct SkillsArgs {
|
||||
#[command(subcommand)]
|
||||
pub action: Option<SkillsAction>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum SkillsAction {
|
||||
#[command(about = "List bundled skills with summaries (default)")]
|
||||
List,
|
||||
#[command(about = "Print a skill's markdown to stdout")]
|
||||
Get(SkillsGetArgs),
|
||||
#[command(about = "Print where bundled skills live")]
|
||||
Path,
|
||||
}
|
||||
|
||||
#[derive(Args, Debug)]
|
||||
pub struct SkillsGetArgs {
|
||||
#[arg(help = "Skill name or alias (desktop, ffi, ...)")]
|
||||
pub name: String,
|
||||
#[arg(
|
||||
help = "Reference filename (e.g. workflows or references/workflows.md). Omit for the main guide."
|
||||
)]
|
||||
pub reference: Option<String>,
|
||||
#[arg(long, help = "Append every reference file to the output")]
|
||||
pub full: bool,
|
||||
}
|
||||
|
|
@ -7,13 +7,13 @@ use agent_desktop_core::{
|
|||
is_check, key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize,
|
||||
minimize, mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press,
|
||||
resize_window, restore, right_click, screenshot, scroll, scroll_to, select, set_value,
|
||||
snapshot, status, toggle, triple_click, type_text, uncheck, version, wait,
|
||||
skills, snapshot, status, toggle, triple_click, type_text, uncheck, version, wait,
|
||||
},
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::cli::Commands;
|
||||
use crate::cli::{Commands, SkillsAction};
|
||||
|
||||
pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
tracing::debug!("dispatch: {}", cmd.name());
|
||||
|
|
@ -301,6 +301,16 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
|
||||
Commands::Version(a) => version::execute(version::VersionArgs { json: a.json }),
|
||||
|
||||
Commands::Skills(a) => match a.action.unwrap_or(SkillsAction::List) {
|
||||
SkillsAction::List => skills::list(),
|
||||
SkillsAction::Path => skills::path(),
|
||||
SkillsAction::Get(g) => skills::get(skills::GetArgs {
|
||||
name: g.name,
|
||||
full: g.full,
|
||||
reference: g.reference,
|
||||
}),
|
||||
},
|
||||
|
||||
Commands::Batch(a) => {
|
||||
let commands = batch::parse_commands(&a.commands_json)?;
|
||||
let mut results = Vec::new();
|
||||
|
|
|
|||
104
src/help_after.txt
Normal file
104
src/help_after.txt
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
OBSERVATION
|
||||
snapshot Accessibility tree as JSON with @ref IDs
|
||||
screenshot PNG screenshot of an application window
|
||||
find Search elements by role, name, value, or text
|
||||
get <ref> <property> Read element property: text, value, title, bounds, role, states
|
||||
is <ref> <property> Check state: visible, enabled, checked, focused, expanded
|
||||
list-surfaces Available surfaces for an app
|
||||
|
||||
INTERACTION
|
||||
click <ref> Click element (kAXPress)
|
||||
double-click <ref> Double-click element
|
||||
triple-click <ref> Triple-click element (select line/paragraph)
|
||||
right-click <ref> Right-click and open context menu
|
||||
type <ref> <text> Focus element and type text
|
||||
set-value <ref> <value> Set value attribute directly
|
||||
clear <ref> Clear element value to empty string
|
||||
focus <ref> Set keyboard focus
|
||||
select <ref> <value> Select option in list or dropdown
|
||||
toggle <ref> Toggle checkbox or switch
|
||||
check <ref> Set checkbox/switch to checked (idempotent)
|
||||
uncheck <ref> Set checkbox/switch to unchecked (idempotent)
|
||||
expand <ref> Expand disclosure triangle or tree item
|
||||
collapse <ref> Collapse disclosure triangle or tree item
|
||||
scroll <ref> Scroll element (--direction, --amount)
|
||||
scroll-to <ref> Scroll element into visible area
|
||||
|
||||
KEYBOARD
|
||||
press <combo> Key combo: return, escape, cmd+c, shift+tab ...
|
||||
key-down <combo> Hold a key or modifier down
|
||||
key-up <combo> Release a held key or modifier
|
||||
|
||||
MOUSE
|
||||
hover <ref|--xy> Move cursor to element or coordinates
|
||||
drag Drag from one element/point to another
|
||||
mouse-move --xy x,y Move cursor to absolute coordinates
|
||||
mouse-click --xy x,y Click at coordinates (--button, --count)
|
||||
mouse-down --xy x,y Press mouse button at coordinates
|
||||
mouse-up --xy x,y Release mouse button at coordinates
|
||||
|
||||
APP & WINDOW
|
||||
launch <app> Launch app and wait until window is visible
|
||||
close-app <app> Quit app gracefully (--force to kill)
|
||||
list-windows All visible windows (--app to filter)
|
||||
list-apps All running GUI applications
|
||||
focus-window Bring window to front
|
||||
resize-window Resize window (--width, --height)
|
||||
move-window Move window (--x, --y)
|
||||
minimize Minimize window
|
||||
maximize Maximize/zoom window
|
||||
restore Restore minimized/maximized window
|
||||
|
||||
NOTIFICATIONS
|
||||
list-notifications List notifications from Notification Center
|
||||
dismiss-notification <n> Dismiss notification by index
|
||||
dismiss-all-notifications Dismiss all notifications
|
||||
notification-action <n> <action> Click action button on notification
|
||||
|
||||
CLIPBOARD
|
||||
clipboard-get Read plain-text clipboard
|
||||
clipboard-set <text> Write text to clipboard
|
||||
clipboard-clear Clear the clipboard
|
||||
|
||||
WAIT
|
||||
wait [ms] Pause for N milliseconds
|
||||
wait --element <ref> Block until element appears (--timeout ms)
|
||||
wait --window <title> Block until window appears
|
||||
wait --text <text> Block until text appears in app
|
||||
wait --notification Block until a new notification arrives
|
||||
|
||||
SYSTEM
|
||||
status Adapter health, platform, and permission state
|
||||
permissions Check accessibility permission (--request to prompt)
|
||||
version Version string (--json for machine-readable)
|
||||
skills Bundled skill docs for AI agents (list, get, path)
|
||||
|
||||
BATCH
|
||||
batch <json> Run commands from a JSON array (--stop-on-error)
|
||||
|
||||
REF IDs
|
||||
snapshot assigns @e1, @e2, ... to interactive elements in depth-first order.
|
||||
Use a ref wherever <ref> appears. Refs are snapshot-scoped; run snapshot
|
||||
again after UI changes.
|
||||
|
||||
KEY COMBOS
|
||||
Single keys: return, escape, tab, space, delete, up, down, left, right
|
||||
Function keys: f1 - f12
|
||||
With modifiers: cmd+c, cmd+v, cmd+z, cmd+shift+z, ctrl+a, shift+tab
|
||||
Modifiers: cmd, ctrl, alt, shift
|
||||
|
||||
EXAMPLES
|
||||
agent-desktop skills get desktop --full Load the primary skill (start here)
|
||||
agent-desktop snapshot --app "System Settings" -i
|
||||
agent-desktop find --role button --name "OK"
|
||||
agent-desktop click @e5
|
||||
agent-desktop check @e3
|
||||
agent-desktop type @e2 "hello@example.com"
|
||||
agent-desktop press cmd+z
|
||||
agent-desktop drag --from @e1 --to @e5
|
||||
agent-desktop hover @e5
|
||||
agent-desktop minimize --app TextEdit
|
||||
agent-desktop resize-window --app TextEdit --width 800 --height 600
|
||||
agent-desktop mouse-click --xy 500,300
|
||||
agent-desktop wait --text "Loading complete" --app Safari --timeout 5000
|
||||
agent-desktop batch '[{"command":"click","args":{"ref_id":"@e1"}}]'
|
||||
15
src/help_before.txt
Normal file
15
src/help_before.txt
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
For AI agents — read this first:
|
||||
agent-desktop skills get desktop --full
|
||||
|
||||
Bundled skill docs travel with the binary, so they always match this
|
||||
exact version. They cover the snapshot/ref loop, JSON envelope contract,
|
||||
error codes with recovery hints, and copy-paste patterns for forms,
|
||||
menus, dialogs, and async waits. Specialized skills go deeper on macOS
|
||||
internals (TCC, AX API, surfaces, Notification Center) and on embedding
|
||||
agent-desktop through its C ABI.
|
||||
|
||||
skills List bundled skills with summaries
|
||||
skills get desktop Primary guide — overview + the observe-act loop
|
||||
skills get desktop --full Same, plus every reference file inlined
|
||||
skills get <name> Open a focused skill (e.g. macos, ffi)
|
||||
skills path Where skills live (compiled into this binary)
|
||||
23
src/main.rs
23
src/main.rs
|
|
@ -3,6 +3,7 @@ mod batch_dispatch_ext;
|
|||
mod cli;
|
||||
mod cli_args;
|
||||
mod cli_args_notifications;
|
||||
mod cli_args_skills;
|
||||
mod dispatch;
|
||||
mod dispatch_notifications;
|
||||
|
||||
|
|
@ -46,23 +47,37 @@ fn main() {
|
|||
|
||||
let cmd_name = cmd.name();
|
||||
|
||||
match &cmd {
|
||||
match cmd {
|
||||
Commands::Version(a) => {
|
||||
let result = agent_desktop_core::commands::version::execute(
|
||||
agent_desktop_core::commands::version::VersionArgs { json: a.json },
|
||||
);
|
||||
finish(cmd_name, result);
|
||||
return;
|
||||
}
|
||||
Commands::Status => {
|
||||
let adapter = build_adapter();
|
||||
let result = agent_desktop_core::commands::status::execute(&adapter);
|
||||
finish(cmd_name, result);
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
Commands::Skills(a) => {
|
||||
let result = match a.action.unwrap_or(cli::SkillsAction::List) {
|
||||
cli::SkillsAction::List => agent_desktop_core::commands::skills::list(),
|
||||
cli::SkillsAction::Path => agent_desktop_core::commands::skills::path(),
|
||||
cli::SkillsAction::Get(g) => agent_desktop_core::commands::skills::get(
|
||||
agent_desktop_core::commands::skills::GetArgs {
|
||||
name: g.name,
|
||||
full: g.full,
|
||||
reference: g.reference,
|
||||
},
|
||||
),
|
||||
};
|
||||
finish(cmd_name, result);
|
||||
}
|
||||
cmd => run_with_adapter(cmd, cmd_name),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_with_adapter(cmd: Commands, cmd_name: &str) {
|
||||
let adapter = build_adapter();
|
||||
|
||||
if let agent_desktop_core::adapter::PermissionStatus::Denied { suggestion } =
|
||||
|
|
|
|||
Loading…
Reference in a new issue