mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-26 17:12:15 +00:00
Merge pull request #1 from lahfir/feat/ax-first-bug-fixes-new-commands
feat: add 19 new commands, AX-first rewrites, LOC compliance
This commit is contained in:
commit
d3f7e03c67
78 changed files with 6409 additions and 776 deletions
|
|
@ -1,19 +1,29 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[non_exhaustive]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum Action {
|
||||
Click,
|
||||
DoubleClick,
|
||||
RightClick,
|
||||
TripleClick,
|
||||
SetValue(String),
|
||||
SetFocus,
|
||||
Expand,
|
||||
Collapse,
|
||||
Select(String),
|
||||
Toggle,
|
||||
Check,
|
||||
Uncheck,
|
||||
Scroll(Direction, u32),
|
||||
ScrollTo,
|
||||
PressKey(KeyCombo),
|
||||
KeyDown(KeyCombo),
|
||||
KeyUp(KeyCombo),
|
||||
TypeText(String),
|
||||
Clear,
|
||||
Hover,
|
||||
Drag(DragParams),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
|
@ -24,6 +34,51 @@ pub enum Direction {
|
|||
Right,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Point {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum MouseButton {
|
||||
Left,
|
||||
Right,
|
||||
Middle,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DragParams {
|
||||
pub from: Point,
|
||||
pub to: Point,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub duration_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum MouseEventKind {
|
||||
Move,
|
||||
Down,
|
||||
Up,
|
||||
Click { count: u32 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MouseEvent {
|
||||
pub kind: MouseEventKind,
|
||||
pub point: Point,
|
||||
pub button: MouseButton,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum WindowOp {
|
||||
Resize { width: f64, height: f64 },
|
||||
Move { x: f64, y: f64 },
|
||||
Minimize,
|
||||
Maximize,
|
||||
Restore,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KeyCombo {
|
||||
pub key: String,
|
||||
|
|
@ -58,7 +113,11 @@ pub struct ElementState {
|
|||
|
||||
impl ActionResult {
|
||||
pub fn new(action: impl Into<String>) -> Self {
|
||||
Self { action: action.into(), ref_id: None, post_state: None }
|
||||
Self {
|
||||
action: action.into(),
|
||||
ref_id: None,
|
||||
post_state: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_ref(mut self, ref_id: impl Into<String>) -> Self {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use crate::{
|
||||
action::{Action, ActionResult},
|
||||
action::{Action, ActionResult, DragParams, MouseEvent, WindowOp},
|
||||
error::AdapterError,
|
||||
node::{AccessibilityNode, AppInfo, WindowInfo},
|
||||
node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo},
|
||||
refs::RefEntry,
|
||||
};
|
||||
use std::marker::PhantomData;
|
||||
|
|
@ -11,11 +11,23 @@ pub struct WindowFilter {
|
|||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub enum SnapshotSurface {
|
||||
#[default]
|
||||
Window,
|
||||
Focused,
|
||||
Menu,
|
||||
Sheet,
|
||||
Popover,
|
||||
Alert,
|
||||
}
|
||||
|
||||
pub struct TreeOptions {
|
||||
pub max_depth: u8,
|
||||
pub include_bounds: bool,
|
||||
pub interactive_only: bool,
|
||||
pub compact: bool,
|
||||
pub surface: SnapshotSurface,
|
||||
}
|
||||
|
||||
impl Default for TreeOptions {
|
||||
|
|
@ -25,13 +37,15 @@ impl Default for TreeOptions {
|
|||
include_bounds: false,
|
||||
interactive_only: false,
|
||||
compact: false,
|
||||
surface: SnapshotSurface::Window,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub enum ScreenshotTarget {
|
||||
Screen(usize),
|
||||
Window(String),
|
||||
/// Capture the frontmost window owned by this process ID.
|
||||
Window(i32),
|
||||
FullScreen,
|
||||
}
|
||||
|
||||
|
|
@ -47,11 +61,17 @@ pub struct NativeHandle {
|
|||
|
||||
impl NativeHandle {
|
||||
pub fn from_ptr(ptr: *const std::ffi::c_void) -> Self {
|
||||
Self { ptr, _not_send_sync: PhantomData }
|
||||
Self {
|
||||
ptr,
|
||||
_not_send_sync: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn null() -> Self {
|
||||
Self { ptr: std::ptr::null(), _not_send_sync: PhantomData }
|
||||
Self {
|
||||
ptr: std::ptr::null(),
|
||||
_not_send_sync: PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -129,7 +149,7 @@ pub trait PlatformAdapter: Send + Sync {
|
|||
Err(AdapterError::not_supported("focus_window"))
|
||||
}
|
||||
|
||||
fn launch_app(&self, _id: &str, _wait: bool) -> Result<WindowInfo, AdapterError> {
|
||||
fn launch_app(&self, _id: &str, _timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
|
||||
Err(AdapterError::not_supported("launch_app"))
|
||||
}
|
||||
|
||||
|
|
@ -152,4 +172,44 @@ pub trait PlatformAdapter: Send + Sync {
|
|||
fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError> {
|
||||
Err(AdapterError::not_supported("focused_window"))
|
||||
}
|
||||
|
||||
fn get_live_value(&self, _handle: &NativeHandle) -> Result<Option<String>, AdapterError> {
|
||||
Err(AdapterError::not_supported("get_live_value"))
|
||||
}
|
||||
|
||||
fn press_key_for_app(
|
||||
&self,
|
||||
_app_name: &str,
|
||||
_combo: &crate::action::KeyCombo,
|
||||
) -> Result<crate::action::ActionResult, AdapterError> {
|
||||
Err(AdapterError::not_supported("press_key_for_app"))
|
||||
}
|
||||
|
||||
fn wait_for_menu(&self, _pid: i32, _open: bool, _timeout_ms: u64) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("wait_for_menu"))
|
||||
}
|
||||
|
||||
fn list_surfaces(&self, _pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError> {
|
||||
Err(AdapterError::not_supported("list_surfaces"))
|
||||
}
|
||||
|
||||
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
|
||||
Err(AdapterError::not_supported("get_element_bounds"))
|
||||
}
|
||||
|
||||
fn window_op(&self, _win: &WindowInfo, _op: WindowOp) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("window_op"))
|
||||
}
|
||||
|
||||
fn mouse_event(&self, _event: MouseEvent) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("mouse_event"))
|
||||
}
|
||||
|
||||
fn drag(&self, _params: DragParams) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("drag"))
|
||||
}
|
||||
|
||||
fn clear_clipboard(&self) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("clear_clipboard"))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use crate::error::AppError;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct BatchArgs {
|
||||
pub commands_json: String,
|
||||
|
|
@ -18,11 +18,3 @@ pub fn parse_commands(json_str: &str) -> Result<Vec<BatchCommand>, AppError> {
|
|||
serde_json::from_str(json_str)
|
||||
.map_err(|e| AppError::invalid_input(format!("Invalid batch JSON: {e}")))
|
||||
}
|
||||
|
||||
pub fn execute(args: BatchArgs, _adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let commands = parse_commands(&args.commands_json)?;
|
||||
Ok(json!({
|
||||
"note": "Batch execution delegated to dispatch layer",
|
||||
"count": commands.len()
|
||||
}))
|
||||
}
|
||||
|
|
|
|||
14
crates/core/src/commands/check.rs
Normal file
14
crates/core/src/commands/check.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct CheckArgs {
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: CheckArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let result = adapter.execute_action(&handle, Action::Check)?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
}
|
||||
14
crates/core/src/commands/clear.rs
Normal file
14
crates/core/src/commands/clear.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct ClearArgs {
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: ClearArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let result = adapter.execute_action(&handle, Action::Clear)?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
}
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub fn execute_get(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let text = adapter.get_clipboard()?;
|
||||
Ok(json!({ "text": text }))
|
||||
}
|
||||
|
||||
pub fn execute_set(text: String, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.set_clipboard(&text)?;
|
||||
Ok(json!({ "ok": true }))
|
||||
}
|
||||
7
crates/core/src/commands/clipboard_clear.rs
Normal file
7
crates/core/src/commands/clipboard_clear.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub fn execute(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.clear_clipboard()?;
|
||||
Ok(json!({ "cleared": true }))
|
||||
}
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
const PROTECTED_PROCESSES: &[&str] =
|
||||
&["loginwindow", "windowserver", "dock", "launchd", "finder"];
|
||||
const PROTECTED_PROCESSES: &[&str] = &["loginwindow", "windowserver", "dock", "launchd", "finder"];
|
||||
|
||||
pub struct CloseAppArgs {
|
||||
pub app: String,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
55
crates/core/src/commands/drag.rs
Normal file
55
crates/core/src/commands/drag.rs
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
use crate::{
|
||||
action::{DragParams, Point},
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct DragArgs {
|
||||
pub from_ref: Option<String>,
|
||||
pub from_xy: Option<(f64, f64)>,
|
||||
pub to_ref: Option<String>,
|
||||
pub to_xy: Option<(f64, f64)>,
|
||||
pub duration_ms: Option<u64>,
|
||||
}
|
||||
|
||||
pub fn execute(args: DragArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let from = resolve_point(&args.from_ref, args.from_xy, "from", adapter)?;
|
||||
let to = resolve_point(&args.to_ref, args.to_xy, "to", adapter)?;
|
||||
let params = DragParams {
|
||||
from: from.clone(),
|
||||
to: to.clone(),
|
||||
duration_ms: args.duration_ms,
|
||||
};
|
||||
adapter.drag(params)?;
|
||||
Ok(json!({
|
||||
"dragged": true,
|
||||
"from": { "x": from.x, "y": from.y },
|
||||
"to": { "x": to.x, "y": to.y }
|
||||
}))
|
||||
}
|
||||
|
||||
fn resolve_point(
|
||||
ref_id: &Option<String>,
|
||||
xy: Option<(f64, f64)>,
|
||||
label: &str,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<Point, AppError> {
|
||||
if let Some(ref_id) = ref_id {
|
||||
let (_entry, handle) = resolve_ref(ref_id, adapter)?;
|
||||
let bounds = adapter
|
||||
.get_element_bounds(&handle)?
|
||||
.ok_or_else(|| AppError::invalid_input(format!("Element {ref_id} has no bounds")))?;
|
||||
Ok(Point {
|
||||
x: bounds.x + bounds.width / 2.0,
|
||||
y: bounds.y + bounds.height / 2.0,
|
||||
})
|
||||
} else if let Some((x, y)) = xy {
|
||||
Ok(Point { x, y })
|
||||
} else {
|
||||
Err(AppError::invalid_input(format!(
|
||||
"Provide --{label} <ref> or --{label}-xy x,y"
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
|
@ -6,15 +6,36 @@ pub struct FindArgs {
|
|||
pub role: Option<String>,
|
||||
pub name: Option<String>,
|
||||
pub value: Option<String>,
|
||||
pub text: Option<String>,
|
||||
pub count: bool,
|
||||
pub first: bool,
|
||||
pub last: bool,
|
||||
pub nth: Option<usize>,
|
||||
}
|
||||
|
||||
pub fn execute(args: FindArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let opts = crate::adapter::TreeOptions::default();
|
||||
let result = snapshot::build(adapter, &opts, args.app.as_deref(), None)?;
|
||||
let result = snapshot::run(adapter, &opts, args.app.as_deref(), None)?;
|
||||
|
||||
let mut matches = Vec::new();
|
||||
search_tree(&result.tree, &args, &mut Vec::new(), &mut matches);
|
||||
|
||||
if args.count {
|
||||
return Ok(json!({ "count": matches.len() }));
|
||||
}
|
||||
|
||||
if args.first {
|
||||
return Ok(json!({ "match": matches.into_iter().next() }));
|
||||
}
|
||||
|
||||
if args.last {
|
||||
return Ok(json!({ "match": matches.into_iter().last() }));
|
||||
}
|
||||
|
||||
if let Some(n) = args.nth {
|
||||
return Ok(json!({ "match": matches.into_iter().nth(n) }));
|
||||
}
|
||||
|
||||
Ok(json!({ "matches": matches }))
|
||||
}
|
||||
|
||||
|
|
@ -26,22 +47,44 @@ fn search_tree(
|
|||
) {
|
||||
let role_match = args.role.as_deref().is_none_or(|r| node.role == r);
|
||||
let name_match = args.name.as_deref().is_none_or(|n| {
|
||||
node.name.as_deref().is_some_and(|name| {
|
||||
name.to_lowercase().contains(&n.to_lowercase())
|
||||
})
|
||||
node.name
|
||||
.as_deref()
|
||||
.is_some_and(|name| name.to_lowercase().contains(&n.to_lowercase()))
|
||||
});
|
||||
let value_match = args.value.as_deref().is_none_or(|v| {
|
||||
node.value.as_deref().is_some_and(|val| {
|
||||
val.to_lowercase().contains(&v.to_lowercase())
|
||||
})
|
||||
node.value
|
||||
.as_deref()
|
||||
.is_some_and(|val| val.to_lowercase().contains(&v.to_lowercase()))
|
||||
});
|
||||
let text_match = args.text.as_deref().is_none_or(|t| {
|
||||
let t_lower = t.to_lowercase();
|
||||
let in_name = node
|
||||
.name
|
||||
.as_deref()
|
||||
.is_some_and(|n| n.to_lowercase().contains(&t_lower));
|
||||
let in_value = node
|
||||
.value
|
||||
.as_deref()
|
||||
.is_some_and(|v| v.to_lowercase().contains(&t_lower));
|
||||
let in_desc = node
|
||||
.description
|
||||
.as_deref()
|
||||
.is_some_and(|d| d.to_lowercase().contains(&t_lower));
|
||||
in_name || in_value || in_desc
|
||||
});
|
||||
|
||||
if role_match && name_match && value_match {
|
||||
if role_match && name_match && value_match && text_match {
|
||||
let interactive = node.ref_id.is_some();
|
||||
let display_name = node
|
||||
.name
|
||||
.as_deref()
|
||||
.or(node.description.as_deref())
|
||||
.map(String::from)
|
||||
.unwrap_or_else(|| format!("(unnamed {})", node.role));
|
||||
matches.push(json!({
|
||||
"ref": node.ref_id,
|
||||
"role": node.role,
|
||||
"name": node.name,
|
||||
"name": display_name,
|
||||
"value": node.value,
|
||||
"interactive": interactive,
|
||||
"path": path.clone()
|
||||
|
|
|
|||
|
|
@ -11,15 +11,22 @@ pub struct FocusWindowArgs {
|
|||
}
|
||||
|
||||
pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let filter = WindowFilter { focused_only: false, app: args.app.clone() };
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: args.app.clone(),
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
|
||||
let window = if let Some(id) = &args.window_id {
|
||||
windows.into_iter().find(|w| &w.id == id)
|
||||
} else if let Some(title) = &args.title {
|
||||
windows.into_iter().find(|w| w.title.contains(title.as_str()))
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.title.contains(title.as_str()))
|
||||
} else if let Some(app) = &args.app {
|
||||
windows.into_iter().find(|w| w.app.eq_ignore_ascii_case(app))
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.app.eq_ignore_ascii_case(app))
|
||||
} else {
|
||||
return Err(AppError::invalid_input(
|
||||
"Provide --window-id, --app, or --title to identify the window",
|
||||
|
|
|
|||
|
|
@ -16,12 +16,15 @@ pub enum GetProperty {
|
|||
}
|
||||
|
||||
pub fn execute(args: GetArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let (entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
|
||||
let value = match args.property {
|
||||
GetProperty::Role => json!(entry.role),
|
||||
GetProperty::Text | GetProperty::Title => json!(entry.name),
|
||||
GetProperty::Value => json!(entry.value),
|
||||
GetProperty::Title => json!(entry.name),
|
||||
GetProperty::Text | GetProperty::Value => {
|
||||
let live = adapter.get_live_value(&handle).ok().flatten();
|
||||
json!(live.or(entry.value))
|
||||
}
|
||||
GetProperty::Bounds => json!(entry.bounds),
|
||||
GetProperty::States => json!(entry.states),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::{
|
||||
adapter::{NativeHandle, PlatformAdapter},
|
||||
adapter::{NativeHandle, PlatformAdapter, WindowFilter},
|
||||
error::AppError,
|
||||
refs::{RefEntry, RefMap},
|
||||
};
|
||||
|
|
@ -14,7 +14,10 @@ pub fn resolve_ref(
|
|||
) -> Result<(RefEntry, NativeHandle), AppError> {
|
||||
validate_ref_id(ref_id)?;
|
||||
let refmap = RefMap::load().map_err(|_| AppError::stale_ref(ref_id))?;
|
||||
let entry = refmap.get(ref_id).ok_or_else(|| AppError::stale_ref(ref_id))?.clone();
|
||||
let entry = refmap
|
||||
.get(ref_id)
|
||||
.ok_or_else(|| AppError::stale_ref(ref_id))?
|
||||
.clone();
|
||||
let handle = adapter.resolve_element(&entry)?;
|
||||
Ok((entry, handle))
|
||||
}
|
||||
|
|
@ -25,13 +28,33 @@ pub fn validate_ref_id(ref_id: &str) -> Result<(), AppError> {
|
|||
&& ref_id.len() <= 12
|
||||
&& ref_id[2..].chars().all(|c| c.is_ascii_digit());
|
||||
if !valid {
|
||||
return Err(AppError::invalid_input(
|
||||
format!("Invalid ref_id '{ref_id}': must match @e{{N}} where N is a positive integer"),
|
||||
));
|
||||
return Err(AppError::invalid_input(format!(
|
||||
"Invalid ref_id '{ref_id}': must match @e{{N}} where N is a positive integer"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn resolve_app_pid(app: Option<&str>, adapter: &dyn PlatformAdapter) -> Result<i32, AppError> {
|
||||
if let Some(name) = app {
|
||||
let apps = adapter.list_apps()?;
|
||||
apps.into_iter()
|
||||
.find(|a| a.name.eq_ignore_ascii_case(name))
|
||||
.map(|a| a.pid)
|
||||
.ok_or_else(|| AppError::invalid_input(format!("App '{name}' not found")))
|
||||
} else {
|
||||
let filter = WindowFilter {
|
||||
focused_only: true,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
windows
|
||||
.first()
|
||||
.map(|w| w.pid)
|
||||
.ok_or_else(|| AppError::invalid_input("No focused window. Use --app to specify."))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
43
crates/core/src/commands/hover.rs
Normal file
43
crates/core/src/commands/hover.rs
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
use crate::{
|
||||
action::{MouseButton, MouseEvent, MouseEventKind, Point},
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct HoverArgs {
|
||||
pub ref_id: Option<String>,
|
||||
pub xy: Option<(f64, f64)>,
|
||||
pub duration_ms: Option<u64>,
|
||||
}
|
||||
|
||||
pub fn execute(args: HoverArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let point = resolve_hover_point(&args, adapter)?;
|
||||
adapter.mouse_event(MouseEvent {
|
||||
kind: MouseEventKind::Move,
|
||||
point: point.clone(),
|
||||
button: MouseButton::Left,
|
||||
})?;
|
||||
if let Some(ms) = args.duration_ms {
|
||||
std::thread::sleep(std::time::Duration::from_millis(ms));
|
||||
}
|
||||
Ok(json!({ "hovered": true, "x": point.x, "y": point.y }))
|
||||
}
|
||||
|
||||
fn resolve_hover_point(args: &HoverArgs, adapter: &dyn PlatformAdapter) -> Result<Point, AppError> {
|
||||
if let Some(ref_id) = &args.ref_id {
|
||||
let (_entry, handle) = resolve_ref(ref_id, adapter)?;
|
||||
let bounds = adapter
|
||||
.get_element_bounds(&handle)?
|
||||
.ok_or_else(|| AppError::invalid_input(format!("Element {ref_id} has no bounds")))?;
|
||||
Ok(Point {
|
||||
x: bounds.x + bounds.width / 2.0,
|
||||
y: bounds.y + bounds.height / 2.0,
|
||||
})
|
||||
} else if let Some((x, y)) = args.xy {
|
||||
Ok(Point { x, y })
|
||||
} else {
|
||||
Err(AppError::invalid_input("Provide a ref (@e1) or --xy x,y"))
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,9 @@ pub enum IsProperty {
|
|||
Expanded,
|
||||
}
|
||||
|
||||
/// States are read from the last snapshot's RefMap. `resolve_ref` verifies the element
|
||||
/// is still live before returning, but the state values themselves are not re-queried
|
||||
/// from the AX API. Run `snapshot` to refresh state before calling `is`.
|
||||
pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
|
||||
|
|
@ -25,6 +28,8 @@ pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, App
|
|||
IsProperty::Expanded => "expanded",
|
||||
};
|
||||
|
||||
let applicable = is_applicable(&args.property, &entry.role);
|
||||
|
||||
let result = match args.property {
|
||||
IsProperty::Visible => !entry.states.contains(&"hidden".to_string()),
|
||||
IsProperty::Enabled => !entry.states.contains(&"disabled".to_string()),
|
||||
|
|
@ -33,5 +38,26 @@ pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, App
|
|||
IsProperty::Expanded => entry.states.contains(&"expanded".to_string()),
|
||||
};
|
||||
|
||||
Ok(json!({ "property": prop_name, "ref": args.ref_id, "result": result }))
|
||||
Ok(
|
||||
json!({ "property": prop_name, "ref": args.ref_id, "result": result, "applicable": applicable }),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_applicable(property: &IsProperty, role: &str) -> bool {
|
||||
match property {
|
||||
IsProperty::Visible | IsProperty::Enabled | IsProperty::Focused => true,
|
||||
IsProperty::Checked => matches!(
|
||||
role,
|
||||
"checkbox"
|
||||
| "switch"
|
||||
| "radiobutton"
|
||||
| "togglebutton"
|
||||
| "menuitemcheckbox"
|
||||
| "menuitemradio"
|
||||
),
|
||||
IsProperty::Expanded => matches!(
|
||||
role,
|
||||
"disclosuretriangle" | "treeitem" | "combobox" | "popupbutton" | "outline" | "row"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
15
crates/core/src/commands/key_down.rs
Normal file
15
crates/core/src/commands/key_down.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::press::parse_combo, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct KeyDownArgs {
|
||||
pub combo: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: KeyDownArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let combo = parse_combo(&args.combo)?;
|
||||
let handle = crate::adapter::NativeHandle::null();
|
||||
adapter.execute_action(&handle, Action::KeyDown(combo))?;
|
||||
Ok(json!({ "key_down": args.combo }))
|
||||
}
|
||||
15
crates/core/src/commands/key_up.rs
Normal file
15
crates/core/src/commands/key_up.rs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::press::parse_combo, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct KeyUpArgs {
|
||||
pub combo: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: KeyUpArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let combo = parse_combo(&args.combo)?;
|
||||
let handle = crate::adapter::NativeHandle::null();
|
||||
adapter.execute_action(&handle, Action::KeyUp(combo))?;
|
||||
Ok(json!({ "key_up": args.combo }))
|
||||
}
|
||||
|
|
@ -3,10 +3,10 @@ use serde_json::Value;
|
|||
|
||||
pub struct LaunchArgs {
|
||||
pub app: String,
|
||||
pub wait: bool,
|
||||
pub timeout_ms: u64,
|
||||
}
|
||||
|
||||
pub fn execute(args: LaunchArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let window = adapter.launch_app(&args.app, args.wait)?;
|
||||
let window = adapter.launch_app(&args.app, args.timeout_ms)?;
|
||||
Ok(serde_json::to_value(window)?)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use serde_json::Value;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub fn execute(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let apps = adapter.list_apps()?;
|
||||
Ok(serde_json::to_value(apps)?)
|
||||
Ok(json!({ "apps": apps }))
|
||||
}
|
||||
|
|
|
|||
12
crates/core/src/commands/list_surfaces.rs
Normal file
12
crates/core/src/commands/list_surfaces.rs
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
use crate::{adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct ListSurfacesArgs {
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
pub fn execute(args: ListSurfacesArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let surfaces = adapter.list_surfaces(pid).map_err(AppError::Adapter)?;
|
||||
Ok(json!({ "pid": pid, "surfaces": surfaces }))
|
||||
}
|
||||
|
|
@ -9,7 +9,10 @@ pub struct ListWindowsArgs {
|
|||
}
|
||||
|
||||
pub fn execute(args: ListWindowsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let filter = WindowFilter { focused_only: false, app: args.app };
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: args.app,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
Ok(serde_json::to_value(windows)?)
|
||||
}
|
||||
|
|
|
|||
30
crates/core/src/commands/maximize.rs
Normal file
30
crates/core/src/commands/maximize.rs
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
use crate::{
|
||||
action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MaximizeArgs {
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
pub fn execute(args: MaximizeArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let win = find_window(pid, adapter)?;
|
||||
adapter.window_op(&win, WindowOp::Maximize)?;
|
||||
Ok(json!({ "maximized": true }))
|
||||
}
|
||||
|
||||
fn find_window(
|
||||
pid: i32,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<crate::node::WindowInfo, AppError> {
|
||||
let filter = crate::adapter::WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.pid == pid)
|
||||
.ok_or_else(|| AppError::invalid_input("No window found for this application"))
|
||||
}
|
||||
30
crates/core/src/commands/minimize.rs
Normal file
30
crates/core/src/commands/minimize.rs
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
use crate::{
|
||||
action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MinimizeArgs {
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
pub fn execute(args: MinimizeArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let win = find_window(pid, adapter)?;
|
||||
adapter.window_op(&win, WindowOp::Minimize)?;
|
||||
Ok(json!({ "minimized": true }))
|
||||
}
|
||||
|
||||
fn find_window(
|
||||
pid: i32,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<crate::node::WindowInfo, AppError> {
|
||||
let filter = crate::adapter::WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.pid == pid)
|
||||
.ok_or_else(|| AppError::invalid_input("No window found for this application"))
|
||||
}
|
||||
|
|
@ -1,31 +1,50 @@
|
|||
pub mod batch;
|
||||
pub mod clipboard;
|
||||
pub mod check;
|
||||
pub mod clear;
|
||||
pub mod click;
|
||||
pub mod clipboard_clear;
|
||||
pub mod clipboard_get;
|
||||
pub mod clipboard_set;
|
||||
pub mod click;
|
||||
pub mod close_app;
|
||||
pub mod collapse;
|
||||
pub mod double_click;
|
||||
pub mod drag;
|
||||
pub mod expand;
|
||||
pub mod find;
|
||||
pub mod focus;
|
||||
pub mod focus_window;
|
||||
pub mod get;
|
||||
pub mod helpers;
|
||||
pub mod hover;
|
||||
pub mod is_check;
|
||||
pub mod key_down;
|
||||
pub mod key_up;
|
||||
pub mod launch;
|
||||
pub mod list_apps;
|
||||
pub mod list_surfaces;
|
||||
pub mod list_windows;
|
||||
pub mod maximize;
|
||||
pub mod minimize;
|
||||
pub mod mouse_click;
|
||||
pub mod mouse_down;
|
||||
pub mod mouse_move;
|
||||
pub mod mouse_up;
|
||||
pub mod move_window;
|
||||
pub mod permissions;
|
||||
pub mod press;
|
||||
pub mod resize_window;
|
||||
pub mod restore;
|
||||
pub mod right_click;
|
||||
pub mod screenshot;
|
||||
pub mod scroll;
|
||||
pub mod scroll_to;
|
||||
pub mod select;
|
||||
pub mod set_value;
|
||||
pub mod snapshot;
|
||||
pub mod status;
|
||||
pub mod toggle;
|
||||
pub mod triple_click;
|
||||
pub mod type_text;
|
||||
pub mod uncheck;
|
||||
pub mod version;
|
||||
pub mod wait;
|
||||
|
|
|
|||
25
crates/core/src/commands/mouse_click.rs
Normal file
25
crates/core/src/commands/mouse_click.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
use crate::{
|
||||
action::{MouseButton, MouseEvent, MouseEventKind, Point},
|
||||
adapter::PlatformAdapter,
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MouseClickArgs {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub button: MouseButton,
|
||||
pub count: u32,
|
||||
}
|
||||
|
||||
pub fn execute(args: MouseClickArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.mouse_event(MouseEvent {
|
||||
kind: MouseEventKind::Click { count: args.count },
|
||||
point: Point {
|
||||
x: args.x,
|
||||
y: args.y,
|
||||
},
|
||||
button: args.button,
|
||||
})?;
|
||||
Ok(json!({ "clicked": true, "x": args.x, "y": args.y, "count": args.count }))
|
||||
}
|
||||
24
crates/core/src/commands/mouse_down.rs
Normal file
24
crates/core/src/commands/mouse_down.rs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
use crate::{
|
||||
action::{MouseButton, MouseEvent, MouseEventKind, Point},
|
||||
adapter::PlatformAdapter,
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MouseDownArgs {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub button: MouseButton,
|
||||
}
|
||||
|
||||
pub fn execute(args: MouseDownArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.mouse_event(MouseEvent {
|
||||
kind: MouseEventKind::Down,
|
||||
point: Point {
|
||||
x: args.x,
|
||||
y: args.y,
|
||||
},
|
||||
button: args.button,
|
||||
})?;
|
||||
Ok(json!({ "pressed": true, "x": args.x, "y": args.y }))
|
||||
}
|
||||
23
crates/core/src/commands/mouse_move.rs
Normal file
23
crates/core/src/commands/mouse_move.rs
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
use crate::{
|
||||
action::{MouseButton, MouseEvent, MouseEventKind, Point},
|
||||
adapter::PlatformAdapter,
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MouseMoveArgs {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
}
|
||||
|
||||
pub fn execute(args: MouseMoveArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.mouse_event(MouseEvent {
|
||||
kind: MouseEventKind::Move,
|
||||
point: Point {
|
||||
x: args.x,
|
||||
y: args.y,
|
||||
},
|
||||
button: MouseButton::Left,
|
||||
})?;
|
||||
Ok(json!({ "moved": true, "x": args.x, "y": args.y }))
|
||||
}
|
||||
24
crates/core/src/commands/mouse_up.rs
Normal file
24
crates/core/src/commands/mouse_up.rs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
use crate::{
|
||||
action::{MouseButton, MouseEvent, MouseEventKind, Point},
|
||||
adapter::PlatformAdapter,
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MouseUpArgs {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub button: MouseButton,
|
||||
}
|
||||
|
||||
pub fn execute(args: MouseUpArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.mouse_event(MouseEvent {
|
||||
kind: MouseEventKind::Up,
|
||||
point: Point {
|
||||
x: args.x,
|
||||
y: args.y,
|
||||
},
|
||||
button: args.button,
|
||||
})?;
|
||||
Ok(json!({ "released": true, "x": args.x, "y": args.y }))
|
||||
}
|
||||
38
crates/core/src/commands/move_window.rs
Normal file
38
crates/core/src/commands/move_window.rs
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
use crate::{
|
||||
action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct MoveWindowArgs {
|
||||
pub app: Option<String>,
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
}
|
||||
|
||||
pub fn execute(args: MoveWindowArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let win = find_window(pid, adapter)?;
|
||||
adapter.window_op(
|
||||
&win,
|
||||
WindowOp::Move {
|
||||
x: args.x,
|
||||
y: args.y,
|
||||
},
|
||||
)?;
|
||||
Ok(json!({ "moved": true, "x": args.x, "y": args.y }))
|
||||
}
|
||||
|
||||
fn find_window(
|
||||
pid: i32,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<crate::node::WindowInfo, AppError> {
|
||||
let filter = crate::adapter::WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.pid == pid)
|
||||
.ok_or_else(|| AppError::invalid_input("No window found for this application"))
|
||||
}
|
||||
|
|
@ -1,4 +1,7 @@
|
|||
use crate::{adapter::{PermissionStatus, PlatformAdapter}, error::AppError};
|
||||
use crate::{
|
||||
adapter::{PermissionStatus, PlatformAdapter},
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct PermissionsArgs {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ const BLOCKED_COMBOS: &[&str] = &[
|
|||
|
||||
pub struct PressArgs {
|
||||
pub combo: String,
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
|
|
@ -27,18 +28,25 @@ pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result<Value,
|
|||
}
|
||||
|
||||
let combo = parse_combo(&normalized)?;
|
||||
|
||||
if let Some(app_name) = &args.app {
|
||||
let result = adapter.press_key_for_app(app_name, &combo)?;
|
||||
return Ok(serde_json::to_value(result)?);
|
||||
}
|
||||
|
||||
let handle = crate::adapter::NativeHandle::null();
|
||||
let result = adapter.execute_action(&handle, Action::PressKey(combo))?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
}
|
||||
|
||||
fn parse_combo(s: &str) -> Result<KeyCombo, AppError> {
|
||||
pub fn parse_combo(s: &str) -> Result<KeyCombo, AppError> {
|
||||
let parts: Vec<&str> = s.split('+').collect();
|
||||
if parts.is_empty() {
|
||||
return Err(AppError::invalid_input("Empty key combo"));
|
||||
}
|
||||
|
||||
let key = parts.last().unwrap().to_string();
|
||||
let key = parts
|
||||
.last()
|
||||
.copied()
|
||||
.filter(|k| !k.is_empty())
|
||||
.ok_or_else(|| AppError::invalid_input("Empty key combo"))?
|
||||
.to_string();
|
||||
let mut modifiers = Vec::new();
|
||||
|
||||
for &part in &parts[..parts.len() - 1] {
|
||||
|
|
|
|||
38
crates/core/src/commands/resize_window.rs
Normal file
38
crates/core/src/commands/resize_window.rs
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
use crate::{
|
||||
action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct ResizeWindowArgs {
|
||||
pub app: Option<String>,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
pub fn execute(args: ResizeWindowArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let win = find_window(pid, adapter)?;
|
||||
adapter.window_op(
|
||||
&win,
|
||||
WindowOp::Resize {
|
||||
width: args.width,
|
||||
height: args.height,
|
||||
},
|
||||
)?;
|
||||
Ok(json!({ "resized": true, "width": args.width, "height": args.height }))
|
||||
}
|
||||
|
||||
fn find_window(
|
||||
pid: i32,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<crate::node::WindowInfo, AppError> {
|
||||
let filter = crate::adapter::WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.pid == pid)
|
||||
.ok_or_else(|| AppError::invalid_input("No window found for this application"))
|
||||
}
|
||||
30
crates/core/src/commands/restore.rs
Normal file
30
crates/core/src/commands/restore.rs
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
use crate::{
|
||||
action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct RestoreArgs {
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
pub fn execute(args: RestoreArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let win = find_window(pid, adapter)?;
|
||||
adapter.window_op(&win, WindowOp::Restore)?;
|
||||
Ok(json!({ "restored": true }))
|
||||
}
|
||||
|
||||
fn find_window(
|
||||
pid: i32,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<crate::node::WindowInfo, AppError> {
|
||||
let filter = crate::adapter::WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.pid == pid)
|
||||
.ok_or_else(|| AppError::invalid_input("No window found for this application"))
|
||||
}
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::{
|
||||
adapter::{PlatformAdapter, ScreenshotTarget},
|
||||
adapter::{PlatformAdapter, ScreenshotTarget, WindowFilter},
|
||||
error::AppError,
|
||||
};
|
||||
use base64::Engine;
|
||||
|
|
@ -13,12 +13,7 @@ pub struct ScreenshotArgs {
|
|||
}
|
||||
|
||||
pub fn execute(args: ScreenshotArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let target = match (&args.window_id, &args.app) {
|
||||
(Some(id), _) => ScreenshotTarget::Window(id.clone()),
|
||||
(None, Some(_app)) => ScreenshotTarget::FullScreen,
|
||||
(None, None) => ScreenshotTarget::FullScreen,
|
||||
};
|
||||
|
||||
let target = resolve_target(&args, adapter)?;
|
||||
let buf = adapter.screenshot(target)?;
|
||||
|
||||
if let Some(path) = args.output_path {
|
||||
|
|
@ -34,3 +29,35 @@ pub fn execute(args: ScreenshotArgs, adapter: &dyn PlatformAdapter) -> Result<Va
|
|||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_target(
|
||||
args: &ScreenshotArgs,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<ScreenshotTarget, AppError> {
|
||||
if let Some(window_id) = &args.window_id {
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: args.app.clone(),
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
let win = windows
|
||||
.into_iter()
|
||||
.find(|w| &w.id == window_id)
|
||||
.ok_or_else(|| AppError::invalid_input(format!("Window '{window_id}' not found")))?;
|
||||
return Ok(ScreenshotTarget::Window(win.pid));
|
||||
}
|
||||
|
||||
if let Some(app_name) = &args.app {
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: Some(app_name.clone()),
|
||||
};
|
||||
let windows = adapter.list_windows(&filter)?;
|
||||
let win = windows.into_iter().next().ok_or_else(|| {
|
||||
AppError::invalid_input(format!("No windows found for app '{app_name}'"))
|
||||
})?;
|
||||
return Ok(ScreenshotTarget::Window(win.pid));
|
||||
}
|
||||
|
||||
Ok(ScreenshotTarget::FullScreen)
|
||||
}
|
||||
|
|
|
|||
14
crates/core/src/commands/scroll_to.rs
Normal file
14
crates/core/src/commands/scroll_to.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct ScrollToArgs {
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: ScrollToArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let result = adapter.execute_action(&handle, Action::ScrollTo)?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
}
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError, snapshot};
|
||||
use crate::{
|
||||
adapter::{PlatformAdapter, SnapshotSurface},
|
||||
error::AppError,
|
||||
snapshot,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub struct SnapshotArgs {
|
||||
|
|
@ -8,6 +12,7 @@ pub struct SnapshotArgs {
|
|||
pub include_bounds: bool,
|
||||
pub interactive_only: bool,
|
||||
pub compact: bool,
|
||||
pub surface: SnapshotSurface,
|
||||
}
|
||||
|
||||
pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
|
|
@ -16,6 +21,7 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Valu
|
|||
include_bounds: args.include_bounds,
|
||||
interactive_only: args.interactive_only,
|
||||
compact: args.compact,
|
||||
surface: args.surface,
|
||||
};
|
||||
|
||||
let result = snapshot::run(
|
||||
|
|
@ -27,8 +33,14 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Valu
|
|||
|
||||
let ref_count = result.refmap.len();
|
||||
let tree = serde_json::to_value(&result.tree)?;
|
||||
let win = &result.window;
|
||||
|
||||
Ok(json!({
|
||||
"app": win.app,
|
||||
"window": {
|
||||
"id": win.id,
|
||||
"title": win.title
|
||||
},
|
||||
"ref_count": ref_count,
|
||||
"tree": tree
|
||||
}))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
use crate::{adapter::{PermissionStatus, PlatformAdapter}, error::AppError, refs::RefMap};
|
||||
use crate::{
|
||||
adapter::{PermissionStatus, PlatformAdapter},
|
||||
error::AppError,
|
||||
refs::RefMap,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub fn execute(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
|
|
|
|||
14
crates/core/src/commands/triple_click.rs
Normal file
14
crates/core/src/commands/triple_click.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct TripleClickArgs {
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: TripleClickArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let result = adapter.execute_action(&handle, Action::TripleClick)?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
}
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::PlatformAdapter,
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
14
crates/core/src/commands/uncheck.rs
Normal file
14
crates/core/src/commands/uncheck.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct UncheckArgs {
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
pub fn execute(args: UncheckArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let result = adapter.execute_action(&handle, Action::Uncheck)?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
}
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
use crate::{
|
||||
adapter::{PlatformAdapter, WindowFilter},
|
||||
commands::helpers::validate_ref_id,
|
||||
commands::helpers::{resolve_app_pid, validate_ref_id},
|
||||
error::AppError,
|
||||
node::AccessibilityNode,
|
||||
refs::RefMap,
|
||||
snapshot,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use std::time::{Duration, Instant};
|
||||
|
|
@ -11,7 +13,11 @@ pub struct WaitArgs {
|
|||
pub ms: Option<u64>,
|
||||
pub element: Option<String>,
|
||||
pub window: Option<String>,
|
||||
pub text: Option<String>,
|
||||
pub timeout_ms: u64,
|
||||
pub menu: bool,
|
||||
pub menu_closed: bool,
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
pub fn execute(args: WaitArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
|
|
@ -20,6 +26,16 @@ pub fn execute(args: WaitArgs, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
return Ok(json!({ "waited_ms": ms }));
|
||||
}
|
||||
|
||||
if args.menu || args.menu_closed {
|
||||
let pid = resolve_app_pid(args.app.as_deref(), adapter)?;
|
||||
let start = Instant::now();
|
||||
adapter
|
||||
.wait_for_menu(pid, args.menu, args.timeout_ms)
|
||||
.map_err(AppError::Adapter)?;
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
return Ok(json!({ "found": true, "elapsed_ms": elapsed }));
|
||||
}
|
||||
|
||||
if let Some(ref_id) = args.element {
|
||||
validate_ref_id(&ref_id)?;
|
||||
return wait_for_element(ref_id, args.timeout_ms, adapter);
|
||||
|
|
@ -29,8 +45,12 @@ pub fn execute(args: WaitArgs, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
return wait_for_window(title, args.timeout_ms, adapter);
|
||||
}
|
||||
|
||||
if let Some(text) = args.text {
|
||||
return wait_for_text(text, args.app, args.timeout_ms, adapter);
|
||||
}
|
||||
|
||||
Err(AppError::invalid_input(
|
||||
"Provide a duration (ms), --element <ref>, or --window <title>",
|
||||
"Provide a duration (ms), --menu, --element <ref>, --window <title>, or --text <text>",
|
||||
))
|
||||
}
|
||||
|
||||
|
|
@ -44,18 +64,18 @@ fn wait_for_element(
|
|||
|
||||
loop {
|
||||
if let Ok(refmap) = RefMap::load() {
|
||||
if refmap.get(&ref_id).is_some()
|
||||
&& adapter.resolve_element(refmap.get(&ref_id).unwrap()).is_ok()
|
||||
{
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed }));
|
||||
if let Some(entry) = refmap.get(&ref_id) {
|
||||
if adapter.resolve_element(entry).is_ok() {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if start.elapsed() >= timeout {
|
||||
return Err(AppError::Adapter(crate::error::AdapterError::timeout(format!(
|
||||
"Element {ref_id} not found within {timeout_ms}ms"
|
||||
))));
|
||||
return Err(AppError::Adapter(crate::error::AdapterError::timeout(
|
||||
format!("Element {ref_id} not found within {timeout_ms}ms"),
|
||||
)));
|
||||
}
|
||||
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
|
|
@ -69,7 +89,10 @@ fn wait_for_window(
|
|||
) -> Result<Value, AppError> {
|
||||
let start = Instant::now();
|
||||
let timeout = Duration::from_millis(timeout_ms);
|
||||
let filter = WindowFilter { focused_only: false, app: None };
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
|
||||
loop {
|
||||
if let Ok(windows) = adapter.list_windows(&filter) {
|
||||
|
|
@ -80,11 +103,80 @@ fn wait_for_window(
|
|||
}
|
||||
|
||||
if start.elapsed() >= timeout {
|
||||
return Err(AppError::Adapter(crate::error::AdapterError::timeout(format!(
|
||||
"Window with title '{title}' not found within {timeout_ms}ms"
|
||||
))));
|
||||
return Err(AppError::Adapter(crate::error::AdapterError::timeout(
|
||||
format!("Window with title '{title}' not found within {timeout_ms}ms"),
|
||||
)));
|
||||
}
|
||||
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_for_text(
|
||||
text: String,
|
||||
app: Option<String>,
|
||||
timeout_ms: u64,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<Value, AppError> {
|
||||
let start = Instant::now();
|
||||
let timeout = Duration::from_millis(timeout_ms);
|
||||
let opts = crate::adapter::TreeOptions::default();
|
||||
let text_lower = text.to_lowercase();
|
||||
|
||||
loop {
|
||||
if let Ok(result) = snapshot::run(adapter, &opts, app.as_deref(), None) {
|
||||
if let Some(found) = find_text_in_tree(&result.tree, &text_lower) {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
return Ok(json!({
|
||||
"found": true,
|
||||
"text": text,
|
||||
"ref": found.ref_id,
|
||||
"role": found.role,
|
||||
"elapsed_ms": elapsed
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if start.elapsed() >= timeout {
|
||||
return Err(AppError::Adapter(crate::error::AdapterError::timeout(
|
||||
format!("Text '{text}' not found within {timeout_ms}ms"),
|
||||
)));
|
||||
}
|
||||
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
}
|
||||
|
||||
struct TextMatch {
|
||||
ref_id: Option<String>,
|
||||
role: String,
|
||||
}
|
||||
|
||||
fn find_text_in_tree(node: &AccessibilityNode, text_lower: &str) -> Option<TextMatch> {
|
||||
let in_name = node
|
||||
.name
|
||||
.as_deref()
|
||||
.is_some_and(|n| n.to_lowercase().contains(text_lower));
|
||||
let in_value = node
|
||||
.value
|
||||
.as_deref()
|
||||
.is_some_and(|v| v.to_lowercase().contains(text_lower));
|
||||
let in_desc = node
|
||||
.description
|
||||
.as_deref()
|
||||
.is_some_and(|d| d.to_lowercase().contains(text_lower));
|
||||
|
||||
if in_name || in_value || in_desc {
|
||||
return Some(TextMatch {
|
||||
ref_id: node.ref_id.clone(),
|
||||
role: node.role.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
for child in &node.children {
|
||||
if let Some(found) = find_text_in_tree(child, text_lower) {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,8 +72,11 @@ impl AdapterError {
|
|||
}
|
||||
|
||||
pub fn stale_ref(ref_id: &str) -> Self {
|
||||
Self::new(ErrorCode::StaleRef, format!("{ref_id} not found in current RefMap"))
|
||||
.with_suggestion("Run 'snapshot' to refresh, then retry with updated ref")
|
||||
Self::new(
|
||||
ErrorCode::StaleRef,
|
||||
format!("{ref_id} not found in current RefMap"),
|
||||
)
|
||||
.with_suggestion("Run 'snapshot' to refresh, then retry with updated ref")
|
||||
}
|
||||
|
||||
pub fn not_supported(method: &str) -> Self {
|
||||
|
|
@ -85,8 +88,11 @@ impl AdapterError {
|
|||
}
|
||||
|
||||
pub fn element_not_found(ref_id: &str) -> Self {
|
||||
Self::new(ErrorCode::ElementNotFound, format!("Element {ref_id} could not be resolved"))
|
||||
.with_suggestion("Run 'snapshot' to get fresh refs")
|
||||
Self::new(
|
||||
ErrorCode::ElementNotFound,
|
||||
format!("Element {ref_id} could not be resolved"),
|
||||
)
|
||||
.with_suggestion("Run 'snapshot' to get fresh refs")
|
||||
}
|
||||
|
||||
pub fn timeout(msg: impl Into<String>) -> Self {
|
||||
|
|
@ -99,10 +105,13 @@ impl AdapterError {
|
|||
}
|
||||
|
||||
pub fn permission_denied() -> Self {
|
||||
Self::new(ErrorCode::PermDenied, "Accessibility permission not granted")
|
||||
.with_suggestion(
|
||||
"Open System Settings > Privacy & Security > Accessibility and add your terminal",
|
||||
)
|
||||
Self::new(
|
||||
ErrorCode::PermDenied,
|
||||
"Accessibility permission not granted",
|
||||
)
|
||||
.with_suggestion(
|
||||
"Open System Settings > Privacy & Security > Accessibility and add your terminal",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@ pub mod output;
|
|||
pub mod refs;
|
||||
pub mod snapshot;
|
||||
|
||||
pub use action::{Action, ActionResult, Direction, ElementState, KeyCombo, Modifier};
|
||||
pub use action::{
|
||||
Action, ActionResult, Direction, DragParams, ElementState, KeyCombo, Modifier, MouseButton,
|
||||
MouseEvent, MouseEventKind, Point, WindowOp,
|
||||
};
|
||||
pub use adapter::{
|
||||
ImageBuffer, ImageFormat, NativeHandle, PermissionStatus, PlatformAdapter, ScreenshotTarget,
|
||||
TreeOptions, WindowFilter,
|
||||
|
|
|
|||
|
|
@ -69,3 +69,13 @@ pub struct AppInfo {
|
|||
pub pid: i32,
|
||||
pub bundle_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SurfaceInfo {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub item_count: Option<usize>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,10 @@ pub struct RefMap {
|
|||
|
||||
impl RefMap {
|
||||
pub fn new() -> Self {
|
||||
Self { inner: HashMap::new(), counter: 0 }
|
||||
Self {
|
||||
inner: HashMap::new(),
|
||||
counter: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn allocate(&mut self, entry: RefEntry) -> String {
|
||||
|
|
@ -60,7 +63,10 @@ impl RefMap {
|
|||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
std::fs::DirBuilder::new().recursive(true).mode(0o700).create(dir)?;
|
||||
std::fs::DirBuilder::new()
|
||||
.recursive(true)
|
||||
.mode(0o700)
|
||||
.create(dir)?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
std::fs::create_dir_all(dir)?;
|
||||
|
|
@ -111,8 +117,7 @@ impl Default for RefMap {
|
|||
}
|
||||
|
||||
fn refmap_path() -> Result<PathBuf, AppError> {
|
||||
let home = home_dir()
|
||||
.ok_or_else(|| AppError::Internal("HOME directory not found".into()))?;
|
||||
let home = home_dir().ok_or_else(|| AppError::Internal("HOME directory not found".into()))?;
|
||||
Ok(home.join(".agent-desktop").join("last_refmap.json"))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,33 @@
|
|||
use crate::{
|
||||
adapter::{PlatformAdapter, TreeOptions, WindowFilter},
|
||||
error::AppError,
|
||||
node::AccessibilityNode,
|
||||
node::{AccessibilityNode, WindowInfo},
|
||||
refs::{RefEntry, RefMap},
|
||||
};
|
||||
|
||||
const INTERACTIVE_ROLES: &[&str] = &[
|
||||
"button", "textfield", "checkbox", "link", "menuitem", "tab", "slider",
|
||||
"combobox", "treeitem", "cell", "radiobutton", "incrementor",
|
||||
"button",
|
||||
"textfield",
|
||||
"checkbox",
|
||||
"link",
|
||||
"menuitem",
|
||||
"tab",
|
||||
"slider",
|
||||
"combobox",
|
||||
"treeitem",
|
||||
"cell",
|
||||
"radiobutton",
|
||||
"incrementor",
|
||||
"menubutton",
|
||||
"switch",
|
||||
"colorwell",
|
||||
"dockitem",
|
||||
];
|
||||
|
||||
pub struct SnapshotResult {
|
||||
pub tree: AccessibilityNode,
|
||||
pub refmap: RefMap,
|
||||
pub window: WindowInfo,
|
||||
}
|
||||
|
||||
pub fn build(
|
||||
|
|
@ -41,7 +56,10 @@ pub fn build(
|
|||
.find(|w| w.app.eq_ignore_ascii_case(app) && w.is_focused)
|
||||
.or_else(|| {
|
||||
adapter
|
||||
.list_windows(&WindowFilter { focused_only: false, app: Some(app.to_string()) })
|
||||
.list_windows(&WindowFilter {
|
||||
focused_only: false,
|
||||
app: Some(app.to_string()),
|
||||
})
|
||||
.ok()
|
||||
.and_then(|ws| ws.into_iter().next())
|
||||
})
|
||||
|
|
@ -52,26 +70,15 @@ pub fn build(
|
|||
))
|
||||
})?
|
||||
} else {
|
||||
windows
|
||||
.into_iter()
|
||||
.find(|w| w.is_focused)
|
||||
.ok_or_else(|| {
|
||||
AppError::Adapter(crate::error::AdapterError::new(
|
||||
crate::error::ErrorCode::WindowNotFound,
|
||||
"No focused window found. Use --app to specify an application.",
|
||||
))
|
||||
})?
|
||||
windows.into_iter().find(|w| w.is_focused).ok_or_else(|| {
|
||||
AppError::Adapter(crate::error::AdapterError::new(
|
||||
crate::error::ErrorCode::WindowNotFound,
|
||||
"No focused window found. Use --app to specify an application.",
|
||||
))
|
||||
})?
|
||||
};
|
||||
|
||||
let capped_depth = opts.max_depth;
|
||||
let tree_opts = TreeOptions {
|
||||
max_depth: capped_depth,
|
||||
include_bounds: opts.include_bounds,
|
||||
interactive_only: opts.interactive_only,
|
||||
compact: opts.compact,
|
||||
};
|
||||
|
||||
let raw_tree = adapter.get_tree(&window, &tree_opts)?;
|
||||
let raw_tree = adapter.get_tree(&window, opts)?;
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let tree = allocate_refs(
|
||||
|
|
@ -83,7 +90,11 @@ pub fn build(
|
|||
Some(window.app.as_str()),
|
||||
);
|
||||
|
||||
Ok(SnapshotResult { tree, refmap })
|
||||
Ok(SnapshotResult {
|
||||
tree,
|
||||
refmap,
|
||||
window,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
|
|
@ -129,7 +140,21 @@ fn allocate_refs(
|
|||
node.children = node
|
||||
.children
|
||||
.into_iter()
|
||||
.map(|child| allocate_refs(child, refmap, include_bounds, interactive_only, window_pid, source_app))
|
||||
.filter_map(|child| {
|
||||
let child = allocate_refs(
|
||||
child,
|
||||
refmap,
|
||||
include_bounds,
|
||||
interactive_only,
|
||||
window_pid,
|
||||
source_app,
|
||||
);
|
||||
if interactive_only && child.ref_id.is_none() && child.children.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(child)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
node
|
||||
|
|
|
|||
|
|
@ -19,4 +19,19 @@ core-foundation = "0.10.1"
|
|||
core-foundation-sys = "0.8.7"
|
||||
core-graphics = { version = "0.25.0", features = ["highsierra"] }
|
||||
|
||||
[features]
|
||||
dev-tools = []
|
||||
|
||||
[[example]]
|
||||
name = "ax_probe"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[[example]]
|
||||
name = "axprobe"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[[example]]
|
||||
name = "axprobe2"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[build-dependencies]
|
||||
|
|
|
|||
396
crates/macos/examples/ax_probe.rs
Normal file
396
crates/macos/examples/ax_probe.rs
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
/// Direct probe of macOS AX APIs — no abstraction.
|
||||
/// Reveals exactly what the raw APIs return and validates click behavior.
|
||||
///
|
||||
/// cargo run -p agent-desktop-macos --example ax_probe -- <AppName>
|
||||
use std::ffi::c_void;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn main() {
|
||||
let app_name = std::env::args().nth(1).unwrap_or_else(|| "Finder".into());
|
||||
println!("=== AX Probe: '{}' ===\n", app_name);
|
||||
|
||||
let pid = find_pid(&app_name).expect("app not running");
|
||||
println!("[pid] {}", pid);
|
||||
|
||||
probe_app(pid);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_pid(name: &str) -> Option<i32> {
|
||||
let out = std::process::Command::new("pgrep")
|
||||
.arg("-xi")
|
||||
.arg(name)
|
||||
.output()
|
||||
.ok()?;
|
||||
String::from_utf8_lossy(&out.stdout)
|
||||
.trim()
|
||||
.lines()
|
||||
.next()?
|
||||
.trim()
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn probe_app(pid: i32) {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::base::CFTypeRef;
|
||||
|
||||
let app = unsafe { AXUIElementCreateApplication(pid) };
|
||||
|
||||
// ── 1. What attributes does the app element expose? ──────────────────────
|
||||
println!("\n[1] App element attribute names:");
|
||||
let attr_names = get_attribute_names(app);
|
||||
for n in &attr_names {
|
||||
println!(" {n}");
|
||||
}
|
||||
|
||||
// ── 2. Get windows the right way ─────────────────────────────────────────
|
||||
println!("\n[2] Windows via kAXWindowsAttribute:");
|
||||
let windows = get_ax_children(app, kAXWindowsAttribute);
|
||||
println!(" count = {}", windows.len());
|
||||
for (i, win) in windows.iter().enumerate() {
|
||||
let role = read_string(*win, kAXRoleAttribute);
|
||||
let title = read_string(*win, kAXTitleAttribute);
|
||||
let pos = read_cgpoint(*win, kAXPositionAttribute);
|
||||
let size = read_cgsize(*win, kAXSizeAttribute);
|
||||
println!(
|
||||
" [{i}] role={:?} title={:?} pos={:?} size={:?}",
|
||||
role, title, pos, size
|
||||
);
|
||||
|
||||
// Children of this window
|
||||
let children = get_ax_children(*win, kAXChildrenAttribute);
|
||||
println!(" children = {}", children.len());
|
||||
for (ci, child) in children.iter().enumerate().take(8) {
|
||||
let cr = read_string(*child, kAXRoleAttribute);
|
||||
let ct = read_string(*child, kAXTitleAttribute);
|
||||
let cd = read_string(*child, kAXDescriptionAttribute);
|
||||
let cv = read_string(*child, kAXValueAttribute);
|
||||
let cpos = read_cgpoint(*child, kAXPositionAttribute);
|
||||
let csz = read_cgsize(*child, kAXSizeAttribute);
|
||||
|
||||
println!(
|
||||
" [{ci}] role={:?} title={:?} desc={:?} val={:?}",
|
||||
cr, ct, cd, cv
|
||||
);
|
||||
println!(" pos={:?} size={:?}", cpos, csz);
|
||||
|
||||
// ── 3. Test kAXPressAction on each child ────────────────────────
|
||||
let ax_err = ax_press(*child);
|
||||
println!(" kAXPressAction → err={} (0=ok, -25200=fail, -25205=not_supported)", ax_err);
|
||||
|
||||
// ── 4. Test CGEvent click at element center ─────────────────────
|
||||
if let (Some(p), Some(s)) = (cpos, csz) {
|
||||
let cx = p.0 + s.0 / 2.0;
|
||||
let cy = p.1 + s.1 / 2.0;
|
||||
let cg_ok = cg_click(cx, cy);
|
||||
println!(
|
||||
" CGEvent click at ({:.0},{:.0}) → {}",
|
||||
cx,
|
||||
cy,
|
||||
if cg_ok { "OK" } else { "FAIL" }
|
||||
);
|
||||
} else {
|
||||
println!(" CGEvent click → no bounds available");
|
||||
}
|
||||
|
||||
release_ax(*child);
|
||||
}
|
||||
for child in children.iter().skip(8) {
|
||||
release_ax(*child);
|
||||
}
|
||||
|
||||
release_ax(*win);
|
||||
if i >= 1 {
|
||||
break;
|
||||
} // only first 2 windows
|
||||
}
|
||||
|
||||
// ── 5. Multi-attribute fetch speed comparison ─────────────────────────────
|
||||
println!("\n[5] AXUIElementCopyMultipleAttributeValues vs individual calls:");
|
||||
speed_test(app, pid);
|
||||
|
||||
// ── 6. Scroll event test ─────────────────────────────────────────────────
|
||||
println!("\n[6] CGEvent scroll at (400, 400):");
|
||||
let ok = cg_scroll(400.0, 400.0, 0, -3);
|
||||
println!(" result = {}", if ok { "OK" } else { "FAIL" });
|
||||
|
||||
unsafe { core_foundation::base::CFRelease(app as CFTypeRef) };
|
||||
println!("\n=== Done ===");
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_attribute_names(el: accessibility_sys::AXUIElementRef) -> Vec<String> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{array::CFArray, base::CFTypeRef, base::TCFType, string::CFString};
|
||||
|
||||
let mut out_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut out_ref as *mut _ as *mut _) };
|
||||
if err != kAXErrorSuccess || out_ref.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFString>::wrap_under_create_rule(out_ref as _) };
|
||||
arr.into_iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
/// Read an array-typed AX attribute, retaining each element so it stays alive.
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_ax_children(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
attr: &str,
|
||||
) -> Vec<accessibility_sys::AXUIElementRef> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(val as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
return None;
|
||||
}
|
||||
// Retain so the element lives past CFArray dealloc
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(ptr)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn release_ax(el: accessibility_sys::AXUIElementRef) {
|
||||
if !el.is_null() {
|
||||
unsafe { core_foundation::base::CFRelease(el as core_foundation::base::CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_string(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<String> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(val) };
|
||||
cf.downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_cgpoint(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<(f64, f64)> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
base::{CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
use core_graphics::geometry::CGPoint;
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut pt = CGPoint::new(0.0, 0.0);
|
||||
let ok = unsafe {
|
||||
AXValueGetValue(
|
||||
val as _,
|
||||
kAXValueTypeCGPoint,
|
||||
&mut pt as *mut _ as *mut std::ffi::c_void,
|
||||
)
|
||||
};
|
||||
unsafe { core_foundation::base::CFRelease(val) };
|
||||
if ok {
|
||||
Some((pt.x, pt.y))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_cgsize(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<(f64, f64)> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
base::{CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
use core_graphics::geometry::CGSize;
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() {
|
||||
return None;
|
||||
}
|
||||
let mut sz = CGSize::new(0.0, 0.0);
|
||||
let ok = unsafe {
|
||||
AXValueGetValue(
|
||||
val as _,
|
||||
kAXValueTypeCGSize,
|
||||
&mut sz as *mut _ as *mut std::ffi::c_void,
|
||||
)
|
||||
};
|
||||
unsafe { core_foundation::base::CFRelease(val) };
|
||||
if ok {
|
||||
Some((sz.width, sz.height))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn ax_press(el: accessibility_sys::AXUIElementRef) -> i32 {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
let action = CFString::new(kAXPressAction);
|
||||
unsafe { AXUIElementPerformAction(el, action.as_concrete_TypeRef()) }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn cg_click(x: f64, y: f64) -> bool {
|
||||
use core_graphics::{
|
||||
event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton},
|
||||
event_source::{CGEventSource, CGEventSourceStateID},
|
||||
geometry::CGPoint,
|
||||
};
|
||||
let src = match CGEventSource::new(CGEventSourceStateID::HIDSystemState) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return false,
|
||||
};
|
||||
let pt = CGPoint::new(x, y);
|
||||
let down = CGEvent::new_mouse_event(
|
||||
src.clone(),
|
||||
CGEventType::LeftMouseDown,
|
||||
pt,
|
||||
CGMouseButton::Left,
|
||||
);
|
||||
let up = CGEvent::new_mouse_event(src, CGEventType::LeftMouseUp, pt, CGMouseButton::Left);
|
||||
match (down, up) {
|
||||
(Ok(d), Ok(u)) => {
|
||||
d.post(CGEventTapLocation::HID);
|
||||
u.post(CGEventTapLocation::HID);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn cg_scroll(x: f64, y: f64, dx: i32, dy: i32) -> bool {
|
||||
use core_graphics::{
|
||||
event::{CGEvent, CGEventTapLocation, ScrollEventUnit},
|
||||
event_source::{CGEventSource, CGEventSourceStateID},
|
||||
};
|
||||
let src = match CGEventSource::new(CGEventSourceStateID::HIDSystemState) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return false,
|
||||
};
|
||||
match CGEvent::new_scroll_event(src, ScrollEventUnit::LINE, 2, dy, dx, 0) {
|
||||
Ok(ev) => {
|
||||
ev.post(CGEventTapLocation::HID);
|
||||
true
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn speed_test(app: accessibility_sys::AXUIElementRef, pid: i32) {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
use std::time::Instant;
|
||||
|
||||
// Get a real window element to test on
|
||||
let windows = get_ax_children(app, kAXWindowsAttribute);
|
||||
let el = if let Some(&w) = windows.first() {
|
||||
w
|
||||
} else {
|
||||
app
|
||||
};
|
||||
|
||||
let attrs = [
|
||||
kAXRoleAttribute,
|
||||
kAXTitleAttribute,
|
||||
kAXDescriptionAttribute,
|
||||
kAXValueAttribute,
|
||||
kAXEnabledAttribute,
|
||||
kAXFocusedAttribute,
|
||||
];
|
||||
let cf_attrs: Vec<CFString> = attrs.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_attrs.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
// Multi-attr
|
||||
let t = Instant::now();
|
||||
for _ in 0..100 {
|
||||
let mut res: CFTypeRef = std::ptr::null_mut();
|
||||
unsafe {
|
||||
AXUIElementCopyMultipleAttributeValues(
|
||||
el,
|
||||
names_arr.as_concrete_TypeRef(),
|
||||
0,
|
||||
&mut res as *mut _ as *mut _,
|
||||
)
|
||||
};
|
||||
if !res.is_null() {
|
||||
unsafe { CFRelease(res) };
|
||||
}
|
||||
}
|
||||
let multi = t.elapsed();
|
||||
|
||||
// Individual attrs
|
||||
let t2 = Instant::now();
|
||||
for _ in 0..100 {
|
||||
for attr in &attrs {
|
||||
let _ = read_string(el, attr);
|
||||
}
|
||||
}
|
||||
let single = t2.elapsed();
|
||||
|
||||
println!(
|
||||
" 100x multi-attr: {:?} ({:?}/call)",
|
||||
multi,
|
||||
multi / 100
|
||||
);
|
||||
println!(
|
||||
" 100x individual: {:?} ({:?}/call)",
|
||||
single,
|
||||
single / 100
|
||||
);
|
||||
println!(
|
||||
" speedup: {:.1}x",
|
||||
single.as_nanos() as f64 / multi.as_nanos().max(1) as f64
|
||||
);
|
||||
|
||||
for w in windows {
|
||||
release_ax(w);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
fn main() {
|
||||
eprintln!("macOS only");
|
||||
}
|
||||
424
crates/macos/examples/axprobe.rs
Normal file
424
crates/macos/examples/axprobe.rs
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
//! AX API probe — discovers what every accessibility function returns on a live app.
|
||||
//! Run: cargo run -p agent-desktop-macos --bin axprobe -- Finder
|
||||
//!
|
||||
//! This is a diagnostic tool used to learn exactly which attributes hold which
|
||||
//! data before writing tree traversal code. Output is deliberately verbose.
|
||||
|
||||
fn main() {
|
||||
#[cfg(target_os = "macos")]
|
||||
run();
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
eprintln!("axprobe only works on macOS");
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn run() {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
url::CFURL,
|
||||
};
|
||||
|
||||
let app_name = std::env::args()
|
||||
.nth(1)
|
||||
.unwrap_or_else(|| "Finder".to_string());
|
||||
let pid = find_pid(&app_name).unwrap_or_else(|| {
|
||||
eprintln!("App '{}' not running", app_name);
|
||||
std::process::exit(1);
|
||||
});
|
||||
|
||||
println!("=== axprobe: {} (pid {}) ===\n", app_name, pid);
|
||||
|
||||
let app_el = unsafe { AXUIElementCreateApplication(pid) };
|
||||
unsafe { AXUIElementSetMessagingTimeout(app_el, 5.0) };
|
||||
|
||||
// ── 1. App-level attributes ──────────────────────────────────────────
|
||||
println!("──────────────────────────────────────────────");
|
||||
println!("SECTION 1: App element attributes");
|
||||
println!("──────────────────────────────────────────────");
|
||||
dump_all_attrs(app_el, 0);
|
||||
|
||||
// ── 2. Windows ───────────────────────────────────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 2: Windows");
|
||||
println!("──────────────────────────────────────────────");
|
||||
let windows = copy_el_array(app_el, "AXWindows");
|
||||
println!("Window count: {}", windows.len());
|
||||
|
||||
for (wi, &win) in windows.iter().enumerate() {
|
||||
unsafe { AXUIElementSetMessagingTimeout(win, 5.0) };
|
||||
let title = fetch_repr(win, "AXTitle");
|
||||
println!("\n Window[{}] AXTitle={}", wi, title);
|
||||
dump_all_attrs(win, 2);
|
||||
|
||||
// ── 3. Direct children of window ─────────────────────────────
|
||||
let children = copy_el_array(win, "AXChildren");
|
||||
println!("\n Direct children: {}", children.len());
|
||||
|
||||
for (ci, &child) in children.iter().enumerate() {
|
||||
let role = fetch_repr(child, "AXRole");
|
||||
let subrole = fetch_repr(child, "AXSubrole");
|
||||
let title = fetch_repr(child, "AXTitle");
|
||||
let desc = fetch_repr(child, "AXDescription");
|
||||
let value = fetch_repr(child, "AXValue");
|
||||
println!(
|
||||
"\n Child[{}] role={} subrole={} title={} desc={} value={}",
|
||||
ci, role, subrole, title, desc, value
|
||||
);
|
||||
dump_all_attrs(child, 6);
|
||||
|
||||
// Grandchildren
|
||||
let grandchildren = copy_el_array(child, "AXChildren");
|
||||
println!(" grandchildren: {}", grandchildren.len());
|
||||
for (gci, &gc) in grandchildren.iter().enumerate().take(10) {
|
||||
let r = fetch_repr(gc, "AXRole");
|
||||
let s = fetch_repr(gc, "AXSubrole");
|
||||
let t = fetch_repr(gc, "AXTitle");
|
||||
let d = fetch_repr(gc, "AXDescription");
|
||||
let v = fetch_repr(gc, "AXValue");
|
||||
println!(
|
||||
" GC[{}] role={} subrole={} title={} desc={} value={}",
|
||||
gci, r, s, t, d, v
|
||||
);
|
||||
dump_all_attrs(gc, 8);
|
||||
|
||||
// Great-grandchildren (just roles/names)
|
||||
let ggc = copy_el_array(gc, "AXChildren");
|
||||
for (ggci, &el) in ggc.iter().enumerate().take(6) {
|
||||
let r2 = fetch_repr(el, "AXRole");
|
||||
let t2 = fetch_repr(el, "AXTitle");
|
||||
let d2 = fetch_repr(el, "AXDescription");
|
||||
let v2 = fetch_repr(el, "AXValue");
|
||||
println!(
|
||||
" GGC[{}] role={} title={} desc={} value={}",
|
||||
ggci, r2, t2, d2, v2
|
||||
);
|
||||
dump_all_attrs(el, 10);
|
||||
|
||||
for &gggel in copy_el_array(el, "AXChildren").iter().take(4) {
|
||||
let r3 = fetch_repr(gggel, "AXRole");
|
||||
let t3 = fetch_repr(gggel, "AXTitle");
|
||||
let v3 = fetch_repr(gggel, "AXValue");
|
||||
println!(" GGGC role={} title={} value={}", r3, t3, v3);
|
||||
unsafe { CFRelease(gggel as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(el as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
|
||||
// Only probe first window in depth
|
||||
break;
|
||||
}
|
||||
|
||||
// ── 4. AXCopyMultipleAttributeValues API test ────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 3: AXUIElementCopyMultipleAttributeValues test");
|
||||
println!("──────────────────────────────────────────────");
|
||||
|
||||
// Use first window child for the multi-attr test
|
||||
if !windows.is_empty() {
|
||||
let win = windows[0];
|
||||
let children = copy_el_array(win, "AXChildren");
|
||||
if !children.is_empty() {
|
||||
let el = children[0];
|
||||
let role = fetch_repr(el, "AXRole");
|
||||
println!("Test element: role={}", role);
|
||||
test_multi_attr(el);
|
||||
unsafe { CFRelease(el as CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
// ── 5. AXUIElementCopyElementAtPosition ─────────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 4: AXUIElementCopyElementAtPosition");
|
||||
println!("──────────────────────────────────────────────");
|
||||
for (x, y) in [(400.0f32, 300.0), (600.0, 300.0), (200.0, 400.0)] {
|
||||
let mut pos_el: AXUIElementRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyElementAtPosition(app_el, x, y, &mut pos_el) };
|
||||
if err == 0 && !pos_el.is_null() {
|
||||
let r = fetch_repr(pos_el, "AXRole");
|
||||
let t = fetch_repr(pos_el, "AXTitle");
|
||||
let d = fetch_repr(pos_el, "AXDescription");
|
||||
let v = fetch_repr(pos_el, "AXValue");
|
||||
println!(
|
||||
" ({},{}): role={} title={} desc={} value={}",
|
||||
x, y, r, t, d, v
|
||||
);
|
||||
unsafe { CFRelease(pos_el as CFTypeRef) };
|
||||
} else {
|
||||
println!(" ({},{}): err={}", x, y, err);
|
||||
}
|
||||
}
|
||||
|
||||
// ── 6. Action names ──────────────────────────────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 5: AXUIElementCopyActionNames");
|
||||
println!("──────────────────────────────────────────────");
|
||||
if !windows.is_empty() {
|
||||
let win = windows[0];
|
||||
let children = copy_el_array(win, "AXChildren");
|
||||
for &child in children.iter().take(5) {
|
||||
let role = fetch_repr(child, "AXRole");
|
||||
let actions = copy_action_names(child);
|
||||
println!(" {} => {:?}", role, actions);
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
for &win in &windows {
|
||||
unsafe { CFRelease(win as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(app_el as CFTypeRef) };
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_pid(app_name: &str) -> Option<i32> {
|
||||
let out = std::process::Command::new("pgrep")
|
||||
.arg("-x")
|
||||
.arg(app_name)
|
||||
.output()
|
||||
.ok()?;
|
||||
String::from_utf8_lossy(&out.stdout)
|
||||
.lines()
|
||||
.next()?
|
||||
.trim()
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn dump_all_attrs(el: accessibility_sys::AXUIElementRef, indent: usize) {
|
||||
use accessibility_sys::AXUIElementCopyAttributeNames;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let pad = " ".repeat(indent);
|
||||
let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) };
|
||||
if err != 0 || names_ref.is_null() {
|
||||
println!("{} <AXUIElementCopyAttributeNames err={}>", pad, err);
|
||||
return;
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(names_ref as _) };
|
||||
let names: Vec<String> = arr
|
||||
.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
|
||||
for name in &names {
|
||||
let val = fetch_repr(el, name);
|
||||
println!("{} [attr] {}: {}", pad, name, val);
|
||||
}
|
||||
|
||||
// Also test AXUIElementCopyParameterizedAttributeNames
|
||||
let mut pnames_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let perr = unsafe {
|
||||
accessibility_sys::AXUIElementCopyParameterizedAttributeNames(el, &mut pnames_ref)
|
||||
};
|
||||
if perr == 0 && !pnames_ref.is_null() {
|
||||
let parr = unsafe { CFArray::<CFType>::wrap_under_create_rule(pnames_ref as _) };
|
||||
let pnames: Vec<String> = parr
|
||||
.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
if !pnames.is_empty() {
|
||||
println!("{} [parameterized attrs]: {:?}", pad, pnames);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn fetch_repr(el: accessibility_sys::AXUIElementRef, attr: &str) -> String {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
url::CFURL,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != 0 {
|
||||
return format!("<err:{}>", err);
|
||||
}
|
||||
if value.is_null() {
|
||||
return "<null>".to_string();
|
||||
}
|
||||
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
|
||||
if let Some(s) = cf.downcast::<CFString>() {
|
||||
return format!("\"{}\"", s.to_string());
|
||||
}
|
||||
if let Some(b) = cf.downcast::<CFBoolean>() {
|
||||
return format!("bool:{}", bool::from(b));
|
||||
}
|
||||
if let Some(n) = cf.downcast::<CFNumber>() {
|
||||
if let Some(i) = n.to_i64() {
|
||||
return format!("num:{}", i);
|
||||
}
|
||||
if let Some(f) = n.to_f64() {
|
||||
return format!("num:{:.2}", f);
|
||||
}
|
||||
}
|
||||
// Check if it's an array by type ID
|
||||
let arr_type_id = unsafe { core_foundation_sys::array::CFArrayGetTypeID() };
|
||||
if cf.type_of() == arr_type_id {
|
||||
let arr = unsafe {
|
||||
CFArray::<CFType>::wrap_under_get_rule(
|
||||
cf.as_concrete_TypeRef() as core_foundation_sys::array::CFArrayRef
|
||||
)
|
||||
};
|
||||
return format!("array[{}]", arr.len());
|
||||
}
|
||||
if let Some(url) = cf.downcast::<CFURL>() {
|
||||
return format!("url:{}", url.get_string().to_string());
|
||||
}
|
||||
|
||||
format!("cftype:{}", cf.type_of())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn copy_el_array(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
attr: &str,
|
||||
) -> Vec<accessibility_sys::AXUIElementRef> {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != 0 || value.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(value as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as accessibility_sys::AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(ptr)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn copy_action_names(el: accessibility_sys::AXUIElementRef) -> Vec<String> {
|
||||
use accessibility_sys::AXUIElementCopyActionNames;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let mut ref_: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyActionNames(el, &mut ref_) };
|
||||
if err != 0 || ref_.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(ref_ as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn test_multi_attr(el: accessibility_sys::AXUIElementRef) {
|
||||
use accessibility_sys::{
|
||||
kAXCopyMultipleAttributeOptionStopOnError, AXUIElementCopyMultipleAttributeValues,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let test_attrs = [
|
||||
"AXRole",
|
||||
"AXSubrole",
|
||||
"AXTitle",
|
||||
"AXDescription",
|
||||
"AXValue",
|
||||
"AXEnabled",
|
||||
"AXFocused",
|
||||
"AXHelp",
|
||||
"AXPlaceholderValue",
|
||||
"AXRoleDescription",
|
||||
];
|
||||
|
||||
for &options in &[0u32, kAXCopyMultipleAttributeOptionStopOnError] {
|
||||
let label = if options == 0 {
|
||||
"AllowPartial(0)"
|
||||
} else {
|
||||
"StopOnError(0x1)"
|
||||
};
|
||||
println!(" options={}", label);
|
||||
|
||||
let cf_names: Vec<CFString> = test_attrs.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
let mut result_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyMultipleAttributeValues(
|
||||
el,
|
||||
names_arr.as_concrete_TypeRef(),
|
||||
options,
|
||||
&mut result_ref as *mut _ as *mut _,
|
||||
)
|
||||
};
|
||||
println!(" err={}", err);
|
||||
|
||||
if err == 0 && !result_ref.is_null() {
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(result_ref as _) };
|
||||
println!(
|
||||
" result_count={} (requested {})",
|
||||
arr.len(),
|
||||
test_attrs.len()
|
||||
);
|
||||
for (i, item) in arr.into_iter().enumerate() {
|
||||
let name = test_attrs.get(i).unwrap_or(&"?");
|
||||
let repr = if let Some(s) = item.downcast::<CFString>() {
|
||||
format!("String(\"{}\")", s.to_string())
|
||||
} else if let Some(b) = item.downcast::<CFBoolean>() {
|
||||
format!("Bool({})", bool::from(b))
|
||||
} else if let Some(n) = item.downcast::<CFNumber>() {
|
||||
format!("Number({})", n.to_i64().unwrap_or(-1))
|
||||
} else {
|
||||
format!("Other(type_id={})", item.type_of())
|
||||
};
|
||||
println!(" [{}] {} = {}", i, name, repr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
447
crates/macos/examples/axprobe2.rs
Normal file
447
crates/macos/examples/axprobe2.rs
Normal file
|
|
@ -0,0 +1,447 @@
|
|||
//! Deep probe of AXOutline sidebar rows and AXBrowser columns in Finder.
|
||||
//! Run: cargo run -p agent-desktop-macos --bin axprobe2
|
||||
|
||||
fn main() {
|
||||
#[cfg(target_os = "macos")]
|
||||
run();
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
eprintln!("macOS only");
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn run() {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let pid = find_pid("Finder").unwrap_or_else(|| {
|
||||
eprintln!("Finder not running");
|
||||
std::process::exit(1);
|
||||
});
|
||||
|
||||
let app_el = unsafe { AXUIElementCreateApplication(pid) };
|
||||
unsafe { AXUIElementSetMessagingTimeout(app_el, 5.0) };
|
||||
|
||||
let windows = copy_el_array(app_el, "AXWindows");
|
||||
let win = match windows.first() {
|
||||
Some(&w) => w,
|
||||
None => {
|
||||
eprintln!("No windows");
|
||||
return;
|
||||
}
|
||||
};
|
||||
unsafe { AXUIElementSetMessagingTimeout(win, 5.0) };
|
||||
|
||||
println!("=== Finder Window: {} ===\n", fetch_str(win, "AXTitle"));
|
||||
|
||||
// ── Sidebar AXOutline → AXRow children ─────────────────────────────
|
||||
println!("────────────────────────────────────────");
|
||||
println!("PART 1: Sidebar AXOutline rows (6 levels deep)");
|
||||
println!("────────────────────────────────────────");
|
||||
|
||||
let sidebar_outline = find_by_desc(win, "sidebar", 0);
|
||||
if let Some(outline) = sidebar_outline {
|
||||
let outline_role = fetch_str(outline, "AXRole");
|
||||
let outline_desc = fetch_str(outline, "AXDescription");
|
||||
println!("Found: role={} desc={}", outline_role, outline_desc);
|
||||
println!("AXRows count: {}", count_attr(outline, "AXRows"));
|
||||
println!("AXChildren count: {}", count_attr(outline, "AXChildren"));
|
||||
|
||||
let rows = copy_el_array(outline, "AXRows");
|
||||
println!("Probing first 8 AXRows (all attributes + full children):");
|
||||
for (i, &row) in rows.iter().enumerate().take(8) {
|
||||
dump_element(row, i, 2);
|
||||
}
|
||||
for &row in &rows {
|
||||
unsafe { CFRelease(row as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(outline as CFTypeRef) };
|
||||
} else {
|
||||
println!("Could not find sidebar outline");
|
||||
}
|
||||
|
||||
// ── AXBrowser column view ───────────────────────────────────────────
|
||||
println!("\n────────────────────────────────────────");
|
||||
println!("PART 2: AXBrowser column view (AXColumns)");
|
||||
println!("────────────────────────────────────────");
|
||||
|
||||
let browser = find_by_role(win, "AXBrowser", 0);
|
||||
if let Some(br) = browser {
|
||||
println!("Found AXBrowser: desc={}", fetch_str(br, "AXDescription"));
|
||||
println!("AXColumns: {}", count_attr(br, "AXColumns"));
|
||||
println!("AXChildren: {}", count_attr(br, "AXChildren"));
|
||||
println!("AXVisibleColumns: {}", count_attr(br, "AXVisibleColumns"));
|
||||
|
||||
// Probe each column
|
||||
let columns = copy_el_array(br, "AXColumns");
|
||||
println!("\nProbing AXColumns:");
|
||||
for (ci, &col) in columns.iter().enumerate() {
|
||||
println!("\n Column[{}]:", ci);
|
||||
dump_all_attrs(col, 4);
|
||||
let col_rows = copy_el_array(col, "AXRows");
|
||||
let col_children = copy_el_array(col, "AXChildren");
|
||||
println!(
|
||||
" AXRows={} AXChildren={}",
|
||||
col_rows.len(),
|
||||
col_children.len()
|
||||
);
|
||||
|
||||
// Probe first few rows
|
||||
for (ri, &row) in col_rows.iter().enumerate().take(6) {
|
||||
dump_element(row, ri, 6);
|
||||
}
|
||||
for &row in &col_rows {
|
||||
unsafe { CFRelease(row as CFTypeRef) };
|
||||
}
|
||||
for &child in &col_children {
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(col as CFTypeRef) };
|
||||
}
|
||||
|
||||
// Also try AXChildren path
|
||||
println!("\nAXChildren of browser:");
|
||||
let br_children = copy_el_array(br, "AXChildren");
|
||||
for (ci, &child) in br_children.iter().enumerate() {
|
||||
let r = fetch_str(child, "AXRole");
|
||||
let d = fetch_str(child, "AXDescription");
|
||||
println!(" Child[{}] role={} desc={}", ci, r, d);
|
||||
// One more level
|
||||
let gchildren = copy_el_array(child, "AXChildren");
|
||||
for (gi, &gc) in gchildren.iter().enumerate().take(4) {
|
||||
let r2 = fetch_str(gc, "AXRole");
|
||||
let d2 = fetch_str(gc, "AXDescription");
|
||||
println!(" GC[{}] role={} desc={}", gi, r2, d2);
|
||||
// Column rows inside scroll area
|
||||
let sc_children = copy_el_array(gc, "AXChildren");
|
||||
for (sci, &sc) in sc_children.iter().enumerate().take(4) {
|
||||
let r3 = fetch_str(sc, "AXRole");
|
||||
let d3 = fetch_str(sc, "AXDescription");
|
||||
println!(" SC[{}] role={} desc={}", sci, r3, d3);
|
||||
let sc2 = copy_el_array(sc, "AXRows");
|
||||
for (s2i, &s2) in sc2.iter().enumerate().take(6) {
|
||||
dump_element(s2, s2i, 10);
|
||||
}
|
||||
for &s2 in &sc2 {
|
||||
unsafe { CFRelease(s2 as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(sc as CFTypeRef) };
|
||||
}
|
||||
for &sc in &sc_children {
|
||||
unsafe { CFRelease(sc as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
for &gc in &gchildren {
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
|
||||
unsafe { CFRelease(br as CFTypeRef) };
|
||||
} else {
|
||||
println!("Could not find AXBrowser");
|
||||
}
|
||||
|
||||
// ── AXCopyMultipleAttributeValues on an AXRow ───────────────────────
|
||||
println!("\n────────────────────────────────────────");
|
||||
println!("PART 3: AXUIElementCopyMultipleAttributeValues on AXRow");
|
||||
println!("────────────────────────────────────────");
|
||||
|
||||
let sidebar2 = find_by_desc(win, "sidebar", 0);
|
||||
if let Some(outline) = sidebar2 {
|
||||
let rows = copy_el_array(outline, "AXRows");
|
||||
if let Some(&row) = rows.first() {
|
||||
println!("Testing on first sidebar AXRow:");
|
||||
test_multi_attr_extended(row);
|
||||
}
|
||||
for &row in &rows {
|
||||
unsafe { CFRelease(row as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(outline as CFTypeRef) };
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
for &win2 in &windows {
|
||||
unsafe { CFRelease(win2 as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(app_el as CFTypeRef) };
|
||||
}
|
||||
|
||||
// ── Deep element dump ────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn dump_element(el: accessibility_sys::AXUIElementRef, idx: usize, indent: usize) {
|
||||
use core_foundation::base::{CFRelease, CFTypeRef};
|
||||
let pad = " ".repeat(indent);
|
||||
let role = fetch_str(el, "AXRole");
|
||||
let sub = fetch_str(el, "AXSubrole");
|
||||
let title = fetch_str(el, "AXTitle");
|
||||
let desc = fetch_str(el, "AXDescription");
|
||||
let val = fetch_str(el, "AXValue");
|
||||
let help = fetch_str(el, "AXHelp");
|
||||
println!(
|
||||
"{}[{}] role={} subrole={} title={} desc={} value={} help={}",
|
||||
pad, idx, role, sub, title, desc, val, help
|
||||
);
|
||||
dump_all_attrs(el, indent + 2);
|
||||
|
||||
let children = copy_el_array(el, "AXChildren");
|
||||
for (ci, &child) in children.iter().enumerate() {
|
||||
let cr = fetch_str(child, "AXRole");
|
||||
let ct = fetch_str(child, "AXTitle");
|
||||
let cd = fetch_str(child, "AXDescription");
|
||||
let cv = fetch_str(child, "AXValue");
|
||||
println!(
|
||||
"{} child[{}] role={} title={} desc={} value={}",
|
||||
pad, ci, cr, ct, cd, cv
|
||||
);
|
||||
dump_all_attrs(child, indent + 4);
|
||||
|
||||
let gchildren = copy_el_array(child, "AXChildren");
|
||||
for (gi, &gc) in gchildren.iter().enumerate() {
|
||||
let gr = fetch_str(gc, "AXRole");
|
||||
let gt = fetch_str(gc, "AXTitle");
|
||||
let gd = fetch_str(gc, "AXDescription");
|
||||
let gv = fetch_str(gc, "AXValue");
|
||||
println!(
|
||||
"{} gc[{}] role={} title={} desc={} value={}",
|
||||
pad, gi, gr, gt, gd, gv
|
||||
);
|
||||
dump_all_attrs(gc, indent + 6);
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn test_multi_attr_extended(el: accessibility_sys::AXUIElementRef) {
|
||||
use accessibility_sys::{
|
||||
AXUIElementCopyAttributeNames, AXUIElementCopyMultipleAttributeValues,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
// First, get ALL attribute names for this element
|
||||
let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) };
|
||||
if err != 0 || names_ref.is_null() {
|
||||
return;
|
||||
}
|
||||
let name_arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(names_ref as _) };
|
||||
let all_names: Vec<String> = name_arr
|
||||
.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
|
||||
println!("Available attributes on AXRow: {:?}", all_names);
|
||||
|
||||
// Batch fetch all of them
|
||||
let cf_names: Vec<CFString> = all_names.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr2 = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
let mut result: CFTypeRef = std::ptr::null_mut();
|
||||
let err2 = unsafe {
|
||||
AXUIElementCopyMultipleAttributeValues(
|
||||
el,
|
||||
names_arr2.as_concrete_TypeRef(),
|
||||
0,
|
||||
&mut result as *mut _ as *mut _,
|
||||
)
|
||||
};
|
||||
println!("AXUIElementCopyMultipleAttributeValues err={}", err2);
|
||||
if err2 == 0 && !result.is_null() {
|
||||
let res_arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(result as _) };
|
||||
for (i, item) in res_arr.into_iter().enumerate() {
|
||||
let name = all_names.get(i).map(|s| s.as_str()).unwrap_or("?");
|
||||
let repr = if let Some(s) = item.downcast::<CFString>() {
|
||||
format!("String(\"{}\")", s.to_string())
|
||||
} else if let Some(b) = item.downcast::<CFBoolean>() {
|
||||
format!("Bool({})", bool::from(b))
|
||||
} else if let Some(n) = item.downcast::<CFNumber>() {
|
||||
format!("Number({})", n.to_i64().unwrap_or(-1))
|
||||
} else {
|
||||
format!("Other(type_id={})", item.type_of())
|
||||
};
|
||||
println!(" [{}] {} = {}", i, name, repr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_pid(name: &str) -> Option<i32> {
|
||||
let out = std::process::Command::new("pgrep")
|
||||
.arg("-x")
|
||||
.arg(name)
|
||||
.output()
|
||||
.ok()?;
|
||||
String::from_utf8_lossy(&out.stdout)
|
||||
.lines()
|
||||
.next()?
|
||||
.trim()
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn count_attr(el: accessibility_sys::AXUIElementRef, attr: &str) -> usize {
|
||||
copy_el_array(el, attr).len()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn fetch_str(el: accessibility_sys::AXUIElementRef, attr: &str) -> String {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != 0 {
|
||||
return format!("<err:{}>", err);
|
||||
}
|
||||
if value.is_null() {
|
||||
return "<null>".to_string();
|
||||
}
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
if let Some(s) = cf.downcast::<CFString>() {
|
||||
return format!("\"{}\"", s.to_string());
|
||||
}
|
||||
if let Some(b) = cf.downcast::<CFBoolean>() {
|
||||
return format!("bool:{}", bool::from(b));
|
||||
}
|
||||
if let Some(n) = cf.downcast::<CFNumber>() {
|
||||
return format!("num:{}", n.to_i64().unwrap_or(-1));
|
||||
}
|
||||
format!("cftype:{}", cf.type_of())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn dump_all_attrs(el: accessibility_sys::AXUIElementRef, indent: usize) {
|
||||
use accessibility_sys::AXUIElementCopyAttributeNames;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let pad = " ".repeat(indent);
|
||||
let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) };
|
||||
if err != 0 || names_ref.is_null() {
|
||||
return;
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(names_ref as _) };
|
||||
let names: Vec<String> = arr
|
||||
.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
for name in &names {
|
||||
let val = fetch_str(el, name);
|
||||
println!("{}{}: {}", pad, name, val);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn copy_el_array(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
attr: &str,
|
||||
) -> Vec<accessibility_sys::AXUIElementRef> {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != 0 || value.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(value as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as accessibility_sys::AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(ptr)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Find first element with given AXDescription recursively
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_by_desc(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
desc: &str,
|
||||
depth: u32,
|
||||
) -> Option<accessibility_sys::AXUIElementRef> {
|
||||
use core_foundation::base::{CFRetain, CFTypeRef};
|
||||
if depth > 8 {
|
||||
return None;
|
||||
}
|
||||
let d = fetch_str(el, "AXDescription");
|
||||
if d == format!("\"{}\"", desc) {
|
||||
unsafe { CFRetain(el as CFTypeRef) };
|
||||
return Some(el);
|
||||
}
|
||||
for child in copy_el_array(el, "AXChildren") {
|
||||
if let Some(found) = find_by_desc(child, desc, depth + 1) {
|
||||
unsafe { core_foundation::base::CFRelease(child as CFTypeRef) };
|
||||
return Some(found);
|
||||
}
|
||||
unsafe { core_foundation::base::CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find first element with given AXRole recursively
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_by_role(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
role: &str,
|
||||
depth: u32,
|
||||
) -> Option<accessibility_sys::AXUIElementRef> {
|
||||
use core_foundation::base::{CFRelease, CFRetain, CFTypeRef};
|
||||
if depth > 8 {
|
||||
return None;
|
||||
}
|
||||
let r = fetch_str(el, "AXRole");
|
||||
if r == format!("\"{}\"", role) {
|
||||
unsafe { CFRetain(el as CFTypeRef) };
|
||||
return Some(el);
|
||||
}
|
||||
for child in copy_el_array(el, "AXChildren") {
|
||||
if let Some(found) = find_by_role(child, role, depth + 1) {
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
return Some(found);
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
None
|
||||
}
|
||||
182
crates/macos/src/action_extras.rs
Normal file
182
crates/macos/src/action_extras.rs
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
#[cfg(target_os = "macos")]
|
||||
use agent_desktop_core::error::{AdapterError, ErrorCode};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use crate::tree::AXElement;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterError> {
|
||||
use crate::actions::{ax_press_or_fail, ax_set_value, element_role};
|
||||
|
||||
let role = element_role(el);
|
||||
match role.as_deref() {
|
||||
Some("combobox") => {
|
||||
ax_set_value(el, value)?;
|
||||
}
|
||||
Some("popupbutton") | Some("menubutton") => {
|
||||
ax_press_or_fail(el, "select (open popup)")?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
if !find_and_press_menu_item(el, value) {
|
||||
press_escape(el);
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ElementNotFound,
|
||||
format!("No menu item matching '{value}' found"),
|
||||
)
|
||||
.with_suggestion(
|
||||
"Use 'click' to open the menu, then 'snapshot' to see available options.",
|
||||
));
|
||||
}
|
||||
}
|
||||
Some("list") | Some("table") | Some("outline") => {
|
||||
if !select_child_by_name(el, value) {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ElementNotFound,
|
||||
format!("No child matching '{value}' found in list"),
|
||||
)
|
||||
.with_suggestion("Use 'find --role' to discover available items."));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if ax_set_value(el, value).is_err() {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
format!(
|
||||
"Select not supported on role '{}'",
|
||||
role.as_deref().unwrap_or("unknown")
|
||||
),
|
||||
)
|
||||
.with_suggestion("Use 'click' or 'set-value' instead."));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_and_press_menu_item(el: &AXElement, target_value: &str) -> bool {
|
||||
use accessibility_sys::{kAXPressAction, AXUIElementPerformAction};
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
|
||||
for child in &children {
|
||||
let title = crate::tree::copy_string_attr(child, "AXTitle");
|
||||
if let Some(t) = &title {
|
||||
if t.eq_ignore_ascii_case(target_value) {
|
||||
let action = CFString::new(kAXPressAction);
|
||||
unsafe { AXUIElementPerformAction(child.0, action.as_concrete_TypeRef()) };
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if find_and_press_menu_item(child, target_value) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn press_escape(el: &AXElement) {
|
||||
use accessibility_sys::AXUIElementPerformAction;
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let cancel = CFString::new("AXCancel");
|
||||
unsafe { AXUIElementPerformAction(el.0, cancel.as_concrete_TypeRef()) };
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn select_child_by_name(el: &AXElement, name: &str) -> bool {
|
||||
use accessibility_sys::{kAXPressAction, AXUIElementPerformAction};
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
|
||||
for child in &children {
|
||||
let child_name = crate::tree::copy_string_attr(child, "AXTitle")
|
||||
.or_else(|| crate::tree::copy_string_attr(child, "AXDescription"));
|
||||
if let Some(n) = &child_name {
|
||||
if n.eq_ignore_ascii_case(name) {
|
||||
let action = CFString::new(kAXPressAction);
|
||||
unsafe { AXUIElementPerformAction(child.0, action.as_concrete_TypeRef()) };
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) fn ax_scroll(
|
||||
el: &AXElement,
|
||||
direction: &agent_desktop_core::action::Direction,
|
||||
amount: u32,
|
||||
) -> Result<(), AdapterError> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
|
||||
use agent_desktop_core::action::Direction;
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let scroll_area = find_scroll_area(el);
|
||||
let target = scroll_area.as_ref().unwrap_or(el);
|
||||
|
||||
let (bar_orientation, action_name) = match direction {
|
||||
Direction::Down => ("AXVerticalScrollBar", "AXIncrement"),
|
||||
Direction::Up => ("AXVerticalScrollBar", "AXDecrement"),
|
||||
Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"),
|
||||
Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"),
|
||||
};
|
||||
|
||||
if let Some(scroll_bar) = get_scroll_bar(target, bar_orientation) {
|
||||
let ax_action = CFString::new(action_name);
|
||||
for _ in 0..amount {
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(scroll_bar.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("{action_name} on scroll bar failed (err={err})"),
|
||||
));
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let scroll_action_name = match direction {
|
||||
Direction::Down => "AXScrollDownByPage",
|
||||
Direction::Up => "AXScrollUpByPage",
|
||||
Direction::Right => "AXScrollRightByPage",
|
||||
Direction::Left => "AXScrollLeftByPage",
|
||||
};
|
||||
|
||||
if crate::actions::has_ax_action(target, scroll_action_name) {
|
||||
let ax_action = CFString::new(scroll_action_name);
|
||||
for _ in 0..amount {
|
||||
unsafe { AXUIElementPerformAction(target.0, ax_action.as_concrete_TypeRef()) };
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
"No scroll bar or scroll action found on this element",
|
||||
)
|
||||
.with_suggestion("Element may not be scrollable, or try scrolling the parent container."))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_scroll_area(el: &AXElement) -> Option<AXElement> {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
|
||||
let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?;
|
||||
if role == "AXScrollArea" {
|
||||
return Some(AXElement(el.0));
|
||||
}
|
||||
let parent = crate::tree::copy_element_attr(el, "AXParent")?;
|
||||
let parent_role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute)?;
|
||||
if parent_role == "AXScrollArea" {
|
||||
return Some(parent);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_scroll_bar(scroll_area: &AXElement, bar_attr: &str) -> Option<AXElement> {
|
||||
crate::tree::copy_element_attr(scroll_area, bar_attr)
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
use agent_desktop_core::{
|
||||
action::{Action, ActionResult},
|
||||
error::AdapterError,
|
||||
error::{AdapterError, ErrorCode},
|
||||
};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
|
|
@ -9,42 +9,75 @@ mod imp {
|
|||
use crate::tree::AXElement;
|
||||
use accessibility_sys::{
|
||||
kAXErrorSuccess, kAXFocusedAttribute, kAXPressAction, kAXValueAttribute,
|
||||
AXUIElementPerformAction, AXUIElementSetAttributeValue,
|
||||
AXUIElementCopyActionNames, AXUIElementPerformAction, AXUIElementSetAttributeValue,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
boolean::CFBoolean,
|
||||
string::CFString,
|
||||
};
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
|
||||
pub fn perform_action(el: &AXElement, action: &Action) -> Result<ActionResult, AdapterError> {
|
||||
let action_name = action_label(action);
|
||||
let label = action_label(action);
|
||||
match action {
|
||||
Action::Click | Action::DoubleClick | Action::RightClick | Action::Toggle => {
|
||||
let ax_action = CFString::new(kAXPressAction);
|
||||
let err = unsafe {
|
||||
AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef())
|
||||
};
|
||||
Action::Click => {
|
||||
ax_press_or_fail(el, "click")?;
|
||||
}
|
||||
|
||||
Action::DoubleClick => {
|
||||
let open_action = CFString::new("AXOpen");
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(el.0, open_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("AXPress failed with error {err}"),
|
||||
));
|
||||
ax_press_or_fail(el, "double-click (first press)")?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_press_or_fail(el, "double-click (second press)")?;
|
||||
}
|
||||
}
|
||||
Action::SetValue(val) => {
|
||||
let cf_attr = CFString::new(kAXValueAttribute);
|
||||
let cf_val = CFString::new(val);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
cf_val.as_CFTypeRef(),
|
||||
|
||||
Action::RightClick => {
|
||||
let ax_action = CFString::new("AXShowMenu");
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("AXShowMenu failed (err={err})"),
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("SetValue failed with error {err}"),
|
||||
));
|
||||
.with_suggestion("Element may not support context menus"));
|
||||
}
|
||||
}
|
||||
|
||||
Action::Toggle => {
|
||||
let role = element_role(el);
|
||||
let toggle_roles = [
|
||||
"checkbox",
|
||||
"switch",
|
||||
"radiobutton",
|
||||
"togglebutton",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
];
|
||||
if !toggle_roles.iter().any(|r| role.as_deref() == Some(*r)) {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
format!(
|
||||
"Toggle not supported on role '{}'",
|
||||
role.as_deref().unwrap_or("unknown")
|
||||
),
|
||||
)
|
||||
.with_suggestion(
|
||||
"Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.",
|
||||
));
|
||||
}
|
||||
ax_press_or_fail(el, "toggle")?;
|
||||
}
|
||||
|
||||
Action::SetValue(val) => {
|
||||
ax_set_value(el, val)?;
|
||||
}
|
||||
|
||||
Action::SetFocus => {
|
||||
let cf_attr = CFString::new(kAXFocusedAttribute);
|
||||
let err = unsafe {
|
||||
|
|
@ -56,74 +89,202 @@ mod imp {
|
|||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("SetFocus failed with error {err}"),
|
||||
ErrorCode::ActionFailed,
|
||||
format!("SetFocus failed (err={err})"),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Action::TypeText(text) => {
|
||||
let cf_attr = CFString::new(kAXFocusedAttribute);
|
||||
unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
crate::input::synthesize_text(text)?;
|
||||
}
|
||||
|
||||
Action::PressKey(combo) => {
|
||||
crate::input::synthesize_key(combo)?;
|
||||
}
|
||||
|
||||
Action::Expand => {
|
||||
if !has_ax_action(el, "AXExpand") {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
"This element doesn't support expand",
|
||||
)
|
||||
.with_suggestion("Try 'click' to open it instead."));
|
||||
}
|
||||
let ax_action = CFString::new("AXExpand");
|
||||
let err = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("AXExpand failed with error {err}"),
|
||||
ErrorCode::ActionFailed,
|
||||
format!("AXExpand failed (err={err})"),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Action::Collapse => {
|
||||
if !has_ax_action(el, "AXCollapse") {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
"This element doesn't support collapse",
|
||||
)
|
||||
.with_suggestion("Try 'click' to close it instead."));
|
||||
}
|
||||
let ax_action = CFString::new("AXCollapse");
|
||||
let err = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("AXCollapse failed with error {err}"),
|
||||
ErrorCode::ActionFailed,
|
||||
format!("AXCollapse failed (err={err})"),
|
||||
));
|
||||
}
|
||||
}
|
||||
Action::Select(_) => {
|
||||
let ax_action = CFString::new(kAXPressAction);
|
||||
let err = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("AXPress (select) failed with error {err}"),
|
||||
));
|
||||
}
|
||||
|
||||
Action::Select(value) => {
|
||||
crate::action_extras::select_value(el, value)?;
|
||||
}
|
||||
|
||||
Action::Scroll(direction, amount) => {
|
||||
use core_graphics::{
|
||||
event::{CGEvent, CGEventTapLocation, ScrollEventUnit},
|
||||
event_source::{CGEventSource, CGEventSourceStateID},
|
||||
};
|
||||
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
|
||||
.map_err(|_| AdapterError::internal("CGEventSource failed"))?;
|
||||
let (dx, dy) = match direction {
|
||||
agent_desktop_core::action::Direction::Up => (0i32, *amount as i32),
|
||||
agent_desktop_core::action::Direction::Down => (0i32, -(*amount as i32)),
|
||||
agent_desktop_core::action::Direction::Left => (-(*amount as i32), 0i32),
|
||||
agent_desktop_core::action::Direction::Right => (*amount as i32, 0i32),
|
||||
};
|
||||
let event = CGEvent::new_scroll_event(
|
||||
source,
|
||||
ScrollEventUnit::LINE,
|
||||
2,
|
||||
dy,
|
||||
dx,
|
||||
0,
|
||||
)
|
||||
.map_err(|_| AdapterError::internal("CGEvent scroll failed"))?;
|
||||
event.post(CGEventTapLocation::HID);
|
||||
crate::action_extras::ax_scroll(el, direction, *amount)?;
|
||||
}
|
||||
|
||||
Action::Check => {
|
||||
check_uncheck(el, true)?;
|
||||
}
|
||||
|
||||
Action::Uncheck => {
|
||||
check_uncheck(el, false)?;
|
||||
}
|
||||
|
||||
Action::TripleClick => {
|
||||
ax_press_or_fail(el, "triple-click (1st)")?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
ax_press_or_fail(el, "triple-click (2nd)")?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
ax_press_or_fail(el, "triple-click (3rd)")?;
|
||||
}
|
||||
|
||||
Action::ScrollTo => {
|
||||
let ax_action = CFString::new("AXScrollToVisible");
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("AXScrollToVisible failed (err={err})"),
|
||||
)
|
||||
.with_suggestion("Element may not be inside a scrollable area"));
|
||||
}
|
||||
}
|
||||
|
||||
Action::Clear => {
|
||||
ax_set_value(el, "")?;
|
||||
}
|
||||
|
||||
Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
format!(
|
||||
"{} requires adapter-level handling, not element action",
|
||||
label
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
_ => {
|
||||
return Err(AdapterError::not_supported(&label));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ActionResult::new(action_name))
|
||||
Ok(ActionResult::new(label))
|
||||
}
|
||||
|
||||
pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> {
|
||||
let action = CFString::new(kAXPressAction);
|
||||
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("{context}: AXPress failed (err={err})"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> {
|
||||
let cf_attr = CFString::new(kAXValueAttribute);
|
||||
let cf_val = CFString::new(val);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("SetValue failed (err={err})"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn element_role(el: &AXElement) -> Option<String> {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
crate::tree::copy_string_attr(el, kAXRoleAttribute)
|
||||
.map(|r| crate::roles::ax_role_to_str(&r).to_string())
|
||||
}
|
||||
|
||||
pub fn has_ax_action(el: &AXElement, action_name: &str) -> bool {
|
||||
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
|
||||
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
|
||||
if err != kAXErrorSuccess || actions_ref.is_null() {
|
||||
return false;
|
||||
}
|
||||
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
|
||||
let target = CFString::new(action_name);
|
||||
for i in 0..actions.len() {
|
||||
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
|
||||
if name == target {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn check_uncheck(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> {
|
||||
let role = element_role(el);
|
||||
let valid_roles = [
|
||||
"checkbox",
|
||||
"switch",
|
||||
"radiobutton",
|
||||
"togglebutton",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
];
|
||||
if !valid_roles.iter().any(|r| role.as_deref() == Some(*r)) {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
format!(
|
||||
"check/uncheck not supported on role '{}'",
|
||||
role.as_deref().unwrap_or("unknown")
|
||||
),
|
||||
)
|
||||
.with_suggestion("Only works on checkboxes, switches, and radio buttons."));
|
||||
}
|
||||
let current = crate::tree::copy_string_attr(el, "AXValue");
|
||||
let is_checked = current.as_deref() == Some("1");
|
||||
if is_checked == want_checked {
|
||||
return Ok(());
|
||||
}
|
||||
ax_press_or_fail(el, if want_checked { "check" } else { "uncheck" })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -139,20 +300,33 @@ mod imp {
|
|||
|
||||
pub use imp::perform_action;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) use imp::{ax_press_or_fail, ax_set_value, element_role, has_ax_action};
|
||||
|
||||
fn action_label(action: &Action) -> String {
|
||||
match action {
|
||||
Action::Click => "click",
|
||||
Action::DoubleClick => "double_click",
|
||||
Action::RightClick => "right_click",
|
||||
Action::TripleClick => "triple_click",
|
||||
Action::SetValue(_) => "set_value",
|
||||
Action::SetFocus => "set_focus",
|
||||
Action::Expand => "expand",
|
||||
Action::Collapse => "collapse",
|
||||
Action::Select(_) => "select",
|
||||
Action::Toggle => "toggle",
|
||||
Action::Check => "check",
|
||||
Action::Uncheck => "uncheck",
|
||||
Action::Scroll(_, _) => "scroll",
|
||||
Action::ScrollTo => "scroll_to",
|
||||
Action::PressKey(_) => "press_key",
|
||||
Action::KeyDown(_) => "key_down",
|
||||
Action::KeyUp(_) => "key_up",
|
||||
Action::TypeText(_) => "type_text",
|
||||
Action::Clear => "clear",
|
||||
Action::Hover => "hover",
|
||||
Action::Drag(_) => "drag",
|
||||
_ => "unknown",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
use agent_desktop_core::{
|
||||
action::{Action, ActionResult},
|
||||
action::{Action, ActionResult, DragParams, MouseEvent, WindowOp},
|
||||
adapter::{
|
||||
ImageBuffer, NativeHandle, PermissionStatus, PlatformAdapter, ScreenshotTarget, TreeOptions,
|
||||
WindowFilter,
|
||||
ImageBuffer, NativeHandle, PermissionStatus, PlatformAdapter, ScreenshotTarget,
|
||||
SnapshotSurface, TreeOptions, WindowFilter,
|
||||
},
|
||||
error::AdapterError,
|
||||
node::{AccessibilityNode, AppInfo, WindowInfo},
|
||||
node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo},
|
||||
refs::RefEntry,
|
||||
};
|
||||
use rustc_hash::FxHashSet;
|
||||
|
|
@ -34,10 +34,22 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
win: &WindowInfo,
|
||||
opts: &TreeOptions,
|
||||
) -> Result<AccessibilityNode, AdapterError> {
|
||||
let el = match opts.surface {
|
||||
SnapshotSurface::Window => crate::tree::window_element_for(win.pid, &win.title),
|
||||
SnapshotSurface::Focused => crate::surfaces::focused_surface_for_pid(win.pid)
|
||||
.ok_or_else(|| AdapterError::internal("No focused surface found"))?,
|
||||
SnapshotSurface::Menu => crate::surfaces::menu_element_for_pid(win.pid)
|
||||
.ok_or_else(|| AdapterError::element_not_found("No open context menu"))?,
|
||||
SnapshotSurface::Sheet => crate::surfaces::sheet_for_pid(win.pid)
|
||||
.ok_or_else(|| AdapterError::element_not_found("No open sheet"))?,
|
||||
SnapshotSurface::Popover => crate::surfaces::popover_for_pid(win.pid)
|
||||
.ok_or_else(|| AdapterError::element_not_found("No visible popover"))?,
|
||||
SnapshotSurface::Alert => crate::surfaces::alert_for_pid(win.pid)
|
||||
.ok_or_else(|| AdapterError::element_not_found("No open alert or dialog"))?,
|
||||
};
|
||||
let mut visited = FxHashSet::default();
|
||||
let el = crate::tree::element_for_pid(win.pid);
|
||||
crate::tree::build_subtree(&el, 0, opts.max_depth, opts.include_bounds, &mut visited)
|
||||
.ok_or_else(|| AdapterError::internal("Empty AX tree for window"))
|
||||
.ok_or_else(|| AdapterError::internal("Empty AX tree for surface"))
|
||||
}
|
||||
|
||||
fn execute_action(
|
||||
|
|
@ -64,8 +76,8 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
crate::app_ops::focus_window_impl(win)
|
||||
}
|
||||
|
||||
fn launch_app(&self, id: &str, wait: bool) -> Result<WindowInfo, AdapterError> {
|
||||
crate::app_ops::launch_app_impl(id, wait)
|
||||
fn launch_app(&self, id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
|
||||
crate::app_ops::launch_app_impl(id, timeout_ms)
|
||||
}
|
||||
|
||||
fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError> {
|
||||
|
|
@ -74,15 +86,7 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
|
||||
fn screenshot(&self, target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> {
|
||||
match target {
|
||||
ScreenshotTarget::Window(id) => {
|
||||
let window_id = id.parse::<u32>().map_err(|_| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::InvalidArgs,
|
||||
format!("Invalid window ID: {id}"),
|
||||
)
|
||||
})?;
|
||||
crate::screenshot::capture_window(window_id)
|
||||
}
|
||||
ScreenshotTarget::Window(pid) => crate::screenshot::capture_app(pid),
|
||||
ScreenshotTarget::Screen(idx) => crate::screenshot::capture_screen(idx),
|
||||
ScreenshotTarget::FullScreen => crate::screenshot::capture_screen(0),
|
||||
}
|
||||
|
|
@ -96,24 +100,99 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
crate::clipboard::set(text)
|
||||
}
|
||||
|
||||
fn press_key_for_app(
|
||||
&self,
|
||||
app_name: &str,
|
||||
combo: &agent_desktop_core::action::KeyCombo,
|
||||
) -> Result<agent_desktop_core::action::ActionResult, AdapterError> {
|
||||
crate::key_dispatch::press_for_app_impl(app_name, combo)
|
||||
}
|
||||
|
||||
fn wait_for_menu(&self, pid: i32, open: bool, timeout_ms: u64) -> Result<(), AdapterError> {
|
||||
crate::wait::wait_for_menu(pid, open, timeout_ms)
|
||||
}
|
||||
|
||||
fn list_surfaces(&self, pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError> {
|
||||
Ok(crate::surfaces::list_surfaces_for_pid(pid))
|
||||
}
|
||||
|
||||
fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError> {
|
||||
let filter = WindowFilter { focused_only: true, app: None };
|
||||
let filter = WindowFilter {
|
||||
focused_only: true,
|
||||
app: None,
|
||||
};
|
||||
let windows = self.list_windows(&filter)?;
|
||||
Ok(windows.into_iter().next())
|
||||
}
|
||||
|
||||
fn get_live_value(&self, handle: &NativeHandle) -> Result<Option<String>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use crate::tree::AXElement;
|
||||
use accessibility_sys::kAXValueAttribute;
|
||||
use std::mem::ManuallyDrop;
|
||||
let el = ManuallyDrop::new(AXElement(
|
||||
handle.as_raw() as accessibility_sys::AXUIElementRef
|
||||
));
|
||||
Ok(crate::tree::copy_string_attr(&el, kAXValueAttribute))
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
Err(AdapterError::not_supported("get_live_value"))
|
||||
}
|
||||
|
||||
fn get_element_bounds(&self, handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use crate::tree::AXElement;
|
||||
use std::mem::ManuallyDrop;
|
||||
let el = ManuallyDrop::new(AXElement(
|
||||
handle.as_raw() as accessibility_sys::AXUIElementRef
|
||||
));
|
||||
Ok(crate::tree::read_bounds(&el))
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
let _ = handle;
|
||||
Err(AdapterError::not_supported("get_element_bounds"))
|
||||
}
|
||||
}
|
||||
|
||||
fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> {
|
||||
crate::window_ops::execute(win, op)
|
||||
}
|
||||
|
||||
fn mouse_event(&self, event: MouseEvent) -> Result<(), AdapterError> {
|
||||
crate::mouse::synthesize_mouse(event)
|
||||
}
|
||||
|
||||
fn drag(&self, params: DragParams) -> Result<(), AdapterError> {
|
||||
crate::mouse::synthesize_drag(params)
|
||||
}
|
||||
|
||||
fn clear_clipboard(&self) -> Result<(), AdapterError> {
|
||||
crate::clipboard::clear()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn execute_action_impl(handle: &NativeHandle, action: Action) -> Result<ActionResult, AdapterError> {
|
||||
fn execute_action_impl(
|
||||
handle: &NativeHandle,
|
||||
action: Action,
|
||||
) -> Result<ActionResult, AdapterError> {
|
||||
use crate::tree::AXElement;
|
||||
use std::mem::ManuallyDrop;
|
||||
|
||||
let el = ManuallyDrop::new(AXElement(handle.as_raw() as accessibility_sys::AXUIElementRef));
|
||||
crate::actions::perform_action(&*el, &action)
|
||||
let el = ManuallyDrop::new(AXElement(
|
||||
handle.as_raw() as accessibility_sys::AXUIElementRef
|
||||
));
|
||||
crate::actions::perform_action(&el, &action)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
fn execute_action_impl(_handle: &NativeHandle, _action: Action) -> Result<ActionResult, AdapterError> {
|
||||
fn execute_action_impl(
|
||||
_handle: &NativeHandle,
|
||||
_action: Action,
|
||||
) -> Result<ActionResult, AdapterError> {
|
||||
Err(AdapterError::not_supported("execute_action"))
|
||||
}
|
||||
|
||||
|
|
@ -132,25 +211,22 @@ fn find_element_recursive(
|
|||
max_depth: u8,
|
||||
visited: &mut FxHashSet<usize>,
|
||||
) -> Result<NativeHandle, AdapterError> {
|
||||
use accessibility_sys::{
|
||||
kAXChildrenAttribute, kAXErrorSuccess, kAXRoleAttribute, kAXTitleAttribute,
|
||||
AXUIElementCopyAttributeValue, AXUIElementRef,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
};
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
use core_foundation::base::{CFRetain, CFTypeRef};
|
||||
|
||||
if !visited.insert(el.0 as usize) {
|
||||
return Err(AdapterError::element_not_found("element"));
|
||||
}
|
||||
|
||||
let role = crate::tree::copy_string_attr(el, kAXRoleAttribute);
|
||||
let normalized = role.as_deref().map(crate::roles::ax_role_to_str).unwrap_or("unknown");
|
||||
let ax_role = crate::tree::copy_string_attr(el, kAXRoleAttribute);
|
||||
let normalized = ax_role
|
||||
.as_deref()
|
||||
.map(crate::roles::ax_role_to_str)
|
||||
.unwrap_or("unknown");
|
||||
|
||||
if normalized == entry.role {
|
||||
let name = crate::tree::copy_string_attr(el, kAXTitleAttribute);
|
||||
let name_match = match (&entry.name, &name) {
|
||||
let elem_name = crate::tree::resolve_element_name(el);
|
||||
let name_match = match (&entry.name, &elem_name) {
|
||||
(Some(en), Some(nn)) => en == nn,
|
||||
(None, None) => true,
|
||||
_ => false,
|
||||
|
|
@ -165,29 +241,18 @@ fn find_element_recursive(
|
|||
return Err(AdapterError::element_not_found("element"));
|
||||
}
|
||||
|
||||
let cf_attr = core_foundation::string::CFString::new(kAXChildrenAttribute);
|
||||
let mut children_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut children_ref,
|
||||
)
|
||||
let child_attr = if ax_role.as_deref() == Some("AXBrowser") {
|
||||
"AXColumns"
|
||||
} else {
|
||||
"AXChildren"
|
||||
};
|
||||
let children = crate::tree::copy_ax_array(el, child_attr)
|
||||
.filter(|v| !v.is_empty())
|
||||
.or_else(|| crate::tree::copy_ax_array(el, "AXContents").filter(|v| !v.is_empty()))
|
||||
.unwrap_or_default();
|
||||
|
||||
if err != kAXErrorSuccess || children_ref.is_null() {
|
||||
return Err(AdapterError::element_not_found("element"));
|
||||
}
|
||||
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(children_ref as _) };
|
||||
for item in arr.into_iter() {
|
||||
let ptr = item.as_concrete_TypeRef() as AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
continue;
|
||||
}
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
let child = crate::tree::AXElement(ptr);
|
||||
if let Ok(handle) = find_element_recursive(&child, entry, depth + 1, max_depth, visited) {
|
||||
for child in &children {
|
||||
if let Ok(handle) = find_element_recursive(child, entry, depth + 1, max_depth, visited) {
|
||||
return Ok(handle);
|
||||
}
|
||||
}
|
||||
|
|
@ -203,54 +268,70 @@ fn resolve_element_impl(_entry: &RefEntry) -> Result<NativeHandle, AdapterError>
|
|||
pub fn list_windows_impl(filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use core_foundation::base::{CFType, TCFType};
|
||||
use core_foundation::number::CFNumber;
|
||||
use core_foundation::string::CFString;
|
||||
use core_foundation_sys::dictionary::CFDictionaryGetValue;
|
||||
use core_graphics::display::CGDisplay;
|
||||
use core_graphics::window::{
|
||||
kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowName, kCGWindowOwnerName,
|
||||
kCGWindowOwnerPID,
|
||||
};
|
||||
use rustc_hash::FxHasher;
|
||||
use std::ffi::c_void;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::process::Command;
|
||||
|
||||
let app_filter = filter.app.as_deref().unwrap_or("").to_string();
|
||||
let script = r#"
|
||||
tell application "System Events"
|
||||
set winList to {}
|
||||
repeat with proc in (processes where background only is false)
|
||||
set pName to name of proc as string
|
||||
set pPid to (unix id of proc) as string
|
||||
set winTitles to name of every window of proc
|
||||
repeat with wTitle in winTitles
|
||||
set winList to winList & {pName & "|" & pPid & "|" & (wTitle as string)}
|
||||
end repeat
|
||||
end repeat
|
||||
set AppleScript's text item delimiters to linefeed
|
||||
return winList as text
|
||||
end tell
|
||||
"#;
|
||||
let output = Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(script)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("osascript failed: {e}")))?;
|
||||
unsafe fn dict_string(dict: *const c_void, key: *const c_void) -> Option<String> {
|
||||
let val = CFDictionaryGetValue(dict as _, key);
|
||||
if val.is_null() {
|
||||
return None;
|
||||
}
|
||||
CFType::wrap_under_get_rule(val as _)
|
||||
.downcast::<CFString>()
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
|
||||
let text = String::from_utf8_lossy(&output.stdout);
|
||||
unsafe fn dict_i64(dict: *const c_void, key: *const c_void) -> Option<i64> {
|
||||
let val = CFDictionaryGetValue(dict as _, key);
|
||||
if val.is_null() {
|
||||
return None;
|
||||
}
|
||||
CFType::wrap_under_get_rule(val as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
}
|
||||
|
||||
let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) {
|
||||
Some(a) => a,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let app_filter = filter.app.as_deref().unwrap_or("").to_lowercase();
|
||||
let mut windows = Vec::new();
|
||||
for (idx, line) in text.lines().enumerate() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
|
||||
for raw in arr.get_all_values() {
|
||||
if raw.is_null() {
|
||||
continue;
|
||||
}
|
||||
let mut parts = line.splitn(3, '|');
|
||||
let app_name = match parts.next() {
|
||||
Some(s) => s.trim().to_string(),
|
||||
None => continue,
|
||||
};
|
||||
let pid: i32 = match parts.next().and_then(|s| s.trim().parse().ok()) {
|
||||
Some(p) => p,
|
||||
None => continue,
|
||||
};
|
||||
let title = parts.next().unwrap_or("").trim().to_string();
|
||||
|
||||
if !app_filter.is_empty() && !app_name.eq_ignore_ascii_case(&app_filter) {
|
||||
let layer = unsafe { dict_i64(raw, kCGWindowLayer as _) }.unwrap_or(99);
|
||||
if layer != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let app_name = match unsafe { dict_string(raw, kCGWindowOwnerName as _) } {
|
||||
Some(n) if !n.is_empty() => n,
|
||||
_ => continue,
|
||||
};
|
||||
if !app_filter.is_empty() && !app_name.to_lowercase().contains(&app_filter) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let title = match unsafe { dict_string(raw, kCGWindowName as _) } {
|
||||
Some(t) if !t.is_empty() => t,
|
||||
_ => app_name.clone(),
|
||||
};
|
||||
|
||||
let pid = unsafe { dict_i64(raw, kCGWindowOwnerPID as _) }.unwrap_or(0) as i32;
|
||||
let mut h = FxHasher::default();
|
||||
pid.hash(&mut h);
|
||||
title.hash(&mut h);
|
||||
|
|
@ -262,7 +343,7 @@ end tell
|
|||
app: app_name,
|
||||
pid,
|
||||
bounds: None,
|
||||
is_focused: idx == 0,
|
||||
is_focused: windows.is_empty(),
|
||||
});
|
||||
}
|
||||
Ok(windows)
|
||||
|
|
@ -277,32 +358,74 @@ end tell
|
|||
fn list_apps_impl() -> Result<Vec<AppInfo>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use std::process::Command;
|
||||
let output = Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(
|
||||
r#"tell application "System Events"
|
||||
set result to ""
|
||||
repeat with proc in (processes where background only is false)
|
||||
set result to result & (name of proc as string) & "|" & ((unix id of proc) as string) & linefeed
|
||||
end repeat
|
||||
return result
|
||||
end tell"#,
|
||||
)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("osascript failed: {e}")))?;
|
||||
use core_foundation::base::{CFType, TCFType};
|
||||
use core_foundation::number::CFNumber;
|
||||
use core_foundation::string::CFString;
|
||||
use core_foundation_sys::dictionary::CFDictionaryGetValue;
|
||||
use core_graphics::display::CGDisplay;
|
||||
use core_graphics::window::{
|
||||
kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowOwnerName, kCGWindowOwnerPID,
|
||||
};
|
||||
|
||||
let text = String::from_utf8_lossy(&output.stdout);
|
||||
let apps = text
|
||||
.lines()
|
||||
.filter(|l| !l.is_empty())
|
||||
.filter_map(|line| {
|
||||
let mut parts = line.split('|');
|
||||
let name = parts.next()?.trim().to_string();
|
||||
let pid: i32 = parts.next()?.trim().parse().ok()?;
|
||||
Some(AppInfo { name, pid, bundle_id: None })
|
||||
})
|
||||
.collect();
|
||||
let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) {
|
||||
Some(a) => a,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let mut seen_pids = std::collections::HashSet::new();
|
||||
let mut apps = Vec::new();
|
||||
|
||||
for raw in arr.get_all_values() {
|
||||
if raw.is_null() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let layer = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowLayer as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.unwrap_or(99)
|
||||
};
|
||||
if layer != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let pid = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerPID as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.unwrap_or(0) as i32
|
||||
};
|
||||
if !seen_pids.insert(pid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerName as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFString>()
|
||||
.map(|s| s.to_string())
|
||||
};
|
||||
|
||||
if let Some(n) = name {
|
||||
apps.push(AppInfo {
|
||||
name: n,
|
||||
pid,
|
||||
bundle_id: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(apps)
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
|
|||
|
|
@ -1,24 +1,46 @@
|
|||
use agent_desktop_core::{
|
||||
adapter::WindowFilter,
|
||||
error::AdapterError,
|
||||
node::WindowInfo,
|
||||
};
|
||||
use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> {
|
||||
use std::process::Command;
|
||||
let script = format!(
|
||||
r#"tell application "System Events"
|
||||
set frontmostProc to first process whose unix id is {}
|
||||
set frontmost of frontmostProc to true
|
||||
end tell"#,
|
||||
win.pid
|
||||
);
|
||||
Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(script)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("focus_window failed: {e}")))?;
|
||||
use accessibility_sys::{
|
||||
kAXErrorSuccess, AXUIElementCreateApplication, AXUIElementPerformAction,
|
||||
AXUIElementSetAttributeValue,
|
||||
};
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
|
||||
let app_el = unsafe { AXUIElementCreateApplication(win.pid) };
|
||||
if app_el.is_null() {
|
||||
return Err(AdapterError::internal("Failed to create AX app element"));
|
||||
}
|
||||
|
||||
let frontmost_attr = CFString::new("AXFrontmost");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
app_el,
|
||||
frontmost_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::internal(format!(
|
||||
"Failed to set AXFrontmost (err={err})"
|
||||
)));
|
||||
}
|
||||
|
||||
let main_win = crate::tree::window_element_for(win.pid, &win.title);
|
||||
let raise_action = CFString::new("AXRaise");
|
||||
let raise_err =
|
||||
unsafe { AXUIElementPerformAction(main_win.0, raise_action.as_concrete_TypeRef()) };
|
||||
if raise_err != kAXErrorSuccess {
|
||||
let main_attr = CFString::new("AXMain");
|
||||
unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
main_win.0,
|
||||
main_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -28,7 +50,7 @@ pub fn focus_window_impl(_win: &WindowInfo) -> Result<(), AdapterError> {
|
|||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn launch_app_impl(id: &str, wait: bool) -> Result<WindowInfo, AdapterError> {
|
||||
pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
|
||||
use crate::adapter::list_windows_impl;
|
||||
use std::process::Command;
|
||||
use std::time::{Duration, Instant};
|
||||
|
|
@ -40,48 +62,53 @@ pub fn launch_app_impl(id: &str, wait: bool) -> Result<WindowInfo, AdapterError>
|
|||
));
|
||||
}
|
||||
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: Some(id.to_string()),
|
||||
};
|
||||
if let Ok(wins) = list_windows_impl(&filter) {
|
||||
if let Some(win) = wins.into_iter().next() {
|
||||
return Ok(win);
|
||||
}
|
||||
}
|
||||
|
||||
Command::new("open")
|
||||
.arg("-a")
|
||||
.arg(id)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("open failed: {e}")))?;
|
||||
|
||||
if wait {
|
||||
let start = Instant::now();
|
||||
let timeout = Duration::from_secs(10);
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
let filter = WindowFilter { focused_only: false, app: Some(id.to_string()) };
|
||||
if let Ok(wins) = list_windows_impl(&filter) {
|
||||
if let Some(win) = wins.into_iter().next() {
|
||||
return Ok(win);
|
||||
}
|
||||
}
|
||||
if start.elapsed() > timeout {
|
||||
break;
|
||||
let start = Instant::now();
|
||||
let timeout = Duration::from_millis(timeout_ms);
|
||||
let mut poll_interval = Duration::from_millis(100);
|
||||
let max_interval = Duration::from_millis(500);
|
||||
|
||||
loop {
|
||||
std::thread::sleep(poll_interval);
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: Some(id.to_string()),
|
||||
};
|
||||
if let Ok(wins) = list_windows_impl(&filter) {
|
||||
if let Some(win) = wins.into_iter().next() {
|
||||
return Ok(win);
|
||||
}
|
||||
}
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::AppNotFound,
|
||||
format!("App '{id}' launched but no window found within timeout"),
|
||||
)
|
||||
.with_suggestion(
|
||||
"Try again with a longer timeout or check that the app has a visible window",
|
||||
));
|
||||
if start.elapsed() > timeout {
|
||||
break;
|
||||
}
|
||||
poll_interval = (poll_interval * 3 / 2).min(max_interval);
|
||||
}
|
||||
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
let filter = WindowFilter { focused_only: false, app: Some(id.to_string()) };
|
||||
if let Ok(wins) = list_windows_impl(&filter) {
|
||||
if let Some(win) = wins.into_iter().next() {
|
||||
return Ok(win);
|
||||
}
|
||||
}
|
||||
Err(AdapterError::internal(format!("App '{id}' launched but no window found")))
|
||||
Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::AppNotFound,
|
||||
format!("App '{id}' launched but no window appeared within {timeout_ms} ms"),
|
||||
)
|
||||
.with_suggestion("The app may take longer to start, or it may not create a visible window"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn launch_app_impl(_id: &str, _wait: bool) -> Result<WindowInfo, AdapterError> {
|
||||
pub fn launch_app_impl(_id: &str, _timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
|
||||
Err(AdapterError::not_supported("launch_app"))
|
||||
}
|
||||
|
||||
|
|
@ -95,22 +122,61 @@ pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> {
|
|||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("pkill failed: {e}")))?;
|
||||
} else {
|
||||
let safe_name = id.replace('"', "");
|
||||
let script = format!(
|
||||
r#"tell application "System Events"
|
||||
let pid = crate::key_dispatch::find_pid_by_name(id)?;
|
||||
let app_ax = crate::tree::element_for_pid(pid);
|
||||
let closed = try_quit_via_menu_bar(&app_ax);
|
||||
if !closed {
|
||||
let safe_name = id.replace('"', "");
|
||||
let script = format!(
|
||||
r#"tell application "System Events"
|
||||
set theProc to first process whose name is "{safe_name}"
|
||||
tell theProc to quit
|
||||
end tell"#
|
||||
);
|
||||
Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(script)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("quit failed: {e}")))?;
|
||||
);
|
||||
Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(script)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("quit failed: {e}")))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let Some(menu_bar) = crate::tree::copy_element_attr(app_el, "AXMenuBar") else {
|
||||
return false;
|
||||
};
|
||||
let Some(bar_items) = crate::tree::copy_ax_array(&menu_bar, "AXChildren") else {
|
||||
return false;
|
||||
};
|
||||
if let Some(app_menu) = bar_items.first() {
|
||||
if let Some(menus) = crate::tree::copy_ax_array(app_menu, "AXChildren") {
|
||||
for menu in &menus {
|
||||
if let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") {
|
||||
for item in &items {
|
||||
let title = crate::tree::copy_string_attr(item, "AXTitle");
|
||||
if let Some(t) = &title {
|
||||
if t.contains("Quit") || t.contains("quit") {
|
||||
let press = CFString::new("AXPress");
|
||||
let err = unsafe {
|
||||
AXUIElementPerformAction(item.0, press.as_concrete_TypeRef())
|
||||
};
|
||||
return err == kAXErrorSuccess;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn close_app_impl(_id: &str, _force: bool) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("close_app"))
|
||||
|
|
|
|||
|
|
@ -29,6 +29,10 @@ mod imp {
|
|||
.map_err(|e| AdapterError::internal(format!("pbcopy wait failed: {e}")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear() -> Result<(), AdapterError> {
|
||||
set("")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
@ -42,6 +46,10 @@ mod imp {
|
|||
pub fn set(_text: &str) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("clipboard_set"))
|
||||
}
|
||||
|
||||
pub fn clear() -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("clipboard_clear"))
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::{get, set};
|
||||
pub use imp::{clear, get, set};
|
||||
|
|
|
|||
|
|
@ -3,84 +3,263 @@ use agent_desktop_core::{action::KeyCombo, error::AdapterError};
|
|||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use core_graphics::event::{
|
||||
CGEvent, CGEventFlags, CGEventTapLocation, CGKeyCode,
|
||||
use accessibility_sys::{
|
||||
kAXErrorSuccess, AXUIElementCreateSystemWide, AXUIElementPostKeyboardEvent,
|
||||
};
|
||||
use core_graphics::event_source::{CGEventSource, CGEventSourceStateID};
|
||||
|
||||
pub fn synthesize_key(combo: &KeyCombo) -> Result<(), AdapterError> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
|
||||
.map_err(|_| AdapterError::internal("Failed to create CGEventSource"))?;
|
||||
|
||||
let key_code = key_name_to_code(&combo.key)?;
|
||||
let mut flags = CGEventFlags::empty();
|
||||
|
||||
for modifier in &combo.modifiers {
|
||||
flags |= modifier_to_flags(modifier);
|
||||
let sys_wide = unsafe { AXUIElementCreateSystemWide() };
|
||||
if sys_wide.is_null() {
|
||||
return Err(AdapterError::internal(
|
||||
"Failed to create system-wide AX element",
|
||||
));
|
||||
}
|
||||
|
||||
let key_down = CGEvent::new_keyboard_event(source.clone(), key_code, true)
|
||||
.map_err(|_| AdapterError::internal("Failed to create key down event"))?;
|
||||
key_down.set_flags(flags);
|
||||
if !combo.modifiers.is_empty() {
|
||||
for m in &combo.modifiers {
|
||||
let mod_code = modifier_keycode(m);
|
||||
let err = unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, mod_code, true) };
|
||||
if err != kAXErrorSuccess {
|
||||
release_modifiers(sys_wide, &combo.modifiers);
|
||||
unsafe { core_foundation::base::CFRelease(sys_wide as _) };
|
||||
return Err(AdapterError::internal(format!(
|
||||
"AXUIElementPostKeyboardEvent modifier-down failed (err={err})"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let key_up = CGEvent::new_keyboard_event(source, key_code, false)
|
||||
.map_err(|_| AdapterError::internal("Failed to create key up event"))?;
|
||||
key_up.set_flags(flags);
|
||||
let err_down = unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, key_code, true) };
|
||||
let err_up = unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, key_code, false) };
|
||||
|
||||
key_down.post(CGEventTapLocation::HID);
|
||||
key_up.post(CGEventTapLocation::HID);
|
||||
if !combo.modifiers.is_empty() {
|
||||
release_modifiers(sys_wide, &combo.modifiers);
|
||||
}
|
||||
|
||||
unsafe { core_foundation::base::CFRelease(sys_wide as _) };
|
||||
|
||||
if err_down != kAXErrorSuccess {
|
||||
return Err(AdapterError::internal(format!(
|
||||
"AXUIElementPostKeyboardEvent key-down failed (err={err_down})"
|
||||
)));
|
||||
}
|
||||
if err_up != kAXErrorSuccess {
|
||||
return Err(AdapterError::internal(format!(
|
||||
"AXUIElementPostKeyboardEvent key-up failed (err={err_up})"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn synthesize_text(text: &str) -> Result<(), AdapterError> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
|
||||
.map_err(|_| AdapterError::internal("Failed to create CGEventSource"))?;
|
||||
let sys_wide = unsafe { AXUIElementCreateSystemWide() };
|
||||
if sys_wide.is_null() {
|
||||
return Err(AdapterError::internal(
|
||||
"Failed to create system-wide AX element",
|
||||
));
|
||||
}
|
||||
|
||||
for ch in text.chars() {
|
||||
let key_down = CGEvent::new_keyboard_event(source.clone(), 0, true)
|
||||
.map_err(|_| AdapterError::internal("Failed to create keyboard event"))?;
|
||||
key_down.set_string(&ch.to_string());
|
||||
key_down.post(CGEventTapLocation::HID);
|
||||
|
||||
let key_up = CGEvent::new_keyboard_event(source.clone(), 0, false)
|
||||
.map_err(|_| AdapterError::internal("Failed to create keyboard event"))?;
|
||||
key_up.post(CGEventTapLocation::HID);
|
||||
if ch == '\n' {
|
||||
let return_code = 36u16;
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(sys_wide, 0, return_code, true);
|
||||
AXUIElementPostKeyboardEvent(sys_wide, 0, return_code, false);
|
||||
};
|
||||
} else if let Some(code) = char_to_keycode(ch) {
|
||||
let needs_shift = ch.is_ascii_uppercase() || is_shifted_char(ch);
|
||||
if needs_shift {
|
||||
unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, 56, true) };
|
||||
}
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(sys_wide, 0, code, true);
|
||||
AXUIElementPostKeyboardEvent(sys_wide, 0, code, false);
|
||||
};
|
||||
if needs_shift {
|
||||
unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, 56, false) };
|
||||
}
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(4));
|
||||
}
|
||||
|
||||
unsafe { core_foundation::base::CFRelease(sys_wide as _) };
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn modifier_to_flags(m: &agent_desktop_core::action::Modifier) -> CGEventFlags {
|
||||
use agent_desktop_core::action::Modifier;
|
||||
match m {
|
||||
Modifier::Cmd => CGEventFlags::CGEventFlagCommand,
|
||||
Modifier::Ctrl => CGEventFlags::CGEventFlagControl,
|
||||
Modifier::Alt => CGEventFlags::CGEventFlagAlternate,
|
||||
Modifier::Shift => CGEventFlags::CGEventFlagShift,
|
||||
fn release_modifiers(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
modifiers: &[agent_desktop_core::action::Modifier],
|
||||
) {
|
||||
for m in modifiers.iter().rev() {
|
||||
let code = modifier_keycode(m);
|
||||
unsafe { AXUIElementPostKeyboardEvent(el, 0, code, false) };
|
||||
}
|
||||
}
|
||||
|
||||
fn key_name_to_code(key: &str) -> Result<CGKeyCode, AdapterError> {
|
||||
fn modifier_keycode(m: &agent_desktop_core::action::Modifier) -> u16 {
|
||||
use agent_desktop_core::action::Modifier;
|
||||
match m {
|
||||
Modifier::Cmd => 55,
|
||||
Modifier::Shift => 56,
|
||||
Modifier::Alt => 58,
|
||||
Modifier::Ctrl => 59,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_shifted_char(ch: char) -> bool {
|
||||
matches!(
|
||||
ch,
|
||||
'!' | '@'
|
||||
| '#'
|
||||
| '$'
|
||||
| '%'
|
||||
| '^'
|
||||
| '&'
|
||||
| '*'
|
||||
| '('
|
||||
| ')'
|
||||
| '_'
|
||||
| '+'
|
||||
| '{'
|
||||
| '}'
|
||||
| '|'
|
||||
| ':'
|
||||
| '"'
|
||||
| '<'
|
||||
| '>'
|
||||
| '?'
|
||||
| '~'
|
||||
)
|
||||
}
|
||||
|
||||
fn char_to_keycode(ch: char) -> Option<u16> {
|
||||
let lower = ch.to_ascii_lowercase();
|
||||
Some(match lower {
|
||||
'a' => 0,
|
||||
'b' => 11,
|
||||
'c' => 8,
|
||||
'd' => 2,
|
||||
'e' => 14,
|
||||
'f' => 3,
|
||||
'g' => 5,
|
||||
'h' => 4,
|
||||
'i' => 34,
|
||||
'j' => 38,
|
||||
'k' => 40,
|
||||
'l' => 37,
|
||||
'm' => 46,
|
||||
'n' => 45,
|
||||
'o' => 31,
|
||||
'p' => 35,
|
||||
'q' => 12,
|
||||
'r' => 15,
|
||||
's' => 1,
|
||||
't' => 17,
|
||||
'u' => 32,
|
||||
'v' => 9,
|
||||
'w' => 13,
|
||||
'x' => 7,
|
||||
'y' => 16,
|
||||
'z' => 6,
|
||||
'0' | ')' => 29,
|
||||
'1' | '!' => 18,
|
||||
'2' | '@' => 19,
|
||||
'3' | '#' => 20,
|
||||
'4' | '$' => 21,
|
||||
'5' | '%' => 23,
|
||||
'6' | '^' => 22,
|
||||
'7' | '&' => 26,
|
||||
'8' | '*' => 28,
|
||||
'9' | '(' => 25,
|
||||
' ' => 49,
|
||||
'-' | '_' => 27,
|
||||
'=' | '+' => 24,
|
||||
'[' | '{' => 33,
|
||||
']' | '}' => 30,
|
||||
'\\' | '|' => 42,
|
||||
';' | ':' => 41,
|
||||
'\'' | '"' => 39,
|
||||
',' | '<' => 43,
|
||||
'.' | '>' => 47,
|
||||
'/' | '?' => 44,
|
||||
'`' | '~' => 50,
|
||||
'\t' => 48,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn key_name_to_code(key: &str) -> Result<u16, AdapterError> {
|
||||
let code = match key {
|
||||
"a" => 0, "b" => 11, "c" => 8, "d" => 2, "e" => 14, "f" => 3,
|
||||
"g" => 5, "h" => 4, "i" => 34, "j" => 38, "k" => 40, "l" => 37,
|
||||
"m" => 46, "n" => 45, "o" => 31, "p" => 35, "q" => 12, "r" => 15,
|
||||
"s" => 1, "t" => 17, "u" => 32, "v" => 9, "w" => 13, "x" => 7,
|
||||
"y" => 16, "z" => 6,
|
||||
"0" => 29, "1" => 18, "2" => 19, "3" => 20, "4" => 21,
|
||||
"5" => 23, "6" => 22, "7" => 26, "8" => 28, "9" => 25,
|
||||
"a" => 0,
|
||||
"b" => 11,
|
||||
"c" => 8,
|
||||
"d" => 2,
|
||||
"e" => 14,
|
||||
"f" => 3,
|
||||
"g" => 5,
|
||||
"h" => 4,
|
||||
"i" => 34,
|
||||
"j" => 38,
|
||||
"k" => 40,
|
||||
"l" => 37,
|
||||
"m" => 46,
|
||||
"n" => 45,
|
||||
"o" => 31,
|
||||
"p" => 35,
|
||||
"q" => 12,
|
||||
"r" => 15,
|
||||
"s" => 1,
|
||||
"t" => 17,
|
||||
"u" => 32,
|
||||
"v" => 9,
|
||||
"w" => 13,
|
||||
"x" => 7,
|
||||
"y" => 16,
|
||||
"z" => 6,
|
||||
"0" => 29,
|
||||
"1" => 18,
|
||||
"2" => 19,
|
||||
"3" => 20,
|
||||
"4" => 21,
|
||||
"5" => 23,
|
||||
"6" => 22,
|
||||
"7" => 26,
|
||||
"8" => 28,
|
||||
"9" => 25,
|
||||
"return" | "enter" => 36,
|
||||
"escape" | "esc" => 53,
|
||||
"tab" => 48,
|
||||
"space" => 49,
|
||||
"delete" | "backspace" => 51,
|
||||
"left" => 123, "right" => 124, "down" => 125, "up" => 126,
|
||||
"f1" => 122, "f2" => 120, "f3" => 99, "f4" => 118,
|
||||
"f5" => 96, "f6" => 97, "f7" => 98, "f8" => 100,
|
||||
"f9" => 101, "f10" => 109, "f11" => 103, "f12" => 111,
|
||||
other => return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::InvalidArgs,
|
||||
format!("Unknown key: '{other}'"),
|
||||
)),
|
||||
"forwarddelete" => 117,
|
||||
"home" => 115,
|
||||
"end" => 119,
|
||||
"pageup" => 116,
|
||||
"pagedown" => 121,
|
||||
"left" => 123,
|
||||
"right" => 124,
|
||||
"down" => 125,
|
||||
"up" => 126,
|
||||
"f1" => 122,
|
||||
"f2" => 120,
|
||||
"f3" => 99,
|
||||
"f4" => 118,
|
||||
"f5" => 96,
|
||||
"f6" => 97,
|
||||
"f7" => 98,
|
||||
"f8" => 100,
|
||||
"f9" => 101,
|
||||
"f10" => 109,
|
||||
"f11" => 103,
|
||||
"f12" => 111,
|
||||
other => {
|
||||
return Err(AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::InvalidArgs,
|
||||
format!("Unknown key: '{other}'"),
|
||||
))
|
||||
}
|
||||
};
|
||||
Ok(code)
|
||||
}
|
||||
|
|
|
|||
320
crates/macos/src/key_dispatch.rs
Normal file
320
crates/macos/src/key_dispatch.rs
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
use agent_desktop_core::{
|
||||
action::{ActionResult, KeyCombo},
|
||||
error::AdapterError,
|
||||
};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use agent_desktop_core::{action::Modifier, adapter::WindowFilter};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn press_for_app_impl(app_name: &str, combo: &KeyCombo) -> Result<ActionResult, AdapterError> {
|
||||
use accessibility_sys::{AXUIElementCreateApplication, AXUIElementSetAttributeValue};
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
|
||||
let pid = find_pid_by_name(app_name)?;
|
||||
let app_el = unsafe { AXUIElementCreateApplication(pid) };
|
||||
if app_el.is_null() {
|
||||
return Err(AdapterError::internal("Failed to create AX app element"));
|
||||
}
|
||||
|
||||
let frontmost_attr = CFString::new("AXFrontmost");
|
||||
unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
app_el,
|
||||
frontmost_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
|
||||
if !combo.modifiers.is_empty() {
|
||||
let app_ax = crate::tree::AXElement(app_el);
|
||||
if let Some(result) = try_menu_bar_shortcut(&app_ax, combo) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let simple_result = try_simple_key_action(app_el, combo);
|
||||
if let Some(result) = simple_result {
|
||||
return result;
|
||||
}
|
||||
|
||||
ax_post_keyboard_event(app_el, combo)?;
|
||||
Ok(ActionResult::new("press_key".to_string()))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_simple_key_action(
|
||||
app_el: accessibility_sys::AXUIElementRef,
|
||||
combo: &KeyCombo,
|
||||
) -> Option<Result<ActionResult, AdapterError>> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
if !combo.modifiers.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let focused = get_focused_element(app_el)?;
|
||||
let action_name = match combo.key.as_str() {
|
||||
"return" | "enter" => "AXConfirm",
|
||||
"escape" | "esc" => "AXCancel",
|
||||
"space" => "AXPress",
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let ax_action = CFString::new(action_name);
|
||||
let err = unsafe { AXUIElementPerformAction(focused.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err == kAXErrorSuccess {
|
||||
Some(Ok(ActionResult::new("press_key".to_string())))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_focused_element(
|
||||
app_el: accessibility_sys::AXUIElementRef,
|
||||
) -> Option<crate::tree::AXElement> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue, AXUIElementRef};
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let attr = CFString::new("AXFocusedUIElement");
|
||||
let mut value: core_foundation_sys::base::CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(app_el, attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
Some(crate::tree::AXElement(value as AXUIElementRef))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_menu_bar_shortcut(
|
||||
app_el: &crate::tree::AXElement,
|
||||
combo: &KeyCombo,
|
||||
) -> Option<Result<ActionResult, AdapterError>> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let menu_bar = crate::tree::copy_element_attr(app_el, "AXMenuBar")?;
|
||||
let menu_bar_items = crate::tree::copy_ax_array(&menu_bar, "AXChildren")?;
|
||||
|
||||
let target_char = if combo.key.len() == 1 {
|
||||
combo.key.to_uppercase()
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let target_mods = combo_to_ax_modifiers(combo);
|
||||
|
||||
for bar_item in &menu_bar_items {
|
||||
if let Some(menu) = crate::tree::copy_ax_array(bar_item, "AXChildren") {
|
||||
for menu_group in &menu {
|
||||
if let Some(items) = crate::tree::copy_ax_array(menu_group, "AXChildren") {
|
||||
for item in &items {
|
||||
let cmd_char = crate::tree::copy_string_attr(item, "AXMenuItemCmdChar");
|
||||
let cmd_mods = read_menu_item_modifiers(item);
|
||||
|
||||
if let Some(ch) = &cmd_char {
|
||||
if ch.to_uppercase() == target_char && cmd_mods == target_mods {
|
||||
let press = CFString::new("AXPress");
|
||||
let err = unsafe {
|
||||
AXUIElementPerformAction(item.0, press.as_concrete_TypeRef())
|
||||
};
|
||||
if err == kAXErrorSuccess {
|
||||
return Some(Ok(ActionResult::new("press_key".to_string())));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_menu_item_modifiers(el: &crate::tree::AXElement) -> u32 {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue};
|
||||
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
|
||||
|
||||
let attr = CFString::new("AXMenuItemCmdModifiers");
|
||||
let mut value: core_foundation_sys::base::CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(el.0, attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return 0;
|
||||
}
|
||||
let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) };
|
||||
cf.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.map(|v| v as u32)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn combo_to_ax_modifiers(combo: &KeyCombo) -> u32 {
|
||||
let mut mods: u32 = 0;
|
||||
for m in &combo.modifiers {
|
||||
match m {
|
||||
Modifier::Shift => mods |= 1 << 1,
|
||||
Modifier::Alt => mods |= 1 << 3,
|
||||
Modifier::Ctrl => mods |= 1 << 2,
|
||||
Modifier::Cmd => {}
|
||||
}
|
||||
}
|
||||
mods
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn ax_post_keyboard_event(
|
||||
app_el: accessibility_sys::AXUIElementRef,
|
||||
combo: &KeyCombo,
|
||||
) -> Result<(), AdapterError> {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent;
|
||||
|
||||
let key_code = key_to_keycode(&combo.key).ok_or_else(|| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionNotSupported,
|
||||
format!(
|
||||
"No AX equivalent for key combo '{}'. This combo has no menu-bar action.",
|
||||
format_combo(combo)
|
||||
),
|
||||
)
|
||||
.with_suggestion("This key combo cannot be executed via accessibility APIs alone.")
|
||||
})?;
|
||||
|
||||
let err = unsafe { AXUIElementPostKeyboardEvent(app_el, 0 as _, key_code, true) };
|
||||
if err != accessibility_sys::kAXErrorSuccess {
|
||||
return Err(AdapterError::internal(format!(
|
||||
"AXUIElementPostKeyboardEvent key-down failed (err={err})"
|
||||
)));
|
||||
}
|
||||
|
||||
let err = unsafe { AXUIElementPostKeyboardEvent(app_el, 0 as _, key_code, false) };
|
||||
if err != accessibility_sys::kAXErrorSuccess {
|
||||
return Err(AdapterError::internal(format!(
|
||||
"AXUIElementPostKeyboardEvent key-up failed (err={err})"
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn format_combo(combo: &KeyCombo) -> String {
|
||||
let mods: Vec<&str> = combo
|
||||
.modifiers
|
||||
.iter()
|
||||
.map(|m| match m {
|
||||
Modifier::Cmd => "cmd",
|
||||
Modifier::Ctrl => "ctrl",
|
||||
Modifier::Alt => "alt",
|
||||
Modifier::Shift => "shift",
|
||||
})
|
||||
.collect();
|
||||
if mods.is_empty() {
|
||||
combo.key.clone()
|
||||
} else {
|
||||
format!("{}+{}", mods.join("+"), combo.key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn key_to_keycode(key: &str) -> Option<u16> {
|
||||
Some(match key {
|
||||
"a" => 0,
|
||||
"b" => 11,
|
||||
"c" => 8,
|
||||
"d" => 2,
|
||||
"e" => 14,
|
||||
"f" => 3,
|
||||
"g" => 5,
|
||||
"h" => 4,
|
||||
"i" => 34,
|
||||
"j" => 38,
|
||||
"k" => 40,
|
||||
"l" => 37,
|
||||
"m" => 46,
|
||||
"n" => 45,
|
||||
"o" => 31,
|
||||
"p" => 35,
|
||||
"q" => 12,
|
||||
"r" => 15,
|
||||
"s" => 1,
|
||||
"t" => 17,
|
||||
"u" => 32,
|
||||
"v" => 9,
|
||||
"w" => 13,
|
||||
"x" => 7,
|
||||
"y" => 16,
|
||||
"z" => 6,
|
||||
"0" => 29,
|
||||
"1" => 18,
|
||||
"2" => 19,
|
||||
"3" => 20,
|
||||
"4" => 21,
|
||||
"5" => 23,
|
||||
"6" => 22,
|
||||
"7" => 26,
|
||||
"8" => 28,
|
||||
"9" => 25,
|
||||
"return" | "enter" => 36,
|
||||
"escape" | "esc" => 53,
|
||||
"tab" => 48,
|
||||
"space" => 49,
|
||||
"delete" | "backspace" => 51,
|
||||
"forwarddelete" => 117,
|
||||
"home" => 115,
|
||||
"end" => 119,
|
||||
"pageup" => 116,
|
||||
"pagedown" => 121,
|
||||
"left" => 123,
|
||||
"right" => 124,
|
||||
"down" => 125,
|
||||
"up" => 126,
|
||||
"f1" => 122,
|
||||
"f2" => 120,
|
||||
"f3" => 99,
|
||||
"f4" => 118,
|
||||
"f5" => 96,
|
||||
"f6" => 97,
|
||||
"f7" => 98,
|
||||
"f8" => 100,
|
||||
"f9" => 101,
|
||||
"f10" => 109,
|
||||
"f11" => 103,
|
||||
"f12" => 111,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) fn find_pid_by_name(app_name: &str) -> Result<i32, AdapterError> {
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: Some(app_name.to_string()),
|
||||
};
|
||||
let windows = crate::adapter::list_windows_impl(&filter)?;
|
||||
windows.first().map(|w| w.pid).ok_or_else(|| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::AppNotFound,
|
||||
format!("App '{app_name}' not found"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn press_for_app_impl(
|
||||
_app_name: &str,
|
||||
_combo: &KeyCombo,
|
||||
) -> Result<ActionResult, AdapterError> {
|
||||
Err(AdapterError::not_supported("press_for_app"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub(crate) fn find_pid_by_name(_app_name: &str) -> Result<i32, AdapterError> {
|
||||
Err(AdapterError::not_supported("find_pid_by_name"))
|
||||
}
|
||||
|
|
@ -1,11 +1,17 @@
|
|||
pub mod action_extras;
|
||||
pub mod actions;
|
||||
pub mod adapter;
|
||||
pub mod app_ops;
|
||||
pub mod clipboard;
|
||||
pub mod input;
|
||||
pub mod key_dispatch;
|
||||
pub mod mouse;
|
||||
pub mod permissions;
|
||||
pub mod roles;
|
||||
pub mod screenshot;
|
||||
pub mod surfaces;
|
||||
pub mod tree;
|
||||
pub mod wait;
|
||||
pub mod window_ops;
|
||||
|
||||
pub use adapter::MacOSAdapter;
|
||||
|
|
|
|||
147
crates/macos/src/mouse.rs
Normal file
147
crates/macos/src/mouse.rs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
use agent_desktop_core::{
|
||||
action::{DragParams, MouseButton, MouseEvent, MouseEventKind},
|
||||
error::AdapterError,
|
||||
};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use core_graphics::event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton};
|
||||
use core_graphics::event_source::{CGEventSource, CGEventSourceStateID};
|
||||
use core_graphics::geometry::CGPoint;
|
||||
|
||||
pub fn synthesize_mouse(event: MouseEvent) -> Result<(), AdapterError> {
|
||||
let point = CGPoint::new(event.point.x, event.point.y);
|
||||
let cg_button = to_cg_button(&event.button);
|
||||
match event.kind {
|
||||
MouseEventKind::Move => post_event(CGEventType::MouseMoved, point, cg_button),
|
||||
MouseEventKind::Down => post_event(down_type(&event.button), point, cg_button),
|
||||
MouseEventKind::Up => post_event(up_type(&event.button), point, cg_button),
|
||||
MouseEventKind::Click { count } => {
|
||||
synthesize_click(point, cg_button, &event.button, count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn synthesize_drag(params: DragParams) -> Result<(), AdapterError> {
|
||||
let from = CGPoint::new(params.from.x, params.from.y);
|
||||
let to = CGPoint::new(params.to.x, params.to.y);
|
||||
let duration_ms = params.duration_ms.unwrap_or(300);
|
||||
let steps = (duration_ms / 16).max(4) as usize;
|
||||
let step_delay = std::time::Duration::from_millis(duration_ms / steps as u64);
|
||||
|
||||
post_event(CGEventType::LeftMouseDown, from, CGMouseButton::Left)?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
|
||||
for i in 1..=steps {
|
||||
let t = i as f64 / steps as f64;
|
||||
let x = params.from.x + (params.to.x - params.from.x) * t;
|
||||
let y = params.from.y + (params.to.y - params.from.y) * t;
|
||||
post_event(
|
||||
CGEventType::LeftMouseDragged,
|
||||
CGPoint::new(x, y),
|
||||
CGMouseButton::Left,
|
||||
)?;
|
||||
std::thread::sleep(step_delay);
|
||||
}
|
||||
|
||||
post_event(CGEventType::LeftMouseUp, to, CGMouseButton::Left)
|
||||
}
|
||||
|
||||
fn synthesize_click(
|
||||
point: CGPoint,
|
||||
cg_button: CGMouseButton,
|
||||
button: &MouseButton,
|
||||
count: u32,
|
||||
) -> Result<(), AdapterError> {
|
||||
let down_ty = down_type(button);
|
||||
let up_ty = up_type(button);
|
||||
for i in 1..=count {
|
||||
let down = create_event(down_ty, point, cg_button)?;
|
||||
let up = create_event(up_ty, point, cg_button)?;
|
||||
set_click_count(&down, i as i64);
|
||||
set_click_count(&up, i as i64);
|
||||
down.post(CGEventTapLocation::HID);
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
up.post(CGEventTapLocation::HID);
|
||||
if i < count {
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_click_count(event: &CGEvent, count: i64) {
|
||||
unsafe {
|
||||
CGEventSetIntegerValueField(
|
||||
event as *const CGEvent as *const std::ffi::c_void,
|
||||
1,
|
||||
count,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
fn CGEventSetIntegerValueField(event: *const std::ffi::c_void, field: u32, value: i64);
|
||||
}
|
||||
|
||||
fn create_event(
|
||||
event_type: CGEventType,
|
||||
point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
) -> Result<CGEvent, AdapterError> {
|
||||
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
|
||||
.map_err(|()| AdapterError::internal("Failed to create CGEventSource"))?;
|
||||
CGEvent::new_mouse_event(source, event_type, point, button)
|
||||
.map_err(|()| AdapterError::internal("CGEvent::new_mouse_event failed"))
|
||||
}
|
||||
|
||||
fn post_event(
|
||||
event_type: CGEventType,
|
||||
point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
) -> Result<(), AdapterError> {
|
||||
let ev = create_event(event_type, point, button)?;
|
||||
ev.post(CGEventTapLocation::HID);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn to_cg_button(button: &MouseButton) -> CGMouseButton {
|
||||
match button {
|
||||
MouseButton::Left => CGMouseButton::Left,
|
||||
MouseButton::Right => CGMouseButton::Right,
|
||||
MouseButton::Middle => CGMouseButton::Center,
|
||||
}
|
||||
}
|
||||
|
||||
fn down_type(button: &MouseButton) -> CGEventType {
|
||||
match button {
|
||||
MouseButton::Left => CGEventType::LeftMouseDown,
|
||||
MouseButton::Right => CGEventType::RightMouseDown,
|
||||
MouseButton::Middle => CGEventType::OtherMouseDown,
|
||||
}
|
||||
}
|
||||
|
||||
fn up_type(button: &MouseButton) -> CGEventType {
|
||||
match button {
|
||||
MouseButton::Left => CGEventType::LeftMouseUp,
|
||||
MouseButton::Right => CGEventType::RightMouseUp,
|
||||
MouseButton::Middle => CGEventType::OtherMouseUp,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
mod imp {
|
||||
use super::*;
|
||||
|
||||
pub fn synthesize_mouse(_event: MouseEvent) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("mouse_event"))
|
||||
}
|
||||
|
||||
pub fn synthesize_drag(_params: DragParams) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("drag"))
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::{synthesize_drag, synthesize_mouse};
|
||||
|
|
@ -6,10 +6,7 @@ mod imp {
|
|||
kAXTrustedCheckOptionPrompt, AXIsProcessTrusted, AXIsProcessTrustedWithOptions,
|
||||
};
|
||||
use core_foundation::{
|
||||
base::TCFType,
|
||||
boolean::CFBoolean,
|
||||
dictionary::CFDictionary,
|
||||
string::CFString,
|
||||
base::TCFType, boolean::CFBoolean, dictionary::CFDictionary, string::CFString,
|
||||
};
|
||||
|
||||
pub fn is_trusted() -> bool {
|
||||
|
|
@ -20,8 +17,7 @@ mod imp {
|
|||
unsafe {
|
||||
let key = CFString::wrap_under_get_rule(kAXTrustedCheckOptionPrompt);
|
||||
let val = CFBoolean::true_value();
|
||||
let dict =
|
||||
CFDictionary::from_CFType_pairs(&[(key.as_CFType(), val.as_CFType())]);
|
||||
let dict = CFDictionary::from_CFType_pairs(&[(key.as_CFType(), val.as_CFType())]);
|
||||
AXIsProcessTrustedWithOptions(dict.as_concrete_TypeRef())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
pub fn ax_role_to_str(ax_role: &str) -> &'static str {
|
||||
match ax_role {
|
||||
"AXApplication" => "application",
|
||||
"AXButton" => "button",
|
||||
"AXMenuButton" => "menubutton",
|
||||
"AXTextField" | "AXTextArea" | "AXSearchField" => "textfield",
|
||||
"AXCheckBox" => "checkbox",
|
||||
"AXSwitch" | "AXToggle" => "switch",
|
||||
"AXLink" => "link",
|
||||
"AXMenuItem" | "AXMenuBarItem" => "menuitem",
|
||||
"AXRadioButton" => "radiobutton",
|
||||
|
|
@ -11,6 +14,7 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str {
|
|||
"AXComboBox" | "AXPopUpButton" => "combobox",
|
||||
"AXOutlineRow" | "AXRow" => "treeitem",
|
||||
"AXCell" => "cell",
|
||||
"AXColumn" => "column",
|
||||
"AXWindow" => "window",
|
||||
"AXSheet" => "sheet",
|
||||
"AXDialog" => "dialog",
|
||||
|
|
@ -22,13 +26,28 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str {
|
|||
"AXList" => "list",
|
||||
"AXOutline" => "outline",
|
||||
"AXScrollArea" | "AXScrollBar" => "scrollarea",
|
||||
"AXSplitter" => "splitter",
|
||||
"AXSplitter" | "AXSplitGroup" => "splitter",
|
||||
"AXMenu" | "AXMenuBar" => "menu",
|
||||
"AXIncrementor" | "AXStepper" => "incrementor",
|
||||
"AXDisclosureTriangle" => "disclosure",
|
||||
"AXProgressIndicator" | "AXBusyIndicator" => "progressbar",
|
||||
"AXColorWell" => "colorwell",
|
||||
"AXWebArea" => "webarea",
|
||||
"AXBrowser" => "browser",
|
||||
"AXGrid" => "grid",
|
||||
"AXHandle" => "handle",
|
||||
"AXPopover" => "popover",
|
||||
"AXDockItem" => "dockitem",
|
||||
"AXRuler" => "ruler",
|
||||
"AXRulerMarker" => "rulermarker",
|
||||
"AXTimeField" => "timefield",
|
||||
"AXDateField" => "datefield",
|
||||
"AXHelpTag" => "helptag",
|
||||
"AXMatte" => "matte",
|
||||
"AXDrawer" => "drawer",
|
||||
"AXLayoutArea" | "AXLayoutItem" => "layoutitem",
|
||||
"AXLevelIndicator" => "levelindicator",
|
||||
"AXRelevanceIndicator" => "relevanceindicator",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
|
@ -37,8 +56,10 @@ pub fn is_interactive_role(role: &str) -> bool {
|
|||
matches!(
|
||||
role,
|
||||
"button"
|
||||
| "menubutton"
|
||||
| "textfield"
|
||||
| "checkbox"
|
||||
| "switch"
|
||||
| "link"
|
||||
| "menuitem"
|
||||
| "tab"
|
||||
|
|
@ -48,5 +69,7 @@ pub fn is_interactive_role(role: &str) -> bool {
|
|||
| "cell"
|
||||
| "radiobutton"
|
||||
| "incrementor"
|
||||
| "colorwell"
|
||||
| "dockitem"
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,53 +3,151 @@ use agent_desktop_core::{adapter::ImageBuffer, adapter::ImageFormat, error::Adap
|
|||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use core_graphics::{
|
||||
display::CGDisplay,
|
||||
geometry::{CGPoint, CGRect, CGSize},
|
||||
window::{
|
||||
create_image, kCGWindowImageDefault, kCGWindowListOptionIncludingWindow,
|
||||
},
|
||||
};
|
||||
use std::process::Command;
|
||||
|
||||
pub fn capture_window(window_id: u32) -> Result<ImageBuffer, AdapterError> {
|
||||
let zero_rect = CGRect::new(&CGPoint::new(0.0, 0.0), &CGSize::new(0.0, 0.0));
|
||||
let image = create_image(
|
||||
zero_rect,
|
||||
kCGWindowListOptionIncludingWindow,
|
||||
window_id,
|
||||
kCGWindowImageDefault,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
format!("Failed to capture window {window_id}"),
|
||||
)
|
||||
})?;
|
||||
pub fn capture_app(pid: i32) -> Result<ImageBuffer, AdapterError> {
|
||||
let temp = temp_path();
|
||||
let cg_id = find_cg_window_id_for_pid(pid);
|
||||
|
||||
let width = image.width() as u32;
|
||||
let height = image.height() as u32;
|
||||
let data = vec![0u8; (width * height * 4) as usize];
|
||||
Ok(ImageBuffer { data, format: ImageFormat::Png, width, height })
|
||||
let status = if let Some(wid) = cg_id {
|
||||
Command::new("screencapture")
|
||||
.args(["-x", "-t", "png", "-l"])
|
||||
.arg(wid.to_string())
|
||||
.arg(&temp)
|
||||
.status()
|
||||
} else {
|
||||
Command::new("screencapture")
|
||||
.args(["-x", "-t", "png"])
|
||||
.arg(&temp)
|
||||
.status()
|
||||
}
|
||||
.map_err(|e| AdapterError::internal(format!("screencapture: {e}")))?;
|
||||
|
||||
if !status.success() {
|
||||
return Err(AdapterError::internal("screencapture exited with error"));
|
||||
}
|
||||
|
||||
read_png(&temp)
|
||||
}
|
||||
|
||||
pub fn capture_screen(display_idx: usize) -> Result<ImageBuffer, AdapterError> {
|
||||
let displays = CGDisplay::active_displays()
|
||||
.map_err(|_| AdapterError::internal("Failed to list displays"))?;
|
||||
let display_id = displays.get(display_idx).copied().ok_or_else(|| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::InvalidArgs,
|
||||
format!("Display index {display_idx} out of range"),
|
||||
)
|
||||
})?;
|
||||
let display = CGDisplay::new(display_id);
|
||||
let image = display
|
||||
.image()
|
||||
.ok_or_else(|| AdapterError::internal("Failed to capture display"))?;
|
||||
pub fn capture_screen(_idx: usize) -> Result<ImageBuffer, AdapterError> {
|
||||
let temp = temp_path();
|
||||
let status = Command::new("screencapture")
|
||||
.args(["-x", "-t", "png"])
|
||||
.arg(&temp)
|
||||
.status()
|
||||
.map_err(|e| AdapterError::internal(format!("screencapture: {e}")))?;
|
||||
|
||||
let width = image.width() as u32;
|
||||
let height = image.height() as u32;
|
||||
let data = vec![0u8; (width * height * 4) as usize];
|
||||
Ok(ImageBuffer { data, format: ImageFormat::Png, width, height })
|
||||
if !status.success() {
|
||||
return Err(AdapterError::internal("screencapture exited with error"));
|
||||
}
|
||||
|
||||
read_png(&temp)
|
||||
}
|
||||
|
||||
fn temp_path() -> String {
|
||||
format!("/tmp/agent-desktop-ss-{}.png", std::process::id())
|
||||
}
|
||||
|
||||
fn read_png(path: &str) -> Result<ImageBuffer, AdapterError> {
|
||||
let data = std::fs::read(path)
|
||||
.map_err(|e| AdapterError::internal(format!("read screenshot: {e}")))?;
|
||||
let _ = std::fs::remove_file(path);
|
||||
let (width, height) = png_dimensions(&data);
|
||||
Ok(ImageBuffer {
|
||||
data,
|
||||
format: ImageFormat::Png,
|
||||
width,
|
||||
height,
|
||||
})
|
||||
}
|
||||
|
||||
fn png_dimensions(data: &[u8]) -> (u32, u32) {
|
||||
if data.len() < 24 {
|
||||
return (0, 0);
|
||||
}
|
||||
let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]);
|
||||
let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]);
|
||||
(w, h)
|
||||
}
|
||||
|
||||
fn find_cg_window_id_for_pid(pid: i32) -> Option<u32> {
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
dictionary::CFDictionary,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
extern "C" {
|
||||
fn CGWindowListCopyWindowInfo(option: u32, window_id: u32) -> CFTypeRef;
|
||||
}
|
||||
|
||||
let info_ref = unsafe { CGWindowListCopyWindowInfo(17, 0) };
|
||||
if info_ref.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let array = unsafe { CFArray::<CFType>::wrap_under_create_rule(info_ref as _) };
|
||||
|
||||
let mut best_id: Option<u32> = None;
|
||||
let mut best_area: f64 = 0.0;
|
||||
|
||||
for item in array.iter() {
|
||||
let dict = unsafe {
|
||||
CFDictionary::<CFString, CFType>::wrap_under_get_rule(
|
||||
item.as_concrete_TypeRef() as _
|
||||
)
|
||||
};
|
||||
|
||||
let int_field = |key: &str| -> Option<i32> {
|
||||
let k = CFString::new(key);
|
||||
dict.find(&k).and_then(|v| {
|
||||
let n = unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) };
|
||||
n.to_i32()
|
||||
})
|
||||
};
|
||||
|
||||
if int_field("kCGWindowOwnerPID") != Some(pid) {
|
||||
continue;
|
||||
}
|
||||
if int_field("kCGWindowLayer").unwrap_or(99) != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let wid = match int_field("kCGWindowNumber") {
|
||||
Some(n) => n as u32,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let bounds_key = CFString::new("kCGWindowBounds");
|
||||
let area = if let Some(bounds_val) = dict.find(&bounds_key) {
|
||||
let bounds_dict = unsafe {
|
||||
CFDictionary::<CFString, CFType>::wrap_under_get_rule(
|
||||
bounds_val.as_concrete_TypeRef() as _,
|
||||
)
|
||||
};
|
||||
let w = bounds_dict.find(CFString::new("Width")).and_then(|v| {
|
||||
let n = unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) };
|
||||
n.to_f64()
|
||||
});
|
||||
let h = bounds_dict.find(CFString::new("Height")).and_then(|v| {
|
||||
let n = unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) };
|
||||
n.to_f64()
|
||||
});
|
||||
w.unwrap_or(0.0) * h.unwrap_or(0.0)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
if area > best_area {
|
||||
best_area = area;
|
||||
best_id = Some(wid);
|
||||
}
|
||||
}
|
||||
|
||||
best_id
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -57,8 +155,8 @@ mod imp {
|
|||
mod imp {
|
||||
use super::*;
|
||||
|
||||
pub fn capture_window(_id: u32) -> Result<ImageBuffer, AdapterError> {
|
||||
Err(AdapterError::not_supported("capture_window"))
|
||||
pub fn capture_app(_pid: i32) -> Result<ImageBuffer, AdapterError> {
|
||||
Err(AdapterError::not_supported("capture_app"))
|
||||
}
|
||||
|
||||
pub fn capture_screen(_idx: usize) -> Result<ImageBuffer, AdapterError> {
|
||||
|
|
@ -66,4 +164,4 @@ mod imp {
|
|||
}
|
||||
}
|
||||
|
||||
pub use imp::{capture_screen, capture_window};
|
||||
pub use imp::{capture_app, capture_screen};
|
||||
|
|
|
|||
252
crates/macos/src/surfaces.rs
Normal file
252
crates/macos/src/surfaces.rs
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
use crate::tree::{copy_ax_array, copy_string_attr, element_for_pid, AXElement};
|
||||
use agent_desktop_core::node::SurfaceInfo;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue, AXUIElementRef};
|
||||
use core_foundation::{
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
fn copy_element_attr(el: &AXElement, attr: &str) -> Option<AXElement> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
Some(AXElement(value as AXUIElementRef))
|
||||
}
|
||||
|
||||
fn copy_bool_attr(el: &AXElement, attr: &str) -> Option<bool> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf_type = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
cf_type.downcast::<CFBoolean>().map(|b| b.into())
|
||||
}
|
||||
|
||||
fn focused_window_element(pid: i32) -> Option<AXElement> {
|
||||
let app = element_for_pid(pid);
|
||||
copy_element_attr(&app, "AXFocusedWindow")
|
||||
}
|
||||
|
||||
/// Find an open menu bar menu by looking for AXMenuBarItem with AXSelected=true.
|
||||
/// This is the correct macOS mechanism — menus are always children of their bar item,
|
||||
/// not direct children of the application.
|
||||
fn open_menubar_menu(pid: i32) -> Option<AXElement> {
|
||||
let app = element_for_pid(pid);
|
||||
let app_children = copy_ax_array(&app, "AXChildren")?;
|
||||
let menubar = app_children
|
||||
.into_iter()
|
||||
.find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenuBar"))?;
|
||||
let items = copy_ax_array(&menubar, "AXChildren")?;
|
||||
for item in &items {
|
||||
if copy_string_attr(item, "AXRole").as_deref() != Some("AXMenuBarItem") {
|
||||
continue;
|
||||
}
|
||||
if !copy_bool_attr(item, "AXSelected").unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
if let Some(children) = copy_ax_array(item, "AXChildren") {
|
||||
return children
|
||||
.into_iter()
|
||||
.find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu"));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find a right-click context menu. After AXShowMenu, the menu appears
|
||||
/// as a child of the focused element (the one that was right-clicked).
|
||||
/// Falls back to scanning direct app children for Electron-style apps.
|
||||
fn context_menu_from_app(pid: i32) -> Option<AXElement> {
|
||||
let app = element_for_pid(pid);
|
||||
if let Some(focused) = copy_element_attr(&app, "AXFocusedUIElement") {
|
||||
if let Some(children) = copy_ax_array(&focused, "AXChildren") {
|
||||
if let Some(menu) = children
|
||||
.into_iter()
|
||||
.find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu"))
|
||||
{
|
||||
return Some(menu);
|
||||
}
|
||||
}
|
||||
}
|
||||
let children = copy_ax_array(&app, "AXChildren")?;
|
||||
children
|
||||
.into_iter()
|
||||
.find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu"))
|
||||
}
|
||||
|
||||
pub fn menu_element_for_pid(pid: i32) -> Option<AXElement> {
|
||||
open_menubar_menu(pid).or_else(|| context_menu_from_app(pid))
|
||||
}
|
||||
|
||||
pub fn focused_surface_for_pid(pid: i32) -> Option<AXElement> {
|
||||
focused_window_element(pid)
|
||||
}
|
||||
|
||||
fn first_child_with_subrole(pid: i32, subrole: &str) -> Option<AXElement> {
|
||||
let win = focused_window_element(pid)?;
|
||||
let children = copy_ax_array(&win, "AXChildren")?;
|
||||
children
|
||||
.into_iter()
|
||||
.find(|child| copy_string_attr(child, "AXSubrole").as_deref() == Some(subrole))
|
||||
}
|
||||
|
||||
pub fn sheet_for_pid(pid: i32) -> Option<AXElement> {
|
||||
first_child_with_subrole(pid, "AXSheet")
|
||||
}
|
||||
|
||||
pub fn popover_for_pid(pid: i32) -> Option<AXElement> {
|
||||
first_child_with_subrole(pid, "AXPopover")
|
||||
}
|
||||
|
||||
pub fn alert_for_pid(pid: i32) -> Option<AXElement> {
|
||||
let win = focused_window_element(pid)?;
|
||||
let children = copy_ax_array(&win, "AXChildren")?;
|
||||
children.into_iter().find(|child| {
|
||||
let subrole = copy_string_attr(child, "AXSubrole");
|
||||
matches!(subrole.as_deref(), Some("AXDialog") | Some("AXAlert"))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_menu_open(pid: i32) -> bool {
|
||||
open_menubar_menu(pid).is_some() || context_menu_from_app(pid).is_some()
|
||||
}
|
||||
|
||||
pub fn list_surfaces_for_pid(pid: i32) -> Vec<SurfaceInfo> {
|
||||
let mut surfaces = Vec::new();
|
||||
let app = element_for_pid(pid);
|
||||
|
||||
if let Some(app_children) = copy_ax_array(&app, "AXChildren") {
|
||||
for ch in &app_children {
|
||||
match copy_string_attr(ch, "AXRole").as_deref() {
|
||||
Some("AXMenuBar") => {
|
||||
if let Some(items) = copy_ax_array(ch, "AXChildren") {
|
||||
for item in &items {
|
||||
if copy_string_attr(item, "AXRole").as_deref()
|
||||
!= Some("AXMenuBarItem")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !copy_bool_attr(item, "AXSelected").unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let title = copy_string_attr(item, "AXTitle");
|
||||
if let Some(menu_children) = copy_ax_array(item, "AXChildren") {
|
||||
for menu in &menu_children {
|
||||
if copy_string_attr(menu, "AXRole").as_deref()
|
||||
== Some("AXMenu")
|
||||
{
|
||||
let item_count =
|
||||
copy_ax_array(menu, "AXChildren").map(|v| v.len());
|
||||
surfaces.push(SurfaceInfo {
|
||||
kind: "menu".into(),
|
||||
title: title.clone(),
|
||||
item_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("AXMenu") => {
|
||||
let title = copy_string_attr(ch, "AXTitle")
|
||||
.or_else(|| copy_string_attr(ch, "AXDescription"));
|
||||
let item_count = copy_ax_array(ch, "AXChildren").map(|v| v.len());
|
||||
surfaces.push(SurfaceInfo {
|
||||
kind: "context_menu".into(),
|
||||
title,
|
||||
item_count,
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(focused) = copy_element_attr(&app, "AXFocusedUIElement") {
|
||||
if let Some(children) = copy_ax_array(&focused, "AXChildren") {
|
||||
for ch in &children {
|
||||
if copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu") {
|
||||
let title = copy_string_attr(ch, "AXTitle")
|
||||
.or_else(|| copy_string_attr(ch, "AXDescription"));
|
||||
let item_count = copy_ax_array(ch, "AXChildren").map(|v| v.len());
|
||||
surfaces.push(SurfaceInfo {
|
||||
kind: "context_menu".into(),
|
||||
title,
|
||||
item_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(win) = focused_window_element(pid) {
|
||||
if let Some(children) = copy_ax_array(&win, "AXChildren") {
|
||||
for child in &children {
|
||||
let subrole = copy_string_attr(child, "AXSubrole");
|
||||
let kind = match subrole.as_deref() {
|
||||
Some("AXSheet") => "sheet",
|
||||
Some("AXPopover") => "popover",
|
||||
Some("AXDialog") | Some("AXAlert") => "alert",
|
||||
_ => continue,
|
||||
};
|
||||
let title = copy_string_attr(child, "AXTitle")
|
||||
.or_else(|| copy_string_attr(child, "AXDescription"));
|
||||
surfaces.push(SurfaceInfo {
|
||||
kind: kind.into(),
|
||||
title,
|
||||
item_count: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
surfaces
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
mod imp {
|
||||
use super::*;
|
||||
|
||||
pub fn menu_element_for_pid(_pid: i32) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn focused_surface_for_pid(_pid: i32) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn sheet_for_pid(_pid: i32) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn popover_for_pid(_pid: i32) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn alert_for_pid(_pid: i32) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn is_menu_open(_pid: i32) -> bool {
|
||||
false
|
||||
}
|
||||
pub fn list_surfaces_for_pid(_pid: i32) -> Vec<SurfaceInfo> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::{
|
||||
alert_for_pid, focused_surface_for_pid, is_menu_open, list_surfaces_for_pid,
|
||||
menu_element_for_pid, popover_for_pid, sheet_for_pid,
|
||||
};
|
||||
|
|
@ -7,15 +7,17 @@ pub const ABSOLUTE_MAX_DEPTH: u8 = 50;
|
|||
mod imp {
|
||||
use super::*;
|
||||
use accessibility_sys::{
|
||||
kAXChildrenAttribute, kAXDescriptionAttribute, kAXEnabledAttribute,
|
||||
kAXErrorSuccess, kAXFocusedAttribute, kAXRoleAttribute,
|
||||
kAXTitleAttribute, kAXValueAttribute,
|
||||
AXUIElementCopyAttributeValue, AXUIElementCreateApplication, AXUIElementRef,
|
||||
kAXChildrenAttribute, kAXContentsAttribute, kAXDescriptionAttribute, kAXEnabledAttribute,
|
||||
kAXErrorSuccess, kAXFocusedAttribute, kAXRoleAttribute, kAXTitleAttribute,
|
||||
kAXValueAttribute, kAXWindowsAttribute, AXUIElementCopyAttributeValue,
|
||||
AXUIElementCopyMultipleAttributeValues, AXUIElementCreateApplication, AXUIElementRef,
|
||||
AXUIElementSetMessagingTimeout,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
|
|
@ -39,7 +41,145 @@ mod imp {
|
|||
}
|
||||
|
||||
pub fn element_for_pid(pid: i32) -> AXElement {
|
||||
AXElement(unsafe { AXUIElementCreateApplication(pid) })
|
||||
let el = AXElement(unsafe { AXUIElementCreateApplication(pid) });
|
||||
if !el.0.is_null() {
|
||||
unsafe { AXUIElementSetMessagingTimeout(el.0, 2.0) };
|
||||
}
|
||||
el
|
||||
}
|
||||
|
||||
/// Find the AXWindow element whose title matches `win_title`.
|
||||
/// Falls back to the first window, then to the app element if no windows.
|
||||
pub fn window_element_for(pid: i32, win_title: &str) -> AXElement {
|
||||
let app = element_for_pid(pid);
|
||||
|
||||
if let Some(windows) = copy_ax_array(&app, kAXWindowsAttribute) {
|
||||
for win in &windows {
|
||||
let title = copy_string_attr(win, kAXTitleAttribute);
|
||||
if title.as_deref() == Some(win_title) {
|
||||
return win.clone();
|
||||
}
|
||||
}
|
||||
for win in &windows {
|
||||
let title = copy_string_attr(win, kAXTitleAttribute);
|
||||
if title
|
||||
.as_deref()
|
||||
.is_some_and(|t| t.contains(win_title) || win_title.contains(t))
|
||||
{
|
||||
return win.clone();
|
||||
}
|
||||
}
|
||||
if let Some(first) = windows.into_iter().next() {
|
||||
return first;
|
||||
}
|
||||
}
|
||||
|
||||
app
|
||||
}
|
||||
|
||||
/// Batch-fetch six most-used attributes. Returns (role, title, desc, value, enabled, focused).
|
||||
/// Value handles CFString, CFBoolean, and CFNumber types.
|
||||
fn fetch_node_attrs(
|
||||
el: &AXElement,
|
||||
) -> (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
bool,
|
||||
bool,
|
||||
) {
|
||||
let attr_names = [
|
||||
kAXRoleAttribute,
|
||||
kAXTitleAttribute,
|
||||
kAXDescriptionAttribute,
|
||||
kAXValueAttribute,
|
||||
kAXEnabledAttribute,
|
||||
kAXFocusedAttribute,
|
||||
];
|
||||
let cf_names: Vec<CFString> = attr_names.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
let mut result_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyMultipleAttributeValues(
|
||||
el.0,
|
||||
names_arr.as_concrete_TypeRef(),
|
||||
0,
|
||||
&mut result_ref as *mut _ as *mut _,
|
||||
)
|
||||
};
|
||||
|
||||
if err != kAXErrorSuccess || result_ref.is_null() {
|
||||
let role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
let val = copy_value_typed(el);
|
||||
let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true);
|
||||
let focused = copy_bool_attr(el, kAXFocusedAttribute).unwrap_or(false);
|
||||
return (role, title, desc, val, enabled, focused);
|
||||
}
|
||||
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(result_ref as _) };
|
||||
let items: Vec<Option<String>> = arr
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(idx, item)| {
|
||||
if let Some(s) = item.downcast::<CFString>() {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
match idx {
|
||||
3 => {
|
||||
if let Some(b) = item.downcast::<CFBoolean>() {
|
||||
return Some(bool::from(b).to_string());
|
||||
}
|
||||
if let Some(n) = item.downcast::<CFNumber>() {
|
||||
if let Some(i) = n.to_i64() {
|
||||
return Some(i.to_string());
|
||||
}
|
||||
if let Some(f) = n.to_f64() {
|
||||
return Some(format!("{:.2}", f));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
4 | 5 => item
|
||||
.downcast::<CFBoolean>()
|
||||
.map(|b| bool::from(b).to_string()),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let get = |i: usize| items.get(i).and_then(|v| v.clone());
|
||||
let role = get(0);
|
||||
let title = get(1);
|
||||
let desc = get(2);
|
||||
let val = get(3);
|
||||
let enabled = get(4).map(|s| s == "true").unwrap_or(true);
|
||||
let focused = get(5).map(|s| s == "true").unwrap_or(false);
|
||||
|
||||
(role, title, desc, val, enabled, focused)
|
||||
}
|
||||
|
||||
/// Compute the effective display name for any element, mirroring `build_subtree` name resolution.
|
||||
pub fn resolve_element_name(el: &AXElement) -> Option<String> {
|
||||
let ax_role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
|
||||
let name = title.or(desc);
|
||||
let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") {
|
||||
copy_string_attr(el, kAXValueAttribute).or(name)
|
||||
} else {
|
||||
name
|
||||
};
|
||||
|
||||
name.or_else(|| {
|
||||
let children = copy_ax_array(el, kAXChildrenAttribute).unwrap_or_default();
|
||||
label_from_children(&children)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn build_subtree(
|
||||
|
|
@ -56,28 +196,41 @@ mod imp {
|
|||
return None;
|
||||
}
|
||||
|
||||
let role = copy_string_attr(el, kAXRoleAttribute)?;
|
||||
let normalized_role = crate::roles::ax_role_to_str(&role).to_string();
|
||||
let (ax_role, title, ax_desc, value, enabled, focused) = fetch_node_attrs(el);
|
||||
|
||||
let role = ax_role
|
||||
.as_deref()
|
||||
.map(crate::roles::ax_role_to_str)
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let ax_desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
let name = title.clone().or_else(|| ax_desc.clone());
|
||||
let description = if title.is_some() { ax_desc } else { None };
|
||||
|
||||
let value = copy_string_attr(el, kAXValueAttribute);
|
||||
let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") {
|
||||
value.clone().or(name)
|
||||
} else {
|
||||
name
|
||||
};
|
||||
|
||||
let mut states = Vec::new();
|
||||
if copy_bool_attr(el, kAXFocusedAttribute) == Some(true) {
|
||||
if focused {
|
||||
states.push("focused".into());
|
||||
}
|
||||
if copy_bool_attr(el, kAXEnabledAttribute) == Some(false) {
|
||||
if !enabled {
|
||||
states.push("disabled".into());
|
||||
}
|
||||
|
||||
let bounds = if include_bounds { read_bounds(el) } else { None };
|
||||
let bounds = if include_bounds {
|
||||
read_bounds(el)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let children = copy_children(el)
|
||||
.unwrap_or_default()
|
||||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
let name = name.or_else(|| label_from_children(&children_raw));
|
||||
|
||||
let children = children_raw
|
||||
.into_iter()
|
||||
.filter_map(|child| {
|
||||
build_subtree(&child, depth + 1, max_depth, include_bounds, visited)
|
||||
|
|
@ -86,7 +239,7 @@ mod imp {
|
|||
|
||||
Some(AccessibilityNode {
|
||||
ref_id: None,
|
||||
role: normalized_role,
|
||||
role,
|
||||
name,
|
||||
value,
|
||||
description,
|
||||
|
|
@ -96,15 +249,64 @@ mod imp {
|
|||
})
|
||||
}
|
||||
|
||||
/// Scan immediate children (and one level deeper through AXCell) to find a text label.
|
||||
/// Resolves the common macOS pattern: AXRow → AXCell → AXStaticText.kAXValue = "label".
|
||||
fn label_from_children(children: &[AXElement]) -> Option<String> {
|
||||
fn text_of(el: &AXElement) -> Option<String> {
|
||||
copy_string_attr(el, kAXValueAttribute)
|
||||
.or_else(|| copy_string_attr(el, kAXTitleAttribute))
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
for child in children.iter().take(5) {
|
||||
match copy_string_attr(child, kAXRoleAttribute).as_deref() {
|
||||
Some("AXStaticText") => {
|
||||
if let Some(s) = text_of(child) {
|
||||
return Some(s);
|
||||
}
|
||||
}
|
||||
Some("AXCell") | Some("AXGroup") => {
|
||||
for gc in copy_ax_array(child, kAXChildrenAttribute).unwrap_or_default() {
|
||||
if copy_string_attr(&gc, kAXRoleAttribute).as_deref()
|
||||
== Some("AXStaticText")
|
||||
{
|
||||
if let Some(s) = text_of(&gc) {
|
||||
return Some(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Read children using the appropriate attribute for this element's role.
|
||||
/// AXBrowser exposes its content via AXColumns, not AXChildren.
|
||||
fn copy_children(el: &AXElement, ax_role: Option<&str>) -> Option<Vec<AXElement>> {
|
||||
if ax_role == Some("AXBrowser") {
|
||||
return copy_ax_array(el, "AXColumns");
|
||||
}
|
||||
for attr in &[
|
||||
kAXChildrenAttribute,
|
||||
kAXContentsAttribute,
|
||||
"AXChildrenInNavigationOrder",
|
||||
] {
|
||||
if let Some(v) = copy_ax_array(el, attr) {
|
||||
if !v.is_empty() {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn copy_string_attr(el: &AXElement, attr: &str) -> Option<String> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut value,
|
||||
)
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
|
|
@ -113,15 +315,39 @@ mod imp {
|
|||
cf_type.downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
/// Read kAXValueAttribute handling CFString, CFBoolean, and CFNumber.
|
||||
fn copy_value_typed(el: &AXElement) -> Option<String> {
|
||||
let cf_attr = CFString::new(kAXValueAttribute);
|
||||
let mut val_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut val_ref)
|
||||
};
|
||||
if err != kAXErrorSuccess || val_ref.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(val_ref) };
|
||||
if let Some(s) = cf.downcast::<CFString>() {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
if let Some(b) = cf.downcast::<CFBoolean>() {
|
||||
return Some(bool::from(b).to_string());
|
||||
}
|
||||
if let Some(n) = cf.downcast::<CFNumber>() {
|
||||
if let Some(i) = n.to_i64() {
|
||||
return Some(i.to_string());
|
||||
}
|
||||
if let Some(f) = n.to_f64() {
|
||||
return Some(format!("{:.2}", f));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn copy_bool_attr(el: &AXElement, attr: &str) -> Option<bool> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut value,
|
||||
)
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
|
|
@ -130,40 +356,48 @@ mod imp {
|
|||
cf_type.downcast::<CFBoolean>().map(|b| b.into())
|
||||
}
|
||||
|
||||
fn copy_children(el: &AXElement) -> Option<Vec<AXElement>> {
|
||||
let cf_attr = CFString::new(kAXChildrenAttribute);
|
||||
/// Read an array-typed AX attribute, retaining each AXUIElement.
|
||||
pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option<Vec<AXElement>> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut value,
|
||||
)
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(value as _) };
|
||||
let children = arr
|
||||
let children: Vec<AXElement> = arr
|
||||
.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(AXElement(ptr))
|
||||
return None;
|
||||
}
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(AXElement(ptr))
|
||||
})
|
||||
.collect();
|
||||
Some(children)
|
||||
}
|
||||
|
||||
fn read_bounds(el: &AXElement) -> Option<Rect> {
|
||||
pub fn copy_element_attr(el: &AXElement, attr: &str) -> Option<AXElement> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let ptr = value as AXUIElementRef;
|
||||
Some(AXElement(ptr))
|
||||
}
|
||||
|
||||
pub fn read_bounds(el: &AXElement) -> Option<Rect> {
|
||||
use accessibility_sys::{
|
||||
kAXPositionAttribute, kAXSizeAttribute,
|
||||
kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint, kAXValueTypeCGSize,
|
||||
AXValueGetValue,
|
||||
kAXValueTypeCGPoint, kAXValueTypeCGSize,
|
||||
};
|
||||
use core_graphics::geometry::{CGPoint, CGSize};
|
||||
use std::ffi::c_void;
|
||||
|
|
@ -176,6 +410,7 @@ mod imp {
|
|||
if pos_ok != kAXErrorSuccess || pos_ref.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut point = CGPoint::new(0.0, 0.0);
|
||||
let got_pos = unsafe {
|
||||
AXValueGetValue(
|
||||
|
|
@ -197,6 +432,7 @@ mod imp {
|
|||
if size_ok != kAXErrorSuccess || size_ref.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut size = CGSize::new(0.0, 0.0);
|
||||
let got_size = unsafe {
|
||||
AXValueGetValue(
|
||||
|
|
@ -210,7 +446,12 @@ mod imp {
|
|||
return None;
|
||||
}
|
||||
|
||||
Some(Rect { x: point.x, y: point.y, width: size.width, height: size.height })
|
||||
Some(Rect {
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -223,7 +464,6 @@ mod imp {
|
|||
impl Drop for AXElement {
|
||||
fn drop(&mut self) {}
|
||||
}
|
||||
|
||||
impl Clone for AXElement {
|
||||
fn clone(&self) -> Self {
|
||||
AXElement(self.0)
|
||||
|
|
@ -233,6 +473,24 @@ mod imp {
|
|||
pub fn element_for_pid(_pid: i32) -> AXElement {
|
||||
AXElement(std::ptr::null())
|
||||
}
|
||||
pub fn window_element_for(_pid: i32, _win_title: &str) -> AXElement {
|
||||
AXElement(std::ptr::null())
|
||||
}
|
||||
pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option<Vec<AXElement>> {
|
||||
None
|
||||
}
|
||||
pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
pub fn copy_element_attr(_el: &AXElement, _attr: &str) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn read_bounds(_el: &AXElement) -> Option<agent_desktop_core::node::Rect> {
|
||||
None
|
||||
}
|
||||
pub fn resolve_element_name(_el: &AXElement) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn build_subtree(
|
||||
_el: &AXElement,
|
||||
|
|
@ -243,10 +501,9 @@ mod imp {
|
|||
) -> Option<AccessibilityNode> {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::{build_subtree, copy_string_attr, element_for_pid, AXElement};
|
||||
pub use imp::{
|
||||
build_subtree, copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid,
|
||||
read_bounds, resolve_element_name, window_element_for, AXElement,
|
||||
};
|
||||
|
|
|
|||
37
crates/macos/src/wait.rs
Normal file
37
crates/macos/src/wait.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
use agent_desktop_core::error::AdapterError;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use crate::surfaces::is_menu_open;
|
||||
|
||||
pub fn wait_for_menu(pid: i32, open: bool, timeout_ms: u64) -> Result<(), AdapterError> {
|
||||
let deadline = Instant::now() + Duration::from_millis(timeout_ms);
|
||||
loop {
|
||||
if is_menu_open(pid) == open {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let msg = if open {
|
||||
format!("No context menu opened within {timeout_ms}ms")
|
||||
} else {
|
||||
format!("Context menu did not close within {timeout_ms}ms")
|
||||
};
|
||||
return Err(AdapterError::timeout(msg));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
mod imp {
|
||||
use super::*;
|
||||
|
||||
pub fn wait_for_menu(_pid: i32, _open: bool, _timeout_ms: u64) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("wait_for_menu"))
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::wait_for_menu;
|
||||
126
crates/macos/src/window_ops.rs
Normal file
126
crates/macos/src/window_ops.rs
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
use agent_desktop_core::{action::WindowOp, error::AdapterError, node::WindowInfo};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use accessibility_sys::{
|
||||
kAXErrorSuccess, kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint,
|
||||
kAXValueTypeCGSize, AXUIElementPerformAction, AXUIElementSetAttributeValue,
|
||||
};
|
||||
use agent_desktop_core::error::ErrorCode;
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
use core_graphics::geometry::{CGPoint, CGSize};
|
||||
use std::ffi::c_void;
|
||||
|
||||
extern "C" {
|
||||
fn AXValueCreate(value_type: u32, value_ptr: *const c_void) -> *mut c_void;
|
||||
}
|
||||
|
||||
pub fn execute(win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> {
|
||||
let win_el = crate::tree::window_element_for(win.pid, &win.title);
|
||||
match op {
|
||||
WindowOp::Resize { width, height } => set_size(&win_el, width, height),
|
||||
WindowOp::Move { x, y } => set_position(&win_el, x, y),
|
||||
WindowOp::Minimize => set_minimized(&win_el, true),
|
||||
WindowOp::Maximize => press_zoom_button(&win_el),
|
||||
WindowOp::Restore => set_minimized(&win_el, false),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_size(el: &crate::tree::AXElement, width: f64, height: f64) -> Result<(), AdapterError> {
|
||||
let size = CGSize::new(width, height);
|
||||
let ax_value =
|
||||
unsafe { AXValueCreate(kAXValueTypeCGSize, &size as *const _ as *const c_void) };
|
||||
if ax_value.is_null() {
|
||||
return Err(AdapterError::internal("Failed to create AXValue for size"));
|
||||
}
|
||||
let cf_attr = CFString::new(kAXSizeAttribute);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), ax_value as _)
|
||||
};
|
||||
unsafe { core_foundation::base::CFRelease(ax_value as _) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("Resize failed (err={err})"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_position(el: &crate::tree::AXElement, x: f64, y: f64) -> Result<(), AdapterError> {
|
||||
let point = CGPoint::new(x, y);
|
||||
let ax_value =
|
||||
unsafe { AXValueCreate(kAXValueTypeCGPoint, &point as *const _ as *const c_void) };
|
||||
if ax_value.is_null() {
|
||||
return Err(AdapterError::internal(
|
||||
"Failed to create AXValue for position",
|
||||
));
|
||||
}
|
||||
let cf_attr = CFString::new(kAXPositionAttribute);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), ax_value as _)
|
||||
};
|
||||
unsafe { core_foundation::base::CFRelease(ax_value as _) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("Move failed (err={err})"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_minimized(el: &crate::tree::AXElement, minimized: bool) -> Result<(), AdapterError> {
|
||||
let cf_attr = CFString::new("AXMinimized");
|
||||
let val = if minimized {
|
||||
CFBoolean::true_value()
|
||||
} else {
|
||||
CFBoolean::false_value()
|
||||
};
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), val.as_CFTypeRef())
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
let op = if minimized { "Minimize" } else { "Restore" };
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("{op} failed (err={err})"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn press_zoom_button(el: &crate::tree::AXElement) -> Result<(), AdapterError> {
|
||||
let zoom = crate::tree::copy_element_attr(el, "AXZoomButton");
|
||||
match zoom {
|
||||
Some(btn) => {
|
||||
let action = CFString::new("AXPress");
|
||||
let err = unsafe { AXUIElementPerformAction(btn.0, action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("Zoom button press failed (err={err})"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
None => Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
"Window has no zoom button",
|
||||
)
|
||||
.with_suggestion("Window may not support maximizing")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
mod imp {
|
||||
use super::*;
|
||||
|
||||
pub fn execute(_win: &WindowInfo, _op: WindowOp) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("window_op"))
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::execute;
|
||||
|
|
@ -11,11 +11,19 @@ deepened: 2026-02-19
|
|||
|
||||
## Enhancement Summary
|
||||
|
||||
**Deepened on:** 2026-02-19
|
||||
**Research agents used:** 10 (Architecture Strategist, Performance Oracle, Security Sentinel, Code Simplicity Reviewer, Agent-Native Reviewer, Pattern Recognition Specialist, Best Practices Researcher, Framework Docs Researcher, Spec Flow Analyzer, Agent-Native Architecture Skill)
|
||||
**Deepened on:** 2026-02-19 (round 1), 2026-02-19 (round 2 — code quality audit)
|
||||
**Research agents used:** 15 (Architecture Strategist, Performance Oracle, Security Sentinel, Code Simplicity Reviewer, Agent-Native Reviewer, Pattern Recognition Specialist, Best Practices Researcher, Framework Docs Researcher, Spec Flow Analyzer, Agent-Native Architecture Skill, Dead Code/LOC Auditor, Test-Before-Implement Researcher, Modular Architecture Researcher, Code Quality Reviewer, Architecture Extensibility Reviewer)
|
||||
**Context7 queries:** clap 4 derive patterns, serde performance patterns
|
||||
**Web research:** macOS AX FFI best practices, cargo-dist distribution, rmcp latest version, accessibility-sys alternatives, thiserror 2.0 changes
|
||||
|
||||
### Round 2 Structural Findings (Fix During Implementation)
|
||||
|
||||
- **Dead code:** `clipboard.rs` (zero callers), `batch::execute` stub (never called)
|
||||
- **LOC violation:** `tree.rs` at 403 lines — split into `ax_element.rs` / `ax_attrs.rs` / `ax_tree.rs`
|
||||
- **Dispatch duplication:** 187/397 lines in `dispatch.rs` are a near-verbatim parallel dispatch table — collapse to single code path via `Commands` enum deserialization
|
||||
- **Probe binaries:** `axprobe.rs` / `axprobe2.rs` in `src/bin/` compile on every build — move to `examples/` with `required-features = ["dev-tools"]`
|
||||
- **Bugs:** `wait.rs` silently discards RefMap load errors; `is_check.rs` reads stale state; `press.rs` undocumented null-handle convention
|
||||
|
||||
### Blockers (Fix Before Writing Code)
|
||||
|
||||
1. **Circular dependency in CommandRegistry** — `CommandRegistry::dispatch` references `crate::cli::Commands` from the binary crate, creating a circular dep (core ↔ binary). **Fix:** Drop `Command` trait and `CommandRegistry` entirely; dispatch via `match` in the binary crate.
|
||||
|
|
@ -941,7 +949,11 @@ Use `serde_json::to_writer(BufWriter::new(stdout.lock()), &data)` instead of `to
|
|||
|
||||
## macOS Adapter Implementation
|
||||
|
||||
### `crates/macos/src/tree.rs` — Key patterns
|
||||
### `crates/macos/src/` — Tree module split
|
||||
|
||||
`tree.rs` exceeds 400 LOC and is split into three files (see "Code Quality and Structural Refactors" section). The patterns below apply to the split files: `AXElement` and `element_for_pid` go in `ax_element.rs`, attribute helpers in `ax_attrs.rs`, traversal logic in `ax_tree.rs`.
|
||||
|
||||
### `crates/macos/src/ax_element.rs` — Key patterns
|
||||
|
||||
```rust
|
||||
use accessibility_sys::{
|
||||
|
|
@ -1495,15 +1507,15 @@ macos_large_tree_test - snapshot Xcode → completes under 2 seconds
|
|||
### Weeks 3–4: macOS Tree Traversal
|
||||
|
||||
**Deliverables:**
|
||||
- `crates/macos/src/tree.rs`: `AXElement` newtype with `Drop`, `element_for_pid()`,
|
||||
`build_subtree()` with cycle detection via `HashSet<usize>`, depth limiting
|
||||
- `crates/macos/src/roles.rs`: AXRole string → unified role enum mapping
|
||||
(AXButton → "button", AXTextField → "textfield", etc.)
|
||||
- `crates/macos/src/ax_element.rs`: `AXElement` newtype with `Drop`, `Clone` (CFRetain), `element_for_pid()`, `window_element_for()`
|
||||
- `crates/macos/src/ax_attrs.rs`: `copy_string_attr`, `copy_ax_array`, `fetch_node_attrs`, `read_bounds`
|
||||
- `crates/macos/src/ax_tree.rs`: `build_subtree()` with cycle detection via `FxHashSet<usize>`, depth limiting, `resolve_element_name`
|
||||
- `crates/macos/src/roles.rs`: AXRole string → unified role enum mapping (AXButton → "button", AXTextField → "textfield", etc.)
|
||||
- `crates/macos/src/permissions.rs`: `is_trusted()`, `request_trust()`
|
||||
- `crates/macos/src/adapter.rs`: `MacOSAdapter::get_tree()` and
|
||||
`MacOSAdapter::check_permissions()` implemented; all other methods stub
|
||||
- Unit tests: role mapping coverage, cycle detection (synthetic circular tree),
|
||||
permission mock
|
||||
- `crates/macos/src/adapter.rs`: `MacOSAdapter::get_tree()` and `MacOSAdapter::check_permissions()` implemented; all other methods stub
|
||||
- `crates/macos/examples/axprobe.rs`: probe binary moved from `src/bin/`, gated on `dev-tools` feature
|
||||
- Unit tests: role mapping coverage, cycle detection (synthetic circular tree), permission mock
|
||||
- Stage 1–3 probe validation completed for each AX attribute used before wiring into adapter
|
||||
- Manual validation: `cargo run -- snapshot --app Finder` produces valid JSON
|
||||
|
||||
**Gotchas from research:**
|
||||
|
|
@ -1720,6 +1732,154 @@ From the Agent-Native Reviewer (score: 26/30, 4 structural gaps):
|
|||
|
||||
---
|
||||
|
||||
## Code Quality and Structural Refactors
|
||||
|
||||
These findings are not blockers for Phase 1 feature work, but must be resolved before the Phase 1 milestone is closed. Each has a clear fix and zero risk of behaviour change.
|
||||
|
||||
### Dead Code to Remove
|
||||
|
||||
**`crates/core/src/commands/clipboard.rs`** — 13 lines, exports `execute_get()` and `execute_set()` with zero callers anywhere in the workspace. The clipboard commands are implemented in the correct split files `clipboard_get.rs` and `clipboard_set.rs` (one command per file), which `dispatch.rs` calls directly. `clipboard.rs` is a leftover from before the split and should be deleted along with its `pub mod clipboard;` line in `mod.rs`.
|
||||
|
||||
**`batch::execute` stub** — `crates/core/src/commands/batch.rs` exports an `execute()` function that returns `Ok(json!({"note": "..."}))`. It is never called; `dispatch.rs` handles batch routing inline. Delete the `execute` function body. Keep `BatchArgs`, `BatchCommand`, and `parse_commands` — they are live.
|
||||
|
||||
### LOC Violations
|
||||
|
||||
**`crates/macos/src/tree.rs` is at 403 lines** — exceeds the 400-line hard limit. Split into three files by single responsibility:
|
||||
|
||||
| New file | Contents | Estimated LOC |
|
||||
|---|---|---|
|
||||
| `ax_element.rs` | `AXElement` newtype, `Drop`, `Clone` (with `CFRetain`), `element_for_pid`, `window_element_for` | ~55 |
|
||||
| `ax_attrs.rs` | `copy_string_attr`, `copy_value_typed`, `copy_bool_attr`, `copy_ax_array`, `read_bounds`, `fetch_node_attrs` | ~200 |
|
||||
| `ax_tree.rs` | `build_subtree`, `resolve_element_name`, `label_from_children`, `copy_children` | ~115 |
|
||||
|
||||
**`crates/macos/src/lib.rs`** — update module declarations:
|
||||
```rust
|
||||
pub mod ax_element;
|
||||
pub mod ax_attrs;
|
||||
pub mod ax_tree;
|
||||
// Remove: pub mod tree;
|
||||
```
|
||||
|
||||
Internal cross-file imports use `super::` paths:
|
||||
```rust
|
||||
// ax_tree.rs
|
||||
use super::ax_element::AXElement;
|
||||
use super::ax_attrs::{copy_string_attr, fetch_node_attrs};
|
||||
```
|
||||
|
||||
**`src/dispatch.rs` is at 397 lines** — 3 lines under the limit but contains 187 lines of duplication (see below). Collapsing the duplication brings it to ~210 lines.
|
||||
|
||||
### Dispatch Duplication
|
||||
|
||||
`dispatch.rs` has two parallel dispatch tables:
|
||||
- Lines 16–169: `dispatch()` — type-safe `Commands` enum, 29 match arms
|
||||
- Lines 171–358: `dispatch_batch_command()` — string-keyed version, same 29 commands parsed differently
|
||||
|
||||
**Fix:** Collapse `dispatch_batch_command()` by deserializing batch JSON into the `Commands` enum. Because `Commands` derives `Deserialize`, `serde_json::from_value(json!({"snapshot": {"app": "Finder"}}))` already works:
|
||||
|
||||
```rust
|
||||
pub fn dispatch_batch_command(
|
||||
name: &str,
|
||||
args: serde_json::Value,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<serde_json::Value, AppError> {
|
||||
let cmd: Commands = serde_json::from_value(serde_json::json!({ name: args }))
|
||||
.map_err(|e| AppError::invalid_args(e.to_string()))?;
|
||||
dispatch(cmd, adapter)
|
||||
}
|
||||
```
|
||||
|
||||
This eliminates ~187 lines of duplication. `batch.rs`'s `parse_commands` routes `{"command": "snapshot", "args": {...}}` entries to `dispatch_batch_command("snapshot", args, adapter)` — one unified code path.
|
||||
|
||||
### Probe Binary Reorganization
|
||||
|
||||
`src/bin/axprobe.rs` and `src/bin/axprobe2.rs` are development probe binaries compiled on every `cargo build`. They should live in `examples/` and only compile when explicitly requested:
|
||||
|
||||
```
|
||||
crates/macos/
|
||||
└── examples/
|
||||
├── axprobe.rs # moved from src/bin/axprobe.rs
|
||||
├── axprobe2.rs # moved from src/bin/axprobe2.rs
|
||||
└── probe_utils.rs # extract shared helpers (find_pid, string readers, array readers)
|
||||
```
|
||||
|
||||
In `crates/macos/Cargo.toml`:
|
||||
```toml
|
||||
[[example]]
|
||||
name = "axprobe"
|
||||
path = "examples/axprobe.rs"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[features]
|
||||
dev-tools = []
|
||||
```
|
||||
|
||||
Run with: `cargo run -p agent-desktop-macos --example axprobe --features dev-tools`
|
||||
|
||||
### Bugs to Fix Before Phase 1 Ships
|
||||
|
||||
| File | Bug | Fix |
|
||||
|---|---|---|
|
||||
| `wait.rs` | `if let Ok(refmap) = RefMap::load()` silently discards load errors on every polling iteration | Propagate with `?` or emit `tracing::warn!` |
|
||||
| `is_check.rs` | States read from stale RefMap entry; `_handle` is resolved (liveness check) but then discarded | Add `///` doc-comment: "Returns state from last snapshot. Run `snapshot` first for live state." |
|
||||
| `press.rs` | `parts.last().unwrap()` — safe but proof depends on preceding `is_empty` guard being visible | Change to `parts.last().ok_or_else(|| AppError::invalid_args("empty key combo"))` |
|
||||
| `press.rs` | `NativeHandle::null()` passed for `PressKey` action — convention undocumented | Add `// SAFETY: PressKey synthesises a global CGEvent; no element handle is required.` |
|
||||
|
||||
---
|
||||
|
||||
## Test-Before-Implement Development Workflow
|
||||
|
||||
Every new AX capability must be confirmed at the OS level before being wired into the adapter. This 4-stage process eliminates "wrote 100 lines then discovered the attribute doesn't exist" failures. Document this in `CLAUDE.md` under `## Development Workflow`.
|
||||
|
||||
### Stage 1 — Accessibility Inspector
|
||||
|
||||
Open `/Applications/Xcode.app/Contents/Applications/Accessibility Inspector.app`. Point at the target application. Confirm the AX attribute exists and has a non-null value. Note the exact attribute name string (e.g. `AXMenus`, `AXFocusedWindow`).
|
||||
|
||||
### Stage 2 — JXA Probe (30-second sanity check)
|
||||
|
||||
```bash
|
||||
osascript -l JavaScript -e '
|
||||
var sys = Application("System Events");
|
||||
var proc = sys.processes.whose({ name: "Finder" })[0];
|
||||
// inspect proc.windows(), proc.menuBars() etc.
|
||||
JSON.stringify(proc.windows[0].attributes.whose({ name: "AXRole" })[0].value())
|
||||
'
|
||||
```
|
||||
|
||||
If this returns the expected value, proceed to Stage 3. If it errors or returns null, the attribute is not accessible via standard AX for this app/macOS version — stop and reassess.
|
||||
|
||||
### Stage 3 — Rust Probe in examples/
|
||||
|
||||
Add a probe function to `examples/axprobe.rs` that calls the candidate AX API directly and prints the result:
|
||||
|
||||
```rust
|
||||
// examples/axprobe.rs
|
||||
fn probe_ax_menus(pid: i32) {
|
||||
let app = ax_element::element_for_pid(pid);
|
||||
let menus = ax_attrs::copy_ax_array(&app, "AXMenus");
|
||||
println!("AXMenus count: {:?}", menus.map(|v| v.len()));
|
||||
}
|
||||
```
|
||||
|
||||
Run: `cargo run -p agent-desktop-macos --example axprobe --features dev-tools -- --pid $(pgrep Finder)`
|
||||
|
||||
Only proceed to Stage 4 when Stage 3 prints the expected value.
|
||||
|
||||
### Stage 4 — Implement as `pub(crate)` + Unit Test
|
||||
|
||||
1. Add the function to the appropriate `ax_*.rs` file with `pub(crate)` visibility
|
||||
2. Write a unit test (with MockAdapter or synthetic tree if real AX not available)
|
||||
3. Wire into `adapter.rs` / `adapter.get_tree()`
|
||||
|
||||
### Why the Stages Matter
|
||||
|
||||
- AX attribute availability varies by app (Electron apps expose nothing; native apps vary)
|
||||
- Some attributes exist but always return `kAXErrorNoValue` for certain element subtypes
|
||||
- Stage 2 costs 30 seconds; Stage 3 costs 5 minutes; Stage 4 costs hours — fail early
|
||||
- The probe files double as executable documentation of how each AX API was validated
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- PRD v2.0: `agent_desktop_prd_v2.pdf`
|
||||
|
|
|
|||
470
src/batch_dispatch.rs
Normal file
470
src/batch_dispatch.rs
Normal file
|
|
@ -0,0 +1,470 @@
|
|||
use agent_desktop_core::{
|
||||
commands::{
|
||||
check, clear, click, clipboard_clear, clipboard_get, clipboard_set, close_app, collapse,
|
||||
double_click, drag, expand, find, focus, focus_window, get, helpers, hover, is_check,
|
||||
key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize, minimize,
|
||||
mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press,
|
||||
resize_window, restore, right_click, screenshot, scroll, scroll_to, select, set_value,
|
||||
snapshot, status, toggle, triple_click, type_text, uncheck, version, wait,
|
||||
},
|
||||
error::AppError,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::dispatch::{parse_direction, parse_mouse_button, parse_xy};
|
||||
|
||||
pub fn dispatch_batch_command(
|
||||
command: &str,
|
||||
args: Value,
|
||||
adapter: &dyn agent_desktop_core::adapter::PlatformAdapter,
|
||||
) -> Result<Value, AppError> {
|
||||
fn str_field(v: &Value, key: &str) -> Option<String> {
|
||||
v.get(key).and_then(|v| v.as_str()).map(String::from)
|
||||
}
|
||||
|
||||
fn req_str(v: &Value, key: &str) -> Result<String, AppError> {
|
||||
str_field(v, key).ok_or_else(|| {
|
||||
AppError::invalid_input(format!("Batch: missing required field '{key}'"))
|
||||
})
|
||||
}
|
||||
|
||||
match command {
|
||||
"snapshot" => snapshot::execute(
|
||||
snapshot::SnapshotArgs {
|
||||
app: str_field(&args, "app"),
|
||||
window_id: str_field(&args, "window_id"),
|
||||
max_depth: args
|
||||
.get("max_depth")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|v| v as u8)
|
||||
.unwrap_or(10),
|
||||
include_bounds: args
|
||||
.get("include_bounds")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false),
|
||||
interactive_only: args
|
||||
.get("interactive_only")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false),
|
||||
compact: args
|
||||
.get("compact")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false),
|
||||
surface: parse_batch_surface(args.get("surface").and_then(|v| v.as_str())),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"find" => find::execute(
|
||||
find::FindArgs {
|
||||
app: str_field(&args, "app"),
|
||||
role: str_field(&args, "role"),
|
||||
name: str_field(&args, "name"),
|
||||
value: str_field(&args, "value"),
|
||||
text: str_field(&args, "text"),
|
||||
count: args.get("count").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
first: args.get("first").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
last: args.get("last").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
nth: args.get("nth").and_then(|v| v.as_u64()).map(|v| v as usize),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"screenshot" => screenshot::execute(
|
||||
screenshot::ScreenshotArgs {
|
||||
app: str_field(&args, "app"),
|
||||
window_id: str_field(&args, "window_id"),
|
||||
output_path: str_field(&args, "output_path").map(std::path::PathBuf::from),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"get" => get::execute(
|
||||
get::GetArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
property: crate::dispatch::parse_get_property(
|
||||
args.get("property")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("text"),
|
||||
)?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"is" => is_check::execute(
|
||||
is_check::IsArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
property: crate::dispatch::parse_is_property(
|
||||
args.get("property")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("visible"),
|
||||
)?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"click" => click::execute(
|
||||
click::ClickArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"double-click" => double_click::execute(
|
||||
double_click::DoubleClickArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"triple-click" => triple_click::execute(
|
||||
triple_click::TripleClickArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"right-click" => right_click::execute(
|
||||
right_click::RightClickArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"focus" => focus::execute(
|
||||
helpers::RefArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"toggle" => toggle::execute(
|
||||
helpers::RefArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"check" => check::execute(
|
||||
check::CheckArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"uncheck" => uncheck::execute(
|
||||
uncheck::UncheckArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"expand" => expand::execute(
|
||||
helpers::RefArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"collapse" => collapse::execute(
|
||||
helpers::RefArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"clear" => clear::execute(
|
||||
clear::ClearArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
"scroll-to" => scroll_to::execute(
|
||||
scroll_to::ScrollToArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"type" => type_text::execute(
|
||||
type_text::TypeArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
text: req_str(&args, "text")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"set-value" => set_value::execute(
|
||||
set_value::SetValueArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
value: req_str(&args, "value")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"select" => select::execute(
|
||||
select::SelectArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
value: req_str(&args, "value")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"scroll" => scroll::execute(
|
||||
scroll::ScrollArgs {
|
||||
ref_id: req_str(&args, "ref_id")?,
|
||||
direction: parse_direction(
|
||||
args.get("direction")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("down"),
|
||||
)?,
|
||||
amount: args
|
||||
.get("amount")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|v| v as u32)
|
||||
.unwrap_or(3),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"press" => press::execute(
|
||||
press::PressArgs {
|
||||
combo: req_str(&args, "combo")?,
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"key-down" => key_down::execute(
|
||||
key_down::KeyDownArgs {
|
||||
combo: req_str(&args, "combo")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"key-up" => key_up::execute(
|
||||
key_up::KeyUpArgs {
|
||||
combo: req_str(&args, "combo")?,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"hover" => {
|
||||
let xy = str_field(&args, "xy").map(|s| parse_xy(&s)).transpose()?;
|
||||
hover::execute(
|
||||
hover::HoverArgs {
|
||||
ref_id: str_field(&args, "ref_id"),
|
||||
xy,
|
||||
duration_ms: args.get("duration_ms").and_then(|v| v.as_u64()),
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
"drag" => {
|
||||
let from_xy = str_field(&args, "from_xy")
|
||||
.map(|s| parse_xy(&s))
|
||||
.transpose()?;
|
||||
let to_xy = str_field(&args, "to_xy")
|
||||
.map(|s| parse_xy(&s))
|
||||
.transpose()?;
|
||||
drag::execute(
|
||||
drag::DragArgs {
|
||||
from_ref: str_field(&args, "from"),
|
||||
from_xy,
|
||||
to_ref: str_field(&args, "to"),
|
||||
to_xy,
|
||||
duration_ms: args.get("duration_ms").and_then(|v| v.as_u64()),
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
"mouse-move" => {
|
||||
let xy = req_str(&args, "xy")?;
|
||||
let (x, y) = parse_xy(&xy)?;
|
||||
mouse_move::execute(mouse_move::MouseMoveArgs { x, y }, adapter)
|
||||
}
|
||||
|
||||
"mouse-click" => {
|
||||
let xy = req_str(&args, "xy")?;
|
||||
let (x, y) = parse_xy(&xy)?;
|
||||
let button = args
|
||||
.get("button")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("left");
|
||||
mouse_click::execute(
|
||||
mouse_click::MouseClickArgs {
|
||||
x,
|
||||
y,
|
||||
button: parse_mouse_button(button)?,
|
||||
count: args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as u32,
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
"mouse-down" => {
|
||||
let xy = req_str(&args, "xy")?;
|
||||
let (x, y) = parse_xy(&xy)?;
|
||||
let button = args
|
||||
.get("button")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("left");
|
||||
mouse_down::execute(
|
||||
mouse_down::MouseDownArgs {
|
||||
x,
|
||||
y,
|
||||
button: parse_mouse_button(button)?,
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
"mouse-up" => {
|
||||
let xy = req_str(&args, "xy")?;
|
||||
let (x, y) = parse_xy(&xy)?;
|
||||
let button = args
|
||||
.get("button")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("left");
|
||||
mouse_up::execute(
|
||||
mouse_up::MouseUpArgs {
|
||||
x,
|
||||
y,
|
||||
button: parse_mouse_button(button)?,
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
"launch" => launch::execute(
|
||||
launch::LaunchArgs {
|
||||
app: req_str(&args, "app")?,
|
||||
timeout_ms: args
|
||||
.get("timeout")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(30000),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"close-app" => close_app::execute(
|
||||
close_app::CloseAppArgs {
|
||||
app: req_str(&args, "app")?,
|
||||
force: args.get("force").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"list-windows" => list_windows::execute(
|
||||
list_windows::ListWindowsArgs {
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"list-apps" => list_apps::execute(adapter),
|
||||
|
||||
"focus-window" => focus_window::execute(
|
||||
focus_window::FocusWindowArgs {
|
||||
window_id: str_field(&args, "window_id"),
|
||||
app: str_field(&args, "app"),
|
||||
title: str_field(&args, "title"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"resize-window" => resize_window::execute(
|
||||
resize_window::ResizeWindowArgs {
|
||||
app: str_field(&args, "app"),
|
||||
width: args.get("width").and_then(|v| v.as_f64()).unwrap_or(800.0),
|
||||
height: args.get("height").and_then(|v| v.as_f64()).unwrap_or(600.0),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"move-window" => move_window::execute(
|
||||
move_window::MoveWindowArgs {
|
||||
app: str_field(&args, "app"),
|
||||
x: args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0),
|
||||
y: args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"minimize" => minimize::execute(
|
||||
minimize::MinimizeArgs {
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"maximize" => maximize::execute(
|
||||
maximize::MaximizeArgs {
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"restore" => restore::execute(
|
||||
restore::RestoreArgs {
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"clipboard-get" => clipboard_get::execute(adapter),
|
||||
"clipboard-set" => clipboard_set::execute(req_str(&args, "text")?, adapter),
|
||||
"clipboard-clear" => clipboard_clear::execute(adapter),
|
||||
|
||||
"wait" => wait::execute(
|
||||
wait::WaitArgs {
|
||||
ms: args.get("ms").and_then(|v| v.as_u64()),
|
||||
element: str_field(&args, "element"),
|
||||
window: str_field(&args, "window"),
|
||||
text: str_field(&args, "text"),
|
||||
timeout_ms: args
|
||||
.get("timeout_ms")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(30000),
|
||||
menu: args.get("menu").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
menu_closed: args
|
||||
.get("menu_closed")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false),
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"list-surfaces" => list_surfaces::execute(
|
||||
list_surfaces::ListSurfacesArgs {
|
||||
app: str_field(&args, "app"),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"status" => status::execute(adapter),
|
||||
|
||||
"permissions" => permissions::execute(
|
||||
permissions::PermissionsArgs {
|
||||
request: args
|
||||
.get("request")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
"version" => version::execute(version::VersionArgs {
|
||||
json: args.get("json").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
}),
|
||||
|
||||
other => Err(AppError::invalid_input(format!(
|
||||
"Unknown batch command '{other}'"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_batch_surface(s: Option<&str>) -> agent_desktop_core::adapter::SnapshotSurface {
|
||||
use agent_desktop_core::adapter::SnapshotSurface;
|
||||
match s {
|
||||
Some("menu") => SnapshotSurface::Menu,
|
||||
Some("sheet") => SnapshotSurface::Sheet,
|
||||
Some("popover") => SnapshotSurface::Popover,
|
||||
Some("alert") => SnapshotSurface::Alert,
|
||||
Some("focused") => SnapshotSurface::Focused,
|
||||
_ => SnapshotSurface::Window,
|
||||
}
|
||||
}
|
||||
376
src/cli.rs
376
src/cli.rs
|
|
@ -1,23 +1,116 @@
|
|||
use clap::{Parser, Subcommand};
|
||||
|
||||
pub use crate::cli_args::*;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "agent-desktop",
|
||||
version,
|
||||
about = "Desktop automation for AI agents",
|
||||
about = "Desktop automation CLI for AI agents",
|
||||
long_about = None,
|
||||
after_help = "\
|
||||
CATEGORIES:
|
||||
Observation: snapshot, find, screenshot, get, is
|
||||
Interaction: click, double-click, right-click, type, set-value, focus, select,
|
||||
toggle, expand, collapse, scroll, press
|
||||
App/Window: launch, close-app, list-windows, list-apps, focus-window
|
||||
Clipboard: clipboard-get, clipboard-set
|
||||
Wait: wait
|
||||
System: status, permissions, version
|
||||
Batch: batch"
|
||||
OBSERVATION
|
||||
snapshot Accessibility tree as JSON with @ref IDs
|
||||
screenshot PNG screenshot of an application window
|
||||
find Search elements by role, name, value, or text
|
||||
get <ref> <property> Read element property: text, value, title, bounds, role, states
|
||||
is <ref> <property> Check state: visible, enabled, checked, focused, expanded
|
||||
list-surfaces Available surfaces for an app
|
||||
|
||||
INTERACTION
|
||||
click <ref> Click element (kAXPress)
|
||||
double-click <ref> Double-click element
|
||||
triple-click <ref> Triple-click element (select line/paragraph)
|
||||
right-click <ref> Right-click and open context menu
|
||||
type <ref> <text> Focus element and type text
|
||||
set-value <ref> <value> Set value attribute directly
|
||||
clear <ref> Clear element value to empty string
|
||||
focus <ref> Set keyboard focus
|
||||
select <ref> <value> Select option in list or dropdown
|
||||
toggle <ref> Toggle checkbox or switch
|
||||
check <ref> Set checkbox/switch to checked (idempotent)
|
||||
uncheck <ref> Set checkbox/switch to unchecked (idempotent)
|
||||
expand <ref> Expand disclosure triangle or tree item
|
||||
collapse <ref> Collapse disclosure triangle or tree item
|
||||
scroll <ref> Scroll element (--direction, --amount)
|
||||
scroll-to <ref> Scroll element into visible area
|
||||
|
||||
KEYBOARD
|
||||
press <combo> Key combo: return, escape, cmd+c, shift+tab ...
|
||||
key-down <combo> Hold a key or modifier down
|
||||
key-up <combo> Release a held key or modifier
|
||||
|
||||
MOUSE
|
||||
hover <ref|--xy> Move cursor to element or coordinates
|
||||
drag Drag from one element/point to another
|
||||
mouse-move --xy x,y Move cursor to absolute coordinates
|
||||
mouse-click --xy x,y Click at coordinates (--button, --count)
|
||||
mouse-down --xy x,y Press mouse button at coordinates
|
||||
mouse-up --xy x,y Release mouse button at coordinates
|
||||
|
||||
APP & WINDOW
|
||||
launch <app> Launch app and wait until window is visible
|
||||
close-app <app> Quit app gracefully (--force to kill)
|
||||
list-windows All visible windows (--app to filter)
|
||||
list-apps All running GUI applications
|
||||
focus-window Bring window to front
|
||||
resize-window Resize window (--width, --height)
|
||||
move-window Move window (--x, --y)
|
||||
minimize Minimize window
|
||||
maximize Maximize/zoom window
|
||||
restore Restore minimized/maximized window
|
||||
|
||||
CLIPBOARD
|
||||
clipboard-get Read plain-text clipboard
|
||||
clipboard-set <text> Write text to clipboard
|
||||
clipboard-clear Clear the clipboard
|
||||
|
||||
WAIT
|
||||
wait [ms] Pause for N milliseconds
|
||||
wait --element <ref> Block until element appears (--timeout ms)
|
||||
wait --window <title> Block until window appears
|
||||
wait --text <text> Block until text appears in app
|
||||
|
||||
SYSTEM
|
||||
status Adapter health, platform, and permission state
|
||||
permissions Check accessibility permission (--request to prompt)
|
||||
version Version string (--json for machine-readable)
|
||||
|
||||
BATCH
|
||||
batch <json> Run commands from a JSON array (--stop-on-error)
|
||||
|
||||
REF IDs
|
||||
snapshot assigns @e1, @e2, ... to interactive elements in depth-first order.
|
||||
Use a ref wherever <ref> appears. Refs are snapshot-scoped; run snapshot
|
||||
again after UI changes.
|
||||
|
||||
KEY COMBOS
|
||||
Single keys: return, escape, tab, space, delete, up, down, left, right
|
||||
Function keys: f1 - f12
|
||||
With modifiers: cmd+c, cmd+v, cmd+z, cmd+shift+z, ctrl+a, shift+tab
|
||||
Modifiers: cmd, ctrl, alt, shift
|
||||
|
||||
EXAMPLES
|
||||
agent-desktop snapshot --app \"System Settings\" -i
|
||||
agent-desktop find --role button --name \"OK\"
|
||||
agent-desktop click @e5
|
||||
agent-desktop check @e3
|
||||
agent-desktop type @e2 \"hello@example.com\"
|
||||
agent-desktop press cmd+z
|
||||
agent-desktop drag --from @e1 --to @e5
|
||||
agent-desktop hover @e5
|
||||
agent-desktop minimize --app TextEdit
|
||||
agent-desktop resize-window --app TextEdit --width 800 --height 600
|
||||
agent-desktop mouse-click --xy 500,300
|
||||
agent-desktop wait --text \"Loading complete\" --app Safari --timeout 5000
|
||||
agent-desktop batch '[{\"command\":\"click\",\"args\":{\"ref_id\":\"@e1\"}}]'"
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[arg(long, short = 'v', global = true)]
|
||||
#[arg(
|
||||
long,
|
||||
short = 'v',
|
||||
global = true,
|
||||
help = "Enable debug logging to stderr"
|
||||
)]
|
||||
pub verbose: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
|
|
@ -26,34 +119,103 @@ pub struct Cli {
|
|||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum Commands {
|
||||
#[command(about = "Capture accessibility tree as structured JSON with @ref IDs")]
|
||||
Snapshot(SnapshotArgs),
|
||||
#[command(about = "Search elements by role, name, value, or text content")]
|
||||
Find(FindArgs),
|
||||
#[command(about = "Take a PNG screenshot of an application window")]
|
||||
Screenshot(ScreenshotArgs),
|
||||
#[command(about = "Read an element property (text, value, title, bounds, role, states)")]
|
||||
Get(GetArgs),
|
||||
#[command(about = "Check element state (visible, enabled, checked, focused, expanded)")]
|
||||
Is(IsArgs),
|
||||
#[command(about = "Click element via accessibility press action")]
|
||||
Click(RefArgs),
|
||||
#[command(about = "Double-click element")]
|
||||
DoubleClick(RefArgs),
|
||||
#[command(about = "Triple-click element to select line or paragraph")]
|
||||
TripleClick(RefArgs),
|
||||
#[command(about = "Right-click element to open context menu")]
|
||||
RightClick(RefArgs),
|
||||
#[command(about = "Focus element and type text")]
|
||||
Type(TypeArgs),
|
||||
#[command(about = "Set element value directly via accessibility attribute")]
|
||||
SetValue(SetValueArgs),
|
||||
#[command(about = "Clear element value to empty string")]
|
||||
Clear(RefArgs),
|
||||
#[command(about = "Set keyboard focus on element")]
|
||||
Focus(RefArgs),
|
||||
#[command(about = "Select an option in a list or dropdown")]
|
||||
Select(SelectArgs),
|
||||
#[command(about = "Toggle a checkbox or switch")]
|
||||
Toggle(RefArgs),
|
||||
#[command(about = "Set checkbox or switch to checked state (idempotent)")]
|
||||
Check(RefArgs),
|
||||
#[command(about = "Set checkbox or switch to unchecked state (idempotent)")]
|
||||
Uncheck(RefArgs),
|
||||
#[command(about = "Expand a disclosure triangle or tree item")]
|
||||
Expand(RefArgs),
|
||||
#[command(about = "Collapse a disclosure triangle or tree item")]
|
||||
Collapse(RefArgs),
|
||||
#[command(about = "Scroll element (--direction up/down/left/right, --amount N)")]
|
||||
Scroll(ScrollArgs),
|
||||
#[command(about = "Scroll element into visible area")]
|
||||
ScrollTo(RefArgs),
|
||||
#[command(about = "Send a key combo: return, escape, cmd+c, shift+tab ...")]
|
||||
Press(PressArgs),
|
||||
#[command(about = "Hold a key or modifier down")]
|
||||
KeyDown(KeyComboArgs),
|
||||
#[command(about = "Release a held key or modifier")]
|
||||
KeyUp(KeyComboArgs),
|
||||
#[command(about = "Move cursor to element center or coordinates")]
|
||||
Hover(HoverArgs),
|
||||
#[command(about = "Drag from one element or point to another")]
|
||||
Drag(DragCliArgs),
|
||||
#[command(about = "Move cursor to absolute screen coordinates")]
|
||||
MouseMove(MouseMoveArgs),
|
||||
#[command(about = "Click at absolute screen coordinates")]
|
||||
MouseClick(MouseClickArgs),
|
||||
#[command(about = "Press mouse button at coordinates (without releasing)")]
|
||||
MouseDown(MousePointArgs),
|
||||
#[command(about = "Release mouse button at coordinates")]
|
||||
MouseUp(MousePointArgs),
|
||||
#[command(about = "Launch application and wait until its window is visible")]
|
||||
Launch(LaunchArgs),
|
||||
#[command(about = "Quit an application gracefully (--force to kill)")]
|
||||
CloseApp(CloseAppArgs),
|
||||
#[command(about = "List all visible windows (--app to filter by application)")]
|
||||
ListWindows(ListWindowsArgs),
|
||||
#[command(about = "List all running GUI applications")]
|
||||
ListApps,
|
||||
#[command(about = "Bring a window to front (--app, --title, or --window-id)")]
|
||||
FocusWindow(FocusWindowArgs),
|
||||
#[command(about = "Resize application window")]
|
||||
ResizeWindow(ResizeWindowCliArgs),
|
||||
#[command(about = "Move application window to new position")]
|
||||
MoveWindow(MoveWindowCliArgs),
|
||||
#[command(about = "Minimize application window")]
|
||||
Minimize(AppRefArgs),
|
||||
#[command(about = "Maximize or zoom application window")]
|
||||
Maximize(AppRefArgs),
|
||||
#[command(about = "Restore minimized or maximized window")]
|
||||
Restore(AppRefArgs),
|
||||
#[command(about = "List accessibility surfaces for an app (window, menu, sheet ...)")]
|
||||
ListSurfaces(ListSurfacesArgs),
|
||||
#[command(about = "Read plain-text clipboard contents")]
|
||||
ClipboardGet,
|
||||
#[command(about = "Write text to the clipboard")]
|
||||
ClipboardSet(ClipboardSetArgs),
|
||||
#[command(about = "Clear the clipboard")]
|
||||
ClipboardClear,
|
||||
#[command(about = "Wait for time (ms), element presence, text, or window appearance")]
|
||||
Wait(WaitArgs),
|
||||
#[command(about = "Show adapter health, platform info, and permission state")]
|
||||
Status,
|
||||
#[command(about = "Check accessibility permission status (--request to prompt system dialog)")]
|
||||
Permissions(PermissionsArgs),
|
||||
#[command(about = "Show version (--json for machine-readable output)")]
|
||||
Version(VersionArgs),
|
||||
#[command(about = "Execute multiple commands from a JSON array (--stop-on-error)")]
|
||||
Batch(BatchArgs),
|
||||
}
|
||||
|
||||
|
|
@ -67,23 +229,43 @@ impl Commands {
|
|||
Self::Is(_) => "is",
|
||||
Self::Click(_) => "click",
|
||||
Self::DoubleClick(_) => "double-click",
|
||||
Self::TripleClick(_) => "triple-click",
|
||||
Self::RightClick(_) => "right-click",
|
||||
Self::Type(_) => "type",
|
||||
Self::SetValue(_) => "set-value",
|
||||
Self::Clear(_) => "clear",
|
||||
Self::Focus(_) => "focus",
|
||||
Self::Select(_) => "select",
|
||||
Self::Toggle(_) => "toggle",
|
||||
Self::Check(_) => "check",
|
||||
Self::Uncheck(_) => "uncheck",
|
||||
Self::Expand(_) => "expand",
|
||||
Self::Collapse(_) => "collapse",
|
||||
Self::Scroll(_) => "scroll",
|
||||
Self::ScrollTo(_) => "scroll-to",
|
||||
Self::Press(_) => "press",
|
||||
Self::KeyDown(_) => "key-down",
|
||||
Self::KeyUp(_) => "key-up",
|
||||
Self::Hover(_) => "hover",
|
||||
Self::Drag(_) => "drag",
|
||||
Self::MouseMove(_) => "mouse-move",
|
||||
Self::MouseClick(_) => "mouse-click",
|
||||
Self::MouseDown(_) => "mouse-down",
|
||||
Self::MouseUp(_) => "mouse-up",
|
||||
Self::Launch(_) => "launch",
|
||||
Self::CloseApp(_) => "close-app",
|
||||
Self::ListWindows(_) => "list-windows",
|
||||
Self::ListApps => "list-apps",
|
||||
Self::FocusWindow(_) => "focus-window",
|
||||
Self::ResizeWindow(_) => "resize-window",
|
||||
Self::MoveWindow(_) => "move-window",
|
||||
Self::Minimize(_) => "minimize",
|
||||
Self::Maximize(_) => "maximize",
|
||||
Self::Restore(_) => "restore",
|
||||
Self::ListSurfaces(_) => "list-surfaces",
|
||||
Self::ClipboardGet => "clipboard-get",
|
||||
Self::ClipboardSet(_) => "clipboard-set",
|
||||
Self::ClipboardClear => "clipboard-clear",
|
||||
Self::Wait(_) => "wait",
|
||||
Self::Status => "status",
|
||||
Self::Permissions(_) => "permissions",
|
||||
|
|
@ -92,173 +274,3 @@ impl Commands {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct SnapshotArgs {
|
||||
#[arg(long)]
|
||||
pub app: Option<String>,
|
||||
#[arg(long, name = "window-id")]
|
||||
pub window_id: Option<String>,
|
||||
#[arg(long, default_value = "10")]
|
||||
pub max_depth: u8,
|
||||
#[arg(long)]
|
||||
pub include_bounds: bool,
|
||||
#[arg(long, short = 'i')]
|
||||
pub interactive_only: bool,
|
||||
#[arg(long)]
|
||||
pub compact: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct FindArgs {
|
||||
#[arg(long)]
|
||||
pub app: Option<String>,
|
||||
#[arg(long)]
|
||||
pub role: Option<String>,
|
||||
#[arg(long)]
|
||||
pub name: Option<String>,
|
||||
#[arg(long)]
|
||||
pub value: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ScreenshotArgs {
|
||||
#[arg(long)]
|
||||
pub app: Option<String>,
|
||||
#[arg(long, name = "window-id")]
|
||||
pub window_id: Option<String>,
|
||||
#[arg(value_name = "PATH")]
|
||||
pub output_path: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct GetArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
#[arg(long, default_value = "text")]
|
||||
pub property: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct IsArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
#[arg(long, default_value = "visible")]
|
||||
pub property: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct RefArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct TypeArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
#[arg(value_name = "TEXT")]
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct SetValueArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
#[arg(value_name = "VALUE")]
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct SelectArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
#[arg(value_name = "VALUE")]
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ScrollArgs {
|
||||
#[arg(value_name = "REF")]
|
||||
pub ref_id: String,
|
||||
#[arg(long, default_value = "down")]
|
||||
pub direction: String,
|
||||
#[arg(long, default_value = "3")]
|
||||
pub amount: u32,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct PressArgs {
|
||||
#[arg(value_name = "COMBO")]
|
||||
pub combo: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct LaunchArgs {
|
||||
#[arg(value_name = "APP")]
|
||||
pub app: String,
|
||||
#[arg(long)]
|
||||
pub wait: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct CloseAppArgs {
|
||||
#[arg(value_name = "APP")]
|
||||
pub app: String,
|
||||
#[arg(long)]
|
||||
pub force: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ListWindowsArgs {
|
||||
#[arg(long)]
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct FocusWindowArgs {
|
||||
#[arg(long, name = "window-id")]
|
||||
pub window_id: Option<String>,
|
||||
#[arg(long)]
|
||||
pub app: Option<String>,
|
||||
#[arg(long)]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ClipboardSetArgs {
|
||||
#[arg(value_name = "TEXT")]
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct WaitArgs {
|
||||
#[arg(value_name = "MS")]
|
||||
pub ms: Option<u64>,
|
||||
#[arg(long)]
|
||||
pub element: Option<String>,
|
||||
#[arg(long)]
|
||||
pub window: Option<String>,
|
||||
#[arg(long, default_value = "30000")]
|
||||
pub timeout: u64,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct PermissionsArgs {
|
||||
#[arg(long)]
|
||||
pub request: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct VersionArgs {
|
||||
#[arg(long)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct BatchArgs {
|
||||
#[arg(value_name = "JSON")]
|
||||
pub commands_json: String,
|
||||
#[arg(long)]
|
||||
pub stop_on_error: bool,
|
||||
}
|
||||
|
|
|
|||
342
src/cli_args.rs
Normal file
342
src/cli_args.rs
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
use clap::{Parser, ValueEnum};
|
||||
|
||||
#[derive(ValueEnum, Clone, Debug, Default)]
|
||||
pub enum Surface {
|
||||
#[default]
|
||||
Window,
|
||||
Focused,
|
||||
Menu,
|
||||
Sheet,
|
||||
Popover,
|
||||
Alert,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct SnapshotArgs {
|
||||
#[arg(long, help = "Filter to application by name")]
|
||||
pub app: Option<String>,
|
||||
#[arg(
|
||||
long,
|
||||
name = "window-id",
|
||||
help = "Filter to window ID (from list-windows)"
|
||||
)]
|
||||
pub window_id: Option<String>,
|
||||
#[arg(long, default_value = "10", help = "Maximum tree depth")]
|
||||
pub max_depth: u8,
|
||||
#[arg(long, help = "Include element bounds (x, y, width, height)")]
|
||||
pub include_bounds: bool,
|
||||
#[arg(long, short = 'i', help = "Include interactive elements only")]
|
||||
pub interactive_only: bool,
|
||||
#[arg(long, help = "Omit empty structural nodes from output")]
|
||||
pub compact: bool,
|
||||
#[arg(long, value_enum, default_value_t = Surface::Window, help = "Surface to snapshot")]
|
||||
pub surface: Surface,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct FindArgs {
|
||||
#[arg(long, help = "Filter to application by name")]
|
||||
pub app: Option<String>,
|
||||
#[arg(
|
||||
long,
|
||||
help = "Match by accessibility role (button, textfield, checkbox ...)"
|
||||
)]
|
||||
pub role: Option<String>,
|
||||
#[arg(long, help = "Match by accessible name or label")]
|
||||
pub name: Option<String>,
|
||||
#[arg(long, help = "Match by current value")]
|
||||
pub value: Option<String>,
|
||||
#[arg(long, help = "Match by text in name, value, title, or description")]
|
||||
pub text: Option<String>,
|
||||
#[arg(long, help = "Return match count only")]
|
||||
pub count: bool,
|
||||
#[arg(long, help = "Return first match only")]
|
||||
pub first: bool,
|
||||
#[arg(long, help = "Return last match only")]
|
||||
pub last: bool,
|
||||
#[arg(long, help = "Return Nth match (0-indexed)")]
|
||||
pub nth: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ScreenshotArgs {
|
||||
#[arg(long, help = "Filter to application by name")]
|
||||
pub app: Option<String>,
|
||||
#[arg(
|
||||
long,
|
||||
name = "window-id",
|
||||
help = "Filter to window ID (from list-windows)"
|
||||
)]
|
||||
pub window_id: Option<String>,
|
||||
#[arg(value_name = "PATH", help = "Save to file instead of returning base64")]
|
||||
pub output_path: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct GetArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "text",
|
||||
help = "Property: text, value, title, bounds, role, states"
|
||||
)]
|
||||
pub property: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct IsArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "visible",
|
||||
help = "State: visible, enabled, checked, focused, expanded"
|
||||
)]
|
||||
pub property: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct RefArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct TypeArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
#[arg(value_name = "TEXT", allow_hyphen_values = true, help = "Text to type")]
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct SetValueArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
#[arg(
|
||||
value_name = "VALUE",
|
||||
allow_hyphen_values = true,
|
||||
help = "Value to set"
|
||||
)]
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct SelectArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
#[arg(value_name = "VALUE", help = "Option to select")]
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ScrollArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")]
|
||||
pub ref_id: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "down",
|
||||
help = "Direction: up, down, left, right"
|
||||
)]
|
||||
pub direction: String,
|
||||
#[arg(long, default_value = "3", help = "Number of scroll units")]
|
||||
pub amount: u32,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct PressArgs {
|
||||
#[arg(
|
||||
value_name = "COMBO",
|
||||
help = "Key combo: return, escape, cmd+c, shift+tab ..."
|
||||
)]
|
||||
pub combo: String,
|
||||
#[arg(long, help = "Target application name (focuses app before pressing)")]
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct KeyComboArgs {
|
||||
#[arg(
|
||||
value_name = "COMBO",
|
||||
help = "Key or modifier to hold/release: shift, cmd, ctrl ..."
|
||||
)]
|
||||
pub combo: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct HoverArgs {
|
||||
#[arg(value_name = "REF", help = "Element ref to hover over")]
|
||||
pub ref_id: Option<String>,
|
||||
#[arg(long, help = "Absolute coordinates as x,y")]
|
||||
pub xy: Option<String>,
|
||||
#[arg(long, help = "Hold hover position for N milliseconds")]
|
||||
pub duration: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct DragCliArgs {
|
||||
#[arg(long, help = "Source element ref")]
|
||||
pub from: Option<String>,
|
||||
#[arg(long, name = "from-xy", help = "Source coordinates as x,y")]
|
||||
pub from_xy: Option<String>,
|
||||
#[arg(long, help = "Destination element ref")]
|
||||
pub to: Option<String>,
|
||||
#[arg(long, name = "to-xy", help = "Destination coordinates as x,y")]
|
||||
pub to_xy: Option<String>,
|
||||
#[arg(long, help = "Drag duration in milliseconds")]
|
||||
pub duration: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct MouseMoveArgs {
|
||||
#[arg(long, help = "Absolute coordinates as x,y")]
|
||||
pub xy: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct MouseClickArgs {
|
||||
#[arg(long, help = "Absolute coordinates as x,y")]
|
||||
pub xy: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "left",
|
||||
help = "Mouse button: left, right, middle"
|
||||
)]
|
||||
pub button: String,
|
||||
#[arg(long, default_value = "1", help = "Number of clicks")]
|
||||
pub count: u32,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct MousePointArgs {
|
||||
#[arg(long, help = "Absolute coordinates as x,y")]
|
||||
pub xy: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "left",
|
||||
help = "Mouse button: left, right, middle"
|
||||
)]
|
||||
pub button: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct LaunchArgs {
|
||||
#[arg(value_name = "APP", help = "Application name or bundle ID")]
|
||||
pub app: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "30000",
|
||||
help = "Max time in ms to wait for the window to appear"
|
||||
)]
|
||||
pub timeout: u64,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct CloseAppArgs {
|
||||
#[arg(value_name = "APP", help = "Application name")]
|
||||
pub app: String,
|
||||
#[arg(long, help = "Force-kill the process instead of quitting gracefully")]
|
||||
pub force: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ListWindowsArgs {
|
||||
#[arg(long, help = "Filter to application by name")]
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct FocusWindowArgs {
|
||||
#[arg(long, name = "window-id", help = "Window ID from list-windows")]
|
||||
pub window_id: Option<String>,
|
||||
#[arg(long, help = "Application name")]
|
||||
pub app: Option<String>,
|
||||
#[arg(long, help = "Window title (partial match accepted)")]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ResizeWindowCliArgs {
|
||||
#[arg(long, help = "Application name")]
|
||||
pub app: Option<String>,
|
||||
#[arg(long, help = "New window width in pixels")]
|
||||
pub width: f64,
|
||||
#[arg(long, help = "New window height in pixels")]
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct MoveWindowCliArgs {
|
||||
#[arg(long, help = "Application name")]
|
||||
pub app: Option<String>,
|
||||
#[arg(long, help = "New window X position")]
|
||||
pub x: f64,
|
||||
#[arg(long, help = "New window Y position")]
|
||||
pub y: f64,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct AppRefArgs {
|
||||
#[arg(long, help = "Application name")]
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ClipboardSetArgs {
|
||||
#[arg(value_name = "TEXT", help = "Text to write to the clipboard")]
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct WaitArgs {
|
||||
#[arg(value_name = "MS", help = "Milliseconds to pause")]
|
||||
pub ms: Option<u64>,
|
||||
#[arg(long, help = "Block until this element ref appears in the tree")]
|
||||
pub element: Option<String>,
|
||||
#[arg(long, help = "Block until a window with this title appears")]
|
||||
pub window: Option<String>,
|
||||
#[arg(
|
||||
long,
|
||||
help = "Block until text appears in the app's accessibility tree"
|
||||
)]
|
||||
pub text: Option<String>,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "30000",
|
||||
help = "Timeout in milliseconds for element/window/text waits"
|
||||
)]
|
||||
pub timeout: u64,
|
||||
#[arg(long, help = "Block until a context menu is open")]
|
||||
pub menu: bool,
|
||||
#[arg(long, help = "Block until the context menu is dismissed")]
|
||||
pub menu_closed: bool,
|
||||
#[arg(long, help = "Scope element, window, or text wait to this application")]
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct ListSurfacesArgs {
|
||||
#[arg(long, help = "Filter to application by name")]
|
||||
pub app: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct PermissionsArgs {
|
||||
#[arg(long, help = "Trigger the system accessibility permission dialog")]
|
||||
pub request: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct VersionArgs {
|
||||
#[arg(long, help = "Output version as JSON object")]
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct BatchArgs {
|
||||
#[arg(value_name = "JSON", help = "JSON array of {command, args} objects")]
|
||||
pub commands_json: String,
|
||||
#[arg(long, help = "Halt the batch on the first failed command")]
|
||||
pub stop_on_error: bool,
|
||||
}
|
||||
256
src/dispatch.rs
256
src/dispatch.rs
|
|
@ -1,11 +1,13 @@
|
|||
use agent_desktop_core::{
|
||||
action::Direction,
|
||||
action::{Direction, MouseButton},
|
||||
adapter::PlatformAdapter,
|
||||
commands::{
|
||||
batch, click, clipboard_get, clipboard_set, close_app, collapse, double_click, expand,
|
||||
find, focus, focus_window, get, helpers, is_check, launch, list_apps, list_windows,
|
||||
permissions, press, right_click, screenshot, scroll, select, set_value, snapshot, status,
|
||||
toggle, type_text, version, wait,
|
||||
batch, check, clear, click, clipboard_clear, clipboard_get, clipboard_set, close_app,
|
||||
collapse, double_click, drag, expand, find, focus, focus_window, get, helpers, hover,
|
||||
is_check, key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize,
|
||||
minimize, mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press,
|
||||
resize_window, restore, right_click, screenshot, scroll, scroll_to, select, set_value,
|
||||
snapshot, status, toggle, triple_click, type_text, uncheck, version, wait,
|
||||
},
|
||||
error::AppError,
|
||||
};
|
||||
|
|
@ -23,12 +25,23 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
include_bounds: a.include_bounds,
|
||||
interactive_only: a.interactive_only,
|
||||
compact: a.compact,
|
||||
surface: cli_surface_to_core(&a.surface),
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::Find(a) => find::execute(
|
||||
find::FindArgs { app: a.app, role: a.role, name: a.name, value: a.value },
|
||||
find::FindArgs {
|
||||
app: a.app,
|
||||
role: a.role,
|
||||
name: a.name,
|
||||
value: a.value,
|
||||
text: a.text,
|
||||
count: a.count,
|
||||
first: a.first,
|
||||
last: a.last,
|
||||
nth: a.nth,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
|
|
@ -61,29 +74,45 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
Commands::DoubleClick(a) => {
|
||||
double_click::execute(double_click::DoubleClickArgs { ref_id: a.ref_id }, adapter)
|
||||
}
|
||||
Commands::TripleClick(a) => {
|
||||
triple_click::execute(triple_click::TripleClickArgs { ref_id: a.ref_id }, adapter)
|
||||
}
|
||||
Commands::RightClick(a) => {
|
||||
right_click::execute(right_click::RightClickArgs { ref_id: a.ref_id }, adapter)
|
||||
}
|
||||
|
||||
Commands::Type(a) => type_text::execute(
|
||||
type_text::TypeArgs { ref_id: a.ref_id, text: a.text },
|
||||
type_text::TypeArgs {
|
||||
ref_id: a.ref_id,
|
||||
text: a.text,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::SetValue(a) => set_value::execute(
|
||||
set_value::SetValueArgs { ref_id: a.ref_id, value: a.value },
|
||||
set_value::SetValueArgs {
|
||||
ref_id: a.ref_id,
|
||||
value: a.value,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::Clear(a) => clear::execute(clear::ClearArgs { ref_id: a.ref_id }, adapter),
|
||||
|
||||
Commands::Focus(a) => focus::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter),
|
||||
Commands::Toggle(a) => toggle::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter),
|
||||
Commands::Expand(a) => expand::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter),
|
||||
Commands::Collapse(a) => {
|
||||
collapse::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter)
|
||||
Commands::Check(a) => check::execute(check::CheckArgs { ref_id: a.ref_id }, adapter),
|
||||
Commands::Uncheck(a) => {
|
||||
uncheck::execute(uncheck::UncheckArgs { ref_id: a.ref_id }, adapter)
|
||||
}
|
||||
Commands::Expand(a) => expand::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter),
|
||||
Commands::Collapse(a) => collapse::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter),
|
||||
|
||||
Commands::Select(a) => select::execute(
|
||||
select::SelectArgs { ref_id: a.ref_id, value: a.value },
|
||||
select::SelectArgs {
|
||||
ref_id: a.ref_id,
|
||||
value: a.value,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
|
|
@ -96,14 +125,99 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
adapter,
|
||||
),
|
||||
|
||||
Commands::Press(a) => press::execute(press::PressArgs { combo: a.combo }, adapter),
|
||||
|
||||
Commands::Launch(a) => {
|
||||
launch::execute(launch::LaunchArgs { app: a.app, wait: a.wait }, adapter)
|
||||
Commands::ScrollTo(a) => {
|
||||
scroll_to::execute(scroll_to::ScrollToArgs { ref_id: a.ref_id }, adapter)
|
||||
}
|
||||
|
||||
Commands::Press(a) => press::execute(
|
||||
press::PressArgs {
|
||||
combo: a.combo,
|
||||
app: a.app,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::KeyDown(a) => {
|
||||
key_down::execute(key_down::KeyDownArgs { combo: a.combo }, adapter)
|
||||
}
|
||||
|
||||
Commands::KeyUp(a) => key_up::execute(key_up::KeyUpArgs { combo: a.combo }, adapter),
|
||||
|
||||
Commands::Hover(a) => hover::execute(
|
||||
hover::HoverArgs {
|
||||
ref_id: a.ref_id,
|
||||
xy: parse_xy_opt(a.xy.as_deref())?,
|
||||
duration_ms: a.duration,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::Drag(a) => drag::execute(
|
||||
drag::DragArgs {
|
||||
from_ref: a.from,
|
||||
from_xy: parse_xy_opt(a.from_xy.as_deref())?,
|
||||
to_ref: a.to,
|
||||
to_xy: parse_xy_opt(a.to_xy.as_deref())?,
|
||||
duration_ms: a.duration,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::MouseMove(a) => {
|
||||
let (x, y) = parse_xy(&a.xy)?;
|
||||
mouse_move::execute(mouse_move::MouseMoveArgs { x, y }, adapter)
|
||||
}
|
||||
|
||||
Commands::MouseClick(a) => {
|
||||
let (x, y) = parse_xy(&a.xy)?;
|
||||
mouse_click::execute(
|
||||
mouse_click::MouseClickArgs {
|
||||
x,
|
||||
y,
|
||||
button: parse_mouse_button(&a.button)?,
|
||||
count: a.count,
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
Commands::MouseDown(a) => {
|
||||
let (x, y) = parse_xy(&a.xy)?;
|
||||
mouse_down::execute(
|
||||
mouse_down::MouseDownArgs {
|
||||
x,
|
||||
y,
|
||||
button: parse_mouse_button(&a.button)?,
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
Commands::MouseUp(a) => {
|
||||
let (x, y) = parse_xy(&a.xy)?;
|
||||
mouse_up::execute(
|
||||
mouse_up::MouseUpArgs {
|
||||
x,
|
||||
y,
|
||||
button: parse_mouse_button(&a.button)?,
|
||||
},
|
||||
adapter,
|
||||
)
|
||||
}
|
||||
|
||||
Commands::Launch(a) => launch::execute(
|
||||
launch::LaunchArgs {
|
||||
app: a.app,
|
||||
timeout_ms: a.timeout,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::CloseApp(a) => close_app::execute(
|
||||
close_app::CloseAppArgs { app: a.app, force: a.force },
|
||||
close_app::CloseAppArgs {
|
||||
app: a.app,
|
||||
force: a.force,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
|
|
@ -113,6 +227,10 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
|
||||
Commands::ListApps => list_apps::execute(adapter),
|
||||
|
||||
Commands::ListSurfaces(a) => {
|
||||
list_surfaces::execute(list_surfaces::ListSurfacesArgs { app: a.app }, adapter)
|
||||
}
|
||||
|
||||
Commands::FocusWindow(a) => focus_window::execute(
|
||||
focus_window::FocusWindowArgs {
|
||||
window_id: a.window_id,
|
||||
|
|
@ -122,25 +240,53 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
adapter,
|
||||
),
|
||||
|
||||
Commands::ResizeWindow(a) => resize_window::execute(
|
||||
resize_window::ResizeWindowArgs {
|
||||
app: a.app,
|
||||
width: a.width,
|
||||
height: a.height,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::MoveWindow(a) => move_window::execute(
|
||||
move_window::MoveWindowArgs {
|
||||
app: a.app,
|
||||
x: a.x,
|
||||
y: a.y,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::Minimize(a) => minimize::execute(minimize::MinimizeArgs { app: a.app }, adapter),
|
||||
|
||||
Commands::Maximize(a) => maximize::execute(maximize::MaximizeArgs { app: a.app }, adapter),
|
||||
|
||||
Commands::Restore(a) => restore::execute(restore::RestoreArgs { app: a.app }, adapter),
|
||||
|
||||
Commands::ClipboardGet => clipboard_get::execute(adapter),
|
||||
Commands::ClipboardSet(a) => clipboard_set::execute(a.text, adapter),
|
||||
Commands::ClipboardClear => clipboard_clear::execute(adapter),
|
||||
|
||||
Commands::Wait(a) => wait::execute(
|
||||
wait::WaitArgs {
|
||||
ms: a.ms,
|
||||
element: a.element,
|
||||
window: a.window,
|
||||
text: a.text,
|
||||
timeout_ms: a.timeout,
|
||||
menu: a.menu,
|
||||
menu_closed: a.menu_closed,
|
||||
app: a.app,
|
||||
},
|
||||
adapter,
|
||||
),
|
||||
|
||||
Commands::Status => status::execute(adapter),
|
||||
|
||||
Commands::Permissions(a) => permissions::execute(
|
||||
permissions::PermissionsArgs { request: a.request },
|
||||
adapter,
|
||||
),
|
||||
Commands::Permissions(a) => {
|
||||
permissions::execute(permissions::PermissionsArgs { request: a.request }, adapter)
|
||||
}
|
||||
|
||||
Commands::Version(a) => version::execute(version::VersionArgs { json: a.json }),
|
||||
|
||||
|
|
@ -148,7 +294,8 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
let commands = batch::parse_commands(&a.commands_json)?;
|
||||
let mut results = Vec::new();
|
||||
for cmd in commands {
|
||||
let result = dispatch_batch_command(&cmd.command, cmd.args, adapter);
|
||||
let result =
|
||||
crate::batch_dispatch::dispatch_batch_command(&cmd.command, cmd.args, adapter);
|
||||
let ok = result.is_ok();
|
||||
let entry = match result {
|
||||
Ok(data) => {
|
||||
|
|
@ -168,17 +315,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
|
|||
}
|
||||
}
|
||||
|
||||
fn dispatch_batch_command(
|
||||
command: &str,
|
||||
_args: serde_json::Value,
|
||||
_adapter: &dyn agent_desktop_core::adapter::PlatformAdapter,
|
||||
) -> Result<serde_json::Value, agent_desktop_core::error::AppError> {
|
||||
Err(agent_desktop_core::error::AppError::invalid_input(format!(
|
||||
"Batch dispatch for '{command}' not yet implemented. Use individual commands."
|
||||
)))
|
||||
}
|
||||
|
||||
fn parse_get_property(s: &str) -> Result<get::GetProperty, AppError> {
|
||||
pub(crate) fn parse_get_property(s: &str) -> Result<get::GetProperty, AppError> {
|
||||
match s {
|
||||
"text" => Ok(get::GetProperty::Text),
|
||||
"value" => Ok(get::GetProperty::Value),
|
||||
|
|
@ -192,7 +329,7 @@ fn parse_get_property(s: &str) -> Result<get::GetProperty, AppError> {
|
|||
}
|
||||
}
|
||||
|
||||
fn parse_is_property(s: &str) -> Result<is_check::IsProperty, AppError> {
|
||||
pub(crate) fn parse_is_property(s: &str) -> Result<is_check::IsProperty, AppError> {
|
||||
match s {
|
||||
"visible" => Ok(is_check::IsProperty::Visible),
|
||||
"enabled" => Ok(is_check::IsProperty::Enabled),
|
||||
|
|
@ -205,7 +342,7 @@ fn parse_is_property(s: &str) -> Result<is_check::IsProperty, AppError> {
|
|||
}
|
||||
}
|
||||
|
||||
fn parse_direction(s: &str) -> Result<Direction, AppError> {
|
||||
pub(crate) fn parse_direction(s: &str) -> Result<Direction, AppError> {
|
||||
match s {
|
||||
"up" => Ok(Direction::Up),
|
||||
"down" => Ok(Direction::Down),
|
||||
|
|
@ -216,3 +353,52 @@ fn parse_direction(s: &str) -> Result<Direction, AppError> {
|
|||
))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_mouse_button(s: &str) -> Result<MouseButton, AppError> {
|
||||
match s {
|
||||
"left" => Ok(MouseButton::Left),
|
||||
"right" => Ok(MouseButton::Right),
|
||||
"middle" => Ok(MouseButton::Middle),
|
||||
other => Err(AppError::invalid_input(format!(
|
||||
"Unknown button '{other}'. Valid: left, right, middle"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_xy(s: &str) -> Result<(f64, f64), AppError> {
|
||||
let parts: Vec<&str> = s.split(',').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err(AppError::invalid_input(format!(
|
||||
"Invalid coordinates '{s}'. Expected format: x,y (e.g., 500,300)"
|
||||
)));
|
||||
}
|
||||
let x: f64 = parts[0]
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| AppError::invalid_input(format!("Invalid x coordinate: '{}'", parts[0])))?;
|
||||
let y: f64 = parts[1]
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| AppError::invalid_input(format!("Invalid y coordinate: '{}'", parts[1])))?;
|
||||
Ok((x, y))
|
||||
}
|
||||
|
||||
fn parse_xy_opt(s: Option<&str>) -> Result<Option<(f64, f64)>, AppError> {
|
||||
match s {
|
||||
Some(s) => parse_xy(s).map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn cli_surface_to_core(s: &crate::cli::Surface) -> agent_desktop_core::adapter::SnapshotSurface {
|
||||
use crate::cli::Surface;
|
||||
use agent_desktop_core::adapter::SnapshotSurface;
|
||||
match s {
|
||||
Surface::Window => SnapshotSurface::Window,
|
||||
Surface::Focused => SnapshotSurface::Focused,
|
||||
Surface::Menu => SnapshotSurface::Menu,
|
||||
Surface::Sheet => SnapshotSurface::Sheet,
|
||||
Surface::Popover => SnapshotSurface::Popover,
|
||||
Surface::Alert => SnapshotSurface::Alert,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
mod batch_dispatch;
|
||||
mod cli;
|
||||
mod cli_args;
|
||||
mod dispatch;
|
||||
|
||||
use agent_desktop_core::adapter::PlatformAdapter;
|
||||
|
|
@ -43,10 +45,9 @@ fn main() {
|
|||
|
||||
match &cmd {
|
||||
Commands::Version(a) => {
|
||||
let result =
|
||||
agent_desktop_core::commands::version::execute(
|
||||
agent_desktop_core::commands::version::VersionArgs { json: a.json },
|
||||
);
|
||||
let result = agent_desktop_core::commands::version::execute(
|
||||
agent_desktop_core::commands::version::VersionArgs { json: a.json },
|
||||
);
|
||||
finish(cmd_name, result);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
305
tests/agentic_test_notes.md
Normal file
305
tests/agentic_test_notes.md
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
# Agent-Desktop Agentic Test Run
|
||||
## Date: 2026-02-19
|
||||
|
||||
## Task: Multi-step desktop automation (TextEdit + Calculator)
|
||||
|
||||
---
|
||||
|
||||
## Command Results
|
||||
|
||||
### Phase 1: Environment Assessment
|
||||
|
||||
#### 1. `status`
|
||||
- Result: OK — returns permissions, platform, ref_count, version
|
||||
- Issues: NONE
|
||||
|
||||
#### 2. `permissions`
|
||||
- Result: OK — `{"granted":true}`
|
||||
- Issues: NONE
|
||||
|
||||
#### 3. `list-apps`
|
||||
- Result: OK — returned 8 apps with name + pid
|
||||
- Issues:
|
||||
- **BUG (minor):** `data` is a bare array, not `{"apps":[...]}`. Inconsistent with other commands that nest under a named key. An agent parsing `d["data"]["apps"]` gets a TypeError. Should be `{"apps": [...]}`.
|
||||
- **MISSING:** `bundle_id` is always `null`. Should be populated (e.g. `com.apple.TextEdit`) — agents need this for reliable app identification since display names can vary by locale.
|
||||
|
||||
### Phase 2: App Launch & Window Management
|
||||
|
||||
#### 4. `launch Calculator`
|
||||
- Result: FAILED — `"App 'Calculator' launched but no window found"`
|
||||
- Issues:
|
||||
- **BUG:** The app did launch successfully and had a window 2 seconds later. `launch` doesn't wait long enough for the window to appear. Should poll for window with a configurable timeout (default ~5s) before declaring failure.
|
||||
- The app DID get launched — the error is misleading. Should differentiate "failed to launch" from "launched but window not yet visible."
|
||||
|
||||
#### 5. `list-windows --app TextEdit`
|
||||
- Result: OK — correct window info with id, title, pid, is_focused
|
||||
- Issues: NONE
|
||||
|
||||
#### 6. `focus-window --app TextEdit`
|
||||
- Result: OK — focused and returned window info
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 3: Observation — Understanding the UI
|
||||
|
||||
#### 7. `snapshot --app TextEdit --interactive-only --compact`
|
||||
- Result: OK — 12 refs, clean tree with roles and names
|
||||
- Issues: NONE — interactive-only and compact flags work well together
|
||||
|
||||
#### 8. `find --app TextEdit --role button`
|
||||
- Result: OK — found 3 buttons with ref, role, path
|
||||
- Issues:
|
||||
- **IMPROVEMENT:** Buttons @e9, @e10, @e11 have `"name": null`. These are the Bold/Italic/Underline toolbar buttons. The `find` output gives no way to distinguish them without snapshotting. Should fall back to showing `description` or `title` when `name` is null.
|
||||
|
||||
#### 9. `find --app TextEdit --role textfield`
|
||||
- Result: OK — found @e1 with path `["window:Untitled.rtf","scrollarea"]`
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 4: Interaction — Writing Content
|
||||
|
||||
#### 10. `click @e1`
|
||||
- Result: OK
|
||||
- Issues: NONE
|
||||
|
||||
#### 11. `type @e1 "multi-line text"`
|
||||
- Result: OK — multi-line text with newlines typed correctly
|
||||
- Issues: NONE — handles newlines in quoted strings well
|
||||
|
||||
#### 12. `get @e1 --property text`
|
||||
- Result: OK — returned full typed text with preserved newlines
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 5: Text Selection & Clipboard
|
||||
|
||||
#### 13. `press --app TextEdit "cmd+a"`
|
||||
- Result: OK
|
||||
- Issues: NONE
|
||||
|
||||
#### 14. `press --app TextEdit "cmd+c"`
|
||||
- Result: OK
|
||||
- Issues: NONE
|
||||
|
||||
#### 15. `clipboard-get`
|
||||
- Result: OK — clipboard had the full document text
|
||||
- Issues: NONE
|
||||
|
||||
#### 16. `clipboard-set "new text"`
|
||||
- Result: OK
|
||||
- Issues: NONE
|
||||
|
||||
#### 17. `clipboard-get` (after set)
|
||||
- Result: OK — verified clipboard-set worked
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 6: Advanced Interactions
|
||||
|
||||
#### 18. `set-value @e1 "replacement"`
|
||||
- Result: OK — text replaced directly
|
||||
- Issues: NONE
|
||||
|
||||
#### 19. `get @e1 --property value`
|
||||
- Result: OK — confirmed set-value text
|
||||
- Issues: NONE — `text` and `value` both return the textfield content (consistent)
|
||||
|
||||
#### 20. `double-click @e1`
|
||||
- Result: OK
|
||||
- Issues: NONE (visually selected a word)
|
||||
|
||||
#### 21. `right-click @e1`
|
||||
- Result: OK — context menu opened
|
||||
- Issues: NONE
|
||||
|
||||
#### 22. `wait --menu --app TextEdit`
|
||||
- Result: OK — `{"elapsed_ms":25,"found":true}`
|
||||
- Issues: NONE
|
||||
|
||||
#### 23. `list-surfaces --app TextEdit` (with menu open)
|
||||
- Result: OK — detected `context_menu` with 31 items
|
||||
- Issues: NONE
|
||||
|
||||
#### 24. `snapshot --surface menu --app TextEdit`
|
||||
- Result: OK — full context menu tree with 31 menu items
|
||||
- Issues: NONE
|
||||
|
||||
#### 25. `press "escape"`
|
||||
- Result: OK — dismissed context menu
|
||||
- Issues:
|
||||
- **BUG (moderate):** After pressing escape, the NEXT `snapshot --app TextEdit` (default surface=window) still included the 31 context menu refs mixed in with the window refs (e.g. @e2-@e32 were menuitems). The menu refs only disappeared on a subsequent snapshot. Likely a timing issue — the menu hasn't fully closed in the AX tree when the snapshot happens immediately. Should either: (a) poll until the menu is gone, or (b) filter out menu items from window-surface snapshots.
|
||||
|
||||
### Phase 7: Scroll
|
||||
|
||||
#### 26. `scroll @e1 --direction down --amount 3`
|
||||
- Result: OK
|
||||
- Issues: NONE — no way to verify scroll position changed (no scroll offset in get properties)
|
||||
|
||||
#### 27. `scroll @e1 --direction up --amount 3`
|
||||
- Result: OK
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 8: Screenshot & State Queries
|
||||
|
||||
#### 28. `screenshot --app TextEdit /tmp/agentic_test.png`
|
||||
- Result: OK — 96KB PNG captured correctly
|
||||
- Issues: NONE
|
||||
|
||||
#### 29. `is @e1 --property visible`
|
||||
- Result: OK — `true`
|
||||
- Issues: NONE
|
||||
|
||||
#### 30. `is @e1 --property enabled`
|
||||
- Result: OK — `true`
|
||||
- Issues: NONE
|
||||
|
||||
#### 31. `is @e1 --property focused`
|
||||
- Result: OK — `true`
|
||||
- Issues: NONE
|
||||
|
||||
#### 32. `is @e1 --property checked`
|
||||
- Result: OK — `false` (textfield isn't checkable)
|
||||
- Issues:
|
||||
- **IMPROVEMENT:** Returns `false` instead of something like `"not_applicable"`. An agent can't tell if the element is unchecked vs. not-a-checkbox. This could cause logic errors (e.g., agent thinks a checkbox is unchecked when it's actually a button).
|
||||
|
||||
#### 33. `is @e1 --property expanded`
|
||||
- Result: OK — `false`
|
||||
- Issues: Same as checked — `false` vs "not applicable" ambiguity.
|
||||
|
||||
#### 34. `get @e1 --property role`
|
||||
- Result: OK — `"textfield"`
|
||||
- Issues: NONE
|
||||
|
||||
#### 35. `get @e1 --property bounds`
|
||||
- Result: OK structurally — `"value": null`
|
||||
- Issues:
|
||||
- **BUG:** Bounds returns `null` for @e1 even though the element clearly has screen position. Need to use `--include-bounds` in snapshot to see bounds. The `get` command should return bounds independently without requiring a snapshot flag.
|
||||
|
||||
#### 36. `get @e1 --property states`
|
||||
- Result: OK — `["focused"]`
|
||||
- Issues: NONE
|
||||
|
||||
#### 37. `get @e1 --property title`
|
||||
- Result: OK — `null` (textfield has no title attribute)
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 9: Expand / Collapse / Toggle / Select
|
||||
|
||||
#### 38. `expand @e4` (combobox)
|
||||
- Result: FAILED — `ACTION_FAILED` (err=-25205)
|
||||
- Issues:
|
||||
- **BUG:** macOS comboboxes don't support AXExpand. The error message is raw AX error code with no suggestion. Should detect that expand isn't in the element's supported actions and return `ACTION_NOT_SUPPORTED` with suggestion: "This element doesn't support expand. Try 'click' to open it instead."
|
||||
|
||||
#### 39. `collapse @e4` (combobox)
|
||||
- Result: FAILED — same as expand
|
||||
- Issues: Same as above — should return `ACTION_NOT_SUPPORTED`.
|
||||
|
||||
#### 40. `toggle @e1` (textfield)
|
||||
- Result: OK (returned success)
|
||||
- Issues:
|
||||
- **BUG:** `toggle` returned `ok:true` on a textfield which doesn't support toggling. No visible effect occurred. Should either fail with `ACTION_NOT_SUPPORTED` or check supported actions before attempting.
|
||||
|
||||
#### 41. `select @e4 "Courier"` (combobox)
|
||||
- Result: OK structurally (returned success)
|
||||
- Issues:
|
||||
- **BUG:** `select` returned `ok:true` but the combobox value remained "Helvetica" — the font did NOT change. The command silently succeeded without actually performing the selection. Should verify the value changed, or at minimum, not return success if the AX select action wasn't available.
|
||||
|
||||
### Phase 10: Calculator (Cross-App Test)
|
||||
|
||||
#### 42. `snapshot --app Calculator --interactive-only --compact`
|
||||
- Result: OK — 53 refs, all buttons properly named ("7", "8", "Add", "Equals", etc.)
|
||||
- Issues: NONE — excellent button labeling from macOS AX
|
||||
|
||||
#### 43. Clicking buttons: `@e27 @e39 @e20 @e18 @e50` (43 × 8 =)
|
||||
- Result: OK — all 5 clicks registered, screenshot confirmed display showed 344
|
||||
- Issues:
|
||||
- **MISSING:** Calculator display value is NOT in the accessibility tree. An agent cannot read the calculation result programmatically — must use screenshot + OCR. The display is likely a custom-rendered view with no AX value. This is a macOS/Apple limitation, not an agent-desktop bug, but worth documenting.
|
||||
|
||||
### Phase 11: Close App
|
||||
|
||||
#### 44. `close-app Calculator`
|
||||
- Result: Reported `ok:true` but app was still running
|
||||
- Issues:
|
||||
- **BUG:** `close-app` reported success but Calculator was still listed in `list-windows` even after 1 second. Only `close-app --force` actually killed it. The non-force variant either isn't sending the right signal or isn't waiting for confirmation. Should either: (a) verify the app actually quit before returning success, or (b) return a different status like `"requested":true` if it can't confirm termination.
|
||||
|
||||
### Phase 12: Error Handling
|
||||
|
||||
#### 45. `click @e999`
|
||||
- Result: OK — `STALE_REF` with helpful suggestion
|
||||
- Issues: NONE — good error UX
|
||||
|
||||
#### 46. `snapshot --app NonExistentApp`
|
||||
- Result: OK — `APP_NOT_FOUND`
|
||||
- Issues: NONE
|
||||
|
||||
#### 47. `wait --element @e999 --timeout 1500`
|
||||
- Result: OK — `TIMEOUT` with suggestion
|
||||
- Issues: NONE
|
||||
|
||||
#### 48. `wait --window "NonExistent" --app TextEdit --timeout 1500`
|
||||
- Result: OK — `TIMEOUT`
|
||||
- Issues: NONE
|
||||
|
||||
#### 49. `batch [...] --stop-on-error`
|
||||
- Result: OK — stopped correctly after failing command, didn't execute remaining
|
||||
- Issues: NONE
|
||||
|
||||
### Phase 13: Help / UX
|
||||
|
||||
#### 50. `--help` (top-level)
|
||||
- Result: OK — shows all commands with categories
|
||||
- Issues:
|
||||
- **IMPROVEMENT:** Subcommand descriptions are empty. Every `Commands:` line just shows the name with no description. Should have one-line descriptions (e.g., `snapshot Capture the accessibility tree of an application`).
|
||||
|
||||
#### 51. `snapshot --help`
|
||||
- Result: OK — shows all options
|
||||
- Issues:
|
||||
- **IMPROVEMENT:** No option descriptions. `--app`, `--max-depth`, `--compact` etc. have no help text. Should describe what each flag does.
|
||||
|
||||
### Phase 14: Global Output Issues
|
||||
|
||||
#### 52. Stderr duplication
|
||||
- Issues:
|
||||
- **BUG:** Every error JSON is printed TWICE (both to stdout and stderr, or duplicated on one stream). Makes parsing harder — an agent doing `json.load(sys.stdin)` gets a parse error if stderr bleeds into stdout. Error should appear once on stdout (for machine parsing) and optionally on stderr (for human debugging with `--verbose`).
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
### Working Well (22/30 commands flawless)
|
||||
- `status`, `permissions`, `version` — solid
|
||||
- `snapshot` (all surface variants) — excellent, the core strength
|
||||
- `find` — works well for element discovery
|
||||
- `get` (text, value, role, states) — reliable
|
||||
- `is` (visible, enabled, focused) — works
|
||||
- `click`, `double-click`, `right-click` — all work
|
||||
- `type`, `set-value` — both work correctly including multi-line
|
||||
- `press` (with `--app` targeting) — keyboard automation solid
|
||||
- `clipboard-get`, `clipboard-set` — roundtrip perfect
|
||||
- `scroll` — works (hard to verify but no errors)
|
||||
- `wait` (all variants: ms, --element, --window, --menu) — timeouts & success both correct
|
||||
- `list-windows`, `focus-window` — reliable
|
||||
- `list-surfaces` — correctly detects menus
|
||||
- `screenshot` — clean captures
|
||||
- `batch` (with and without --stop-on-error) — correct behavior
|
||||
|
||||
### Bugs Found (7)
|
||||
1. **`list-apps` data shape** — bare array instead of `{"apps":[...]}`
|
||||
2. **`launch` window detection** — doesn't wait long enough, reports failure when app launches fine
|
||||
3. **`close-app` (non-force)** — reports success but app keeps running
|
||||
4. **`expand`/`collapse` error code** — returns `ACTION_FAILED` instead of `ACTION_NOT_SUPPORTED`
|
||||
5. **`toggle` false success** — returns ok on elements that don't support toggle
|
||||
6. **`select` false success** — returns ok but value doesn't change
|
||||
7. **Stderr duplication** — all errors printed twice
|
||||
|
||||
### Needs Improvement (5)
|
||||
1. **`get --property bounds`** returns null — should work independently of snapshot flags
|
||||
2. **`is` checked/expanded** — returns `false` instead of "not applicable" for irrelevant properties
|
||||
3. **`find` output** — buttons with null names are indistinguishable without secondary info
|
||||
4. **`--help` descriptions** — all subcommands and options lack description text
|
||||
5. **`list-apps` bundle_id** — always null, should be populated
|
||||
|
||||
### Missing Features / Gaps (2)
|
||||
1. **Calculator display** — not accessible via AX tree (macOS limitation, not a bug)
|
||||
2. **Scroll position** — no way to query current scroll offset
|
||||
|
||||
### Severity Ranking
|
||||
1. **P0 (breaks agent workflows):** stderr duplication (JSON parse failures), `select`/`toggle` false success (agent thinks action worked when it didn't)
|
||||
2. **P1 (degrades reliability):** `launch` window detection, `close-app` not actually closing, `expand`/`collapse` wrong error code
|
||||
3. **P2 (minor inconsistencies):** `list-apps` shape, `get bounds` null, `is` false vs N/A, help text missing, bundle_id null
|
||||
Loading…
Reference in a new issue