diff --git a/.gitignore b/.gitignore index 23809ef..ae25984 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,7 @@ dist/ *.deb *.rpm *.snap + +# Compound Engineering +docs/ +todos/ \ No newline at end of file diff --git a/crates/core/src/adapter.rs b/crates/core/src/adapter.rs index 4475933..4f26568 100644 --- a/crates/core/src/adapter.rs +++ b/crates/core/src/adapter.rs @@ -63,8 +63,9 @@ impl NativeHandle { } } -// SAFETY: Phase 1 is single-threaded CLI. All AX calls happen on the calling thread. -// NativeHandle is never stored across thread boundaries. +// SAFETY: Phase 1 is single-threaded CLI. NativeHandle is never sent across thread +// boundaries. The unsafe impls are required for use with dyn PlatformAdapter (which +// is Send + Sync). Remove in Phase 4 when async daemon is introduced. unsafe impl Send for NativeHandle {} unsafe impl Sync for NativeHandle {} diff --git a/crates/core/src/commands/batch.rs b/crates/core/src/commands/batch.rs index f64149d..77b6238 100644 --- a/crates/core/src/commands/batch.rs +++ b/crates/core/src/commands/batch.rs @@ -8,19 +8,21 @@ pub struct BatchArgs { } #[derive(Debug, Deserialize)] -#[allow(dead_code)] -struct BatchCommand { - command: String, +pub struct BatchCommand { + pub command: String, #[serde(default)] - args: Value, + pub args: Value, +} + +pub fn parse_commands(json_str: &str) -> Result, AppError> { + serde_json::from_str(json_str) + .map_err(|e| AppError::invalid_input(format!("Invalid batch JSON: {e}"))) } pub fn execute(args: BatchArgs, _adapter: &dyn PlatformAdapter) -> Result { - let commands: Vec = serde_json::from_str(&args.commands_json) - .map_err(|e| AppError::invalid_input(format!("Invalid batch JSON: {e}")))?; - + let commands = parse_commands(&args.commands_json)?; Ok(json!({ - "note": "Batch dispatch is implemented in the binary crate dispatch layer", + "note": "Batch execution delegated to dispatch layer", "count": commands.len() })) } diff --git a/crates/core/src/commands/click.rs b/crates/core/src/commands/click.rs index 37d618a..69ef927 100644 --- a/crates/core/src/commands/click.rs +++ b/crates/core/src/commands/click.rs @@ -15,15 +15,3 @@ pub fn execute(args: ClickArgs, adapter: &dyn PlatformAdapter) -> Result Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::DoubleClick)?; - Ok(serde_json::to_value(result)?) -} - -pub fn execute_right(args: ClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::RightClick)?; - Ok(serde_json::to_value(result)?) -} diff --git a/crates/core/src/commands/clipboard_get.rs b/crates/core/src/commands/clipboard_get.rs new file mode 100644 index 0000000..3869fcd --- /dev/null +++ b/crates/core/src/commands/clipboard_get.rs @@ -0,0 +1,7 @@ +use crate::{adapter::PlatformAdapter, error::AppError}; +use serde_json::{json, Value}; + +pub fn execute(adapter: &dyn PlatformAdapter) -> Result { + let text = adapter.get_clipboard()?; + Ok(json!({ "text": text })) +} diff --git a/crates/core/src/commands/clipboard_set.rs b/crates/core/src/commands/clipboard_set.rs new file mode 100644 index 0000000..9da9352 --- /dev/null +++ b/crates/core/src/commands/clipboard_set.rs @@ -0,0 +1,7 @@ +use crate::{adapter::PlatformAdapter, error::AppError}; +use serde_json::{json, Value}; + +pub fn execute(text: String, adapter: &dyn PlatformAdapter) -> Result { + adapter.set_clipboard(&text)?; + Ok(json!({ "ok": true })) +} diff --git a/crates/core/src/commands/collapse.rs b/crates/core/src/commands/collapse.rs index 26c31b8..19d181c 100644 --- a/crates/core/src/commands/collapse.rs +++ b/crates/core/src/commands/collapse.rs @@ -1,15 +1,11 @@ use crate::{ action::Action, adapter::PlatformAdapter, - commands::helpers::resolve_ref, + commands::helpers::{resolve_ref, RefArgs}, error::AppError, }; use serde_json::Value; -pub struct RefArgs { - pub ref_id: String, -} - pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; let result = adapter.execute_action(&handle, Action::Collapse)?; diff --git a/crates/core/src/commands/double_click.rs b/crates/core/src/commands/double_click.rs new file mode 100644 index 0000000..13a68e8 --- /dev/null +++ b/crates/core/src/commands/double_click.rs @@ -0,0 +1,17 @@ +use crate::{ + action::Action, + adapter::PlatformAdapter, + commands::helpers::resolve_ref, + error::AppError, +}; +use serde_json::Value; + +pub struct DoubleClickArgs { + pub ref_id: String, +} + +pub fn execute(args: DoubleClickArgs, adapter: &dyn PlatformAdapter) -> Result { + let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; + let result = adapter.execute_action(&handle, Action::DoubleClick)?; + Ok(serde_json::to_value(result)?) +} diff --git a/crates/core/src/commands/expand.rs b/crates/core/src/commands/expand.rs index 3da6e9c..37cdaf7 100644 --- a/crates/core/src/commands/expand.rs +++ b/crates/core/src/commands/expand.rs @@ -1,15 +1,11 @@ use crate::{ action::Action, adapter::PlatformAdapter, - commands::helpers::resolve_ref, + commands::helpers::{resolve_ref, RefArgs}, error::AppError, }; use serde_json::Value; -pub struct RefArgs { - pub ref_id: String, -} - pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; let result = adapter.execute_action(&handle, Action::Expand)?; diff --git a/crates/core/src/commands/find.rs b/crates/core/src/commands/find.rs index e8f0022..1adf37c 100644 --- a/crates/core/src/commands/find.rs +++ b/crates/core/src/commands/find.rs @@ -10,7 +10,7 @@ pub struct FindArgs { pub fn execute(args: FindArgs, adapter: &dyn PlatformAdapter) -> Result { let opts = crate::adapter::TreeOptions::default(); - let result = snapshot::run(adapter, &opts, args.app.as_deref(), None)?; + let result = snapshot::build(adapter, &opts, args.app.as_deref(), None)?; let mut matches = Vec::new(); search_tree(&result.tree, &args, &mut Vec::new(), &mut matches); @@ -37,16 +37,15 @@ fn search_tree( }); if role_match && name_match && value_match { - if let Some(ref_id) = &node.ref_id { - let path_str: Vec = path.clone(); - matches.push(json!({ - "ref": ref_id, - "role": node.role, - "name": node.name, - "value": node.value, - "path": path_str - })); - } + let interactive = node.ref_id.is_some(); + matches.push(json!({ + "ref": node.ref_id, + "role": node.role, + "name": node.name, + "value": node.value, + "interactive": interactive, + "path": path.clone() + })); } let label = if let Some(name) = &node.name { diff --git a/crates/core/src/commands/focus.rs b/crates/core/src/commands/focus.rs index e27787f..db8587f 100644 --- a/crates/core/src/commands/focus.rs +++ b/crates/core/src/commands/focus.rs @@ -1,15 +1,11 @@ use crate::{ action::Action, adapter::PlatformAdapter, - commands::helpers::resolve_ref, + commands::helpers::{resolve_ref, RefArgs}, error::AppError, }; use serde_json::Value; -pub struct RefArgs { - pub ref_id: String, -} - pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; let result = adapter.execute_action(&handle, Action::SetFocus)?; diff --git a/crates/core/src/commands/get.rs b/crates/core/src/commands/get.rs index a3d3426..75409b9 100644 --- a/crates/core/src/commands/get.rs +++ b/crates/core/src/commands/get.rs @@ -20,12 +20,10 @@ pub fn execute(args: GetArgs, adapter: &dyn PlatformAdapter) -> Result json!(entry.role), - GetProperty::Text | GetProperty::Title => { - json!(entry.name) - } - GetProperty::Value => json!(null), - GetProperty::Bounds => json!(null), - GetProperty::States => json!([]), + GetProperty::Text | GetProperty::Title => json!(entry.name), + GetProperty::Value => json!(entry.value), + GetProperty::Bounds => json!(entry.bounds), + GetProperty::States => json!(entry.states), }; let prop_name = match args.property { diff --git a/crates/core/src/commands/helpers.rs b/crates/core/src/commands/helpers.rs index 89e9c9e..5bf3eab 100644 --- a/crates/core/src/commands/helpers.rs +++ b/crates/core/src/commands/helpers.rs @@ -4,6 +4,10 @@ use crate::{ refs::{RefEntry, RefMap}, }; +pub struct RefArgs { + pub ref_id: String, +} + pub fn resolve_ref( ref_id: &str, adapter: &dyn PlatformAdapter, diff --git a/crates/core/src/commands/is_check.rs b/crates/core/src/commands/is_check.rs index 6c21edf..3f56670 100644 --- a/crates/core/src/commands/is_check.rs +++ b/crates/core/src/commands/is_check.rs @@ -26,11 +26,11 @@ pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result true, - IsProperty::Enabled => !entry.role.is_empty(), - IsProperty::Checked => false, - IsProperty::Focused => false, - IsProperty::Expanded => false, + IsProperty::Visible => !entry.states.contains(&"hidden".to_string()), + IsProperty::Enabled => !entry.states.contains(&"disabled".to_string()), + IsProperty::Checked => entry.states.contains(&"checked".to_string()), + IsProperty::Focused => entry.states.contains(&"focused".to_string()), + IsProperty::Expanded => entry.states.contains(&"expanded".to_string()), }; Ok(json!({ "property": prop_name, "ref": args.ref_id, "result": result })) diff --git a/crates/core/src/commands/mod.rs b/crates/core/src/commands/mod.rs index 5771dd3..ccc3605 100644 --- a/crates/core/src/commands/mod.rs +++ b/crates/core/src/commands/mod.rs @@ -1,8 +1,11 @@ pub mod batch; pub mod clipboard; +pub mod clipboard_get; +pub mod clipboard_set; pub mod click; pub mod close_app; pub mod collapse; +pub mod double_click; pub mod expand; pub mod find; pub mod focus; @@ -15,6 +18,7 @@ pub mod list_apps; pub mod list_windows; pub mod permissions; pub mod press; +pub mod right_click; pub mod screenshot; pub mod scroll; pub mod select; diff --git a/crates/core/src/commands/right_click.rs b/crates/core/src/commands/right_click.rs new file mode 100644 index 0000000..bdc0fc8 --- /dev/null +++ b/crates/core/src/commands/right_click.rs @@ -0,0 +1,17 @@ +use crate::{ + action::Action, + adapter::PlatformAdapter, + commands::helpers::resolve_ref, + error::AppError, +}; +use serde_json::Value; + +pub struct RightClickArgs { + pub ref_id: String, +} + +pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result { + let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; + let result = adapter.execute_action(&handle, Action::RightClick)?; + Ok(serde_json::to_value(result)?) +} diff --git a/crates/core/src/commands/toggle.rs b/crates/core/src/commands/toggle.rs index 4914b7e..0d317af 100644 --- a/crates/core/src/commands/toggle.rs +++ b/crates/core/src/commands/toggle.rs @@ -1,15 +1,11 @@ use crate::{ action::Action, adapter::PlatformAdapter, - commands::helpers::resolve_ref, + commands::helpers::{resolve_ref, RefArgs}, error::AppError, }; use serde_json::Value; -pub struct RefArgs { - pub ref_id: String, -} - pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; let result = adapter.execute_action(&handle, Action::Toggle)?; diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 0db489c..4d57f21 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -17,6 +17,24 @@ pub enum ErrorCode { Internal, } +impl ErrorCode { + pub fn as_str(&self) -> &'static str { + match self { + ErrorCode::PermissionDenied => "PERMISSION_DENIED", + ErrorCode::ElementNotFound => "ELEMENT_NOT_FOUND", + ErrorCode::ApplicationNotFound => "APPLICATION_NOT_FOUND", + ErrorCode::ActionFailed => "ACTION_FAILED", + ErrorCode::ActionNotSupported => "ACTION_NOT_SUPPORTED", + ErrorCode::StaleRef => "STALE_REF", + ErrorCode::WindowNotFound => "WINDOW_NOT_FOUND", + ErrorCode::PlatformNotSupported => "PLATFORM_NOT_SUPPORTED", + ErrorCode::Timeout => "TIMEOUT", + ErrorCode::InvalidArgs => "INVALID_ARGS", + ErrorCode::Internal => "INTERNAL", + } + } +} + #[derive(Debug, Error, Clone)] #[error("{message}")] pub struct AdapterError { @@ -106,22 +124,8 @@ pub enum AppError { impl AppError { pub fn code(&self) -> &str { match self { - AppError::Adapter(e) => match e.code { - ErrorCode::PermissionDenied => "PERMISSION_DENIED", - ErrorCode::ElementNotFound => "ELEMENT_NOT_FOUND", - ErrorCode::ApplicationNotFound => "APPLICATION_NOT_FOUND", - ErrorCode::ActionFailed => "ACTION_FAILED", - ErrorCode::ActionNotSupported => "ACTION_NOT_SUPPORTED", - ErrorCode::StaleRef => "STALE_REF", - ErrorCode::WindowNotFound => "WINDOW_NOT_FOUND", - ErrorCode::PlatformNotSupported => "PLATFORM_NOT_SUPPORTED", - ErrorCode::Timeout => "TIMEOUT", - ErrorCode::InvalidArgs => "INVALID_ARGS", - ErrorCode::Internal => "INTERNAL", - }, - AppError::Io(_) => "INTERNAL", - AppError::Json(_) => "INTERNAL", - AppError::Internal(_) => "INTERNAL", + AppError::Adapter(e) => e.code.as_str(), + AppError::Io(_) | AppError::Json(_) | AppError::Internal(_) => "INTERNAL", } } diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 29b1230..d0004a2 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -36,9 +36,9 @@ pub struct Rect { impl Rect { pub fn bounds_hash(&self) -> u64 { - use std::collections::hash_map::DefaultHasher; + use rustc_hash::FxHasher; use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); + let mut h = FxHasher::default(); let x = (self.x * 100.0) as i64; let y = (self.y * 100.0) as i64; let w = (self.width * 100.0) as i64; @@ -55,6 +55,7 @@ impl Rect { pub struct WindowInfo { pub id: String, pub title: String, + #[serde(rename = "app_name")] pub app: String, pub pid: i32, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/core/src/output.rs b/crates/core/src/output.rs index 54778dd..3afae57 100644 --- a/crates/core/src/output.rs +++ b/crates/core/src/output.rs @@ -1,6 +1,9 @@ use serde::Serialize; use serde_json::Value; +/// Structured output envelope used by the Phase 3 MCP server transport layer. +/// CLI commands currently build responses via inline `serde_json::json!` calls in `main.rs`; +/// this type provides the typed equivalent for programmatic consumers. #[derive(Debug, Serialize)] pub struct Response { pub version: &'static str, diff --git a/crates/core/src/refs.rs b/crates/core/src/refs.rs index 39879a9..3fe0ca6 100644 --- a/crates/core/src/refs.rs +++ b/crates/core/src/refs.rs @@ -10,6 +10,12 @@ pub struct RefEntry { pub pid: i32, pub role: String, pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub states: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bounds: Option, pub bounds_hash: Option, pub available_actions: Vec, pub source_app: Option, @@ -73,6 +79,7 @@ impl RefMap { .mode(0o600) .open(&tmp)?; file.write_all(json.as_bytes())?; + file.flush()?; } #[cfg(not(unix))] std::fs::write(&tmp, json.as_bytes())?; @@ -110,7 +117,9 @@ fn refmap_path() -> Result { } fn home_dir() -> Option { - std::env::var_os("HOME").map(PathBuf::from) + std::env::var_os("HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("USERPROFILE").map(PathBuf::from)) } #[cfg(test)] @@ -124,6 +133,9 @@ mod tests { pid: 1, role: "button".into(), name: Some("OK".into()), + value: None, + states: vec![], + bounds: None, bounds_hash: None, available_actions: vec!["Click".into()], source_app: None, @@ -142,6 +154,9 @@ mod tests { pid: 42, role: "textfield".into(), name: None, + value: None, + states: vec![], + bounds: None, bounds_hash: Some(12345), available_actions: vec![], source_app: Some("Finder".into()), diff --git a/crates/core/src/snapshot.rs b/crates/core/src/snapshot.rs index b3d9a3e..af37a80 100644 --- a/crates/core/src/snapshot.rs +++ b/crates/core/src/snapshot.rs @@ -10,14 +10,12 @@ const INTERACTIVE_ROLES: &[&str] = &[ "combobox", "treeitem", "cell", "radiobutton", "incrementor", ]; -const ABSOLUTE_MAX_DEPTH: u8 = 50; - pub struct SnapshotResult { pub tree: AccessibilityNode, pub refmap: RefMap, } -pub fn run( +pub fn build( adapter: &dyn PlatformAdapter, opts: &TreeOptions, app_name: Option<&str>, @@ -65,7 +63,7 @@ pub fn run( })? }; - let capped_depth = opts.max_depth.min(ABSOLUTE_MAX_DEPTH); + let capped_depth = opts.max_depth; let tree_opts = TreeOptions { max_depth: capped_depth, include_bounds: opts.include_bounds, @@ -76,33 +74,52 @@ pub fn run( let raw_tree = adapter.get_tree(&window, &tree_opts)?; let mut refmap = RefMap::new(); - let tree = allocate_refs(raw_tree, &mut refmap, opts.include_bounds, opts.interactive_only); - - refmap.save()?; + let tree = allocate_refs( + raw_tree, + &mut refmap, + opts.include_bounds, + opts.interactive_only, + window.pid, + Some(window.app.as_str()), + ); Ok(SnapshotResult { tree, refmap }) } +pub fn run( + adapter: &dyn PlatformAdapter, + opts: &TreeOptions, + app_name: Option<&str>, + window_id: Option<&str>, +) -> Result { + let result = build(adapter, opts, app_name, window_id)?; + result.refmap.save()?; + Ok(result) +} + fn allocate_refs( mut node: AccessibilityNode, refmap: &mut RefMap, include_bounds: bool, interactive_only: bool, + window_pid: i32, + source_app: Option<&str>, ) -> AccessibilityNode { let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str()); if is_interactive { let entry = RefEntry { - pid: 0, + pid: window_pid, role: node.role.clone(), name: node.name.clone(), + value: node.value.clone(), + states: node.states.clone(), + bounds: node.bounds, bounds_hash: node.bounds.as_ref().map(|b| b.bounds_hash()), available_actions: actions_for_role(&node.role), - source_app: None, + source_app: source_app.map(str::to_string), }; node.ref_id = Some(refmap.allocate(entry)); - } else if interactive_only { - return node; } if !include_bounds { @@ -112,7 +129,7 @@ fn allocate_refs( node.children = node .children .into_iter() - .map(|child| allocate_refs(child, refmap, include_bounds, interactive_only)) + .map(|child| allocate_refs(child, refmap, include_bounds, interactive_only, window_pid, source_app)) .collect(); node diff --git a/crates/macos/Cargo.toml b/crates/macos/Cargo.toml index 10203c1..776fed9 100644 --- a/crates/macos/Cargo.toml +++ b/crates/macos/Cargo.toml @@ -17,6 +17,6 @@ rustc-hash.workspace = true accessibility-sys = "0.2.0" core-foundation = "0.10.1" core-foundation-sys = "0.8.7" -core-graphics = "0.25.0" +core-graphics = { version = "0.25.0", features = ["highsierra"] } [build-dependencies] diff --git a/crates/macos/src/actions.rs b/crates/macos/src/actions.rs index 2bbb51d..ff44ce1 100644 --- a/crates/macos/src/actions.rs +++ b/crates/macos/src/actions.rs @@ -69,19 +69,57 @@ mod imp { } Action::Expand => { let ax_action = CFString::new("AXExpand"); - let _ = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + let err = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + if err != kAXErrorSuccess { + return Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::ActionFailed, + format!("AXExpand failed with error {err}"), + )); + } } Action::Collapse => { let ax_action = CFString::new("AXCollapse"); - let _ = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + let err = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + if err != kAXErrorSuccess { + return Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::ActionFailed, + format!("AXCollapse failed with error {err}"), + )); + } } Action::Select(_) => { let ax_action = CFString::new(kAXPressAction); - let _ = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + let err = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + if err != kAXErrorSuccess { + return Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::ActionFailed, + format!("AXPress (select) failed with error {err}"), + )); + } } - Action::Scroll(_, _) => { - let ax_action = CFString::new(kAXPressAction); - let _ = unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; + Action::Scroll(direction, amount) => { + use core_graphics::{ + event::{CGEvent, CGEventTapLocation, ScrollEventUnit}, + event_source::{CGEventSource, CGEventSourceStateID}, + }; + let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) + .map_err(|_| AdapterError::internal("CGEventSource failed"))?; + let (dx, dy) = match direction { + agent_desktop_core::action::Direction::Up => (0i32, *amount as i32), + agent_desktop_core::action::Direction::Down => (0i32, -(*amount as i32)), + agent_desktop_core::action::Direction::Left => (-(*amount as i32), 0i32), + agent_desktop_core::action::Direction::Right => (*amount as i32, 0i32), + }; + let event = CGEvent::new_scroll_event( + source, + ScrollEventUnit::LINE, + 2, + dy, + dx, + 0, + ) + .map_err(|_| AdapterError::internal("CGEvent scroll failed"))?; + event.post(CGEventTapLocation::HID); } } diff --git a/crates/macos/src/adapter.rs b/crates/macos/src/adapter.rs index 1d54b1e..0e2410f 100644 --- a/crates/macos/src/adapter.rs +++ b/crates/macos/src/adapter.rs @@ -61,15 +61,15 @@ impl PlatformAdapter for MacOSAdapter { } fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError> { - focus_window_impl(win) + crate::app_ops::focus_window_impl(win) } fn launch_app(&self, id: &str, wait: bool) -> Result { - launch_app_impl(id, wait) + crate::app_ops::launch_app_impl(id, wait) } fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError> { - close_app_impl(id, force) + crate::app_ops::close_app_impl(id, force) } fn screenshot(&self, target: ScreenshotTarget) -> Result { @@ -106,14 +106,10 @@ impl PlatformAdapter for MacOSAdapter { #[cfg(target_os = "macos")] fn execute_action_impl(handle: &NativeHandle, action: Action) -> Result { use crate::tree::AXElement; - use core_foundation::base::CFRetain; - use core_foundation_sys::base::CFTypeRef; + use std::mem::ManuallyDrop; - unsafe { CFRetain(handle.as_raw() as CFTypeRef) }; - let el = AXElement(handle.as_raw() as accessibility_sys::AXUIElementRef); - let result = crate::actions::perform_action(&el, &action)?; - std::mem::forget(el); - Ok(result) + let el = ManuallyDrop::new(AXElement(handle.as_raw() as accessibility_sys::AXUIElementRef)); + crate::actions::perform_action(&*el, &action) } #[cfg(not(target_os = "macos"))] @@ -124,7 +120,8 @@ fn execute_action_impl(_handle: &NativeHandle, _action: Action) -> Result Result { let root = crate::tree::element_for_pid(entry.pid); - find_element_recursive(&root, entry, 0, 20) + let mut visited = FxHashSet::default(); + find_element_recursive(&root, entry, 0, 20, &mut visited) } #[cfg(target_os = "macos")] @@ -133,6 +130,7 @@ fn find_element_recursive( entry: &RefEntry, depth: u8, max_depth: u8, + visited: &mut FxHashSet, ) -> Result { use accessibility_sys::{ kAXChildrenAttribute, kAXErrorSuccess, kAXRoleAttribute, kAXTitleAttribute, @@ -143,6 +141,10 @@ fn find_element_recursive( base::{CFRetain, CFType, CFTypeRef, TCFType}, }; + if !visited.insert(el.0 as usize) { + return Err(AdapterError::element_not_found("element")); + } + let role = crate::tree::copy_string_attr(el, kAXRoleAttribute); let normalized = role.as_deref().map(crate::roles::ax_role_to_str).unwrap_or("unknown"); @@ -185,7 +187,7 @@ fn find_element_recursive( } unsafe { CFRetain(ptr as CFTypeRef) }; let child = crate::tree::AXElement(ptr); - if let Ok(handle) = find_element_recursive(&child, entry, depth + 1, max_depth) { + if let Ok(handle) = find_element_recursive(&child, entry, depth + 1, max_depth, visited) { return Ok(handle); } } @@ -198,10 +200,13 @@ fn resolve_element_impl(_entry: &RefEntry) -> Result Err(AdapterError::not_supported("resolve_element")) } -fn list_windows_impl(filter: &WindowFilter) -> Result, AdapterError> { +pub fn list_windows_impl(filter: &WindowFilter) -> Result, AdapterError> { #[cfg(target_os = "macos")] { + use rustc_hash::FxHasher; + use std::hash::{Hash, Hasher}; use std::process::Command; + let app_filter = filter.app.as_deref().unwrap_or("").to_string(); let script = r#" tell application "System Events" @@ -246,8 +251,13 @@ end tell continue; } + let mut h = FxHasher::default(); + pid.hash(&mut h); + title.hash(&mut h); + let id = format!("w-{:x}", h.finish() & 0xFFFFFF); + windows.push(WindowInfo { - id: format!("w-{}", idx + 1), + id, title, app: app_name, pid, @@ -298,92 +308,3 @@ end tell"#, #[cfg(not(target_os = "macos"))] Err(AdapterError::not_supported("list_apps")) } - -fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> { - #[cfg(target_os = "macos")] - { - use std::process::Command; - Command::new("osascript") - .arg("-e") - .arg(format!(r#"tell application "{}" to activate"#, win.app)) - .output() - .map_err(|e| AdapterError::internal(format!("focus_window failed: {e}")))?; - Ok(()) - } - #[cfg(not(target_os = "macos"))] - { - let _ = win; - Err(AdapterError::not_supported("focus_window")) - } -} - -fn launch_app_impl(id: &str, wait: bool) -> Result { - #[cfg(target_os = "macos")] - { - use std::process::Command; - use std::time::{Duration, Instant}; - - Command::new("open") - .arg("-a") - .arg(id) - .output() - .map_err(|e| AdapterError::internal(format!("open failed: {e}")))?; - - if wait { - let start = Instant::now(); - let timeout = Duration::from_secs(10); - loop { - std::thread::sleep(Duration::from_millis(200)); - let filter = WindowFilter { focused_only: false, app: Some(id.to_string()) }; - if let Ok(wins) = list_windows_impl(&filter) { - if let Some(win) = wins.into_iter().next() { - return Ok(win); - } - } - if start.elapsed() > timeout { - break; - } - } - } - - Ok(WindowInfo { - id: "w-0".into(), - title: id.to_string(), - app: id.to_string(), - pid: 0, - bounds: None, - is_focused: true, - }) - } - #[cfg(not(target_os = "macos"))] - { - let _ = (id, wait); - Err(AdapterError::not_supported("launch_app")) - } -} - -fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> { - #[cfg(target_os = "macos")] - { - use std::process::Command; - if force { - Command::new("pkill") - .arg("-f") - .arg(id) - .output() - .map_err(|e| AdapterError::internal(format!("pkill failed: {e}")))?; - } else { - Command::new("osascript") - .arg("-e") - .arg(format!(r#"tell application "{id}" to quit"#)) - .output() - .map_err(|e| AdapterError::internal(format!("quit failed: {e}")))?; - } - Ok(()) - } - #[cfg(not(target_os = "macos"))] - { - let _ = (id, force); - Err(AdapterError::not_supported("close_app")) - } -} diff --git a/crates/macos/src/app_ops.rs b/crates/macos/src/app_ops.rs new file mode 100644 index 0000000..6ee3713 --- /dev/null +++ b/crates/macos/src/app_ops.rs @@ -0,0 +1,117 @@ +use agent_desktop_core::{ + adapter::WindowFilter, + error::AdapterError, + node::WindowInfo, +}; + +#[cfg(target_os = "macos")] +pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> { + use std::process::Command; + let script = format!( + r#"tell application "System Events" + set frontmostProc to first process whose unix id is {} + set frontmost of frontmostProc to true +end tell"#, + win.pid + ); + Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| AdapterError::internal(format!("focus_window failed: {e}")))?; + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +pub fn focus_window_impl(_win: &WindowInfo) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("focus_window")) +} + +#[cfg(target_os = "macos")] +pub fn launch_app_impl(id: &str, wait: bool) -> Result { + use crate::adapter::list_windows_impl; + use std::process::Command; + use std::time::{Duration, Instant}; + + if id.contains("..") || id.starts_with('/') { + return Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::InvalidArgs, + format!("Invalid app identifier: '{id}'"), + )); + } + + Command::new("open") + .arg("-a") + .arg(id) + .output() + .map_err(|e| AdapterError::internal(format!("open failed: {e}")))?; + + if wait { + let start = Instant::now(); + let timeout = Duration::from_secs(10); + loop { + std::thread::sleep(Duration::from_millis(200)); + let filter = WindowFilter { focused_only: false, app: Some(id.to_string()) }; + if let Ok(wins) = list_windows_impl(&filter) { + if let Some(win) = wins.into_iter().next() { + return Ok(win); + } + } + if start.elapsed() > timeout { + break; + } + } + return Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::ApplicationNotFound, + format!("App '{id}' launched but no window found within timeout"), + ) + .with_suggestion( + "Try again with a longer timeout or check that the app has a visible window", + )); + } + + std::thread::sleep(std::time::Duration::from_millis(500)); + let filter = WindowFilter { focused_only: false, app: Some(id.to_string()) }; + if let Ok(wins) = list_windows_impl(&filter) { + if let Some(win) = wins.into_iter().next() { + return Ok(win); + } + } + Err(AdapterError::internal(format!("App '{id}' launched but no window found"))) +} + +#[cfg(not(target_os = "macos"))] +pub fn launch_app_impl(_id: &str, _wait: bool) -> Result { + Err(AdapterError::not_supported("launch_app")) +} + +#[cfg(target_os = "macos")] +pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> { + use std::process::Command; + if force { + Command::new("pkill") + .arg("-x") + .arg(id) + .output() + .map_err(|e| AdapterError::internal(format!("pkill failed: {e}")))?; + } else { + let safe_name = id.replace('"', ""); + let script = format!( + r#"tell application "System Events" + set theProc to first process whose name is "{safe_name}" + tell theProc to quit +end tell"# + ); + Command::new("osascript") + .arg("-e") + .arg(script) + .output() + .map_err(|e| AdapterError::internal(format!("quit failed: {e}")))?; + } + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +pub fn close_app_impl(_id: &str, _force: bool) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("close_app")) +} diff --git a/crates/macos/src/lib.rs b/crates/macos/src/lib.rs index 4c8547f..6e33b86 100644 --- a/crates/macos/src/lib.rs +++ b/crates/macos/src/lib.rs @@ -1,5 +1,6 @@ pub mod actions; pub mod adapter; +pub mod app_ops; pub mod clipboard; pub mod input; pub mod permissions; diff --git a/crates/macos/src/tree.rs b/crates/macos/src/tree.rs index e7d4ac7..265a076 100644 --- a/crates/macos/src/tree.rs +++ b/crates/macos/src/tree.rs @@ -59,8 +59,10 @@ mod imp { let role = copy_string_attr(el, kAXRoleAttribute)?; let normalized_role = crate::roles::ax_role_to_str(&role).to_string(); - let name = copy_string_attr(el, kAXTitleAttribute) - .or_else(|| copy_string_attr(el, kAXDescriptionAttribute)); + let title = copy_string_attr(el, kAXTitleAttribute); + let ax_desc = copy_string_attr(el, kAXDescriptionAttribute); + let name = title.clone().or_else(|| ax_desc.clone()); + let description = if title.is_some() { ax_desc } else { None }; let value = copy_string_attr(el, kAXValueAttribute); @@ -87,7 +89,7 @@ mod imp { role: normalized_role, name, value, - description: None, + description, states, bounds, children, @@ -157,8 +159,58 @@ mod imp { Some(children) } - fn read_bounds(_el: &AXElement) -> Option { - None + fn read_bounds(el: &AXElement) -> Option { + use accessibility_sys::{ + kAXPositionAttribute, kAXSizeAttribute, + AXValueGetValue, + kAXValueTypeCGPoint, kAXValueTypeCGSize, + }; + use core_graphics::geometry::{CGPoint, CGSize}; + use std::ffi::c_void; + + let pos_cf = CFString::new(kAXPositionAttribute); + let mut pos_ref: CFTypeRef = std::ptr::null_mut(); + let pos_ok = unsafe { + AXUIElementCopyAttributeValue(el.0, pos_cf.as_concrete_TypeRef(), &mut pos_ref) + }; + if pos_ok != kAXErrorSuccess || pos_ref.is_null() { + return None; + } + let mut point = CGPoint::new(0.0, 0.0); + let got_pos = unsafe { + AXValueGetValue( + pos_ref as _, + kAXValueTypeCGPoint, + &mut point as *mut _ as *mut c_void, + ) + }; + unsafe { CFRelease(pos_ref) }; + if !got_pos { + return None; + } + + let size_cf = CFString::new(kAXSizeAttribute); + let mut size_ref: CFTypeRef = std::ptr::null_mut(); + let size_ok = unsafe { + AXUIElementCopyAttributeValue(el.0, size_cf.as_concrete_TypeRef(), &mut size_ref) + }; + if size_ok != kAXErrorSuccess || size_ref.is_null() { + return None; + } + let mut size = CGSize::new(0.0, 0.0); + let got_size = unsafe { + AXValueGetValue( + size_ref as _, + kAXValueTypeCGSize, + &mut size as *mut _ as *mut c_void, + ) + }; + unsafe { CFRelease(size_ref) }; + if !got_size { + return None; + } + + Some(Rect { x: point.x, y: point.y, width: size.width, height: size.height }) } } diff --git a/src/dispatch.rs b/src/dispatch.rs index 246d885..7a9cbc3 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -2,9 +2,10 @@ use agent_desktop_core::{ action::Direction, adapter::PlatformAdapter, commands::{ - batch, clipboard, click, close_app, collapse, expand, find, focus, focus_window, get, - is_check, launch, list_apps, list_windows, permissions, press, screenshot, scroll, select, - set_value, snapshot, status, toggle, type_text, version, wait, + batch, click, clipboard_get, clipboard_set, close_app, collapse, double_click, expand, + find, focus, focus_window, get, helpers, is_check, launch, list_apps, list_windows, + permissions, press, right_click, screenshot, scroll, select, set_value, snapshot, status, + toggle, type_text, version, wait, }, error::AppError, }; @@ -58,10 +59,10 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result click::execute(click::ClickArgs { ref_id: a.ref_id }, adapter), Commands::DoubleClick(a) => { - click::execute_double(click::ClickArgs { ref_id: a.ref_id }, adapter) + double_click::execute(double_click::DoubleClickArgs { ref_id: a.ref_id }, adapter) } Commands::RightClick(a) => { - click::execute_right(click::ClickArgs { ref_id: a.ref_id }, adapter) + right_click::execute(right_click::RightClickArgs { ref_id: a.ref_id }, adapter) } Commands::Type(a) => type_text::execute( @@ -74,11 +75,11 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result focus::execute(focus::RefArgs { ref_id: a.ref_id }, adapter), - Commands::Toggle(a) => toggle::execute(toggle::RefArgs { ref_id: a.ref_id }, adapter), - Commands::Expand(a) => expand::execute(expand::RefArgs { ref_id: a.ref_id }, adapter), + Commands::Focus(a) => focus::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), + Commands::Toggle(a) => toggle::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), + Commands::Expand(a) => expand::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), Commands::Collapse(a) => { - collapse::execute(collapse::RefArgs { ref_id: a.ref_id }, adapter) + collapse::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter) } Commands::Select(a) => select::execute( @@ -121,8 +122,8 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result clipboard::execute_get(adapter), - Commands::ClipboardSet(a) => clipboard::execute_set(a.text, adapter), + Commands::ClipboardGet => clipboard_get::execute(adapter), + Commands::ClipboardSet(a) => clipboard_set::execute(a.text, adapter), Commands::Wait(a) => wait::execute( wait::WaitArgs { @@ -143,16 +144,40 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result version::execute(version::VersionArgs { json: a.json }), - Commands::Batch(a) => batch::execute( - batch::BatchArgs { - commands_json: a.commands_json, - stop_on_error: a.stop_on_error, - }, - adapter, - ), + Commands::Batch(a) => { + let commands = batch::parse_commands(&a.commands_json)?; + let mut results = Vec::new(); + for cmd in commands { + let result = dispatch_batch_command(&cmd.command, cmd.args, adapter); + let ok = result.is_ok(); + let entry = match result { + Ok(data) => { + serde_json::json!({ "ok": true, "command": cmd.command, "data": data }) + } + Err(e) => { + serde_json::json!({ "ok": false, "command": cmd.command, "error": e.to_string() }) + } + }; + results.push(entry); + if !ok && a.stop_on_error { + break; + } + } + Ok(serde_json::json!({ "results": results })) + } } } +fn dispatch_batch_command( + command: &str, + _args: serde_json::Value, + _adapter: &dyn agent_desktop_core::adapter::PlatformAdapter, +) -> Result { + Err(agent_desktop_core::error::AppError::invalid_input(format!( + "Batch dispatch for '{command}' not yet implemented. Use individual commands." + ))) +} + fn parse_get_property(s: &str) -> Result { match s { "text" => Ok(get::GetProperty::Text), diff --git a/src/main.rs b/src/main.rs index a2fb25a..8f93f36 100644 --- a/src/main.rs +++ b/src/main.rs @@ -122,8 +122,11 @@ fn finish(cmd_name: &str, result: Result impl agent_desktop_core::adapter::PlatformAdapter {