diff --git a/crates/core/src/adapter.rs b/crates/core/src/adapter.rs index ce208f3..9f71f8f 100644 --- a/crates/core/src/adapter.rs +++ b/crates/core/src/adapter.rs @@ -153,4 +153,8 @@ pub trait PlatformAdapter: Send + Sync { fn focused_window(&self) -> Result, AdapterError> { Err(AdapterError::not_supported("focused_window")) } + + fn get_live_value(&self, _handle: &NativeHandle) -> Result, AdapterError> { + Err(AdapterError::not_supported("get_live_value")) + } } diff --git a/crates/core/src/commands/batch.rs b/crates/core/src/commands/batch.rs index 77b6238..c778e4b 100644 --- a/crates/core/src/commands/batch.rs +++ b/crates/core/src/commands/batch.rs @@ -1,6 +1,6 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::error::AppError; use serde::Deserialize; -use serde_json::{json, Value}; +use serde_json::Value; pub struct BatchArgs { pub commands_json: String, @@ -18,11 +18,3 @@ pub fn parse_commands(json_str: &str) -> Result, AppError> { serde_json::from_str(json_str) .map_err(|e| AppError::invalid_input(format!("Invalid batch JSON: {e}"))) } - -pub fn execute(args: BatchArgs, _adapter: &dyn PlatformAdapter) -> Result { - let commands = parse_commands(&args.commands_json)?; - Ok(json!({ - "note": "Batch execution delegated to dispatch layer", - "count": commands.len() - })) -} diff --git a/crates/core/src/commands/clipboard.rs b/crates/core/src/commands/clipboard.rs deleted file mode 100644 index 24017f8..0000000 --- a/crates/core/src/commands/clipboard.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; -use serde_json::{json, Value}; - -pub fn execute_get(adapter: &dyn PlatformAdapter) -> Result { - let text = adapter.get_clipboard()?; - Ok(json!({ "text": text })) -} - -pub fn execute_set(text: String, adapter: &dyn PlatformAdapter) -> Result { - adapter.set_clipboard(&text)?; - Ok(json!({ "ok": true })) -} diff --git a/crates/core/src/commands/get.rs b/crates/core/src/commands/get.rs index 75409b9..ee09410 100644 --- a/crates/core/src/commands/get.rs +++ b/crates/core/src/commands/get.rs @@ -16,12 +16,15 @@ pub enum GetProperty { } pub fn execute(args: GetArgs, adapter: &dyn PlatformAdapter) -> Result { - let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?; + let (entry, handle) = resolve_ref(&args.ref_id, adapter)?; let value = match args.property { GetProperty::Role => json!(entry.role), - GetProperty::Text | GetProperty::Title => json!(entry.name), - GetProperty::Value => json!(entry.value), + GetProperty::Title => json!(entry.name), + GetProperty::Text | GetProperty::Value => { + let live = adapter.get_live_value(&handle).ok().flatten(); + json!(live.or(entry.value)) + } GetProperty::Bounds => json!(entry.bounds), GetProperty::States => json!(entry.states), }; diff --git a/crates/core/src/commands/is_check.rs b/crates/core/src/commands/is_check.rs index 3f56670..68ba5b3 100644 --- a/crates/core/src/commands/is_check.rs +++ b/crates/core/src/commands/is_check.rs @@ -14,6 +14,9 @@ pub enum IsProperty { Expanded, } +/// States are read from the last snapshot's RefMap. `resolve_ref` verifies the element +/// is still live before returning, but the state values themselves are not re-queried +/// from the AX API. Run `snapshot` to refresh state before calling `is`. pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result { let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?; diff --git a/crates/core/src/commands/mod.rs b/crates/core/src/commands/mod.rs index ccc3605..e5b21a7 100644 --- a/crates/core/src/commands/mod.rs +++ b/crates/core/src/commands/mod.rs @@ -1,5 +1,4 @@ pub mod batch; -pub mod clipboard; pub mod clipboard_get; pub mod clipboard_set; pub mod click; diff --git a/crates/core/src/commands/press.rs b/crates/core/src/commands/press.rs index cfc2609..139619e 100644 --- a/crates/core/src/commands/press.rs +++ b/crates/core/src/commands/press.rs @@ -34,11 +34,12 @@ pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result Result { let parts: Vec<&str> = s.split('+').collect(); - if parts.is_empty() { - return Err(AppError::invalid_input("Empty key combo")); - } - - let key = parts.last().unwrap().to_string(); + let key = parts + .last() + .copied() + .filter(|k| !k.is_empty()) + .ok_or_else(|| AppError::invalid_input("Empty key combo"))? + .to_string(); let mut modifiers = Vec::new(); for &part in &parts[..parts.len() - 1] { diff --git a/crates/core/src/commands/wait.rs b/crates/core/src/commands/wait.rs index 8fde27d..31c3eb3 100644 --- a/crates/core/src/commands/wait.rs +++ b/crates/core/src/commands/wait.rs @@ -44,11 +44,11 @@ fn wait_for_element( loop { if let Ok(refmap) = RefMap::load() { - if refmap.get(&ref_id).is_some() - && adapter.resolve_element(refmap.get(&ref_id).unwrap()).is_ok() - { - let elapsed = start.elapsed().as_millis(); - return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed })); + if let Some(entry) = refmap.get(&ref_id) { + if adapter.resolve_element(entry).is_ok() { + let elapsed = start.elapsed().as_millis(); + return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed })); + } } } diff --git a/crates/core/src/snapshot.rs b/crates/core/src/snapshot.rs index 673fe0f..88ece52 100644 --- a/crates/core/src/snapshot.rs +++ b/crates/core/src/snapshot.rs @@ -8,6 +8,7 @@ use crate::{ const INTERACTIVE_ROLES: &[&str] = &[ "button", "textfield", "checkbox", "link", "menuitem", "tab", "slider", "combobox", "treeitem", "cell", "radiobutton", "incrementor", + "menubutton", "switch", "colorwell", "dockitem", ]; pub struct SnapshotResult { @@ -129,7 +130,14 @@ fn allocate_refs( node.children = node .children .into_iter() - .map(|child| allocate_refs(child, refmap, include_bounds, interactive_only, window_pid, source_app)) + .filter_map(|child| { + let child = allocate_refs(child, refmap, include_bounds, interactive_only, window_pid, source_app); + if interactive_only && child.ref_id.is_none() && child.children.is_empty() { + None + } else { + Some(child) + } + }) .collect(); node diff --git a/crates/macos/Cargo.toml b/crates/macos/Cargo.toml index 776fed9..178960d 100644 --- a/crates/macos/Cargo.toml +++ b/crates/macos/Cargo.toml @@ -19,4 +19,19 @@ core-foundation = "0.10.1" core-foundation-sys = "0.8.7" core-graphics = { version = "0.25.0", features = ["highsierra"] } +[features] +dev-tools = [] + +[[example]] +name = "ax_probe" +required-features = ["dev-tools"] + +[[example]] +name = "axprobe" +required-features = ["dev-tools"] + +[[example]] +name = "axprobe2" +required-features = ["dev-tools"] + [build-dependencies] diff --git a/crates/macos/examples/ax_probe.rs b/crates/macos/examples/ax_probe.rs new file mode 100644 index 0000000..e3322b1 --- /dev/null +++ b/crates/macos/examples/ax_probe.rs @@ -0,0 +1,261 @@ +/// Direct probe of macOS AX APIs — no abstraction. +/// Reveals exactly what the raw APIs return and validates click behavior. +/// +/// cargo run -p agent-desktop-macos --example ax_probe -- +use std::ffi::c_void; + +#[cfg(target_os = "macos")] +fn main() { + let app_name = std::env::args().nth(1).unwrap_or_else(|| "Finder".into()); + println!("=== AX Probe: '{}' ===\n", app_name); + + let pid = find_pid(&app_name).expect("app not running"); + println!("[pid] {}", pid); + + probe_app(pid); +} + +#[cfg(target_os = "macos")] +fn find_pid(name: &str) -> Option { + let out = std::process::Command::new("pgrep").arg("-xi").arg(name).output().ok()?; + String::from_utf8_lossy(&out.stdout).trim().lines().next()?.trim().parse().ok() +} + +#[cfg(target_os = "macos")] +fn probe_app(pid: i32) { + use accessibility_sys::*; + use core_foundation::base::CFTypeRef; + + let app = unsafe { AXUIElementCreateApplication(pid) }; + + // ── 1. What attributes does the app element expose? ────────────────────── + println!("\n[1] App element attribute names:"); + let attr_names = get_attribute_names(app); + for n in &attr_names { println!(" {n}"); } + + // ── 2. Get windows the right way ───────────────────────────────────────── + println!("\n[2] Windows via kAXWindowsAttribute:"); + let windows = get_ax_children(app, kAXWindowsAttribute); + println!(" count = {}", windows.len()); + for (i, win) in windows.iter().enumerate() { + let role = read_string(*win, kAXRoleAttribute); + let title = read_string(*win, kAXTitleAttribute); + let pos = read_cgpoint(*win, kAXPositionAttribute); + let size = read_cgsize(*win, kAXSizeAttribute); + println!(" [{i}] role={:?} title={:?} pos={:?} size={:?}", role, title, pos, size); + + // Children of this window + let children = get_ax_children(*win, kAXChildrenAttribute); + println!(" children = {}", children.len()); + for (ci, child) in children.iter().enumerate().take(8) { + let cr = read_string(*child, kAXRoleAttribute); + let ct = read_string(*child, kAXTitleAttribute); + let cd = read_string(*child, kAXDescriptionAttribute); + let cv = read_string(*child, kAXValueAttribute); + let cpos = read_cgpoint(*child, kAXPositionAttribute); + let csz = read_cgsize(*child, kAXSizeAttribute); + + println!(" [{ci}] role={:?} title={:?} desc={:?} val={:?}", cr, ct, cd, cv); + println!(" pos={:?} size={:?}", cpos, csz); + + // ── 3. Test kAXPressAction on each child ──────────────────────── + let ax_err = ax_press(*child); + println!(" kAXPressAction → err={} (0=ok, -25200=fail, -25205=not_supported)", ax_err); + + // ── 4. Test CGEvent click at element center ───────────────────── + if let (Some(p), Some(s)) = (cpos, csz) { + let cx = p.0 + s.0 / 2.0; + let cy = p.1 + s.1 / 2.0; + let cg_ok = cg_click(cx, cy); + println!(" CGEvent click at ({:.0},{:.0}) → {}", cx, cy, if cg_ok { "OK" } else { "FAIL" }); + } else { + println!(" CGEvent click → no bounds available"); + } + + release_ax(*child); + } + for child in children.iter().skip(8) { release_ax(*child); } + + release_ax(*win); + if i >= 1 { break; } // only first 2 windows + } + + // ── 5. Multi-attribute fetch speed comparison ───────────────────────────── + println!("\n[5] AXUIElementCopyMultipleAttributeValues vs individual calls:"); + speed_test(app, pid); + + // ── 6. Scroll event test ───────────────────────────────────────────────── + println!("\n[6] CGEvent scroll at (400, 400):"); + let ok = cg_scroll(400.0, 400.0, 0, -3); + println!(" result = {}", if ok { "OK" } else { "FAIL" }); + + unsafe { core_foundation::base::CFRelease(app as CFTypeRef) }; + println!("\n=== Done ==="); +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +#[cfg(target_os = "macos")] +fn get_attribute_names(el: accessibility_sys::AXUIElementRef) -> Vec { + use accessibility_sys::*; + use core_foundation::{array::CFArray, base::CFTypeRef, string::CFString, base::TCFType}; + + let mut out_ref: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeNames(el, &mut out_ref as *mut _ as *mut _) }; + if err != kAXErrorSuccess || out_ref.is_null() { return vec![]; } + let arr = unsafe { CFArray::::wrap_under_create_rule(out_ref as _) }; + arr.into_iter().map(|s| s.to_string()).collect() +} + +/// Read an array-typed AX attribute, retaining each element so it stays alive. +#[cfg(target_os = "macos")] +fn get_ax_children(el: accessibility_sys::AXUIElementRef, attr: &str) -> Vec { + use accessibility_sys::*; + use core_foundation::{array::CFArray, base::{CFRetain, CFType, CFTypeRef, TCFType}, string::CFString}; + + let key = CFString::new(attr); + let mut val: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) }; + if err != kAXErrorSuccess || val.is_null() { return vec![]; } + + let arr = unsafe { CFArray::::wrap_under_create_rule(val as _) }; + arr.into_iter().filter_map(|item| { + let ptr = item.as_concrete_TypeRef() as AXUIElementRef; + if ptr.is_null() { return None; } + // Retain so the element lives past CFArray dealloc + unsafe { CFRetain(ptr as CFTypeRef) }; + Some(ptr) + }).collect() +} + +#[cfg(target_os = "macos")] +fn release_ax(el: accessibility_sys::AXUIElementRef) { + if !el.is_null() { + unsafe { core_foundation::base::CFRelease(el as core_foundation::base::CFTypeRef) }; + } +} + +#[cfg(target_os = "macos")] +fn read_string(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option { + use accessibility_sys::*; + use core_foundation::{base::{CFType, CFTypeRef, TCFType}, string::CFString}; + + let key = CFString::new(attr); + let mut val: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) }; + if err != kAXErrorSuccess || val.is_null() { return None; } + let cf = unsafe { CFType::wrap_under_create_rule(val) }; + cf.downcast::().map(|s| s.to_string()) +} + +#[cfg(target_os = "macos")] +fn read_cgpoint(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<(f64, f64)> { + use accessibility_sys::*; + use core_foundation::{base::{CFTypeRef, TCFType}, string::CFString}; + use core_graphics::geometry::CGPoint; + + let key = CFString::new(attr); + let mut val: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) }; + if err != kAXErrorSuccess || val.is_null() { return None; } + let mut pt = CGPoint::new(0.0, 0.0); + let ok = unsafe { AXValueGetValue(val as _, kAXValueTypeCGPoint, &mut pt as *mut _ as *mut std::ffi::c_void) }; + unsafe { core_foundation::base::CFRelease(val) }; + if ok { Some((pt.x, pt.y)) } else { None } +} + +#[cfg(target_os = "macos")] +fn read_cgsize(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<(f64, f64)> { + use accessibility_sys::*; + use core_foundation::{base::{CFTypeRef, TCFType}, string::CFString}; + use core_graphics::geometry::CGSize; + + let key = CFString::new(attr); + let mut val: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) }; + if err != kAXErrorSuccess || val.is_null() { return None; } + let mut sz = CGSize::new(0.0, 0.0); + let ok = unsafe { AXValueGetValue(val as _, kAXValueTypeCGSize, &mut sz as *mut _ as *mut std::ffi::c_void) }; + unsafe { core_foundation::base::CFRelease(val) }; + if ok { Some((sz.width, sz.height)) } else { None } +} + +#[cfg(target_os = "macos")] +fn ax_press(el: accessibility_sys::AXUIElementRef) -> i32 { + use accessibility_sys::*; + use core_foundation::{base::TCFType, string::CFString}; + let action = CFString::new(kAXPressAction); + unsafe { AXUIElementPerformAction(el, action.as_concrete_TypeRef()) } +} + +#[cfg(target_os = "macos")] +fn cg_click(x: f64, y: f64) -> bool { + use core_graphics::{ + event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton}, + event_source::{CGEventSource, CGEventSourceStateID}, + geometry::CGPoint, + }; + let src = match CGEventSource::new(CGEventSourceStateID::HIDSystemState) { Ok(s) => s, Err(_) => return false }; + let pt = CGPoint::new(x, y); + let down = CGEvent::new_mouse_event(src.clone(), CGEventType::LeftMouseDown, pt, CGMouseButton::Left); + let up = CGEvent::new_mouse_event(src, CGEventType::LeftMouseUp, pt, CGMouseButton::Left); + match (down, up) { + (Ok(d), Ok(u)) => { d.post(CGEventTapLocation::HID); u.post(CGEventTapLocation::HID); true } + _ => false, + } +} + +#[cfg(target_os = "macos")] +fn cg_scroll(x: f64, y: f64, dx: i32, dy: i32) -> bool { + use core_graphics::{ + event::{CGEvent, CGEventTapLocation, ScrollEventUnit}, + event_source::{CGEventSource, CGEventSourceStateID}, + }; + let src = match CGEventSource::new(CGEventSourceStateID::HIDSystemState) { Ok(s) => s, Err(_) => return false }; + match CGEvent::new_scroll_event(src, ScrollEventUnit::LINE, 2, dy, dx, 0) { + Ok(ev) => { ev.post(CGEventTapLocation::HID); true } + Err(_) => false, + } +} + +#[cfg(target_os = "macos")] +fn speed_test(app: accessibility_sys::AXUIElementRef, pid: i32) { + use accessibility_sys::*; + use core_foundation::{array::CFArray, base::{CFRelease, CFType, CFTypeRef, TCFType}, string::CFString}; + use std::time::Instant; + + // Get a real window element to test on + let windows = get_ax_children(app, kAXWindowsAttribute); + let el = if let Some(&w) = windows.first() { w } else { app }; + + let attrs = [kAXRoleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, + kAXValueAttribute, kAXEnabledAttribute, kAXFocusedAttribute]; + let cf_attrs: Vec = attrs.iter().map(|a| CFString::new(a)).collect(); + let cf_refs: Vec<_> = cf_attrs.iter().map(|s| s.as_concrete_TypeRef()).collect(); + let names_arr = CFArray::from_copyable(&cf_refs); + + // Multi-attr + let t = Instant::now(); + for _ in 0..100 { + let mut res: CFTypeRef = std::ptr::null_mut(); + unsafe { AXUIElementCopyMultipleAttributeValues(el, names_arr.as_concrete_TypeRef(), 0, &mut res as *mut _ as *mut _) }; + if !res.is_null() { unsafe { CFRelease(res) }; } + } + let multi = t.elapsed(); + + // Individual attrs + let t2 = Instant::now(); + for _ in 0..100 { + for attr in &attrs { let _ = read_string(el, attr); } + } + let single = t2.elapsed(); + + println!(" 100x multi-attr: {:?} ({:?}/call)", multi, multi / 100); + println!(" 100x individual: {:?} ({:?}/call)", single, single / 100); + println!(" speedup: {:.1}x", single.as_nanos() as f64 / multi.as_nanos().max(1) as f64); + + for w in windows { release_ax(w); } +} + +#[cfg(not(target_os = "macos"))] +fn main() { eprintln!("macOS only"); } diff --git a/crates/macos/examples/axprobe.rs b/crates/macos/examples/axprobe.rs new file mode 100644 index 0000000..eb2036c --- /dev/null +++ b/crates/macos/examples/axprobe.rs @@ -0,0 +1,395 @@ +//! AX API probe — discovers what every accessibility function returns on a live app. +//! Run: cargo run -p agent-desktop-macos --bin axprobe -- Finder +//! +//! This is a diagnostic tool used to learn exactly which attributes hold which +//! data before writing tree traversal code. Output is deliberately verbose. + +fn main() { + #[cfg(target_os = "macos")] + run(); + + #[cfg(not(target_os = "macos"))] + eprintln!("axprobe only works on macOS"); +} + +#[cfg(target_os = "macos")] +fn run() { + use accessibility_sys::*; + use core_foundation::{ + array::CFArray, + base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + url::CFURL, + }; + + let app_name = std::env::args().nth(1).unwrap_or_else(|| "Finder".to_string()); + let pid = find_pid(&app_name).unwrap_or_else(|| { + eprintln!("App '{}' not running", app_name); + std::process::exit(1); + }); + + println!("=== axprobe: {} (pid {}) ===\n", app_name, pid); + + let app_el = unsafe { AXUIElementCreateApplication(pid) }; + unsafe { AXUIElementSetMessagingTimeout(app_el, 5.0) }; + + // ── 1. App-level attributes ────────────────────────────────────────── + println!("──────────────────────────────────────────────"); + println!("SECTION 1: App element attributes"); + println!("──────────────────────────────────────────────"); + dump_all_attrs(app_el, 0); + + // ── 2. Windows ─────────────────────────────────────────────────────── + println!("\n──────────────────────────────────────────────"); + println!("SECTION 2: Windows"); + println!("──────────────────────────────────────────────"); + let windows = copy_el_array(app_el, "AXWindows"); + println!("Window count: {}", windows.len()); + + for (wi, &win) in windows.iter().enumerate() { + unsafe { AXUIElementSetMessagingTimeout(win, 5.0) }; + let title = fetch_repr(win, "AXTitle"); + println!("\n Window[{}] AXTitle={}", wi, title); + dump_all_attrs(win, 2); + + // ── 3. Direct children of window ───────────────────────────── + let children = copy_el_array(win, "AXChildren"); + println!("\n Direct children: {}", children.len()); + + for (ci, &child) in children.iter().enumerate() { + let role = fetch_repr(child, "AXRole"); + let subrole = fetch_repr(child, "AXSubrole"); + let title = fetch_repr(child, "AXTitle"); + let desc = fetch_repr(child, "AXDescription"); + let value = fetch_repr(child, "AXValue"); + println!("\n Child[{}] role={} subrole={} title={} desc={} value={}", + ci, role, subrole, title, desc, value); + dump_all_attrs(child, 6); + + // Grandchildren + let grandchildren = copy_el_array(child, "AXChildren"); + println!(" grandchildren: {}", grandchildren.len()); + for (gci, &gc) in grandchildren.iter().enumerate().take(10) { + let r = fetch_repr(gc, "AXRole"); + let s = fetch_repr(gc, "AXSubrole"); + let t = fetch_repr(gc, "AXTitle"); + let d = fetch_repr(gc, "AXDescription"); + let v = fetch_repr(gc, "AXValue"); + println!(" GC[{}] role={} subrole={} title={} desc={} value={}", + gci, r, s, t, d, v); + dump_all_attrs(gc, 8); + + // Great-grandchildren (just roles/names) + let ggc = copy_el_array(gc, "AXChildren"); + for (ggci, &el) in ggc.iter().enumerate().take(6) { + let r2 = fetch_repr(el, "AXRole"); + let t2 = fetch_repr(el, "AXTitle"); + let d2 = fetch_repr(el, "AXDescription"); + let v2 = fetch_repr(el, "AXValue"); + println!(" GGC[{}] role={} title={} desc={} value={}", + ggci, r2, t2, d2, v2); + dump_all_attrs(el, 10); + + for &gggel in copy_el_array(el, "AXChildren").iter().take(4) { + let r3 = fetch_repr(gggel, "AXRole"); + let t3 = fetch_repr(gggel, "AXTitle"); + let v3 = fetch_repr(gggel, "AXValue"); + println!(" GGGC role={} title={} value={}", r3, t3, v3); + unsafe { CFRelease(gggel as CFTypeRef) }; + } + unsafe { CFRelease(el as CFTypeRef) }; + } + unsafe { CFRelease(gc as CFTypeRef) }; + } + unsafe { CFRelease(child as CFTypeRef) }; + } + + // Only probe first window in depth + break; + } + + // ── 4. AXCopyMultipleAttributeValues API test ──────────────────────── + println!("\n──────────────────────────────────────────────"); + println!("SECTION 3: AXUIElementCopyMultipleAttributeValues test"); + println!("──────────────────────────────────────────────"); + + // Use first window child for the multi-attr test + if !windows.is_empty() { + let win = windows[0]; + let children = copy_el_array(win, "AXChildren"); + if !children.is_empty() { + let el = children[0]; + let role = fetch_repr(el, "AXRole"); + println!("Test element: role={}", role); + test_multi_attr(el); + unsafe { CFRelease(el as CFTypeRef) }; + } + } + + // ── 5. AXUIElementCopyElementAtPosition ───────────────────────────── + println!("\n──────────────────────────────────────────────"); + println!("SECTION 4: AXUIElementCopyElementAtPosition"); + println!("──────────────────────────────────────────────"); + for (x, y) in [(400.0f32, 300.0), (600.0, 300.0), (200.0, 400.0)] { + let mut pos_el: AXUIElementRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyElementAtPosition(app_el, x, y, &mut pos_el) }; + if err == 0 && !pos_el.is_null() { + let r = fetch_repr(pos_el, "AXRole"); + let t = fetch_repr(pos_el, "AXTitle"); + let d = fetch_repr(pos_el, "AXDescription"); + let v = fetch_repr(pos_el, "AXValue"); + println!(" ({},{}): role={} title={} desc={} value={}", x, y, r, t, d, v); + unsafe { CFRelease(pos_el as CFTypeRef) }; + } else { + println!(" ({},{}): err={}", x, y, err); + } + } + + // ── 6. Action names ────────────────────────────────────────────────── + println!("\n──────────────────────────────────────────────"); + println!("SECTION 5: AXUIElementCopyActionNames"); + println!("──────────────────────────────────────────────"); + if !windows.is_empty() { + let win = windows[0]; + let children = copy_el_array(win, "AXChildren"); + for &child in children.iter().take(5) { + let role = fetch_repr(child, "AXRole"); + let actions = copy_action_names(child); + println!(" {} => {:?}", role, actions); + unsafe { CFRelease(child as CFTypeRef) }; + } + } + + // Cleanup + for &win in &windows { + unsafe { CFRelease(win as CFTypeRef) }; + } + unsafe { CFRelease(app_el as CFTypeRef) }; +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +#[cfg(target_os = "macos")] +fn find_pid(app_name: &str) -> Option { + let out = std::process::Command::new("pgrep") + .arg("-x") + .arg(app_name) + .output() + .ok()?; + String::from_utf8_lossy(&out.stdout).lines().next()?.trim().parse().ok() +} + +#[cfg(target_os = "macos")] +fn dump_all_attrs(el: accessibility_sys::AXUIElementRef, indent: usize) { + use accessibility_sys::AXUIElementCopyAttributeNames; + use core_foundation::{ + array::CFArray, + base::{CFType, TCFType}, + string::CFString, + }; + + let pad = " ".repeat(indent); + let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) }; + if err != 0 || names_ref.is_null() { + println!("{} ", pad, err); + return; + } + let arr = unsafe { CFArray::::wrap_under_create_rule(names_ref as _) }; + let names: Vec = arr + .into_iter() + .filter_map(|item| item.downcast::().map(|s| s.to_string())) + .collect(); + + for name in &names { + let val = fetch_repr(el, name); + println!("{} [attr] {}: {}", pad, name, val); + } + + // Also test AXUIElementCopyParameterizedAttributeNames + let mut pnames_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut(); + let perr = unsafe { + accessibility_sys::AXUIElementCopyParameterizedAttributeNames(el, &mut pnames_ref) + }; + if perr == 0 && !pnames_ref.is_null() { + let parr = unsafe { CFArray::::wrap_under_create_rule(pnames_ref as _) }; + let pnames: Vec = parr + .into_iter() + .filter_map(|item| item.downcast::().map(|s| s.to_string())) + .collect(); + if !pnames.is_empty() { + println!("{} [parameterized attrs]: {:?}", pad, pnames); + } + } +} + +#[cfg(target_os = "macos")] +fn fetch_repr(el: accessibility_sys::AXUIElementRef, attr: &str) -> String { + use accessibility_sys::AXUIElementCopyAttributeValue; + use core_foundation::{ + array::CFArray, + base::{CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + url::CFURL, + }; + + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != 0 { + return format!("", err); + } + if value.is_null() { + return "".to_string(); + } + + let cf = unsafe { CFType::wrap_under_create_rule(value) }; + + if let Some(s) = cf.downcast::() { + return format!("\"{}\"", s.to_string()); + } + if let Some(b) = cf.downcast::() { + return format!("bool:{}", bool::from(b)); + } + if let Some(n) = cf.downcast::() { + if let Some(i) = n.to_i64() { + return format!("num:{}", i); + } + if let Some(f) = n.to_f64() { + return format!("num:{:.2}", f); + } + } + // Check if it's an array by type ID + let arr_type_id = unsafe { core_foundation_sys::array::CFArrayGetTypeID() }; + if cf.type_of() == arr_type_id { + let arr = unsafe { + CFArray::::wrap_under_get_rule( + cf.as_concrete_TypeRef() as core_foundation_sys::array::CFArrayRef + ) + }; + return format!("array[{}]", arr.len()); + } + if let Some(url) = cf.downcast::() { + return format!("url:{}", url.get_string().to_string()); + } + + format!("cftype:{}", cf.type_of()) +} + +#[cfg(target_os = "macos")] +fn copy_el_array( + el: accessibility_sys::AXUIElementRef, + attr: &str, +) -> Vec { + use accessibility_sys::AXUIElementCopyAttributeValue; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != 0 || value.is_null() { + return vec![]; + } + let arr = unsafe { CFArray::::wrap_under_create_rule(value as _) }; + arr.into_iter() + .filter_map(|item| { + let ptr = item.as_concrete_TypeRef() as accessibility_sys::AXUIElementRef; + if ptr.is_null() { + None + } else { + unsafe { CFRetain(ptr as CFTypeRef) }; + Some(ptr) + } + }) + .collect() +} + +#[cfg(target_os = "macos")] +fn copy_action_names(el: accessibility_sys::AXUIElementRef) -> Vec { + use accessibility_sys::AXUIElementCopyActionNames; + use core_foundation::{ + array::CFArray, + base::{CFType, TCFType}, + string::CFString, + }; + + let mut ref_: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyActionNames(el, &mut ref_) }; + if err != 0 || ref_.is_null() { + return vec![]; + } + let arr = unsafe { CFArray::::wrap_under_create_rule(ref_ as _) }; + arr.into_iter() + .filter_map(|item| item.downcast::().map(|s| s.to_string())) + .collect() +} + +#[cfg(target_os = "macos")] +fn test_multi_attr(el: accessibility_sys::AXUIElementRef) { + use accessibility_sys::{ + AXUIElementCopyMultipleAttributeValues, kAXCopyMultipleAttributeOptionStopOnError, + }; + use core_foundation::{ + array::CFArray, + base::{CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + }; + + let test_attrs = [ + "AXRole", "AXSubrole", "AXTitle", "AXDescription", "AXValue", + "AXEnabled", "AXFocused", "AXHelp", "AXPlaceholderValue", + "AXRoleDescription", + ]; + + for &options in &[0u32, kAXCopyMultipleAttributeOptionStopOnError] { + let label = if options == 0 { "AllowPartial(0)" } else { "StopOnError(0x1)" }; + println!(" options={}", label); + + let cf_names: Vec = test_attrs.iter().map(|a| CFString::new(a)).collect(); + let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect(); + let names_arr = CFArray::from_copyable(&cf_refs); + + let mut result_ref: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyMultipleAttributeValues( + el, + names_arr.as_concrete_TypeRef(), + options, + &mut result_ref as *mut _ as *mut _, + ) + }; + println!(" err={}", err); + + if err == 0 && !result_ref.is_null() { + let arr = unsafe { CFArray::::wrap_under_create_rule(result_ref as _) }; + println!(" result_count={} (requested {})", arr.len(), test_attrs.len()); + for (i, item) in arr.into_iter().enumerate() { + let name = test_attrs.get(i).unwrap_or(&"?"); + let repr = if let Some(s) = item.downcast::() { + format!("String(\"{}\")", s.to_string()) + } else if let Some(b) = item.downcast::() { + format!("Bool({})", bool::from(b)) + } else if let Some(n) = item.downcast::() { + format!("Number({})", n.to_i64().unwrap_or(-1)) + } else { + format!("Other(type_id={})", item.type_of()) + }; + println!(" [{}] {} = {}", i, name, repr); + } + } + } +} diff --git a/crates/macos/examples/axprobe2.rs b/crates/macos/examples/axprobe2.rs new file mode 100644 index 0000000..33b942f --- /dev/null +++ b/crates/macos/examples/axprobe2.rs @@ -0,0 +1,378 @@ +//! Deep probe of AXOutline sidebar rows and AXBrowser columns in Finder. +//! Run: cargo run -p agent-desktop-macos --bin axprobe2 + +fn main() { + #[cfg(target_os = "macos")] + run(); + #[cfg(not(target_os = "macos"))] + eprintln!("macOS only"); +} + +#[cfg(target_os = "macos")] +fn run() { + use accessibility_sys::*; + use core_foundation::{ + array::CFArray, + base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + }; + + let pid = find_pid("Finder").unwrap_or_else(|| { + eprintln!("Finder not running"); + std::process::exit(1); + }); + + let app_el = unsafe { AXUIElementCreateApplication(pid) }; + unsafe { AXUIElementSetMessagingTimeout(app_el, 5.0) }; + + let windows = copy_el_array(app_el, "AXWindows"); + let win = match windows.first() { + Some(&w) => w, + None => { eprintln!("No windows"); return; } + }; + unsafe { AXUIElementSetMessagingTimeout(win, 5.0) }; + + println!("=== Finder Window: {} ===\n", fetch_str(win, "AXTitle")); + + // ── Sidebar AXOutline → AXRow children ───────────────────────────── + println!("────────────────────────────────────────"); + println!("PART 1: Sidebar AXOutline rows (6 levels deep)"); + println!("────────────────────────────────────────"); + + let sidebar_outline = find_by_desc(win, "sidebar", 0); + if let Some(outline) = sidebar_outline { + let outline_role = fetch_str(outline, "AXRole"); + let outline_desc = fetch_str(outline, "AXDescription"); + println!("Found: role={} desc={}", outline_role, outline_desc); + println!("AXRows count: {}", count_attr(outline, "AXRows")); + println!("AXChildren count: {}", count_attr(outline, "AXChildren")); + + let rows = copy_el_array(outline, "AXRows"); + println!("Probing first 8 AXRows (all attributes + full children):"); + for (i, &row) in rows.iter().enumerate().take(8) { + dump_element(row, i, 2); + } + for &row in &rows { unsafe { CFRelease(row as CFTypeRef) }; } + unsafe { CFRelease(outline as CFTypeRef) }; + } else { + println!("Could not find sidebar outline"); + } + + // ── AXBrowser column view ─────────────────────────────────────────── + println!("\n────────────────────────────────────────"); + println!("PART 2: AXBrowser column view (AXColumns)"); + println!("────────────────────────────────────────"); + + let browser = find_by_role(win, "AXBrowser", 0); + if let Some(br) = browser { + println!("Found AXBrowser: desc={}", fetch_str(br, "AXDescription")); + println!("AXColumns: {}", count_attr(br, "AXColumns")); + println!("AXChildren: {}", count_attr(br, "AXChildren")); + println!("AXVisibleColumns: {}", count_attr(br, "AXVisibleColumns")); + + // Probe each column + let columns = copy_el_array(br, "AXColumns"); + println!("\nProbing AXColumns:"); + for (ci, &col) in columns.iter().enumerate() { + println!("\n Column[{}]:", ci); + dump_all_attrs(col, 4); + let col_rows = copy_el_array(col, "AXRows"); + let col_children = copy_el_array(col, "AXChildren"); + println!(" AXRows={} AXChildren={}", col_rows.len(), col_children.len()); + + // Probe first few rows + for (ri, &row) in col_rows.iter().enumerate().take(6) { + dump_element(row, ri, 6); + } + for &row in &col_rows { unsafe { CFRelease(row as CFTypeRef) }; } + for &child in &col_children { unsafe { CFRelease(child as CFTypeRef) }; } + unsafe { CFRelease(col as CFTypeRef) }; + } + + // Also try AXChildren path + println!("\nAXChildren of browser:"); + let br_children = copy_el_array(br, "AXChildren"); + for (ci, &child) in br_children.iter().enumerate() { + let r = fetch_str(child, "AXRole"); + let d = fetch_str(child, "AXDescription"); + println!(" Child[{}] role={} desc={}", ci, r, d); + // One more level + let gchildren = copy_el_array(child, "AXChildren"); + for (gi, &gc) in gchildren.iter().enumerate().take(4) { + let r2 = fetch_str(gc, "AXRole"); + let d2 = fetch_str(gc, "AXDescription"); + println!(" GC[{}] role={} desc={}", gi, r2, d2); + // Column rows inside scroll area + let sc_children = copy_el_array(gc, "AXChildren"); + for (sci, &sc) in sc_children.iter().enumerate().take(4) { + let r3 = fetch_str(sc, "AXRole"); + let d3 = fetch_str(sc, "AXDescription"); + println!(" SC[{}] role={} desc={}", sci, r3, d3); + let sc2 = copy_el_array(sc, "AXRows"); + for (s2i, &s2) in sc2.iter().enumerate().take(6) { + dump_element(s2, s2i, 10); + } + for &s2 in &sc2 { unsafe { CFRelease(s2 as CFTypeRef) }; } + unsafe { CFRelease(sc as CFTypeRef) }; + } + for &sc in &sc_children { unsafe { CFRelease(sc as CFTypeRef) }; } + unsafe { CFRelease(gc as CFTypeRef) }; + } + for &gc in &gchildren { unsafe { CFRelease(gc as CFTypeRef) }; } + unsafe { CFRelease(child as CFTypeRef) }; + } + + unsafe { CFRelease(br as CFTypeRef) }; + } else { + println!("Could not find AXBrowser"); + } + + // ── AXCopyMultipleAttributeValues on an AXRow ─────────────────────── + println!("\n────────────────────────────────────────"); + println!("PART 3: AXUIElementCopyMultipleAttributeValues on AXRow"); + println!("────────────────────────────────────────"); + + let sidebar2 = find_by_desc(win, "sidebar", 0); + if let Some(outline) = sidebar2 { + let rows = copy_el_array(outline, "AXRows"); + if let Some(&row) = rows.first() { + println!("Testing on first sidebar AXRow:"); + test_multi_attr_extended(row); + } + for &row in &rows { unsafe { CFRelease(row as CFTypeRef) }; } + unsafe { CFRelease(outline as CFTypeRef) }; + } + + // Cleanup + for &win2 in &windows { unsafe { CFRelease(win2 as CFTypeRef) }; } + unsafe { CFRelease(app_el as CFTypeRef) }; +} + +// ── Deep element dump ──────────────────────────────────────────────────────── + +#[cfg(target_os = "macos")] +fn dump_element(el: accessibility_sys::AXUIElementRef, idx: usize, indent: usize) { + use core_foundation::base::{CFRelease, CFTypeRef}; + let pad = " ".repeat(indent); + let role = fetch_str(el, "AXRole"); + let sub = fetch_str(el, "AXSubrole"); + let title = fetch_str(el, "AXTitle"); + let desc = fetch_str(el, "AXDescription"); + let val = fetch_str(el, "AXValue"); + let help = fetch_str(el, "AXHelp"); + println!("{}[{}] role={} subrole={} title={} desc={} value={} help={}", + pad, idx, role, sub, title, desc, val, help); + dump_all_attrs(el, indent + 2); + + let children = copy_el_array(el, "AXChildren"); + for (ci, &child) in children.iter().enumerate() { + let cr = fetch_str(child, "AXRole"); + let ct = fetch_str(child, "AXTitle"); + let cd = fetch_str(child, "AXDescription"); + let cv = fetch_str(child, "AXValue"); + println!("{} child[{}] role={} title={} desc={} value={}", pad, ci, cr, ct, cd, cv); + dump_all_attrs(child, indent + 4); + + let gchildren = copy_el_array(child, "AXChildren"); + for (gi, &gc) in gchildren.iter().enumerate() { + let gr = fetch_str(gc, "AXRole"); + let gt = fetch_str(gc, "AXTitle"); + let gd = fetch_str(gc, "AXDescription"); + let gv = fetch_str(gc, "AXValue"); + println!("{} gc[{}] role={} title={} desc={} value={}", pad, gi, gr, gt, gd, gv); + dump_all_attrs(gc, indent + 6); + unsafe { CFRelease(gc as CFTypeRef) }; + } + unsafe { CFRelease(child as CFTypeRef) }; + } +} + +#[cfg(target_os = "macos")] +fn test_multi_attr_extended(el: accessibility_sys::AXUIElementRef) { + use accessibility_sys::{AXUIElementCopyMultipleAttributeValues, AXUIElementCopyAttributeNames}; + use core_foundation::{ + array::CFArray, + base::{CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + }; + + // First, get ALL attribute names for this element + let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) }; + if err != 0 || names_ref.is_null() { return; } + let name_arr = unsafe { CFArray::::wrap_under_create_rule(names_ref as _) }; + let all_names: Vec = name_arr.into_iter() + .filter_map(|item| item.downcast::().map(|s| s.to_string())) + .collect(); + + println!("Available attributes on AXRow: {:?}", all_names); + + // Batch fetch all of them + let cf_names: Vec = all_names.iter().map(|a| CFString::new(a)).collect(); + let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect(); + let names_arr2 = CFArray::from_copyable(&cf_refs); + + let mut result: CFTypeRef = std::ptr::null_mut(); + let err2 = unsafe { + AXUIElementCopyMultipleAttributeValues( + el, names_arr2.as_concrete_TypeRef(), 0, + &mut result as *mut _ as *mut _, + ) + }; + println!("AXUIElementCopyMultipleAttributeValues err={}", err2); + if err2 == 0 && !result.is_null() { + let res_arr = unsafe { CFArray::::wrap_under_create_rule(result as _) }; + for (i, item) in res_arr.into_iter().enumerate() { + let name = all_names.get(i).map(|s| s.as_str()).unwrap_or("?"); + let repr = if let Some(s) = item.downcast::() { + format!("String(\"{}\")", s.to_string()) + } else if let Some(b) = item.downcast::() { + format!("Bool({})", bool::from(b)) + } else if let Some(n) = item.downcast::() { + format!("Number({})", n.to_i64().unwrap_or(-1)) + } else { + format!("Other(type_id={})", item.type_of()) + }; + println!(" [{}] {} = {}", i, name, repr); + } + } +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +#[cfg(target_os = "macos")] +fn find_pid(name: &str) -> Option { + let out = std::process::Command::new("pgrep").arg("-x").arg(name).output().ok()?; + String::from_utf8_lossy(&out.stdout).lines().next()?.trim().parse().ok() +} + +#[cfg(target_os = "macos")] +fn count_attr(el: accessibility_sys::AXUIElementRef, attr: &str) -> usize { + copy_el_array(el, attr).len() +} + +#[cfg(target_os = "macos")] +fn fetch_str(el: accessibility_sys::AXUIElementRef, attr: &str) -> String { + use accessibility_sys::AXUIElementCopyAttributeValue; + use core_foundation::{ + base::{CFType, CFTypeRef, TCFType}, + boolean::CFBoolean, + number::CFNumber, + string::CFString, + }; + + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != 0 { return format!("", err); } + if value.is_null() { return "".to_string(); } + let cf = unsafe { CFType::wrap_under_create_rule(value) }; + if let Some(s) = cf.downcast::() { return format!("\"{}\"", s.to_string()); } + if let Some(b) = cf.downcast::() { return format!("bool:{}", bool::from(b)); } + if let Some(n) = cf.downcast::() { return format!("num:{}", n.to_i64().unwrap_or(-1)); } + format!("cftype:{}", cf.type_of()) +} + +#[cfg(target_os = "macos")] +fn dump_all_attrs(el: accessibility_sys::AXUIElementRef, indent: usize) { + use accessibility_sys::AXUIElementCopyAttributeNames; + use core_foundation::{array::CFArray, base::{CFType, TCFType}, string::CFString}; + + let pad = " ".repeat(indent); + let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut(); + let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) }; + if err != 0 || names_ref.is_null() { return; } + let arr = unsafe { CFArray::::wrap_under_create_rule(names_ref as _) }; + let names: Vec = arr.into_iter() + .filter_map(|item| item.downcast::().map(|s| s.to_string())) + .collect(); + for name in &names { + let val = fetch_str(el, name); + println!("{}{}: {}", pad, name, val); + } +} + +#[cfg(target_os = "macos")] +fn copy_el_array( + el: accessibility_sys::AXUIElementRef, + attr: &str, +) -> Vec { + use accessibility_sys::AXUIElementCopyAttributeValue; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != 0 || value.is_null() { return vec![]; } + let arr = unsafe { CFArray::::wrap_under_create_rule(value as _) }; + arr.into_iter() + .filter_map(|item| { + let ptr = item.as_concrete_TypeRef() as accessibility_sys::AXUIElementRef; + if ptr.is_null() { None } + else { + unsafe { CFRetain(ptr as CFTypeRef) }; + Some(ptr) + } + }) + .collect() +} + +/// Find first element with given AXDescription recursively +#[cfg(target_os = "macos")] +fn find_by_desc( + el: accessibility_sys::AXUIElementRef, + desc: &str, + depth: u32, +) -> Option { + use core_foundation::base::{CFRetain, CFTypeRef}; + if depth > 8 { return None; } + let d = fetch_str(el, "AXDescription"); + if d == format!("\"{}\"", desc) { + unsafe { CFRetain(el as CFTypeRef) }; + return Some(el); + } + for child in copy_el_array(el, "AXChildren") { + if let Some(found) = find_by_desc(child, desc, depth + 1) { + unsafe { core_foundation::base::CFRelease(child as CFTypeRef) }; + return Some(found); + } + unsafe { core_foundation::base::CFRelease(child as CFTypeRef) }; + } + None +} + +/// Find first element with given AXRole recursively +#[cfg(target_os = "macos")] +fn find_by_role( + el: accessibility_sys::AXUIElementRef, + role: &str, + depth: u32, +) -> Option { + use core_foundation::base::{CFRetain, CFRelease, CFTypeRef}; + if depth > 8 { return None; } + let r = fetch_str(el, "AXRole"); + if r == format!("\"{}\"", role) { + unsafe { CFRetain(el as CFTypeRef) }; + return Some(el); + } + for child in copy_el_array(el, "AXChildren") { + if let Some(found) = find_by_role(child, role, depth + 1) { + unsafe { CFRelease(child as CFTypeRef) }; + return Some(found); + } + unsafe { CFRelease(child as CFTypeRef) }; + } + None +} diff --git a/crates/macos/src/actions.rs b/crates/macos/src/actions.rs index b88d947..4f15b0e 100644 --- a/crates/macos/src/actions.rs +++ b/crates/macos/src/actions.rs @@ -22,14 +22,13 @@ mod imp { let label = action_label(action); match action { Action::Click => { - // Try AXPress first (works for native controls). - // Always follow up with a CGEvent click to handle Electron/web elements. - let _ = ax_press(el); - cg_mouse_click(el, 1, CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left)?; + let err = ax_press(el); + if err != kAXErrorSuccess { + cg_mouse_click(el, 1, CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left)?; + } } Action::DoubleClick => { - let _ = ax_press(el); cg_mouse_click(el, 2, CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left)?; } @@ -80,6 +79,14 @@ mod imp { } Action::TypeText(text) => { + let cf_attr = CFString::new(kAXFocusedAttribute); + unsafe { + AXUIElementSetAttributeValue( + el.0, + cf_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; crate::input::synthesize_text(text)?; } @@ -163,7 +170,7 @@ mod imp { let center = element_center(el).ok_or_else(|| { AdapterError::new( agent_desktop_core::error::ErrorCode::ActionFailed, - "Cannot click: element has no position/size. Run snapshot --include-bounds first.", + "Cannot click: element has no accessible position or size", ) })?; diff --git a/crates/macos/src/adapter.rs b/crates/macos/src/adapter.rs index e56da83..5d36853 100644 --- a/crates/macos/src/adapter.rs +++ b/crates/macos/src/adapter.rs @@ -93,6 +93,19 @@ impl PlatformAdapter for MacOSAdapter { let windows = self.list_windows(&filter)?; Ok(windows.into_iter().next()) } + + fn get_live_value(&self, handle: &NativeHandle) -> Result, AdapterError> { + #[cfg(target_os = "macos")] + { + use accessibility_sys::kAXValueAttribute; + use crate::tree::AXElement; + use std::mem::ManuallyDrop; + let el = ManuallyDrop::new(AXElement(handle.as_raw() as accessibility_sys::AXUIElementRef)); + Ok(crate::tree::copy_string_attr(&el, kAXValueAttribute)) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("get_live_value")) + } } #[cfg(target_os = "macos")] @@ -101,7 +114,7 @@ fn execute_action_impl(handle: &NativeHandle, action: Action) -> Result, ) -> Result { - use accessibility_sys::{ - kAXChildrenAttribute, kAXErrorSuccess, kAXRoleAttribute, kAXTitleAttribute, - AXUIElementCopyAttributeValue, AXUIElementRef, - }; - use core_foundation::{ - array::CFArray, - base::{CFRetain, CFType, CFTypeRef, TCFType}, - }; + use accessibility_sys::kAXRoleAttribute; + use core_foundation::base::{CFRetain, CFTypeRef}; if !visited.insert(el.0 as usize) { return Err(AdapterError::element_not_found("element")); } - let role = crate::tree::copy_string_attr(el, kAXRoleAttribute); - let normalized = role.as_deref().map(crate::roles::ax_role_to_str).unwrap_or("unknown"); + let ax_role = crate::tree::copy_string_attr(el, kAXRoleAttribute); + let normalized = ax_role.as_deref().map(crate::roles::ax_role_to_str).unwrap_or("unknown"); if normalized == entry.role { - let name = crate::tree::copy_string_attr(el, kAXTitleAttribute); - let name_match = match (&entry.name, &name) { + let elem_name = crate::tree::resolve_element_name(el); + let name_match = match (&entry.name, &elem_name) { (Some(en), Some(nn)) => en == nn, (None, None) => true, _ => false, @@ -157,29 +164,14 @@ fn find_element_recursive( return Err(AdapterError::element_not_found("element")); } - let cf_attr = core_foundation::string::CFString::new(kAXChildrenAttribute); - let mut children_ref: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - &mut children_ref, - ) - }; + let child_attr = if ax_role.as_deref() == Some("AXBrowser") { "AXColumns" } else { "AXChildren" }; + let children = crate::tree::copy_ax_array(el, child_attr) + .filter(|v| !v.is_empty()) + .or_else(|| crate::tree::copy_ax_array(el, "AXContents").filter(|v| !v.is_empty())) + .unwrap_or_default(); - if err != kAXErrorSuccess || children_ref.is_null() { - return Err(AdapterError::element_not_found("element")); - } - - let arr = unsafe { CFArray::::wrap_under_create_rule(children_ref as _) }; - for item in arr.into_iter() { - let ptr = item.as_concrete_TypeRef() as AXUIElementRef; - if ptr.is_null() { - continue; - } - unsafe { CFRetain(ptr as CFTypeRef) }; - let child = crate::tree::AXElement(ptr); - if let Ok(handle) = find_element_recursive(&child, entry, depth + 1, max_depth, visited) { + for child in &children { + if let Ok(handle) = find_element_recursive(child, entry, depth + 1, max_depth, visited) { return Ok(handle); } } @@ -195,54 +187,58 @@ fn resolve_element_impl(_entry: &RefEntry) -> Result pub fn list_windows_impl(filter: &WindowFilter) -> Result, AdapterError> { #[cfg(target_os = "macos")] { + use core_foundation::base::{CFType, TCFType}; + use core_foundation::number::CFNumber; + use core_foundation::string::CFString; + use core_foundation_sys::dictionary::CFDictionaryGetValue; + use core_graphics::display::CGDisplay; + use core_graphics::window::{ + kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowName, + kCGWindowOwnerName, kCGWindowOwnerPID, + }; use rustc_hash::FxHasher; + use std::ffi::c_void; use std::hash::{Hash, Hasher}; - use std::process::Command; - let app_filter = filter.app.as_deref().unwrap_or("").to_string(); - let script = r#" -tell application "System Events" - set winList to {} - repeat with proc in (processes where background only is false) - set pName to name of proc as string - set pPid to (unix id of proc) as string - set winTitles to name of every window of proc - repeat with wTitle in winTitles - set winList to winList & {pName & "|" & pPid & "|" & (wTitle as string)} - end repeat - end repeat - set AppleScript's text item delimiters to linefeed - return winList as text -end tell -"#; - let output = Command::new("osascript") - .arg("-e") - .arg(script) - .output() - .map_err(|e| AdapterError::internal(format!("osascript failed: {e}")))?; + unsafe fn dict_string(dict: *const c_void, key: *const c_void) -> Option { + let val = CFDictionaryGetValue(dict as _, key); + if val.is_null() { return None; } + CFType::wrap_under_get_rule(val as _).downcast::().map(|s| s.to_string()) + } - let text = String::from_utf8_lossy(&output.stdout); + unsafe fn dict_i64(dict: *const c_void, key: *const c_void) -> Option { + let val = CFDictionaryGetValue(dict as _, key); + if val.is_null() { return None; } + CFType::wrap_under_get_rule(val as _).downcast::().and_then(|n| n.to_i64()) + } + + let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) { + Some(a) => a, + None => return Ok(vec![]), + }; + + let app_filter = filter.app.as_deref().unwrap_or("").to_lowercase(); let mut windows = Vec::new(); - for (idx, line) in text.lines().enumerate() { - let line = line.trim(); - if line.is_empty() { - continue; - } - let mut parts = line.splitn(3, '|'); - let app_name = match parts.next() { - Some(s) => s.trim().to_string(), - None => continue, - }; - let pid: i32 = match parts.next().and_then(|s| s.trim().parse().ok()) { - Some(p) => p, - None => continue, - }; - let title = parts.next().unwrap_or("").trim().to_string(); - if !app_filter.is_empty() && !app_name.eq_ignore_ascii_case(&app_filter) { + for raw in arr.get_all_values() { + if raw.is_null() { continue; } + let layer = unsafe { dict_i64(raw, kCGWindowLayer as _) }.unwrap_or(99); + if layer != 0 { continue; } + + let app_name = match unsafe { dict_string(raw, kCGWindowOwnerName as _) } { + Some(n) if !n.is_empty() => n, + _ => continue, + }; + if !app_filter.is_empty() && !app_name.to_lowercase().contains(&app_filter) { continue; } + let title = match unsafe { dict_string(raw, kCGWindowName as _) } { + Some(t) if !t.is_empty() => t, + _ => continue, + }; + + let pid = unsafe { dict_i64(raw, kCGWindowOwnerPID as _) }.unwrap_or(0) as i32; let mut h = FxHasher::default(); pid.hash(&mut h); title.hash(&mut h); @@ -254,7 +250,7 @@ end tell app: app_name, pid, bounds: None, - is_focused: idx == 0, + is_focused: windows.is_empty(), }); } Ok(windows) @@ -269,32 +265,50 @@ end tell fn list_apps_impl() -> Result, AdapterError> { #[cfg(target_os = "macos")] { - use std::process::Command; - let output = Command::new("osascript") - .arg("-e") - .arg( - r#"tell application "System Events" - set result to "" - repeat with proc in (processes where background only is false) - set result to result & (name of proc as string) & "|" & ((unix id of proc) as string) & linefeed - end repeat - return result -end tell"#, - ) - .output() - .map_err(|e| AdapterError::internal(format!("osascript failed: {e}")))?; + use core_foundation::base::{CFType, TCFType}; + use core_foundation::number::CFNumber; + use core_foundation::string::CFString; + use core_foundation_sys::dictionary::CFDictionaryGetValue; + use core_graphics::display::CGDisplay; + use core_graphics::window::{ + kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowOwnerName, kCGWindowOwnerPID, + }; - let text = String::from_utf8_lossy(&output.stdout); - let apps = text - .lines() - .filter(|l| !l.is_empty()) - .filter_map(|line| { - let mut parts = line.split('|'); - let name = parts.next()?.trim().to_string(); - let pid: i32 = parts.next()?.trim().parse().ok()?; - Some(AppInfo { name, pid, bundle_id: None }) - }) - .collect(); + let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) { + Some(a) => a, + None => return Ok(vec![]), + }; + + let mut seen_pids = std::collections::HashSet::new(); + let mut apps = Vec::new(); + + for raw in arr.get_all_values() { + if raw.is_null() { continue; } + + let layer = unsafe { + let v = CFDictionaryGetValue(raw as _, kCGWindowLayer as _); + if v.is_null() { continue; } + CFType::wrap_under_get_rule(v as _).downcast::().and_then(|n| n.to_i64()).unwrap_or(99) + }; + if layer != 0 { continue; } + + let pid = unsafe { + let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerPID as _); + if v.is_null() { continue; } + CFType::wrap_under_get_rule(v as _).downcast::().and_then(|n| n.to_i64()).unwrap_or(0) as i32 + }; + if !seen_pids.insert(pid) { continue; } + + let name = unsafe { + let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerName as _); + if v.is_null() { continue; } + CFType::wrap_under_get_rule(v as _).downcast::().map(|s| s.to_string()) + }; + + if let Some(n) = name { + apps.push(AppInfo { name: n, pid, bundle_id: None }); + } + } Ok(apps) } #[cfg(not(target_os = "macos"))] diff --git a/crates/macos/src/roles.rs b/crates/macos/src/roles.rs index 331a114..44540bc 100644 --- a/crates/macos/src/roles.rs +++ b/crates/macos/src/roles.rs @@ -1,36 +1,54 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str { match ax_role { - "AXApplication" => "application", - "AXButton" => "button", + "AXApplication" => "application", + "AXButton" => "button", + "AXMenuButton" => "menubutton", "AXTextField" | "AXTextArea" | "AXSearchField" => "textfield", - "AXCheckBox" => "checkbox", - "AXLink" => "link", - "AXMenuItem" | "AXMenuBarItem" => "menuitem", - "AXRadioButton" => "radiobutton", - "AXTab" | "AXTabGroup" => "tab", - "AXSlider" | "AXValueIndicator" => "slider", - "AXComboBox" | "AXPopUpButton" => "combobox", - "AXOutlineRow" | "AXRow" => "treeitem", - "AXCell" => "cell", - "AXWindow" => "window", - "AXSheet" => "sheet", - "AXDialog" => "dialog", - "AXGroup" | "AXGenericElement" => "group", - "AXToolbar" => "toolbar", - "AXStaticText" => "statictext", - "AXImage" => "image", - "AXTable" => "table", - "AXList" => "list", - "AXOutline" => "outline", - "AXScrollArea" | "AXScrollBar" => "scrollarea", - "AXSplitter" | "AXSplitGroup" => "splitter", - "AXMenu" | "AXMenuBar" => "menu", - "AXIncrementor" | "AXStepper" => "incrementor", - "AXDisclosureTriangle" => "disclosure", - "AXProgressIndicator" | "AXBusyIndicator" => "progressbar", - "AXColorWell" => "colorwell", - "AXWebArea" => "webarea", - _ => "unknown", + "AXCheckBox" => "checkbox", + "AXSwitch" | "AXToggle" => "switch", + "AXLink" => "link", + "AXMenuItem" | "AXMenuBarItem" => "menuitem", + "AXRadioButton" => "radiobutton", + "AXTab" | "AXTabGroup" => "tab", + "AXSlider" | "AXValueIndicator" => "slider", + "AXComboBox" | "AXPopUpButton" => "combobox", + "AXOutlineRow" | "AXRow" => "treeitem", + "AXCell" => "cell", + "AXColumn" => "column", + "AXWindow" => "window", + "AXSheet" => "sheet", + "AXDialog" => "dialog", + "AXGroup" | "AXGenericElement" => "group", + "AXToolbar" => "toolbar", + "AXStaticText" => "statictext", + "AXImage" => "image", + "AXTable" => "table", + "AXList" => "list", + "AXOutline" => "outline", + "AXScrollArea" | "AXScrollBar" => "scrollarea", + "AXSplitter" | "AXSplitGroup" => "splitter", + "AXMenu" | "AXMenuBar" => "menu", + "AXIncrementor" | "AXStepper" => "incrementor", + "AXDisclosureTriangle" => "disclosure", + "AXProgressIndicator" | "AXBusyIndicator" => "progressbar", + "AXColorWell" => "colorwell", + "AXWebArea" => "webarea", + "AXBrowser" => "browser", + "AXGrid" => "grid", + "AXHandle" => "handle", + "AXPopover" => "popover", + "AXDockItem" => "dockitem", + "AXRuler" => "ruler", + "AXRulerMarker" => "rulermarker", + "AXTimeField" => "timefield", + "AXDateField" => "datefield", + "AXHelpTag" => "helptag", + "AXMatte" => "matte", + "AXDrawer" => "drawer", + "AXLayoutArea" | "AXLayoutItem" => "layoutitem", + "AXLevelIndicator" => "levelindicator", + "AXRelevanceIndicator" => "relevanceindicator", + _ => "unknown", } } @@ -38,8 +56,10 @@ pub fn is_interactive_role(role: &str) -> bool { matches!( role, "button" + | "menubutton" | "textfield" | "checkbox" + | "switch" | "link" | "menuitem" | "tab" @@ -49,5 +69,7 @@ pub fn is_interactive_role(role: &str) -> bool { | "cell" | "radiobutton" | "incrementor" + | "colorwell" + | "dockitem" ) } diff --git a/crates/macos/src/screenshot.rs b/crates/macos/src/screenshot.rs index 39ddc73..d6853f7 100644 --- a/crates/macos/src/screenshot.rs +++ b/crates/macos/src/screenshot.rs @@ -53,7 +53,17 @@ mod imp { let data = std::fs::read(path) .map_err(|e| AdapterError::internal(format!("read screenshot: {e}")))?; let _ = std::fs::remove_file(path); - Ok(ImageBuffer { data, format: ImageFormat::Png, width: 0, height: 0 }) + let (width, height) = png_dimensions(&data); + Ok(ImageBuffer { data, format: ImageFormat::Png, width, height }) + } + + fn png_dimensions(data: &[u8]) -> (u32, u32) { + if data.len() < 24 { + return (0, 0); + } + let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); + let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); + (w, h) } fn find_cg_window_id_for_pid(pid: i32) -> Option { @@ -116,12 +126,12 @@ mod imp { bounds_val.as_concrete_TypeRef() as _, ) }; - let w = bounds_dict.find(&CFString::new("Width")).and_then(|v| { + let w = bounds_dict.find(CFString::new("Width")).and_then(|v| { let n = unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) }; n.to_f64() }); - let h = bounds_dict.find(&CFString::new("Height")).and_then(|v| { + let h = bounds_dict.find(CFString::new("Height")).and_then(|v| { let n = unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) }; n.to_f64() diff --git a/crates/macos/src/tree.rs b/crates/macos/src/tree.rs index 02fe47c..85a988d 100644 --- a/crates/macos/src/tree.rs +++ b/crates/macos/src/tree.rs @@ -11,12 +11,13 @@ mod imp { kAXEnabledAttribute, kAXErrorSuccess, kAXFocusedAttribute, kAXRoleAttribute, kAXTitleAttribute, kAXValueAttribute, kAXWindowsAttribute, AXUIElementCopyAttributeValue, AXUIElementCopyMultipleAttributeValues, - AXUIElementCreateApplication, AXUIElementRef, + AXUIElementCreateApplication, AXUIElementRef, AXUIElementSetMessagingTimeout, }; use core_foundation::{ array::CFArray, base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType}, boolean::CFBoolean, + number::CFNumber, string::CFString, }; @@ -40,7 +41,12 @@ mod imp { } pub fn element_for_pid(pid: i32) -> AXElement { - AXElement(unsafe { AXUIElementCreateApplication(pid) }) + let el = AXElement(unsafe { AXUIElementCreateApplication(pid) }); + if !el.0.is_null() { + // 2-second timeout prevents hung/slow processes from blocking the tree walk + unsafe { AXUIElementSetMessagingTimeout(el.0, 2.0) }; + } + el } /// Find the AXWindow element whose title matches `win_title`. @@ -48,37 +54,32 @@ mod imp { pub fn window_element_for(pid: i32, win_title: &str) -> AXElement { let app = element_for_pid(pid); - // Try kAXWindowsAttribute if let Some(windows) = copy_ax_array(&app, kAXWindowsAttribute) { - // Exact title match for win in &windows { let title = copy_string_attr(win, kAXTitleAttribute); if title.as_deref() == Some(win_title) { - let matched = win.clone(); - return matched; + return win.clone(); } } - // Partial match for win in &windows { let title = copy_string_attr(win, kAXTitleAttribute); if title.as_deref().is_some_and(|t| t.contains(win_title) || win_title.contains(t)) { - let matched = win.clone(); - return matched; + return win.clone(); } } - // First available window if let Some(first) = windows.into_iter().next() { return first; } } - // Fallback: app root app } - /// Batch-fetch the six most-used attributes in a single AX API call. - /// Returns (role, title, description, value, enabled, focused). - fn fetch_node_attrs(el: &AXElement) -> (Option, Option, Option, Option, bool, bool) { + /// Batch-fetch six most-used attributes. Returns (role, title, desc, value, enabled, focused). + /// Value handles CFString, CFBoolean, and CFNumber types. + fn fetch_node_attrs( + el: &AXElement, + ) -> (Option, Option, Option, Option, bool, bool) { let attr_names = [ kAXRoleAttribute, kAXTitleAttribute, @@ -102,19 +103,34 @@ mod imp { }; if err != kAXErrorSuccess || result_ref.is_null() { - // Fallback to individual calls - let role = copy_string_attr(el, kAXRoleAttribute); - let title = copy_string_attr(el, kAXTitleAttribute); - let desc = copy_string_attr(el, kAXDescriptionAttribute); - let val = copy_string_attr(el, kAXValueAttribute); + let role = copy_string_attr(el, kAXRoleAttribute); + let title = copy_string_attr(el, kAXTitleAttribute); + let desc = copy_string_attr(el, kAXDescriptionAttribute); + let val = copy_value_typed(el); let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true); let focused = copy_bool_attr(el, kAXFocusedAttribute).unwrap_or(false); return (role, title, desc, val, enabled, focused); } let arr = unsafe { CFArray::::wrap_under_create_rule(result_ref as _) }; - let items: Vec> = arr.into_iter().map(|item| { - item.downcast::().map(|s| s.to_string()) + let items: Vec> = arr.into_iter().enumerate().map(|(idx, item)| { + if let Some(s) = item.downcast::() { + return Some(s.to_string()); + } + match idx { + // value: may be CFBoolean (checkbox) or CFNumber (slider, stepper) + 3 => { + if let Some(b) = item.downcast::() { return Some(bool::from(b).to_string()); } + if let Some(n) = item.downcast::() { + if let Some(i) = n.to_i64() { return Some(i.to_string()); } + if let Some(f) = n.to_f64() { return Some(format!("{:.2}", f)); } + } + None + } + // enabled / focused are always CFBoolean + 4 | 5 => item.downcast::().map(|b| bool::from(b).to_string()), + _ => None, + } }).collect(); let get = |i: usize| items.get(i).and_then(|v| v.clone()); @@ -122,14 +138,31 @@ mod imp { let title = get(1); let desc = get(2); let val = get(3); - // enabled/focused are CFBoolean not CFString, so they'll be None from downcast - // re-read them individually (cheap since it's only 2 attrs) - let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true); - let focused = copy_bool_attr(el, kAXFocusedAttribute).unwrap_or(false); + let enabled = get(4).map(|s| s == "true").unwrap_or(true); + let focused = get(5).map(|s| s == "true").unwrap_or(false); (role, title, desc, val, enabled, focused) } + /// Compute the effective display name for any element, mirroring `build_subtree` name resolution. + pub fn resolve_element_name(el: &AXElement) -> Option { + let ax_role = copy_string_attr(el, kAXRoleAttribute); + let title = copy_string_attr(el, kAXTitleAttribute); + let desc = copy_string_attr(el, kAXDescriptionAttribute); + + let name = title.or(desc); + let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") { + copy_string_attr(el, kAXValueAttribute).or(name) + } else { + name + }; + + name.or_else(|| { + let children = copy_ax_array(el, kAXChildrenAttribute).unwrap_or_default(); + label_from_children(&children) + }) + } + pub fn build_subtree( el: &AXElement, depth: u8, @@ -155,22 +188,29 @@ mod imp { let name = title.clone().or_else(|| ax_desc.clone()); let description = if title.is_some() { ax_desc } else { None }; + // AXStaticText stores its visible text in kAXValueAttribute, not title/description + let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") { + value.clone().or(name) + } else { + name + }; + let mut states = Vec::new(); - if focused { - states.push("focused".into()); - } - if !enabled { - states.push("disabled".into()); - } + if focused { states.push("focused".into()); } + if !enabled { states.push("disabled".into()); } let bounds = if include_bounds { read_bounds(el) } else { None }; - let children = copy_children(el) - .unwrap_or_default() + // Children fetched before name resolution so we can extract labels from them + let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default(); + + // Last-resort name: walk immediate children for AXStaticText or AXCell→AXStaticText + // This resolves AXRow (sidebar items, list rows) whose label lives in a child text node + let name = name.or_else(|| label_from_children(&children_raw)); + + let children = children_raw .into_iter() - .filter_map(|child| { - build_subtree(&child, depth + 1, max_depth, include_bounds, visited) - }) + .filter_map(|child| build_subtree(&child, depth + 1, max_depth, include_bounds, visited)) .collect(); Some(AccessibilityNode { @@ -185,43 +225,48 @@ mod imp { }) } + /// Scan immediate children (and one level deeper through AXCell) to find a text label. + /// Resolves the common macOS pattern: AXRow → AXCell → AXStaticText.kAXValue = "label". + fn label_from_children(children: &[AXElement]) -> Option { + fn text_of(el: &AXElement) -> Option { + copy_string_attr(el, kAXValueAttribute) + .or_else(|| copy_string_attr(el, kAXTitleAttribute)) + .filter(|s| !s.is_empty()) + } + + for child in children.iter().take(5) { + match copy_string_attr(child, kAXRoleAttribute).as_deref() { + Some("AXStaticText") => { + if let Some(s) = text_of(child) { return Some(s); } + } + Some("AXCell") | Some("AXGroup") => { + for gc in copy_ax_array(child, kAXChildrenAttribute).unwrap_or_default() { + if copy_string_attr(&gc, kAXRoleAttribute).as_deref() == Some("AXStaticText") { + if let Some(s) = text_of(&gc) { return Some(s); } + } + } + } + _ => {} + } + } + None + } + + /// Read children using the appropriate attribute for this element's role. + /// AXBrowser exposes its content via AXColumns, not AXChildren. + fn copy_children(el: &AXElement, ax_role: Option<&str>) -> Option> { + if ax_role == Some("AXBrowser") { + return copy_ax_array(el, "AXColumns"); + } + for attr in &[kAXChildrenAttribute, kAXContentsAttribute, "AXChildrenInNavigationOrder"] { + if let Some(v) = copy_ax_array(el, attr) { + if !v.is_empty() { return Some(v); } + } + } + None + } + pub fn copy_string_attr(el: &AXElement, attr: &str) -> Option { - let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - &mut value, - ) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; - cf_type.downcast::().map(|s| s.to_string()) - } - - fn copy_bool_attr(el: &AXElement, attr: &str) -> Option { - let cf_attr = CFString::new(attr); - let mut value: CFTypeRef = std::ptr::null_mut(); - let err = unsafe { - AXUIElementCopyAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - &mut value, - ) - }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; - cf_type.downcast::().map(|b| b.into()) - } - - /// Read an array-typed AX attribute, retaining each AXUIElement so they - /// stay alive past CFArray deallocation. - pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option> { let cf_attr = CFString::new(attr); let mut value: CFTypeRef = std::ptr::null_mut(); let err = unsafe { @@ -230,37 +275,60 @@ mod imp { if err != kAXErrorSuccess || value.is_null() { return None; } + let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; + cf_type.downcast::().map(|s| s.to_string()) + } + + /// Read kAXValueAttribute handling CFString, CFBoolean, and CFNumber. + fn copy_value_typed(el: &AXElement) -> Option { + let cf_attr = CFString::new(kAXValueAttribute); + let mut val_ref: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut val_ref) + }; + if err != kAXErrorSuccess || val_ref.is_null() { return None; } + let cf = unsafe { CFType::wrap_under_create_rule(val_ref) }; + if let Some(s) = cf.downcast::() { return Some(s.to_string()); } + if let Some(b) = cf.downcast::() { return Some(bool::from(b).to_string()); } + if let Some(n) = cf.downcast::() { + if let Some(i) = n.to_i64() { return Some(i.to_string()); } + if let Some(f) = n.to_f64() { return Some(format!("{:.2}", f)); } + } + None + } + + fn copy_bool_attr(el: &AXElement, attr: &str) -> Option { + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != kAXErrorSuccess || value.is_null() { return None; } + let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; + cf_type.downcast::().map(|b| b.into()) + } + + /// Read an array-typed AX attribute, retaining each AXUIElement. + pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option> { + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != kAXErrorSuccess || value.is_null() { return None; } let arr = unsafe { CFArray::::wrap_under_create_rule(value as _) }; let children: Vec = arr .into_iter() .filter_map(|item| { let ptr = item.as_concrete_TypeRef() as AXUIElementRef; - if ptr.is_null() { - None - } else { - unsafe { CFRetain(ptr as CFTypeRef) }; - Some(AXElement(ptr)) - } + if ptr.is_null() { return None; } + unsafe { CFRetain(ptr as CFTypeRef) }; + Some(AXElement(ptr)) }) .collect(); Some(children) } - fn copy_children(el: &AXElement) -> Option> { - for attr in &[ - kAXChildrenAttribute, - kAXContentsAttribute, - "AXChildrenInNavigationOrder", - ] { - if let Some(v) = copy_ax_array(el, attr) { - if !v.is_empty() { - return Some(v); - } - } - } - None - } - pub fn read_bounds(el: &AXElement) -> Option { use accessibility_sys::{ kAXPositionAttribute, kAXSizeAttribute, AXValueGetValue, @@ -274,34 +342,28 @@ mod imp { let pos_ok = unsafe { AXUIElementCopyAttributeValue(el.0, pos_cf.as_concrete_TypeRef(), &mut pos_ref) }; - if pos_ok != kAXErrorSuccess || pos_ref.is_null() { - return None; - } + if pos_ok != kAXErrorSuccess || pos_ref.is_null() { return None; } + let mut point = CGPoint::new(0.0, 0.0); let got_pos = unsafe { AXValueGetValue(pos_ref as _, kAXValueTypeCGPoint, &mut point as *mut _ as *mut c_void) }; unsafe { CFRelease(pos_ref) }; - if !got_pos { - return None; - } + if !got_pos { return None; } let size_cf = CFString::new(kAXSizeAttribute); let mut size_ref: CFTypeRef = std::ptr::null_mut(); let size_ok = unsafe { AXUIElementCopyAttributeValue(el.0, size_cf.as_concrete_TypeRef(), &mut size_ref) }; - if size_ok != kAXErrorSuccess || size_ref.is_null() { - return None; - } + if size_ok != kAXErrorSuccess || size_ref.is_null() { return None; } + let mut size = CGSize::new(0.0, 0.0); let got_size = unsafe { AXValueGetValue(size_ref as _, kAXValueTypeCGSize, &mut size as *mut _ as *mut c_void) }; unsafe { CFRelease(size_ref) }; - if !got_size { - return None; - } + if !got_size { return None; } Some(Rect { x: point.x, y: point.y, width: size.width, height: size.height }) } @@ -313,45 +375,20 @@ mod imp { pub struct AXElement(pub(crate) *const std::ffi::c_void); - impl Drop for AXElement { - fn drop(&mut self) {} - } + impl Drop for AXElement { fn drop(&mut self) {} } + impl Clone for AXElement { fn clone(&self) -> Self { AXElement(self.0) } } - impl Clone for AXElement { - fn clone(&self) -> Self { - AXElement(self.0) - } - } + pub fn element_for_pid(_pid: i32) -> AXElement { AXElement(std::ptr::null()) } + pub fn window_element_for(_pid: i32, _win_title: &str) -> AXElement { AXElement(std::ptr::null()) } + pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option> { None } + pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option { None } + pub fn read_bounds(_el: &AXElement) -> Option { None } + pub fn resolve_element_name(_el: &AXElement) -> Option { None } - pub fn element_for_pid(_pid: i32) -> AXElement { - AXElement(std::ptr::null()) - } - - pub fn window_element_for(_pid: i32, _win_title: &str) -> AXElement { - AXElement(std::ptr::null()) - } - - pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option> { - None - } - - pub fn build_subtree( - _el: &AXElement, - _depth: u8, - _max_depth: u8, - _include_bounds: bool, - _visited: &mut FxHashSet, - ) -> Option { - None - } - - pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option { - None - } - - pub fn read_bounds(_el: &AXElement) -> Option { - None - } + pub fn build_subtree(_el: &AXElement, _depth: u8, _max_depth: u8, _include_bounds: bool, _visited: &mut FxHashSet) -> Option { None } } -pub use imp::{build_subtree, copy_ax_array, copy_string_attr, element_for_pid, read_bounds, window_element_for, AXElement}; +pub use imp::{ + build_subtree, copy_ax_array, copy_string_attr, element_for_pid, read_bounds, + resolve_element_name, window_element_for, AXElement, +};