mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-08-17 20:45:50 +00:00
refactor: Phase A quality fixes — dead code, bugs, LOC compliance
- Delete clipboard.rs (superseded by clipboard_get/set; zero callers) - Remove batch::execute() stub (dispatch layer owns batch execution) - Fix wait.rs: replace double get+unwrap with if-let pattern - Fix press.rs: replace dead is_empty check + unwrap with ok_or_else - Add doc comment to is_check.rs documenting stale-state semantics - Trim tree.rs to 394 LOC (was 403; compress non-macos stub + remove redundant comment) - Move probe binaries to examples/ with required-features = ["dev-tools"] - Fix clippy::explicit_auto_deref in adapter.rs - Fix clippy::needless_borrows_for_generic_args in screenshot.rs - Apply prior session core/macos fixes (adapter, actions, roles, snapshot, get) All targets pass cargo clippy -D warnings.
This commit is contained in:
Lahfir
parent
1d98ab828c
commit
f1ed36046c
18 changed files with 1456 additions and 319 deletions
|
|
@ -153,4 +153,8 @@ pub trait PlatformAdapter: Send + Sync {
|
|||
fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError> {
|
||||
Err(AdapterError::not_supported("focused_window"))
|
||||
}
|
||||
|
||||
fn get_live_value(&self, _handle: &NativeHandle) -> Result<Option<String>, AdapterError> {
|
||||
Err(AdapterError::not_supported("get_live_value"))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use crate::error::AppError;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct BatchArgs {
|
||||
pub commands_json: String,
|
||||
|
|
@ -18,11 +18,3 @@ pub fn parse_commands(json_str: &str) -> Result<Vec<BatchCommand>, AppError> {
|
|||
serde_json::from_str(json_str)
|
||||
.map_err(|e| AppError::invalid_input(format!("Invalid batch JSON: {e}")))
|
||||
}
|
||||
|
||||
pub fn execute(args: BatchArgs, _adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let commands = parse_commands(&args.commands_json)?;
|
||||
Ok(json!({
|
||||
"note": "Batch execution delegated to dispatch layer",
|
||||
"count": commands.len()
|
||||
}))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
use crate::{adapter::PlatformAdapter, error::AppError};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub fn execute_get(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let text = adapter.get_clipboard()?;
|
||||
Ok(json!({ "text": text }))
|
||||
}
|
||||
|
||||
pub fn execute_set(text: String, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
adapter.set_clipboard(&text)?;
|
||||
Ok(json!({ "ok": true }))
|
||||
}
|
||||
|
|
@ -16,12 +16,15 @@ pub enum GetProperty {
|
|||
}
|
||||
|
||||
pub fn execute(args: GetArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let (entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
|
||||
let value = match args.property {
|
||||
GetProperty::Role => json!(entry.role),
|
||||
GetProperty::Text | GetProperty::Title => json!(entry.name),
|
||||
GetProperty::Value => json!(entry.value),
|
||||
GetProperty::Title => json!(entry.name),
|
||||
GetProperty::Text | GetProperty::Value => {
|
||||
let live = adapter.get_live_value(&handle).ok().flatten();
|
||||
json!(live.or(entry.value))
|
||||
}
|
||||
GetProperty::Bounds => json!(entry.bounds),
|
||||
GetProperty::States => json!(entry.states),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ pub enum IsProperty {
|
|||
Expanded,
|
||||
}
|
||||
|
||||
/// States are read from the last snapshot's RefMap. `resolve_ref` verifies the element
|
||||
/// is still live before returning, but the state values themselves are not re-queried
|
||||
/// from the AX API. Run `snapshot` to refresh state before calling `is`.
|
||||
pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
pub mod batch;
|
||||
pub mod clipboard;
|
||||
pub mod clipboard_get;
|
||||
pub mod clipboard_set;
|
||||
pub mod click;
|
||||
|
|
|
|||
|
|
@ -34,11 +34,12 @@ pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result<Value,
|
|||
|
||||
fn parse_combo(s: &str) -> Result<KeyCombo, AppError> {
|
||||
let parts: Vec<&str> = s.split('+').collect();
|
||||
if parts.is_empty() {
|
||||
return Err(AppError::invalid_input("Empty key combo"));
|
||||
}
|
||||
|
||||
let key = parts.last().unwrap().to_string();
|
||||
let key = parts
|
||||
.last()
|
||||
.copied()
|
||||
.filter(|k| !k.is_empty())
|
||||
.ok_or_else(|| AppError::invalid_input("Empty key combo"))?
|
||||
.to_string();
|
||||
let mut modifiers = Vec::new();
|
||||
|
||||
for &part in &parts[..parts.len() - 1] {
|
||||
|
|
|
|||
|
|
@ -44,11 +44,11 @@ fn wait_for_element(
|
|||
|
||||
loop {
|
||||
if let Ok(refmap) = RefMap::load() {
|
||||
if refmap.get(&ref_id).is_some()
|
||||
&& adapter.resolve_element(refmap.get(&ref_id).unwrap()).is_ok()
|
||||
{
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed }));
|
||||
if let Some(entry) = refmap.get(&ref_id) {
|
||||
if adapter.resolve_element(entry).is_ok() {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use crate::{
|
|||
const INTERACTIVE_ROLES: &[&str] = &[
|
||||
"button", "textfield", "checkbox", "link", "menuitem", "tab", "slider",
|
||||
"combobox", "treeitem", "cell", "radiobutton", "incrementor",
|
||||
"menubutton", "switch", "colorwell", "dockitem",
|
||||
];
|
||||
|
||||
pub struct SnapshotResult {
|
||||
|
|
@ -129,7 +130,14 @@ fn allocate_refs(
|
|||
node.children = node
|
||||
.children
|
||||
.into_iter()
|
||||
.map(|child| allocate_refs(child, refmap, include_bounds, interactive_only, window_pid, source_app))
|
||||
.filter_map(|child| {
|
||||
let child = allocate_refs(child, refmap, include_bounds, interactive_only, window_pid, source_app);
|
||||
if interactive_only && child.ref_id.is_none() && child.children.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(child)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
node
|
||||
|
|
|
|||
|
|
@ -19,4 +19,19 @@ core-foundation = "0.10.1"
|
|||
core-foundation-sys = "0.8.7"
|
||||
core-graphics = { version = "0.25.0", features = ["highsierra"] }
|
||||
|
||||
[features]
|
||||
dev-tools = []
|
||||
|
||||
[[example]]
|
||||
name = "ax_probe"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[[example]]
|
||||
name = "axprobe"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[[example]]
|
||||
name = "axprobe2"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[build-dependencies]
|
||||
|
|
|
|||
261
crates/macos/examples/ax_probe.rs
Normal file
261
crates/macos/examples/ax_probe.rs
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
/// Direct probe of macOS AX APIs — no abstraction.
|
||||
/// Reveals exactly what the raw APIs return and validates click behavior.
|
||||
///
|
||||
/// cargo run -p agent-desktop-macos --example ax_probe -- <AppName>
|
||||
use std::ffi::c_void;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn main() {
|
||||
let app_name = std::env::args().nth(1).unwrap_or_else(|| "Finder".into());
|
||||
println!("=== AX Probe: '{}' ===\n", app_name);
|
||||
|
||||
let pid = find_pid(&app_name).expect("app not running");
|
||||
println!("[pid] {}", pid);
|
||||
|
||||
probe_app(pid);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_pid(name: &str) -> Option<i32> {
|
||||
let out = std::process::Command::new("pgrep").arg("-xi").arg(name).output().ok()?;
|
||||
String::from_utf8_lossy(&out.stdout).trim().lines().next()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn probe_app(pid: i32) {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::base::CFTypeRef;
|
||||
|
||||
let app = unsafe { AXUIElementCreateApplication(pid) };
|
||||
|
||||
// ── 1. What attributes does the app element expose? ──────────────────────
|
||||
println!("\n[1] App element attribute names:");
|
||||
let attr_names = get_attribute_names(app);
|
||||
for n in &attr_names { println!(" {n}"); }
|
||||
|
||||
// ── 2. Get windows the right way ─────────────────────────────────────────
|
||||
println!("\n[2] Windows via kAXWindowsAttribute:");
|
||||
let windows = get_ax_children(app, kAXWindowsAttribute);
|
||||
println!(" count = {}", windows.len());
|
||||
for (i, win) in windows.iter().enumerate() {
|
||||
let role = read_string(*win, kAXRoleAttribute);
|
||||
let title = read_string(*win, kAXTitleAttribute);
|
||||
let pos = read_cgpoint(*win, kAXPositionAttribute);
|
||||
let size = read_cgsize(*win, kAXSizeAttribute);
|
||||
println!(" [{i}] role={:?} title={:?} pos={:?} size={:?}", role, title, pos, size);
|
||||
|
||||
// Children of this window
|
||||
let children = get_ax_children(*win, kAXChildrenAttribute);
|
||||
println!(" children = {}", children.len());
|
||||
for (ci, child) in children.iter().enumerate().take(8) {
|
||||
let cr = read_string(*child, kAXRoleAttribute);
|
||||
let ct = read_string(*child, kAXTitleAttribute);
|
||||
let cd = read_string(*child, kAXDescriptionAttribute);
|
||||
let cv = read_string(*child, kAXValueAttribute);
|
||||
let cpos = read_cgpoint(*child, kAXPositionAttribute);
|
||||
let csz = read_cgsize(*child, kAXSizeAttribute);
|
||||
|
||||
println!(" [{ci}] role={:?} title={:?} desc={:?} val={:?}", cr, ct, cd, cv);
|
||||
println!(" pos={:?} size={:?}", cpos, csz);
|
||||
|
||||
// ── 3. Test kAXPressAction on each child ────────────────────────
|
||||
let ax_err = ax_press(*child);
|
||||
println!(" kAXPressAction → err={} (0=ok, -25200=fail, -25205=not_supported)", ax_err);
|
||||
|
||||
// ── 4. Test CGEvent click at element center ─────────────────────
|
||||
if let (Some(p), Some(s)) = (cpos, csz) {
|
||||
let cx = p.0 + s.0 / 2.0;
|
||||
let cy = p.1 + s.1 / 2.0;
|
||||
let cg_ok = cg_click(cx, cy);
|
||||
println!(" CGEvent click at ({:.0},{:.0}) → {}", cx, cy, if cg_ok { "OK" } else { "FAIL" });
|
||||
} else {
|
||||
println!(" CGEvent click → no bounds available");
|
||||
}
|
||||
|
||||
release_ax(*child);
|
||||
}
|
||||
for child in children.iter().skip(8) { release_ax(*child); }
|
||||
|
||||
release_ax(*win);
|
||||
if i >= 1 { break; } // only first 2 windows
|
||||
}
|
||||
|
||||
// ── 5. Multi-attribute fetch speed comparison ─────────────────────────────
|
||||
println!("\n[5] AXUIElementCopyMultipleAttributeValues vs individual calls:");
|
||||
speed_test(app, pid);
|
||||
|
||||
// ── 6. Scroll event test ─────────────────────────────────────────────────
|
||||
println!("\n[6] CGEvent scroll at (400, 400):");
|
||||
let ok = cg_scroll(400.0, 400.0, 0, -3);
|
||||
println!(" result = {}", if ok { "OK" } else { "FAIL" });
|
||||
|
||||
unsafe { core_foundation::base::CFRelease(app as CFTypeRef) };
|
||||
println!("\n=== Done ===");
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_attribute_names(el: accessibility_sys::AXUIElementRef) -> Vec<String> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{array::CFArray, base::CFTypeRef, string::CFString, base::TCFType};
|
||||
|
||||
let mut out_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut out_ref as *mut _ as *mut _) };
|
||||
if err != kAXErrorSuccess || out_ref.is_null() { return vec![]; }
|
||||
let arr = unsafe { CFArray::<CFString>::wrap_under_create_rule(out_ref as _) };
|
||||
arr.into_iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
/// Read an array-typed AX attribute, retaining each element so it stays alive.
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_ax_children(el: accessibility_sys::AXUIElementRef, attr: &str) -> Vec<accessibility_sys::AXUIElementRef> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{array::CFArray, base::{CFRetain, CFType, CFTypeRef, TCFType}, string::CFString};
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() { return vec![]; }
|
||||
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(val as _) };
|
||||
arr.into_iter().filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as AXUIElementRef;
|
||||
if ptr.is_null() { return None; }
|
||||
// Retain so the element lives past CFArray dealloc
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(ptr)
|
||||
}).collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn release_ax(el: accessibility_sys::AXUIElementRef) {
|
||||
if !el.is_null() {
|
||||
unsafe { core_foundation::base::CFRelease(el as core_foundation::base::CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_string(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<String> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{base::{CFType, CFTypeRef, TCFType}, string::CFString};
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() { return None; }
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(val) };
|
||||
cf.downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_cgpoint(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<(f64, f64)> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{base::{CFTypeRef, TCFType}, string::CFString};
|
||||
use core_graphics::geometry::CGPoint;
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() { return None; }
|
||||
let mut pt = CGPoint::new(0.0, 0.0);
|
||||
let ok = unsafe { AXValueGetValue(val as _, kAXValueTypeCGPoint, &mut pt as *mut _ as *mut std::ffi::c_void) };
|
||||
unsafe { core_foundation::base::CFRelease(val) };
|
||||
if ok { Some((pt.x, pt.y)) } else { None }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_cgsize(el: accessibility_sys::AXUIElementRef, attr: &str) -> Option<(f64, f64)> {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{base::{CFTypeRef, TCFType}, string::CFString};
|
||||
use core_graphics::geometry::CGSize;
|
||||
|
||||
let key = CFString::new(attr);
|
||||
let mut val: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeValue(el, key.as_concrete_TypeRef(), &mut val) };
|
||||
if err != kAXErrorSuccess || val.is_null() { return None; }
|
||||
let mut sz = CGSize::new(0.0, 0.0);
|
||||
let ok = unsafe { AXValueGetValue(val as _, kAXValueTypeCGSize, &mut sz as *mut _ as *mut std::ffi::c_void) };
|
||||
unsafe { core_foundation::base::CFRelease(val) };
|
||||
if ok { Some((sz.width, sz.height)) } else { None }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn ax_press(el: accessibility_sys::AXUIElementRef) -> i32 {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
let action = CFString::new(kAXPressAction);
|
||||
unsafe { AXUIElementPerformAction(el, action.as_concrete_TypeRef()) }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn cg_click(x: f64, y: f64) -> bool {
|
||||
use core_graphics::{
|
||||
event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton},
|
||||
event_source::{CGEventSource, CGEventSourceStateID},
|
||||
geometry::CGPoint,
|
||||
};
|
||||
let src = match CGEventSource::new(CGEventSourceStateID::HIDSystemState) { Ok(s) => s, Err(_) => return false };
|
||||
let pt = CGPoint::new(x, y);
|
||||
let down = CGEvent::new_mouse_event(src.clone(), CGEventType::LeftMouseDown, pt, CGMouseButton::Left);
|
||||
let up = CGEvent::new_mouse_event(src, CGEventType::LeftMouseUp, pt, CGMouseButton::Left);
|
||||
match (down, up) {
|
||||
(Ok(d), Ok(u)) => { d.post(CGEventTapLocation::HID); u.post(CGEventTapLocation::HID); true }
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn cg_scroll(x: f64, y: f64, dx: i32, dy: i32) -> bool {
|
||||
use core_graphics::{
|
||||
event::{CGEvent, CGEventTapLocation, ScrollEventUnit},
|
||||
event_source::{CGEventSource, CGEventSourceStateID},
|
||||
};
|
||||
let src = match CGEventSource::new(CGEventSourceStateID::HIDSystemState) { Ok(s) => s, Err(_) => return false };
|
||||
match CGEvent::new_scroll_event(src, ScrollEventUnit::LINE, 2, dy, dx, 0) {
|
||||
Ok(ev) => { ev.post(CGEventTapLocation::HID); true }
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn speed_test(app: accessibility_sys::AXUIElementRef, pid: i32) {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{array::CFArray, base::{CFRelease, CFType, CFTypeRef, TCFType}, string::CFString};
|
||||
use std::time::Instant;
|
||||
|
||||
// Get a real window element to test on
|
||||
let windows = get_ax_children(app, kAXWindowsAttribute);
|
||||
let el = if let Some(&w) = windows.first() { w } else { app };
|
||||
|
||||
let attrs = [kAXRoleAttribute, kAXTitleAttribute, kAXDescriptionAttribute,
|
||||
kAXValueAttribute, kAXEnabledAttribute, kAXFocusedAttribute];
|
||||
let cf_attrs: Vec<CFString> = attrs.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_attrs.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
// Multi-attr
|
||||
let t = Instant::now();
|
||||
for _ in 0..100 {
|
||||
let mut res: CFTypeRef = std::ptr::null_mut();
|
||||
unsafe { AXUIElementCopyMultipleAttributeValues(el, names_arr.as_concrete_TypeRef(), 0, &mut res as *mut _ as *mut _) };
|
||||
if !res.is_null() { unsafe { CFRelease(res) }; }
|
||||
}
|
||||
let multi = t.elapsed();
|
||||
|
||||
// Individual attrs
|
||||
let t2 = Instant::now();
|
||||
for _ in 0..100 {
|
||||
for attr in &attrs { let _ = read_string(el, attr); }
|
||||
}
|
||||
let single = t2.elapsed();
|
||||
|
||||
println!(" 100x multi-attr: {:?} ({:?}/call)", multi, multi / 100);
|
||||
println!(" 100x individual: {:?} ({:?}/call)", single, single / 100);
|
||||
println!(" speedup: {:.1}x", single.as_nanos() as f64 / multi.as_nanos().max(1) as f64);
|
||||
|
||||
for w in windows { release_ax(w); }
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
fn main() { eprintln!("macOS only"); }
|
||||
395
crates/macos/examples/axprobe.rs
Normal file
395
crates/macos/examples/axprobe.rs
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
//! AX API probe — discovers what every accessibility function returns on a live app.
|
||||
//! Run: cargo run -p agent-desktop-macos --bin axprobe -- Finder
|
||||
//!
|
||||
//! This is a diagnostic tool used to learn exactly which attributes hold which
|
||||
//! data before writing tree traversal code. Output is deliberately verbose.
|
||||
|
||||
fn main() {
|
||||
#[cfg(target_os = "macos")]
|
||||
run();
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
eprintln!("axprobe only works on macOS");
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn run() {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
url::CFURL,
|
||||
};
|
||||
|
||||
let app_name = std::env::args().nth(1).unwrap_or_else(|| "Finder".to_string());
|
||||
let pid = find_pid(&app_name).unwrap_or_else(|| {
|
||||
eprintln!("App '{}' not running", app_name);
|
||||
std::process::exit(1);
|
||||
});
|
||||
|
||||
println!("=== axprobe: {} (pid {}) ===\n", app_name, pid);
|
||||
|
||||
let app_el = unsafe { AXUIElementCreateApplication(pid) };
|
||||
unsafe { AXUIElementSetMessagingTimeout(app_el, 5.0) };
|
||||
|
||||
// ── 1. App-level attributes ──────────────────────────────────────────
|
||||
println!("──────────────────────────────────────────────");
|
||||
println!("SECTION 1: App element attributes");
|
||||
println!("──────────────────────────────────────────────");
|
||||
dump_all_attrs(app_el, 0);
|
||||
|
||||
// ── 2. Windows ───────────────────────────────────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 2: Windows");
|
||||
println!("──────────────────────────────────────────────");
|
||||
let windows = copy_el_array(app_el, "AXWindows");
|
||||
println!("Window count: {}", windows.len());
|
||||
|
||||
for (wi, &win) in windows.iter().enumerate() {
|
||||
unsafe { AXUIElementSetMessagingTimeout(win, 5.0) };
|
||||
let title = fetch_repr(win, "AXTitle");
|
||||
println!("\n Window[{}] AXTitle={}", wi, title);
|
||||
dump_all_attrs(win, 2);
|
||||
|
||||
// ── 3. Direct children of window ─────────────────────────────
|
||||
let children = copy_el_array(win, "AXChildren");
|
||||
println!("\n Direct children: {}", children.len());
|
||||
|
||||
for (ci, &child) in children.iter().enumerate() {
|
||||
let role = fetch_repr(child, "AXRole");
|
||||
let subrole = fetch_repr(child, "AXSubrole");
|
||||
let title = fetch_repr(child, "AXTitle");
|
||||
let desc = fetch_repr(child, "AXDescription");
|
||||
let value = fetch_repr(child, "AXValue");
|
||||
println!("\n Child[{}] role={} subrole={} title={} desc={} value={}",
|
||||
ci, role, subrole, title, desc, value);
|
||||
dump_all_attrs(child, 6);
|
||||
|
||||
// Grandchildren
|
||||
let grandchildren = copy_el_array(child, "AXChildren");
|
||||
println!(" grandchildren: {}", grandchildren.len());
|
||||
for (gci, &gc) in grandchildren.iter().enumerate().take(10) {
|
||||
let r = fetch_repr(gc, "AXRole");
|
||||
let s = fetch_repr(gc, "AXSubrole");
|
||||
let t = fetch_repr(gc, "AXTitle");
|
||||
let d = fetch_repr(gc, "AXDescription");
|
||||
let v = fetch_repr(gc, "AXValue");
|
||||
println!(" GC[{}] role={} subrole={} title={} desc={} value={}",
|
||||
gci, r, s, t, d, v);
|
||||
dump_all_attrs(gc, 8);
|
||||
|
||||
// Great-grandchildren (just roles/names)
|
||||
let ggc = copy_el_array(gc, "AXChildren");
|
||||
for (ggci, &el) in ggc.iter().enumerate().take(6) {
|
||||
let r2 = fetch_repr(el, "AXRole");
|
||||
let t2 = fetch_repr(el, "AXTitle");
|
||||
let d2 = fetch_repr(el, "AXDescription");
|
||||
let v2 = fetch_repr(el, "AXValue");
|
||||
println!(" GGC[{}] role={} title={} desc={} value={}",
|
||||
ggci, r2, t2, d2, v2);
|
||||
dump_all_attrs(el, 10);
|
||||
|
||||
for &gggel in copy_el_array(el, "AXChildren").iter().take(4) {
|
||||
let r3 = fetch_repr(gggel, "AXRole");
|
||||
let t3 = fetch_repr(gggel, "AXTitle");
|
||||
let v3 = fetch_repr(gggel, "AXValue");
|
||||
println!(" GGGC role={} title={} value={}", r3, t3, v3);
|
||||
unsafe { CFRelease(gggel as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(el as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
|
||||
// Only probe first window in depth
|
||||
break;
|
||||
}
|
||||
|
||||
// ── 4. AXCopyMultipleAttributeValues API test ────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 3: AXUIElementCopyMultipleAttributeValues test");
|
||||
println!("──────────────────────────────────────────────");
|
||||
|
||||
// Use first window child for the multi-attr test
|
||||
if !windows.is_empty() {
|
||||
let win = windows[0];
|
||||
let children = copy_el_array(win, "AXChildren");
|
||||
if !children.is_empty() {
|
||||
let el = children[0];
|
||||
let role = fetch_repr(el, "AXRole");
|
||||
println!("Test element: role={}", role);
|
||||
test_multi_attr(el);
|
||||
unsafe { CFRelease(el as CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
// ── 5. AXUIElementCopyElementAtPosition ─────────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 4: AXUIElementCopyElementAtPosition");
|
||||
println!("──────────────────────────────────────────────");
|
||||
for (x, y) in [(400.0f32, 300.0), (600.0, 300.0), (200.0, 400.0)] {
|
||||
let mut pos_el: AXUIElementRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyElementAtPosition(app_el, x, y, &mut pos_el) };
|
||||
if err == 0 && !pos_el.is_null() {
|
||||
let r = fetch_repr(pos_el, "AXRole");
|
||||
let t = fetch_repr(pos_el, "AXTitle");
|
||||
let d = fetch_repr(pos_el, "AXDescription");
|
||||
let v = fetch_repr(pos_el, "AXValue");
|
||||
println!(" ({},{}): role={} title={} desc={} value={}", x, y, r, t, d, v);
|
||||
unsafe { CFRelease(pos_el as CFTypeRef) };
|
||||
} else {
|
||||
println!(" ({},{}): err={}", x, y, err);
|
||||
}
|
||||
}
|
||||
|
||||
// ── 6. Action names ──────────────────────────────────────────────────
|
||||
println!("\n──────────────────────────────────────────────");
|
||||
println!("SECTION 5: AXUIElementCopyActionNames");
|
||||
println!("──────────────────────────────────────────────");
|
||||
if !windows.is_empty() {
|
||||
let win = windows[0];
|
||||
let children = copy_el_array(win, "AXChildren");
|
||||
for &child in children.iter().take(5) {
|
||||
let role = fetch_repr(child, "AXRole");
|
||||
let actions = copy_action_names(child);
|
||||
println!(" {} => {:?}", role, actions);
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
for &win in &windows {
|
||||
unsafe { CFRelease(win as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(app_el as CFTypeRef) };
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_pid(app_name: &str) -> Option<i32> {
|
||||
let out = std::process::Command::new("pgrep")
|
||||
.arg("-x")
|
||||
.arg(app_name)
|
||||
.output()
|
||||
.ok()?;
|
||||
String::from_utf8_lossy(&out.stdout).lines().next()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn dump_all_attrs(el: accessibility_sys::AXUIElementRef, indent: usize) {
|
||||
use accessibility_sys::AXUIElementCopyAttributeNames;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let pad = " ".repeat(indent);
|
||||
let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) };
|
||||
if err != 0 || names_ref.is_null() {
|
||||
println!("{} <AXUIElementCopyAttributeNames err={}>", pad, err);
|
||||
return;
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(names_ref as _) };
|
||||
let names: Vec<String> = arr
|
||||
.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
|
||||
for name in &names {
|
||||
let val = fetch_repr(el, name);
|
||||
println!("{} [attr] {}: {}", pad, name, val);
|
||||
}
|
||||
|
||||
// Also test AXUIElementCopyParameterizedAttributeNames
|
||||
let mut pnames_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let perr = unsafe {
|
||||
accessibility_sys::AXUIElementCopyParameterizedAttributeNames(el, &mut pnames_ref)
|
||||
};
|
||||
if perr == 0 && !pnames_ref.is_null() {
|
||||
let parr = unsafe { CFArray::<CFType>::wrap_under_create_rule(pnames_ref as _) };
|
||||
let pnames: Vec<String> = parr
|
||||
.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
if !pnames.is_empty() {
|
||||
println!("{} [parameterized attrs]: {:?}", pad, pnames);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn fetch_repr(el: accessibility_sys::AXUIElementRef, attr: &str) -> String {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
url::CFURL,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != 0 {
|
||||
return format!("<err:{}>", err);
|
||||
}
|
||||
if value.is_null() {
|
||||
return "<null>".to_string();
|
||||
}
|
||||
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
|
||||
if let Some(s) = cf.downcast::<CFString>() {
|
||||
return format!("\"{}\"", s.to_string());
|
||||
}
|
||||
if let Some(b) = cf.downcast::<CFBoolean>() {
|
||||
return format!("bool:{}", bool::from(b));
|
||||
}
|
||||
if let Some(n) = cf.downcast::<CFNumber>() {
|
||||
if let Some(i) = n.to_i64() {
|
||||
return format!("num:{}", i);
|
||||
}
|
||||
if let Some(f) = n.to_f64() {
|
||||
return format!("num:{:.2}", f);
|
||||
}
|
||||
}
|
||||
// Check if it's an array by type ID
|
||||
let arr_type_id = unsafe { core_foundation_sys::array::CFArrayGetTypeID() };
|
||||
if cf.type_of() == arr_type_id {
|
||||
let arr = unsafe {
|
||||
CFArray::<CFType>::wrap_under_get_rule(
|
||||
cf.as_concrete_TypeRef() as core_foundation_sys::array::CFArrayRef
|
||||
)
|
||||
};
|
||||
return format!("array[{}]", arr.len());
|
||||
}
|
||||
if let Some(url) = cf.downcast::<CFURL>() {
|
||||
return format!("url:{}", url.get_string().to_string());
|
||||
}
|
||||
|
||||
format!("cftype:{}", cf.type_of())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn copy_el_array(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
attr: &str,
|
||||
) -> Vec<accessibility_sys::AXUIElementRef> {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != 0 || value.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(value as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as accessibility_sys::AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(ptr)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn copy_action_names(el: accessibility_sys::AXUIElementRef) -> Vec<String> {
|
||||
use accessibility_sys::AXUIElementCopyActionNames;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let mut ref_: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyActionNames(el, &mut ref_) };
|
||||
if err != 0 || ref_.is_null() {
|
||||
return vec![];
|
||||
}
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(ref_ as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn test_multi_attr(el: accessibility_sys::AXUIElementRef) {
|
||||
use accessibility_sys::{
|
||||
AXUIElementCopyMultipleAttributeValues, kAXCopyMultipleAttributeOptionStopOnError,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let test_attrs = [
|
||||
"AXRole", "AXSubrole", "AXTitle", "AXDescription", "AXValue",
|
||||
"AXEnabled", "AXFocused", "AXHelp", "AXPlaceholderValue",
|
||||
"AXRoleDescription",
|
||||
];
|
||||
|
||||
for &options in &[0u32, kAXCopyMultipleAttributeOptionStopOnError] {
|
||||
let label = if options == 0 { "AllowPartial(0)" } else { "StopOnError(0x1)" };
|
||||
println!(" options={}", label);
|
||||
|
||||
let cf_names: Vec<CFString> = test_attrs.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
let mut result_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyMultipleAttributeValues(
|
||||
el,
|
||||
names_arr.as_concrete_TypeRef(),
|
||||
options,
|
||||
&mut result_ref as *mut _ as *mut _,
|
||||
)
|
||||
};
|
||||
println!(" err={}", err);
|
||||
|
||||
if err == 0 && !result_ref.is_null() {
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(result_ref as _) };
|
||||
println!(" result_count={} (requested {})", arr.len(), test_attrs.len());
|
||||
for (i, item) in arr.into_iter().enumerate() {
|
||||
let name = test_attrs.get(i).unwrap_or(&"?");
|
||||
let repr = if let Some(s) = item.downcast::<CFString>() {
|
||||
format!("String(\"{}\")", s.to_string())
|
||||
} else if let Some(b) = item.downcast::<CFBoolean>() {
|
||||
format!("Bool({})", bool::from(b))
|
||||
} else if let Some(n) = item.downcast::<CFNumber>() {
|
||||
format!("Number({})", n.to_i64().unwrap_or(-1))
|
||||
} else {
|
||||
format!("Other(type_id={})", item.type_of())
|
||||
};
|
||||
println!(" [{}] {} = {}", i, name, repr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
378
crates/macos/examples/axprobe2.rs
Normal file
378
crates/macos/examples/axprobe2.rs
Normal file
|
|
@ -0,0 +1,378 @@
|
|||
//! Deep probe of AXOutline sidebar rows and AXBrowser columns in Finder.
|
||||
//! Run: cargo run -p agent-desktop-macos --bin axprobe2
|
||||
|
||||
fn main() {
|
||||
#[cfg(target_os = "macos")]
|
||||
run();
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
eprintln!("macOS only");
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn run() {
|
||||
use accessibility_sys::*;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let pid = find_pid("Finder").unwrap_or_else(|| {
|
||||
eprintln!("Finder not running");
|
||||
std::process::exit(1);
|
||||
});
|
||||
|
||||
let app_el = unsafe { AXUIElementCreateApplication(pid) };
|
||||
unsafe { AXUIElementSetMessagingTimeout(app_el, 5.0) };
|
||||
|
||||
let windows = copy_el_array(app_el, "AXWindows");
|
||||
let win = match windows.first() {
|
||||
Some(&w) => w,
|
||||
None => { eprintln!("No windows"); return; }
|
||||
};
|
||||
unsafe { AXUIElementSetMessagingTimeout(win, 5.0) };
|
||||
|
||||
println!("=== Finder Window: {} ===\n", fetch_str(win, "AXTitle"));
|
||||
|
||||
// ── Sidebar AXOutline → AXRow children ─────────────────────────────
|
||||
println!("────────────────────────────────────────");
|
||||
println!("PART 1: Sidebar AXOutline rows (6 levels deep)");
|
||||
println!("────────────────────────────────────────");
|
||||
|
||||
let sidebar_outline = find_by_desc(win, "sidebar", 0);
|
||||
if let Some(outline) = sidebar_outline {
|
||||
let outline_role = fetch_str(outline, "AXRole");
|
||||
let outline_desc = fetch_str(outline, "AXDescription");
|
||||
println!("Found: role={} desc={}", outline_role, outline_desc);
|
||||
println!("AXRows count: {}", count_attr(outline, "AXRows"));
|
||||
println!("AXChildren count: {}", count_attr(outline, "AXChildren"));
|
||||
|
||||
let rows = copy_el_array(outline, "AXRows");
|
||||
println!("Probing first 8 AXRows (all attributes + full children):");
|
||||
for (i, &row) in rows.iter().enumerate().take(8) {
|
||||
dump_element(row, i, 2);
|
||||
}
|
||||
for &row in &rows { unsafe { CFRelease(row as CFTypeRef) }; }
|
||||
unsafe { CFRelease(outline as CFTypeRef) };
|
||||
} else {
|
||||
println!("Could not find sidebar outline");
|
||||
}
|
||||
|
||||
// ── AXBrowser column view ───────────────────────────────────────────
|
||||
println!("\n────────────────────────────────────────");
|
||||
println!("PART 2: AXBrowser column view (AXColumns)");
|
||||
println!("────────────────────────────────────────");
|
||||
|
||||
let browser = find_by_role(win, "AXBrowser", 0);
|
||||
if let Some(br) = browser {
|
||||
println!("Found AXBrowser: desc={}", fetch_str(br, "AXDescription"));
|
||||
println!("AXColumns: {}", count_attr(br, "AXColumns"));
|
||||
println!("AXChildren: {}", count_attr(br, "AXChildren"));
|
||||
println!("AXVisibleColumns: {}", count_attr(br, "AXVisibleColumns"));
|
||||
|
||||
// Probe each column
|
||||
let columns = copy_el_array(br, "AXColumns");
|
||||
println!("\nProbing AXColumns:");
|
||||
for (ci, &col) in columns.iter().enumerate() {
|
||||
println!("\n Column[{}]:", ci);
|
||||
dump_all_attrs(col, 4);
|
||||
let col_rows = copy_el_array(col, "AXRows");
|
||||
let col_children = copy_el_array(col, "AXChildren");
|
||||
println!(" AXRows={} AXChildren={}", col_rows.len(), col_children.len());
|
||||
|
||||
// Probe first few rows
|
||||
for (ri, &row) in col_rows.iter().enumerate().take(6) {
|
||||
dump_element(row, ri, 6);
|
||||
}
|
||||
for &row in &col_rows { unsafe { CFRelease(row as CFTypeRef) }; }
|
||||
for &child in &col_children { unsafe { CFRelease(child as CFTypeRef) }; }
|
||||
unsafe { CFRelease(col as CFTypeRef) };
|
||||
}
|
||||
|
||||
// Also try AXChildren path
|
||||
println!("\nAXChildren of browser:");
|
||||
let br_children = copy_el_array(br, "AXChildren");
|
||||
for (ci, &child) in br_children.iter().enumerate() {
|
||||
let r = fetch_str(child, "AXRole");
|
||||
let d = fetch_str(child, "AXDescription");
|
||||
println!(" Child[{}] role={} desc={}", ci, r, d);
|
||||
// One more level
|
||||
let gchildren = copy_el_array(child, "AXChildren");
|
||||
for (gi, &gc) in gchildren.iter().enumerate().take(4) {
|
||||
let r2 = fetch_str(gc, "AXRole");
|
||||
let d2 = fetch_str(gc, "AXDescription");
|
||||
println!(" GC[{}] role={} desc={}", gi, r2, d2);
|
||||
// Column rows inside scroll area
|
||||
let sc_children = copy_el_array(gc, "AXChildren");
|
||||
for (sci, &sc) in sc_children.iter().enumerate().take(4) {
|
||||
let r3 = fetch_str(sc, "AXRole");
|
||||
let d3 = fetch_str(sc, "AXDescription");
|
||||
println!(" SC[{}] role={} desc={}", sci, r3, d3);
|
||||
let sc2 = copy_el_array(sc, "AXRows");
|
||||
for (s2i, &s2) in sc2.iter().enumerate().take(6) {
|
||||
dump_element(s2, s2i, 10);
|
||||
}
|
||||
for &s2 in &sc2 { unsafe { CFRelease(s2 as CFTypeRef) }; }
|
||||
unsafe { CFRelease(sc as CFTypeRef) };
|
||||
}
|
||||
for &sc in &sc_children { unsafe { CFRelease(sc as CFTypeRef) }; }
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
for &gc in &gchildren { unsafe { CFRelease(gc as CFTypeRef) }; }
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
|
||||
unsafe { CFRelease(br as CFTypeRef) };
|
||||
} else {
|
||||
println!("Could not find AXBrowser");
|
||||
}
|
||||
|
||||
// ── AXCopyMultipleAttributeValues on an AXRow ───────────────────────
|
||||
println!("\n────────────────────────────────────────");
|
||||
println!("PART 3: AXUIElementCopyMultipleAttributeValues on AXRow");
|
||||
println!("────────────────────────────────────────");
|
||||
|
||||
let sidebar2 = find_by_desc(win, "sidebar", 0);
|
||||
if let Some(outline) = sidebar2 {
|
||||
let rows = copy_el_array(outline, "AXRows");
|
||||
if let Some(&row) = rows.first() {
|
||||
println!("Testing on first sidebar AXRow:");
|
||||
test_multi_attr_extended(row);
|
||||
}
|
||||
for &row in &rows { unsafe { CFRelease(row as CFTypeRef) }; }
|
||||
unsafe { CFRelease(outline as CFTypeRef) };
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
for &win2 in &windows { unsafe { CFRelease(win2 as CFTypeRef) }; }
|
||||
unsafe { CFRelease(app_el as CFTypeRef) };
|
||||
}
|
||||
|
||||
// ── Deep element dump ────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn dump_element(el: accessibility_sys::AXUIElementRef, idx: usize, indent: usize) {
|
||||
use core_foundation::base::{CFRelease, CFTypeRef};
|
||||
let pad = " ".repeat(indent);
|
||||
let role = fetch_str(el, "AXRole");
|
||||
let sub = fetch_str(el, "AXSubrole");
|
||||
let title = fetch_str(el, "AXTitle");
|
||||
let desc = fetch_str(el, "AXDescription");
|
||||
let val = fetch_str(el, "AXValue");
|
||||
let help = fetch_str(el, "AXHelp");
|
||||
println!("{}[{}] role={} subrole={} title={} desc={} value={} help={}",
|
||||
pad, idx, role, sub, title, desc, val, help);
|
||||
dump_all_attrs(el, indent + 2);
|
||||
|
||||
let children = copy_el_array(el, "AXChildren");
|
||||
for (ci, &child) in children.iter().enumerate() {
|
||||
let cr = fetch_str(child, "AXRole");
|
||||
let ct = fetch_str(child, "AXTitle");
|
||||
let cd = fetch_str(child, "AXDescription");
|
||||
let cv = fetch_str(child, "AXValue");
|
||||
println!("{} child[{}] role={} title={} desc={} value={}", pad, ci, cr, ct, cd, cv);
|
||||
dump_all_attrs(child, indent + 4);
|
||||
|
||||
let gchildren = copy_el_array(child, "AXChildren");
|
||||
for (gi, &gc) in gchildren.iter().enumerate() {
|
||||
let gr = fetch_str(gc, "AXRole");
|
||||
let gt = fetch_str(gc, "AXTitle");
|
||||
let gd = fetch_str(gc, "AXDescription");
|
||||
let gv = fetch_str(gc, "AXValue");
|
||||
println!("{} gc[{}] role={} title={} desc={} value={}", pad, gi, gr, gt, gd, gv);
|
||||
dump_all_attrs(gc, indent + 6);
|
||||
unsafe { CFRelease(gc as CFTypeRef) };
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn test_multi_attr_extended(el: accessibility_sys::AXUIElementRef) {
|
||||
use accessibility_sys::{AXUIElementCopyMultipleAttributeValues, AXUIElementCopyAttributeNames};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
// First, get ALL attribute names for this element
|
||||
let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) };
|
||||
if err != 0 || names_ref.is_null() { return; }
|
||||
let name_arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(names_ref as _) };
|
||||
let all_names: Vec<String> = name_arr.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
|
||||
println!("Available attributes on AXRow: {:?}", all_names);
|
||||
|
||||
// Batch fetch all of them
|
||||
let cf_names: Vec<CFString> = all_names.iter().map(|a| CFString::new(a)).collect();
|
||||
let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect();
|
||||
let names_arr2 = CFArray::from_copyable(&cf_refs);
|
||||
|
||||
let mut result: CFTypeRef = std::ptr::null_mut();
|
||||
let err2 = unsafe {
|
||||
AXUIElementCopyMultipleAttributeValues(
|
||||
el, names_arr2.as_concrete_TypeRef(), 0,
|
||||
&mut result as *mut _ as *mut _,
|
||||
)
|
||||
};
|
||||
println!("AXUIElementCopyMultipleAttributeValues err={}", err2);
|
||||
if err2 == 0 && !result.is_null() {
|
||||
let res_arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(result as _) };
|
||||
for (i, item) in res_arr.into_iter().enumerate() {
|
||||
let name = all_names.get(i).map(|s| s.as_str()).unwrap_or("?");
|
||||
let repr = if let Some(s) = item.downcast::<CFString>() {
|
||||
format!("String(\"{}\")", s.to_string())
|
||||
} else if let Some(b) = item.downcast::<CFBoolean>() {
|
||||
format!("Bool({})", bool::from(b))
|
||||
} else if let Some(n) = item.downcast::<CFNumber>() {
|
||||
format!("Number({})", n.to_i64().unwrap_or(-1))
|
||||
} else {
|
||||
format!("Other(type_id={})", item.type_of())
|
||||
};
|
||||
println!(" [{}] {} = {}", i, name, repr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_pid(name: &str) -> Option<i32> {
|
||||
let out = std::process::Command::new("pgrep").arg("-x").arg(name).output().ok()?;
|
||||
String::from_utf8_lossy(&out.stdout).lines().next()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn count_attr(el: accessibility_sys::AXUIElementRef, attr: &str) -> usize {
|
||||
copy_el_array(el, attr).len()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn fetch_str(el: accessibility_sys::AXUIElementRef, attr: &str) -> String {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
base::{CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != 0 { return format!("<err:{}>", err); }
|
||||
if value.is_null() { return "<null>".to_string(); }
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
if let Some(s) = cf.downcast::<CFString>() { return format!("\"{}\"", s.to_string()); }
|
||||
if let Some(b) = cf.downcast::<CFBoolean>() { return format!("bool:{}", bool::from(b)); }
|
||||
if let Some(n) = cf.downcast::<CFNumber>() { return format!("num:{}", n.to_i64().unwrap_or(-1)); }
|
||||
format!("cftype:{}", cf.type_of())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn dump_all_attrs(el: accessibility_sys::AXUIElementRef, indent: usize) {
|
||||
use accessibility_sys::AXUIElementCopyAttributeNames;
|
||||
use core_foundation::{array::CFArray, base::{CFType, TCFType}, string::CFString};
|
||||
|
||||
let pad = " ".repeat(indent);
|
||||
let mut names_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null_mut();
|
||||
let err = unsafe { AXUIElementCopyAttributeNames(el, &mut names_ref) };
|
||||
if err != 0 || names_ref.is_null() { return; }
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(names_ref as _) };
|
||||
let names: Vec<String> = arr.into_iter()
|
||||
.filter_map(|item| item.downcast::<CFString>().map(|s| s.to_string()))
|
||||
.collect();
|
||||
for name in &names {
|
||||
let val = fetch_str(el, name);
|
||||
println!("{}{}: {}", pad, name, val);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn copy_el_array(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
attr: &str,
|
||||
) -> Vec<accessibility_sys::AXUIElementRef> {
|
||||
use accessibility_sys::AXUIElementCopyAttributeValue;
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != 0 || value.is_null() { return vec![]; }
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(value as _) };
|
||||
arr.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as accessibility_sys::AXUIElementRef;
|
||||
if ptr.is_null() { None }
|
||||
else {
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(ptr)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Find first element with given AXDescription recursively
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_by_desc(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
desc: &str,
|
||||
depth: u32,
|
||||
) -> Option<accessibility_sys::AXUIElementRef> {
|
||||
use core_foundation::base::{CFRetain, CFTypeRef};
|
||||
if depth > 8 { return None; }
|
||||
let d = fetch_str(el, "AXDescription");
|
||||
if d == format!("\"{}\"", desc) {
|
||||
unsafe { CFRetain(el as CFTypeRef) };
|
||||
return Some(el);
|
||||
}
|
||||
for child in copy_el_array(el, "AXChildren") {
|
||||
if let Some(found) = find_by_desc(child, desc, depth + 1) {
|
||||
unsafe { core_foundation::base::CFRelease(child as CFTypeRef) };
|
||||
return Some(found);
|
||||
}
|
||||
unsafe { core_foundation::base::CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find first element with given AXRole recursively
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_by_role(
|
||||
el: accessibility_sys::AXUIElementRef,
|
||||
role: &str,
|
||||
depth: u32,
|
||||
) -> Option<accessibility_sys::AXUIElementRef> {
|
||||
use core_foundation::base::{CFRetain, CFRelease, CFTypeRef};
|
||||
if depth > 8 { return None; }
|
||||
let r = fetch_str(el, "AXRole");
|
||||
if r == format!("\"{}\"", role) {
|
||||
unsafe { CFRetain(el as CFTypeRef) };
|
||||
return Some(el);
|
||||
}
|
||||
for child in copy_el_array(el, "AXChildren") {
|
||||
if let Some(found) = find_by_role(child, role, depth + 1) {
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
return Some(found);
|
||||
}
|
||||
unsafe { CFRelease(child as CFTypeRef) };
|
||||
}
|
||||
None
|
||||
}
|
||||
|
|
@ -22,14 +22,13 @@ mod imp {
|
|||
let label = action_label(action);
|
||||
match action {
|
||||
Action::Click => {
|
||||
// Try AXPress first (works for native controls).
|
||||
// Always follow up with a CGEvent click to handle Electron/web elements.
|
||||
let _ = ax_press(el);
|
||||
cg_mouse_click(el, 1, CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left)?;
|
||||
let err = ax_press(el);
|
||||
if err != kAXErrorSuccess {
|
||||
cg_mouse_click(el, 1, CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left)?;
|
||||
}
|
||||
}
|
||||
|
||||
Action::DoubleClick => {
|
||||
let _ = ax_press(el);
|
||||
cg_mouse_click(el, 2, CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left)?;
|
||||
}
|
||||
|
||||
|
|
@ -80,6 +79,14 @@ mod imp {
|
|||
}
|
||||
|
||||
Action::TypeText(text) => {
|
||||
let cf_attr = CFString::new(kAXFocusedAttribute);
|
||||
unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
crate::input::synthesize_text(text)?;
|
||||
}
|
||||
|
||||
|
|
@ -163,7 +170,7 @@ mod imp {
|
|||
let center = element_center(el).ok_or_else(|| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ActionFailed,
|
||||
"Cannot click: element has no position/size. Run snapshot --include-bounds first.",
|
||||
"Cannot click: element has no accessible position or size",
|
||||
)
|
||||
})?;
|
||||
|
||||
|
|
|
|||
|
|
@ -93,6 +93,19 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
let windows = self.list_windows(&filter)?;
|
||||
Ok(windows.into_iter().next())
|
||||
}
|
||||
|
||||
fn get_live_value(&self, handle: &NativeHandle) -> Result<Option<String>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use accessibility_sys::kAXValueAttribute;
|
||||
use crate::tree::AXElement;
|
||||
use std::mem::ManuallyDrop;
|
||||
let el = ManuallyDrop::new(AXElement(handle.as_raw() as accessibility_sys::AXUIElementRef));
|
||||
Ok(crate::tree::copy_string_attr(&el, kAXValueAttribute))
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
Err(AdapterError::not_supported("get_live_value"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
|
|
@ -101,7 +114,7 @@ fn execute_action_impl(handle: &NativeHandle, action: Action) -> Result<ActionRe
|
|||
use std::mem::ManuallyDrop;
|
||||
|
||||
let el = ManuallyDrop::new(AXElement(handle.as_raw() as accessibility_sys::AXUIElementRef));
|
||||
crate::actions::perform_action(&*el, &action)
|
||||
crate::actions::perform_action(&el, &action)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
@ -124,25 +137,19 @@ fn find_element_recursive(
|
|||
max_depth: u8,
|
||||
visited: &mut FxHashSet<usize>,
|
||||
) -> Result<NativeHandle, AdapterError> {
|
||||
use accessibility_sys::{
|
||||
kAXChildrenAttribute, kAXErrorSuccess, kAXRoleAttribute, kAXTitleAttribute,
|
||||
AXUIElementCopyAttributeValue, AXUIElementRef,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
};
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
use core_foundation::base::{CFRetain, CFTypeRef};
|
||||
|
||||
if !visited.insert(el.0 as usize) {
|
||||
return Err(AdapterError::element_not_found("element"));
|
||||
}
|
||||
|
||||
let role = crate::tree::copy_string_attr(el, kAXRoleAttribute);
|
||||
let normalized = role.as_deref().map(crate::roles::ax_role_to_str).unwrap_or("unknown");
|
||||
let ax_role = crate::tree::copy_string_attr(el, kAXRoleAttribute);
|
||||
let normalized = ax_role.as_deref().map(crate::roles::ax_role_to_str).unwrap_or("unknown");
|
||||
|
||||
if normalized == entry.role {
|
||||
let name = crate::tree::copy_string_attr(el, kAXTitleAttribute);
|
||||
let name_match = match (&entry.name, &name) {
|
||||
let elem_name = crate::tree::resolve_element_name(el);
|
||||
let name_match = match (&entry.name, &elem_name) {
|
||||
(Some(en), Some(nn)) => en == nn,
|
||||
(None, None) => true,
|
||||
_ => false,
|
||||
|
|
@ -157,29 +164,14 @@ fn find_element_recursive(
|
|||
return Err(AdapterError::element_not_found("element"));
|
||||
}
|
||||
|
||||
let cf_attr = core_foundation::string::CFString::new(kAXChildrenAttribute);
|
||||
let mut children_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut children_ref,
|
||||
)
|
||||
};
|
||||
let child_attr = if ax_role.as_deref() == Some("AXBrowser") { "AXColumns" } else { "AXChildren" };
|
||||
let children = crate::tree::copy_ax_array(el, child_attr)
|
||||
.filter(|v| !v.is_empty())
|
||||
.or_else(|| crate::tree::copy_ax_array(el, "AXContents").filter(|v| !v.is_empty()))
|
||||
.unwrap_or_default();
|
||||
|
||||
if err != kAXErrorSuccess || children_ref.is_null() {
|
||||
return Err(AdapterError::element_not_found("element"));
|
||||
}
|
||||
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(children_ref as _) };
|
||||
for item in arr.into_iter() {
|
||||
let ptr = item.as_concrete_TypeRef() as AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
continue;
|
||||
}
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
let child = crate::tree::AXElement(ptr);
|
||||
if let Ok(handle) = find_element_recursive(&child, entry, depth + 1, max_depth, visited) {
|
||||
for child in &children {
|
||||
if let Ok(handle) = find_element_recursive(child, entry, depth + 1, max_depth, visited) {
|
||||
return Ok(handle);
|
||||
}
|
||||
}
|
||||
|
|
@ -195,54 +187,58 @@ fn resolve_element_impl(_entry: &RefEntry) -> Result<NativeHandle, AdapterError>
|
|||
pub fn list_windows_impl(filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use core_foundation::base::{CFType, TCFType};
|
||||
use core_foundation::number::CFNumber;
|
||||
use core_foundation::string::CFString;
|
||||
use core_foundation_sys::dictionary::CFDictionaryGetValue;
|
||||
use core_graphics::display::CGDisplay;
|
||||
use core_graphics::window::{
|
||||
kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowName,
|
||||
kCGWindowOwnerName, kCGWindowOwnerPID,
|
||||
};
|
||||
use rustc_hash::FxHasher;
|
||||
use std::ffi::c_void;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::process::Command;
|
||||
|
||||
let app_filter = filter.app.as_deref().unwrap_or("").to_string();
|
||||
let script = r#"
|
||||
tell application "System Events"
|
||||
set winList to {}
|
||||
repeat with proc in (processes where background only is false)
|
||||
set pName to name of proc as string
|
||||
set pPid to (unix id of proc) as string
|
||||
set winTitles to name of every window of proc
|
||||
repeat with wTitle in winTitles
|
||||
set winList to winList & {pName & "|" & pPid & "|" & (wTitle as string)}
|
||||
end repeat
|
||||
end repeat
|
||||
set AppleScript's text item delimiters to linefeed
|
||||
return winList as text
|
||||
end tell
|
||||
"#;
|
||||
let output = Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(script)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("osascript failed: {e}")))?;
|
||||
unsafe fn dict_string(dict: *const c_void, key: *const c_void) -> Option<String> {
|
||||
let val = CFDictionaryGetValue(dict as _, key);
|
||||
if val.is_null() { return None; }
|
||||
CFType::wrap_under_get_rule(val as _).downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
let text = String::from_utf8_lossy(&output.stdout);
|
||||
unsafe fn dict_i64(dict: *const c_void, key: *const c_void) -> Option<i64> {
|
||||
let val = CFDictionaryGetValue(dict as _, key);
|
||||
if val.is_null() { return None; }
|
||||
CFType::wrap_under_get_rule(val as _).downcast::<CFNumber>().and_then(|n| n.to_i64())
|
||||
}
|
||||
|
||||
let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) {
|
||||
Some(a) => a,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let app_filter = filter.app.as_deref().unwrap_or("").to_lowercase();
|
||||
let mut windows = Vec::new();
|
||||
for (idx, line) in text.lines().enumerate() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut parts = line.splitn(3, '|');
|
||||
let app_name = match parts.next() {
|
||||
Some(s) => s.trim().to_string(),
|
||||
None => continue,
|
||||
};
|
||||
let pid: i32 = match parts.next().and_then(|s| s.trim().parse().ok()) {
|
||||
Some(p) => p,
|
||||
None => continue,
|
||||
};
|
||||
let title = parts.next().unwrap_or("").trim().to_string();
|
||||
|
||||
if !app_filter.is_empty() && !app_name.eq_ignore_ascii_case(&app_filter) {
|
||||
for raw in arr.get_all_values() {
|
||||
if raw.is_null() { continue; }
|
||||
let layer = unsafe { dict_i64(raw, kCGWindowLayer as _) }.unwrap_or(99);
|
||||
if layer != 0 { continue; }
|
||||
|
||||
let app_name = match unsafe { dict_string(raw, kCGWindowOwnerName as _) } {
|
||||
Some(n) if !n.is_empty() => n,
|
||||
_ => continue,
|
||||
};
|
||||
if !app_filter.is_empty() && !app_name.to_lowercase().contains(&app_filter) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let title = match unsafe { dict_string(raw, kCGWindowName as _) } {
|
||||
Some(t) if !t.is_empty() => t,
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let pid = unsafe { dict_i64(raw, kCGWindowOwnerPID as _) }.unwrap_or(0) as i32;
|
||||
let mut h = FxHasher::default();
|
||||
pid.hash(&mut h);
|
||||
title.hash(&mut h);
|
||||
|
|
@ -254,7 +250,7 @@ end tell
|
|||
app: app_name,
|
||||
pid,
|
||||
bounds: None,
|
||||
is_focused: idx == 0,
|
||||
is_focused: windows.is_empty(),
|
||||
});
|
||||
}
|
||||
Ok(windows)
|
||||
|
|
@ -269,32 +265,50 @@ end tell
|
|||
fn list_apps_impl() -> Result<Vec<AppInfo>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use std::process::Command;
|
||||
let output = Command::new("osascript")
|
||||
.arg("-e")
|
||||
.arg(
|
||||
r#"tell application "System Events"
|
||||
set result to ""
|
||||
repeat with proc in (processes where background only is false)
|
||||
set result to result & (name of proc as string) & "|" & ((unix id of proc) as string) & linefeed
|
||||
end repeat
|
||||
return result
|
||||
end tell"#,
|
||||
)
|
||||
.output()
|
||||
.map_err(|e| AdapterError::internal(format!("osascript failed: {e}")))?;
|
||||
use core_foundation::base::{CFType, TCFType};
|
||||
use core_foundation::number::CFNumber;
|
||||
use core_foundation::string::CFString;
|
||||
use core_foundation_sys::dictionary::CFDictionaryGetValue;
|
||||
use core_graphics::display::CGDisplay;
|
||||
use core_graphics::window::{
|
||||
kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowOwnerName, kCGWindowOwnerPID,
|
||||
};
|
||||
|
||||
let text = String::from_utf8_lossy(&output.stdout);
|
||||
let apps = text
|
||||
.lines()
|
||||
.filter(|l| !l.is_empty())
|
||||
.filter_map(|line| {
|
||||
let mut parts = line.split('|');
|
||||
let name = parts.next()?.trim().to_string();
|
||||
let pid: i32 = parts.next()?.trim().parse().ok()?;
|
||||
Some(AppInfo { name, pid, bundle_id: None })
|
||||
})
|
||||
.collect();
|
||||
let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) {
|
||||
Some(a) => a,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let mut seen_pids = std::collections::HashSet::new();
|
||||
let mut apps = Vec::new();
|
||||
|
||||
for raw in arr.get_all_values() {
|
||||
if raw.is_null() { continue; }
|
||||
|
||||
let layer = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowLayer as _);
|
||||
if v.is_null() { continue; }
|
||||
CFType::wrap_under_get_rule(v as _).downcast::<CFNumber>().and_then(|n| n.to_i64()).unwrap_or(99)
|
||||
};
|
||||
if layer != 0 { continue; }
|
||||
|
||||
let pid = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerPID as _);
|
||||
if v.is_null() { continue; }
|
||||
CFType::wrap_under_get_rule(v as _).downcast::<CFNumber>().and_then(|n| n.to_i64()).unwrap_or(0) as i32
|
||||
};
|
||||
if !seen_pids.insert(pid) { continue; }
|
||||
|
||||
let name = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerName as _);
|
||||
if v.is_null() { continue; }
|
||||
CFType::wrap_under_get_rule(v as _).downcast::<CFString>().map(|s| s.to_string())
|
||||
};
|
||||
|
||||
if let Some(n) = name {
|
||||
apps.push(AppInfo { name: n, pid, bundle_id: None });
|
||||
}
|
||||
}
|
||||
Ok(apps)
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
|
|||
|
|
@ -1,36 +1,54 @@
|
|||
pub fn ax_role_to_str(ax_role: &str) -> &'static str {
|
||||
match ax_role {
|
||||
"AXApplication" => "application",
|
||||
"AXButton" => "button",
|
||||
"AXApplication" => "application",
|
||||
"AXButton" => "button",
|
||||
"AXMenuButton" => "menubutton",
|
||||
"AXTextField" | "AXTextArea" | "AXSearchField" => "textfield",
|
||||
"AXCheckBox" => "checkbox",
|
||||
"AXLink" => "link",
|
||||
"AXMenuItem" | "AXMenuBarItem" => "menuitem",
|
||||
"AXRadioButton" => "radiobutton",
|
||||
"AXTab" | "AXTabGroup" => "tab",
|
||||
"AXSlider" | "AXValueIndicator" => "slider",
|
||||
"AXComboBox" | "AXPopUpButton" => "combobox",
|
||||
"AXOutlineRow" | "AXRow" => "treeitem",
|
||||
"AXCell" => "cell",
|
||||
"AXWindow" => "window",
|
||||
"AXSheet" => "sheet",
|
||||
"AXDialog" => "dialog",
|
||||
"AXGroup" | "AXGenericElement" => "group",
|
||||
"AXToolbar" => "toolbar",
|
||||
"AXStaticText" => "statictext",
|
||||
"AXImage" => "image",
|
||||
"AXTable" => "table",
|
||||
"AXList" => "list",
|
||||
"AXOutline" => "outline",
|
||||
"AXScrollArea" | "AXScrollBar" => "scrollarea",
|
||||
"AXSplitter" | "AXSplitGroup" => "splitter",
|
||||
"AXMenu" | "AXMenuBar" => "menu",
|
||||
"AXIncrementor" | "AXStepper" => "incrementor",
|
||||
"AXDisclosureTriangle" => "disclosure",
|
||||
"AXProgressIndicator" | "AXBusyIndicator" => "progressbar",
|
||||
"AXColorWell" => "colorwell",
|
||||
"AXWebArea" => "webarea",
|
||||
_ => "unknown",
|
||||
"AXCheckBox" => "checkbox",
|
||||
"AXSwitch" | "AXToggle" => "switch",
|
||||
"AXLink" => "link",
|
||||
"AXMenuItem" | "AXMenuBarItem" => "menuitem",
|
||||
"AXRadioButton" => "radiobutton",
|
||||
"AXTab" | "AXTabGroup" => "tab",
|
||||
"AXSlider" | "AXValueIndicator" => "slider",
|
||||
"AXComboBox" | "AXPopUpButton" => "combobox",
|
||||
"AXOutlineRow" | "AXRow" => "treeitem",
|
||||
"AXCell" => "cell",
|
||||
"AXColumn" => "column",
|
||||
"AXWindow" => "window",
|
||||
"AXSheet" => "sheet",
|
||||
"AXDialog" => "dialog",
|
||||
"AXGroup" | "AXGenericElement" => "group",
|
||||
"AXToolbar" => "toolbar",
|
||||
"AXStaticText" => "statictext",
|
||||
"AXImage" => "image",
|
||||
"AXTable" => "table",
|
||||
"AXList" => "list",
|
||||
"AXOutline" => "outline",
|
||||
"AXScrollArea" | "AXScrollBar" => "scrollarea",
|
||||
"AXSplitter" | "AXSplitGroup" => "splitter",
|
||||
"AXMenu" | "AXMenuBar" => "menu",
|
||||
"AXIncrementor" | "AXStepper" => "incrementor",
|
||||
"AXDisclosureTriangle" => "disclosure",
|
||||
"AXProgressIndicator" | "AXBusyIndicator" => "progressbar",
|
||||
"AXColorWell" => "colorwell",
|
||||
"AXWebArea" => "webarea",
|
||||
"AXBrowser" => "browser",
|
||||
"AXGrid" => "grid",
|
||||
"AXHandle" => "handle",
|
||||
"AXPopover" => "popover",
|
||||
"AXDockItem" => "dockitem",
|
||||
"AXRuler" => "ruler",
|
||||
"AXRulerMarker" => "rulermarker",
|
||||
"AXTimeField" => "timefield",
|
||||
"AXDateField" => "datefield",
|
||||
"AXHelpTag" => "helptag",
|
||||
"AXMatte" => "matte",
|
||||
"AXDrawer" => "drawer",
|
||||
"AXLayoutArea" | "AXLayoutItem" => "layoutitem",
|
||||
"AXLevelIndicator" => "levelindicator",
|
||||
"AXRelevanceIndicator" => "relevanceindicator",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -38,8 +56,10 @@ pub fn is_interactive_role(role: &str) -> bool {
|
|||
matches!(
|
||||
role,
|
||||
"button"
|
||||
| "menubutton"
|
||||
| "textfield"
|
||||
| "checkbox"
|
||||
| "switch"
|
||||
| "link"
|
||||
| "menuitem"
|
||||
| "tab"
|
||||
|
|
@ -49,5 +69,7 @@ pub fn is_interactive_role(role: &str) -> bool {
|
|||
| "cell"
|
||||
| "radiobutton"
|
||||
| "incrementor"
|
||||
| "colorwell"
|
||||
| "dockitem"
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,7 +53,17 @@ mod imp {
|
|||
let data = std::fs::read(path)
|
||||
.map_err(|e| AdapterError::internal(format!("read screenshot: {e}")))?;
|
||||
let _ = std::fs::remove_file(path);
|
||||
Ok(ImageBuffer { data, format: ImageFormat::Png, width: 0, height: 0 })
|
||||
let (width, height) = png_dimensions(&data);
|
||||
Ok(ImageBuffer { data, format: ImageFormat::Png, width, height })
|
||||
}
|
||||
|
||||
fn png_dimensions(data: &[u8]) -> (u32, u32) {
|
||||
if data.len() < 24 {
|
||||
return (0, 0);
|
||||
}
|
||||
let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]);
|
||||
let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]);
|
||||
(w, h)
|
||||
}
|
||||
|
||||
fn find_cg_window_id_for_pid(pid: i32) -> Option<u32> {
|
||||
|
|
@ -116,12 +126,12 @@ mod imp {
|
|||
bounds_val.as_concrete_TypeRef() as _,
|
||||
)
|
||||
};
|
||||
let w = bounds_dict.find(&CFString::new("Width")).and_then(|v| {
|
||||
let w = bounds_dict.find(CFString::new("Width")).and_then(|v| {
|
||||
let n =
|
||||
unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) };
|
||||
n.to_f64()
|
||||
});
|
||||
let h = bounds_dict.find(&CFString::new("Height")).and_then(|v| {
|
||||
let h = bounds_dict.find(CFString::new("Height")).and_then(|v| {
|
||||
let n =
|
||||
unsafe { CFNumber::wrap_under_get_rule(v.as_concrete_TypeRef() as _) };
|
||||
n.to_f64()
|
||||
|
|
|
|||
|
|
@ -11,12 +11,13 @@ mod imp {
|
|||
kAXEnabledAttribute, kAXErrorSuccess, kAXFocusedAttribute, kAXRoleAttribute,
|
||||
kAXTitleAttribute, kAXValueAttribute, kAXWindowsAttribute,
|
||||
AXUIElementCopyAttributeValue, AXUIElementCopyMultipleAttributeValues,
|
||||
AXUIElementCreateApplication, AXUIElementRef,
|
||||
AXUIElementCreateApplication, AXUIElementRef, AXUIElementSetMessagingTimeout,
|
||||
};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRelease, CFRetain, CFType, CFTypeRef, TCFType},
|
||||
boolean::CFBoolean,
|
||||
number::CFNumber,
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
|
|
@ -40,7 +41,12 @@ mod imp {
|
|||
}
|
||||
|
||||
pub fn element_for_pid(pid: i32) -> AXElement {
|
||||
AXElement(unsafe { AXUIElementCreateApplication(pid) })
|
||||
let el = AXElement(unsafe { AXUIElementCreateApplication(pid) });
|
||||
if !el.0.is_null() {
|
||||
// 2-second timeout prevents hung/slow processes from blocking the tree walk
|
||||
unsafe { AXUIElementSetMessagingTimeout(el.0, 2.0) };
|
||||
}
|
||||
el
|
||||
}
|
||||
|
||||
/// Find the AXWindow element whose title matches `win_title`.
|
||||
|
|
@ -48,37 +54,32 @@ mod imp {
|
|||
pub fn window_element_for(pid: i32, win_title: &str) -> AXElement {
|
||||
let app = element_for_pid(pid);
|
||||
|
||||
// Try kAXWindowsAttribute
|
||||
if let Some(windows) = copy_ax_array(&app, kAXWindowsAttribute) {
|
||||
// Exact title match
|
||||
for win in &windows {
|
||||
let title = copy_string_attr(win, kAXTitleAttribute);
|
||||
if title.as_deref() == Some(win_title) {
|
||||
let matched = win.clone();
|
||||
return matched;
|
||||
return win.clone();
|
||||
}
|
||||
}
|
||||
// Partial match
|
||||
for win in &windows {
|
||||
let title = copy_string_attr(win, kAXTitleAttribute);
|
||||
if title.as_deref().is_some_and(|t| t.contains(win_title) || win_title.contains(t)) {
|
||||
let matched = win.clone();
|
||||
return matched;
|
||||
return win.clone();
|
||||
}
|
||||
}
|
||||
// First available window
|
||||
if let Some(first) = windows.into_iter().next() {
|
||||
return first;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: app root
|
||||
app
|
||||
}
|
||||
|
||||
/// Batch-fetch the six most-used attributes in a single AX API call.
|
||||
/// Returns (role, title, description, value, enabled, focused).
|
||||
fn fetch_node_attrs(el: &AXElement) -> (Option<String>, Option<String>, Option<String>, Option<String>, bool, bool) {
|
||||
/// Batch-fetch six most-used attributes. Returns (role, title, desc, value, enabled, focused).
|
||||
/// Value handles CFString, CFBoolean, and CFNumber types.
|
||||
fn fetch_node_attrs(
|
||||
el: &AXElement,
|
||||
) -> (Option<String>, Option<String>, Option<String>, Option<String>, bool, bool) {
|
||||
let attr_names = [
|
||||
kAXRoleAttribute,
|
||||
kAXTitleAttribute,
|
||||
|
|
@ -102,19 +103,34 @@ mod imp {
|
|||
};
|
||||
|
||||
if err != kAXErrorSuccess || result_ref.is_null() {
|
||||
// Fallback to individual calls
|
||||
let role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
let val = copy_string_attr(el, kAXValueAttribute);
|
||||
let role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
let val = copy_value_typed(el);
|
||||
let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true);
|
||||
let focused = copy_bool_attr(el, kAXFocusedAttribute).unwrap_or(false);
|
||||
return (role, title, desc, val, enabled, focused);
|
||||
}
|
||||
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(result_ref as _) };
|
||||
let items: Vec<Option<String>> = arr.into_iter().map(|item| {
|
||||
item.downcast::<CFString>().map(|s| s.to_string())
|
||||
let items: Vec<Option<String>> = arr.into_iter().enumerate().map(|(idx, item)| {
|
||||
if let Some(s) = item.downcast::<CFString>() {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
match idx {
|
||||
// value: may be CFBoolean (checkbox) or CFNumber (slider, stepper)
|
||||
3 => {
|
||||
if let Some(b) = item.downcast::<CFBoolean>() { return Some(bool::from(b).to_string()); }
|
||||
if let Some(n) = item.downcast::<CFNumber>() {
|
||||
if let Some(i) = n.to_i64() { return Some(i.to_string()); }
|
||||
if let Some(f) = n.to_f64() { return Some(format!("{:.2}", f)); }
|
||||
}
|
||||
None
|
||||
}
|
||||
// enabled / focused are always CFBoolean
|
||||
4 | 5 => item.downcast::<CFBoolean>().map(|b| bool::from(b).to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let get = |i: usize| items.get(i).and_then(|v| v.clone());
|
||||
|
|
@ -122,14 +138,31 @@ mod imp {
|
|||
let title = get(1);
|
||||
let desc = get(2);
|
||||
let val = get(3);
|
||||
// enabled/focused are CFBoolean not CFString, so they'll be None from downcast
|
||||
// re-read them individually (cheap since it's only 2 attrs)
|
||||
let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true);
|
||||
let focused = copy_bool_attr(el, kAXFocusedAttribute).unwrap_or(false);
|
||||
let enabled = get(4).map(|s| s == "true").unwrap_or(true);
|
||||
let focused = get(5).map(|s| s == "true").unwrap_or(false);
|
||||
|
||||
(role, title, desc, val, enabled, focused)
|
||||
}
|
||||
|
||||
/// Compute the effective display name for any element, mirroring `build_subtree` name resolution.
|
||||
pub fn resolve_element_name(el: &AXElement) -> Option<String> {
|
||||
let ax_role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
|
||||
let name = title.or(desc);
|
||||
let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") {
|
||||
copy_string_attr(el, kAXValueAttribute).or(name)
|
||||
} else {
|
||||
name
|
||||
};
|
||||
|
||||
name.or_else(|| {
|
||||
let children = copy_ax_array(el, kAXChildrenAttribute).unwrap_or_default();
|
||||
label_from_children(&children)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn build_subtree(
|
||||
el: &AXElement,
|
||||
depth: u8,
|
||||
|
|
@ -155,22 +188,29 @@ mod imp {
|
|||
let name = title.clone().or_else(|| ax_desc.clone());
|
||||
let description = if title.is_some() { ax_desc } else { None };
|
||||
|
||||
// AXStaticText stores its visible text in kAXValueAttribute, not title/description
|
||||
let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") {
|
||||
value.clone().or(name)
|
||||
} else {
|
||||
name
|
||||
};
|
||||
|
||||
let mut states = Vec::new();
|
||||
if focused {
|
||||
states.push("focused".into());
|
||||
}
|
||||
if !enabled {
|
||||
states.push("disabled".into());
|
||||
}
|
||||
if focused { states.push("focused".into()); }
|
||||
if !enabled { states.push("disabled".into()); }
|
||||
|
||||
let bounds = if include_bounds { read_bounds(el) } else { None };
|
||||
|
||||
let children = copy_children(el)
|
||||
.unwrap_or_default()
|
||||
// Children fetched before name resolution so we can extract labels from them
|
||||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
|
||||
// Last-resort name: walk immediate children for AXStaticText or AXCell→AXStaticText
|
||||
// This resolves AXRow (sidebar items, list rows) whose label lives in a child text node
|
||||
let name = name.or_else(|| label_from_children(&children_raw));
|
||||
|
||||
let children = children_raw
|
||||
.into_iter()
|
||||
.filter_map(|child| {
|
||||
build_subtree(&child, depth + 1, max_depth, include_bounds, visited)
|
||||
})
|
||||
.filter_map(|child| build_subtree(&child, depth + 1, max_depth, include_bounds, visited))
|
||||
.collect();
|
||||
|
||||
Some(AccessibilityNode {
|
||||
|
|
@ -185,43 +225,48 @@ mod imp {
|
|||
})
|
||||
}
|
||||
|
||||
/// Scan immediate children (and one level deeper through AXCell) to find a text label.
|
||||
/// Resolves the common macOS pattern: AXRow → AXCell → AXStaticText.kAXValue = "label".
|
||||
fn label_from_children(children: &[AXElement]) -> Option<String> {
|
||||
fn text_of(el: &AXElement) -> Option<String> {
|
||||
copy_string_attr(el, kAXValueAttribute)
|
||||
.or_else(|| copy_string_attr(el, kAXTitleAttribute))
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
for child in children.iter().take(5) {
|
||||
match copy_string_attr(child, kAXRoleAttribute).as_deref() {
|
||||
Some("AXStaticText") => {
|
||||
if let Some(s) = text_of(child) { return Some(s); }
|
||||
}
|
||||
Some("AXCell") | Some("AXGroup") => {
|
||||
for gc in copy_ax_array(child, kAXChildrenAttribute).unwrap_or_default() {
|
||||
if copy_string_attr(&gc, kAXRoleAttribute).as_deref() == Some("AXStaticText") {
|
||||
if let Some(s) = text_of(&gc) { return Some(s); }
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Read children using the appropriate attribute for this element's role.
|
||||
/// AXBrowser exposes its content via AXColumns, not AXChildren.
|
||||
fn copy_children(el: &AXElement, ax_role: Option<&str>) -> Option<Vec<AXElement>> {
|
||||
if ax_role == Some("AXBrowser") {
|
||||
return copy_ax_array(el, "AXColumns");
|
||||
}
|
||||
for attr in &[kAXChildrenAttribute, kAXContentsAttribute, "AXChildrenInNavigationOrder"] {
|
||||
if let Some(v) = copy_ax_array(el, attr) {
|
||||
if !v.is_empty() { return Some(v); }
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn copy_string_attr(el: &AXElement, attr: &str) -> Option<String> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut value,
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf_type = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
cf_type.downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
fn copy_bool_attr(el: &AXElement, attr: &str) -> Option<bool> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
&mut value,
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf_type = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
cf_type.downcast::<CFBoolean>().map(|b| b.into())
|
||||
}
|
||||
|
||||
/// Read an array-typed AX attribute, retaining each AXUIElement so they
|
||||
/// stay alive past CFArray deallocation.
|
||||
pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option<Vec<AXElement>> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
|
|
@ -230,37 +275,60 @@ mod imp {
|
|||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf_type = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
cf_type.downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
/// Read kAXValueAttribute handling CFString, CFBoolean, and CFNumber.
|
||||
fn copy_value_typed(el: &AXElement) -> Option<String> {
|
||||
let cf_attr = CFString::new(kAXValueAttribute);
|
||||
let mut val_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut val_ref)
|
||||
};
|
||||
if err != kAXErrorSuccess || val_ref.is_null() { return None; }
|
||||
let cf = unsafe { CFType::wrap_under_create_rule(val_ref) };
|
||||
if let Some(s) = cf.downcast::<CFString>() { return Some(s.to_string()); }
|
||||
if let Some(b) = cf.downcast::<CFBoolean>() { return Some(bool::from(b).to_string()); }
|
||||
if let Some(n) = cf.downcast::<CFNumber>() {
|
||||
if let Some(i) = n.to_i64() { return Some(i.to_string()); }
|
||||
if let Some(f) = n.to_f64() { return Some(format!("{:.2}", f)); }
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn copy_bool_attr(el: &AXElement, attr: &str) -> Option<bool> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() { return None; }
|
||||
let cf_type = unsafe { CFType::wrap_under_create_rule(value) };
|
||||
cf_type.downcast::<CFBoolean>().map(|b| b.into())
|
||||
}
|
||||
|
||||
/// Read an array-typed AX attribute, retaining each AXUIElement.
|
||||
pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option<Vec<AXElement>> {
|
||||
let cf_attr = CFString::new(attr);
|
||||
let mut value: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value)
|
||||
};
|
||||
if err != kAXErrorSuccess || value.is_null() { return None; }
|
||||
let arr = unsafe { CFArray::<CFType>::wrap_under_create_rule(value as _) };
|
||||
let children: Vec<AXElement> = arr
|
||||
.into_iter()
|
||||
.filter_map(|item| {
|
||||
let ptr = item.as_concrete_TypeRef() as AXUIElementRef;
|
||||
if ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(AXElement(ptr))
|
||||
}
|
||||
if ptr.is_null() { return None; }
|
||||
unsafe { CFRetain(ptr as CFTypeRef) };
|
||||
Some(AXElement(ptr))
|
||||
})
|
||||
.collect();
|
||||
Some(children)
|
||||
}
|
||||
|
||||
fn copy_children(el: &AXElement) -> Option<Vec<AXElement>> {
|
||||
for attr in &[
|
||||
kAXChildrenAttribute,
|
||||
kAXContentsAttribute,
|
||||
"AXChildrenInNavigationOrder",
|
||||
] {
|
||||
if let Some(v) = copy_ax_array(el, attr) {
|
||||
if !v.is_empty() {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn read_bounds(el: &AXElement) -> Option<Rect> {
|
||||
use accessibility_sys::{
|
||||
kAXPositionAttribute, kAXSizeAttribute, AXValueGetValue,
|
||||
|
|
@ -274,34 +342,28 @@ mod imp {
|
|||
let pos_ok = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, pos_cf.as_concrete_TypeRef(), &mut pos_ref)
|
||||
};
|
||||
if pos_ok != kAXErrorSuccess || pos_ref.is_null() {
|
||||
return None;
|
||||
}
|
||||
if pos_ok != kAXErrorSuccess || pos_ref.is_null() { return None; }
|
||||
|
||||
let mut point = CGPoint::new(0.0, 0.0);
|
||||
let got_pos = unsafe {
|
||||
AXValueGetValue(pos_ref as _, kAXValueTypeCGPoint, &mut point as *mut _ as *mut c_void)
|
||||
};
|
||||
unsafe { CFRelease(pos_ref) };
|
||||
if !got_pos {
|
||||
return None;
|
||||
}
|
||||
if !got_pos { return None; }
|
||||
|
||||
let size_cf = CFString::new(kAXSizeAttribute);
|
||||
let mut size_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let size_ok = unsafe {
|
||||
AXUIElementCopyAttributeValue(el.0, size_cf.as_concrete_TypeRef(), &mut size_ref)
|
||||
};
|
||||
if size_ok != kAXErrorSuccess || size_ref.is_null() {
|
||||
return None;
|
||||
}
|
||||
if size_ok != kAXErrorSuccess || size_ref.is_null() { return None; }
|
||||
|
||||
let mut size = CGSize::new(0.0, 0.0);
|
||||
let got_size = unsafe {
|
||||
AXValueGetValue(size_ref as _, kAXValueTypeCGSize, &mut size as *mut _ as *mut c_void)
|
||||
};
|
||||
unsafe { CFRelease(size_ref) };
|
||||
if !got_size {
|
||||
return None;
|
||||
}
|
||||
if !got_size { return None; }
|
||||
|
||||
Some(Rect { x: point.x, y: point.y, width: size.width, height: size.height })
|
||||
}
|
||||
|
|
@ -313,45 +375,20 @@ mod imp {
|
|||
|
||||
pub struct AXElement(pub(crate) *const std::ffi::c_void);
|
||||
|
||||
impl Drop for AXElement {
|
||||
fn drop(&mut self) {}
|
||||
}
|
||||
impl Drop for AXElement { fn drop(&mut self) {} }
|
||||
impl Clone for AXElement { fn clone(&self) -> Self { AXElement(self.0) } }
|
||||
|
||||
impl Clone for AXElement {
|
||||
fn clone(&self) -> Self {
|
||||
AXElement(self.0)
|
||||
}
|
||||
}
|
||||
pub fn element_for_pid(_pid: i32) -> AXElement { AXElement(std::ptr::null()) }
|
||||
pub fn window_element_for(_pid: i32, _win_title: &str) -> AXElement { AXElement(std::ptr::null()) }
|
||||
pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option<Vec<AXElement>> { None }
|
||||
pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option<String> { None }
|
||||
pub fn read_bounds(_el: &AXElement) -> Option<agent_desktop_core::node::Rect> { None }
|
||||
pub fn resolve_element_name(_el: &AXElement) -> Option<String> { None }
|
||||
|
||||
pub fn element_for_pid(_pid: i32) -> AXElement {
|
||||
AXElement(std::ptr::null())
|
||||
}
|
||||
|
||||
pub fn window_element_for(_pid: i32, _win_title: &str) -> AXElement {
|
||||
AXElement(std::ptr::null())
|
||||
}
|
||||
|
||||
pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option<Vec<AXElement>> {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn build_subtree(
|
||||
_el: &AXElement,
|
||||
_depth: u8,
|
||||
_max_depth: u8,
|
||||
_include_bounds: bool,
|
||||
_visited: &mut FxHashSet<usize>,
|
||||
) -> Option<AccessibilityNode> {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn read_bounds(_el: &AXElement) -> Option<agent_desktop_core::node::Rect> {
|
||||
None
|
||||
}
|
||||
pub fn build_subtree(_el: &AXElement, _depth: u8, _max_depth: u8, _include_bounds: bool, _visited: &mut FxHashSet<usize>) -> Option<AccessibilityNode> { None }
|
||||
}
|
||||
|
||||
pub use imp::{build_subtree, copy_ax_array, copy_string_attr, element_for_pid, read_bounds, window_element_for, AXElement};
|
||||
pub use imp::{
|
||||
build_subtree, copy_ax_array, copy_string_attr, element_for_pid, read_bounds,
|
||||
resolve_element_name, window_element_for, AXElement,
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue