mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-08-09 08:27:25 +00:00
feat: implement progressive skeleton traversal with ref-rooted drill-down
Add --skeleton and --root flags to snapshot command for token-efficient accessibility tree exploration. Skeleton mode clamps depth to 3 levels and annotates truncated containers with children_count, allowing AI agents to discover regions before drilling into them. Named containers at skeleton boundaries (via name or description) receive refs as drill-down targets. The --root flag starts traversal from a previous ref with scoped invalidation — only refs from that drill-down are replaced on re-drill. Key changes: - New ref_alloc.rs: shared ref helpers (INTERACTIVE_ROLES, actions_for_role, ref_entry_from_node, is_collapsible) extracted from snapshot.rs - New snapshot_ref.rs: drill-down logic with DrillDownConfig, scoped invalidation via root_ref tagging on RefEntry - macOS count_children() uses raw CFArrayGetCount without materializing AXElement wrappers for performance at skeleton boundaries - RefMap write-side size check prevents >1MB files - Skeleton anchors consider both name and description for Electron compat
This commit is contained in:
parent
b6a15ae0e6
commit
b13dc69afd
12 changed files with 562 additions and 153 deletions
|
|
@ -1,7 +1,7 @@
|
|||
use crate::{
|
||||
adapter::{PlatformAdapter, SnapshotSurface},
|
||||
error::AppError,
|
||||
snapshot,
|
||||
snapshot, snapshot_ref,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
|
|
@ -27,8 +27,14 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Valu
|
|||
args.compact
|
||||
);
|
||||
|
||||
let effective_depth = if args.skeleton {
|
||||
args.max_depth.min(3)
|
||||
} else {
|
||||
args.max_depth
|
||||
};
|
||||
|
||||
let opts = crate::adapter::TreeOptions {
|
||||
max_depth: args.max_depth,
|
||||
max_depth: effective_depth,
|
||||
include_bounds: args.include_bounds,
|
||||
interactive_only: args.interactive_only,
|
||||
compact: args.compact,
|
||||
|
|
@ -37,6 +43,15 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Valu
|
|||
root_ref: args.root_ref.clone(),
|
||||
};
|
||||
|
||||
if let Some(ref root) = args.root_ref {
|
||||
if !matches!(args.surface, SnapshotSurface::Window) {
|
||||
return Err(AppError::invalid_input(
|
||||
"--root cannot be combined with --surface",
|
||||
));
|
||||
}
|
||||
return format_result(snapshot_ref::run_from_ref(adapter, &opts, root)?);
|
||||
}
|
||||
|
||||
let result = snapshot::run(
|
||||
adapter,
|
||||
&opts,
|
||||
|
|
@ -44,6 +59,10 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Valu
|
|||
args.window_id.as_deref(),
|
||||
)?;
|
||||
|
||||
format_result(result)
|
||||
}
|
||||
|
||||
fn format_result(result: snapshot::SnapshotResult) -> Result<Value, AppError> {
|
||||
let ref_count = result.refmap.len();
|
||||
let tree = serde_json::to_value(&result.tree)?;
|
||||
let win = &result.window;
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ pub mod hints;
|
|||
pub mod node;
|
||||
pub mod notification;
|
||||
pub mod output;
|
||||
pub mod ref_alloc;
|
||||
pub mod refs;
|
||||
pub mod snapshot;
|
||||
pub mod snapshot_ref;
|
||||
|
||||
pub use action::{
|
||||
Action, ActionResult, Direction, DragParams, ElementState, KeyCombo, Modifier, MouseButton,
|
||||
|
|
|
|||
63
crates/core/src/ref_alloc.rs
Normal file
63
crates/core/src/ref_alloc.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
use crate::node::AccessibilityNode;
|
||||
use crate::refs::RefEntry;
|
||||
|
||||
pub(crate) const INTERACTIVE_ROLES: &[&str] = &[
|
||||
"button",
|
||||
"textfield",
|
||||
"checkbox",
|
||||
"link",
|
||||
"menuitem",
|
||||
"tab",
|
||||
"slider",
|
||||
"combobox",
|
||||
"treeitem",
|
||||
"cell",
|
||||
"radiobutton",
|
||||
"incrementor",
|
||||
"menubutton",
|
||||
"switch",
|
||||
"colorwell",
|
||||
"dockitem",
|
||||
];
|
||||
|
||||
pub(crate) fn actions_for_role(role: &str) -> Vec<String> {
|
||||
match role {
|
||||
"button" | "link" | "menuitem" | "tab" | "radiobutton" => vec!["Click".into()],
|
||||
"textfield" | "incrementor" => vec!["Click".into(), "SetValue".into(), "SetFocus".into()],
|
||||
"checkbox" => vec!["Click".into(), "Toggle".into()],
|
||||
"combobox" => vec!["Click".into(), "Select".into()],
|
||||
"treeitem" => vec!["Click".into(), "Expand".into(), "Collapse".into()],
|
||||
"slider" => vec!["SetValue".into()],
|
||||
"cell" => vec!["Click".into()],
|
||||
_ => vec!["Click".into()],
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn ref_entry_from_node(
|
||||
node: &AccessibilityNode,
|
||||
pid: i32,
|
||||
source_app: Option<&str>,
|
||||
root_ref: Option<String>,
|
||||
) -> RefEntry {
|
||||
RefEntry {
|
||||
pid,
|
||||
role: node.role.clone(),
|
||||
name: node.name.clone(),
|
||||
value: node.value.clone(),
|
||||
states: node.states.clone(),
|
||||
bounds: node.bounds,
|
||||
bounds_hash: node.bounds.as_ref().map(|b| b.bounds_hash()),
|
||||
available_actions: actions_for_role(&node.role),
|
||||
source_app: source_app.map(str::to_string),
|
||||
root_ref,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_collapsible(node: &AccessibilityNode) -> bool {
|
||||
node.ref_id.is_none()
|
||||
&& node.name.as_deref().is_none_or(str::is_empty)
|
||||
&& node.value.as_deref().is_none_or(str::is_empty)
|
||||
&& node.description.as_deref().is_none_or(str::is_empty)
|
||||
&& node.states.is_empty()
|
||||
&& node.children.len() == 1
|
||||
}
|
||||
|
|
@ -57,9 +57,8 @@ impl RefMap {
|
|||
}
|
||||
|
||||
pub fn remove_by_root_ref(&mut self, root: &str) {
|
||||
self.inner.retain(|_, entry| {
|
||||
entry.root_ref.as_deref() != Some(root)
|
||||
});
|
||||
self.inner
|
||||
.retain(|_, entry| entry.root_ref.as_deref() != Some(root));
|
||||
}
|
||||
|
||||
pub fn save(&self) -> Result<(), AppError> {
|
||||
|
|
|
|||
|
|
@ -2,28 +2,10 @@ use crate::{
|
|||
adapter::{PlatformAdapter, SnapshotSurface, TreeOptions, WindowFilter},
|
||||
error::AppError,
|
||||
node::{AccessibilityNode, WindowInfo},
|
||||
refs::{RefEntry, RefMap},
|
||||
ref_alloc::{is_collapsible, ref_entry_from_node, INTERACTIVE_ROLES},
|
||||
refs::RefMap,
|
||||
};
|
||||
|
||||
const INTERACTIVE_ROLES: &[&str] = &[
|
||||
"button",
|
||||
"textfield",
|
||||
"checkbox",
|
||||
"link",
|
||||
"menuitem",
|
||||
"tab",
|
||||
"slider",
|
||||
"combobox",
|
||||
"treeitem",
|
||||
"cell",
|
||||
"radiobutton",
|
||||
"incrementor",
|
||||
"menubutton",
|
||||
"switch",
|
||||
"colorwell",
|
||||
"dockitem",
|
||||
];
|
||||
|
||||
pub struct SnapshotResult {
|
||||
pub tree: AccessibilityNode,
|
||||
pub refmap: RefMap,
|
||||
|
|
@ -148,15 +130,6 @@ pub fn append_surface_refs(
|
|||
Some(tree)
|
||||
}
|
||||
|
||||
fn is_collapsible(node: &AccessibilityNode) -> bool {
|
||||
node.ref_id.is_none()
|
||||
&& node.name.as_deref().is_none_or(str::is_empty)
|
||||
&& node.value.as_deref().is_none_or(str::is_empty)
|
||||
&& node.description.as_deref().is_none_or(str::is_empty)
|
||||
&& node.states.is_empty()
|
||||
&& node.children.len() == 1
|
||||
}
|
||||
|
||||
fn allocate_refs(
|
||||
mut node: AccessibilityNode,
|
||||
refmap: &mut RefMap,
|
||||
|
|
@ -169,18 +142,17 @@ fn allocate_refs(
|
|||
let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str());
|
||||
|
||||
if is_interactive {
|
||||
let entry = RefEntry {
|
||||
pid: window_pid,
|
||||
role: node.role.clone(),
|
||||
name: node.name.clone(),
|
||||
value: node.value.clone(),
|
||||
states: node.states.clone(),
|
||||
bounds: node.bounds,
|
||||
bounds_hash: node.bounds.as_ref().map(|b| b.bounds_hash()),
|
||||
available_actions: actions_for_role(&node.role),
|
||||
source_app: source_app.map(str::to_string),
|
||||
root_ref: None,
|
||||
};
|
||||
let entry = ref_entry_from_node(&node, window_pid, source_app, None);
|
||||
node.ref_id = Some(refmap.allocate(entry));
|
||||
}
|
||||
|
||||
let has_label = node.name.as_deref().is_some_and(|n| !n.is_empty())
|
||||
|| node.description.as_deref().is_some_and(|d| !d.is_empty());
|
||||
let is_skeleton_anchor = !is_interactive && node.children_count.is_some() && has_label;
|
||||
|
||||
if is_skeleton_anchor {
|
||||
let mut entry = ref_entry_from_node(&node, window_pid, source_app, None);
|
||||
entry.available_actions = vec![];
|
||||
node.ref_id = Some(refmap.allocate(entry));
|
||||
}
|
||||
|
||||
|
|
@ -204,7 +176,11 @@ fn allocate_refs(
|
|||
if compact && is_collapsible(&child) {
|
||||
return child.children.into_iter().next();
|
||||
}
|
||||
if interactive_only && child.ref_id.is_none() && child.children.is_empty() {
|
||||
if interactive_only
|
||||
&& child.ref_id.is_none()
|
||||
&& child.children.is_empty()
|
||||
&& child.children_count.is_none()
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(child)
|
||||
|
|
@ -215,19 +191,6 @@ fn allocate_refs(
|
|||
node
|
||||
}
|
||||
|
||||
fn actions_for_role(role: &str) -> Vec<String> {
|
||||
match role {
|
||||
"button" | "link" | "menuitem" | "tab" | "radiobutton" => vec!["Click".into()],
|
||||
"textfield" | "incrementor" => vec!["Click".into(), "SetValue".into(), "SetFocus".into()],
|
||||
"checkbox" => vec!["Click".into(), "Toggle".into()],
|
||||
"combobox" => vec!["Click".into(), "Select".into()],
|
||||
"treeitem" => vec!["Click".into(), "Expand".into(), "Collapse".into()],
|
||||
"slider" => vec!["SetValue".into()],
|
||||
"cell" => vec!["Click".into()],
|
||||
_ => vec!["Click".into()],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -353,4 +316,67 @@ mod tests {
|
|||
assert_eq!(result.children[0].role, "button");
|
||||
assert!(result.children[0].ref_id.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skeleton_named_container_gets_ref() {
|
||||
let mut container = node("group");
|
||||
container.name = Some("Sidebar".into());
|
||||
container.children_count = Some(5);
|
||||
let mut root = node("window");
|
||||
root.children = vec![container];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let result = allocate_refs(root, &mut refmap, false, false, false, 1, Some("Test"));
|
||||
|
||||
assert!(result.children[0].ref_id.is_some());
|
||||
assert_eq!(refmap.len(), 1);
|
||||
let entry = refmap
|
||||
.get(result.children[0].ref_id.as_deref().unwrap())
|
||||
.unwrap();
|
||||
assert!(entry.available_actions.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skeleton_unnamed_container_no_ref() {
|
||||
let mut container = node("group");
|
||||
container.children_count = Some(5);
|
||||
let mut root = node("window");
|
||||
root.children = vec![container];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let result = allocate_refs(root, &mut refmap, false, false, false, 1, Some("Test"));
|
||||
|
||||
assert!(result.children[0].ref_id.is_none());
|
||||
assert_eq!(refmap.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skeleton_described_container_gets_ref() {
|
||||
let mut container = node("group");
|
||||
container.description = Some("Channels and direct messages".into());
|
||||
container.children_count = Some(12);
|
||||
let mut root = node("window");
|
||||
root.children = vec![container];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let result = allocate_refs(root, &mut refmap, false, false, false, 1, Some("Test"));
|
||||
|
||||
assert!(result.children[0].ref_id.is_some());
|
||||
assert_eq!(refmap.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skeleton_truncated_node_survives_interactive_only() {
|
||||
let mut container = node("group");
|
||||
container.name = Some("Content".into());
|
||||
container.children_count = Some(10);
|
||||
let mut root = node("window");
|
||||
root.children = vec![container];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let result = allocate_refs(root, &mut refmap, false, true, false, 1, Some("Test"));
|
||||
|
||||
assert_eq!(result.children.len(), 1);
|
||||
assert_eq!(result.children[0].children_count, Some(10));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
202
crates/core/src/snapshot_ref.rs
Normal file
202
crates/core/src/snapshot_ref.rs
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
use crate::{
|
||||
adapter::{PlatformAdapter, TreeOptions},
|
||||
error::AppError,
|
||||
node::{AccessibilityNode, WindowInfo},
|
||||
ref_alloc::{is_collapsible, ref_entry_from_node, INTERACTIVE_ROLES},
|
||||
refs::RefMap,
|
||||
snapshot::SnapshotResult,
|
||||
};
|
||||
|
||||
struct DrillDownConfig<'a> {
|
||||
include_bounds: bool,
|
||||
interactive_only: bool,
|
||||
compact: bool,
|
||||
pid: i32,
|
||||
source_app: Option<&'a str>,
|
||||
root_ref_id: &'a str,
|
||||
}
|
||||
|
||||
pub fn run_from_ref(
|
||||
adapter: &dyn PlatformAdapter,
|
||||
opts: &TreeOptions,
|
||||
root_ref_id: &str,
|
||||
) -> Result<SnapshotResult, AppError> {
|
||||
let mut refmap = RefMap::load()?;
|
||||
|
||||
let entry = refmap
|
||||
.get(root_ref_id)
|
||||
.ok_or_else(|| AppError::stale_ref(root_ref_id))?
|
||||
.clone();
|
||||
|
||||
let handle = adapter.resolve_element(&entry)?;
|
||||
|
||||
let raw_tree = adapter.get_subtree(&handle, opts)?;
|
||||
|
||||
refmap.remove_by_root_ref(root_ref_id);
|
||||
|
||||
let config = DrillDownConfig {
|
||||
include_bounds: opts.include_bounds,
|
||||
interactive_only: opts.interactive_only,
|
||||
compact: opts.compact,
|
||||
pid: entry.pid,
|
||||
source_app: entry.source_app.as_deref(),
|
||||
root_ref_id,
|
||||
};
|
||||
|
||||
let mut tree = allocate_refs_with_root(raw_tree, &mut refmap, &config);
|
||||
|
||||
crate::hints::add_structural_hints(&mut tree);
|
||||
|
||||
refmap.save()?;
|
||||
|
||||
let window = WindowInfo {
|
||||
id: String::new(),
|
||||
title: format!("subtree from {root_ref_id}"),
|
||||
app: entry.source_app.unwrap_or_default(),
|
||||
pid: entry.pid,
|
||||
bounds: None,
|
||||
is_focused: true,
|
||||
};
|
||||
|
||||
Ok(SnapshotResult {
|
||||
tree,
|
||||
refmap,
|
||||
window,
|
||||
})
|
||||
}
|
||||
|
||||
fn allocate_refs_with_root(
|
||||
mut node: AccessibilityNode,
|
||||
refmap: &mut RefMap,
|
||||
config: &DrillDownConfig,
|
||||
) -> AccessibilityNode {
|
||||
let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str());
|
||||
|
||||
if is_interactive {
|
||||
let entry = ref_entry_from_node(
|
||||
&node,
|
||||
config.pid,
|
||||
config.source_app,
|
||||
Some(config.root_ref_id.to_string()),
|
||||
);
|
||||
node.ref_id = Some(refmap.allocate(entry));
|
||||
}
|
||||
|
||||
if !config.include_bounds {
|
||||
node.bounds = None;
|
||||
}
|
||||
|
||||
node.children = node
|
||||
.children
|
||||
.into_iter()
|
||||
.filter_map(|child| {
|
||||
let child = allocate_refs_with_root(child, refmap, config);
|
||||
if config.compact && is_collapsible(&child) {
|
||||
return child.children.into_iter().next();
|
||||
}
|
||||
if config.interactive_only && child.ref_id.is_none() && child.children.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(child)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
node
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::node::AccessibilityNode;
|
||||
|
||||
fn node(role: &str) -> AccessibilityNode {
|
||||
AccessibilityNode {
|
||||
ref_id: None,
|
||||
role: role.into(),
|
||||
name: None,
|
||||
value: None,
|
||||
description: None,
|
||||
hint: None,
|
||||
states: vec![],
|
||||
bounds: None,
|
||||
children_count: None,
|
||||
children: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allocate_refs_with_root_tags_entries() {
|
||||
let mut btn = node("button");
|
||||
btn.name = Some("Submit".into());
|
||||
let mut root = node("group");
|
||||
root.children = vec![btn];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let config = DrillDownConfig {
|
||||
include_bounds: false,
|
||||
interactive_only: false,
|
||||
compact: false,
|
||||
pid: 42,
|
||||
source_app: Some("TestApp"),
|
||||
root_ref_id: "@e5",
|
||||
};
|
||||
let tree = allocate_refs_with_root(root, &mut refmap, &config);
|
||||
|
||||
assert_eq!(refmap.len(), 1);
|
||||
let btn_ref = tree.children[0]
|
||||
.ref_id
|
||||
.as_deref()
|
||||
.expect("button should have ref");
|
||||
let entry = refmap.get(btn_ref).expect("entry should exist");
|
||||
assert_eq!(entry.root_ref.as_deref(), Some("@e5"));
|
||||
assert_eq!(entry.pid, 42);
|
||||
assert_eq!(entry.source_app.as_deref(), Some("TestApp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allocate_refs_with_root_respects_interactive_only() {
|
||||
let btn = node("button");
|
||||
let text = node("statictext");
|
||||
let mut root = node("group");
|
||||
root.children = vec![btn, text];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let config = DrillDownConfig {
|
||||
include_bounds: false,
|
||||
interactive_only: true,
|
||||
compact: false,
|
||||
pid: 1,
|
||||
source_app: None,
|
||||
root_ref_id: "@e1",
|
||||
};
|
||||
let tree = allocate_refs_with_root(root, &mut refmap, &config);
|
||||
|
||||
assert_eq!(tree.children.len(), 1);
|
||||
assert_eq!(tree.children[0].role, "button");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allocate_refs_with_root_compact() {
|
||||
let mut btn = node("button");
|
||||
btn.name = Some("OK".into());
|
||||
let mut wrapper = node("group");
|
||||
wrapper.children = vec![btn];
|
||||
let mut root = node("window");
|
||||
root.children = vec![wrapper];
|
||||
|
||||
let mut refmap = RefMap::new();
|
||||
let config = DrillDownConfig {
|
||||
include_bounds: false,
|
||||
interactive_only: false,
|
||||
compact: true,
|
||||
pid: 1,
|
||||
source_app: None,
|
||||
root_ref_id: "@e1",
|
||||
};
|
||||
let tree = allocate_refs_with_root(root, &mut refmap, &config);
|
||||
|
||||
assert_eq!(tree.children.len(), 1);
|
||||
assert_eq!(tree.children[0].role, "button");
|
||||
}
|
||||
}
|
||||
|
|
@ -51,8 +51,15 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
.ok_or_else(|| AdapterError::element_not_found("No open alert or dialog"))?,
|
||||
};
|
||||
let mut visited = FxHashSet::default();
|
||||
crate::tree::build_subtree(&el, 0, opts.max_depth, opts.include_bounds, &mut visited)
|
||||
.ok_or_else(|| AdapterError::internal("Empty AX tree for surface"))
|
||||
crate::tree::build_subtree(
|
||||
&el,
|
||||
0,
|
||||
opts.max_depth,
|
||||
opts.include_bounds,
|
||||
&mut visited,
|
||||
opts.skeleton,
|
||||
)
|
||||
.ok_or_else(|| AdapterError::internal("Empty AX tree for surface"))
|
||||
}
|
||||
|
||||
fn execute_action(
|
||||
|
|
@ -72,7 +79,7 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
}
|
||||
|
||||
fn list_apps(&self) -> Result<Vec<AppInfo>, AdapterError> {
|
||||
list_apps_impl()
|
||||
crate::system::app_ops::list_apps_impl()
|
||||
}
|
||||
|
||||
fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError> {
|
||||
|
|
@ -205,6 +212,35 @@ impl PlatformAdapter for MacOSAdapter {
|
|||
) -> Result<ActionResult, AdapterError> {
|
||||
crate::notifications::actions::notification_action(index, action_name)
|
||||
}
|
||||
|
||||
fn get_subtree(
|
||||
&self,
|
||||
handle: &NativeHandle,
|
||||
opts: &TreeOptions,
|
||||
) -> Result<AccessibilityNode, AdapterError> {
|
||||
use crate::tree::AXElement;
|
||||
use std::mem::ManuallyDrop;
|
||||
|
||||
let el = ManuallyDrop::new(AXElement(
|
||||
handle.as_raw() as accessibility_sys::AXUIElementRef
|
||||
));
|
||||
let mut ancestors = FxHashSet::default();
|
||||
crate::tree::build_subtree(
|
||||
&el,
|
||||
0,
|
||||
opts.max_depth,
|
||||
opts.include_bounds,
|
||||
&mut ancestors,
|
||||
false,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::ElementNotFound,
|
||||
"Element no longer exists in accessibility tree",
|
||||
)
|
||||
.with_suggestion("Run 'snapshot' to refresh refs, then retry.")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
|
|
@ -318,80 +354,3 @@ pub(crate) fn list_windows_impl(filter: &WindowFilter) -> Result<Vec<WindowInfo>
|
|||
Err(AdapterError::not_supported("list_windows"))
|
||||
}
|
||||
}
|
||||
|
||||
fn list_apps_impl() -> Result<Vec<AppInfo>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use core_foundation::base::{CFType, TCFType};
|
||||
use core_foundation::number::CFNumber;
|
||||
use core_foundation::string::CFString;
|
||||
use core_foundation_sys::dictionary::CFDictionaryGetValue;
|
||||
use core_graphics::display::CGDisplay;
|
||||
use core_graphics::window::{
|
||||
kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowOwnerName, kCGWindowOwnerPID,
|
||||
};
|
||||
|
||||
let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) {
|
||||
Some(a) => a,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let mut seen_pids = std::collections::HashSet::new();
|
||||
let mut apps = Vec::new();
|
||||
|
||||
for raw in arr.get_all_values() {
|
||||
if raw.is_null() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let layer = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowLayer as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.unwrap_or(99)
|
||||
};
|
||||
if layer != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let pid = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerPID as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.unwrap_or(0) as i32
|
||||
};
|
||||
if !seen_pids.insert(pid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerName as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFString>()
|
||||
.map(|s| s.to_string())
|
||||
};
|
||||
|
||||
if let Some(n) = name {
|
||||
apps.push(AppInfo {
|
||||
name: n,
|
||||
pid,
|
||||
bundle_id: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(apps)
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
Err(AdapterError::not_supported("list_apps"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo};
|
||||
use agent_desktop_core::{
|
||||
adapter::WindowFilter,
|
||||
error::AdapterError,
|
||||
node::{AppInfo, WindowInfo},
|
||||
};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn pid_from_element(el: &crate::tree::AXElement) -> Option<i32> {
|
||||
|
|
@ -234,3 +238,80 @@ fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool {
|
|||
pub fn close_app_impl(_id: &str, _force: bool) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("close_app"))
|
||||
}
|
||||
|
||||
pub fn list_apps_impl() -> Result<Vec<AppInfo>, AdapterError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
use core_foundation::base::{CFType, TCFType};
|
||||
use core_foundation::number::CFNumber;
|
||||
use core_foundation::string::CFString;
|
||||
use core_foundation_sys::dictionary::CFDictionaryGetValue;
|
||||
use core_graphics::display::CGDisplay;
|
||||
use core_graphics::window::{
|
||||
kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowOwnerName, kCGWindowOwnerPID,
|
||||
};
|
||||
|
||||
let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) {
|
||||
Some(a) => a,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let mut seen_pids = std::collections::HashSet::new();
|
||||
let mut apps = Vec::new();
|
||||
|
||||
for raw in arr.get_all_values() {
|
||||
if raw.is_null() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let layer = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowLayer as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.unwrap_or(99)
|
||||
};
|
||||
if layer != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let pid = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerPID as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFNumber>()
|
||||
.and_then(|n| n.to_i64())
|
||||
.unwrap_or(0) as i32
|
||||
};
|
||||
if !seen_pids.insert(pid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = unsafe {
|
||||
let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerName as _);
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
CFType::wrap_under_get_rule(v as _)
|
||||
.downcast::<CFString>()
|
||||
.map(|s| s.to_string())
|
||||
};
|
||||
|
||||
if let Some(n) = name {
|
||||
apps.push(AppInfo {
|
||||
name: n,
|
||||
pid,
|
||||
bundle_id: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(apps)
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
Err(AdapterError::not_supported("list_apps"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ use agent_desktop_core::node::AccessibilityNode;
|
|||
use rustc_hash::FxHashSet;
|
||||
|
||||
use super::element::{
|
||||
copy_ax_array, copy_string_attr, element_for_pid, fetch_node_attrs, read_bounds, AXElement,
|
||||
ABSOLUTE_MAX_DEPTH,
|
||||
copy_ax_array, copy_string_attr, count_children, element_for_pid, fetch_node_attrs,
|
||||
read_bounds, AXElement, ABSOLUTE_MAX_DEPTH,
|
||||
};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
|
|
@ -47,6 +47,7 @@ pub fn build_subtree(
|
|||
max_depth: u8,
|
||||
_include_bounds: bool,
|
||||
ancestors: &mut FxHashSet<usize>,
|
||||
skeleton: bool,
|
||||
) -> Option<AccessibilityNode> {
|
||||
if depth > max_depth || depth >= ABSOLUTE_MAX_DEPTH {
|
||||
return None;
|
||||
|
|
@ -83,11 +84,6 @@ pub fn build_subtree(
|
|||
|
||||
let bounds = read_bounds(el);
|
||||
|
||||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
let name = name.or_else(|| label_from_children(&children_raw));
|
||||
|
||||
// Non-semantic groups inside web content don't cost depth budget.
|
||||
// A nameless AXGroup/AXGenericElement is just a <div> wrapper — skip it.
|
||||
let is_web_wrapper = matches!(
|
||||
ax_role.as_deref(),
|
||||
Some("AXGroup") | Some("AXGenericElement")
|
||||
|
|
@ -96,10 +92,49 @@ pub fn build_subtree(
|
|||
|
||||
let child_depth = if is_web_wrapper { depth } else { depth + 1 };
|
||||
|
||||
let at_skeleton_boundary =
|
||||
skeleton && child_depth > max_depth && child_depth < ABSOLUTE_MAX_DEPTH;
|
||||
|
||||
if at_skeleton_boundary {
|
||||
let child_count = count_children(el);
|
||||
let children_count = if child_count > 0 {
|
||||
Some(child_count)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let name = name.or_else(|| {
|
||||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
label_from_children(&children_raw)
|
||||
});
|
||||
ancestors.remove(&ptr_key);
|
||||
return Some(AccessibilityNode {
|
||||
ref_id: None,
|
||||
role,
|
||||
name,
|
||||
value,
|
||||
description,
|
||||
hint: None,
|
||||
states,
|
||||
bounds,
|
||||
children_count,
|
||||
children: vec![],
|
||||
});
|
||||
}
|
||||
|
||||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
let name = name.or_else(|| label_from_children(&children_raw));
|
||||
|
||||
let children = children_raw
|
||||
.into_iter()
|
||||
.filter_map(|child| {
|
||||
build_subtree(&child, child_depth, max_depth, _include_bounds, ancestors)
|
||||
build_subtree(
|
||||
&child,
|
||||
child_depth,
|
||||
max_depth,
|
||||
_include_bounds,
|
||||
ancestors,
|
||||
skeleton,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
|
|
@ -189,6 +224,7 @@ pub fn build_subtree(
|
|||
_max_depth: u8,
|
||||
_include_bounds: bool,
|
||||
_visited: &mut FxHashSet<usize>,
|
||||
_skeleton: bool,
|
||||
) -> Option<AccessibilityNode> {
|
||||
None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -238,6 +238,21 @@ mod imp {
|
|||
Some(AXElement(ptr))
|
||||
}
|
||||
|
||||
pub fn count_children(element: &AXElement) -> u32 {
|
||||
unsafe {
|
||||
let mut value: core_foundation::base::CFTypeRef = std::ptr::null();
|
||||
let attr = CFString::from_static_string("AXChildren");
|
||||
let err =
|
||||
AXUIElementCopyAttributeValue(element.0, attr.as_concrete_TypeRef(), &mut value);
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return 0;
|
||||
}
|
||||
let count = core_foundation_sys::array::CFArrayGetCount(value as _);
|
||||
CFRelease(value);
|
||||
count as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub fn read_bounds(el: &AXElement) -> Option<Rect> {
|
||||
use accessibility_sys::{
|
||||
kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint, kAXValueTypeCGSize,
|
||||
|
|
@ -337,6 +352,9 @@ mod imp {
|
|||
pub fn copy_element_attr(_el: &AXElement, _attr: &str) -> Option<AXElement> {
|
||||
None
|
||||
}
|
||||
pub fn count_children(_element: &AXElement) -> u32 {
|
||||
0
|
||||
}
|
||||
pub fn read_bounds(_el: &AXElement) -> Option<Rect> {
|
||||
None
|
||||
}
|
||||
|
|
@ -362,5 +380,6 @@ mod imp {
|
|||
|
||||
pub use imp::{
|
||||
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed,
|
||||
element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
|
||||
count_children, element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name,
|
||||
AXElement,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ pub mod surfaces;
|
|||
|
||||
pub use builder::{build_subtree, window_element_for};
|
||||
pub use element::{
|
||||
copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, element_for_pid,
|
||||
read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
|
||||
copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, count_children,
|
||||
element_for_pid, read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
|
||||
};
|
||||
pub use resolve::{find_element_recursive, resolve_element_impl};
|
||||
pub use roles::{ax_role_to_str, is_interactive_role};
|
||||
|
|
|
|||
|
|
@ -50,7 +50,10 @@ pub struct SnapshotArgs {
|
|||
pub compact: bool,
|
||||
#[arg(long, value_enum, default_value_t = Surface::Window, help = "Surface to snapshot")]
|
||||
pub surface: Surface,
|
||||
#[arg(long, help = "Shallow overview with children_count on truncated containers")]
|
||||
#[arg(
|
||||
long,
|
||||
help = "Shallow overview with children_count on truncated containers"
|
||||
)]
|
||||
pub skeleton: bool,
|
||||
#[arg(long, help = "Start traversal from this ref instead of window root")]
|
||||
pub root: Option<String>,
|
||||
|
|
|
|||
Loading…
Reference in a new issue