mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-26 17:12:15 +00:00
feat: add electron/web app compatibility for accessibility tree traversal
Skip depth budget for non-semantic AXGroup/AXGenericElement wrappers with empty name and value, allowing default --max-depth 10 to find 100+ refs in Electron apps like Slack and VS Code (previously found only 3). Raise resolver search depth from 20 to ABSOLUTE_MAX_DEPTH (50) so ref resolution succeeds for deeply nested Electron elements. Fix surface detection to check if focused window itself is the target surface (Electron reports dialogs as focused window, not as children) and check both AXRole and AXSubrole for surface matching. Update Phase 2 (Windows) and Phase 3 (Linux) docs with equivalent web/Electron compatibility patterns for their respective APIs.
This commit is contained in:
parent
4d9daeadce
commit
a19c1b5132
4 changed files with 113 additions and 13 deletions
|
|
@ -86,9 +86,21 @@ pub fn build_subtree(
|
|||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
let name = name.or_else(|| label_from_children(&children_raw));
|
||||
|
||||
// Non-semantic groups inside web content don't cost depth budget.
|
||||
// A nameless AXGroup/AXGenericElement is just a <div> wrapper — skip it.
|
||||
let is_web_wrapper = matches!(
|
||||
ax_role.as_deref(),
|
||||
Some("AXGroup") | Some("AXGenericElement")
|
||||
) && title.as_deref().is_none_or(str::is_empty)
|
||||
&& value.as_deref().is_none_or(str::is_empty);
|
||||
|
||||
let child_depth = if is_web_wrapper { depth } else { depth + 1 };
|
||||
|
||||
let children = children_raw
|
||||
.into_iter()
|
||||
.filter_map(|child| build_subtree(&child, depth + 1, max_depth, _include_bounds, ancestors))
|
||||
.filter_map(|child| {
|
||||
build_subtree(&child, child_depth, max_depth, _include_bounds, ancestors)
|
||||
})
|
||||
.collect();
|
||||
|
||||
ancestors.remove(&ptr_key);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ pub fn resolve_element_impl(entry: &RefEntry) -> Result<NativeHandle, AdapterErr
|
|||
);
|
||||
let root = element_for_pid(entry.pid);
|
||||
let mut visited = FxHashSet::default();
|
||||
if let Ok(handle) = find_element_recursive(&root, entry, 0, 20, &mut visited) {
|
||||
// Electron/web apps nest 25+ levels deep; use ABSOLUTE_MAX_DEPTH (50) for resolution.
|
||||
let resolve_depth: u8 = 50;
|
||||
if let Ok(handle) = find_element_recursive(&root, entry, 0, resolve_depth, &mut visited) {
|
||||
tracing::debug!("resolve: found exact match");
|
||||
return Ok(handle);
|
||||
}
|
||||
|
|
@ -27,7 +29,8 @@ pub fn resolve_element_impl(entry: &RefEntry) -> Result<NativeHandle, AdapterErr
|
|||
..entry.clone()
|
||||
};
|
||||
visited.clear();
|
||||
if let Ok(handle) = find_element_recursive(&root, &relaxed, 0, 20, &mut visited) {
|
||||
if let Ok(handle) = find_element_recursive(&root, &relaxed, 0, resolve_depth, &mut visited)
|
||||
{
|
||||
tracing::debug!("resolve: found via relaxed match (bounds changed)");
|
||||
return Ok(handle);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,20 +69,27 @@ mod imp {
|
|||
focused_window_element(pid)
|
||||
}
|
||||
|
||||
fn first_child_with_subrole(pid: i32, subrole: &str) -> Option<AXElement> {
|
||||
fn first_child_with_role_or_subrole(pid: i32, target: &str) -> Option<AXElement> {
|
||||
let win = focused_window_element(pid)?;
|
||||
// The focused window itself might be the target surface (e.g. Electron sheets).
|
||||
if copy_string_attr(&win, "AXRole").as_deref() == Some(target)
|
||||
|| copy_string_attr(&win, "AXSubrole").as_deref() == Some(target)
|
||||
{
|
||||
return Some(win);
|
||||
}
|
||||
let children = copy_ax_array(&win, "AXChildren")?;
|
||||
children
|
||||
.into_iter()
|
||||
.find(|child| copy_string_attr(child, "AXSubrole").as_deref() == Some(subrole))
|
||||
children.into_iter().find(|child| {
|
||||
copy_string_attr(child, "AXSubrole").as_deref() == Some(target)
|
||||
|| copy_string_attr(child, "AXRole").as_deref() == Some(target)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn sheet_for_pid(pid: i32) -> Option<AXElement> {
|
||||
first_child_with_subrole(pid, "AXSheet")
|
||||
first_child_with_role_or_subrole(pid, "AXSheet")
|
||||
}
|
||||
|
||||
pub fn popover_for_pid(pid: i32) -> Option<AXElement> {
|
||||
first_child_with_subrole(pid, "AXPopover")
|
||||
first_child_with_role_or_subrole(pid, "AXPopover")
|
||||
}
|
||||
|
||||
pub fn alert_for_pid(pid: i32) -> Option<AXElement> {
|
||||
|
|
@ -199,14 +206,42 @@ mod imp {
|
|||
}
|
||||
|
||||
if let Some(win) = focused_window_element(pid) {
|
||||
// The focused window itself might be a surface (e.g. Electron sheets).
|
||||
let win_role = copy_string_attr(&win, "AXRole");
|
||||
let win_subrole = copy_string_attr(&win, "AXSubrole");
|
||||
let win_kind = match win_subrole.as_deref() {
|
||||
Some("AXSheet") => Some("sheet"),
|
||||
Some("AXPopover") => Some("popover"),
|
||||
Some("AXDialog") | Some("AXAlert") => Some("alert"),
|
||||
_ => match win_role.as_deref() {
|
||||
Some("AXSheet") => Some("sheet"),
|
||||
Some("AXPopover") => Some("popover"),
|
||||
_ => None,
|
||||
},
|
||||
};
|
||||
if let Some(kind) = win_kind {
|
||||
let title = copy_string_attr(&win, "AXTitle")
|
||||
.or_else(|| copy_string_attr(&win, "AXDescription"));
|
||||
surfaces.push(SurfaceInfo {
|
||||
kind: kind.into(),
|
||||
title,
|
||||
item_count: None,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(children) = copy_ax_array(&win, "AXChildren") {
|
||||
for child in &children {
|
||||
let role = copy_string_attr(child, "AXRole");
|
||||
let subrole = copy_string_attr(child, "AXSubrole");
|
||||
let kind = match subrole.as_deref() {
|
||||
Some("AXSheet") => "sheet",
|
||||
Some("AXPopover") => "popover",
|
||||
Some("AXDialog") | Some("AXAlert") => "alert",
|
||||
_ => continue,
|
||||
_ => match role.as_deref() {
|
||||
Some("AXSheet") => "sheet",
|
||||
Some("AXPopover") => "popover",
|
||||
_ => continue,
|
||||
},
|
||||
};
|
||||
let title = copy_string_attr(child, "AXTitle")
|
||||
.or_else(|| copy_string_attr(child, "AXDescription"));
|
||||
|
|
|
|||
|
|
@ -486,12 +486,32 @@ System tray interaction must be implemented from scratch as part of Phase 2.
|
|||
- **Click:** `InvokePattern` on tray items, falling back to coordinate-based `SendInput` for items that don't expose UIA patterns
|
||||
- **Open menu:** After clicking a tray item, detect the resulting popup menu via UIA focus-changed events and expose it for ref-based interaction
|
||||
|
||||
### Chromium Detection
|
||||
### Web/Electron App Compatibility
|
||||
|
||||
- Detect Chromium-based windows (Electron, Chrome, Edge, VS Code) via UIA process name or class name matching
|
||||
Chromium-based apps (Electron, Chrome, Edge, VS Code) expose deep, noisy accessibility trees where every HTML `<div>` becomes a UIA Group element. The macOS adapter solved this with three patterns that must be replicated identically on Windows.
|
||||
|
||||
**Chromium detection:**
|
||||
- Detect Chromium-based windows via UIA process name or `Chrome_WidgetWin_1` window class matching
|
||||
- If tree is empty or minimal for a Chromium window, warn: "This appears to be a Chromium app. Run the app with `--force-renderer-accessibility` to expose the accessibility tree"
|
||||
- Include this guidance in the `platform_detail` field of the error response
|
||||
|
||||
**Web-aware tree traversal (depth-skip):**
|
||||
- Non-semantic wrapper elements (`UIA_GroupControlTypeId` / `UIA_CustomControlTypeId`) with empty `Name` AND empty `Value` properties do NOT consume depth budget during tree traversal
|
||||
- This matches the macOS pattern where `AXGroup`/`AXGenericElement` wrappers are skipped
|
||||
- Without this, default `--max-depth 10` finds ~3 refs in Slack; with it, finds 100+ refs
|
||||
- Implement in `crates/windows/src/tree/builder.rs` with the same `is_web_wrapper` logic
|
||||
|
||||
**Resolver depth:**
|
||||
- Element re-identification must search up to `ABSOLUTE_MAX_DEPTH` (50), not a lower hardcoded limit
|
||||
- Electron elements commonly sit at depth 25+ in the raw tree; a shallow resolver cap causes `STALE_REF` errors
|
||||
- Implement in `crates/windows/src/tree/resolve.rs` matching the macOS pattern
|
||||
|
||||
**Surface detection for Electron:**
|
||||
- When an Electron app opens a modal (file picker, dialog), UIA may report the dialog as the focused window itself rather than a child of the parent window
|
||||
- Surface detection (`list-surfaces`, `--surface sheet/alert`) must check if the focused window IS the target surface, not only search its children
|
||||
- Check both `ControlType` and `LocalizedControlType` / UIA patterns (analogous to macOS checking both AXRole and AXSubrole)
|
||||
- Implement in `crates/windows/src/tree/surfaces.rs`
|
||||
|
||||
### Minimum OS Requirements
|
||||
|
||||
- Windows 10 1809+ (October 2018 update)
|
||||
|
|
@ -531,6 +551,8 @@ agent-desktop-windows = { path = "crates/windows" }
|
|||
- Screenshot produces valid PNG
|
||||
- Large tree snapshot performance validation
|
||||
- Chromium detection — verify warning when tree is empty
|
||||
- Electron app snapshot (VS Code) — default depth finds 50+ refs via web-aware depth-skip
|
||||
- Electron surface detection — file picker dialog detected as sheet surface
|
||||
- List notifications — returns non-empty list when notifications exist
|
||||
- Dismiss notification — verify notification removed from Action Center
|
||||
- Notification action — click action button on a test toast notification
|
||||
|
|
@ -561,7 +583,8 @@ Per [Skill Maintenance Addendum](./prd-addendum-skill-maintenance.md):
|
|||
- [ ] Create `.claude/skills/agent-desktop-windows/SKILL.md`:
|
||||
- UIA permission model and UAC handling
|
||||
- Windows-specific behaviors (UIA patterns, WinUI3 quirks, COM initialization)
|
||||
- Chromium detection and `--force-renderer-accessibility` guidance
|
||||
- Chromium/Electron compatibility: depth-skip, resolver depth, surface detection patterns
|
||||
- `--force-renderer-accessibility` guidance for empty trees
|
||||
- Windows error codes and `platform_detail` examples (HRESULT codes)
|
||||
- Troubleshooting guide (empty trees, COM errors, elevation failures)
|
||||
- [ ] Update core `SKILL.md`:
|
||||
|
|
@ -710,6 +733,31 @@ Runtime detection required for input, clipboard, and screenshot since Linux runs
|
|||
- Input tools: verify `xdotool` (X11) or `ydotool` (Wayland) is installed; error with install instructions if missing
|
||||
- Clipboard tools: verify `xclip` (X11) or `wl-clipboard` (Wayland) is installed; error with install instructions if missing
|
||||
|
||||
### Web/Electron App Compatibility
|
||||
|
||||
Same Chromium/Electron compatibility patterns as Phase 2 (Windows), adapted for AT-SPI2. These patterns ensure default `--max-depth 10` works with Electron apps like Slack, VS Code, and Chrome.
|
||||
|
||||
**Web-aware tree traversal (depth-skip):**
|
||||
- Non-semantic wrapper elements with AT-SPI roles `ROLE_PANEL`, `ROLE_SECTION`, or `ROLE_FILLER` that have empty `Name` AND empty `Value` do NOT consume depth budget during tree traversal
|
||||
- This is the AT-SPI equivalent of macOS `AXGroup`/`AXGenericElement` and Windows `UIA_GroupControlTypeId` skipping
|
||||
- Implement in `crates/linux/src/tree/builder.rs` with the same `is_web_wrapper` logic
|
||||
|
||||
**Resolver depth:**
|
||||
- Element re-identification must search up to `ABSOLUTE_MAX_DEPTH` (50), not a lower hardcoded limit
|
||||
- Electron elements commonly sit at depth 25+ in the raw AT-SPI tree
|
||||
- Implement in `crates/linux/src/tree/resolve.rs` matching the macOS/Windows pattern
|
||||
|
||||
**Surface detection for Electron:**
|
||||
- When an Electron app opens a modal (file picker, dialog), AT-SPI may report the dialog as the active window itself rather than a child of the parent window
|
||||
- Surface detection must check if the focused window IS the target surface, not only search its children
|
||||
- Check both `Role` and `RelationSet` / `RELATION_EMBEDS` for dialog detection (analogous to macOS AXRole + AXSubrole)
|
||||
- Implement in `crates/linux/src/tree/surfaces.rs`
|
||||
|
||||
**Chromium detection:**
|
||||
- Detect Chromium-based apps via process name matching (electron, chrome, chromium, code)
|
||||
- If AT-SPI tree is empty for a Chromium app, warn about `--force-renderer-accessibility`
|
||||
- On Linux, Chromium respects `ACCESSIBILITY_ENABLED=1` environment variable as an alternative
|
||||
|
||||
### AT-SPI2 Bus Detection
|
||||
|
||||
- Check for `org.a11y.Bus` presence on the D-Bus session bus
|
||||
|
|
@ -771,6 +819,8 @@ Note: `tokio` is introduced here for the first time. Phases 1-2 are fully synchr
|
|||
- Resize, move, minimize, maximize, restore window operations
|
||||
- Screenshot produces valid PNG
|
||||
- AT-SPI2 bus not running — correct error code and guidance
|
||||
- Electron app snapshot (VS Code) — default depth finds 50+ refs via web-aware depth-skip
|
||||
- Electron surface detection — file picker dialog detected as sheet surface
|
||||
- List notifications — returns non-empty list when notifications exist (GNOME)
|
||||
- Dismiss notification — verify notification dismissed via D-Bus `CloseNotification`
|
||||
- List tray items — returns known SNI items (if running under KDE or with AppIndicator extension)
|
||||
|
|
|
|||
Loading…
Reference in a new issue