fix: handle null bounds in refmap and improve sidebar click resolution

Rect deserialization now tolerates null/missing fields (defaults to 0.0),
preventing corrupt refmaps from breaking all subsequent click commands.
read_bounds rejects NaN/Inf at the source. Sidebar cells now resolve via
parent row AXSelected instead of falling through to CGClick. Extracted
chain step functions into chain_steps.rs to stay within 400 LOC limit.
This commit is contained in:
Lahfir 2026-02-24 05:55:44 -08:00
parent a33b86649c
commit d4197e8f6f
6 changed files with 350 additions and 175 deletions

View file

@ -13,7 +13,10 @@ pub fn resolve_ref(
adapter: &dyn PlatformAdapter,
) -> Result<(RefEntry, NativeHandle), AppError> {
validate_ref_id(ref_id)?;
let refmap = RefMap::load().map_err(|_| AppError::stale_ref(ref_id))?;
let refmap = RefMap::load().map_err(|e| {
tracing::debug!("refmap load failed: {e}");
AppError::stale_ref(ref_id)
})?;
let entry = refmap
.get(ref_id)
.ok_or_else(|| AppError::stale_ref(ref_id))?

View file

@ -1,4 +1,4 @@
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AccessibilityNode {
@ -31,12 +31,20 @@ pub struct AccessibilityNode {
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct Rect {
#[serde(default, deserialize_with = "f64_or_zero")]
pub x: f64,
#[serde(default, deserialize_with = "f64_or_zero")]
pub y: f64,
#[serde(default, deserialize_with = "f64_or_zero")]
pub width: f64,
#[serde(default, deserialize_with = "f64_or_zero")]
pub height: f64,
}
fn f64_or_zero<'de, D: Deserializer<'de>>(deserializer: D) -> Result<f64, D::Error> {
Option::<f64>::deserialize(deserializer).map(|opt| opt.unwrap_or(0.0))
}
impl Rect {
pub fn bounds_hash(&self) -> u64 {
use rustc_hash::FxHasher;
@ -82,3 +90,39 @@ pub struct SurfaceInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub item_count: Option<usize>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_rect_null_fields_deserialize() {
let json = r#"{"x": null, "y": null, "width": 0.0, "height": 0.0}"#;
let rect: Rect = serde_json::from_str(json).unwrap();
assert_eq!(rect.x, 0.0);
assert_eq!(rect.y, 0.0);
}
#[test]
fn test_rect_missing_fields_deserialize() {
let json = r#"{"width": 100.0, "height": 50.0}"#;
let rect: Rect = serde_json::from_str(json).unwrap();
assert_eq!(rect.x, 0.0);
assert_eq!(rect.y, 0.0);
assert_eq!(rect.width, 100.0);
}
#[test]
fn test_rect_normal_roundtrip() {
let rect = Rect {
x: 10.5,
y: 20.3,
width: 100.0,
height: 50.0,
};
let json = serde_json::to_string(&rect).unwrap();
let back: Rect = serde_json::from_str(&json).unwrap();
assert_eq!(back.x, 10.5);
assert_eq!(back.width, 100.0);
}
}

View file

@ -6,6 +6,7 @@ mod imp {
use crate::actions::{
ax_helpers,
chain::{execute_chain, ChainContext, ChainDef, ChainStep},
chain_steps,
discovery::ElementCaps,
};
use crate::tree::AXElement;
@ -14,34 +15,48 @@ mod imp {
pub static CLICK_CHAIN: ChainDef = ChainDef {
pre_scroll: true,
steps: &[
ChainStep::Action("AXPress"),
ChainStep::Custom {
label: "verified_press",
func: chain_steps::do_verified_press,
},
ChainStep::Action("AXConfirm"),
ChainStep::Action("AXOpen"),
ChainStep::Action("AXPick"),
ChainStep::Custom {
label: "show_alternate_ui",
func: try_show_alternate_ui,
func: chain_steps::try_show_alternate_ui,
},
ChainStep::ChildActions {
actions: &["AXPress", "AXConfirm", "AXOpen"],
limit: 3,
},
ChainStep::Custom {
label: "value_relay",
func: chain_steps::try_value_relay,
},
ChainStep::SetBool {
attr: "AXSelected",
value: true,
},
ChainStep::Custom {
label: "parent_row_select",
func: chain_steps::try_parent_row_select,
},
ChainStep::Custom {
label: "select_via_parent",
func: try_select_via_parent,
func: chain_steps::try_select_via_parent,
},
ChainStep::Custom {
label: "custom_actions",
func: try_custom_actions,
func: chain_steps::try_custom_actions,
},
ChainStep::Custom {
label: "focus_verified_confirm_or_press",
func: chain_steps::try_focus_then_verified_confirm_or_press,
},
ChainStep::FocusThenConfirmOrPress,
ChainStep::Custom {
label: "keyboard_activate",
func: try_keyboard_activate,
func: chain_steps::try_keyboard_activate,
},
ChainStep::AncestorActions {
actions: &["AXPress", "AXConfirm"],
@ -61,11 +76,11 @@ mod imp {
ChainStep::Action("AXShowMenu"),
ChainStep::Custom {
label: "focus_app_show_menu",
func: focus_app_then_show_menu,
func: chain_steps::focus_app_then_show_menu,
},
ChainStep::Custom {
label: "select_then_show_menu",
func: select_then_show_menu,
func: chain_steps::select_then_show_menu,
},
ChainStep::FocusThenAction("AXShowMenu"),
ChainStep::AncestorActions {
@ -124,7 +139,7 @@ mod imp {
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
ChainStep::Custom {
label: "select_all_delete",
func: select_all_then_delete,
func: chain_steps::select_all_then_delete,
},
],
suggestion: "Try 'press cmd+a' then 'press delete'.",
@ -157,175 +172,12 @@ mod imp {
ChainStep::Action("AXScrollToVisible"),
ChainStep::Custom {
label: "walk_parents_scroll",
func: walk_parents_and_scroll,
func: chain_steps::walk_parents_and_scroll,
},
],
suggestion: "Element may not be in a scrollable container.",
};
fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 55, true);
AXUIElementPostKeyboardEvent(app.0, 0, 0, true);
AXUIElementPostKeyboardEvent(app.0, 0, 0, false);
AXUIElementPostKeyboardEvent(app.0, 0, 55, false);
};
std::thread::sleep(std::time::Duration::from_millis(30));
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 51, true);
AXUIElementPostKeyboardEvent(app.0, 0, 51, false);
};
true
}
fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::kAXRoleAttribute;
let bounds = match crate::tree::read_bounds(el) {
Some(b) => b,
None => return false,
};
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..8 {
let parent = match &current {
Some(p) => p,
None => return false,
};
let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute);
if role.as_deref() == Some("AXScrollArea") {
let parent_bounds = match crate::tree::read_bounds(parent) {
Some(b) => b,
None => return false,
};
let target_y = bounds.y + bounds.height / 2.0;
let visible_mid = parent_bounds.y + parent_bounds.height / 2.0;
if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height {
let dy = if target_y > visible_mid { -5 } else { 5 };
let cx = parent_bounds.x + parent_bounds.width / 2.0;
let cy = parent_bounds.y + parent_bounds.height / 2.0;
for _ in 0..20 {
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
std::thread::sleep(std::time::Duration::from_millis(16));
}
}
return true;
}
current = crate::tree::copy_element_attr(parent, "AXParent");
}
false
}
fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
return false;
}
ax_helpers::try_ax_action(el, "AXShowAlternateUI");
std::thread::sleep(std::time::Duration::from_millis(100));
ax_helpers::try_each_child(
el,
|child| {
let ca = ax_helpers::list_ax_actions(child);
ax_helpers::try_action_from_list(child, &ca, &["AXPress"])
},
5,
)
}
fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
string::CFString,
};
let parent = match crate::tree::copy_element_attr(el, "AXParent") {
Some(p) => p,
None => return false,
};
let role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) {
Some(r) => r,
None => return false,
};
if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") {
return false;
}
if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") {
return false;
}
unsafe { CFRetain(el.0 as CFTypeRef) };
let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_cf]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
parent.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool {
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
if custom.is_empty() {
return false;
}
ax_helpers::try_ax_action(el, "AXPerformCustomAction")
}
fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
};
true
}
fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let _ = crate::system::app_ops::ensure_app_focused(pid);
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::is_attr_settable(el, "AXSelected") {
return false;
}
if !ax_helpers::set_ax_bool(el, "AXSelected", true) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> {
if ax_helpers::try_ax_action(el, "AXOpen") {
return Ok(());

View file

@ -0,0 +1,267 @@
#[cfg(target_os = "macos")]
mod imp {
use crate::actions::{ax_helpers, discovery::ElementCaps};
use crate::tree::AXElement;
pub fn do_verified_press(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::kAXRoleAttribute;
let parent = crate::tree::copy_element_attr(el, "AXParent");
let in_container = parent.as_ref().is_some_and(|p| {
matches!(
crate::tree::copy_string_attr(p, kAXRoleAttribute).as_deref(),
Some("AXOutline" | "AXList" | "AXTable")
)
});
if !in_container {
return ax_helpers::try_ax_action_retried(el, "AXPress");
}
let selected_before = crate::tree::element::copy_bool_attr(el, "AXSelected");
if !ax_helpers::try_ax_action_retried(el, "AXPress") {
return false;
}
if selected_before == Some(true) {
return true;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let selected_after = crate::tree::element::copy_bool_attr(el, "AXSelected");
if selected_after == Some(true) {
return true;
}
if crate::tree::copy_string_attr(el, kAXRoleAttribute).is_none() {
return true;
}
tracing::debug!("verified_press: AXPress ok but no state change in selection container");
false
}
pub fn try_focus_then_verified_confirm_or_press(el: &AXElement, caps: &ElementCaps) -> bool {
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
if ax_helpers::try_ax_action_retried(el, "AXConfirm") {
return true;
}
do_verified_press(el, caps)
}
pub fn try_value_relay(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::list_ax_actions(el).is_empty() {
return false;
}
let win = crate::tree::copy_element_attr(el, "AXWindow");
let is_dialog = win.as_ref().is_some_and(|w| {
crate::tree::copy_string_attr(w, "AXSubrole").as_deref() == Some("AXDialog")
});
if !is_dialog {
return false;
}
let label = std::cell::RefCell::new(None::<String>);
ax_helpers::try_each_child(
el,
|child| {
let d = crate::tree::copy_string_attr(child, "AXDescription").unwrap_or_default();
if d.is_empty() {
return false;
}
*label.borrow_mut() = Some(d.split(',').next().unwrap_or(&d).trim().to_owned());
true
},
5,
);
let Some(label) = label.into_inner() else {
return false;
};
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
return false;
};
let app = crate::tree::element_for_pid(pid);
let Some(owner) = crate::tree::copy_element_attr(&app, "AXFocusedUIElement") else {
return false;
};
if !ax_helpers::is_attr_settable(&owner, "AXValue") {
return false;
}
let orig = crate::tree::copy_string_attr(&owner, "AXValue");
if ax_helpers::set_ax_string_or_err(&owner, "AXValue", &label).is_err() {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(150));
if !ax_helpers::try_ax_action(&owner, "AXConfirm") {
if let Some(o) = &orig {
let _ = ax_helpers::set_ax_string_or_err(&owner, "AXValue", o);
}
return false;
}
std::thread::sleep(std::time::Duration::from_millis(150));
let final_val = crate::tree::copy_string_attr(&owner, "AXValue");
if final_val.as_deref() != Some(label.as_str()) {
tracing::debug!("value_relay: reverted to {final_val:?}, expected {label:?}");
}
final_val.as_deref() == Some(label.as_str())
}
pub fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
return false;
};
let a = crate::tree::element_for_pid(pid);
unsafe {
PostKey(a.0, 0, 55, true);
PostKey(a.0, 0, 0, true);
PostKey(a.0, 0, 0, false);
PostKey(a.0, 0, 55, false);
}
std::thread::sleep(std::time::Duration::from_millis(30));
unsafe {
PostKey(a.0, 0, 51, true);
PostKey(a.0, 0, 51, false);
}
true
}
pub fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::kAXRoleAttribute;
let Some(bounds) = crate::tree::read_bounds(el) else {
return false;
};
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..8 {
let Some(parent) = &current else { return false };
if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref()
== Some("AXScrollArea")
{
let Some(pb) = crate::tree::read_bounds(parent) else {
return false;
};
let ty = bounds.y + bounds.height / 2.0;
if ty < pb.y || ty > pb.y + pb.height {
let dy = if ty > pb.y + pb.height / 2.0 { -5 } else { 5 };
let (cx, cy) = (pb.x + pb.width / 2.0, pb.y + pb.height / 2.0);
for _ in 0..20 {
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
std::thread::sleep(std::time::Duration::from_millis(16));
}
}
return true;
}
current = crate::tree::copy_element_attr(parent, "AXParent");
}
false
}
pub fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
return false;
}
ax_helpers::try_ax_action(el, "AXShowAlternateUI");
std::thread::sleep(std::time::Duration::from_millis(100));
ax_helpers::try_each_child(
el,
|child| {
let ca = ax_helpers::list_ax_actions(child);
ax_helpers::try_action_from_list(child, &ca, &["AXPress"])
},
5,
)
}
pub fn try_parent_row_select(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::kAXRoleAttribute;
let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else {
return false;
};
let role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute).unwrap_or_default();
if !matches!(role.as_str(), "AXRow" | "AXOutlineRow") {
return false;
}
if !ax_helpers::is_attr_settable(&parent, "AXSelected") {
return false;
}
ax_helpers::set_ax_bool(&parent, "AXSelected", true)
}
pub fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
string::CFString,
};
let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else {
return false;
};
let Some(role) = crate::tree::copy_string_attr(&parent, kAXRoleAttribute) else {
return false;
};
if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") {
return false;
}
if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") {
return false;
}
unsafe { CFRetain(el.0 as CFTypeRef) };
let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_cf]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
parent.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
pub fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool {
let has = !crate::tree::copy_ax_array(el, "AXCustomActions")
.unwrap_or_default()
.is_empty();
has && ax_helpers::try_ax_action(el, "AXPerformCustomAction")
}
pub fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
return false;
};
let a = crate::tree::element_for_pid(pid);
unsafe {
PostKey(a.0, 0, 49, true);
PostKey(a.0, 0, 49, false);
}
true
}
pub fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
return false;
};
let _ = crate::system::app_ops::ensure_app_focused(pid);
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
pub fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::is_attr_settable(el, "AXSelected")
|| !ax_helpers::set_ax_bool(el, "AXSelected", true)
{
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
}
#[cfg(target_os = "macos")]
pub(crate) use imp::*;

View file

@ -1,6 +1,7 @@
pub mod ax_helpers;
pub mod chain;
pub mod chain_defs;
pub mod chain_steps;
pub mod discovery;
pub mod dispatch;
pub mod extras;

View file

@ -290,6 +290,14 @@ mod imp {
return None;
}
if !point.x.is_finite()
|| !point.y.is_finite()
|| !size.width.is_finite()
|| !size.height.is_finite()
{
return None;
}
Some(Rect {
x: point.x,
y: point.y,