mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 01:22:16 +00:00
fix: handle null bounds in refmap and improve sidebar click resolution
Rect deserialization now tolerates null/missing fields (defaults to 0.0), preventing corrupt refmaps from breaking all subsequent click commands. read_bounds rejects NaN/Inf at the source. Sidebar cells now resolve via parent row AXSelected instead of falling through to CGClick. Extracted chain step functions into chain_steps.rs to stay within 400 LOC limit.
This commit is contained in:
parent
a33b86649c
commit
d4197e8f6f
6 changed files with 350 additions and 175 deletions
|
|
@ -13,7 +13,10 @@ pub fn resolve_ref(
|
|||
adapter: &dyn PlatformAdapter,
|
||||
) -> Result<(RefEntry, NativeHandle), AppError> {
|
||||
validate_ref_id(ref_id)?;
|
||||
let refmap = RefMap::load().map_err(|_| AppError::stale_ref(ref_id))?;
|
||||
let refmap = RefMap::load().map_err(|e| {
|
||||
tracing::debug!("refmap load failed: {e}");
|
||||
AppError::stale_ref(ref_id)
|
||||
})?;
|
||||
let entry = refmap
|
||||
.get(ref_id)
|
||||
.ok_or_else(|| AppError::stale_ref(ref_id))?
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AccessibilityNode {
|
||||
|
|
@ -31,12 +31,20 @@ pub struct AccessibilityNode {
|
|||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct Rect {
|
||||
#[serde(default, deserialize_with = "f64_or_zero")]
|
||||
pub x: f64,
|
||||
#[serde(default, deserialize_with = "f64_or_zero")]
|
||||
pub y: f64,
|
||||
#[serde(default, deserialize_with = "f64_or_zero")]
|
||||
pub width: f64,
|
||||
#[serde(default, deserialize_with = "f64_or_zero")]
|
||||
pub height: f64,
|
||||
}
|
||||
|
||||
fn f64_or_zero<'de, D: Deserializer<'de>>(deserializer: D) -> Result<f64, D::Error> {
|
||||
Option::<f64>::deserialize(deserializer).map(|opt| opt.unwrap_or(0.0))
|
||||
}
|
||||
|
||||
impl Rect {
|
||||
pub fn bounds_hash(&self) -> u64 {
|
||||
use rustc_hash::FxHasher;
|
||||
|
|
@ -82,3 +90,39 @@ pub struct SurfaceInfo {
|
|||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub item_count: Option<usize>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_rect_null_fields_deserialize() {
|
||||
let json = r#"{"x": null, "y": null, "width": 0.0, "height": 0.0}"#;
|
||||
let rect: Rect = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(rect.x, 0.0);
|
||||
assert_eq!(rect.y, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rect_missing_fields_deserialize() {
|
||||
let json = r#"{"width": 100.0, "height": 50.0}"#;
|
||||
let rect: Rect = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(rect.x, 0.0);
|
||||
assert_eq!(rect.y, 0.0);
|
||||
assert_eq!(rect.width, 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rect_normal_roundtrip() {
|
||||
let rect = Rect {
|
||||
x: 10.5,
|
||||
y: 20.3,
|
||||
width: 100.0,
|
||||
height: 50.0,
|
||||
};
|
||||
let json = serde_json::to_string(&rect).unwrap();
|
||||
let back: Rect = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back.x, 10.5);
|
||||
assert_eq!(back.width, 100.0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ mod imp {
|
|||
use crate::actions::{
|
||||
ax_helpers,
|
||||
chain::{execute_chain, ChainContext, ChainDef, ChainStep},
|
||||
chain_steps,
|
||||
discovery::ElementCaps,
|
||||
};
|
||||
use crate::tree::AXElement;
|
||||
|
|
@ -14,34 +15,48 @@ mod imp {
|
|||
pub static CLICK_CHAIN: ChainDef = ChainDef {
|
||||
pre_scroll: true,
|
||||
steps: &[
|
||||
ChainStep::Action("AXPress"),
|
||||
ChainStep::Custom {
|
||||
label: "verified_press",
|
||||
func: chain_steps::do_verified_press,
|
||||
},
|
||||
ChainStep::Action("AXConfirm"),
|
||||
ChainStep::Action("AXOpen"),
|
||||
ChainStep::Action("AXPick"),
|
||||
ChainStep::Custom {
|
||||
label: "show_alternate_ui",
|
||||
func: try_show_alternate_ui,
|
||||
func: chain_steps::try_show_alternate_ui,
|
||||
},
|
||||
ChainStep::ChildActions {
|
||||
actions: &["AXPress", "AXConfirm", "AXOpen"],
|
||||
limit: 3,
|
||||
},
|
||||
ChainStep::Custom {
|
||||
label: "value_relay",
|
||||
func: chain_steps::try_value_relay,
|
||||
},
|
||||
ChainStep::SetBool {
|
||||
attr: "AXSelected",
|
||||
value: true,
|
||||
},
|
||||
ChainStep::Custom {
|
||||
label: "parent_row_select",
|
||||
func: chain_steps::try_parent_row_select,
|
||||
},
|
||||
ChainStep::Custom {
|
||||
label: "select_via_parent",
|
||||
func: try_select_via_parent,
|
||||
func: chain_steps::try_select_via_parent,
|
||||
},
|
||||
ChainStep::Custom {
|
||||
label: "custom_actions",
|
||||
func: try_custom_actions,
|
||||
func: chain_steps::try_custom_actions,
|
||||
},
|
||||
ChainStep::Custom {
|
||||
label: "focus_verified_confirm_or_press",
|
||||
func: chain_steps::try_focus_then_verified_confirm_or_press,
|
||||
},
|
||||
ChainStep::FocusThenConfirmOrPress,
|
||||
ChainStep::Custom {
|
||||
label: "keyboard_activate",
|
||||
func: try_keyboard_activate,
|
||||
func: chain_steps::try_keyboard_activate,
|
||||
},
|
||||
ChainStep::AncestorActions {
|
||||
actions: &["AXPress", "AXConfirm"],
|
||||
|
|
@ -61,11 +76,11 @@ mod imp {
|
|||
ChainStep::Action("AXShowMenu"),
|
||||
ChainStep::Custom {
|
||||
label: "focus_app_show_menu",
|
||||
func: focus_app_then_show_menu,
|
||||
func: chain_steps::focus_app_then_show_menu,
|
||||
},
|
||||
ChainStep::Custom {
|
||||
label: "select_then_show_menu",
|
||||
func: select_then_show_menu,
|
||||
func: chain_steps::select_then_show_menu,
|
||||
},
|
||||
ChainStep::FocusThenAction("AXShowMenu"),
|
||||
ChainStep::AncestorActions {
|
||||
|
|
@ -124,7 +139,7 @@ mod imp {
|
|||
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
|
||||
ChainStep::Custom {
|
||||
label: "select_all_delete",
|
||||
func: select_all_then_delete,
|
||||
func: chain_steps::select_all_then_delete,
|
||||
},
|
||||
],
|
||||
suggestion: "Try 'press cmd+a' then 'press delete'.",
|
||||
|
|
@ -157,175 +172,12 @@ mod imp {
|
|||
ChainStep::Action("AXScrollToVisible"),
|
||||
ChainStep::Custom {
|
||||
label: "walk_parents_scroll",
|
||||
func: walk_parents_and_scroll,
|
||||
func: chain_steps::walk_parents_and_scroll,
|
||||
},
|
||||
],
|
||||
suggestion: "Element may not be in a scrollable container.",
|
||||
};
|
||||
|
||||
fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent;
|
||||
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let pid = match crate::system::app_ops::pid_from_element(el) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let app = crate::tree::element_for_pid(pid);
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 55, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 0, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 0, false);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 55, false);
|
||||
};
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 51, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 51, false);
|
||||
};
|
||||
true
|
||||
}
|
||||
|
||||
fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
|
||||
let bounds = match crate::tree::read_bounds(el) {
|
||||
Some(b) => b,
|
||||
None => return false,
|
||||
};
|
||||
let mut current = crate::tree::copy_element_attr(el, "AXParent");
|
||||
for _ in 0..8 {
|
||||
let parent = match ¤t {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute);
|
||||
if role.as_deref() == Some("AXScrollArea") {
|
||||
let parent_bounds = match crate::tree::read_bounds(parent) {
|
||||
Some(b) => b,
|
||||
None => return false,
|
||||
};
|
||||
let target_y = bounds.y + bounds.height / 2.0;
|
||||
let visible_mid = parent_bounds.y + parent_bounds.height / 2.0;
|
||||
if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height {
|
||||
let dy = if target_y > visible_mid { -5 } else { 5 };
|
||||
let cx = parent_bounds.x + parent_bounds.width / 2.0;
|
||||
let cy = parent_bounds.y + parent_bounds.height / 2.0;
|
||||
for _ in 0..20 {
|
||||
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
|
||||
std::thread::sleep(std::time::Duration::from_millis(16));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
current = crate::tree::copy_element_attr(parent, "AXParent");
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
|
||||
return false;
|
||||
}
|
||||
ax_helpers::try_ax_action(el, "AXShowAlternateUI");
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
ax_helpers::try_each_child(
|
||||
el,
|
||||
|child| {
|
||||
let ca = ax_helpers::list_ax_actions(child);
|
||||
ax_helpers::try_action_from_list(child, &ca, &["AXPress"])
|
||||
},
|
||||
5,
|
||||
)
|
||||
}
|
||||
|
||||
fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let parent = match crate::tree::copy_element_attr(el, "AXParent") {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) {
|
||||
Some(r) => r,
|
||||
None => return false,
|
||||
};
|
||||
if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") {
|
||||
return false;
|
||||
}
|
||||
if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") {
|
||||
return false;
|
||||
}
|
||||
unsafe { CFRetain(el.0 as CFTypeRef) };
|
||||
let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
|
||||
let arr = CFArray::from_CFTypes(&[el_cf]);
|
||||
let cf_attr = CFString::new("AXSelectedRows");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
parent.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
arr.as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
err == kAXErrorSuccess
|
||||
}
|
||||
|
||||
fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
|
||||
if custom.is_empty() {
|
||||
return false;
|
||||
}
|
||||
ax_helpers::try_ax_action(el, "AXPerformCustomAction")
|
||||
}
|
||||
|
||||
fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent;
|
||||
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let pid = match crate::system::app_ops::pid_from_element(el) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let app = crate::tree::element_for_pid(pid);
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
|
||||
};
|
||||
true
|
||||
}
|
||||
|
||||
fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
let pid = match crate::system::app_ops::pid_from_element(el) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let _ = crate::system::app_ops::ensure_app_focused(pid);
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_helpers::try_ax_action(el, "AXShowMenu")
|
||||
}
|
||||
|
||||
fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::is_attr_settable(el, "AXSelected") {
|
||||
return false;
|
||||
}
|
||||
if !ax_helpers::set_ax_bool(el, "AXSelected", true) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_helpers::try_ax_action(el, "AXShowMenu")
|
||||
}
|
||||
|
||||
pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> {
|
||||
if ax_helpers::try_ax_action(el, "AXOpen") {
|
||||
return Ok(());
|
||||
|
|
|
|||
267
crates/macos/src/actions/chain_steps.rs
Normal file
267
crates/macos/src/actions/chain_steps.rs
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use crate::actions::{ax_helpers, discovery::ElementCaps};
|
||||
use crate::tree::AXElement;
|
||||
|
||||
pub fn do_verified_press(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
let parent = crate::tree::copy_element_attr(el, "AXParent");
|
||||
let in_container = parent.as_ref().is_some_and(|p| {
|
||||
matches!(
|
||||
crate::tree::copy_string_attr(p, kAXRoleAttribute).as_deref(),
|
||||
Some("AXOutline" | "AXList" | "AXTable")
|
||||
)
|
||||
});
|
||||
if !in_container {
|
||||
return ax_helpers::try_ax_action_retried(el, "AXPress");
|
||||
}
|
||||
let selected_before = crate::tree::element::copy_bool_attr(el, "AXSelected");
|
||||
if !ax_helpers::try_ax_action_retried(el, "AXPress") {
|
||||
return false;
|
||||
}
|
||||
if selected_before == Some(true) {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let selected_after = crate::tree::element::copy_bool_attr(el, "AXSelected");
|
||||
if selected_after == Some(true) {
|
||||
return true;
|
||||
}
|
||||
if crate::tree::copy_string_attr(el, kAXRoleAttribute).is_none() {
|
||||
return true;
|
||||
}
|
||||
tracing::debug!("verified_press: AXPress ok but no state change in selection container");
|
||||
false
|
||||
}
|
||||
|
||||
pub fn try_focus_then_verified_confirm_or_press(el: &AXElement, caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
if ax_helpers::try_ax_action_retried(el, "AXConfirm") {
|
||||
return true;
|
||||
}
|
||||
do_verified_press(el, caps)
|
||||
}
|
||||
|
||||
pub fn try_value_relay(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::list_ax_actions(el).is_empty() {
|
||||
return false;
|
||||
}
|
||||
let win = crate::tree::copy_element_attr(el, "AXWindow");
|
||||
let is_dialog = win.as_ref().is_some_and(|w| {
|
||||
crate::tree::copy_string_attr(w, "AXSubrole").as_deref() == Some("AXDialog")
|
||||
});
|
||||
if !is_dialog {
|
||||
return false;
|
||||
}
|
||||
let label = std::cell::RefCell::new(None::<String>);
|
||||
ax_helpers::try_each_child(
|
||||
el,
|
||||
|child| {
|
||||
let d = crate::tree::copy_string_attr(child, "AXDescription").unwrap_or_default();
|
||||
if d.is_empty() {
|
||||
return false;
|
||||
}
|
||||
*label.borrow_mut() = Some(d.split(',').next().unwrap_or(&d).trim().to_owned());
|
||||
true
|
||||
},
|
||||
5,
|
||||
);
|
||||
let Some(label) = label.into_inner() else {
|
||||
return false;
|
||||
};
|
||||
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
|
||||
return false;
|
||||
};
|
||||
let app = crate::tree::element_for_pid(pid);
|
||||
let Some(owner) = crate::tree::copy_element_attr(&app, "AXFocusedUIElement") else {
|
||||
return false;
|
||||
};
|
||||
if !ax_helpers::is_attr_settable(&owner, "AXValue") {
|
||||
return false;
|
||||
}
|
||||
let orig = crate::tree::copy_string_attr(&owner, "AXValue");
|
||||
if ax_helpers::set_ax_string_or_err(&owner, "AXValue", &label).is_err() {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||
if !ax_helpers::try_ax_action(&owner, "AXConfirm") {
|
||||
if let Some(o) = &orig {
|
||||
let _ = ax_helpers::set_ax_string_or_err(&owner, "AXValue", o);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(150));
|
||||
let final_val = crate::tree::copy_string_attr(&owner, "AXValue");
|
||||
if final_val.as_deref() != Some(label.as_str()) {
|
||||
tracing::debug!("value_relay: reverted to {final_val:?}, expected {label:?}");
|
||||
}
|
||||
final_val.as_deref() == Some(label.as_str())
|
||||
}
|
||||
|
||||
pub fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey;
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
|
||||
return false;
|
||||
};
|
||||
let a = crate::tree::element_for_pid(pid);
|
||||
unsafe {
|
||||
PostKey(a.0, 0, 55, true);
|
||||
PostKey(a.0, 0, 0, true);
|
||||
PostKey(a.0, 0, 0, false);
|
||||
PostKey(a.0, 0, 55, false);
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
unsafe {
|
||||
PostKey(a.0, 0, 51, true);
|
||||
PostKey(a.0, 0, 51, false);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
let Some(bounds) = crate::tree::read_bounds(el) else {
|
||||
return false;
|
||||
};
|
||||
let mut current = crate::tree::copy_element_attr(el, "AXParent");
|
||||
for _ in 0..8 {
|
||||
let Some(parent) = ¤t else { return false };
|
||||
if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref()
|
||||
== Some("AXScrollArea")
|
||||
{
|
||||
let Some(pb) = crate::tree::read_bounds(parent) else {
|
||||
return false;
|
||||
};
|
||||
let ty = bounds.y + bounds.height / 2.0;
|
||||
if ty < pb.y || ty > pb.y + pb.height {
|
||||
let dy = if ty > pb.y + pb.height / 2.0 { -5 } else { 5 };
|
||||
let (cx, cy) = (pb.x + pb.width / 2.0, pb.y + pb.height / 2.0);
|
||||
for _ in 0..20 {
|
||||
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
|
||||
std::thread::sleep(std::time::Duration::from_millis(16));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
current = crate::tree::copy_element_attr(parent, "AXParent");
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
|
||||
return false;
|
||||
}
|
||||
ax_helpers::try_ax_action(el, "AXShowAlternateUI");
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
ax_helpers::try_each_child(
|
||||
el,
|
||||
|child| {
|
||||
let ca = ax_helpers::list_ax_actions(child);
|
||||
ax_helpers::try_action_from_list(child, &ca, &["AXPress"])
|
||||
},
|
||||
5,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn try_parent_row_select(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else {
|
||||
return false;
|
||||
};
|
||||
let role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute).unwrap_or_default();
|
||||
if !matches!(role.as_str(), "AXRow" | "AXOutlineRow") {
|
||||
return false;
|
||||
}
|
||||
if !ax_helpers::is_attr_settable(&parent, "AXSelected") {
|
||||
return false;
|
||||
}
|
||||
ax_helpers::set_ax_bool(&parent, "AXSelected", true)
|
||||
}
|
||||
|
||||
pub fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else {
|
||||
return false;
|
||||
};
|
||||
let Some(role) = crate::tree::copy_string_attr(&parent, kAXRoleAttribute) else {
|
||||
return false;
|
||||
};
|
||||
if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") {
|
||||
return false;
|
||||
}
|
||||
if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") {
|
||||
return false;
|
||||
}
|
||||
unsafe { CFRetain(el.0 as CFTypeRef) };
|
||||
let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
|
||||
let arr = CFArray::from_CFTypes(&[el_cf]);
|
||||
let cf_attr = CFString::new("AXSelectedRows");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
parent.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
arr.as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
err == kAXErrorSuccess
|
||||
}
|
||||
|
||||
pub fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
let has = !crate::tree::copy_ax_array(el, "AXCustomActions")
|
||||
.unwrap_or_default()
|
||||
.is_empty();
|
||||
has && ax_helpers::try_ax_action(el, "AXPerformCustomAction")
|
||||
}
|
||||
|
||||
pub fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey;
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
|
||||
return false;
|
||||
};
|
||||
let a = crate::tree::element_for_pid(pid);
|
||||
unsafe {
|
||||
PostKey(a.0, 0, 49, true);
|
||||
PostKey(a.0, 0, 49, false);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
let Some(pid) = crate::system::app_ops::pid_from_element(el) else {
|
||||
return false;
|
||||
};
|
||||
let _ = crate::system::app_ops::ensure_app_focused(pid);
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_helpers::try_ax_action(el, "AXShowMenu")
|
||||
}
|
||||
|
||||
pub fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::is_attr_settable(el, "AXSelected")
|
||||
|| !ax_helpers::set_ax_bool(el, "AXSelected", true)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_helpers::try_ax_action(el, "AXShowMenu")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) use imp::*;
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
pub mod ax_helpers;
|
||||
pub mod chain;
|
||||
pub mod chain_defs;
|
||||
pub mod chain_steps;
|
||||
pub mod discovery;
|
||||
pub mod dispatch;
|
||||
pub mod extras;
|
||||
|
|
|
|||
|
|
@ -290,6 +290,14 @@ mod imp {
|
|||
return None;
|
||||
}
|
||||
|
||||
if !point.x.is_finite()
|
||||
|| !point.y.is_finite()
|
||||
|| !size.width.is_finite()
|
||||
|| !size.height.is_finite()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(Rect {
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
|
|
|
|||
Loading…
Reference in a new issue