mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 17:42:13 +00:00
feat: 10-step scroll chain, focus guards, enhanced click chain, bounds fix
- Rewrite ax_scroll with 10-step AX-first chain (scroll-to-visible, increment/decrement, page scroll, value shift, sub-element press, focus child, select rows, keyboard, arrow keys, CGEvent last resort) - Add ensure_app_focused() guard before all CGEvent calls - Enhance smart_activate with AXScrollToVisible pre-step, AXShowAlternateUI, AXCustomActions, and keyboard space fallback - Fix bounds_hash always null in RefMap (always read bounds in builder) - Add right-click menu capture in response data - Add append_surface_refs for post-action menu detection
This commit is contained in:
parent
2f495ffb69
commit
595ccb6cc4
9 changed files with 460 additions and 56 deletions
|
|
@ -1,5 +1,9 @@
|
|||
use crate::{
|
||||
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
|
||||
action::Action,
|
||||
adapter::{PlatformAdapter, SnapshotSurface},
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
snapshot,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -8,7 +12,22 @@ pub struct RightClickArgs {
|
|||
}
|
||||
|
||||
pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
|
||||
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let (entry, handle) = resolve_ref(&args.ref_id, adapter)?;
|
||||
let result = adapter.execute_action(&handle, Action::RightClick)?;
|
||||
Ok(serde_json::to_value(result)?)
|
||||
let mut response = serde_json::to_value(&result)?;
|
||||
|
||||
std::thread::sleep(std::time::Duration::from_millis(250));
|
||||
|
||||
if let Some(menu_tree) = snapshot::append_surface_refs(
|
||||
adapter,
|
||||
entry.pid,
|
||||
entry.source_app.as_deref(),
|
||||
SnapshotSurface::Menu,
|
||||
) {
|
||||
if let Ok(menu_json) = serde_json::to_value(&menu_tree) {
|
||||
response["menu"] = menu_json;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::{
|
||||
adapter::{PlatformAdapter, TreeOptions, WindowFilter},
|
||||
adapter::{PlatformAdapter, SnapshotSurface, TreeOptions, WindowFilter},
|
||||
error::AppError,
|
||||
node::{AccessibilityNode, WindowInfo},
|
||||
refs::{RefEntry, RefMap},
|
||||
|
|
@ -108,6 +108,30 @@ pub fn run(
|
|||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn append_surface_refs(
|
||||
adapter: &dyn PlatformAdapter,
|
||||
pid: i32,
|
||||
source_app: Option<&str>,
|
||||
surface: SnapshotSurface,
|
||||
) -> Option<AccessibilityNode> {
|
||||
let filter = WindowFilter {
|
||||
focused_only: false,
|
||||
app: None,
|
||||
};
|
||||
let windows = adapter.list_windows(&filter).ok()?;
|
||||
let window = windows.into_iter().find(|w| w.pid == pid)?;
|
||||
let opts = TreeOptions {
|
||||
surface,
|
||||
interactive_only: true,
|
||||
..Default::default()
|
||||
};
|
||||
let raw_tree = adapter.get_tree(&window, &opts).ok()?;
|
||||
let mut refmap = RefMap::load().ok()?;
|
||||
let tree = allocate_refs(raw_tree, &mut refmap, false, true, pid, source_app);
|
||||
refmap.save().ok()?;
|
||||
Some(tree)
|
||||
}
|
||||
|
||||
fn allocate_refs(
|
||||
mut node: AccessibilityNode,
|
||||
refmap: &mut RefMap,
|
||||
|
|
|
|||
|
|
@ -16,8 +16,10 @@ mod imp {
|
|||
};
|
||||
use std::os::raw::c_uchar;
|
||||
|
||||
/// Attempts a 10-step AX-first activation chain before falling back to CGEvent.
|
||||
pub fn smart_activate(el: &AXElement) -> Result<(), AdapterError> {
|
||||
let scroll_action = CFString::new("AXScrollToVisible");
|
||||
unsafe { AXUIElementPerformAction(el.0, scroll_action.as_concrete_TypeRef()) };
|
||||
|
||||
let actions = list_ax_actions(el);
|
||||
|
||||
if try_action_from_list(el, &actions, &["AXPress"]) {
|
||||
|
|
@ -32,6 +34,9 @@ mod imp {
|
|||
if try_action_from_list(el, &actions, &["AXPick"]) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_show_alternate_ui(el) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if try_child_activation(el) {
|
||||
return Ok(());
|
||||
|
|
@ -42,9 +47,15 @@ mod imp {
|
|||
if try_select_via_parent(el) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_custom_actions(el) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_focus_then_activate(el) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_keyboard_activate(el) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_parent_activation(el) {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -66,7 +77,6 @@ mod imp {
|
|||
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2)
|
||||
}
|
||||
|
||||
/// AXShowMenu first, then CGEvent right-click.
|
||||
pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> {
|
||||
let actions = list_ax_actions(el);
|
||||
if try_action_from_list(el, &actions, &["AXShowMenu"]) {
|
||||
|
|
@ -225,6 +235,64 @@ mod imp {
|
|||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn try_show_alternate_ui(el: &AXElement) -> bool {
|
||||
let actions = list_ax_actions(el);
|
||||
if !actions.iter().any(|a| a == "AXShowAlternateUI") {
|
||||
return false;
|
||||
}
|
||||
let action = CFString::new("AXShowAlternateUI");
|
||||
unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
|
||||
for child in children.iter().take(5) {
|
||||
let child_actions = list_ax_actions(child);
|
||||
if try_action_from_list(child, &child_actions, &["AXPress"]) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn try_custom_actions(el: &AXElement) -> bool {
|
||||
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
|
||||
if custom.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let action = CFString::new("AXPerformCustomAction");
|
||||
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
|
||||
err == kAXErrorSuccess
|
||||
}
|
||||
|
||||
fn try_keyboard_activate(el: &AXElement) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent;
|
||||
|
||||
let cf_focused = CFString::new(kAXFocusedAttribute);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
el.0,
|
||||
cf_focused.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
|
||||
let pid = match crate::system::app_ops::pid_from_element(el) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let app = crate::tree::element_for_pid(pid);
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
|
||||
};
|
||||
true
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ mod imp {
|
|||
button: MouseButton,
|
||||
count: u32,
|
||||
) -> Result<(), AdapterError> {
|
||||
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
|
||||
let _ = crate::system::app_ops::ensure_app_focused(pid);
|
||||
}
|
||||
let bounds = crate::tree::read_bounds(el).ok_or_else(|| {
|
||||
AdapterError::new(ErrorCode::ActionFailed, "Element has no readable bounds")
|
||||
.with_suggestion("AX action failed and CGEvent fallback unavailable")
|
||||
|
|
|
|||
|
|
@ -109,69 +109,282 @@ pub(crate) fn ax_scroll(
|
|||
direction: &agent_desktop_core::action::Direction,
|
||||
amount: u32,
|
||||
) -> Result<(), AdapterError> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
|
||||
use accessibility_sys::{
|
||||
kAXErrorSuccess, AXUIElementPerformAction, AXUIElementPostKeyboardEvent,
|
||||
AXUIElementSetAttributeValue,
|
||||
};
|
||||
use agent_desktop_core::action::Direction;
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
|
||||
let scroll_area = find_scroll_area(el);
|
||||
let target = scroll_area.as_ref().unwrap_or(el);
|
||||
|
||||
let (bar_orientation, action_name) = match direction {
|
||||
let scroll_visible = CFString::new("AXScrollToVisible");
|
||||
if unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) }
|
||||
== kAXErrorSuccess
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (bar_attr, inc_action) = match direction {
|
||||
Direction::Down => ("AXVerticalScrollBar", "AXIncrement"),
|
||||
Direction::Up => ("AXVerticalScrollBar", "AXDecrement"),
|
||||
Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"),
|
||||
Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"),
|
||||
};
|
||||
|
||||
if let Some(scroll_bar) = get_scroll_bar(target, bar_orientation) {
|
||||
let ax_action = CFString::new(action_name);
|
||||
if let Some(bar) = get_scroll_bar(target, bar_attr) {
|
||||
let ax_action = CFString::new(inc_action);
|
||||
let mut ok = true;
|
||||
for _ in 0..amount {
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(scroll_bar.0, ax_action.as_concrete_TypeRef()) };
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
format!("{action_name} on scroll bar failed (err={err})"),
|
||||
));
|
||||
if unsafe { AXUIElementPerformAction(bar.0, ax_action.as_concrete_TypeRef()) }
|
||||
!= kAXErrorSuccess
|
||||
{
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
if ok {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let scroll_action_name = match direction {
|
||||
let page_action = match direction {
|
||||
Direction::Down => "AXScrollDownByPage",
|
||||
Direction::Up => "AXScrollUpByPage",
|
||||
Direction::Right => "AXScrollRightByPage",
|
||||
Direction::Left => "AXScrollLeftByPage",
|
||||
};
|
||||
|
||||
if crate::actions::dispatch::has_ax_action(target, scroll_action_name) {
|
||||
let ax_action = CFString::new(scroll_action_name);
|
||||
if crate::actions::dispatch::has_ax_action(target, page_action) {
|
||||
let ax = CFString::new(page_action);
|
||||
for _ in 0..amount {
|
||||
unsafe { AXUIElementPerformAction(target.0, ax_action.as_concrete_TypeRef()) };
|
||||
unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) };
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(bar) = get_scroll_bar(target, bar_attr) {
|
||||
if try_scroll_bar_value_shift(&bar, direction, amount) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_scroll_bar_sub_elements(&bar, direction) {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
if try_focus_child_in_direction(target, direction) {
|
||||
return Ok(());
|
||||
}
|
||||
if try_select_row_in_direction(target, direction) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
|
||||
let keycode: u16 = match direction {
|
||||
Direction::Down => 121,
|
||||
Direction::Up => 116,
|
||||
Direction::Right => 124,
|
||||
Direction::Left => 123,
|
||||
};
|
||||
let cf_focused = CFString::new("AXFocused");
|
||||
unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
target.0,
|
||||
cf_focused.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let app = crate::tree::element_for_pid(pid);
|
||||
for _ in 0..amount {
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, keycode, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, keycode, false);
|
||||
};
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
|
||||
let _ = crate::system::app_ops::ensure_app_focused(pid);
|
||||
if let Some(b) = crate::tree::read_bounds(target) {
|
||||
let (dy, dx) = match direction {
|
||||
Direction::Down => (-(amount as i32) * 5, 0),
|
||||
Direction::Up => (amount as i32 * 5, 0),
|
||||
Direction::Right => (0, -(amount as i32) * 5),
|
||||
Direction::Left => (0, amount as i32 * 5),
|
||||
};
|
||||
return crate::input::mouse::synthesize_scroll_at(
|
||||
b.x + b.width / 2.0,
|
||||
b.y + b.height / 2.0,
|
||||
dy,
|
||||
dx,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
"No scroll bar or scroll action found on this element",
|
||||
"No scroll mechanism found on element",
|
||||
)
|
||||
.with_suggestion("Element may not be scrollable, or try scrolling the parent container."))
|
||||
.with_suggestion("Element may not be scrollable, or try the parent container."))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_scroll_bar_value_shift(
|
||||
bar: &AXElement,
|
||||
direction: &agent_desktop_core::action::Direction,
|
||||
amount: u32,
|
||||
) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
|
||||
use agent_desktop_core::action::Direction;
|
||||
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
|
||||
|
||||
if !crate::actions::activate::is_attr_settable(bar, "AXValue") {
|
||||
return false;
|
||||
}
|
||||
let current = read_scroll_bar_value(bar).unwrap_or(0.0);
|
||||
let delta = 0.1 * amount as f64;
|
||||
let new_val = match direction {
|
||||
Direction::Down | Direction::Right => (current + delta).min(1.0),
|
||||
Direction::Up | Direction::Left => (current - delta).max(0.0),
|
||||
};
|
||||
let cf_num = CFNumber::from(new_val as f32);
|
||||
let cf_attr = CFString::new("AXValue");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), cf_num.as_CFTypeRef())
|
||||
};
|
||||
err == kAXErrorSuccess
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn read_scroll_bar_value(bar: &AXElement) -> Option<f64> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue};
|
||||
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
|
||||
|
||||
let cf_attr = CFString::new("AXValue");
|
||||
let mut value: core_foundation::base::CFTypeRef = std::ptr::null_mut();
|
||||
let err =
|
||||
unsafe { AXUIElementCopyAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), &mut value) };
|
||||
if err != kAXErrorSuccess || value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) };
|
||||
cf.downcast::<CFNumber>().and_then(|n| n.to_f64())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_scroll_bar_sub_elements(
|
||||
bar: &AXElement,
|
||||
direction: &agent_desktop_core::action::Direction,
|
||||
) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
|
||||
use agent_desktop_core::action::Direction;
|
||||
use core_foundation::{base::TCFType, string::CFString};
|
||||
|
||||
let children = crate::tree::copy_ax_array(bar, "AXChildren").unwrap_or_default();
|
||||
let target_subroles = match direction {
|
||||
Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"],
|
||||
Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"],
|
||||
};
|
||||
let press = CFString::new("AXPress");
|
||||
for child in &children {
|
||||
let sr = crate::tree::copy_string_attr(child, "AXSubrole").unwrap_or_default();
|
||||
if target_subroles.iter().any(|t| *t == sr)
|
||||
&& unsafe { AXUIElementPerformAction(child.0, press.as_concrete_TypeRef()) }
|
||||
== kAXErrorSuccess
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_focus_child_in_direction(
|
||||
scroll_area: &AXElement,
|
||||
_direction: &agent_desktop_core::action::Direction,
|
||||
) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
|
||||
let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default();
|
||||
let child = match children.first() {
|
||||
Some(c) => c,
|
||||
None => return false,
|
||||
};
|
||||
let grandchildren = crate::tree::copy_ax_array(child, "AXChildren").unwrap_or_default();
|
||||
let target = match grandchildren.last() {
|
||||
Some(t) => t,
|
||||
None => return false,
|
||||
};
|
||||
let cf_attr = CFString::new("AXFocused");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
target.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
err == kAXErrorSuccess
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn try_select_row_in_direction(
|
||||
scroll_area: &AXElement,
|
||||
_direction: &agent_desktop_core::action::Direction,
|
||||
) -> bool {
|
||||
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
|
||||
use core_foundation::{
|
||||
array::CFArray,
|
||||
base::{CFRetain, CFType, CFTypeRef, TCFType},
|
||||
string::CFString,
|
||||
};
|
||||
|
||||
let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default();
|
||||
for child in &children {
|
||||
let role = crate::tree::copy_string_attr(child, kAXRoleAttribute);
|
||||
if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) {
|
||||
continue;
|
||||
}
|
||||
if !crate::actions::activate::is_attr_settable(child, "AXSelectedRows") {
|
||||
continue;
|
||||
}
|
||||
let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default();
|
||||
if let Some(last) = rows.last() {
|
||||
unsafe { CFRetain(last.0 as CFTypeRef) };
|
||||
let el_as_cftype = unsafe { CFType::wrap_under_create_rule(last.0 as CFTypeRef) };
|
||||
let arr = CFArray::from_CFTypes(&[el_as_cftype]);
|
||||
let cf_attr = CFString::new("AXSelectedRows");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
child.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
arr.as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
if err == kAXErrorSuccess {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn find_scroll_area(el: &AXElement) -> Option<AXElement> {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
|
||||
let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?;
|
||||
if role == "AXScrollArea" {
|
||||
return Some(el.clone());
|
||||
}
|
||||
let parent = crate::tree::copy_element_attr(el, "AXParent")?;
|
||||
let parent_role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute)?;
|
||||
if parent_role == "AXScrollArea" {
|
||||
return Some(parent);
|
||||
let mut current = crate::tree::copy_element_attr(el, "AXParent")?;
|
||||
for _ in 0..5 {
|
||||
let r = crate::tree::copy_string_attr(¤t, kAXRoleAttribute)?;
|
||||
if r == "AXScrollArea" {
|
||||
return Some(current);
|
||||
}
|
||||
current = crate::tree::copy_element_attr(¤t, "AXParent")?;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,6 +129,33 @@ mod imp {
|
|||
MouseButton::Middle => CGEventType::OtherMouseUp,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn synthesize_scroll_at(x: f64, y: f64, dy: i32, dx: i32) -> Result<(), AdapterError> {
|
||||
use core_graphics::geometry::CGPoint;
|
||||
|
||||
extern "C" {
|
||||
fn CGEventCreateScrollWheelEvent(
|
||||
source: *const std::ffi::c_void,
|
||||
units: u32,
|
||||
wheel_count: u32,
|
||||
wheel1: i32,
|
||||
wheel2: i32,
|
||||
) -> *mut std::ffi::c_void;
|
||||
fn CGEventSetLocation(event: *mut std::ffi::c_void, point: CGPoint);
|
||||
fn CGEventPost(tap: u32, event: *mut std::ffi::c_void);
|
||||
}
|
||||
|
||||
let event = unsafe { CGEventCreateScrollWheelEvent(std::ptr::null(), 0, 2, dy, dx) };
|
||||
if event.is_null() {
|
||||
return Err(AdapterError::internal("scroll event creation failed"));
|
||||
}
|
||||
unsafe {
|
||||
CGEventSetLocation(event, CGPoint::new(x, y));
|
||||
CGEventPost(0, event);
|
||||
core_foundation::base::CFRelease(event as _);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
@ -142,6 +169,10 @@ mod imp {
|
|||
pub fn synthesize_drag(_params: DragParams) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("drag"))
|
||||
}
|
||||
|
||||
pub fn synthesize_scroll_at(_x: f64, _y: f64, _dy: i32, _dx: i32) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("scroll"))
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::{synthesize_drag, synthesize_mouse};
|
||||
pub use imp::{synthesize_drag, synthesize_mouse, synthesize_scroll_at};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,39 @@
|
|||
use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn pid_from_element(el: &crate::tree::AXElement) -> Option<i32> {
|
||||
let mut pid: i32 = 0;
|
||||
let err = unsafe { accessibility_sys::AXUIElementGetPid(el.0, &mut pid) };
|
||||
if err == accessibility_sys::kAXErrorSuccess {
|
||||
Some(pid)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> {
|
||||
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
|
||||
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
|
||||
|
||||
let app_el = crate::tree::element_for_pid(pid);
|
||||
let frontmost_attr = CFString::new("AXFrontmost");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
app_el.0,
|
||||
frontmost_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return Err(AdapterError::internal(format!(
|
||||
"Failed to focus app pid={pid}"
|
||||
)));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> {
|
||||
use accessibility_sys::{
|
||||
|
|
@ -162,22 +196,23 @@ fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool {
|
|||
let Some(bar_items) = crate::tree::copy_ax_array(&menu_bar, "AXChildren") else {
|
||||
return false;
|
||||
};
|
||||
if let Some(app_menu) = bar_items.first() {
|
||||
if let Some(menus) = crate::tree::copy_ax_array(app_menu, "AXChildren") {
|
||||
for menu in &menus {
|
||||
if let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") {
|
||||
for item in &items {
|
||||
let title = crate::tree::copy_string_attr(item, "AXTitle");
|
||||
if let Some(t) = &title {
|
||||
if t.contains("Quit") || t.contains("quit") {
|
||||
let press = CFString::new("AXPress");
|
||||
let err = unsafe {
|
||||
AXUIElementPerformAction(item.0, press.as_concrete_TypeRef())
|
||||
};
|
||||
return err == kAXErrorSuccess;
|
||||
}
|
||||
}
|
||||
}
|
||||
for bar_item in bar_items.iter().skip(1) {
|
||||
let Some(menus) = crate::tree::copy_ax_array(bar_item, "AXChildren") else {
|
||||
continue;
|
||||
};
|
||||
for menu in &menus {
|
||||
let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") else {
|
||||
continue;
|
||||
};
|
||||
for item in &items {
|
||||
let Some(t) = crate::tree::copy_string_attr(item, "AXTitle") else {
|
||||
continue;
|
||||
};
|
||||
if t.starts_with("Quit") {
|
||||
let press = CFString::new("AXPress");
|
||||
let err =
|
||||
unsafe { AXUIElementPerformAction(item.0, press.as_concrete_TypeRef()) };
|
||||
return err == kAXErrorSuccess;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ pub fn build_subtree(
|
|||
el: &AXElement,
|
||||
depth: u8,
|
||||
max_depth: u8,
|
||||
include_bounds: bool,
|
||||
_include_bounds: bool,
|
||||
ancestors: &mut FxHashSet<usize>,
|
||||
) -> Option<AccessibilityNode> {
|
||||
if depth > max_depth || depth >= ABSOLUTE_MAX_DEPTH {
|
||||
|
|
@ -81,18 +81,14 @@ pub fn build_subtree(
|
|||
states.push("disabled".into());
|
||||
}
|
||||
|
||||
let bounds = if include_bounds {
|
||||
read_bounds(el)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let bounds = read_bounds(el);
|
||||
|
||||
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
|
||||
let name = name.or_else(|| label_from_children(&children_raw));
|
||||
|
||||
let children = children_raw
|
||||
.into_iter()
|
||||
.filter_map(|child| build_subtree(&child, depth + 1, max_depth, include_bounds, ancestors))
|
||||
.filter_map(|child| build_subtree(&child, depth + 1, max_depth, _include_bounds, ancestors))
|
||||
.collect();
|
||||
|
||||
ancestors.remove(&ptr_key);
|
||||
|
|
|
|||
|
|
@ -41,7 +41,22 @@ pub fn find_element_recursive(
|
|||
(None, None) => true,
|
||||
_ => false,
|
||||
};
|
||||
if name_match {
|
||||
let bounds_match = match entry.bounds_hash {
|
||||
Some(expected) => {
|
||||
let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash());
|
||||
tracing::debug!(
|
||||
role = normalized,
|
||||
?elem_name,
|
||||
?actual,
|
||||
expected,
|
||||
name_match,
|
||||
"resolve candidate"
|
||||
);
|
||||
actual.map(|h| h == expected).unwrap_or(false)
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
if name_match && bounds_match {
|
||||
ancestors.remove(&ptr_key);
|
||||
unsafe { CFRetain(el.0 as CFTypeRef) };
|
||||
return Ok(NativeHandle::from_ptr(el.0 as *const _));
|
||||
|
|
|
|||
Loading…
Reference in a new issue