mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-08-09 08:27:25 +00:00
fix: implement core accessible-name precedence algorithm
accname.rs previously reduced NameEvidence with a 2-way title-or-description fallback despite the trait already exposing 7 evidence fields. Implement the documented KTD6 precedence (explicit label -> labelled-by text -> native title -> static-role value -> aggregated child label -> placeholder -> description last) as compute_name/compute_description over NameEvidence, plus a join_child_labels aggregation primitive. Migrate macOS's resolve_element_name to a thin wrapper that gathers raw NameEvidence (name_evidence_impl) and reduces it via core's compute_name, removing the inline fallback chain it used to own. name_evidence_impl now reads AXTitleUIElement (labelled-by), AXPlaceholderValue, and aggregates multiple child labels via join_child_labels instead of returning only the first match. resolve_search.rs, resolve_classify.rs, and chain_menu_steps.rs need no changes: they already consume resolve_element_name's return value and now get the correct precedence for free.
This commit is contained in:
parent
b5c595468d
commit
2a518d8a2a
5 changed files with 360 additions and 55 deletions
|
|
@ -1,17 +1,102 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
/// Raw, unreduced evidence an adapter gathers for one element's accessible
|
||||
/// name/description. Fields carry native attribute text as-is; the
|
||||
/// precedence between them is decided exclusively by [`compute_name`] and
|
||||
/// [`compute_description`] in this module — adapters must never apply their
|
||||
/// own fallback chain (see `docs/plans/.../KTD6`).
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct NameEvidence {
|
||||
/// Rung 1 — an explicit, authoritative label distinct from a
|
||||
/// labelled-by reference. UIA: `Name` when sourced from an explicit
|
||||
/// `AutomationProperties.Name`-style override. AT-SPI: an explicit
|
||||
/// `label` property, when the toolkit exposes one separately from the
|
||||
/// `LABELLED_BY` relation. macOS has no attribute distinct from
|
||||
/// `AXTitleUIElement`, so this rung is always `None` on macOS today.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
pub explicit_label: Option<String>,
|
||||
/// Rung 2 — text drawn from another element the platform designates as
|
||||
/// this element's label. macOS: `AXTitleUIElement` (resolved to that
|
||||
/// element's own title/value text). UIA: `LabeledBy`. AT-SPI:
|
||||
/// `LABELLED_BY` relation target's name.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub labelled_by_text: Option<String>,
|
||||
/// Rung 3 — the element's own native title/name string. macOS:
|
||||
/// `AXTitle`. UIA: `Name` (default, unlabelled case). AT-SPI: `name`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub native_title: Option<String>,
|
||||
/// Rung 4 — the element's value promoted to a name, only for
|
||||
/// non-interactive, static/read-only roles (e.g. macOS `AXValue` on
|
||||
/// `AXStaticText`; UIA `ValuePattern.Value` on a `Text` control). The
|
||||
/// adapter is responsible for the role gating; core treats presence of
|
||||
/// this field as sufficient.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub static_role_value: Option<String>,
|
||||
/// Rung 5 — text aggregated from descendant labels in document order,
|
||||
/// for containers with no direct label of their own (a toolbar button
|
||||
/// wrapping an icon + text child, for example). Adapters build this
|
||||
/// with [`join_child_labels`] over the per-child text they collect.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub child_label: Option<String>,
|
||||
/// Rung 6 — placeholder/hint text shown while the control is empty.
|
||||
/// macOS: `AXPlaceholderValue`. UIA: `HelpText` used as a placeholder
|
||||
/// fallback. AT-SPI: `placeholder-text`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub placeholder: Option<String>,
|
||||
/// Rung 7 (name fallback of last resort) and also the accessible
|
||||
/// description proper via [`compute_description`]. macOS: `AXDescription`.
|
||||
/// UIA: `HelpText`. AT-SPI: `description`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub value_promoted: bool,
|
||||
pub child_label: bool,
|
||||
}
|
||||
|
||||
impl NameEvidence {
|
||||
pub fn resolved_name(&self) -> Option<String> {
|
||||
self.title.clone().or_else(|| self.description.clone())
|
||||
fn non_empty(value: &Option<String>) -> Option<String> {
|
||||
value
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Reduces [`NameEvidence`] to a single accessible name following the
|
||||
/// documented 7-rung precedence: explicit label, labelled-by text, native
|
||||
/// title, static-role value, aggregated child label, placeholder,
|
||||
/// description last. The earliest non-empty rung wins; core never re-derives
|
||||
/// evidence, it only picks among what the adapter supplied.
|
||||
pub fn compute_name(evidence: &NameEvidence) -> Option<String> {
|
||||
non_empty(&evidence.explicit_label)
|
||||
.or_else(|| non_empty(&evidence.labelled_by_text))
|
||||
.or_else(|| non_empty(&evidence.native_title))
|
||||
.or_else(|| non_empty(&evidence.static_role_value))
|
||||
.or_else(|| non_empty(&evidence.child_label))
|
||||
.or_else(|| non_empty(&evidence.placeholder))
|
||||
.or_else(|| non_empty(&evidence.description))
|
||||
}
|
||||
|
||||
/// Reduces [`NameEvidence`] to the accessible description: the raw
|
||||
/// description rung, independent of whether it was also consumed as the
|
||||
/// name's rung-7 fallback.
|
||||
pub fn compute_description(evidence: &NameEvidence) -> Option<String> {
|
||||
non_empty(&evidence.description)
|
||||
}
|
||||
|
||||
/// Joins per-child label text into a single aggregated child-label rung,
|
||||
/// in the document order the caller supplies them, trimming and dropping
|
||||
/// empty entries. Returns `None` when nothing survives.
|
||||
pub fn join_child_labels<'a, I: IntoIterator<Item = &'a str>>(labels: I) -> Option<String> {
|
||||
let joined = labels
|
||||
.into_iter()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
if joined.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(joined)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "accname_tests.rs"]
|
||||
mod tests;
|
||||
|
|
|
|||
147
crates/core/src/accname_tests.rs
Normal file
147
crates/core/src/accname_tests.rs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
use super::*;
|
||||
|
||||
fn evidence_all_rungs() -> NameEvidence {
|
||||
NameEvidence {
|
||||
explicit_label: Some("explicit".into()),
|
||||
labelled_by_text: Some("labelled-by".into()),
|
||||
native_title: Some("native-title".into()),
|
||||
static_role_value: Some("static-value".into()),
|
||||
child_label: Some("child-label".into()),
|
||||
placeholder: Some("placeholder".into()),
|
||||
description: Some("description".into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung1_explicit_label_wins_over_every_lower_rung() {
|
||||
let evidence = evidence_all_rungs();
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("explicit"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung2_labelled_by_text_wins_when_explicit_absent() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.explicit_label = None;
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("labelled-by"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung3_native_title_wins_when_rungs_1_2_absent() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.explicit_label = None;
|
||||
evidence.labelled_by_text = None;
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("native-title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung4_static_role_value_wins_when_rungs_1_3_absent() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.explicit_label = None;
|
||||
evidence.labelled_by_text = None;
|
||||
evidence.native_title = None;
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("static-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung5_child_label_wins_when_rungs_1_4_absent() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.explicit_label = None;
|
||||
evidence.labelled_by_text = None;
|
||||
evidence.native_title = None;
|
||||
evidence.static_role_value = None;
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("child-label"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung6_placeholder_wins_when_rungs_1_5_absent() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.explicit_label = None;
|
||||
evidence.labelled_by_text = None;
|
||||
evidence.native_title = None;
|
||||
evidence.static_role_value = None;
|
||||
evidence.child_label = None;
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("placeholder"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung7_description_is_last_resort_for_name() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.explicit_label = None;
|
||||
evidence.labelled_by_text = None;
|
||||
evidence.native_title = None;
|
||||
evidence.static_role_value = None;
|
||||
evidence.child_label = None;
|
||||
evidence.placeholder = None;
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("description"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_absent_evidence_computes_no_name() {
|
||||
let evidence = NameEvidence::default();
|
||||
assert_eq!(compute_name(&evidence), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_blank_evidence_computes_no_name() {
|
||||
let evidence = NameEvidence {
|
||||
explicit_label: Some(" ".into()),
|
||||
labelled_by_text: Some("".into()),
|
||||
native_title: Some("\t".into()),
|
||||
static_role_value: Some(String::new()),
|
||||
child_label: Some(" ".into()),
|
||||
placeholder: Some(String::new()),
|
||||
description: Some(" ".into()),
|
||||
};
|
||||
assert_eq!(compute_name(&evidence), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_description_returns_description_rung_independent_of_name() {
|
||||
let evidence = evidence_all_rungs();
|
||||
assert_eq!(
|
||||
compute_description(&evidence).as_deref(),
|
||||
Some("description")
|
||||
);
|
||||
assert_eq!(compute_name(&evidence).as_deref(), Some("explicit"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_description_none_when_description_absent() {
|
||||
let mut evidence = evidence_all_rungs();
|
||||
evidence.description = None;
|
||||
assert_eq!(compute_description(&evidence), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_label_aggregation_joins_in_document_order() {
|
||||
let joined = join_child_labels(["Save", "As...", "PDF"]);
|
||||
assert_eq!(joined.as_deref(), Some("Save As... PDF"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_label_aggregation_drops_blank_entries_but_preserves_order() {
|
||||
let joined = join_child_labels(["", "First", " ", "Second"]);
|
||||
assert_eq!(joined.as_deref(), Some("First Second"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_label_aggregation_of_all_blank_entries_is_none() {
|
||||
assert_eq!(join_child_labels(["", " ", "\t"]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_label_aggregation_of_empty_iterator_is_none() {
|
||||
assert_eq!(join_child_labels(std::iter::empty()), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_evidence_serde_roundtrip_skips_absent_fields() {
|
||||
let evidence = NameEvidence {
|
||||
native_title: Some("Only Title".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let json = serde_json::to_value(&evidence).expect("serialize");
|
||||
assert_eq!(json, serde_json::json!({ "native_title": "Only Title" }));
|
||||
let round_tripped: NameEvidence = serde_json::from_value(json).expect("deserialize");
|
||||
assert_eq!(round_tripped, evidence);
|
||||
}
|
||||
|
|
@ -18,8 +18,7 @@ mod imp {
|
|||
tree::{
|
||||
NodeAttrs,
|
||||
attributes::{
|
||||
copy_ax_array, copy_bool_attr, copy_first_element_attr, copy_string_attr,
|
||||
copy_value_typed,
|
||||
copy_bool_attr, copy_first_element_attr, copy_string_attr, copy_value_typed,
|
||||
},
|
||||
ax_element::AXElement,
|
||||
ax_value,
|
||||
|
|
@ -240,27 +239,6 @@ mod imp {
|
|||
)
|
||||
}
|
||||
|
||||
pub fn resolve_element_name(el: &AXElement) -> Option<String> {
|
||||
let ax_role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, kAXTitleAttribute);
|
||||
let desc = copy_string_attr(el, kAXDescriptionAttribute);
|
||||
|
||||
let name = title.or(desc);
|
||||
let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") {
|
||||
copy_string_attr(el, kAXValueAttribute).or(name)
|
||||
} else {
|
||||
name
|
||||
};
|
||||
|
||||
name.or_else(|| {
|
||||
let children = super::child_attributes(ax_role.as_deref())
|
||||
.iter()
|
||||
.find_map(|attr| copy_ax_array(el, attr).filter(|v| !v.is_empty()))
|
||||
.unwrap_or_default();
|
||||
crate::tree::builder::label_from_children(&children)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn count_children(element: &AXElement, ax_role: Option<&str>) -> u32 {
|
||||
for attr_name in child_attributes(ax_role) {
|
||||
let mut count: core_foundation_sys::base::CFIndex = 0;
|
||||
|
|
@ -291,13 +269,22 @@ mod imp {
|
|||
0
|
||||
}
|
||||
|
||||
pub fn resolve_element_name(_el: &AXElement) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn fetch_node_attrs(_el: &AXElement) -> NodeAttrs {
|
||||
NodeAttrs::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::{count_children, element_for_pid, fetch_node_attrs, resolve_element_name};
|
||||
pub use imp::{count_children, element_for_pid, fetch_node_attrs};
|
||||
|
||||
/// The element's accessible name, reduced from adapter-supplied
|
||||
/// [`agent_desktop_core::accname::NameEvidence`] by core's precedence
|
||||
/// algorithm. This function is a thin caller-facing wrapper — it owns no
|
||||
/// fallback chain of its own; `name_evidence` module supplies raw evidence,
|
||||
/// `compute_name` decides precedence.
|
||||
pub fn resolve_element_name(el: &super::ax_element::AXElement) -> Option<String> {
|
||||
agent_desktop_core::accname::compute_name(&super::name_evidence::name_evidence_impl(el))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "element_tests.rs"]
|
||||
mod tests;
|
||||
|
|
|
|||
29
crates/macos/src/tree/element_tests.rs
Normal file
29
crates/macos/src/tree/element_tests.rs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
const ELEMENT_SOURCE: &str = include_str!("element.rs");
|
||||
|
||||
#[test]
|
||||
fn resolve_element_name_delegates_precedence_to_core_accname() {
|
||||
assert!(
|
||||
ELEMENT_SOURCE.contains("agent_desktop_core::accname::compute_name"),
|
||||
"resolve_element_name must reduce NameEvidence through core's compute_name (KTD6); \
|
||||
found no call to accname::compute_name in element.rs"
|
||||
);
|
||||
assert!(
|
||||
ELEMENT_SOURCE.contains("super::name_evidence::name_evidence_impl"),
|
||||
"resolve_element_name must gather evidence via the name_evidence supplier, not read \
|
||||
AX attributes itself"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_element_name_owns_no_local_fallback_chain() {
|
||||
assert!(
|
||||
!ELEMENT_SOURCE.contains("title.or(desc)"),
|
||||
"resolve_element_name regressed to owning its own title/description fallback chain; \
|
||||
precedence belongs in accname::compute_name (KTD6)"
|
||||
);
|
||||
assert!(
|
||||
!ELEMENT_SOURCE.contains("kAXValueAttribute).or(name)"),
|
||||
"resolve_element_name regressed to owning its own static-role-value fallback; \
|
||||
precedence belongs in accname::compute_name (KTD6)"
|
||||
);
|
||||
}
|
||||
|
|
@ -1,25 +1,87 @@
|
|||
#[cfg(target_os = "macos")]
|
||||
mod imp {
|
||||
use crate::tree::{AXElement, copy_string_attr, resolve_element_name};
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
use agent_desktop_core::accname::NameEvidence;
|
||||
use crate::tree::AXElement;
|
||||
use crate::tree::attributes::{
|
||||
copy_ax_array, copy_ax_array_prefix, copy_element_attr, copy_string_attr,
|
||||
};
|
||||
use crate::tree::element::child_attributes;
|
||||
use accessibility_sys::{
|
||||
kAXChildrenAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute,
|
||||
kAXRoleAttribute, kAXTitleAttribute, kAXTitleUIElementAttribute, kAXValueAttribute,
|
||||
};
|
||||
use agent_desktop_core::accname::{NameEvidence, join_child_labels};
|
||||
|
||||
const STATIC_TEXT_ROLE: &str = "AXStaticText";
|
||||
|
||||
/// Gathers raw `NameEvidence` from an element's own AX attributes.
|
||||
/// Every rung is a direct attribute read — precedence between rungs is
|
||||
/// core's job via `accname::compute_name`, never decided here.
|
||||
pub fn name_evidence_impl(el: &AXElement) -> NameEvidence {
|
||||
let ax_role = copy_string_attr(el, kAXRoleAttribute);
|
||||
let title = copy_string_attr(el, "AXTitle");
|
||||
let description = copy_string_attr(el, "AXDescription");
|
||||
let value = copy_string_attr(el, "AXValue");
|
||||
let value_promoted =
|
||||
title.is_none() && description.is_none() && ax_role.as_deref() == Some("AXStaticText");
|
||||
let resolved = resolve_element_name(el);
|
||||
let child_label = resolved.is_some() && title.is_none() && description.is_none();
|
||||
NameEvidence {
|
||||
title,
|
||||
description,
|
||||
value_promoted: value_promoted && value.is_some(),
|
||||
child_label,
|
||||
explicit_label: None,
|
||||
labelled_by_text: labelled_by_text(el),
|
||||
native_title: copy_string_attr(el, kAXTitleAttribute),
|
||||
static_role_value: static_role_value(el, ax_role.as_deref()),
|
||||
child_label: aggregated_child_label(el, ax_role.as_deref()),
|
||||
placeholder: copy_string_attr(el, kAXPlaceholderValueAttribute),
|
||||
description: copy_string_attr(el, kAXDescriptionAttribute),
|
||||
}
|
||||
}
|
||||
|
||||
fn labelled_by_text(el: &AXElement) -> Option<String> {
|
||||
let label_el = copy_element_attr(el, kAXTitleUIElementAttribute)?;
|
||||
copy_string_attr(&label_el, kAXTitleAttribute)
|
||||
.or_else(|| copy_string_attr(&label_el, kAXValueAttribute))
|
||||
}
|
||||
|
||||
fn static_role_value(el: &AXElement, ax_role: Option<&str>) -> Option<String> {
|
||||
if ax_role != Some(STATIC_TEXT_ROLE) {
|
||||
return None;
|
||||
}
|
||||
copy_string_attr(el, kAXValueAttribute)
|
||||
}
|
||||
|
||||
fn aggregated_child_label(el: &AXElement, ax_role: Option<&str>) -> Option<String> {
|
||||
let children = child_attributes(ax_role)
|
||||
.iter()
|
||||
.find_map(|attr| copy_ax_array(el, attr).filter(|v| !v.is_empty()))
|
||||
.unwrap_or_default();
|
||||
let texts = collect_child_texts(&children);
|
||||
join_child_labels(texts.iter().map(String::as_str))
|
||||
}
|
||||
|
||||
fn collect_child_texts(children: &[AXElement]) -> Vec<String> {
|
||||
fn text_of(el: &AXElement) -> Option<String> {
|
||||
copy_string_attr(el, kAXValueAttribute)
|
||||
.or_else(|| copy_string_attr(el, kAXTitleAttribute))
|
||||
}
|
||||
|
||||
let mut texts = Vec::new();
|
||||
for child in children.iter().take(5) {
|
||||
match copy_string_attr(child, kAXRoleAttribute).as_deref() {
|
||||
Some("AXStaticText") => {
|
||||
if let Some(text) = text_of(child) {
|
||||
texts.push(text);
|
||||
}
|
||||
}
|
||||
Some("AXCell") | Some("AXGroup") => {
|
||||
for grandchild in
|
||||
copy_ax_array_prefix(child, kAXChildrenAttribute, 5).unwrap_or_default()
|
||||
{
|
||||
if copy_string_attr(&grandchild, kAXRoleAttribute).as_deref()
|
||||
== Some("AXStaticText")
|
||||
&& let Some(text) = text_of(&grandchild)
|
||||
{
|
||||
texts.push(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
texts
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
@ -28,12 +90,7 @@ mod imp {
|
|||
use agent_desktop_core::accname::NameEvidence;
|
||||
|
||||
pub fn name_evidence_impl(_el: &AXElement) -> NameEvidence {
|
||||
NameEvidence {
|
||||
title: None,
|
||||
description: None,
|
||||
value_promoted: false,
|
||||
child_label: false,
|
||||
}
|
||||
NameEvidence::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue