fix: implement core accessible-name precedence algorithm

accname.rs previously reduced NameEvidence with a 2-way title-or-description
fallback despite the trait already exposing 7 evidence fields. Implement the
documented KTD6 precedence (explicit label -> labelled-by text -> native
title -> static-role value -> aggregated child label -> placeholder ->
description last) as compute_name/compute_description over NameEvidence,
plus a join_child_labels aggregation primitive.

Migrate macOS's resolve_element_name to a thin wrapper that gathers raw
NameEvidence (name_evidence_impl) and reduces it via core's compute_name,
removing the inline fallback chain it used to own. name_evidence_impl now
reads AXTitleUIElement (labelled-by), AXPlaceholderValue, and aggregates
multiple child labels via join_child_labels instead of returning only the
first match. resolve_search.rs, resolve_classify.rs, and chain_menu_steps.rs
need no changes: they already consume resolve_element_name's return value
and now get the correct precedence for free.
This commit is contained in:
Lahfir 2026-07-03 02:43:01 -07:00
parent b5c595468d
commit 2a518d8a2a
5 changed files with 360 additions and 55 deletions

View file

@ -1,17 +1,102 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
/// Raw, unreduced evidence an adapter gathers for one element's accessible
/// name/description. Fields carry native attribute text as-is; the
/// precedence between them is decided exclusively by [`compute_name`] and
/// [`compute_description`] in this module — adapters must never apply their
/// own fallback chain (see `docs/plans/.../KTD6`).
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct NameEvidence {
/// Rung 1 — an explicit, authoritative label distinct from a
/// labelled-by reference. UIA: `Name` when sourced from an explicit
/// `AutomationProperties.Name`-style override. AT-SPI: an explicit
/// `label` property, when the toolkit exposes one separately from the
/// `LABELLED_BY` relation. macOS has no attribute distinct from
/// `AXTitleUIElement`, so this rung is always `None` on macOS today.
#[serde(skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
pub explicit_label: Option<String>,
/// Rung 2 — text drawn from another element the platform designates as
/// this element's label. macOS: `AXTitleUIElement` (resolved to that
/// element's own title/value text). UIA: `LabeledBy`. AT-SPI:
/// `LABELLED_BY` relation target's name.
#[serde(skip_serializing_if = "Option::is_none")]
pub labelled_by_text: Option<String>,
/// Rung 3 — the element's own native title/name string. macOS:
/// `AXTitle`. UIA: `Name` (default, unlabelled case). AT-SPI: `name`.
#[serde(skip_serializing_if = "Option::is_none")]
pub native_title: Option<String>,
/// Rung 4 — the element's value promoted to a name, only for
/// non-interactive, static/read-only roles (e.g. macOS `AXValue` on
/// `AXStaticText`; UIA `ValuePattern.Value` on a `Text` control). The
/// adapter is responsible for the role gating; core treats presence of
/// this field as sufficient.
#[serde(skip_serializing_if = "Option::is_none")]
pub static_role_value: Option<String>,
/// Rung 5 — text aggregated from descendant labels in document order,
/// for containers with no direct label of their own (a toolbar button
/// wrapping an icon + text child, for example). Adapters build this
/// with [`join_child_labels`] over the per-child text they collect.
#[serde(skip_serializing_if = "Option::is_none")]
pub child_label: Option<String>,
/// Rung 6 — placeholder/hint text shown while the control is empty.
/// macOS: `AXPlaceholderValue`. UIA: `HelpText` used as a placeholder
/// fallback. AT-SPI: `placeholder-text`.
#[serde(skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
/// Rung 7 (name fallback of last resort) and also the accessible
/// description proper via [`compute_description`]. macOS: `AXDescription`.
/// UIA: `HelpText`. AT-SPI: `description`.
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
pub value_promoted: bool,
pub child_label: bool,
}
impl NameEvidence {
pub fn resolved_name(&self) -> Option<String> {
self.title.clone().or_else(|| self.description.clone())
fn non_empty(value: &Option<String>) -> Option<String> {
value
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
}
/// Reduces [`NameEvidence`] to a single accessible name following the
/// documented 7-rung precedence: explicit label, labelled-by text, native
/// title, static-role value, aggregated child label, placeholder,
/// description last. The earliest non-empty rung wins; core never re-derives
/// evidence, it only picks among what the adapter supplied.
pub fn compute_name(evidence: &NameEvidence) -> Option<String> {
non_empty(&evidence.explicit_label)
.or_else(|| non_empty(&evidence.labelled_by_text))
.or_else(|| non_empty(&evidence.native_title))
.or_else(|| non_empty(&evidence.static_role_value))
.or_else(|| non_empty(&evidence.child_label))
.or_else(|| non_empty(&evidence.placeholder))
.or_else(|| non_empty(&evidence.description))
}
/// Reduces [`NameEvidence`] to the accessible description: the raw
/// description rung, independent of whether it was also consumed as the
/// name's rung-7 fallback.
pub fn compute_description(evidence: &NameEvidence) -> Option<String> {
non_empty(&evidence.description)
}
/// Joins per-child label text into a single aggregated child-label rung,
/// in the document order the caller supplies them, trimming and dropping
/// empty entries. Returns `None` when nothing survives.
pub fn join_child_labels<'a, I: IntoIterator<Item = &'a str>>(labels: I) -> Option<String> {
let joined = labels
.into_iter()
.map(str::trim)
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
.join(" ");
if joined.is_empty() {
None
} else {
Some(joined)
}
}
#[cfg(test)]
#[path = "accname_tests.rs"]
mod tests;

View file

@ -0,0 +1,147 @@
use super::*;
fn evidence_all_rungs() -> NameEvidence {
NameEvidence {
explicit_label: Some("explicit".into()),
labelled_by_text: Some("labelled-by".into()),
native_title: Some("native-title".into()),
static_role_value: Some("static-value".into()),
child_label: Some("child-label".into()),
placeholder: Some("placeholder".into()),
description: Some("description".into()),
}
}
#[test]
fn rung1_explicit_label_wins_over_every_lower_rung() {
let evidence = evidence_all_rungs();
assert_eq!(compute_name(&evidence).as_deref(), Some("explicit"));
}
#[test]
fn rung2_labelled_by_text_wins_when_explicit_absent() {
let mut evidence = evidence_all_rungs();
evidence.explicit_label = None;
assert_eq!(compute_name(&evidence).as_deref(), Some("labelled-by"));
}
#[test]
fn rung3_native_title_wins_when_rungs_1_2_absent() {
let mut evidence = evidence_all_rungs();
evidence.explicit_label = None;
evidence.labelled_by_text = None;
assert_eq!(compute_name(&evidence).as_deref(), Some("native-title"));
}
#[test]
fn rung4_static_role_value_wins_when_rungs_1_3_absent() {
let mut evidence = evidence_all_rungs();
evidence.explicit_label = None;
evidence.labelled_by_text = None;
evidence.native_title = None;
assert_eq!(compute_name(&evidence).as_deref(), Some("static-value"));
}
#[test]
fn rung5_child_label_wins_when_rungs_1_4_absent() {
let mut evidence = evidence_all_rungs();
evidence.explicit_label = None;
evidence.labelled_by_text = None;
evidence.native_title = None;
evidence.static_role_value = None;
assert_eq!(compute_name(&evidence).as_deref(), Some("child-label"));
}
#[test]
fn rung6_placeholder_wins_when_rungs_1_5_absent() {
let mut evidence = evidence_all_rungs();
evidence.explicit_label = None;
evidence.labelled_by_text = None;
evidence.native_title = None;
evidence.static_role_value = None;
evidence.child_label = None;
assert_eq!(compute_name(&evidence).as_deref(), Some("placeholder"));
}
#[test]
fn rung7_description_is_last_resort_for_name() {
let mut evidence = evidence_all_rungs();
evidence.explicit_label = None;
evidence.labelled_by_text = None;
evidence.native_title = None;
evidence.static_role_value = None;
evidence.child_label = None;
evidence.placeholder = None;
assert_eq!(compute_name(&evidence).as_deref(), Some("description"));
}
#[test]
fn all_absent_evidence_computes_no_name() {
let evidence = NameEvidence::default();
assert_eq!(compute_name(&evidence), None);
}
#[test]
fn all_blank_evidence_computes_no_name() {
let evidence = NameEvidence {
explicit_label: Some(" ".into()),
labelled_by_text: Some("".into()),
native_title: Some("\t".into()),
static_role_value: Some(String::new()),
child_label: Some(" ".into()),
placeholder: Some(String::new()),
description: Some(" ".into()),
};
assert_eq!(compute_name(&evidence), None);
}
#[test]
fn compute_description_returns_description_rung_independent_of_name() {
let evidence = evidence_all_rungs();
assert_eq!(
compute_description(&evidence).as_deref(),
Some("description")
);
assert_eq!(compute_name(&evidence).as_deref(), Some("explicit"));
}
#[test]
fn compute_description_none_when_description_absent() {
let mut evidence = evidence_all_rungs();
evidence.description = None;
assert_eq!(compute_description(&evidence), None);
}
#[test]
fn child_label_aggregation_joins_in_document_order() {
let joined = join_child_labels(["Save", "As...", "PDF"]);
assert_eq!(joined.as_deref(), Some("Save As... PDF"));
}
#[test]
fn child_label_aggregation_drops_blank_entries_but_preserves_order() {
let joined = join_child_labels(["", "First", " ", "Second"]);
assert_eq!(joined.as_deref(), Some("First Second"));
}
#[test]
fn child_label_aggregation_of_all_blank_entries_is_none() {
assert_eq!(join_child_labels(["", " ", "\t"]), None);
}
#[test]
fn child_label_aggregation_of_empty_iterator_is_none() {
assert_eq!(join_child_labels(std::iter::empty()), None);
}
#[test]
fn name_evidence_serde_roundtrip_skips_absent_fields() {
let evidence = NameEvidence {
native_title: Some("Only Title".into()),
..Default::default()
};
let json = serde_json::to_value(&evidence).expect("serialize");
assert_eq!(json, serde_json::json!({ "native_title": "Only Title" }));
let round_tripped: NameEvidence = serde_json::from_value(json).expect("deserialize");
assert_eq!(round_tripped, evidence);
}

View file

@ -18,8 +18,7 @@ mod imp {
tree::{
NodeAttrs,
attributes::{
copy_ax_array, copy_bool_attr, copy_first_element_attr, copy_string_attr,
copy_value_typed,
copy_bool_attr, copy_first_element_attr, copy_string_attr, copy_value_typed,
},
ax_element::AXElement,
ax_value,
@ -240,27 +239,6 @@ mod imp {
)
}
pub fn resolve_element_name(el: &AXElement) -> Option<String> {
let ax_role = copy_string_attr(el, kAXRoleAttribute);
let title = copy_string_attr(el, kAXTitleAttribute);
let desc = copy_string_attr(el, kAXDescriptionAttribute);
let name = title.or(desc);
let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") {
copy_string_attr(el, kAXValueAttribute).or(name)
} else {
name
};
name.or_else(|| {
let children = super::child_attributes(ax_role.as_deref())
.iter()
.find_map(|attr| copy_ax_array(el, attr).filter(|v| !v.is_empty()))
.unwrap_or_default();
crate::tree::builder::label_from_children(&children)
})
}
pub fn count_children(element: &AXElement, ax_role: Option<&str>) -> u32 {
for attr_name in child_attributes(ax_role) {
let mut count: core_foundation_sys::base::CFIndex = 0;
@ -291,13 +269,22 @@ mod imp {
0
}
pub fn resolve_element_name(_el: &AXElement) -> Option<String> {
None
}
pub fn fetch_node_attrs(_el: &AXElement) -> NodeAttrs {
NodeAttrs::default()
}
}
pub use imp::{count_children, element_for_pid, fetch_node_attrs, resolve_element_name};
pub use imp::{count_children, element_for_pid, fetch_node_attrs};
/// The element's accessible name, reduced from adapter-supplied
/// [`agent_desktop_core::accname::NameEvidence`] by core's precedence
/// algorithm. This function is a thin caller-facing wrapper — it owns no
/// fallback chain of its own; `name_evidence` module supplies raw evidence,
/// `compute_name` decides precedence.
pub fn resolve_element_name(el: &super::ax_element::AXElement) -> Option<String> {
agent_desktop_core::accname::compute_name(&super::name_evidence::name_evidence_impl(el))
}
#[cfg(test)]
#[path = "element_tests.rs"]
mod tests;

View file

@ -0,0 +1,29 @@
const ELEMENT_SOURCE: &str = include_str!("element.rs");
#[test]
fn resolve_element_name_delegates_precedence_to_core_accname() {
assert!(
ELEMENT_SOURCE.contains("agent_desktop_core::accname::compute_name"),
"resolve_element_name must reduce NameEvidence through core's compute_name (KTD6); \
found no call to accname::compute_name in element.rs"
);
assert!(
ELEMENT_SOURCE.contains("super::name_evidence::name_evidence_impl"),
"resolve_element_name must gather evidence via the name_evidence supplier, not read \
AX attributes itself"
);
}
#[test]
fn resolve_element_name_owns_no_local_fallback_chain() {
assert!(
!ELEMENT_SOURCE.contains("title.or(desc)"),
"resolve_element_name regressed to owning its own title/description fallback chain; \
precedence belongs in accname::compute_name (KTD6)"
);
assert!(
!ELEMENT_SOURCE.contains("kAXValueAttribute).or(name)"),
"resolve_element_name regressed to owning its own static-role-value fallback; \
precedence belongs in accname::compute_name (KTD6)"
);
}

View file

@ -1,25 +1,87 @@
#[cfg(target_os = "macos")]
mod imp {
use crate::tree::{AXElement, copy_string_attr, resolve_element_name};
use accessibility_sys::kAXRoleAttribute;
use agent_desktop_core::accname::NameEvidence;
use crate::tree::AXElement;
use crate::tree::attributes::{
copy_ax_array, copy_ax_array_prefix, copy_element_attr, copy_string_attr,
};
use crate::tree::element::child_attributes;
use accessibility_sys::{
kAXChildrenAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute,
kAXRoleAttribute, kAXTitleAttribute, kAXTitleUIElementAttribute, kAXValueAttribute,
};
use agent_desktop_core::accname::{NameEvidence, join_child_labels};
const STATIC_TEXT_ROLE: &str = "AXStaticText";
/// Gathers raw `NameEvidence` from an element's own AX attributes.
/// Every rung is a direct attribute read — precedence between rungs is
/// core's job via `accname::compute_name`, never decided here.
pub fn name_evidence_impl(el: &AXElement) -> NameEvidence {
let ax_role = copy_string_attr(el, kAXRoleAttribute);
let title = copy_string_attr(el, "AXTitle");
let description = copy_string_attr(el, "AXDescription");
let value = copy_string_attr(el, "AXValue");
let value_promoted =
title.is_none() && description.is_none() && ax_role.as_deref() == Some("AXStaticText");
let resolved = resolve_element_name(el);
let child_label = resolved.is_some() && title.is_none() && description.is_none();
NameEvidence {
title,
description,
value_promoted: value_promoted && value.is_some(),
child_label,
explicit_label: None,
labelled_by_text: labelled_by_text(el),
native_title: copy_string_attr(el, kAXTitleAttribute),
static_role_value: static_role_value(el, ax_role.as_deref()),
child_label: aggregated_child_label(el, ax_role.as_deref()),
placeholder: copy_string_attr(el, kAXPlaceholderValueAttribute),
description: copy_string_attr(el, kAXDescriptionAttribute),
}
}
fn labelled_by_text(el: &AXElement) -> Option<String> {
let label_el = copy_element_attr(el, kAXTitleUIElementAttribute)?;
copy_string_attr(&label_el, kAXTitleAttribute)
.or_else(|| copy_string_attr(&label_el, kAXValueAttribute))
}
fn static_role_value(el: &AXElement, ax_role: Option<&str>) -> Option<String> {
if ax_role != Some(STATIC_TEXT_ROLE) {
return None;
}
copy_string_attr(el, kAXValueAttribute)
}
fn aggregated_child_label(el: &AXElement, ax_role: Option<&str>) -> Option<String> {
let children = child_attributes(ax_role)
.iter()
.find_map(|attr| copy_ax_array(el, attr).filter(|v| !v.is_empty()))
.unwrap_or_default();
let texts = collect_child_texts(&children);
join_child_labels(texts.iter().map(String::as_str))
}
fn collect_child_texts(children: &[AXElement]) -> Vec<String> {
fn text_of(el: &AXElement) -> Option<String> {
copy_string_attr(el, kAXValueAttribute)
.or_else(|| copy_string_attr(el, kAXTitleAttribute))
}
let mut texts = Vec::new();
for child in children.iter().take(5) {
match copy_string_attr(child, kAXRoleAttribute).as_deref() {
Some("AXStaticText") => {
if let Some(text) = text_of(child) {
texts.push(text);
}
}
Some("AXCell") | Some("AXGroup") => {
for grandchild in
copy_ax_array_prefix(child, kAXChildrenAttribute, 5).unwrap_or_default()
{
if copy_string_attr(&grandchild, kAXRoleAttribute).as_deref()
== Some("AXStaticText")
&& let Some(text) = text_of(&grandchild)
{
texts.push(text);
}
}
}
_ => {}
}
}
texts
}
}
#[cfg(not(target_os = "macos"))]
@ -28,12 +90,7 @@ mod imp {
use agent_desktop_core::accname::NameEvidence;
pub fn name_evidence_impl(_el: &AXElement) -> NameEvidence {
NameEvidence {
title: None,
description: None,
value_promoted: false,
child_label: false,
}
NameEvidence::default()
}
}