// Phase-2 J1: the raw-byte story scanner. The XML tree the reader builds
// retains no source offsets, so annotation positions (and, later, L0's
// byte-span surgery) come from scanning each story part's ORIGINAL bytes:
// a namespace-aware tokenizer walks the markup, mirrors the projection
// counting the path grammar uses (p/r/tbl/tr/tc/hyperlink ordinals among
// same-kind direct siblings), records each projection node's byte span
// and sequence interval, and locates annotation markers. J1 uses it
// read-only; L0 builds mutation on the same foundation.
//
// Deliberate limits (revisited at L0's go/no-go gate): DOCTYPE subsets
// are skipped naively (stories don't carry them); attribute values ARE
// entity-decoded (predefined + numeric references); only UTF-8 parts
// are scanned (UTF-16 parts raise); legacy VML content is not modeled
// (the index's verification pass degrades positions inside it).
///|
/// Transitional and Strict WML namespace URIs — the reader supports both.
fn is_wml_uri(uri : String) -> Bool {
uri == "http://schemas.openxmlformats.org/wordprocessingml/2006/main" ||
uri == "http://purl.oclc.org/ooxml/wordprocessingml/main"
}
///|
/// The w14 (Office 2010 WordprocessingML extensions) namespace that
/// carries `w14:paraId`. MS-DOCX defines exactly this URI; Strict OOXML
/// has no alternate spelling for it.
fn is_w14_uri(uri : String) -> Bool {
uri == "http://schemas.microsoft.com/office/word/2010/wordml"
}
///|
/// A marker the scanner can locate.
priv enum ScanMarkerKind {
CommentStart
CommentEnd
CommentRef
FootnoteRef
EndnoteRef
}
///|
impl Show for ScanMarkerKind with fn output(self, logger) {
logger.write_string(
match self {
CommentStart => "commentRangeStart"
CommentEnd => "commentRangeEnd"
CommentRef => "commentReference"
FootnoteRef => "footnoteReference"
EndnoteRef => "endnoteReference"
},
)
}
///|
/// A tracked-change container the scanner can locate. The reader flattens
/// `w:ins` and drops `w:del`, so neither survives into the parsed tree:
/// revision identity exists only here, in the raw bytes.
priv enum ScanRevisionKind {
Insertion
Deletion
}
///|
impl Show for ScanRevisionKind with fn output(self, logger) {
logger.write_string(
match self {
Insertion => "ins"
Deletion => "del"
},
)
}
///|
/// One located tracked-change container. `author`/`date`/`id` are the
/// attributes AS SPELLED and stay absent when the source omits them —
/// `w:date` is optional in CT_TrackChange, and inventing one would make
/// the index assert a revision time the document never recorded.
/// `container_path` follows the marker contract above; document order is
/// the array order, so no separate sequence number is retained (adding one
/// would perturb the existing marker/node sequence arithmetic for no reader).
priv struct ScannedRevision {
kind : ScanRevisionKind
id : String?
author : String?
date : String?
container_path : String?
}
///|
/// Insertion or deletion.
fn ScannedRevision::kind(self : ScannedRevision) -> ScanRevisionKind {
self.kind
}
///|
/// The w:id attribute, as spelled, when present.
fn ScannedRevision::id(self : ScannedRevision) -> String? {
self.id
}
///|
/// The w:author attribute, as spelled, when present.
fn ScannedRevision::author(self : ScannedRevision) -> String? {
self.author
}
///|
/// The w:date attribute, LEXICAL (never converted), when present.
fn ScannedRevision::date(self : ScannedRevision) -> String? {
self.date
}
///|
/// The innermost open projection node, or None at story level.
fn ScannedRevision::container_path(self : ScannedRevision) -> String? {
self.container_path
}
///|
/// One element-name occurrence to rewrite in place, as the byte range of
/// the LOCAL NAME only (never the prefix, never the attributes). Rejecting a
/// deletion turns each `w:delText` back into `w:t` by overwriting exactly
/// these ranges, so the source namespace prefix, `xml:space`, and every
/// surrounding byte survive verbatim. `close_start` is -1 for the
/// self-closing form, which has no closing tag to rewrite.
priv struct ScannedNameEdit {
open_start : Int
open_end : Int
mut close_start : Int
mut close_end : Int
replacement : String
}
///|
/// One tracked-change ELEMENT located in the original bytes, with the byte
/// extent a resolver needs to unwrap or remove it.
///
/// This is deliberately a SEPARATE record from `ScannedRevision`, which
/// describes the revisions the reader's projection retains and is therefore
/// purged whenever a subtree is retracted (a merged cell, a deleted row).
/// Mutation cares about bytes, not about the projection: a site is recorded
/// for every tracked-change element in the part, is never retracted, and is
/// never gated on suppression — an element the resolver cannot see is an
/// element it would silently leave behind.
///
/// `supported` is false for every construct outside the resolvable set
/// (property revisions such as `w:rPr/w:ins` and `w:trPr/w:del`, moves, and
/// every `*PrChange`). Those still have to be RECORDED so a selection that
/// reaches one can refuse instead of half-resolving the document.
priv struct ScannedRevisionSite {
local_name : String
supported : Bool
id : String?
author : String?
date : String?
container_path : String?
byte_start : Int
// -1 for the self-closing form (no interior).
content_start : Int
mut close_tag_start : Int
mut byte_end : Int
// Whether the element's own start tag binds any namespace prefix. Unwrapping
// such an element would drop the binding while its former children keep
// using it, so an unwrap resolution has to refuse.
declares_namespaces : Bool
// `w:delText`/`w:delInstrText` names inside a deletion, in document order.
name_edits : Array[ScannedNameEdit]
}
///|
/// One located marker. `seq` is a total document order shared with node
/// intervals; `container_path` is the innermost open projection node
/// ("p[2]/r[1]", story-relative) or None at story level; `after_child`
/// is the last projection child closed inside that container before the
/// marker (for before/after boundary rendering).
priv struct ScannedMarker {
kind : ScanMarkerKind
id : String
id_was_present : Bool
seq : Int
container_path : String?
after_child : String?
}
///|
/// The marker kind.
fn ScannedMarker::kind(self : ScannedMarker) -> ScanMarkerKind {
self.kind
}
///|
/// The marker's w:id, as spelled.
fn ScannedMarker::id(self : ScannedMarker) -> String {
self.id
}
///|
/// Whether the source marker spelled a `w:id` attribute. The tolerant
/// projection intentionally maps an absent id to `""`, so presence must stay
/// separate to prevent a literal empty value from aliasing that sentinel.
fn ScannedMarker::id_was_present(self : ScannedMarker) -> Bool {
self.id_was_present
}
///|
/// Position in the story's total document order.
fn ScannedMarker::seq(self : ScannedMarker) -> Int {
self.seq
}
///|
/// The innermost open projection node, or None at story level.
fn ScannedMarker::container_path(self : ScannedMarker) -> String? {
self.container_path
}
///|
/// The last projection child closed inside the container before this marker.
fn ScannedMarker::after_child(self : ScannedMarker) -> String? {
self.after_child
}
///|
/// One projection node's identity and extent: path ("p[3]" or
/// "tbl[1]/tr[2]/tc[1]/p[1]", story-relative), the projection kind,
/// [open_seq, close_seq] for coverage arithmetic, and the byte span of
/// the whole element in the original part (open tag start .. close tag
/// end, exclusive) for L0.
priv struct ScannedNode {
path : String
kind : String
// Monotonic ordinal assigned when the projection node opens. Close-order
// storage alone cannot recover open order without sorting; this ordinal
// lets the scanner rebuild it with one bounded array pass.
open_order : Int
open_seq : Int
close_seq : Int
byte_start : Int
byte_end : Int
// L0 insertion offsets: content_start is just past the open tag AND
// past a leading pPr (the earliest schema-legal point for run-level
// content); close_tag_start is the '<' of the close tag. Both -1 for
// self-closing elements (no interior to insert into).
content_start : Int
close_tag_start : Int
}
///|
/// One namespace-resolved source element in opening-tag order. Its array
/// index is its stable identity; parent/child/sibling links and the same-name
/// sibling ordinal form the canonical physical tree independently of the
/// reader's flattened logical projection.
#warnings("-unused_field")
priv struct ScannedElement {
identity : Int
uri : String
local_name : String
same_name_ordinal : Int
field_char_type : String?
// A WML `w:p` element's `w14:paraId` value AS SPELLED in the source
// (never normalized here — validity and canonical case are the
// identity layer's judgment). None when the attribute is absent.
para_id_raw : String?
parent_index : Int
mut first_child_index : Int
mut next_sibling_index : Int
byte_start : Int
content_start : Int
mut content_end : Int
mut byte_end : Int
// Source-retaining mode populates this map for non-empty `w:t` and
// `w:instrText` content. Both element kinds draw from the same cumulative
// lexical budget.
mut text_map : WtContentMap?
}
///|
/// Caller-owned cumulative limits for the retained N0b source tree. The
/// lexical budget is shared by every `w:t` and `w:instrText` in one story, so
/// many individually small text nodes cannot multiply the retention allowance.
priv struct ProjectionSourceBudget {
mut elements_left : Int
// Cumulative source bytes retained as element QNames or relevant
// attribute names/values, plus derived canonical sibling-counter keys.
// Charging raw UTF-8 spans and derived output lengths before allocation is
// a conservative upper bound on their eventual UTF-16 String storage.
mut retained_chars_left : Int
// Fixed-size retained objects need their own ceiling: short attributes and
// namespace declarations can otherwise stay under the character budget
// while multiplying tuples, map entries, scope stacks, and declarations.
mut retained_tokens_left : Int
max_depth : Int
text_budget : TokenMapBudget
}
///|
let max_projection_source_elements : Int = 1_000_000
///|
let max_projection_source_attribute_chars : Int = 8 * 1024 * 1024
///|
let max_projection_source_retained_tokens : Int = 1_000_000
///|
let max_projection_source_depth : Int = 256
///|
let max_projection_source_text_bytes : Int = 64 * 1024 * 1024
///|
let max_projection_source_text_tokens : Int = 1_000_000
///|
fn projection_source_budget(
max_elements? : Int = max_projection_source_elements,
max_attribute_chars? : Int = max_projection_source_attribute_chars,
max_retained_tokens? : Int = max_projection_source_retained_tokens,
max_depth? : Int = max_projection_source_depth,
max_text_bytes? : Int = max_projection_source_text_bytes,
max_text_tokens? : Int = max_projection_source_text_tokens,
) -> ProjectionSourceBudget raise DocxError {
guard max_elements >= 0 &&
max_attribute_chars >= 0 &&
max_retained_tokens >= 0 &&
max_depth >= 0 &&
max_text_bytes >= 0 &&
max_text_tokens >= 0 else {
raise Unsupported(
message="projection source-tree budget limits must be non-negative",
)
}
{
elements_left: max_elements,
retained_chars_left: max_attribute_chars,
retained_tokens_left: max_retained_tokens,
max_depth,
text_budget: token_map_budget(max_text_bytes, max_text_tokens) catch {
TokenMapError(message) => raise Unsupported(message~)
},
}
}
///|
fn ProjectionSourceBudget::charge_element(
self : ProjectionSourceBudget,
depth : Int,
) -> Unit raise DocxError {
guard self.elements_left > 0 else {
raise @core.docx_xml_resource_limit_error(DocxXmlTokens)
}
guard depth > 0 && depth <= self.max_depth else {
raise @core.docx_xml_resource_limit_error(DocxXmlNestingDepth)
}
self.elements_left -= 1
}
///|
fn ProjectionSourceBudget::charge_retained_chars(
self : ProjectionSourceBudget,
chars : Int,
) -> Unit raise DocxError {
guard chars >= 0 && chars <= self.retained_chars_left else {
raise @core.docx_xml_resource_limit_error(DocxXmlMaterializedCharacters)
}
self.retained_chars_left -= chars
}
///|
fn ProjectionSourceBudget::charge_retained_token(
self : ProjectionSourceBudget,
) -> Unit raise DocxError {
guard self.retained_tokens_left > 0 else {
raise @core.docx_xml_resource_limit_error(DocxXmlTokens)
}
self.retained_tokens_left -= 1
}
///|
/// Precomputed fallback for a marker outside every projection node. When
/// `node_is_after` is true the marker is before `path`; otherwise it is after
/// the nearest preceding node.
priv struct StoryLevelMarkerPosition {
path : String
node_is_after : Bool
}
///|
/// The story-relative projection path ("p[3]", "tbl[1]/tr[1]/tc[1]").
fn ScannedNode::path(self : ScannedNode) -> String {
self.path
}
///|
/// Sequence number of the node's open tag.
fn ScannedNode::open_seq(self : ScannedNode) -> Int {
self.open_seq
}
///|
/// Sequence number of the node's close tag.
fn ScannedNode::close_seq(self : ScannedNode) -> Int {
self.close_seq
}
///|
fn canonical_physical_name(uri : String, local_name : String) -> String {
if is_wml_uri(uri) {
"w:\{local_name}"
} else if uri == MC_URI {
"mc:\{local_name}"
} else if uri == "" {
local_name
} else {
// Length-prefix arbitrary namespace URIs. Namespace names may legally
// contain every delimiter used by the human-readable ancestry syntax;
// the prefix keeps the concatenated path injective without depending on
// URI escaping rules.
"{\{uri.length()}:\{uri}}\{local_name}"
}
}
///|
fn nonnegative_decimal_digits(value : Int) -> Int {
let mut remaining = value
let mut digits = 1
while remaining >= 10 {
remaining /= 10
digits += 1
}
digits
}
///|
fn canonical_physical_name_chars(uri : String, local_name : String) -> Int {
if is_wml_uri(uri) {
2 + local_name.length()
} else if uri == MC_URI {
3 + local_name.length()
} else if uri == "" {
local_name.length()
} else {
// {uri-length:uri}local
3 +
nonnegative_decimal_digits(uri.length()) +
uri.length() +
local_name.length()
}
}
///|
/// Canonical physical path derived from identity-bearing parent links. Prefix
/// spelling never enters the path: namespace-equivalent Transitional/Strict
/// and alternate-prefix documents therefore describe the same tree shape.
#warnings("-unused_value")
fn ScannedElement::physical_path(
self : ScannedElement,
elements : Array[ScannedElement],
) -> String {
let ancestry : Array[Int] = []
let mut at = self.identity
while at >= 0 &&
at < elements.length() &&
ancestry.length() <= elements.length() {
ancestry.push(at)
at = elements[at].parent_index
}
let out = StringBuilder()
for index = ancestry.length() - 1; index >= 0; index = index - 1 {
let element = elements[ancestry[index]]
out.write_string(
"/\{canonical_physical_name(element.uri, element.local_name)}[\{element.same_name_ordinal}]",
)
}
out.to_string()
}
///|
/// Scanner output for one story part: nodes in CLOSE order (a node is
/// complete only at its close tag), markers in
/// document order, and non-fatal diagnostics.
///
/// FROZEN after `scan_story` returns. Since #462 phase C the annotation
/// index adopts the retained reader-projection scans, so one instance is
/// read by the span surfaces, `covering_comment_ids`, and the mutation
/// resolver at once. Nothing may mutate a returned scan — surgery produces
/// a NEW archive that callers re-read; the old annotated result stays a
/// stale source-pinned snapshot.
priv struct StoryScan {
nodes : Array[ScannedNode]
elements : Array[ScannedElement]
markers : Array[ScannedMarker]
// Tracked-change containers in document order. Retained separately from
// markers: a revision is a range container, not a point, and it survives
// in stories the reader flattens (w:ins) or drops (w:del) entirely.
revisions : Array[ScannedRevision]
// Every tracked-change ELEMENT in the part, with byte extents, in document
// order. Never purged and never suppression-gated (see ScannedRevisionSite).
revision_sites : Array[ScannedRevisionSite]
warnings : Array[String]
story_level_positions : Map[Int, StoryLevelMarkerPosition]
// The story's ROOT element span (kind = its local name, path = "").
// L1 splices new definitions just before root.close_tag_start.
root : ScannedNode?
// Namespace URI that expanded the root element's QName. Kept separately
// from the canonicalized reader DOM so mutations can preserve Strict vs
// Transitional WML exactly.
root_namespace_uri : String?
// Private structural accounting used by the complexity regression tests.
// Every retraction step represents one checkpoint operation or one item
// discarded; no retained prefix is inspected.
work : AnnotationScanWork
}
///|
/// Structural work counters for the annotation scanner's adversarial paths.
/// Keep these counters in sync whenever those paths inspect another slot: the
/// white-box tests assert input-linear bounds rather than wall-clock timing.
priv struct AnnotationScanWork {
mut projection_opens : Int
mut emitted_nodes : Int
mut emitted_markers : Int
mut emitted_revisions : Int
mut ordering_steps : Int
mut story_position_steps : Int
mut retraction_calls : Int
mut retraction_steps : Int
}
///|
/// Scanner and verification paths are retained as UTF-16 strings. One shared
/// counter covers both copies across every story; limited transaction reads
/// additionally charge the caller's cumulative XML materialization budget.
priv struct AnnotationPathBudget {
shared_xml_budget : @xml.XmlReadBudget?
mut remaining_chars : Int
}
///|
let max_annotation_path_chars : Int = 8 * 1024 * 1024
///|
/// The scanner only consumes namespace bindings plus WML id/type/value
/// attributes. Those values are intrinsically small; bounding them before
/// UTF-8 decoding keeps this source-offset pass inside the transaction working
/// reserve even when an otherwise valid story carries a padded reference.
let max_annotation_relevant_attribute_bytes : Int = 4 * 1024
///|
fn AnnotationPathBudget::new(
shared_xml_budget : @xml.XmlReadBudget?,
max_chars? : Int = max_annotation_path_chars,
) -> AnnotationPathBudget {
{
shared_xml_budget,
remaining_chars: if max_chars > 0 {
max_chars
} else {
0
},
}
}
///|
fn AnnotationPathBudget::charge(
self : AnnotationPathBudget,
chars : Int,
) -> Unit raise DocxError {
if chars < 0 || chars > self.remaining_chars {
raise @core.docx_xml_resource_limit_error(DocxXmlMaterializedCharacters)
}
match self.shared_xml_budget {
Some(budget) => budget.charge_derived_chars(chars)
None => ()
}
self.remaining_chars -= chars
}
///|
fn decimal_char_count(value : Int) -> Int {
let mut remaining = value
let mut digits = 1
while remaining >= 10 {
remaining /= 10
digits += 1
}
digits
}
///|
fn AnnotationPathBudget::charge_descendant(
self : AnnotationPathBudget,
parent : String?,
kind : String,
ordinal : Int,
) -> Unit raise DocxError {
let parent_chars = match parent {
Some(path) => path.length() + 1
None => 0
}
self.charge(parent_chars + kind.length() + 2 + decimal_char_count(ordinal))
}
///|
/// The story's root-element span, when the part had one.
fn StoryScan::root(self : StoryScan) -> ScannedNode? {
self.root
}
///|
fn StoryScan::root_namespace_uri(self : StoryScan) -> String? {
self.root_namespace_uri
}
///|
/// Projection nodes, in close order (see the struct doc).
fn StoryScan::nodes(self : StoryScan) -> Array[ScannedNode] {
self.nodes
}
///|
#warnings("-unused_value")
fn StoryScan::elements(self : StoryScan) -> Array[ScannedElement] {
self.elements
}
///|
/// Annotation markers, in document order.
fn StoryScan::markers(self : StoryScan) -> Array[ScannedMarker] {
self.markers
}
///|
/// Tracked-change containers, in document order.
fn StoryScan::revisions(self : StoryScan) -> Array[ScannedRevision] {
self.revisions
}
///|
/// Every tracked-change ELEMENT with byte extents, in document order.
fn StoryScan::revision_sites(self : StoryScan) -> Array[ScannedRevisionSite] {
self.revision_sites
}
///|
/// Non-fatal scan diagnostics.
fn StoryScan::warnings(self : StoryScan) -> Array[String] {
self.warnings
}
///|
fn StoryScan::story_level_position(
self : StoryScan,
marker_seq : Int,
) -> StoryLevelMarkerPosition? {
self.story_level_positions.get(marker_seq)
}
///|
/// White-box complexity counters. Kept as one private tuple so production
/// callers cannot depend on instrumentation details.
fn StoryScan::complexity_counters(
self : StoryScan,
) -> (Int, Int, Int, Int, Int, Int, Int, Int) {
let work = self.work
(
work.projection_opens,
work.emitted_nodes,
work.emitted_markers,
work.emitted_revisions,
work.ordering_steps,
work.story_position_steps,
work.retraction_calls,
work.retraction_steps,
)
}
///|
/// One open element during the walk.
priv struct ScanFrame {
// Exact source QName plus resolved local name. Source-retaining mode
// compares the QName on close because namespace-equivalent but differently
// prefixed open/close tags are not well-formed XML.
qualified_name : String
local_name : String
is_wml : Bool
// Namespace prefixes THIS element declared (to pop on close).
declared : Array[String]
// Projection bookkeeping: Some(path) when this element is a projection
// node (kind = the PROJECTION kind, e.g. "note" for w:footnote — not
// the local name); child ordinal counters; last closed child.
mut projection_path : String?
projection_kind : String?
counters : StableStringMap[Int]
mut last_closed_child : String?
projection_open_order : Int
open_seq : Int
byte_start : Int
// Identity in StoryScan.elements when source-tree retention is enabled.
element_index : Int
// Raw source-tree sibling ordinals and links are deliberately separate from
// the reader-flattened projection counters above.
physical_counters : StableStringMap[Int]
mut last_element_child : Int
// Earliest interior insertion offset (just past the open tag, moved
// past a leading pPr when one closes directly inside this element).
mut content_start : Int
// TRANSPARENT containers (sdt/sdtContent/smartTag/ins) are flattened by
// the reader: their children count in the nearest non-transparent
// ancestor's counters, so projection ordinals mirror the parsed AST.
transparent : Bool
// SUPPRESSED subtrees (w:del content, plumbing notes) do not exist in
// the parsed AST: nothing inside gets a projection path. Mutable for
// vMerge retraction (tcPr precedes cell content, so flipping before
// any child opens is safe).
mut suppressed : Bool
// Where this frame's own ordinal was bumped, for vMerge retraction.
mut counter_owner : StableStringMap[Int]?
// Property containers (tcPr/pPr/rPr/...): the reader never surfaces
// markers from inside them.
in_properties : Bool
// Grid tracking mirroring the reader's calculate_typed_table_row_spans
// EXACTLY: origins are keyed by a cell's STARTING column only; every
// kept cell registers itself as the origin at its column; an unmatched
// continuation becomes an origin (fail-open); origins are never
// removed. tbl frames: the origin-column set. tr frames: the next
// cell's column (plus trPr/w:del row deletion, like read_table_row).
// tc frames: column/span (FIRST gridSpan via the reader's
// parse_grid_span) and the FIRST vMerge declaration; keep-or-retract
// decided at tcPr close.
origin_columns : Set[Int]
mut column_cursor : Int
grid_column : Int
mut grid_span : Int
mut grid_span_seen : Bool
mut vmerge_seen : Bool
mut vmerge_continue : Bool
mut row_deleted : Bool
// The reader honors only the FIRST tcPr/trPr (first_or_empty): a tc/tr
// frame records having seen one; a tcPr/trPr frame records whether it
// IS that first one (declarations from later ones are ignored).
mut properties_seen : Bool
first_properties : Bool
// Origin columns THIS row newly registered (absent before), so a
// late-deleted row can roll back exactly its own contributions —
// the reader excludes deleted rows BEFORE span calculation.
registered_columns : Array[Int]
// Exact output checkpoints at element open. Late tcPr/trPr decisions can
// discard the subtree with tail truncation instead of rescanning/copying
// everything accumulated earlier in the story.
node_checkpoint : Int
marker_checkpoint : Int
revision_checkpoint : Int
// Index into StoryScan.revision_sites when THIS element is a tracked-change
// element, else -1. Its closing tag fills in the site's closing offsets.
revision_site_index : Int
// Index into the enclosing site's name_edits when THIS element is a
// `w:delText`/`w:delInstrText`, else -1. Its closing tag fills in the
// record's closing-name range.
name_edit_index : Int
}
///|
fn is_property_container(uri : String, local_name : String) -> Bool {
is_wml_uri(uri) &&
local_name
is ("tcPr" | "pPr" | "rPr" | "tblPr" | "trPr" | "sectPr" | "tblGrid")
}
///|
fn projection_kind(uri : String, local_name : String) -> String? {
// DELIBERATE LIMIT (J1): image nodes are NOT projected. The reader
// emits zero..many Images per drawing (blip resolution needs the
// relationship graph) and wraps them in Hyperlink when a:hlinkClick is
// present — faithful mirroring is L0-scope work. Consequences, pinned:
// image-path coverage queries return empty, image byte spans arrive
// with L0, and markers inside drawings resolve to their RUN (correct).
if !is_wml_uri(uri) {
return None
}
match local_name {
"p" | "r" | "tbl" | "tr" | "tc" | "hyperlink" => Some(local_name)
// Container identity inside the notes/comments stories (these
// elements never occur in body/header/footer stories).
"footnote" | "endnote" => Some("note")
"comment" => Some("comment")
_ => None
}
}
///|
/// Containers the reader flattens into their parent flow.
fn is_transparent_container(uri : String, local_name : String) -> Bool {
(
is_wml_uri(uri) &&
local_name
is ("sdt" | "sdtContent" | "smartTag" | "ins" | "drawing" | "object")
) ||
(uri == MC_URI && local_name is ("AlternateContent" | "Fallback"))
}
///|
const MC_URI : String = "http://schemas.openxmlformats.org/markup-compatibility/2006"
///|
/// XML Namespaces binds this prefix implicitly in every document. Keeping it
/// in the physical-identity table prevents `xml:x` from aliasing unqualified
/// `x` even when the source correctly omits an explicit declaration.
const XML_NAMESPACE_URI : String = "http://www.w3.org/XML/1998/namespace"
///|
const XMLNS_NAMESPACE_URI : String = "http://www.w3.org/2000/xmlns/"
///|
fn projection_source_is_xml_name_start(char : Char) -> Bool {
let code = char.to_int()
char.is_ascii_alphabetic() ||
char == ':' ||
char == '_' ||
(code >= 0xc0 && code <= 0xd6) ||
(code >= 0xd8 && code <= 0xf6) ||
(code >= 0xf8 && code <= 0x2ff) ||
(code >= 0x370 && code <= 0x37d) ||
(code >= 0x37f && code <= 0x1fff) ||
(code >= 0x200c && code <= 0x200d) ||
(code >= 0x2070 && code <= 0x218f) ||
(code >= 0x2c00 && code <= 0x2fef) ||
(code >= 0x3001 && code <= 0xd7ff) ||
(code >= 0xf900 && code <= 0xfdcf) ||
(code >= 0xfdf0 && code <= 0xfffd) ||
(code >= 0x10000 && code <= 0xeffff)
}
///|
fn projection_source_is_xml_name_char(char : Char) -> Bool {
let code = char.to_int()
projection_source_is_xml_name_start(char) ||
char == '-' ||
char == '.' ||
char.is_ascii_digit() ||
code == 0xb7 ||
(code >= 0x300 && code <= 0x36f) ||
(code >= 0x203f && code <= 0x2040)
}
///|
fn projection_source_is_xml_ncname(value : StringView) -> Bool {
let mut offset = 0
let mut first = true
while offset < value.length() {
guard value.get_char(offset) is Some(char) else { return false }
if char == ':' ||
(if first {
!projection_source_is_xml_name_start(char)
} else {
!projection_source_is_xml_name_char(char)
}) {
return false
}
first = false
offset += char.utf16_len()
}
!first
}
///|
fn projection_source_is_xml_qname(value : String) -> Bool {
match value.find(":") {
Some(colon) => {
let prefix = value[:colon]
let local_name = value[colon + 1:]
projection_source_is_xml_ncname(prefix) &&
projection_source_is_xml_ncname(local_name) &&
!local_name.contains(":")
}
None => projection_source_is_xml_ncname(value)
}
}
///|
fn validate_projection_source_namespace(
prefix : String,
namespace_uri : String,
) -> Unit raise DocxError {
for char in namespace_uri {
if char == ' ' || char == '\t' || char == '\n' || char == '\r' {
raise Unsupported(
message="the projection source scanner found whitespace in a namespace name",
)
}
}
if prefix == "xmlns" || namespace_uri == XMLNS_NAMESPACE_URI {
raise Unsupported(
message="the projection source scanner found an invalid xmlns namespace binding",
)
}
if prefix == "xml" {
if namespace_uri != XML_NAMESPACE_URI {
raise Unsupported(
message="the projection source scanner found an invalid xml namespace binding",
)
}
} else if namespace_uri == XML_NAMESPACE_URI {
raise Unsupported(
message="the projection source scanner found the reserved XML namespace on a non-xml prefix",
)
}
if prefix != "" && namespace_uri == "" {
raise Unsupported(
message="the projection source scanner found an undeclared namespace prefix",
)
}
}
///|
/// Subtrees the reader removes entirely.
fn is_suppressed_container(uri : String, local_name : String) -> Bool {
(is_wml_uri(uri) && local_name == "del") ||
(uri == MC_URI && local_name == "Choice")
}
///|
fn marker_kind(uri : String, local_name : String) -> ScanMarkerKind? {
if !is_wml_uri(uri) {
return None
}
match local_name {
"commentRangeStart" => Some(CommentStart)
"commentRangeEnd" => Some(CommentEnd)
"commentReference" => Some(CommentRef)
"footnoteReference" => Some(FootnoteRef)
"endnoteReference" => Some(EndnoteRef)
_ => None
}
}
///|
/// A tracked-change container the scanner records. `w:moveFrom`/`w:moveTo`
/// are deliberately out of scope: they are a paired MOVE, and reporting one
/// half of a move as a plain deletion would misdescribe the edit.
fn revision_kind(uri : String, local_name : String) -> ScanRevisionKind? {
if !is_wml_uri(uri) {
return None
}
match local_name {
"ins" => Some(Insertion)
"del" => Some(Deletion)
_ => None
}
}
///|
/// Every WML element that RECORDS a tracked change, whether or not the
/// resolver can act on it. Content insertions and deletions are the two the
/// resolver handles; the rest are recorded so a selection reaching one can
/// refuse loudly. Keeping the whole vocabulary in one place is what makes
/// "accept everything by Reviewer" honest: a construct missing from this list
/// would survive publication with no finding.
fn is_revision_site_name(uri : String, local_name : String) -> Bool {
if !is_wml_uri(uri) {
return false
}
local_name
is ("ins"
| "del"
// paired moves: resolving one half as a plain insertion or deletion
// would corrupt the other half, so they are recorded, never acted on
| "moveFrom"
| "moveTo"
| "moveFromRangeStart"
| "moveFromRangeEnd"
| "moveToRangeStart"
| "moveToRangeEnd"
// property revisions: these carry the PREVIOUS properties, and accepting
// or rejecting one rewrites a property container, not a content range
| "rPrChange"
| "pPrChange"
| "tblPrChange"
| "trPrChange"
| "tcPrChange"
| "sectPrChange"
| "tblGridChange"
| "tblPrExChange"
| "numberingChange"
// table-structure revisions
| "cellIns"
| "cellDel"
| "cellMerge"
// custom-XML range revisions
| "customXmlInsRangeStart"
| "customXmlInsRangeEnd"
| "customXmlDelRangeStart"
| "customXmlDelRangeEnd"
| "customXmlMoveFromRangeStart"
| "customXmlMoveFromRangeEnd"
| "customXmlMoveToRangeStart"
| "customXmlMoveToRangeEnd")
}
///|
/// The element name a deleted text run reverts to when its deletion is
/// rejected. `w:delText` is `w:t` and `w:delInstrText` is `w:instrText`;
/// nothing else is a deleted-text carrier.
fn deleted_text_replacement(uri : String, local_name : String) -> String? {
if !is_wml_uri(uri) {
return None
}
match local_name {
"delText" => Some("t")
"delInstrText" => Some("instrText")
_ => None
}
}
///|
/// The byte range of a tag's LOCAL NAME. `name_start` points at the first
/// byte of the QName (just past '<', or just past ' (Int, Int) {
let mut at = name_start
while at < limit && !is_name_end(part[at]) {
at += 1
}
let mut local_start = name_start
for scan in name_start.. (String, String) {
match name.find(":") {
Some(colon) => (name[0:colon].to_owned(), name[colon + 1:].to_owned())
None => ("", name)
}
}
///|
fn scan_closing_name(
part : BytesView,
start : Int,
close_end : Int,
) -> String raise DocxError {
let mut at = start + 2
guard at < close_end && !is_xml_space(part[at]) else {
raise Unsupported(
message="the projection source scanner found whitespace after a closing-tag opener",
)
}
let name_start = at
while at < close_end && !is_name_end(part[at]) {
at += 1
}
guard at > name_start else {
raise Unsupported(
message="the projection source scanner found an empty closing tag",
)
}
guard at - name_start <= max_annotation_relevant_attribute_bytes else {
raise @core.docx_xml_resource_limit_error(DocxXmlTokenLength)
}
let name = utf8_slice(part, name_start, at)
guard projection_source_is_xml_qname(name) else {
raise Unsupported(
message="the projection source scanner found an invalid closing-tag QName",
)
}
while at < close_end - 1 && is_xml_space(part[at]) {
at += 1
}
guard at == close_end - 1 else {
raise Unsupported(
message="the projection source scanner found malformed closing-tag content",
)
}
name
}
///|
/// Scans one story part. `part` is the ORIGINAL part bytes (UTF-8).
fn scan_story(
part : BytesView,
path_budget? : AnnotationPathBudget,
warning_collector? : AnnotationWarningCollector,
source_budget? : ProjectionSourceBudget,
) -> StoryScan raise DocxError {
// A UTF-16 BOM means byte-level markup scanning would be wrong.
if part.length() >= 2 &&
(
(part[0] == 0xFF && part[1] == 0xFE) ||
(part[0] == 0xFE && part[1] == 0xFF)
) {
raise Unsupported(
message="the annotation scanner supports UTF-8 story parts only (found a UTF-16 byte-order mark)",
)
}
let nodes : Array[ScannedNode] = []
let elements : Array[ScannedElement] = []
let markers : Array[ScannedMarker] = []
let revisions : Array[ScannedRevision] = []
let revision_sites : Array[ScannedRevisionSite] = []
let owns_warning_collector = warning_collector is None
let warnings = match warning_collector {
Some(value) => value
None =>
AnnotationWarningCollector::new(
MAX_STORY_SCAN_WARNINGS,
MAX_ANNOTATION_WARNING_CHARS,
)
}
let mut root : ScannedNode? = None
let mut root_namespace_uri : String? = None
let frames : Array[ScanFrame] = []
let root_counters : StableStringMap[Int] = SortedMap([])
// Prefix -> stack of URIs (innermost last). "" is the default namespace;
// `xml` is the reserved implicit binding supplied by the XML standard.
let namespaces : StableStringMap[Array[String]] = SortedMap([
("xml", [XML_NAMESPACE_URI]),
])
let mut seq = 0
let mut projection_open_count = 0
let work = AnnotationScanWork::{
projection_opens: 0,
emitted_nodes: 0,
emitted_markers: 0,
emitted_revisions: 0,
ordering_steps: 0,
story_position_steps: 0,
retraction_calls: 0,
retraction_steps: 0,
}
let path_budget = match path_budget {
Some(value) => value
None => AnnotationPathBudget::new(None)
}
let length = part.length()
let mut index = 0
fn current_projection() -> ScanFrame? {
for at = frames.length() - 1; at >= 0; at = at - 1 {
if frames[at].projection_path is Some(_) {
return Some(frames[at])
}
}
None
}
fn skip_until(pattern : Bytes, from : Int) -> Int raise DocxError {
let mut at = from
while at + pattern.length() <= length {
let mut matched = true
for offset in 0..= length {
match source_budget {
Some(_) =>
raise Unsupported(
message="the projection source scanner found an unterminated markup opener",
)
None => break
}
}
let next = part[index + 1]
if next == b'?' {
index = skip_until(b"?>", index + 2)
continue
}
if next == b'!' {
if index + 3 < length &&
part[index + 2] == b'-' &&
part[index + 3] == b'-' {
index = skip_until(b"-->", index + 4)
} else if index + 8 < length &&
part[index + 2] == b'[' &&
part[index + 3] == b'C' {
index = skip_until(b"]]>", index + 9)
} else {
// DOCTYPE and friends: skip to the next '>' (stories don't carry
// internal subsets; L0's gate revisits).
index = skip_until(b">", index + 2)
}
continue
}
if next == b'/' {
// Closing tag.
let close_end = skip_until(b">", index + 2)
let closing_name = match source_budget {
Some(_) => Some(scan_closing_name(part, index, close_end))
None => None
}
match frames.pop() {
Some(frame) => {
match source_budget {
Some(budget) => {
let name = closing_name.unwrap_or("")
guard name == frame.qualified_name else {
raise Unsupported(
message="the projection source scanner found a mismatched closing tag",
)
}
guard frame.element_index >= 0 &&
frame.element_index < elements.length() else {
raise Unsupported(
message="the projection source scanner lost an element identity",
)
}
let element = elements[frame.element_index]
element.content_end = index
element.byte_end = close_end
if is_wml_uri(element.uri) &&
element.local_name is ("t" | "instrText") {
element.text_map = Some(
map_wt_content(
part,
element.content_start,
element.content_end,
budget.text_budget,
) catch {
TokenMapError(message) =>
raise Unsupported(
message="projection source scanner: \{message}",
)
},
)
}
}
None => ()
}
seq += 1
// Byte extents for the mutation-facing site records. Both are
// filled here, at the CLOSING tag, because that is the first
// moment either offset is known.
if frame.revision_site_index >= 0 &&
frame.revision_site_index < revision_sites.length() {
let site = revision_sites[frame.revision_site_index]
site.close_tag_start = index
site.byte_end = close_end
}
if frame.name_edit_index >= 0 {
match nearest_revision_site(frames, revision_sites) {
Some(site) =>
if frame.name_edit_index < site.name_edits.length() {
let record = site.name_edits[frame.name_edit_index]
let (local_start, local_end) = tag_local_name_span(
part,
index + 2,
close_end,
)
record.close_start = local_start
record.close_end = local_end
}
None => ()
}
}
for prefix in frame.declared {
match namespaces.get(prefix) {
Some(stack) => {
let _ = stack.pop()
}
None => ()
}
}
// A pPr closing directly inside a paragraph (or an rPr
// inside a run) moves the parent's earliest insertion offset
// past itself: CT_P and CT_R are SEQUENCES with the property
// container first, so content must land after it.
if frame.is_wml && frame.local_name == "pPr" {
match frames.last() {
Some(parent) if parent.is_wml && parent.local_name == "p" =>
parent.content_start = close_end
_ => ()
}
}
if frame.is_wml && frame.local_name == "rPr" {
match frames.last() {
Some(parent) if parent.is_wml && parent.local_name == "r" =>
parent.content_start = close_end
_ => ()
}
}
// vMerge DECISION at tcPr close (declarations final, before any
// cell content): reader-exact — a continuation whose STARTING
// column has a registered origin is merged away (retracted);
// an unmatched continuation is KEPT and becomes the origin.
if frame.is_wml &&
frame.local_name == "tcPr" &&
frame.first_properties {
match (frames.last(), nearest_frame_by_kind(frames, "tbl")) {
(Some(cell), Some(table)) =>
if cell.is_wml &&
cell.local_name == "tc" &&
cell.vmerge_continue &&
table.origin_columns.contains(cell.grid_column) {
retract_cell(cell, nodes, markers, revisions, work)
}
_ => ()
}
}
// trPr/w:del deletes the whole row, reader-exact.
if frame.is_wml &&
frame.local_name == "trPr" &&
frame.first_properties {
match frames.last() {
Some(row) if row.is_wml && row.local_name == "tr" =>
if row.row_deleted {
retract_row(
row,
nearest_frame_by_kind(frames, "tbl"),
nodes,
markers,
revisions,
work,
)
}
_ => ()
}
}
// Close bookkeeping at tc close: EVERY kept cell registers as
// the origin at its starting column (origins are never
// removed — a later continuation merges into whatever cell is
// above it, exactly like the reader); the row cursor advances
// by the span (the reader's parse can yield zero or negative
// spans, mirrored verbatim).
if frame.is_wml && frame.local_name == "tc" {
match
(
nearest_frame_by_kind(frames, "tr"),
nearest_frame_by_kind(frames, "tbl"),
) {
(Some(row), Some(table)) => {
if !frame.suppressed &&
!table.origin_columns.contains(frame.grid_column) {
table.origin_columns.add(frame.grid_column)
row.registered_columns.push(frame.grid_column)
}
row.column_cursor = frame.grid_column + frame.grid_span
}
_ => ()
}
}
finish_frame(frame, seq, index, close_end, nodes, frames, work)
if frames.length() == 0 && root is None {
root = Some({
path: "",
kind: frame.local_name,
open_order: -1,
open_seq: frame.open_seq,
close_seq: seq,
byte_start: frame.byte_start,
byte_end: close_end,
content_start: frame.content_start,
close_tag_start: index,
})
}
}
None =>
match source_budget {
Some(_) =>
raise Unsupported(
message="the projection source scanner found an unbalanced closing tag",
)
None => warnings.add("annotation scanner: unbalanced closing tag")
}
}
index = close_end
continue
}
// Opening (or empty) element. Source mode refuses before scan_tag
// materializes this element's QName or any retained attribute.
match source_budget {
Some(budget) => {
guard frames.length() > 0 || elements.length() == 0 else {
raise Unsupported(
message="the projection source scanner found multiple root elements",
)
}
budget.charge_element(frames.length() + 1)
}
None => ()
}
let (name, attributes, unretained_attribute_names, self_closing, tag_end) = scan_tag(
part, index, length, source_budget,
)
// Namespace declarations bind before resolving this element's name.
let declared : Array[String] = []
let declared_prefixes : StableStringMap[Bool] = SortedMap([])
for pair in attributes {
let (attr_name, attr_value) = pair
if attr_name == "xmlns" || attr_name.has_prefix("xmlns:") {
match source_budget {
// Charge the binding before allocating its prefix, scope-stack/map
// state, or closing-scope declaration entry.
Some(budget) => budget.charge_retained_token()
None => ()
}
let prefix = if attr_name == "xmlns" {
""
} else {
attr_name[6:].to_owned()
}
match source_budget {
Some(budget) => {
// The per-tag duplicate detector is another fixed-size retained
// entry during start-tag processing; charge it before insertion.
budget.charge_retained_token()
guard !declared_prefixes.contains(prefix) else {
raise Unsupported(
message="the projection source scanner found a duplicate namespace declaration",
)
}
validate_projection_source_namespace(prefix, attr_value)
declared_prefixes[prefix] = true
}
None => ()
}
match namespaces.get(prefix) {
Some(stack) => stack.push(attr_value)
None => namespaces[prefix] = [attr_value]
}
declared.push(prefix)
}
}
let (prefix, local_name) = split_qualified_name(name)
let uri = match namespaces.get(prefix) {
Some(stack) if stack.length() > 0 => stack[stack.length() - 1]
_ => {
guard source_budget is None || prefix == "" else {
raise Unsupported(
message="the projection source scanner found an unbound namespace prefix",
)
}
""
}
}
match source_budget {
Some(_) => {
for attr_name in unretained_attribute_names {
let (attr_prefix, _) = split_qualified_name(attr_name)
if attr_prefix != "" {
guard namespaces.get(attr_prefix) is Some(stack) &&
stack.length() > 0 else {
raise Unsupported(
message="the projection source scanner found an unbound attribute namespace prefix",
)
}
}
}
for pair in attributes {
let attr_name = pair.0
if attr_name != "xmlns" && !attr_name.has_prefix("xmlns:") {
let (attr_prefix, _) = split_qualified_name(attr_name)
if attr_prefix != "" {
guard namespaces.get(attr_prefix) is Some(stack) &&
stack.length() > 0 else {
raise Unsupported(
message="the projection source scanner found an unbound attribute namespace prefix",
)
}
}
}
}
}
None => ()
}
let element_index = match source_budget {
Some(budget) => {
let parent_index = match frames.last() {
Some(parent) => parent.element_index
None => -1
}
let same_name_ordinal = match frames.last() {
Some(parent) => {
// For arbitrary namespaces the canonical key contains the URI.
// Charge it before concatenation so a long URI repeated across
// many child names cannot amplify into unbounded retained keys.
budget.charge_retained_chars(
canonical_physical_name_chars(uri, local_name),
)
bump_counter(
parent.physical_counters,
physical_counter_key(uri, local_name),
)
}
None => 1
}
let identity = elements.length()
elements.push({
identity,
uri,
local_name,
same_name_ordinal,
field_char_type: if is_wml_uri(uri) && local_name == "fldChar" {
attribute_value(attributes, "fldCharType", namespaces)
} else {
None
},
para_id_raw: if is_wml_uri(uri) && local_name == "p" {
uri_attribute_value(attributes, "paraId", namespaces, is_w14_uri)
} else {
None
},
parent_index,
first_child_index: -1,
next_sibling_index: -1,
byte_start: index,
content_start: tag_end,
content_end: if self_closing {
tag_end
} else {
-1
},
byte_end: if self_closing {
tag_end
} else {
-1
},
text_map: None,
})
match frames.last() {
Some(parent) => {
if parent.last_element_child >= 0 {
elements[parent.last_element_child].next_sibling_index = identity
} else {
elements[parent_index].first_child_index = identity
}
parent.last_element_child = identity
}
None => ()
}
// A self-closing w:t has no interior insertion point. Leave its map
// absent so later surgery cannot treat the byte after '/>' as text
// content; rewriting the whole empty-element tag is a later slice.
identity
}
None => -1
}
if frames.length() == 0 && root_namespace_uri is None {
root_namespace_uri = Some(uri)
}
seq += 1
let transparent = is_transparent_container(uri, local_name)
// A subtree is suppressed if the reader drops it: w:del content,
// plumbing notes (separator/continuationSeparator/continuationNotice
// never surface as user notes), or anything inside an already
// suppressed ancestor.
let inherited_suppression = match frames.last() {
Some(parent) => parent.suppressed
None => false
}
let plumbing_note = is_wml_uri(uri) &&
local_name is ("footnote" | "endnote") &&
attribute_value(attributes, "type", namespaces)
is (Some("separator")
| Some("continuationSeparator")
| Some("continuationNotice"))
let suppressed = inherited_suppression ||
is_suppressed_container(uri, local_name) ||
plumbing_note
// Markers respect suppression: the reader drops w:del content,
// mc:Choice branches, and plumbing notes wholesale — markers
// included. (A parent's suppression may have been set after its
// open — vMerge retraction — which this naturally observes.)
let inside_properties = match frames.last() {
Some(parent) => parent.in_properties
None => false
}
if !suppressed && !inside_properties {
match marker_kind(uri, local_name) {
Some(kind) => {
// Recorded whether self-closing or not (Word emits them
// self-closing; a non-empty form still marks its position).
seq += 1
let id = attribute_value(attributes, "id", namespaces)
let (container_path, after_child) = match current_projection() {
Some(frame) => (frame.projection_path, frame.last_closed_child)
None => (None, None)
}
markers.push({
kind,
id: id.unwrap_or(""),
id_was_present: id is Some(_),
seq,
container_path,
after_child,
})
work.emitted_markers += 1
if id is None {
warnings.add(
"annotation scanner: a \{kind} marker has no w:id attribute",
)
}
}
None => ()
}
}
// Tracked-change containers are located from the CONTAINER's own frame,
// because `w:del` suppresses everything inside itself: gating on
// `suppressed` (as markers do) would make every deletion invisible,
// which is exactly the blind spot this record exists to close. An
// `inherited_suppression` gate still applies, so a revision the reader
// drops wholesale (inside an unselected mc:Choice branch) stays out.
//
// DELIBERATE LIMIT: property-container revisions are not recorded.
// `w:rPr/w:ins` marks an inserted PARAGRAPH MARK and `w:trPr/w:del` a
// deleted table ROW; both are revisions of a property, not of content,
// and the row case is retracted from the projection anyway, leaving no
// path to name. Content revisions are the ones a review agent reads.
if !inherited_suppression && !inside_properties {
match revision_kind(uri, local_name) {
Some(kind) => {
let id = attribute_value(attributes, "id", namespaces)
let author = attribute_value(attributes, "author", namespaces)
let date = attribute_value(attributes, "date", namespaces)
// Charge the retained metadata before it is stored, so a story
// padded with revision attributes cannot outgrow the scanner's
// cumulative retention allowance.
path_budget.charge(
id.map(String::length).unwrap_or(0) +
author.map(String::length).unwrap_or(0) +
date.map(String::length).unwrap_or(0),
)
let container_path = match current_projection() {
Some(frame) => frame.projection_path
None => None
}
revisions.push({ kind, id, author, date, container_path, })
work.emitted_revisions += 1
if author is None {
warnings.add(
"annotation scanner: a w:\{kind} revision has no w:author attribute",
)
}
}
None => ()
}
}
// The mutation-facing site record. Recorded for EVERY tracked-change
// element regardless of suppression or property context: the resolver
// must be able to see (and refuse on) constructs the reader's projection
// drops, or "accept everything" would quietly leave them behind.
let revision_site_index = if is_revision_site_name(uri, local_name) {
let id = attribute_value(attributes, "id", namespaces)
let author = attribute_value(attributes, "author", namespaces)
let date = attribute_value(attributes, "date", namespaces)
path_budget.charge(
local_name.length() +
id.map(String::length).unwrap_or(0) +
author.map(String::length).unwrap_or(0) +
date.map(String::length).unwrap_or(0),
)
let container_path = match current_projection() {
Some(frame) => frame.projection_path
None => None
}
revision_sites.push({
local_name,
// Only a CONTENT insertion or deletion is resolvable. The same
// element names inside a property container mark an inserted or
// deleted PARAGRAPH MARK (or table row), which is a different edit.
supported: local_name is ("ins" | "del") && !inside_properties,
id,
author,
date,
container_path,
byte_start: index,
content_start: if self_closing {
-1
} else {
tag_end
},
close_tag_start: -1,
byte_end: if self_closing {
tag_end
} else {
-1
},
declares_namespaces: !declared.is_empty(),
name_edits: [],
})
revision_sites.length() - 1
} else {
-1
}
// A deleted-text carrier records the byte range of its own name in both
// tags, so rejecting the enclosing deletion can rename it in place.
let name_edit_index = match deleted_text_replacement(uri, local_name) {
Some(replacement) =>
match nearest_revision_site(frames, revision_sites) {
Some(site) => {
path_budget.charge(replacement.length())
let (local_start, local_end) = tag_local_name_span(
part,
index + 1,
tag_end,
)
site.name_edits.push({
open_start: local_start,
open_end: local_end,
close_start: -1,
close_end: -1,
replacement,
})
site.name_edits.length() - 1
}
None => -1
}
None => -1
}
let mut counter_owner : StableStringMap[Int]? = None
let mut projection_kind_value : String? = None
let mut projection_open_order = -1
let projection = if suppressed {
None
} else {
match projection_kind(uri, local_name) {
Some(kind) => {
// Ordinals live on the nearest NON-transparent frame, mirroring
// the reader's flattening; root_counters backs the degenerate
// projection-at-root case.
let counters = nearest_counters(frames, root_counters)
let ordinal = bump_counter(counters, kind)
counter_owner = Some(counters)
projection_kind_value = Some(kind)
projection_open_order = projection_open_count
projection_open_count += 1
work.projection_opens += 1
let parent_path = match current_projection() {
Some(frame) => frame.projection_path
None => None
}
path_budget.charge_descendant(parent_path, kind, ordinal)
let path = match parent_path {
Some(base) => "\{base}/\{kind}[\{ordinal}]"
None => "\{kind}[\{ordinal}]"
}
Some(path)
}
None => None
}
}
// Grid declarations, DIRECT tcPr children only, mirroring the
// reader: FIRST vMerge wins (continue iff val absent or "continue");
// FIRST gridSpan wins, parsed with the reader's own parse_grid_span.
let parent_is = fn(name : String) -> Bool {
match frames.last() {
Some(parent) =>
parent.is_wml && parent.local_name == name && parent.first_properties
None => false
}
}
if is_wml_uri(uri) && local_name == "vMerge" && parent_is("tcPr") {
match nearest_frame_by_kind(frames, "tc") {
Some(cell) =>
if !cell.vmerge_seen {
cell.vmerge_seen = true
cell.vmerge_continue = attribute_value(
attributes, "val", namespaces,
)
is (None | Some("continue"))
}
None =>
warnings.add("annotation scanner: a vMerge outside any table cell")
}
}
if is_wml_uri(uri) && local_name == "gridSpan" && parent_is("tcPr") {
match nearest_frame_by_kind(frames, "tc") {
Some(cell) =>
if !cell.grid_span_seen {
cell.grid_span_seen = true
cell.grid_span = parse_grid_span(
attribute_value(attributes, "val", namespaces).unwrap_or("1"),
)
}
None => ()
}
}
// trPr/w:del deletes the WHOLE ROW (reader's read_table_row).
if is_wml_uri(uri) && local_name == "del" && parent_is("trPr") {
match nearest_frame_by_kind(frames, "tr") {
Some(row) => row.row_deleted = true
None => ()
}
}
if self_closing {
// Open+close in place; only projection nodes matter for spans.
seq += 1
// A self-closing ` belongs to the inner `w:del`.
fn nearest_revision_site(
frames : Array[ScanFrame],
sites : Array[ScannedRevisionSite],
) -> ScannedRevisionSite? {
for at = frames.length() - 1; at >= 0; at = at - 1 {
let index = frames[at].revision_site_index
if index >= 0 && index < sites.length() && sites[index].local_name == "del" {
return Some(sites[index])
}
}
None
}
///|
fn nearest_frame_by_kind(
frames : Array[ScanFrame],
kind : String,
) -> ScanFrame? {
for at = frames.length() - 1; at >= 0; at = at - 1 {
if frames[at].projection_kind is Some(k) && k == kind {
return Some(frames[at])
}
// Also match retracted cells (projection cleared but still the
// structural tc frame).
if frames[at].is_wml && frames[at].local_name == kind {
return Some(frames[at])
}
}
None
}
///|
/// Removes already-emitted nodes/markers from inside a retracted frame:
/// the reader drops the WHOLE subtree even when properties arrive after
/// content (tolerated malformed input), so anything recorded since the
/// frame opened must go.
fn purge_subtree(
frame : ScanFrame,
nodes : Array[ScannedNode],
markers : Array[ScannedMarker],
revisions : Array[ScannedRevision],
work : AnnotationScanWork,
) -> Unit {
work.retraction_calls += 1
work.retraction_steps += 1 +
(nodes.length() - frame.node_checkpoint) +
(markers.length() - frame.marker_checkpoint) +
(revisions.length() - frame.revision_checkpoint)
nodes.truncate(frame.node_checkpoint)
markers.truncate(frame.marker_checkpoint)
// A retracted subtree has no projection node left to attach a revision
// path to, so its revisions go with it — the same rule markers follow.
revisions.truncate(frame.revision_checkpoint)
}
///|
/// Retracts a deleted row (trPr/w:del): un-counts its ordinal and
/// suppresses the whole subtree, like the reader's read_table_row.
fn retract_row(
row : ScanFrame,
table : ScanFrame?,
nodes : Array[ScannedNode],
markers : Array[ScannedMarker],
revisions : Array[ScannedRevision],
work : AnnotationScanWork,
) -> Unit {
match (row.projection_path, row.counter_owner) {
(Some(_), Some(counters)) => {
let current = counters.get_or_default("tr", 0)
if current > 0 {
counters["tr"] = current - 1
}
row.projection_path = None
row.counter_owner = None
row.suppressed = true
purge_subtree(row, nodes, markers, revisions, work)
// Roll back exactly the origins this row's cells introduced —
// the reader never lets a deleted row participate in merging.
match table {
Some(owner) =>
for column in row.registered_columns {
owner.origin_columns.remove(column)
}
None => ()
}
row.registered_columns.clear()
}
_ => ()
}
}
///|
/// Retracts a continuation cell the reader strips: un-counts its
/// ordinal and suppresses the whole subtree.
fn retract_cell(
cell : ScanFrame,
nodes : Array[ScannedNode],
markers : Array[ScannedMarker],
revisions : Array[ScannedRevision],
work : AnnotationScanWork,
) -> Unit {
match (cell.projection_path, cell.counter_owner) {
(Some(_), Some(counters)) => {
let current = counters.get_or_default("tc", 0)
if current > 0 {
counters["tc"] = current - 1
}
cell.projection_path = None
cell.counter_owner = None
cell.suppressed = true
purge_subtree(cell, nodes, markers, revisions, work)
}
_ => ()
}
}
///|
fn physical_counter_key(uri : String, local_name : String) -> String {
canonical_physical_name(uri, local_name)
}
///|
fn bump_counter(counters : StableStringMap[Int], kind : String) -> Int {
let next = counters.get_or_default(kind, 0) + 1
counters[kind] = next
next
}
///|
fn record_closed_child(frames : Array[ScanFrame], path : String) -> Unit {
// Boundary bookkeeping follows the FLATTENED structure too.
for at = frames.length() - 1; at >= 0; at = at - 1 {
if !frames[at].transparent {
frames[at].last_closed_child = Some(path)
return
}
}
}
///|
fn finish_frame(
frame : ScanFrame,
close_seq : Int,
close_tag_start : Int,
byte_end : Int,
nodes : Array[ScannedNode],
frames : Array[ScanFrame],
work : AnnotationScanWork,
) -> Unit {
match frame.projection_path {
Some(path) => {
nodes.push({
path,
kind: frame.projection_kind.unwrap_or(frame.local_name),
open_order: frame.projection_open_order,
open_seq: frame.open_seq,
close_seq,
byte_start: frame.byte_start,
byte_end,
content_start: frame.content_start,
close_tag_start,
})
work.emitted_nodes += 1
record_closed_child(frames, path)
}
None => ()
}
}
///|
/// Reads the qualified name and attributes of the tag opening at `start`
/// (which points at '<'). In source-retaining mode, attribute QNames that the
/// semantic scanner does not otherwise need are returned separately so their
/// namespace prefixes can still be validated. Returns (name, retained
/// attributes, unretained attribute names, self_closing, index-just-past '>').
fn scan_tag(
part : BytesView,
start : Int,
length : Int,
source_budget : ProjectionSourceBudget?,
) -> (String, Array[(String, String)], Array[String], Bool, Int) raise DocxError {
let mut at = start + 1
let name_start = at
while at < length && !is_name_end(part[at]) {
at += 1
}
match source_budget {
Some(budget) => {
guard at - name_start <= max_annotation_relevant_attribute_bytes else {
raise @core.docx_xml_resource_limit_error(DocxXmlTokenLength)
}
budget.charge_retained_chars(at - name_start)
}
None => ()
}
let name = utf8_slice(part, name_start, at)
match source_budget {
Some(_) => {
guard projection_source_is_xml_qname(name) else {
raise Unsupported(
message="the projection source scanner found an invalid element QName",
)
}
}
None => ()
}
let attributes : Array[(String, String)] = []
let unretained_attribute_names : Array[String] = []
let mut self_closing = false
while at < length {
// Skip whitespace.
let mut separated = false
while at < length && is_xml_space(part[at]) {
separated = true
at += 1
}
if at >= length {
break
}
if part[at] == b'>' {
at += 1
return (name, attributes, unretained_attribute_names, self_closing, at)
}
if part[at] == b'/' {
match source_budget {
Some(_) => {
guard at + 1 < length && part[at + 1] == b'>' else {
raise Unsupported(
message="the projection source scanner found malformed self-closing-tag content",
)
}
return (name, attributes, unretained_attribute_names, true, at + 2)
}
None => ()
}
self_closing = true
at += 1
continue
}
match source_budget {
Some(_) => {
guard separated else {
raise Unsupported(
message="the projection source scanner found attributes without separating whitespace",
)
}
}
None => ()
}
// Attribute name.
let attr_start = at
while at < length &&
part[at] != b'=' &&
!is_xml_space(part[at]) &&
part[at] != b'>' {
at += 1
}
let attr_end = at
let relevant_attribute = annotation_attribute_is_relevant(
part,
attr_start,
attr_end,
source_budget is Some(_),
)
let validated_source_name = match source_budget {
Some(budget) => {
guard attr_end - attr_start <= max_annotation_relevant_attribute_bytes else {
raise @core.docx_xml_resource_limit_error(DocxXmlTokenLength)
}
// Refuse before materializing every source attribute QName. Relevant
// names live in the retained attribute array; other names live just
// long enough for scan_story's post-declaration namespace check.
budget.charge_retained_token()
budget.charge_retained_chars(attr_end - attr_start)
let candidate = utf8_slice(part, attr_start, attr_end)
guard projection_source_is_xml_qname(candidate) else {
raise Unsupported(
message="the projection source scanner found an invalid attribute QName",
)
}
if !relevant_attribute {
unretained_attribute_names.push(candidate)
}
Some(candidate)
}
None => None
}
while at < length && is_xml_space(part[at]) {
at += 1
}
if at >= length || part[at] != b'=' {
match source_budget {
Some(_) =>
raise Unsupported(
message="the projection source scanner found a valueless attribute",
)
// The ordinary annotation scanner historically tolerates valueless
// attributes; source retention alone needs strict physical identity.
None => continue
}
}
at += 1
while at < length && is_xml_space(part[at]) {
at += 1
}
if at >= length || (part[at] != b'"' && part[at] != b'\'') {
raise Unsupported(
message="the annotation scanner found an unquoted attribute value",
)
}
let quote = part[at]
at += 1
let value_start = at
while at < length && part[at] != quote {
at += 1
}
if at >= length {
raise Unsupported(
message="the annotation scanner found an unterminated attribute value",
)
}
if relevant_attribute {
if attr_end - attr_start > max_annotation_relevant_attribute_bytes ||
at - value_start > max_annotation_relevant_attribute_bytes {
raise @core.docx_xml_resource_limit_error(DocxXmlTokenLength)
}
match source_budget {
Some(budget) =>
// The QName and its fixed-size retained slot were charged before the
// QName allocation above; only the value remains.
budget.charge_retained_chars(at - value_start)
None => ()
}
let attr_name = match validated_source_name {
Some(value) => value
None => utf8_slice(part, attr_start, attr_end)
}
let raw_value = @utf8.decode(part[value_start:at]) catch {
_ =>
raise Unsupported(
message="the annotation scanner found a non-UTF-8 attribute value",
)
}
// Attribute values must be entity-decoded: w:id="1" IS id "1"
// (and namespace URIs may legally carry &).
attributes.push((attr_name, decode_entities(raw_value)))
}
at += 1
}
raise Unsupported(message="the annotation scanner found an unterminated tag")
}
///|
fn bytes_range_equals_ascii(
part : BytesView,
start : Int,
end : Int,
expected : String,
) -> Bool {
if end - start != expected.length() {
return false
}
for offset in 0.. Bool {
if end - start < expected.length() {
return false
}
bytes_range_equals_ascii(part, start, start + expected.length(), expected)
}
///|
fn bytes_range_has_ascii_suffix(
part : BytesView,
start : Int,
end : Int,
expected : String,
) -> Bool {
if end - start < expected.length() {
return false
}
bytes_range_equals_ascii(part, end - expected.length(), end, expected)
}
///|
/// Filters before materialization. Namespace declarations are needed to
/// resolve names; tracked-change containers additionally need the CT_TrackChange
/// attributes that say who revised the document and when; source-retaining
/// reader projection additionally needs the few attributes that make BodyReader
/// emit zero versus one logical element.
fn annotation_attribute_is_relevant(
part : BytesView,
start : Int,
end : Int,
retain_projection_attributes : Bool,
) -> Bool {
bytes_range_equals_ascii(part, start, end, "xmlns") ||
bytes_range_has_ascii_prefix(part, start, end, "xmlns:") ||
bytes_range_has_ascii_suffix(part, start, end, ":id") ||
bytes_range_has_ascii_suffix(part, start, end, ":type") ||
bytes_range_has_ascii_suffix(part, start, end, ":val") ||
bytes_range_has_ascii_suffix(part, start, end, ":author") ||
bytes_range_has_ascii_suffix(part, start, end, ":date") ||
(
retain_projection_attributes &&
(
bytes_range_has_ascii_suffix(part, start, end, ":fldCharType") ||
// Paragraph identity: `w14:paraId` values are retained AS SPELLED
// for the identity layer. The suffix admits any prefix; the
// URI-aware lookup at capture time is the namespace judgment.
bytes_range_has_ascii_suffix(part, start, end, ":paraId")
)
)
}
///|
fn is_xml_space(byte : Byte) -> Bool {
byte == b' ' || byte == b'\t' || byte == b'\r' || byte == b'\n'
}
///|
fn is_name_end(byte : Byte) -> Bool {
is_xml_space(byte) || byte == b'>' || byte == b'/'
}
///|
fn utf8_slice(
part : BytesView,
start : Int,
end : Int,
) -> String raise DocxError {
@utf8.decode(part[start:end]) catch {
_ =>
raise Unsupported(
message="the annotation scanner found a non-UTF-8 XML name or value",
)
}
}
///|
/// The marker's id attribute is namespace-qualified (`w:id`): any prefix
/// bound to a WML URI matches; unprefixed attributes never do (default
/// namespaces do not apply to attributes). When mixed Strict/Transitional
/// prefixes both qualify, the LAST physical match wins in every scan mode —
/// the XML reader's canonical-map overwrite precedence. Legacy annotation
/// scans historically took the first match; #462 phase C unified them so a
/// document resolves identical marker/revision identities whether its scan
/// is rebuilt or adopted from the retained reader projection.
fn uri_attribute_value(
attributes : Array[(String, String)],
local_name : String,
namespaces : StableStringMap[Array[String]],
uri_matches : (String) -> Bool,
) -> String? {
let mut matched : String? = None
for pair in attributes {
let (name, value) = pair
match name.find(":") {
Some(colon) => {
let prefix = name[0:colon].to_owned()
let attr_local = name[colon + 1:].to_owned()
if attr_local == local_name {
match namespaces.get(prefix) {
Some(stack) if stack.length() > 0 &&
uri_matches(stack[stack.length() - 1]) => matched = Some(value)
_ => ()
}
}
}
// Unprefixed attributes are in NO namespace (default xmlns does not
// apply to attributes) — they never match a namespaced attribute.
None => ()
}
}
matched
}
///|
fn attribute_value(
attributes : Array[(String, String)],
local_name : String,
namespaces : StableStringMap[Array[String]],
) -> String? {
let mut matched : String? = None
for pair in attributes {
let (name, value) = pair
match name.find(":") {
Some(colon) => {
let prefix = name[0:colon].to_owned()
let attr_local = name[colon + 1:].to_owned()
if attr_local == local_name {
match namespaces.get(prefix) {
Some(stack) if stack.length() > 0 &&
is_wml_uri(stack[stack.length() - 1]) => matched = Some(value)
_ => ()
}
}
}
// Unprefixed attributes are in NO namespace (default xmlns does not
// apply to attributes) — they never match a w: attribute.
None => ()
}
}
matched
}
///|
/// Decodes the five XML predefined entities and numeric character
/// references. Malformed references are kept verbatim — @json-style
/// tolerance; the XML parser is the strict gate.
fn decode_entities(value : String) -> String {
if value.find("&") is None {
return value
}
let units = value.code_units()
let builder = StringBuilder()
let length = units.length()
let mut at = 0
while at < length {
let unit = units[at].to_int()
if unit != '&'.to_int() {
match unit.to_char() {
Some(ch) => builder.write_char(ch)
None =>
// Keep lone surrogates as-is via a paired write when possible.
match
(if at + 1 < length {
(unit, units[at + 1].to_int())
} else {
(unit, 0)
}) {
(high, low) if high >= 0xD800 &&
high <= 0xDBFF &&
low >= 0xDC00 &&
low <= 0xDFFF => {
match
(0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00)).to_char() {
Some(ch) => builder.write_char(ch)
None => ()
}
at += 1
}
_ => ()
}
}
at += 1
continue
}
// Named entities have a fixed small vocabulary. Numeric references may be
// arbitrarily zero-padded, so scan their digit run without an artificial
// length ceiling. Stopping at the first non-digit keeps malformed tolerant
// input linear rather than repeatedly searching to a distant semicolon.
let numeric = at + 1 < length && units[at + 1].to_int() == '#'.to_int()
let mut numeric_base = 10
let mut digits_start = at + 2
if numeric &&
digits_start < length &&
(
units[digits_start].to_int() == 'x'.to_int() ||
units[digits_start].to_int() == 'X'.to_int()
) {
numeric_base = 16
digits_start += 1
}
let mut semi = -1
let mut probe = at + 1
while probe < length && (numeric || probe <= at + 24) {
if units[probe].to_int() == ';'.to_int() {
semi = probe
break
}
if numeric && probe >= digits_start {
let code = units[probe].to_int()
let digit = (code >= '0'.to_int() && code <= '9'.to_int()) ||
(
numeric_base == 16 &&
(
(code >= 'a'.to_int() && code <= 'f'.to_int()) ||
(code >= 'A'.to_int() && code <= 'F'.to_int())
)
)
if !digit {
break
}
}
probe += 1
}
if semi < 0 {
builder.write_char('&')
at += 1
continue
}
let name_builder = StringBuilder()
for i in (at + 1).. name_builder.write_char(ch)
None => ()
}
}
let entity = name_builder.to_string()
let decoded : Char? = match entity {
"amp" => Some('&')
"lt" => Some('<')
"gt" => Some('>')
"quot" => Some('"')
"apos" => Some('\'')
_ =>
if entity.has_prefix("#x") || entity.has_prefix("#X") {
parse_char_reference(entity[2:].to_owned(), 16)
} else if entity.has_prefix("#") {
parse_char_reference(entity[1:].to_owned(), 10)
} else {
None
}
}
match decoded {
Some(ch) => {
builder.write_char(ch)
at = semi + 1
}
None => {
builder.write_char('&')
at += 1
}
}
}
builder.to_string()
}
///|
fn parse_char_reference(digits : String, base : Int) -> Char? {
// Value-based, overflow-aware: leading-zero padding is valid XML
// (1 is "1"); anything past U+10FFFF is invalid however it
// is spelled.
if digits.length() == 0 {
return None
}
let mut value = 0
for ch in digits {
let digit = if ch is ('0'..='9') {
ch.to_int() - '0'.to_int()
} else if base == 16 && ch is ('a'..='f') {
ch.to_int() - 'a'.to_int() + 10
} else if base == 16 && ch is ('A'..='F') {
ch.to_int() - 'A'.to_int() + 10
} else {
return None
}
value = value * base + digit
if value > 0x10FFFF {
return None
}
}
value.to_char()
}
///|
#warnings("-unused_value")
fn scan_projection_source_tree(
part : BytesView,
budget? : ProjectionSourceBudget,
path_budget? : AnnotationPathBudget,
) -> StoryScan raise DocxError {
let budget = match budget {
Some(value) => value
None => projection_source_budget()
}
scan_story(part, path_budget?, source_budget=budget)
}
///|
fn scan_text(xml : String) -> StoryScan raise DocxError {
scan_story(@utf8.encode(xml))
}
///|
test "scanner: projection paths, marker containment, and document order" {
let scan = scan_text(
(
#|
#|
#|
#|plain
#|annotated
#|cell link
#|
),
)
debug_inspect(scan.warnings().length(), content="0")
let paths : Array[String] = []
for node in scan.nodes() {
paths.push("\{node.kind} \{node.path()}")
}
paths.sort()
debug_inspect(
paths,
content=(
#|[
#| "p p[1]",
#| "p p[2]",
#| "p p[3]",
#| "tbl tbl[1]",
#| "r p[1]/r[1]",
#| "r p[2]/r[1]",
#| "r p[2]/r[2]",
#| "tr tbl[1]/tr[1]",
#| "tc tbl[1]/tr[1]/tc[1]",
#| "r p[3]/hyperlink[1]/r[1]",
#| "p tbl[1]/tr[1]/tc[1]/p[1]",
#| "hyperlink p[3]/hyperlink[1]",
#| "r tbl[1]/tr[1]/tc[1]/p[1]/r[1]",
#|]
),
)
let marks : Array[String] = []
for marker in scan.markers() {
marks.push(
"\{marker.kind()} id=\{marker.id()} in=\{marker.container_path().unwrap_or("")} after=\{marker.after_child().unwrap_or("-")}",
)
}
debug_inspect(
marks,
content=(
#|[
#| "commentRangeStart id=1 in=p[2] after=-",
#| "commentRangeEnd id=1 in=p[2] after=p[2]/r[1]",
#| "commentReference id=1 in=p[2]/r[2] after=-",
#| "footnoteReference id=2 in=p[3]/hyperlink[1]/r[1] after=-",
#|]
),
)
// Document order and interval sanity: the range start precedes r[1]'s
// open; the end follows r[1]'s close; the covered run's interval nests
// inside [start.seq, end.seq].
let markers = scan.markers()
let mut covered_open = 0
let mut covered_close = 0
for node in scan.nodes() {
if node.path() == "p[2]/r[1]" {
covered_open = node.open_seq()
covered_close = node.close_seq()
}
}
debug_inspect(markers[0].seq() < covered_open, content="true")
debug_inspect(covered_close < markers[1].seq(), content="true")
}
///|
test "scanner: tracked-change containers survive the reader's flattening" {
// The reader flattens w:ins into the accepted text and drops w:del
// wholesale, so neither reaches the parsed tree. The scan is the only
// place the revision identity exists.
let scan = scan_text(
(
#|
#|
#|The revenue was flat up 18% cell
#|
),
)
let found : Array[String] = []
for revision in scan.revisions() {
found.push(
"\{revision.kind()} id=\{revision.id().unwrap_or("-")} author=\{revision.author().unwrap_or("-")} date=\{revision.date().unwrap_or("-")} in=\{revision.container_path().unwrap_or("")}",
)
}
// The dateless insertion reports "-", not a substituted timestamp: w:date
// is optional in CT_TrackChange and defaulting it would invent a fact.
debug_inspect(
found,
content=(
#|[
#| "del id=1 author=Reviewer date=2026-01-01T00:00:00Z in=p[1]",
#| "ins id=2 author=Reviewer date=- in=p[1]",
#| "ins id=3 author=Ravi date=- in=tbl[1]/tr[1]/tc[1]/p[1]",
#|]
),
)
debug_inspect(scan.warnings().length(), content="0")
}
///|
test "scanner: revisions the reader never renders are not reported" {
// An unselected mc:Choice branch is dropped by the reader, so a revision
// inside it describes text nobody reads. Property-container revisions
// (an inserted paragraph MARK, a deleted table ROW) are deliberately out
// of scope for the same reason markers are: they revise a property, not
// content, and the deleted row is retracted from the projection entirely.
let scan = scan_text(
(
#|
#|
#|a x b c d
#|
),
)
let found : Array[String] = []
for revision in scan.revisions() {
found.push("\{revision.kind()} id=\{revision.id().unwrap_or("-")}")
}
// Only the revision inside a rendered paragraph survives. id=4 was
// retracted with its deleted row; id=1 and id=3 are property revisions;
// id=2 sits in a branch the reader discards.
debug_inspect(
found,
content=(
#|["ins id=5"]
),
)
}
///|
test "scanner: a revision without an author is reported and diagnosed" {
let scan = scan_text(
(
#|
#|x
),
)
guard scan.revisions() is [revision] else { fail("expected one revision") }
debug_inspect(revision.author(), content="None")
debug_inspect(
scan.warnings(),
content=(
#|["annotation scanner: a w:ins revision has no w:author attribute"]
),
)
}
///|
/// Three scanner behaviours no fixture reached: the endnote marker kind, the
/// explicit `w:vMerge w:val="continue"` spelling, and the `w:delInstrText`
/// deleted-instruction carrier.
test "scanner: endnote markers, explicit continuation, deleted instructions" {
let scan = scan_text(
(
#|
#|
#|gone PAGE
#|A gone
#|
#|
),
)
debug_inspect(scan.warnings().length(), content="0")
guard scan.markers() is [endnote_marker] else {
fail("expected the endnote marker")
}
inspect("\{endnote_marker.kind()}", content="endnoteReference")
inspect(endnote_marker.id(), content="9")
// the deleted carriers belong to their deletion, each recorded under its
// replacement name: delInstrText edits to instrText, delText to t. This is
// the mapping the mutation resolver renames through, and asserting only
// that a deletion existed left the whole mapping deletable -- review
// measured the entire suite green without it.
guard scan.revisions() is [deletion] else { fail("expected the deletion") }
inspect("\{deletion.kind()}", content="del")
guard scan.revision_sites() is [del_site] else {
fail("expected the deletion site")
}
inspect(
del_site.name_edits.map(edit => edit.replacement).join(","),
content="t,instrText",
)
// the endnote STORY projects note nodes, distinct from the endnote marker
// pinned above: removing "endnote" from projection_kind loses the node.
// The separator and continuation plumbing notes are suppressed -- only the
// user endnote appears, and as note[1] -- which pins the endnote arm of
// the plumbing filter the footnote fixtures could not reach.
let endnotes = scan_text(
(
#|
#|sep cont E
),
)
let note_kinds : Array[String] = []
for node in endnotes.nodes() {
note_kinds.push("\{node.kind} \{node.path()}")
}
note_kinds.sort()
debug_inspect(
note_kinds,
content=(
#|["note note[1]", "p note[1]/p[1]", "r note[1]/p[1]/r[1]"]
),
)
}
///|
/// Three more scanner behaviours review proved unwitnessed: the MC container
/// tables at scanner level (the generated pair proves the reader and
/// projection agree, not what the scanner's own tables do), the five named
/// entities in the scanner's OWN decoder (the token map's entity tests
/// exercise a different implementation), and retention of the exact `xmlns`
/// default-namespace attribute (every fixture bound prefixes).
test "scanner: MC selection, named entities, and the default namespace" {
// AlternateContent flattens, Choice suppresses, Fallback flattens: the
// fallback paragraph projects with ordinal continuity, the choice one
// never exists. Removing any of the three table entries breaks this.
let mc = scan_text(
(
#|
#|
#|before choice fallback after
#|
),
)
let mc_paths : Array[String] = []
for node in mc.nodes() {
if node.kind == "p" {
mc_paths.push(node.path())
}
}
mc_paths.sort()
debug_inspect(
mc_paths,
content=(
#|["p[1]", "p[2]", "p[3]"]
),
)
// object and smartTag transparency with ordinal CONTINUITY: the runs
// inside both wrappers keep the paragraph's run numbering, so r[2] and
// r[3] follow r[1]. Text equality alone could not see either table
// entry vanish.
let object_scan = scan_text(
(
#|
#|a b c
),
)
let object_paths : Array[String] = []
for node in object_scan.nodes() {
object_paths.push("\{node.kind} \{node.path()}")
}
object_paths.sort()
debug_inspect(
object_paths,
content=(
#|["p p[1]", "r p[1]/r[1]", "r p[1]/r[2]", "r p[1]/r[3]"]
),
)
// the five named entities decode in the scanner's own attribute decoder;
// numeric references were tested, the named arms were not
let entities = scan_text(
(
#|
#|x
),
)
guard entities.revisions() is [insertion] else {
fail("expected the insertion")
}
inspect(insertion.author().unwrap_or(""), content="a&bd\"e'f")
// the DEFAULT namespace: an unprefixed document relies on retention of the
// exact `xmlns` attribute to mean WML at all
let unprefixed = scan_text(
(
#|
#|plain
#|
),
)
let plain_paths : Array[String] = []
for node in unprefixed.nodes() {
plain_paths.push("\{node.kind} \{node.path()}")
}
plain_paths.sort()
debug_inspect(
plain_paths,
content=(
#|["p p[1]", "r p[1]/r[1]"]
),
)
}
///|
/// The complete revision-site table, name by name.
///
/// `is_revision_site_name` lists twenty-eight tracked-change element names,
/// and every one must push a site record -- the mutation resolver refuses on
/// constructs the projection drops, so a name missing from the table is a
/// construct "accept everything" would silently leave behind. Only content
/// `ins`/`del` are resolvable; the other twenty-six (and ins/del inside a
/// property container, pinned elsewhere) record as unsupported.
test "scanner: every revision-site name records a site" {
let names = [
"cellDel", "cellIns", "cellMerge", "customXmlDelRangeEnd", "customXmlDelRangeStart",
"customXmlInsRangeEnd", "customXmlInsRangeStart", "customXmlMoveFromRangeEnd",
"customXmlMoveFromRangeStart", "customXmlMoveToRangeEnd", "customXmlMoveToRangeStart",
"del", "ins", "moveFrom", "moveFromRangeEnd", "moveFromRangeStart", "moveTo",
"moveToRangeEnd", "moveToRangeStart", "numberingChange", "pPrChange", "rPrChange",
"sectPrChange", "tblGridChange", "tblPrChange", "tblPrExChange", "tcPrChange",
"trPrChange",
]
let body = StringBuilder()
for name in names {
body.write_string("")
}
let scan = scan_text(
"" +
body.to_string() +
" " +
body.replace(old="CONTAINED", new=site) +
"
#|
#|
#|hi
#|
#|
),
)
let marks : Array[String] = []
for marker in scan.markers() {
marks.push("\{marker.kind()} id=\{marker.id()}")
}
debug_inspect(
marks,
content=(
#|["commentRangeStart id=3", "commentRangeEnd id=3"]
),
)
debug_inspect(scan.nodes().length(), content="2")
}
///|
test "scanner: comments story counts each comment body separately" {
// w:comment is a projection CONTAINER (kind "comment") in the comments
// story, so paragraph paths carry the container identity and each
// comment's paragraphs count independently — review round 1 finding 3.
let scan = scan_text(
(
#|
#|a b c
),
)
let paragraph_paths : Array[String] = []
for node in scan.nodes() {
if node.kind == "p" {
paragraph_paths.push(node.path())
}
}
paragraph_paths.sort()
debug_inspect(
paragraph_paths,
content=(
#|["comment[1]/p[1]", "comment[1]/p[2]", "comment[2]/p[1]"]
),
)
}
///|
test "scanner: unterminated tag and UTF-16 parts fail closed" {
try {
let _ = scan_text(
"
inspect(
message.has_prefix("the annotation scanner found an unterminated"),
content="true",
)
err => fail("unexpected error: \{repr(err)}")
} noraise {
_ => fail("expected the scan to fail")
}
let utf16 : Bytes = b"\xFF\xFE<\x00a\x00>\x00"
try {
let _ = scan_story(utf16)
} catch {
Unsupported(message~) =>
inspect(
message.has_prefix("the annotation scanner supports UTF-8"),
content="true",
)
err => fail("unexpected error: \{repr(err)}")
} noraise {
_ => fail("expected the scan to fail")
}
}
///|
test "scanner review round 1: flattening parity — sdt, ins, del, vMerge" {
// The reader flattens sdt/sdtContent/smartTag/ins and drops w:del and
// vMerge continuation cells; scanner ordinals must mirror that.
let scan = scan_text(
(
#|
#|
#|a inserted gone
#|tall b b2
#|
#|
),
)
let paths : Array[String] = []
for node in scan.nodes() {
paths.push(node.path())
}
paths.sort()
// The sdt-wrapped paragraph is p[2] (NOT a nested p[1]); the inserted
// run is p[2]/r[1]; the deleted run has no path at all. Row 2's
// continuation cell is stripped, so its sibling is tc[1] in the AST —
// and the continuation cell's paragraph/run get no paths.
debug_inspect(
paths,
content=(
#|[
#| "p[1]",
#| "p[2]",
#| "tbl[1]",
#| "p[1]/r[1]",
#| "p[2]/r[1]",
#| "tbl[1]/tr[1]",
#| "tbl[1]/tr[2]",
#| "tbl[1]/tr[1]/tc[1]",
#| "tbl[1]/tr[1]/tc[2]",
#| "tbl[1]/tr[2]/tc[1]",
#| "tbl[1]/tr[1]/tc[1]/p[1]",
#| "tbl[1]/tr[1]/tc[2]/p[1]",
#| "tbl[1]/tr[2]/tc[1]/p[1]",
#| "tbl[1]/tr[1]/tc[1]/p[1]/r[1]",
#| "tbl[1]/tr[1]/tc[2]/p[1]/r[1]",
#| "tbl[1]/tr[2]/tc[1]/p[1]/r[1]",
#|]
),
)
}
///|
test "scanner review round 1: entity-decoded ids, Strict WML, CDATA immunity" {
// 1 IS id "1"; the Strict namespace URI is recognized; marker
// lookalikes inside CDATA are never counted.
let scan = scan_text(
(
#|
#|
#|]]>
#|
),
)
let marks : Array[String] = []
for marker in scan.markers() {
marks.push("\{marker.kind()} id=\{marker.id()}")
}
debug_inspect(
marks,
content=(
#|[
#| "commentRangeStart id=1",
#| "commentRangeEnd id=1",
#| "commentReference id=1",
#|]
),
)
}
///|
test "scanner skips oversized irrelevant attributes before decoding" {
let padding = "0".repeat(32 * 1024)
let scan = scan_text(
"
#|real
),
)
let paths : Array[String] = []
for node in scan.nodes() {
paths.push(node.path())
}
paths.sort()
debug_inspect(
paths,
content=(
#|["note[1]", "note[1]/p[1]", "note[1]/p[1]/r[1]"]
),
)
}
///|
test "scanner review round 3: vMerge fail-open, property-container markers, projection kinds" {
// A continuation in the FIRST row has no merge to continue — the
// reader keeps the cell (fail-open) and so must the scanner. A marker
// inside tcPr never surfaces (the reader ignores property-container
// content). Note containers report the PROJECTION kind "note".
let scan = scan_text(
(
#|
#|
#|
#|orphan continuation, kept tall
#|
#|
),
)
debug_inspect(scan.markers().length(), content="0")
let cells : Array[String] = []
for node in scan.nodes() {
if node.kind == "tc" {
cells.push(node.path())
}
}
cells.sort()
// Row 1's orphan continuation kept; row 3's true continuation gone.
debug_inspect(
cells,
content=(
#|["tbl[1]/tr[1]/tc[1]", "tbl[1]/tr[2]/tc[1]"]
),
)
let note_scan = scan_text(
(
#|
#|real
),
)
let mut container_kind = ""
for node in note_scan.nodes() {
if node.path() == "note[1]" {
container_kind = node.kind
}
}
debug_inspect(
container_kind,
content=(
#|"note"
),
)
}
///|
test "scanner review round 4: reader-exact merge origins, spans, and row deletion" {
let scan = scan_text(
(
#|
#|
#|
#|orphan kept, becomes origin second orphan merges away span2 restart at col0 plain col0 misaligned col1 continuation KEPT (origins key on start column) deleted row first vMerge wins: continuation, merges into col0 origin above after self-closing cell
#|
#|
),
)
let cells : Array[String] = []
for node in scan.nodes() {
if node.kind is ("tc" | "tr") {
cells.push("\{node.kind} \{node.path()}")
}
}
cells.sort()
// Rows: r1 orphan kept (tr[1]/tc[1]); r2 second orphan MERGED (tr[2]
// has no tc); r3 span-2 restart kept; r4 plain col0 + misaligned col1
// continuation BOTH kept; r5 deleted (no tr — later rows renumber);
// r6 (now tr[5]) first-vMerge-wins continuation MERGED into the col0
// origin (the r4 plain cell); r7 (tr[6]) self-closing tc + sibling.
debug_inspect(
cells,
content=(
#|[
#| "tr tbl[1]/tr[1]",
#| "tr tbl[1]/tr[2]",
#| "tr tbl[1]/tr[3]",
#| "tr tbl[1]/tr[4]",
#| "tr tbl[1]/tr[5]",
#| "tr tbl[1]/tr[6]",
#| "tc tbl[1]/tr[1]/tc[1]",
#| "tc tbl[1]/tr[3]/tc[1]",
#| "tc tbl[1]/tr[4]/tc[1]",
#| "tc tbl[1]/tr[4]/tc[2]",
#| "tc tbl[1]/tr[6]/tc[1]",
#| "tc tbl[1]/tr[6]/tc[2]",
#|]
),
)
// gridSpan parse parity with the reader (parse_grid_span): the span-2
// restart advanced the cursor so r3's would-be second cell (none) —
// pinned indirectly by r4's col assignments above.
}
///|
test "scanner review round 5: span-sensitive merging, first-tcPr-only, retraction purges" {
// Row 1: a span-"2abc" cell (reader's parse_grid_span -> 2) then a
// plain sibling — the sibling lands at column 2 ONLY if the span
// parsed as 2. Row 2: continuations at columns 0 and 2 — BOTH merge
// away iff the row-1 columns were registered at 0 and 2 (a span-1
// misparse would register the sibling at column 1 and leave the
// column-2 continuation kept). Row 3: a SECOND tcPr carrying vMerge
// must be ignored (reader honors only the first), so the cell is
// KEPT. Row 4: a first-tcPr continuation arriving AFTER content —
// the retraction purges the already-emitted paragraph and marker.
let scan = scan_text(
(
#|
#|
#|
#|span two at col 2 second tcPr ignored, kept emitted then purged
#|
#|
),
)
// Wait — row 2's second continuation sits at column 1 (cursor), not 2.
// The cursor claims columns in document order: cell 1 at col 0
// (span 1 default? no: vMerge only) then cell 2 at col 1. Column 1
// has NO origin (row 1 registered 0 and 2), so cell 2 is KEPT.
let cells : Array[String] = []
for node in scan.nodes() {
if node.kind == "tc" {
cells.push(node.path())
}
}
cells.sort()
debug_inspect(
cells,
content=(
#|[
#| "tbl[1]/tr[1]/tc[1]",
#| "tbl[1]/tr[1]/tc[2]",
#| "tbl[1]/tr[2]/tc[1]",
#| "tbl[1]/tr[3]/tc[1]",
#|]
),
)
// The purged row-4 cell left no paragraph, run, or marker behind.
// The row node itself survives (only the CELL was retracted), so the
// no-leak check targets the row's CHILDREN — round-6 review caught the
// earlier prefix matching the row and the pin asserting a tautology.
let mut leaked = false
for node in scan.nodes() {
if node.path().has_prefix("tbl[1]/tr[4]/") {
leaked = true
}
}
debug_inspect(leaked, content="false")
debug_inspect(scan.markers().length(), content="0")
}
///|
test "scanner review round 6: a deleted row rolls back its origin registrations" {
// Row 1 is deleted AFTER its cell closed (late first trPr/w:del) —
// its origin at column 0 must be rolled back, so row 2's orphan
// continuation is KEPT (the reader excludes deleted rows before span
// calculation). Row 3's continuation then merges into ROW 2's cell.
let scan = scan_text(
(
#|
#|
#|
#|closes before the row is deleted orphan kept
#|
#|
),
)
let rows_and_cells : Array[String] = []
for node in scan.nodes() {
if node.kind is ("tr" | "tc") {
rows_and_cells.push("\{node.kind} \{node.path()}")
}
}
rows_and_cells.sort()
debug_inspect(
rows_and_cells,
content=(
#|["tr tbl[1]/tr[1]", "tr tbl[1]/tr[2]", "tc tbl[1]/tr[1]/tc[1]"]
),
)
}
///|
test "scanner complexity: alternating late row and cell retractions stay linear" {
let count = 128
let xml = StringBuilder()
xml.write_string(
"",
)
// Establish the column-zero origin required for each late vMerge
// continuation below to be retracted.
xml.write_string(" ")
// Retained story-level markers exercise both monotonic nearest-node sweeps.
for index in 0..