///|
/// What a corpus actually covers: which universal tags and relations appear,
/// which tags it uses that the UD spec does not define, how well-formed its
/// trees are, and the shape of its sentences. Answers the questions a corpus
/// maintainer asks before trusting a treebank.
pub struct Coverage {
mut sentences : Int32
mut words : Int32
upos_used : Array[String]
mut upos_missing : Array[String]
deprel_used : Array[String]
mut deprel_missing : Array[String]
unknown_upos : Array[String]
unknown_deprel : Array[String]
/// Word rows whose UPOS is `_`, i.e. tokens left without a tag.
mut untagged : Int32
mut projective : Int32
mut non_projective : Int32
mut single_rooted : Int32
mut multi_rooted : Int32
mut max_depth : Int32
mut avg_depth : Double
/// Sum of every word's tree depth, kept so merged reports can recompute an
/// average weighted by tokens instead of averaging averages.
mut depth_total : Int32
/// Sentences with more than 50 words; long inputs are where parsers and
/// memory budgets break first.
mut long_sentences : Int32
mut empty_nodes : Int32
mut multiword_tokens : Int32
mut documents : Int32
mut paragraphs : Int32
}
///|
fn join(items : Array[String], separator : String) -> String {
if items.length() == 0 {
return "none"
}
let mut out = ""
let mut i = 0
while i < items.length() {
if i > 0 {
out = out + separator
}
out = out + items.at(i)
i = i + 1
}
out
}
///|
fn push_unique(items : Array[String], value : String) -> Unit {
if !items.contains(value) {
items.push(value)
}
}
///|
fn missing_from(spec : Array[String], used : Array[String]) -> Array[String] {
let out : Array[String] = []
for tag in spec {
if !used.contains(tag) {
out.push(tag)
}
}
out
}
///|
/// Measures one corpus.
pub fn coverage(sentences : Array[Sentence]) -> Coverage {
let upos_used : Array[String] = []
let deprel_used : Array[String] = []
let unknown_upos : Array[String] = []
let unknown_deprel : Array[String] = []
let cov : Coverage = {
sentences: 0,
words: 0,
upos_used,
upos_missing: [],
deprel_used,
deprel_missing: [],
unknown_upos,
unknown_deprel,
untagged: 0,
projective: 0,
non_projective: 0,
single_rooted: 0,
multi_rooted: 0,
max_depth: 0,
avg_depth: 0.0,
depth_total: 0,
long_sentences: 0,
empty_nodes: 0,
multiword_tokens: 0,
documents: 0,
paragraphs: 0,
}
let mut depth_sum : Int32 = 0
for sentence in sentences {
cov.sentences = cov.sentences + 1
let words = sentence.words()
let count = words.length()
cov.words = cov.words + count
if count > 50 {
cov.long_sentences = cov.long_sentences + 1
}
if is_projective(sentence) {
cov.projective = cov.projective + 1
} else {
cov.non_projective = cov.non_projective + 1
}
if is_single_rooted(sentence) {
cov.single_rooted = cov.single_rooted + 1
} else {
cov.multi_rooted = cov.multi_rooted + 1
}
let d = max_depth(sentence)
if d > cov.max_depth {
cov.max_depth = d
}
for row in sentence.rows() {
if row.kind is Multiword {
cov.multiword_tokens = cov.multiword_tokens + 1
}
if row.kind is Empty {
cov.empty_nodes = cov.empty_nodes + 1
}
}
for row in words {
let tag = row.upos
if tag == "_" || tag.is_empty() {
cov.untagged = cov.untagged + 1
} else if is_known_upos(tag) {
push_unique(cov.upos_used, tag)
} else {
push_unique(cov.unknown_upos, tag)
}
let rel = row.deprel
if rel == "_" || rel.is_empty() {
// Enhanced-only rows and empty nodes may leave the basic relation
// empty; that is a different problem from an unknown label.
continue
}
let base = base_deprel(rel)
if is_known_deprel(base) {
push_unique(cov.deprel_used, base)
} else {
push_unique(cov.unknown_deprel, base)
}
}
let mut k = 0
while k < count {
depth_sum = depth_sum + depth(sentence, words.at(k).start)
k = k + 1
}
}
cov.avg_depth = if cov.words == 0 {
0.0
} else {
depth_sum.to_double() / cov.words.to_double()
}
cov.depth_total = depth_sum
cov.upos_used.sort_by(compare_labels)
cov.deprel_used.sort_by(compare_labels)
cov.unknown_upos.sort_by(compare_labels)
cov.unknown_deprel.sort_by(compare_labels)
cov.upos_missing = missing_from(upos_tags, cov.upos_used)
cov.deprel_missing = missing_from(deprel_tags, cov.deprel_used)
let docs = documents(sentences)
cov.documents = docs.length()
let mut pars = 0
for doc in docs {
pars = pars + doc.paragraph_count()
}
cov.paragraphs = pars
cov
}
///|
fn is_known_upos(tag : String) -> Bool {
upos_tags.contains(tag)
}
///|
fn is_known_deprel(base : String) -> Bool {
deprel_tags.contains(base)
}
///|
fn merge_into(target : Array[String], source : Array[String]) -> Unit {
for item in source {
push_unique(target, item)
}
}
///|
/// Combines per-file coverages into one corpus-wide view: counters add up,
/// used labels are unioned, missing labels are recomputed against the spec, and
/// the average depth is rebuilt from the token-weighted depth total so a merged
/// report does not average averages.
pub fn merge_coverage(items : Array[Coverage]) -> Coverage {
let merged : Coverage = {
sentences: 0,
words: 0,
upos_used: [],
upos_missing: [],
deprel_used: [],
deprel_missing: [],
unknown_upos: [],
unknown_deprel: [],
untagged: 0,
projective: 0,
non_projective: 0,
single_rooted: 0,
multi_rooted: 0,
max_depth: 0,
avg_depth: 0.0,
depth_total: 0,
long_sentences: 0,
empty_nodes: 0,
multiword_tokens: 0,
documents: 0,
paragraphs: 0,
}
for item in items {
merged.sentences = merged.sentences + item.sentences
merged.words = merged.words + item.words
merged.untagged = merged.untagged + item.untagged
merged.projective = merged.projective + item.projective
merged.non_projective = merged.non_projective + item.non_projective
merged.single_rooted = merged.single_rooted + item.single_rooted
merged.multi_rooted = merged.multi_rooted + item.multi_rooted
merged.long_sentences = merged.long_sentences + item.long_sentences
merged.empty_nodes = merged.empty_nodes + item.empty_nodes
merged.multiword_tokens = merged.multiword_tokens + item.multiword_tokens
merged.documents = merged.documents + item.documents
merged.paragraphs = merged.paragraphs + item.paragraphs
merged.depth_total = merged.depth_total + item.depth_total
if item.max_depth > merged.max_depth {
merged.max_depth = item.max_depth
}
merge_into(merged.upos_used, item.upos_used)
merge_into(merged.deprel_used, item.deprel_used)
merge_into(merged.unknown_upos, item.unknown_upos)
merge_into(merged.unknown_deprel, item.unknown_deprel)
}
merged.upos_used.sort_by(compare_labels)
merged.deprel_used.sort_by(compare_labels)
merged.unknown_upos.sort_by(compare_labels)
merged.unknown_deprel.sort_by(compare_labels)
merged.upos_missing = missing_from(upos_tags, merged.upos_used)
merged.deprel_missing = missing_from(deprel_tags, merged.deprel_used)
merged.avg_depth = if merged.words == 0 {
0.0
} else {
merged.depth_total.to_double() / merged.words.to_double()
}
merged
}
///|
/// Share of sentences whose tree is projective, in percent.
pub fn Coverage::projectivity(self : Coverage) -> Double {
if self.sentences == 0 {
return 0.0
}
self.projective.to_double() / self.sentences.to_double() * 100.0
}
///|
/// True when every universal tag and relation of the spec appears at least
/// once. A `false` here is not a defect, but it does tell you the corpus will
/// not exercise those labels in a parser evaluation.
pub fn Coverage::full_upos(self : Coverage) -> Bool {
self.upos_missing.length() == 0
}
///|
/// Same for the 37 universal relations.
pub fn Coverage::full_deprel(self : Coverage) -> Bool {
self.deprel_missing.length() == 0
}
///|
fn percent(value : Double) -> String {
let rounded = (value * 100.0).to_int().to_double() / 100.0
rounded.to_string()
}
///|
/// The same report as one JSON object, for callers that build dashboards or
/// diff two corpora programmatically. Label arrays are plain string arrays.
pub fn Coverage::to_json(self : Coverage) -> String {
let sb = StringBuilder::StringBuilder()
sb.write_string("{\"sentences\":")
sb.write_string(self.sentences.to_string())
sb.write_string(",\"words\":")
sb.write_string(self.words.to_string())
sb.write_string(",\"documents\":")
sb.write_string(self.documents.to_string())
sb.write_string(",\"paragraphs\":")
sb.write_string(self.paragraphs.to_string())
sb.write_string(",\"upos_used\":")
sb.write_string(json_str_array(self.upos_used))
sb.write_string(",\"upos_missing\":")
sb.write_string(json_str_array(self.upos_missing))
sb.write_string(",\"deprel_used\":")
sb.write_string(json_str_array(self.deprel_used))
sb.write_string(",\"deprel_missing\":")
sb.write_string(json_str_array(self.deprel_missing))
sb.write_string(",\"unknown_upos\":")
sb.write_string(json_str_array(self.unknown_upos))
sb.write_string(",\"unknown_deprel\":")
sb.write_string(json_str_array(self.unknown_deprel))
sb.write_string(",\"untagged\":")
sb.write_string(self.untagged.to_string())
sb.write_string(",\"projective\":")
sb.write_string(self.projective.to_string())
sb.write_string(",\"non_projective\":")
sb.write_string(self.non_projective.to_string())
sb.write_string(",\"projectivity\":")
sb.write_string(percent(self.projectivity()))
sb.write_string(",\"single_rooted\":")
sb.write_string(self.single_rooted.to_string())
sb.write_string(",\"multi_rooted\":")
sb.write_string(self.multi_rooted.to_string())
sb.write_string(",\"max_depth\":")
sb.write_string(self.max_depth.to_string())
sb.write_string(",\"avg_depth\":")
sb.write_string(percent(self.avg_depth))
sb.write_string(",\"long_sentences\":")
sb.write_string(self.long_sentences.to_string())
sb.write_string(",\"empty_nodes\":")
sb.write_string(self.empty_nodes.to_string())
sb.write_string(",\"multiword_tokens\":")
sb.write_string(self.multiword_tokens.to_string())
sb.write_string("}")
sb.to_string()
}
///|
/// Human-readable report, one section per dimension. Printed by
/// `conllu analyze`.
pub fn Coverage::to_string(self : Coverage) -> String {
let sb = StringBuilder::StringBuilder()
sb.write_string(
"sentences=\{self.sentences} words=\{self.words} documents=\{self.documents} paragraphs=\{self.paragraphs}",
)
sb.write_string("\n")
sb.write_string(
"upos: \{self.upos_used.length()}/\{upos_tags.length()} used, missing: \{join(self.upos_missing, ",")}",
)
sb.write_string("\n")
sb.write_string(
"deprel: \{self.deprel_used.length()}/\{deprel_tags.length()} used, missing: \{join(self.deprel_missing, ",")}",
)
sb.write_string("\n")
sb.write_string(
"unknown: upos=\{join(self.unknown_upos, ",")} deprel=\{join(self.unknown_deprel, ",")} untagged=\{self.untagged}",
)
sb.write_string("\n")
sb.write_string(
"trees: projective=\{percent(self.projectivity())}% non_projective=\{self.non_projective} single_rooted=\{self.single_rooted} multi_rooted=\{self.multi_rooted} max_depth=\{self.max_depth} avg_depth=\{percent(self.avg_depth)}",
)
sb.write_string("\n")
sb.write_string(
"surface: multiword=\{self.multiword_tokens} empty_nodes=\{self.empty_nodes} long_sentences(>50w)=\{self.long_sentences}",
)
sb.to_string()
}