// Language -- a BCP 47 / RFC 5646 language tag. The `langs` on a post, the
// `lang` on a video caption.
//
// https://atproto.com/specs/lexicon#language
// Ported from @atproto/syntax packages/syntax/src/language.ts.
//
// There are TWO levels here, and the difference is not academic -- the
// protocol's own interop corpus has a whole file of tags that pass one and fail
// the other:
//
// `is_well_formed` -- RFC 5646 §2.1 well-formedness. This is what the Lexicon
// `language` format uses, so it is what a decoder applies to a field
// arriving off the wire. It accepts `JA` and `jaja`.
// `parse` / `is_valid` -- additionally requires a lowercase 2-3 letter
// primary subtag (§2.1.1) and forbids repeated variant subtags or
// repeated extension singletons (§2.2.9, §4.1).
//
// The duplicate rules cannot be expressed as a pattern at all. Upstream uses a
// regex with named groups and then walks the subtags anyway, because JavaScript
// named captures keep only the LAST occurrence and so cannot see a repeat. The
// walk below is that second pass, doing the whole job.
///|
/// Tags that predate the grammar and do not fit it. Matched case-insensitively
/// and in full, before any structural parsing -- `i-default` has a one-letter
/// primary subtag and `sgn-BE-NL` has two region-shaped subtags, so neither can
/// be reached any other way.
let grandfathered_tags : Array[String] = [
// Irregular.
"en-gb-oed", "i-ami", "i-bnn", "i-default", "i-enochian", "i-hak", "i-klingon",
"i-lux", "i-mingo", "i-navajo", "i-pwn", "i-tao", "i-tay", "i-tsu", "sgn-be-fr",
"sgn-be-nl", "sgn-ch-de",
// Regular.
"art-lojban", "cel-gaulish", "no-bok", "no-nyn", "zh-guoyu", "zh-hakka", "zh-min",
"zh-min-nan", "zh-xiang",
]
///|
/// A well-formed BCP 47 language tag, holding the bytes it was parsed from.
/// Case is meaningful to readers by convention but not to equality in the
/// protocol; nothing here normalizes, because a record must round-trip.
pub struct Language(String) derive(Eq, Debug)
///|
pub impl Show for Language with fn output(self, logger) {
logger.write_string(self.0)
}
///|
pub fn Language::to_string(self : Self) -> String {
self.0
}
///|
pub fn Language::unchecked(tag : String) -> Language {
Language(tag)
}
///|
/// RFC 5646 §2.1 well-formedness, and nothing more. **This is the level the
/// Lexicon `language` format applies**, so it is the one a decoder should use
/// on a value arriving from a server -- being stricter than the protocol would
/// mean rejecting records other clients happily wrote.
pub fn Language::is_well_formed(tag : String) -> Bool {
scan_language(tag) is Some(_)
}
///|
/// Well-formedness only. See `parse` for the stricter reading.
pub fn Language::parse_lenient(tag : String) -> Language raise SyntaxError {
guard scan_language(tag) is Some(_) else {
raise SyntaxError(
kind=Language,
input=tag,
reason="language is not a well-formed BCP 47 tag",
)
}
Language(tag)
}
///|
pub fn Language::is_valid(tag : String) -> Bool {
try {
Language::parse(tag) |> ignore
true
} catch {
_ => false
}
}
///|
/// Well-formed *and* valid: a lowercase 2-3 letter primary subtag, no repeated
/// variant, no repeated extension singleton.
pub fn Language::parse(tag : String) -> Language raise SyntaxError {
fn bad(reason : String) -> SyntaxError {
SyntaxError(kind=Language, input=tag, reason~)
}
guard scan_language(tag) is Some(form) else {
raise bad("language is not a well-formed BCP 47 tag")
}
match form {
// Both are exempt from the primary-subtag rule: a grandfathered tag does
// not have a conforming one, and a private-use tag has no language at all.
Grandfathered | PrivateUseOnly => ()
Langtag(primary~, variants~, singletons~) => {
guard primary.length() >= 2 &&
primary.length() <= 3 &&
all_chars(primary, is_ascii_lower) else {
raise bad(
"language primary subtag must be 2-3 lower-case letters (RFC 5646 2.1.1)",
)
}
guard !has_case_insensitive_duplicate(variants) else {
raise bad("language has a repeated variant subtag")
}
guard !has_case_insensitive_duplicate(singletons) else {
raise bad("language has a repeated extension singleton subtag")
}
}
}
Language(tag)
}
///|
/// The primary language subtag -- `en` in `en-GB-boont`. `None` for a
/// private-use-only tag, which names no language.
pub fn Language::primary(self : Self) -> String? {
match scan_language(self.0) {
Some(Langtag(primary~, ..)) => Some(primary)
// A grandfathered tag's first subtag is its closest thing to a primary.
Some(Grandfathered) =>
match self.0.split("-").collect() {
[first, ..] => Some(first.to_owned())
_ => None
}
_ => None
}
}
///|
priv enum LanguageForm {
Grandfathered
PrivateUseOnly
Langtag(
primary~ : String,
variants~ : Array[String],
singletons~ : Array[String]
)
}
///|
/// The RFC 5646 grammar, as a left-to-right walk over the `-`-separated
/// subtags. Each step consumes what it can and the next step sees the rest; a
/// tag is well-formed exactly when every subtag is consumed.
///
/// langtag = language ["-" script] ["-" region] *("-" variant)
/// *("-" extension) ["-" privateuse]
/// language = 2*3ALPHA ["-" extlang] / 4ALPHA / 5*8ALPHA
/// extlang = 3ALPHA *2("-" 3ALPHA)
/// script = 4ALPHA
/// region = 2ALPHA / 3DIGIT
/// variant = 5*8alphanum / (DIGIT 3alphanum)
/// extension = singleton 1*("-" 2*8alphanum)
/// privateuse = "x" 1*("-" 1*8alphanum)
fn scan_language(tag : String) -> LanguageForm? {
guard tag.length() > 0 else { return None }
let subtags = []
for part in tag.split("-") {
// An empty subtag means a leading, trailing or doubled `-`.
guard part.length() > 0 else { return None }
subtags.push(part.to_owned())
}
let lowered = tag.to_lower()
for candidate in grandfathered_tags {
if lowered == candidate {
return Some(Grandfathered)
}
}
if is_singleton_x(subtags[0]) {
return if scan_private_use(subtags, 0) == subtags.length() {
Some(PrivateUseOnly)
} else {
None
}
}
let primary = subtags[0]
guard all_chars(primary, is_ascii_alpha) else { return None }
guard primary.length() >= 2 && primary.length() <= 8 else { return None }
let mut i = 1
// extlang: only after a 2-3 letter primary, at most three, each exactly three
// letters. Unambiguous against script (4) and numeric region (3 digits).
if primary.length() <= 3 {
let mut taken = 0
while i < subtags.length() &&
taken < 3 &&
subtags[i].length() == 3 &&
all_chars(subtags[i], is_ascii_alpha) {
i = i + 1
taken = taken + 1
}
}
if i < subtags.length() &&
subtags[i].length() == 4 &&
all_chars(subtags[i], is_ascii_alpha) {
i = i + 1 // script
}
if i < subtags.length() && is_region(subtags[i]) {
i = i + 1
}
let variants = []
while i < subtags.length() && is_variant(subtags[i]) {
variants.push(subtags[i])
i = i + 1
}
let singletons = []
while i < subtags.length() && is_singleton(subtags[i]) {
singletons.push(subtags[i])
i = i + 1
// A singleton must be followed by at least one 2-8 character subtag.
let start = i
while i < subtags.length() &&
subtags[i].length() >= 2 &&
subtags[i].length() <= 8 &&
all_chars(subtags[i], is_ascii_alnum) {
i = i + 1
}
guard i > start else { return None }
}
if i < subtags.length() && is_singleton_x(subtags[i]) {
i = scan_private_use(subtags, i)
}
guard i == subtags.length() else { return None }
Some(Langtag(primary~, variants~, singletons~))
}
///|
/// Consumes `x` and its 1-8 character subtags, returning the new index. Returns
/// the starting index unchanged when there is nothing after the `x`, which the
/// callers read as failure.
fn scan_private_use(subtags : Array[String], start : Int) -> Int {
let mut i = start + 1
let first = i
while i < subtags.length() &&
subtags[i].length() >= 1 &&
subtags[i].length() <= 8 &&
all_chars(subtags[i], is_ascii_alnum) {
i = i + 1
}
if i == first {
start
} else {
i
}
}
///|
/// `2ALPHA / 3DIGIT`.
fn is_region(subtag : String) -> Bool {
(subtag.length() == 2 && all_chars(subtag, is_ascii_alpha)) ||
(subtag.length() == 3 && all_chars(subtag, is_ascii_digit))
}
///|
/// `5*8alphanum / (DIGIT 3alphanum)`.
fn is_variant(subtag : String) -> Bool {
if subtag.length() >= 5 && subtag.length() <= 8 {
all_chars(subtag, is_ascii_alnum)
} else if subtag.length() == 4 {
code_unit_is_ascii_digit(subtag[0]) && all_chars(subtag, is_ascii_alnum)
} else {
false
}
}
///|
/// One alphanumeric character, other than `x`, which starts the private-use
/// section rather than an extension.
fn is_singleton(subtag : String) -> Bool {
subtag.length() == 1 &&
all_chars(subtag, is_ascii_alnum) &&
!is_singleton_x(subtag)
}
///|
fn is_singleton_x(subtag : String) -> Bool {
subtag == "x" || subtag == "X"
}
///|
/// RFC 5646 compares subtags case-insensitively, so `rozaj` and `ROZAJ` are one
/// variant repeated -- which is a case the corpus makes a point of.
fn has_case_insensitive_duplicate(subtags : Array[String]) -> Bool {
let seen = []
for subtag in subtags {
let lowered = subtag.to_lower()
for previous in seen {
if previous == lowered {
return true
}
}
seen.push(lowered)
}
false
}