///|
const EMU_TAG = re"]*)>([^<]*)"

///|
const PRE_BLOCK = re"
(.*?)
"

///|
/// Fetch the spec HTML and extract grammar terminals.
async fn fetch_and_extract() -> (Array[String], Array[String]) {
  let (_response, body) = @http.get("https://webidl.spec.whatwg.org/")
  let html = body.text()
  extract_terminals(html)
}

///|
fn escape_str(s : String) -> String {
  let mut out = ""
  for c in s.iter() {
    if c == '"' {
      out = out + "\\\""
    } else if c == '\\' {
      out = out + "\\\\"
    } else {
      out = out + c.to_string()
    }
  }
  out
}

///|
fn sort_by_length_desc(arr : Array[String]) -> Unit {
  arr.sort_by((a, b) => {
    let length_cmp = b.length().compare(a.length())
    if length_cmp == 0 {
      a.compare(b)
    } else {
      length_cmp
    }
  })
}

///|
fn generate_output(kw : Array[String], punc : Array[String]) -> String {
  fn build_lines(arr : Array[String]) -> String {
    match arr {
      [] => ""
      _ => arr.map(item => "  \"" + escape_str(item) + "\"").join(",\n") + "\n"
    }
  }
  let kw_body = build_lines(kw)
  let punc_body = build_lines(punc)
  let head =
    #|/// Auto-generated by `cmd/build`. Do not edit manually.
    #|/// Literal terminals from the WebIDL grammar.
    #|/// Source: https://webidl.spec.whatwg.org/#idl-grammar
    #|
    #|///|
    #|/// Grammar keyword literals, sorted by length descending.
    #|let keywords : Array[String] = [

  let mid =
    #|]
    #|
    #|///|
    #|/// Grammar punctuation literals, sorted by length descending.
    #|let punctuation : Array[String] = [
  let tail =
    #|]
    #|
    #|///|
    #|/// Returns whether `text` is a grammar keyword literal.
    #|fn is_keyword(text : StringView) -> Bool {
    #|  for kw in keywords {
    #|    if text == kw[:] {
    #|      return true
    #|    }
    #|  }
    #|  false
    #|}
    #|
    #|///|
    #|/// Try to match a grammar punctuation literal at the start of `text`.
    #|fn match_punctuation(text : StringView) -> Int {
    #|  for p in punctuation {
    #|    if text.has_prefix(p[:]) {
    #|      return p.length()
    #|    }
    #|  }
    #|  0
    #|}
  head + kw_body + mid + punc_body + tail
}

///|
async fn main {
  println("Fetching the WebIDL specification; this may take a moment...")
  let (raw_kw, raw_punc) = fetch_and_extract()
  sort_by_length_desc(raw_kw)
  sort_by_length_desc(raw_punc)
  println("Writing lexer/literals.mbt...")
  @fs.write_file("lexer/literals.mbt", generate_output(raw_kw, raw_punc))
  println("Generated lexer/literals.mbt.")
}

///|
/// Grammar terminals in the spec are inside 
 blocks
/// as  tags. We use regex for both the pre blocks and emu-t tags.
fn extract_terminals(html : String) -> (Array[String], Array[String]) {
  let kw_set = @set.Set([])
  let punc_set = @set.Set([])
  let mut rem : StringView = html
  while true {
    match PRE_BLOCK.execute(rem) {
      None => break
      Some(m) => {
        if m.group(1) is Some(pre_body) {
          let mut inner : StringView = pre_body
          while true {
            match EMU_TAG.execute(inner) {
              None => break
              Some(m2) => {
                let is_regex = m2.group(1) is Some(attr) &&
                  attr.contains("regex")
                if !is_regex {
                  if m2.group(2) is Some(text) {
                    let decoded = if text == "<" {
                      "<"
                    } else if text == ">" {
                      ">"
                    } else {
                      text.to_owned()
                    }
                    if decoded.length() == 1 {
                      let c = decoded[0].to_int()
                      let is_punc = !((c >= 48 && c <= 57) ||
                        (c >= 65 && c <= 90) ||
                        (c >= 97 && c <= 122) ||
                        c == 95)
                      if is_punc {
                        punc_set.add(decoded)
                      } else {
                        kw_set.add(decoded)
                      }
                    } else if decoded.length() > 1 {
                      if decoded == "..." {
                        punc_set.add(decoded)
                      } else {
                        kw_set.add(decoded)
                      }
                    }
                  }
                }
                inner = m2.after()
              }
            }
          }
        }
        rem = m.after()
      }
    }
  }
  (kw_set.to_array(), punc_set.to_array())
}