///|
pub(all) enum TokenKind {
  Word(String)
  Tag(String)
  Text(String)
  Number(Int)
  Symbol(Char)
  End
} derive(Eq, Debug)

///|
pub(all) struct Token {
  kind : TokenKind
  span : Span
} derive(Eq, Debug)

///|
priv struct Scanner {
  chars : Array[Char]
  mut pos : Int
  mut line : Int
  mut column : Int
  limits : Limits
}

///|
fn Scanner::peek(self : Scanner, offset : Int) -> Char? {
  let at = self.pos + offset
  if at >= self.chars.length() {
    None
  } else {
    Some(self.chars[at])
  }
}

///|
fn Scanner::take(self : Scanner) -> Char {
  let c = self.chars[self.pos]
  self.pos += 1
  if c == '\n' {
    self.line += 1
    self.column = 1
  } else {
    self.column += 1
  }
  c
}

///|
fn Scanner::mark(self : Scanner) -> Span {
  { start: self.pos, end: self.pos, line: self.line, column: self.column }
}

///|
fn Scanner::finish(self : Scanner, start : Span, kind : TokenKind) -> Token {
  { kind, span: { ..start, end: self.pos } }
}

///|
fn name_start(c : Char) -> Bool {
  c.is_ascii_alphabetic() || c == '_'
}

///|
fn name_char(c : Char) -> Bool {
  name_start(c) || c.is_ascii_digit()
}

///|
fn Scanner::skip(self : Scanner) -> Unit raise SieveError {
  while self.pos < self.chars.length() {
    match self.peek(0) {
      Some(' ' | '\t' | '\r' | '\n') => ignore(self.take())
      Some('#') =>
        while self.pos < self.chars.length() && self.peek(0) != Some('\n') {
          ignore(self.take())
        }
      Some('/') if self.peek(1) == Some('*') => {
        let start = self.mark()
        ignore(self.take())
        ignore(self.take())
        let mut closed = false
        while self.pos < self.chars.length() {
          if self.peek(0) == Some('*') && self.peek(1) == Some('/') {
            ignore(self.take())
            ignore(self.take())
            closed = true
            break
          }
          ignore(self.take())
        }
        if !closed {
          fail("lex.comment", "unterminated block comment", start)
        }
      }
      _ => break
    }
  }
}

///|
fn Scanner::quoted(self : Scanner, start : Span) -> String raise SieveError {
  ignore(self.take())
  let b = StringBuilder()
  let mut count = 0
  while self.pos < self.chars.length() {
    let c = self.take()
    if c == '"' {
      return b.to_string()
    }
    if c == '\u{0}' {
      fail("lex.nul", "NUL is not permitted in a string", start)
    }
    if c == '\\' {
      if self.pos == self.chars.length() {
        fail("lex.escape", "incomplete quoted escape", start)
      }
      let next = self.take()
      if next == '\u{0}' {
        fail("lex.nul", "NUL is not permitted in a string", start)
      }
      count += next.utf16_len()
      b.write_char(next)
    } else if c == '\r' {
      if self.peek(0) != Some('\n') {
        fail("lex.newline", "bare CR in quoted string", start)
      }
      ignore(self.take())
      count += 2
      b.write_string("\r\n")
    } else if c == '\n' {
      count += 2
      b.write_string("\r\n")
    } else {
      count += c.utf16_len()
      b.write_char(c)
    }
    if count > self.limits.string_chars {
      fail("limit.string", "string exceeds configured limit", start)
    }
  }
  fail("lex.string", "unterminated quoted string", start)
  ""
}

///|
fn Scanner::multiline(self : Scanner, start : Span) -> String raise SieveError {
  ignore(self.take())
  while self.peek(0) == Some(' ') || self.peek(0) == Some('\t') {
    ignore(self.take())
  }
  if self.peek(0) == Some('#') {
    while self.pos < self.chars.length() &&
          self.peek(0) != Some('\r') &&
          self.peek(0) != Some('\n') {
      ignore(self.take())
    }
  }
  if self.peek(0) == Some('\r') {
    ignore(self.take())
  }
  if self.peek(0) != Some('\n') {
    fail("lex.text", "text: must be followed by a line break", start)
  }
  ignore(self.take())
  let b = StringBuilder()
  let mut count = 0
  while self.pos < self.chars.length() {
    let line_start = self.pos
    while self.pos < self.chars.length() &&
          self.peek(0) != Some('\n') &&
          self.peek(0) != Some('\r') {
      if self.peek(0) == Some('\u{0}') {
        fail("lex.nul", "NUL is not permitted", self.mark())
      }
      ignore(self.take())
    }
    let end = self.pos
    if self.peek(0) == Some('\r') {
      ignore(self.take())
    }
    if self.peek(0) != Some('\n') {
      fail("lex.text", "multiline text needs a terminating dot line", start)
    }
    ignore(self.take())
    if end == line_start + 1 && self.chars[line_start] == '.' {
      return b.to_string()
    }
    let from = if end > line_start && self.chars[line_start] == '.' {
      line_start + 1
    } else {
      line_start
    }
    let line = chars_text(self.chars, from, end)
    count += line.length() + 2
    b.write_string(line)
    b.write_string("\r\n")
    if count > self.limits.string_chars {
      fail("limit.string", "multiline text exceeds limit", start)
    }
  }
  fail("lex.text", "missing multiline dot terminator", start)
  ""
}

///|
fn Scanner::number(self : Scanner, start : Span) -> Int raise SieveError {
  let mut n = 0
  while self.pos < self.chars.length() && self.chars[self.pos].is_ascii_digit() {
    let d = self.take().to_int() - 48
    if n > (2147483647 - d) / 10 {
      fail("lex.number", "number exceeds signed 31-bit range", start)
    }
    n = n * 10 + d
  }
  let factor = match self.peek(0) {
    Some('k' | 'K') => 1024
    Some('m' | 'M') => 1048576
    Some('g' | 'G') => 1073741824
    _ => 1
  }
  if factor != 1 {
    ignore(self.take())
  }
  if n > 2147483647 / factor {
    fail("lex.number", "quantity exceeds signed 31-bit range", start)
  }
  match self.peek(0) {
    Some(c) if name_char(c) =>
      fail("lex.number", "invalid numeric suffix", start)
    _ => ()
  }
  n * factor
}

///|
pub fn tokenize(
  source : String,
  limits? : Limits = Limits::default(),
) -> Array[Token] raise SieveError {
  limits.check()
  if source.length() > limits.source_chars {
    fail("limit.source", "source exceeds configured UTF-16 limit", origin())
  }
  let s : Scanner = {
    chars: source.to_array(),
    pos: 0,
    line: 1,
    column: 1,
    limits,
  }
  let out : Array[Token] = []
  while true {
    s.skip()
    let start = s.mark()
    if s.pos == s.chars.length() {
      out.push(s.finish(start, End))
      break
    }
    if out.length() >= limits.tokens {
      fail("limit.tokens", "token limit exceeded", start)
    }
    let c = s.chars[s.pos]
    let kind = if c == '"' {
      Text(s.quoted(start))
    } else if c == ':' {
      ignore(s.take())
      match s.peek(0) {
        Some(first) if name_start(first) => ()
        _ => fail("lex.tag", "expected tag name after colon", start)
      }
      let begin = s.pos
      while s.pos < s.chars.length() && name_char(s.chars[s.pos]) {
        ignore(s.take())
      }
      Tag(ascii_lower(chars_text(s.chars, begin, s.pos)))
    } else if name_start(c) {
      let begin = s.pos
      while s.pos < s.chars.length() && name_char(s.chars[s.pos]) {
        ignore(s.take())
      }
      let word = ascii_lower(chars_text(s.chars, begin, s.pos))
      if word == "text" && s.peek(0) == Some(':') {
        Text(s.multiline(start))
      } else {
        Word(word)
      }
    } else if c.is_ascii_digit() {
      Number(s.number(start))
    } else if "{}[](),;".contains_char(c) {
      ignore(s.take())
      Symbol(c)
    } else {
      fail("lex.character", "unexpected character", start)
      End
    }
    out.push(s.finish(start, kind))
  }
  out
}