///|
/// HTML5 Tokenizer
/// Based on WHATWG HTML5 Parsing Specification
/// https://html.spec.whatwg.org/multipage/parsing.html

///|
/// HTML Token types
pub(all) enum Token {
  Doctype(String)
  StartTag(String, Map[String, String], Bool) // tag, attrs, self_closing
  EndTag(String)
  Character(Char)
  Characters(String) // Batch of consecutive text characters (optimization)
  Comment(String)
  EOF
}

///|
/// Tokenizer state
pub(all) struct Tokenizer {
  input : String
  mut pos : Int
  len : Int
  // When set, tokenizer is in raw text mode and only emits text until
  // the matching end tag (e.g., "") is found.
  mut raw_text_end_tag : String?
  // Deferred end tag to emit after raw text content
  mut pending_end_tag : String?
}

///|
pub fn Tokenizer::new(input : String) -> Tokenizer {
  {
    input,
    pos: 0,
    len: input.length(),
    raw_text_end_tag: None,
    pending_end_tag: None,
  }
}

///|
/// Enter raw text mode: all content until  is emitted as text.
pub fn Tokenizer::enter_raw_text_mode(self : Tokenizer, tag : String) -> Unit {
  self.raw_text_end_tag = Some(tag.to_lower())
}

///|
/// Consume raw text content until matching  is found.
/// Returns Characters token with accumulated text, or EndTag if no text.
/// When text is found, the end tag position is saved and will be emitted
/// on the next call (raw_text_end_tag is cleared but pending_end_tag is set).
fn Tokenizer::consume_raw_text_token(
  self : Tokenizer,
  end_tag : String,
) -> Token {
  // If we already have a pending end tag from previous call, emit it now
  match self.pending_end_tag {
    Some(tag) => {
      self.pending_end_tag = None
      return Token::EndTag(tag)
    }
    None => ()
  }
  let buf = StringBuilder::new()
  while !self.is_eof() {
    // Check for closing tag
    if self.peek() is Some('<') && self.peek_at(1) is Some('/') {
      let saved_pos = self.pos
      let _ = self.consume() // <
      let _ = self.consume() // /
      self.skip_whitespace()
      let name = self.consume_name().to_lower()
      self.skip_whitespace()
      if name == end_tag {
        // Found end tag — consume '>' and exit raw text mode
        if self.peek() is Some('>') {
          let _ = self.consume()
        }
        self.raw_text_end_tag = None
        let text = buf.to_string()
        if text.length() > 0 {
          // Return text first, save end tag for next call
          self.pending_end_tag = Some(end_tag)
          return Token::Characters(text)
        }
        return Token::EndTag(end_tag)
      }
      // Not our end tag — restore position and add '<' to buffer
      self.pos = saved_pos
    }
    match self.consume() {
      Some(c) => buf.write_char(c)
      None => break
    }
  }
  self.raw_text_end_tag = None
  let text = buf.to_string()
  if text.length() > 0 {
    Token::Characters(text)
  } else {
    Token::EOF
  }
}

///|
fn Tokenizer::is_eof(self : Tokenizer) -> Bool {
  self.pos >= self.len
}

///|
fn Tokenizer::peek(self : Tokenizer) -> Char? {
  if self.pos >= self.len {
    None
  } else {
    Some(self.input[self.pos].to_int().unsafe_to_char())
  }
}

///|
/// Fast peek without Option wrapping - returns '\x00' for EOF
fn Tokenizer::peek_fast(self : Tokenizer) -> Char {
  if self.pos >= self.len {
    '\u0000'
  } else {
    self.input[self.pos].to_int().unsafe_to_char()
  }
}

///|
fn Tokenizer::peek_at(self : Tokenizer, offset : Int) -> Char? {
  let idx = self.pos + offset
  if idx >= self.len || idx < 0 {
    None
  } else {
    Some(self.input[idx].to_int().unsafe_to_char())
  }
}

///|
fn Tokenizer::consume(self : Tokenizer) -> Char? {
  if self.pos >= self.len {
    None
  } else {
    let c = self.input[self.pos].to_int().unsafe_to_char()
    self.pos += 1
    Some(c)
  }
}

///|
/// Fast consume - advances position and returns the character (caller must check bounds)
fn Tokenizer::advance(self : Tokenizer) -> Unit {
  self.pos += 1
}

///|
fn tok_is_whitespace(c : Char) -> Bool {
  c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\u000C'
}

///|
fn is_ascii_alpha(c : Char) -> Bool {
  (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

///|
fn tok_is_name_char(c : Char) -> Bool {
  is_ascii_alpha(c) ||
  (c >= '0' && c <= '9') ||
  c == '-' ||
  c == '_' ||
  c == ':'
}

///|
fn is_unquoted_attr_end(c : Char) -> Bool {
  tok_is_whitespace(c) ||
  c == '>' ||
  c == '=' ||
  c == '"' ||
  c == '\'' ||
  c == '<'
}

///|
fn Tokenizer::skip_whitespace(self : Tokenizer) -> Unit {
  while self.pos < self.len {
    let c = self.peek_fast()
    if tok_is_whitespace(c) {
      self.advance()
    } else {
      break
    }
  }
}

///|
/// Check for a case-insensitive match at current position
fn Tokenizer::matches_ci(self : Tokenizer, s : String) -> Bool {
  let slen = s.length()
  if self.pos + slen > self.len {
    return false
  }
  for i = 0; i < slen; i = i + 1 {
    let c1 = self.input[self.pos + i].to_int()
    let c2 = s[i].to_int()
    // Fast lowercase comparison using byte arithmetic
    let c1_lower = if c1 >= 65 && c1 <= 90 { c1 + 32 } else { c1 }
    let c2_lower = if c2 >= 65 && c2 <= 90 { c2 + 32 } else { c2 }
    if c1_lower != c2_lower {
      return false
    }
  }
  true
}

///|
/// Consume characters matching a name (tag name, attribute name)
fn Tokenizer::consume_name(self : Tokenizer) -> String {
  let start = self.pos
  while self.pos < self.len {
    let c = self.peek_fast()
    if tok_is_name_char(c) {
      self.advance()
    } else {
      break
    }
  }
  self.input.unsafe_substring(start~, end=self.pos)
}

///|
/// Consume name and convert to lowercase (for tag names)
fn Tokenizer::consume_name_lower(self : Tokenizer) -> String {
  let start = self.pos
  while self.pos < self.len {
    let c = self.peek_fast()
    if tok_is_name_char(c) {
      self.advance()
    } else {
      break
    }
  }
  self.input.unsafe_substring(start~, end=self.pos).to_lower()
}

///|
/// Consume a quoted or unquoted attribute value
fn Tokenizer::consume_attribute_value(self : Tokenizer) -> String {
  self.skip_whitespace()
  if self.pos >= self.len {
    return ""
  }
  let c = self.peek_fast()
  if c == '"' {
    self.advance()
    let start = self.pos
    while self.pos < self.len {
      if self.peek_fast() == '"' {
        let end = self.pos
        self.advance()
        return decode_html_entities(self.input.unsafe_substring(start~, end~))
      }
      self.advance()
    }
    decode_html_entities(self.input.unsafe_substring(start~, end=self.pos))
  } else if c == '\'' {
    self.advance()
    let start = self.pos
    while self.pos < self.len {
      if self.peek_fast() == '\'' {
        let end = self.pos
        self.advance()
        return decode_html_entities(self.input.unsafe_substring(start~, end~))
      }
      self.advance()
    }
    decode_html_entities(self.input.unsafe_substring(start~, end=self.pos))
  } else {
    // Unquoted attribute value
    let start = self.pos
    while self.pos < self.len {
      let ch = self.peek_fast()
      if is_unquoted_attr_end(ch) {
        break
      }
      self.advance()
    }
    decode_html_entities(self.input.unsafe_substring(start~, end=self.pos))
  }
}

///|
/// Skip DOCTYPE declaration
fn Tokenizer::skip_doctype(self : Tokenizer) -> Token? {
  if self.matches_ci("
    while self.pos < self.len && self.peek_fast() != '>' {
      self.advance()
    }
    if self.pos < self.len && self.peek_fast() == '>' {
      self.advance()
    }
    Some(Token::Doctype("html"))
  } else {
    None
  }
}

///|
/// Skip bogus markup declaration  as a comment token.
/// This matches HTML "bogus comment" handling for invalid doctypes like .
fn Tokenizer::skip_bogus_markup_declaration(self : Tokenizer) -> Token? {
  if self.peek_at(0) != Some('<') || self.peek_at(1) != Some('!') {
    return None
  }
  self.pos += 2 // Skip "' {
    self.advance()
  }
  let content = self.input.unsafe_substring(start~, end=self.pos)
  if self.pos < self.len && self.peek_fast() == '>' {
    self.advance()
  }
  Some(Token::Comment(content))
}

///|
/// Skip comment 
/// Handles HTML5 abrupt closing of empty comment ( and )
fn Tokenizer::skip_comment(self : Tokenizer) -> Token? {
  // Check for " or )
  if self.pos < self.len && self.peek_fast() == '>' {
    self.advance()
    return Some(Token::Comment(""))
  }
  if self.pos + 1 < self.len &&
    self.peek_fast() == '-' &&
    self.input[self.pos + 1].to_int().unsafe_to_char() == '>' {
    self.pos += 2
    return Some(Token::Comment(""))
  }
  // Find end of comment "-->"
  let start = self.pos
  while self.pos + 2 < self.len {
    if self.peek_fast() == '-' &&
      self.input[self.pos + 1].to_int().unsafe_to_char() == '-' &&
      self.input[self.pos + 2].to_int().unsafe_to_char() == '>' {
      let content = self.input.unsafe_substring(start~, end=self.pos)
      self.pos += 3 // Skip "-->"
      return Some(Token::Comment(content))
    }
    self.advance()
  }
  // No closing found, return rest as comment
  let content = self.input.unsafe_substring(start~, end=self.len)
  self.pos = self.len
  Some(Token::Comment(content))
}

///|
/// Parse a start tag
fn Tokenizer::parse_start_tag(self : Tokenizer) -> Token {
  let tag = self.consume_name_lower()
  let attrs : Map[String, String] = {}
  let mut self_closing = false
  // Parse attributes
  while self.pos < self.len {
    self.skip_whitespace()
    if self.pos >= self.len {
      break
    }
    let c = self.peek_fast()
    if c == '>' {
      self.advance()
      break
    } else if c == '/' {
      self.advance()
      self.skip_whitespace()
      if self.pos < self.len && self.peek_fast() == '>' {
        self.advance()
        self_closing = true
      }
      break
    } else if tok_is_name_char(c) {
      let attr_name = self.consume_name()
      self.skip_whitespace()
      let attr_value = if self.pos < self.len && self.peek_fast() == '=' {
        self.advance()
        self.consume_attribute_value()
      } else {
        ""
      }
      attrs.set(attr_name, attr_value)
    } else {
      // Skip unknown character
      self.advance()
    }
  }
  Token::StartTag(tag, attrs, self_closing)
}

///|
/// Parse an end tag
fn Tokenizer::parse_end_tag(self : Tokenizer) -> Token {
  let tag = self.consume_name_lower()
  self.skip_whitespace()
  if self.pos < self.len && self.peek_fast() == '>' {
    self.advance()
  }
  Token::EndTag(tag)
}

///|
/// Get the next token
pub fn Tokenizer::next_token(self : Tokenizer) -> Token {
  // Check for pending end tag from raw text mode
  match self.pending_end_tag {
    Some(tag) => {
      self.pending_end_tag = None
      return Token::EndTag(tag)
    }
    None => ()
  }
  if self.is_eof() {
    return Token::EOF
  }
  // Raw text mode: emit text until matching end tag
  match self.raw_text_end_tag {
    Some(end_tag) => return self.consume_raw_text_token(end_tag)
    None => ()
  }
  match self.peek() {
    Some('<') =>
      // Check for DOCTYPE
      if self.matches_ci(" t
          None => {
            let _ = self.consume()
            Token::Character('<')
          }
        }
      } else if self.peek_at(1) is Some('!') &&
        self.peek_at(2) is Some('-') &&
        self.peek_at(3) is Some('-') {
        // Comment
        match self.skip_comment() {
          Some(t) => t
          None => {
            let _ = self.consume()
            Token::Character('<')
          }
        }
      } else if self.peek_at(1) is Some('!') {
        // Bogus declaration (e.g. ) is parsed as comment.
        match self.skip_bogus_markup_declaration() {
          Some(t) => t
          None => {
            let _ = self.consume()
            Token::Character('<')
          }
        }
      } else if self.peek_at(1) is Some('/') {
        // End tag
        let _ = self.consume() // <
        let _ = self.consume() // /
        self.skip_whitespace()
        self.parse_end_tag()
      } else {
        match self.peek_at(1) {
          Some(c) if is_ascii_alpha(c) => {
            // Start tag
            let _ = self.consume() // <
            self.parse_start_tag()
          }
          _ =>
            // Not a valid tag, emit as character
            match self.consume() {
              Some(c) => Token::Character(c)
              None => Token::EOF
            }
        }
      }
    Some(_) => {
      // Collect consecutive text characters as a single token
      let start = self.pos
      while self.pos < self.len {
        let c = self.peek_fast()
        if c == '<' {
          break
        }
        self.advance()
      }
      let text = self.input.unsafe_substring(start~, end=self.pos)
      if text.length() == 1 {
        // Single character - use Character token for compatibility
        Token::Character(text[0].to_int().unsafe_to_char())
      } else {
        Token::Characters(text)
      }
    }
    None => Token::EOF
  }
}

///|
/// Collect all tokens (for debugging/testing)
pub fn Tokenizer::collect_all(self : Tokenizer) -> Array[Token] {
  let tokens : Array[Token] = []
  while true {
    let token = self.next_token()
    tokens.push(token)
    match token {
      Token::EOF => break
      _ => continue
    }
  }
  tokens
}