// Port of tree-sitter lib/src/lexer.h and lib/src/lexer.c.

///|
priv struct ColumnData {
  mut value : UInt32
  mut valid : Bool
}

///|
let byte_order_mark : Int32 = 0xFEFF

///|
let default_range : Range = {
  start_point: { row: 0, column: 0, },
  end_point: { row: 0xFFFFFFFFU, column: 0xFFFFFFFFU, },
  start_byte: 0,
  end_byte: 0xFFFFFFFFU,
}

///|
/// The lexer state. Its public methods form the `TSLexer` interface used by
/// lex functions and external scanners.
pub(all) struct Lexer {
  /// `TSLexer.lookahead`.
  mut lookahead : Int32
  /// `TSLexer.result_symbol`.
  mut result_symbol : Int32
  priv mut current_position : Length
  priv mut token_start_position : Length
  priv mut token_end_position : Length
  priv mut included_ranges : Array[Range]
  /// The current chunk, meaningful only when `has_chunk` (C: `chunk` is
  /// non-NULL).
  priv mut chunk : BytesView
  priv mut has_chunk : Bool
  priv mut input : Input
  priv mut logger : Logger?
  priv mut current_included_range_index : Int32
  priv mut chunk_start : UInt32
  priv mut chunk_size : UInt32
  priv mut lookahead_size : UInt32
  priv mut did_get_column : Bool
  priv column_data : ColumnData
  /// `debug_buffer`: every log message is formatted into it, and the parser
  /// also passes it to the external scanner's `serialize`.
  priv debug_buffer : FixedArray[Byte]
}

///|
/// The effect of `snprintf(self->debug_buffer, 1024, ...)` producing
/// `message`: its UTF-8 bytes, truncated to 1023, followed by a NUL.
///
/// Returns the message as the Rust binding's logger callback receives it:
/// the C string in the buffer (up to its first NUL), or `None` when that is
/// not valid UTF-8 (a multi-byte character cut by the truncation), in which
/// case `CStr::to_str` fails and the binding drops the message.
fn Lexer::write_debug_buffer(self : Lexer, message : String) -> String? {
  let bytes = @utf8.encode(message)
  let n = if bytes.length() < self.debug_buffer.length() {
    bytes.length()
  } else {
    self.debug_buffer.length() - 1
  }
  for i in 0.. None
  } noraise {
    message => Some(message)
  }
}

///|
/// The length of the C string in the debug buffer.
fn Lexer::debug_message_length(self : Lexer) -> Int32 {
  for i in 0.. Bytes {
  let buffer = self.debug_buffer
  Bytes::makei(self.debug_message_length(), i => buffer[i])
}

///|
fn Lexer::log_char(self : Lexer, message : String, character : Int32) -> Unit {
  if self.logger is Some(log) {
    let text = if 32 <= character && character < 127 {
      "\{message} character:'\{Int::unsafe_to_char(character)}'"
    } else {
      "\{message} character:\{character}"
    }
    if self.write_debug_buffer(text) is Some(text) {
      log(Lex, text)
    }
  }
}

///|
/// Sets the column data to the given value and marks it valid.
fn Lexer::set_column_data(self : Lexer, val : UInt32) -> Unit {
  self.column_data.valid = true
  self.column_data.value = val
}

///|
/// Increments the value of the column data; no-op if invalid.
fn Lexer::increment_column_data(self : Lexer) -> Unit {
  if self.column_data.valid {
    self.column_data.value += 1
  }
}

///|
/// Marks the column data as invalid.
fn Lexer::invalidate_column_data(self : Lexer) -> Unit {
  self.column_data.valid = false
  self.column_data.value = 0
}

///|
/// `ts_lexer__eof`: the lexer has consumed all of its available ranges.
pub fn Lexer::eof(self : Lexer) -> Bool {
  self.current_included_range_index == self.included_ranges.length()
}

///|
/// Clear the currently stored chunk of source code, because the lexer's
/// position has changed.
fn Lexer::clear_chunk(self : Lexer) -> Unit {
  self.chunk = b""[:]
  self.has_chunk = false
  self.chunk_size = 0
  self.chunk_start = 0
}

///|
/// Call the lexer's input callback to obtain a new chunk of source code for
/// the current position.
fn Lexer::get_chunk(self : Lexer) -> Unit {
  self.chunk_start = self.current_position.bytes
  let chunk = (self.input.read)(
    self.current_position.bytes,
    self.current_position.extent,
  )
  self.chunk = chunk
  self.has_chunk = true
  self.chunk_size = chunk.length().reinterpret_as_uint()
  if self.chunk_size == 0 {
    self.current_included_range_index = self.included_ranges.length()
    self.chunk = b""[:]
    self.has_chunk = false
  }
}

///|
fn Lexer::decode(self : Lexer, chunk : BytesView) -> (UInt32, Int32) {
  match self.input.encoding {
    UTF8 => decode_utf8(chunk)
    UTF16LE => decode_utf16_le(chunk)
    UTF16BE => decode_utf16_be(chunk)
    Custom =>
      match self.input.decode {
        Some(decode) => decode(chunk)
        None => decode_utf8(chunk)
      }
  }
}

///|
/// Decode the next unicode character in the current chunk of source code.
/// This assumes that the lexer has already retrieved a chunk of source code
/// that spans the current position.
fn Lexer::get_lookahead(self : Lexer) -> Unit {
  let position_in_chunk = self.current_position.bytes - self.chunk_start
  let size = self.chunk_size - position_in_chunk
  if size == 0 {
    self.lookahead_size = 1
    self.lookahead = 0
    return
  }
  if !self.has_chunk {
    abort("no chunk")
  }
  let data = self.chunk
  let chunk = data[position_in_chunk.reinterpret_as_int():self.chunk_size.reinterpret_as_int()]
  if self.input.encoding is UTF8 && chunk[0] < 0x80 {
    self.lookahead = chunk[0].to_int()
    self.lookahead_size = 1
    return
  }
  let (lookahead_size, lookahead) = self.decode(chunk)
  self.lookahead_size = lookahead_size
  self.lookahead = lookahead
  // If this chunk ended in the middle of a multi-byte character, try again
  // with a fresh chunk.
  if self.lookahead == ts_decode_error && size < 4 {
    self.get_chunk()
    let chunk = if self.has_chunk { self.chunk } else { b""[:] }
    // Upstream decodes even an empty chunk here; the decoders read at least
    // one byte, so an empty chunk is treated as an invalid character.
    let (lookahead_size, lookahead) = if chunk.length() == 0 {
      (0U, ts_decode_error)
    } else {
      self.decode(chunk)
    }
    self.lookahead_size = lookahead_size
    self.lookahead = lookahead
  }
  if self.lookahead == ts_decode_error {
    self.lookahead_size = 1
  }
}

///|
fn Lexer::goto(self : Lexer, position : Length) -> Unit {
  if position.bytes != self.current_position.bytes {
    self.invalidate_column_data()
  }
  self.current_position = position
  // Move to the first valid position at or after the given position.
  let mut found_included_range = false
  for i, included_range in self.included_ranges {
    if included_range.end_byte > self.current_position.bytes &&
      included_range.end_byte > included_range.start_byte {
      if included_range.start_byte >= self.current_position.bytes {
        self.current_position = {
          bytes: included_range.start_byte,
          extent: included_range.start_point,
        }
      }
      self.current_included_range_index = i
      found_included_range = true
      break
    }
  }
  if found_included_range {
    // If the current position is outside of the current chunk of text, then
    // clear out the current chunk of text.
    if self.has_chunk &&
      (
        self.current_position.bytes < self.chunk_start ||
        self.current_position.bytes >= self.chunk_start + self.chunk_size
      ) {
      self.clear_chunk()
    }
    self.lookahead_size = 0
    self.lookahead = 0
  } else {
    // If the given position is beyond any of included ranges, move to the
    // EOF state - past the end of the included ranges.
    self.current_included_range_index = self.included_ranges.length()
    let last_included_range = self.included_ranges[self.included_ranges.length() -
      1]
    self.current_position = {
      bytes: last_included_range.end_byte,
      extent: last_included_range.end_point,
    }
    self.clear_chunk()
    self.lookahead_size = 1
    self.lookahead = 0
  }
}

///|
/// Actually advances the lexer. Does not log anything.
fn Lexer::do_advance(self : Lexer, skip : Bool) -> Unit {
  if self.lookahead_size != 0 {
    if self.lookahead == '\n' {
      self.current_position = {
        ..self.current_position,
        extent: { row: self.current_position.extent.row + 1, column: 0, },
      }
      self.set_column_data(0)
    } else {
      let is_bom = self.current_position.bytes == 0 &&
        self.lookahead == byte_order_mark
      if !is_bom {
        self.increment_column_data()
      }
      self.current_position = {
        ..self.current_position,
        extent: {
          ..self.current_position.extent,
          column: self.current_position.extent.column + self.lookahead_size,
        },
      }
    }
    self.current_position = {
      ..self.current_position,
      bytes: self.current_position.bytes + self.lookahead_size,
    }
  }
  let count = self.included_ranges.length()
  let mut current_range : Range? = Some(
    self.included_ranges[self.current_included_range_index],
  )
  while current_range is Some(range) &&
        (
          self.current_position.bytes >= range.end_byte ||
          range.end_byte == range.start_byte
        ) {
    if self.current_included_range_index < count {
      self.current_included_range_index += 1
    }
    if self.current_included_range_index < count {
      let next = self.included_ranges[self.current_included_range_index]
      current_range = Some(next)
      self.current_position = {
        bytes: next.start_byte,
        extent: next.start_point,
      }
    } else {
      current_range = None
      break
    }
  }
  if skip {
    self.token_start_position = self.current_position
  }
  if current_range is Some(_) {
    if self.current_position.bytes < self.chunk_start ||
      self.current_position.bytes >= self.chunk_start + self.chunk_size {
      self.get_chunk()
    }
    self.get_lookahead()
  } else {
    self.clear_chunk()
    self.lookahead = 0
    self.lookahead_size = 1
  }
}

///|
/// `TSLexer.advance`: advance to the next character in the source code,
/// retrieving a new chunk of source code if needed.
pub fn Lexer::advance(self : Lexer, skip : Bool) -> Unit {
  guard self.has_chunk else { return }
  let chunk = self.chunk
  if self.logger is Some(_) {
    if skip {
      self.log_char("skip", self.lookahead)
    } else {
      self.log_char("consume", self.lookahead)
    }
  }
  let next_position = self.current_position.bytes + 1
  let current_range_end = self.included_ranges[self.current_included_range_index].end_byte
  if self.input.encoding is UTF8 &&
    self.lookahead_size == 1 &&
    self.lookahead != '\n' &&
    next_position < current_range_end &&
    next_position < self.chunk_start + self.chunk_size {
    let next_byte = chunk[(next_position - self.chunk_start).reinterpret_as_int()]
    if next_byte < 0x80 {
      self.increment_column_data()
      self.current_position = {
        bytes: self.current_position.bytes + 1,
        extent: {
          ..self.current_position.extent,
          column: self.current_position.extent.column + 1,
        },
      }
      if skip {
        self.token_start_position = self.current_position
      }
      self.lookahead = next_byte.to_int()
      return
    }
  }
  self.do_advance(skip)
}

///|
/// `TSLexer.mark_end`: mark that a token match has completed. This can be
/// called multiple times if a longer match is found later.
pub fn Lexer::mark_end(self : Lexer) -> Unit {
  if !self.eof() {
    // If the lexer is right at the beginning of included range, then the
    // token should be considered to end at the *end* of the previous included
    // range, rather than here.
    let current_included_range = self.included_ranges[self.current_included_range_index]
    if self.current_included_range_index > 0 &&
      self.current_position.bytes == current_included_range.start_byte {
      let previous_included_range = self.included_ranges[self.current_included_range_index -
        1]
      self.token_end_position = {
        bytes: previous_included_range.end_byte,
        extent: previous_included_range.end_point,
      }
      return
    }
  }
  self.token_end_position = self.current_position
}

///|
/// `TSLexer.get_column`.
pub fn Lexer::get_column(self : Lexer) -> UInt32 {
  self.did_get_column = true
  if !self.column_data.valid {
    // Record current position
    let goal_byte = self.current_position.bytes
    // Back up to the beginning of the line
    let start_of_col : Length = {
      bytes: self.current_position.bytes - self.current_position.extent.column,
      extent: { row: self.current_position.extent.row, column: 0, },
    }
    self.goto(start_of_col)
    self.set_column_data(0)
    self.get_chunk()
    if !self.eof() {
      self.get_lookahead()
      // Advance to the recorded position
      while self.current_position.bytes < goal_byte &&
            !self.eof() &&
            self.has_chunk {
        self.do_advance(false)
        if self.eof() {
          break
        }
      }
    }
  }
  self.column_data.value
}

///|
/// `TSLexer.is_at_included_range_start`: is the lexer at a boundary between
/// two disjoint included ranges of source code?
pub fn Lexer::is_at_included_range_start(self : Lexer) -> Bool {
  if self.current_included_range_index < self.included_ranges.length() {
    let current_range = self.included_ranges[self.current_included_range_index]
    self.current_position.bytes == current_range.start_byte
  } else {
    false
  }
}

///|
/// `TSLexer.log`.
pub fn Lexer::log(self : Lexer, message : String) -> Unit {
  if self.logger is Some(log) {
    // Upstream formats into the 1024-byte debug buffer.
    if self.write_debug_buffer(message) is Some(message) {
      log(Lex, message)
    }
  }
}

///|
/// `ts_lexer_init`.
pub fn Lexer::new() -> Lexer {
  let self = {
    lookahead: 0,
    result_symbol: 0,
    current_position: length_zero(),
    token_start_position: length_zero(),
    token_end_position: length_zero(),
    included_ranges: [],
    chunk: b""[:],
    has_chunk: false,
    input: { read: (_, _) => b"", encoding: UTF8, decode: None, },
    logger: None,
    current_included_range_index: 0,
    chunk_start: 0,
    chunk_size: 0,
    lookahead_size: 0,
    did_get_column: false,
    column_data: { value: 0, valid: false, },
    debug_buffer: FixedArray::make(
      @language.tree_sitter_serialization_buffer_size, 0,
    ),
  }
  ignore(self.set_included_ranges([]))
  self
}

///|
/// `ts_lexer_set_input`.
pub fn Lexer::set_input(self : Lexer, input : Input) -> Unit {
  self.input = input
  self.clear_chunk()
  self.goto(self.current_position)
}

///|
pub fn Lexer::set_logger(self : Lexer, logger : Logger?) -> Unit {
  self.logger = logger
}

///|
/// `ts_lexer_reset`: move the lexer to the given position. This doesn't do
/// any work if the parser is already at the given position.
pub fn Lexer::reset(self : Lexer, position : Length) -> Unit {
  if position.bytes != self.current_position.bytes {
    self.goto(position)
  }
}

///|
/// `ts_lexer_start`.
pub fn Lexer::start(self : Lexer) -> Unit {
  self.token_start_position = self.current_position
  self.token_end_position = length_undefined
  self.result_symbol = 0
  self.did_get_column = false
  if !self.eof() {
    if self.chunk_size == 0 {
      self.get_chunk()
    }
    if self.lookahead_size == 0 {
      self.get_lookahead()
    }
    if self.current_position.bytes == 0 {
      if self.lookahead == byte_order_mark {
        self.advance(true)
      }
      self.set_column_data(0)
    }
  }
}

///|
/// `ts_lexer_finish`: returns the updated `lookahead_end_byte`.
pub fn Lexer::finish(self : Lexer, lookahead_end_byte : UInt32) -> UInt32 {
  if length_is_undefined(self.token_end_position) {
    self.mark_end()
  }
  // If the token ended at an included range boundary, then its end position
  // will have been reset to the end of the preceding range. Reset the start
  // position to match.
  if self.token_end_position.bytes < self.token_start_position.bytes {
    self.token_start_position = self.token_end_position
  }
  let mut current_lookahead_end_byte = self.current_position.bytes + 1
  // In order to determine that a byte sequence is invalid UTF8 or UTF16, the
  // character decoding algorithm may have looked at the following byte.
  // Therefore, the next byte *after* the current (invalid) character affects
  // the interpretation of the current character.
  if self.lookahead == ts_decode_error {
    current_lookahead_end_byte += 4
  }
  if current_lookahead_end_byte > lookahead_end_byte {
    current_lookahead_end_byte
  } else {
    lookahead_end_byte
  }
}

///|
/// `ts_lexer_set_included_ranges`.
pub fn Lexer::set_included_ranges(self : Lexer, ranges : Array[Range]) -> Bool {
  let ranges = if ranges.is_empty() {
    [default_range]
  } else {
    let mut previous_byte : UInt32 = 0
    for range in ranges {
      if range.start_byte < previous_byte || range.end_byte < range.start_byte {
        return false
      }
      previous_byte = range.end_byte
    }
    ranges.copy()
  }
  self.included_ranges = ranges
  self.goto(self.current_position)
  true
}

///|
/// `ts_lexer_included_ranges`.
pub fn Lexer::included_ranges(self : Lexer) -> Array[Range] {
  self.included_ranges
}

///|
pub fn Lexer::current_position(self : Lexer) -> Length {
  self.current_position
}

///|
pub fn Lexer::token_start_position(self : Lexer) -> Length {
  self.token_start_position
}

///|
pub fn Lexer::token_end_position(self : Lexer) -> Length {
  self.token_end_position
}

///|
pub fn Lexer::did_get_column(self : Lexer) -> Bool {
  self.did_get_column
}

///|
pub fn Lexer::has_logger(self : Lexer) -> Bool {
  self.logger is Some(_)
}