///|
let use_utf16_location : Ref[Bool] = Ref(false)

///|
fn lex_tokens(
  input : StringView,
  base~ : StringView,
  env~ : LexEnv,
  preserve_comment~ : (Comment, Int, Int) -> Unit,
) -> Unit {
  fn metavar_enabled() -> Bool {
    env.enable_metavar && !env.is_interpolation
  }

  fn typed_metavar_payload(
    prefix : StringView,
    name : StringView,
    kind : StringView,
  ) -> String {
    "\{prefix}(\{name}:\{kind})"
  }

  fn inferred_metavar_payload(
    prefix : StringView,
    name : StringView,
  ) -> String {
    "\{prefix}\{name}"
  }

  for input = input {
    // These helpers return the view to continue lexing from.
    fn reject_disabled_metavar(ch : Char) -> StringView {
      let start_pos = env.calc_offset(input, base~)
      env.add_lexing_error(
        IllegalCharacter(ch),
        start=start_pos,
        end=start_pos + 1,
      )
      input[1:]
    }

    fn resume_disabled_dot_metavar() -> StringView {
      let dot_pos = env.calc_offset(input, base~) + 1
      env.add_token_with_loc(DOT_LIDENT(""), start=dot_pos, end=dot_pos)
      input[1:]
    }

    fn add_metavar_token(
      tok : @tokens.Token,
      rest : StringView,
      disabled_char : Char,
      start_offset : Int,
    ) -> StringView {
      if metavar_enabled() {
        env.add_token_with_loc(
          tok,
          start=env.calc_offset(input, base~) + start_offset,
          end=env.calc_offset(rest, base~),
        )
        rest
      } else if start_offset > 0 {
        resume_disabled_dot_metavar()
      } else {
        reject_disabled_metavar(disabled_char)
      }
    }

    fn add_metavar_error(
      raw : StringView,
      rest : StringView,
      disabled_char : Char,
      start_offset : Int,
    ) -> StringView {
      if metavar_enabled() {
        env.add_lexing_error(
          InvalidMetavarSyntax(raw.to_owned()),
          start=env.calc_offset(input, base~) + start_offset,
          end=env.calc_offset(rest, base~),
        )
        rest
      } else if start_offset > 0 {
        resume_disabled_dot_metavar()
      } else {
        reject_disabled_metavar(disabled_char)
      }
    }

    fn add_metavar_error_to_rest(
      rest : StringView,
      disabled_char : Char,
      start_offset : Int,
    ) -> StringView {
      let raw_end = env.calc_offset(rest, base~) - env.calc_offset(input, base~)
      add_metavar_error(
        input.sub(start=0, end=raw_end),
        rest,
        disabled_char,
        start_offset,
      )
    }

    lexmatch input with longest {
      (
        re"^" +
        RE_LINE_BREAK,
        // Handle newlines
        after=rest,
      ) => {
        let start_pos = env.calc_offset(input, base~)
        let end_pos = env.calc_offset(rest, base~)
        env.add_token_with_loc(NEWLINE, start=start_pos, end=end_pos)
        env.current_bol = end_pos
        env.current_line += 1
        continue rest
      }

      // Handle whitespace (Unicode spaces)

      (re"^" + RE_UNICODE_SPACES1, after=rest) => continue rest

      // Fat arrow
      (re"^(?:=>)", after=rest) => {
        env.add_token_with_loc(
          FAT_ARROW,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Thin arrow
      (re"^(?:->)", after=rest) => {
        env.add_token_with_loc(
          THIN_ARROW,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Comments
      (re"^" + (RE_LINE_COMMENT as raw), after=rest) => {
        let start_pos = env.calc_offset(input, base~)
        env.register_surrogate_pair(raw)
        let end_pos = env.calc_offset(rest, base~)
        let comment_text = raw.to_owned()
        if env.is_interpolation {
          env.add_lexing_error(
            start=start_pos,
            end=end_pos,
            InterpInvalidComment,
          )
        }
        if env.comment {
          let comment = Comment::{
            content: comment_text,
            kind: InlineTrailing,
            consumed_by_docstring: false,
          }
          preserve_comment(comment, start_pos, end_pos)
          env.add_token_with_loc(COMMENT(comment), start=start_pos, end=end_pos)
        }
        continue rest
      }

      // Character literals - basic cases
      (
        re"^" +
        RE_CHAR_QUOTE +
        (RE_CHAR_CONTENT as raw) +
        RE_CHAR_QUOTE,
        after=rest,
      ) => {
        let start_pos = env.calc_offset(input, base~)
        env.register_surrogate_pair_for_char(raw)
        let end_pos = env.calc_offset(rest, base~)
        env.add_token_with_loc(CHAR([raw]), start=start_pos, end=end_pos)
        continue rest
      }

      // Character literals - escape sequences
      (
        re"^" +
        RE_CHAR_QUOTE +
        (RE_CHAR_ESCAPE as raw) +
        RE_CHAR_QUOTE,
        after=rest,
      ) => {
        env.add_token_with_loc(
          CHAR(raw.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Character literals - hex escape
      (
        re"^" +
        RE_CHAR_QUOTE +
        (RE_HEX_ESCAPE as raw) +
        RE_CHAR_QUOTE,
        after=rest,
      ) => {
        env.add_token_with_loc(
          CHAR(raw.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Character literals - octal escape
      (
        re"^" +
        RE_CHAR_QUOTE +
        (RE_OCTAL_ESCAPE as raw) +
        RE_CHAR_QUOTE,
        after=rest,
      ) => {
        env.add_token_with_loc(
          CHAR(raw.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Character literals - unicode escape
      (
        re"^" +
        RE_CHAR_QUOTE +
        (RE_SHORT_UNICODE_ESCAPE as raw) +
        RE_CHAR_QUOTE,
        after=rest,
      ) => {
        env.add_token_with_loc(
          CHAR(raw.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Character literals - unicode escape with braces
      (
        re"^" +
        RE_CHAR_QUOTE +
        (RE_BRACED_UNICODE_ESCAPE as raw) +
        RE_CHAR_QUOTE,
        after=rest,
      ) => {
        let char_text = raw.to_owned()
        env.add_token_with_loc(
          CHAR(char_text),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Regex literals - re"..."
      (re"^(?:re\")", after=rest) => {
        let start_pos = env.calc_offset(input, base~)
        let (rest, interps) = lex_string(
          rest,
          base~,
          env~,
          end_with_newline=false,
          allow_interp=true,
          start_pos~,
        )
        let tok : @tokens.Token = match interps {
          [InterpLit(repr~, ..)] => REGEX_LITERAL(repr)
          interps => REGEX_INTERP(interps)
        }
        let end_pos = env.calc_offset(rest, base~)
        env.add_token_with_loc(tok, start=start_pos, end=end_pos)
        continue rest
      }

      // String literals - basic double-quoted string
      (re"^(?:\")", after=rest) => {
        let start_pos = env.calc_offset(input, base~)
        let (rest, interps) = lex_string(
          rest,
          base~,
          env~,
          end_with_newline=false,
          allow_interp=true,
          start_pos~,
        )
        let tok : @tokens.Token = match interps {
          [InterpLit(repr~, ..)] => STRING(repr)
          interps => INTERP(interps)
        }
        let end_pos = env.calc_offset(rest, base~)
        env.add_token_with_loc(tok, start=start_pos, end=end_pos)
        continue rest
      }

      // Byte string literal
      (re"^(?:b\")", after=rest) => {
        let start_pos = env.calc_offset(input, base~)
        let (rest, interps) = lex_string(
          rest,
          base~,
          env~,
          end_with_newline=false,
          allow_interp=true,
          start_pos~,
        )
        let tok : @tokens.Token = match interps {
          [InterpLit(repr~, ..)] => BYTES(repr)
          interps => BYTES_INTERP(interps)
        }
        env.add_token_with_loc(
          tok,
          start=start_pos,
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Byte literals - hex escape
      (re"^(?:b'\\x[0-9a-fA-F]{2}')" as raw, after=rest) => {
        let literal = raw.sub(start=2, end=raw.length() - 1).to_owned() // Remove b' and '
        env.add_token_with_loc(
          BYTE(literal),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Byte literals - octal escape
      (re"^(?:b'\\o[0-3][0-7]{2}')" as raw, after=rest) => {
        let literal = raw.sub(start=2, end=raw.length() - 1).to_owned() // Remove b' and '
        env.add_token_with_loc(
          BYTE(literal),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Byte literals - ASCII character
      (re"^(?:b'[\x00-\x7F]')" as raw, after=rest) => {
        let ascii_char = raw.sub(start=2, end=raw.length() - 1).to_owned() // Remove b' and '
        env.add_token_with_loc(
          BYTE(ascii_char),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Byte literals - escape sequences
      (re"^(?:b'\\[\\'\"`ntbrf/ ]')" as raw, after=rest) => {
        let literal = raw.sub(start=2, end=raw.length() - 1).to_owned() // Remove b' and '
        env.add_token_with_loc(
          BYTE(literal),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Invalid byte literal
      (re"^(?:b')", after=rest) => {
        let start = env.calc_offset(base~, input)
        let rest = lex_invalid_byte(rest, base~, env~, start~)
        continue rest
      }

      // Float literals
      (re"^" + (RE_FLOAT as float), after=rest) => {
        env.add_token_with_loc(
          FLOAT(float.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Double literals
      (re"^" + (RE_DOUBLE as double), after=rest) => {
        env.add_token_with_loc(
          DOUBLE(double.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Integer literals with range operator handling

      (re"^" + ((RE_INTEGER + re"\.\.") as integer_with_range), after=_rest) => {
        // Need to handle integer..range specially
        let integer_end = integer_with_range.length() - 2 // Remove ".."
        let integer = integer_with_range
          .sub(start=0, end=integer_end)
          .to_owned()
        env.add_token_with_loc(
          INT(integer),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(input, base~) + integer_end,
        )
        // Put back the ".." part
        let dotdot_input = input[integer_end:]
        continue dotdot_input
      }

      // Regular integer literals
      (re"^" + (RE_INTEGER as integer), after=rest) => {
        env.add_token_with_loc(
          INT(integer.to_owned()),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Upper case identifiers (types) - simplified pattern for ASCII and basic Unicode

      (re"^" + (RE_UIDENT as raw), after=rest) => {
        let start_pos = env.calc_offset(input, base~)
        env.register_surrogate_pair(raw)
        let end_pos = env.calc_offset(rest, base~)
        env.add_token_with_loc(
          UIDENT(raw.to_owned()),
          start=start_pos,
          end=end_pos,
        )
        continue rest
      }

      // Transparent post label identifiers (name~~) must be recognized before
      // ordinary post labels so the second `~` is part of the token.
      (re"^(?:[a-z_][a-zA-Z0-9_]*~~)" as raw, after=rest) => {
        let ident = raw.sub(start=0, end=raw.length() - 2).to_owned()
        env.add_token_with_loc(
          POST_LABEL_TRANSPARENT(ident),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Post label identifiers (name~) - simplified pattern
      (re"^(?:[a-z_][a-zA-Z0-9_]*~)" as raw, after=rest) => {
        let ident = raw.sub(start=0, end=raw.length() - 1).to_owned() // Remove ~
        env.add_token_with_loc(
          POST_LABEL(ident),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Lower case identifiers and keywords

      (re"^" + (RE_LIDENT as raw), after=rest) => {
        let raw_str = raw.to_owned()
        let start_pos = env.calc_offset(input, base~)
        env.register_surrogate_pair(raw)
        let end_pos = env.calc_offset(rest, base~)
        if reserved_keyword_table.contains(raw_str) {
          env.add_lexing_error(
            Reserved_keyword(raw_str),
            start=start_pos,
            end=end_pos,
          )
        }
        let token = match keyword_table.get(raw_str) {
          None => @tokens.Token::LIDENT(raw_str)
          Some(tok) => tok
        }
        env.add_token_with_loc(token, start=start_pos, end=end_pos)
        continue rest
      }

      // Operators and punctuation - order matters for longer operators first

      (re"^(?:\+=)", after=rest) => {
        env.add_token_with_loc(
          PLUS_EQUAL,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Augmented assignment operators
      (re"^(?:[+\-*/%]=)" as op, after=rest) => {
        let op_char = op.sub(start=0, end=1).to_owned()
        env.add_token_with_loc(
          AUGMENTED_ASSIGNMENT(op_char),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Multiple character operators
      (re"^(?:&&)", after=rest) => {
        env.add_token_with_loc(
          AMPERAMPER,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\|\|)", after=rest) => {
        env.add_token_with_loc(
          BARBAR,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\|>)", after=rest) => {
        env.add_token_with_loc(
          PIPE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\|\])", after=rest) => {
        env.add_token_with_loc(
          BAR_RBRACKET,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:<\|)", after=rest) => {
        env.add_token_with_loc(
          PIPE_LEFT,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:<\+)", after=rest) => {
        env.add_token_with_loc(
          LT_PLUS,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:<\?)", after=rest) => {
        env.add_token_with_loc(
          LT_QUESTION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:try\?)", after=rest) => {
        env.add_token_with_loc(
          TRY_QUESTION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:try!)", after=rest) => {
        env.add_token_with_loc(
          TRY_EXCLAMATION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:guard!)", after=rest) => {
        env.add_token_with_loc(
          GUARD_EXCLAMATION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:lexmatch\?)", after=rest) => {
        env.add_token_with_loc(
          LEXMATCH_QUESTION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:==)", after=rest) => {
        env.add_token_with_loc(
          INFIX1("=="),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:=~)", after=rest) => {
        env.add_token_with_loc(
          EQ_TILDE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:!=)", after=rest) => {
        env.add_token_with_loc(
          INFIX1("!="),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:<=)", after=rest) => {
        env.add_token_with_loc(
          INFIX1("<="),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:>\.\.)", after=rest) => {
        env.add_token_with_loc(
          RANGE_EXCLUSIVE_REV,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:>=\.\.)", after=rest) => {
        env.add_token_with_loc(
          RANGE_INCLUSIVE_REV,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:>=)", after=rest) => {
        env.add_token_with_loc(
          INFIX1(">="),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:<<)", after=rest) => {
        env.add_token_with_loc(
          INFIX2("<<"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:>>)", after=rest) => {
        env.add_token_with_loc(
          INFIX2(">>"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\u{2200})", after=rest) => {
        env.add_token_with_loc(
          FORALL,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\u{2203})", after=rest) => {
        env.add_token_with_loc(
          EXISTS,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\u{2192})", after=rest) => {
        env.add_token_with_loc(
          IMPLIES,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:::)", after=rest) => {
        env.add_token_with_loc(
          COLONCOLON,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Dots and ranges
      (re"^(?:\.\.\.)", after=rest) => {
        env.add_token_with_loc(
          ELLIPSIS,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\.\.<=)", after=rest) => {
        env.add_token_with_loc(
          RANGE_LT_INCLUSIVE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\.\.=)", after=rest) => {
        env.add_token_with_loc(
          RANGE_INCLUSIVE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\.\.<)", after=rest) => {
        env.add_token_with_loc(
          RANGE_EXCLUSIVE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\.\.)", after=rest) => {
        env.add_token_with_loc(
          DOTDOT,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Dot with identifier

      // Dot metavariable labels, only enabled for pattern parsing.
      (re"^(?:\.\$\$\([^\)\r\n]*\))" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 1)
      (re"^(?:\.\$\$[A-Za-z_][A-Za-z0-9_]*)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 1)
      (re"^(?:\.\$\$)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 1)
      (
        re"^(?:\.)" +
        (RE_METAVAR_PREFIX as prefix) +
        re"(?:\()" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_NAME as name) +
        RE_METAVAR_SPACES +
        re"(?::)" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_KIND as kind) +
        RE_METAVAR_SPACES +
        re"(?:\))",
        after=rest,
      ) =>
        continue add_metavar_token(
            DOT_LIDENT(typed_metavar_payload(prefix, name, kind)),
            rest,
            '$',
            1,
          )
      (
        re"^(?:\.)" +
        (RE_METAVAR_PREFIX as prefix) +
        (RE_METAVAR_NAME as name),
        after=rest,
      ) =>
        continue add_metavar_token(
            DOT_LIDENT(inferred_metavar_payload(prefix, name)),
            rest,
            '$',
            1,
          )
      (re"^(?:\.\$(\$\$)?\([ ]*:)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 1)
      (re"^(?:\.\$(\$\$)?\([^\)\r\n]*\))" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 1)
      (
        re"^(?:\.\$(\$\$)?\()" +
        re"(?:[ ]*)" +
        re"(?:[A-Za-z_][A-Za-z0-9_]*)" +
        re"(?:[ ]*)" +
        re"(?::)" +
        re"(?:[ ]*)" +
        re"(?:[a-z][a-z]*)",
        after=rest,
      ) => continue add_metavar_error_to_rest(rest, '$', 1)
      (re"^(?:\.\$(\$\$)?[a-z][a-z]*:[a-z_][A-Za-z0-9_]*)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 1)
      (re"^(?:\.)" + (RE_UIDENT as ident), after=rest) => {
        let name = ident.to_owned()
        let start_pos = env.calc_offset(input, base~) + 1
        env.register_surrogate_pair(ident)
        let end_pos = env.calc_offset(rest, base~)
        env.add_token_with_loc(DOT_UIDENT(name), start=start_pos, end=end_pos)
        continue rest
      }
      (re"^(?:\.)" + (RE_OPTIONAL_LIDENT as ident), after=rest) => {
        let name = ident.to_owned()
        let dot_start = env.calc_offset(input, base~)
        let start_pos = env.calc_offset(input, base~) + 1
        env.register_surrogate_pair(ident)
        let end_pos = env.calc_offset(rest, base~)
        if name == "" {
          env.add_lexing_error(
            MissingIdentifierAfterDot,
            start=dot_start,
            end=end_pos,
          )
        }
        env.add_token_with_loc(DOT_LIDENT(name), start=start_pos, end=end_pos)
        continue rest
      }

      // Dot with number
      (re"^(?:\.[0-9]+)" as dot_int, after=rest) => {
        let digits_str = dot_int.sub(start=1, end=dot_int.length()) // Remove the '.'
        let idx = @string.parse_int(digits_str) catch {
          _ => {
            let full_str = dot_int.to_owned()
            env.add_lexing_error(
              InvalidDotInt(full_str),
              start=env.calc_offset(input, base~),
              end=env.calc_offset(rest, base~),
            )
            0
          }
        }
        env.add_token_with_loc(
          DOT_INT(idx),
          start=env.calc_offset(input, base~) + 1,
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Dot with parenthesis
      (re"^(?:\.\()", after=rest) => {
        env.add_token_with_loc(
          DOT_LPAREN,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Single character operators and punctuation
      (re"^(?:&)", after=rest) => {
        env.add_token_with_loc(
          AMPER,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\|)", after=rest) => {
        env.add_token_with_loc(
          BAR,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\^)", after=rest) => {
        env.add_token_with_loc(
          CARET,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\()", after=rest) => {
        env.add_token_with_loc(
          LPAREN,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\))", after=rest) => {
        env.add_token_with_loc(
          RPAREN,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\*)", after=rest) => {
        env.add_token_with_loc(
          INFIX3("*"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:/)", after=rest) => {
        env.add_token_with_loc(
          INFIX3("/"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:%)", after=rest) => {
        env.add_token_with_loc(
          INFIX3("%"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:,)", after=rest) => {
        env.add_token_with_loc(
          COMMA,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?::)", after=rest) => {
        env.add_token_with_loc(
          COLON,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:;)", after=rest) => {
        env.add_token_with_loc(
          SEMI(true),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:=)", after=rest) => {
        env.add_token_with_loc(
          EQUAL,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:<)", after=rest) => {
        env.add_token_with_loc(
          INFIX1("<"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:>)", after=rest) => {
        env.add_token_with_loc(
          INFIX1(">"),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\[\|)", after=rest) => {
        env.add_token_with_loc(
          LBRACKET_BAR,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\[)", after=rest) => {
        env.add_token_with_loc(
          LBRACKET,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\])", after=rest) => {
        env.add_token_with_loc(
          RBRACKET,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:[{])", after=rest) => {
        env.add_token_with_loc(
          LBRACE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:[}])", after=rest) => {
        env.add_token_with_loc(
          RBRACE,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\+)", after=rest) => {
        env.add_token_with_loc(
          PLUS,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:-)", after=rest) => {
        env.add_token_with_loc(
          MINUS,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:\?)", after=rest) => {
        env.add_token_with_loc(
          QUESTION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }
      (re"^(?:!)", after=rest) => {
        env.add_token_with_loc(
          EXCLAMATION,
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Attributes

      // #ident.dot_ident with payload

      (
        re"^(?:#)" +
        (re"(?:[a-zA-Z_][a-zA-Z0-9_]*)" as ident) +
        re"(?:\.)" +
        (re"(?:[a-zA-Z_][a-zA-Z0-9_]*)" as dot_ident) +
        (re"(?:[^\r\n]*)" as raw_payload),
        after=rest,
      ) => {
        if env.is_interpolation {
          env.add_lexing_error(
            start=env.calc_offset(input, base~),
            end=env.calc_offset(rest, base~),
            InterpInvalidAttribute,
          )
        } else {
          let ident = ident.to_owned()
          let dot_ident = dot_ident.to_owned()
          let raw_payload = raw_payload.to_owned()
          env.add_token_with_loc(
            ATTRIBUTE((ident, Some(dot_ident), raw_payload)),
            start=env.calc_offset(input, base~),
            end=env.calc_offset(rest, base~),
          )
        }
        continue rest
      }

      // #ident with payload

      (
        re"^(?:#)" +
        (re"(?:[a-zA-Z_][a-zA-Z0-9_]*)" as ident) +
        (re"(?:[^\r\n]*)" as raw_payload),
        after=rest,
      ) => {
        if env.is_interpolation {
          env.add_lexing_error(
            start=env.calc_offset(input, base~),
            end=env.calc_offset(rest, base~),
            InterpInvalidAttribute,
          )
        } else {
          let ident = ident.to_owned()
          let raw_payload = raw_payload.to_owned()
          env.add_token_with_loc(
            ATTRIBUTE((ident, None, raw_payload)),
            start=env.calc_offset(input, base~),
            end=env.calc_offset(rest, base~),
          )
        }
        continue rest
      }

      // Multiline string interpolation $|
      (re"^(?:\$\|)", after=rest) => {
        if env.is_interpolation {
          env.add_lexing_error(
            start=env.calc_offset(input, base~),
            end=env.calc_offset(rest, base~),
            InterpInvalidMultilineString,
          )
        }
        let start_pos = env.calc_offset(input, base~)
        let (rest, interps) = lex_string(
          rest,
          base~,
          env~,
          end_with_newline=true,
          allow_interp=true,
          start_pos~,
        )
        let tok = @tokens.Token::MULTILINE_INTERP(interps)
        env.add_token_with_loc(
          tok,
          start=start_pos,
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // Pattern metavariables, only enabled for pattern parsing.
      (re"^(?:\$\$\([^\)\r\n]*\)~?)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)
      (re"^(?:\$\$[A-Za-z_][A-Za-z0-9_]*~?)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)
      (re"^(?:\$\$)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)
      (
        re"^" +
        (RE_METAVAR_PREFIX as prefix) +
        re"(?:\()" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_NAME as name) +
        RE_METAVAR_SPACES +
        re"(?::)" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_KIND as kind) +
        RE_METAVAR_SPACES +
        re"(?:\)~)",
        after=rest,
      ) =>
        continue add_metavar_token(
            POST_LABEL(typed_metavar_payload(prefix, name, kind)),
            rest,
            '$',
            0,
          )
      (
        re"^" +
        (RE_METAVAR_PREFIX as prefix) +
        (RE_METAVAR_NAME as name) +
        re"(?:~)",
        after=rest,
      ) =>
        continue add_metavar_token(
            POST_LABEL(inferred_metavar_payload(prefix, name)),
            rest,
            '$',
            0,
          )
      (
        re"^" +
        (RE_METAVAR_PREFIX as prefix) +
        re"(?:\()" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_UIDENT as name) +
        RE_METAVAR_SPACES +
        re"(?::)" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_KIND as kind) +
        RE_METAVAR_SPACES +
        re"(?:\))",
        after=rest,
      ) =>
        continue add_metavar_token(
            UIDENT(typed_metavar_payload(prefix, name, kind)),
            rest,
            '$',
            0,
          )
      (
        re"^" +
        (RE_METAVAR_PREFIX as prefix) +
        re"(?:\()" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_LIDENT as name) +
        RE_METAVAR_SPACES +
        re"(?::)" +
        RE_METAVAR_SPACES +
        (RE_METAVAR_KIND as kind) +
        RE_METAVAR_SPACES +
        re"(?:\))",
        after=rest,
      ) =>
        continue add_metavar_token(
            LIDENT(typed_metavar_payload(prefix, name, kind)),
            rest,
            '$',
            0,
          )
      (
        re"^" +
        (RE_METAVAR_PREFIX as prefix) +
        (RE_METAVAR_UIDENT as name),
        after=rest,
      ) =>
        continue add_metavar_token(
            UIDENT(inferred_metavar_payload(prefix, name)),
            rest,
            '$',
            0,
          )
      (
        re"^" +
        (RE_METAVAR_PREFIX as prefix) +
        (RE_METAVAR_LIDENT as name),
        after=rest,
      ) =>
        continue add_metavar_token(
            LIDENT(inferred_metavar_payload(prefix, name)),
            rest,
            '$',
            0,
          )
      (re"^(?:\$(\$\$)?\([ ]*:)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)
      (re"^(?:\$(\$\$)?\([^\)\r\n]*\))" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)
      (
        re"^(?:\$(\$\$)?\()" +
        re"(?:[ ]*)" +
        re"(?:[A-Za-z_][A-Za-z0-9_]*)" +
        re"(?:[ ]*)" +
        re"(?::)" +
        re"(?:[ ]*)" +
        re"(?:[a-z][a-z]*)",
        after=rest,
      ) => continue add_metavar_error_to_rest(rest, '$', 0)
      (re"^(?:\$(\$\$)?[a-z][a-z]*:[A-Za-z_][A-Za-z0-9_]*~)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)
      (re"^(?:\$(\$\$)?[a-z][a-z]*:[a-z_][A-Za-z0-9_]*)" as raw, after=rest) =>
        continue add_metavar_error(raw, rest, '$', 0)

      // Multiline string #|
      (re"^(?:#\|[^\r\n]*)" as multiline_str, after=rest) => {
        let start_pos = env.calc_offset(input, base~)
        env.register_surrogate_pair(multiline_str)
        let end_pos = env.calc_offset(rest, base~)
        if env.is_interpolation {
          env.add_lexing_error(
            start=start_pos,
            end=end_pos,
            InterpInvalidMultilineString,
          )
        }
        let content_str = multiline_str.to_owned()
        let content = content_str.sub(start=2, end=content_str.length()) // Remove #|
        env.add_token_with_loc(
          MULTILINE_STRING(content.to_owned()),
          start=start_pos,
          end=end_pos,
        )
        continue rest
      }

      // Package name @package/name
      (
        re"^(?:@[a-zA-Z_][a-zA-Z0-9_\-]*(/[a-zA-Z_][a-zA-Z0-9_\-]*)*)" as raw,
        after=rest,
      ) => {
        let pkg_name_without_at = raw.sub(start=1, end=raw.length()).to_owned()
        env.add_token_with_loc(
          PACKAGE_NAME(pkg_name_without_at),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(rest, base~),
        )
        continue rest
      }

      // EOF case
      (re"^(?:$)", after=_) => {
        let end_pos = env.calc_offset(input, base~)
        env.add_token_with_loc(EOF, start=end_pos, end=end_pos)
        break
      }

      // Error case - any remaining character
      (re"^(?:.)" as c, after=rest) => {
        env.add_lexing_error(
          IllegalCharacter(c),
          start=env.calc_offset(input, base~),
          end=env.calc_offset(input, base~) + 1,
        )
        continue rest
      }
    }
  }
}

///|
fn interpolation_write_view(
  env : LexEnv,
  buf : StringBuilder,
  raw : StringView,
) -> Unit {
  env.register_surrogate_pair(raw)
  buf.write_view(raw)
}

///|
fn interpolation_write_char(
  env : LexEnv,
  buf : StringBuilder,
  c : Char,
) -> Unit {
  env.register_surrogate_pair_for_char(c)
  buf.write_char(c)
}

///|
fn add_interpolation_lexing_error(
  env : LexEnv,
  base : StringView,
  input : StringView,
  rest : StringView,
  err : LexicalError,
) -> Unit {
  env.add_lexing_error(
    start=env.calc_offset(base~, input),
    end=env.calc_offset(base~, rest),
    err,
  )
}

///|
// These interpolation scanners currently form a mutually recursive state
// machine. The `break scan_interpolation_*` transitions below are ordinary
// cross-function calls: the callee must return before the caller can break out
// of its loop, so every caller frame remains live.
//
// Consequently, sequential atoms grow the call stack even when the source is
// not deeply nested. For example, `\{["", "", ...]}` alternates between
// `scan_interpolation_source` and `scan_interpolation_string_atom` twice per
// string atom. A sufficiently long sequence overflows the JS/Wasm call stack
// and can crash the native backend with a stack-overflow SIGSEGV.
//
// The local `for` loops only remove each scanner's direct self-recursion; they
// do not make these cross-scanner transitions stack-safe. A complete fix should
// drive the scanner modes from one loop, a trampoline, or an explicit state
// stack, so sequential atom count cannot consume call stack space; genuinely
// nested interpolation should be represented explicitly by that state machine.
fn scan_interpolation_source(
  input : StringView,
  base~ : StringView,
  env~ : LexEnv,
  buf~ : StringBuilder,
  brace_depth~ : Int,
  preserve~ : Bool,
) -> (StringView, Int) {
  for input = input, brace_depth = brace_depth {
    lexmatch input with longest {
      (re"^" + ((RE_UNICODE_SPACES + re"\}") as raw), after=rest) =>
        if brace_depth == 0 {
          if preserve {
            interpolation_write_view(env, buf, raw)
            break (rest, env.calc_offset(base~, rest))
          } else {
            break (rest, env.calc_offset(base~, input))
          }
        } else {
          interpolation_write_view(env, buf, raw)
          continue rest, brace_depth - 1
        }
      re"^$" => {
        env.add_lexing_error(
          start=env.calc_offset(base~, input),
          end=env.calc_offset(base~, input),
          UnterminatedString,
        )
        break (input, env.calc_offset(base~, input))
      }
      (re"^" + RE_ASCII_LINE_BREAK, after=_) => {
        env.add_lexing_error(
          start=env.calc_offset(base~, input),
          end=env.calc_offset(base~, input),
          UnterminatedStringInVariableInterploation,
        )
        break (input, env.calc_offset(base~, input))
      }
      (re"^(?:b')" as raw, after=rest) => {
        interpolation_write_view(env, buf, raw)
        break scan_interpolation_char_atom(
          rest,
          base~,
          env~,
          buf~,
          brace_depth~,
          preserve~,
          escaped=false,
        )
      }
      (re"^(?:')" as raw, after=rest) => {
        interpolation_write_char(env, buf, raw)
        break scan_interpolation_char_atom(
          rest,
          base~,
          env~,
          buf~,
          brace_depth~,
          preserve~,
          escaped=false,
        )
      }
      (re"^(?:b\")" as raw, after=rest) => {
        interpolation_write_view(env, buf, raw)
        break scan_interpolation_string_atom(
          rest,
          base~,
          env~,
          buf~,
          brace_depth~,
          preserve~,
          allow_interp=true,
          escaped=false,
        )
      }
      (re"^(?:re\")" as raw, after=rest) => {
        interpolation_write_view(env, buf, raw)
        break scan_interpolation_string_atom(
          rest,
          base~,
          env~,
          buf~,
          brace_depth~,
          preserve~,
          allow_interp=true,
          escaped=false,
        )
      }
      (re"^(?:\")" as raw, after=rest) => {
        interpolation_write_char(env, buf, raw)
        // This mode switch retains the current source-scanner stack frame; see
        // the stack-safety note above `scan_interpolation_source`.
        break scan_interpolation_string_atom(
          rest,
          base~,
          env~,
          buf~,
          brace_depth~,
          preserve~,
          allow_interp=true,
          escaped=false,
        )
      }
      (re"^" + (RE_LINE_COMMENT as raw), after=rest) => {
        add_interpolation_lexing_error(
          env,
          base,
          input,
          rest,
          InterpInvalidComment,
        )
        interpolation_write_view(env, buf, raw)
        continue rest, brace_depth
      }
      (re"^(?:\$\|[^\r\n]*)" as raw, after=rest) => {
        add_interpolation_lexing_error(
          env,
          base,
          input,
          rest,
          InterpInvalidMultilineString,
        )
        interpolation_write_view(env, buf, raw)
        continue rest, brace_depth
      }
      (re"^(?:#\|[^\r\n]*)" as raw, after=rest) => {
        add_interpolation_lexing_error(
          env,
          base,
          input,
          rest,
          InterpInvalidMultilineString,
        )
        interpolation_write_view(env, buf, raw)
        continue rest, brace_depth
      }
      (re"^(?:#[a-zA-Z_][a-zA-Z0-9_]*[^\r\n]*)" as raw, after=rest) => {
        add_interpolation_lexing_error(
          env,
          base,
          input,
          rest,
          InterpInvalidAttribute,
        )
        interpolation_write_view(env, buf, raw)
        continue rest, brace_depth
      }
      (re"^(?:[{])" as raw, after=rest) => {
        interpolation_write_char(env, buf, raw)
        continue rest, brace_depth + 1
      }
      (re"^(?:.)" as c, after=rest) => {
        interpolation_write_char(env, buf, c)
        continue rest, brace_depth
      }
    }
  }
}

///|
fn scan_interpolation_string_atom(
  input : StringView,
  base~ : StringView,
  env~ : LexEnv,
  buf~ : StringBuilder,
  brace_depth~ : Int,
  preserve~ : Bool,
  allow_interp~ : Bool,
  escaped~ : Bool,
) -> (StringView, Int) {
  for input = input, escaped = escaped {
    lexmatch input with longest {
      re"^$" => {
        env.add_lexing_error(
          start=env.calc_offset(base~, input),
          end=env.calc_offset(base~, input),
          UnterminatedString,
        )
        break (input, env.calc_offset(base~, input))
      }
      (re"^" + RE_ASCII_LINE_BREAK, after=_) => {
        env.add_lexing_error(
          start=env.calc_offset(base~, input),
          end=env.calc_offset(base~, input),
          UnterminatedStringInVariableInterploation,
        )
        break (input, env.calc_offset(base~, input))
      }
      (re"^(?:\\[{])" as raw, after=rest) => {
        interpolation_write_view(env, buf, raw)
        if allow_interp && !escaped {
          let (rest, _) = scan_interpolation_source(
            rest,
            base~,
            env~,
            buf~,
            brace_depth=0,
            preserve=true,
          )
          continue rest, false
        } else {
          continue rest, false
        }
      }
      (re"^(?:.)" as c, after=rest) => {
        interpolation_write_char(env, buf, c)
        match (c.to_int(), escaped) {
          (34, false) =>
            // This switches back by calling, rather than continuing a shared
            // loop, so repeated string atoms accumulate mutual-recursion frames.
            break scan_interpolation_source(
              rest,
              base~,
              env~,
              buf~,
              brace_depth~,
              preserve~,
            )
          (92, false) => continue rest, true
          _ => continue rest, false
        }
      }
    }
  }
}

///|
fn scan_interpolation_char_atom(
  input : StringView,
  base~ : StringView,
  env~ : LexEnv,
  buf~ : StringBuilder,
  brace_depth~ : Int,
  preserve~ : Bool,
  escaped~ : Bool,
) -> (StringView, Int) {
  for input = input, escaped = escaped {
    lexmatch input with longest {
      re"^$" => {
        env.add_lexing_error(
          start=env.calc_offset(base~, input),
          end=env.calc_offset(base~, input),
          UnterminatedString,
        )
        break (input, env.calc_offset(base~, input))
      }
      (re"^" + RE_ASCII_LINE_BREAK, after=_) => {
        env.add_lexing_error(
          start=env.calc_offset(base~, input),
          end=env.calc_offset(base~, input),
          UnterminatedStringInVariableInterploation,
        )
        break (input, env.calc_offset(base~, input))
      }
      (re"^(?:.)" as c, after=rest) => {
        interpolation_write_char(env, buf, c)
        match (c.to_int(), escaped) {
          (39, false) =>
            break scan_interpolation_source(
              rest,
              base~,
              env~,
              buf~,
              brace_depth~,
              preserve~,
            )
          (92, false) => continue rest, true
          _ => continue rest, false
        }
      }
    }
  }
}

///|
fn lex_invalid_byte(
  input : StringView,
  base~ : StringView,
  env~ : LexEnv,
  start~ : Int,
) -> StringView {
  let invalid_byte_repr_buf = StringBuilder::new()
  fn add_invalid_byte(rest : StringView) -> StringView {
    let payload = invalid_byte_repr_buf.to_string()
    let end_pos = env.calc_offset(base~, rest)
    env.add_lexing_error(InvalidByteLiteral(payload), start~, end=end_pos)
    env.add_token_with_loc(BYTE(payload), start~, end=end_pos)
    rest
  }

  fn process_invalid_byte(input : StringView) -> StringView {
    for input = input {
      lexmatch input with longest {
        (re"^(?:')", after=rest) => break add_invalid_byte(rest)
        (re"^" + RE_ASCII_LINE_BREAK, after=_) => {
          env.add_lexing_error(
            start=env.calc_offset(base~, input),
            end=env.calc_offset(base~, input),
            UnterminatedStringInVariableInterploation,
          )
          break input
        }
        re"^$" => break add_invalid_byte(input)
        (re"^(?:.)" as c, after=rest) => {
          env.register_surrogate_pair_for_char(c)
          invalid_byte_repr_buf.write_char(c)
          continue rest
        }
      }
    }
  }

  process_invalid_byte(input)
}

///|
fn lex_string(
  input : StringView,
  base~ : StringView,
  env~ : LexEnv,
  end_with_newline~ : Bool,
  allow_interp~ : Bool,
  start_pos~ : Int,
) -> (StringView, Array[InterpElem]) {
  let string_repr_buf = StringBuilder::new()
  let interps = []
  fn add_literal(repr : String, start : Int, end : Int) -> Unit {
    interps.push(
      @tokens.InterpElem::InterpLit(repr~, loc={
        start: env.make_pos(start),
        end: env.make_pos(end),
      }),
    )
  }

  fn process_string(input : StringView, start_pos : Int) -> StringView {
    for input = input, start_pos = start_pos {
      lexmatch input with longest {
        (
          re"^(?:\")",
          // End of string
          after=rest,
        ) =>
          if end_with_newline {
            string_repr_buf.write_char('"')
            continue rest, start_pos
          } else {
            if !string_repr_buf.is_empty() {
              add_literal(
                string_repr_buf.to_string(),
                start_pos,
                env.calc_offset(base~, rest),
              )
            }
            break rest
          }

        // Escape sequences
        (re"^" + (RE_STRING_ESCAPE as raw), after=rest) => {
          string_repr_buf.write_string(raw.to_owned())
          continue rest, start_pos
        }

        // Hex escape
        (re"^" + (RE_HEX_ESCAPE as raw), after=rest) => {
          string_repr_buf.write_string(raw.to_owned())
          continue rest, start_pos
        }

        // Octal escape
        (re"^" + (RE_OCTAL_ESCAPE as raw), after=rest) => {
          string_repr_buf.write_string(raw.to_owned())
          continue rest, start_pos
        }

        // Unicode escape
        (re"^" + (RE_SHORT_UNICODE_ESCAPE as raw), after=rest) => {
          string_repr_buf.write_string(raw.to_owned())
          continue rest, start_pos
        }

        // Unicode escape with braces
        (re"^" + (RE_BRACED_UNICODE_ESCAPE as raw), after=rest) => {
          string_repr_buf.write_string(raw.to_owned())
          continue rest, start_pos
        }

        // String interpolation

        (re"^" + ((re"\\[{]" + RE_UNICODE_SPACES) as raw), after=rest) =>
          if allow_interp {
            if !string_repr_buf.is_empty() {
              add_literal(
                string_repr_buf.to_string(),
                start_pos,
                env.calc_offset(base~, input),
              )
            }
            string_repr_buf.reset()
            let interp_start_pos = env.make_pos(env.calc_offset(base~, rest))
            let (rest, interp_end) = scan_interpolation_source(
              rest,
              base~,
              env~,
              buf=string_repr_buf,
              brace_depth=0,
              preserve=false,
            )
            let loc = Location::{
              start: interp_start_pos,
              end: env.make_pos(interp_end),
            }
            if string_repr_buf.is_empty() {
              env.add_lexing_error(
                start=env.calc_offset(base~, input),
                end=env.calc_offset(base~, rest),
                InterpMissingExpression,
              )
            } else {
              let source = string_repr_buf.to_string()
              interps.push(InterpSource({ source, loc }))
            }
            string_repr_buf.reset()
            continue rest, env.calc_offset(base~, rest)
          } else {
            string_repr_buf.write_string(raw.to_owned())
            continue rest, env.calc_offset(base~, rest)
          }

        // EOF
        re"^$" => {
          env.add_lexing_error(
            start=start_pos,
            end=env.calc_offset(base~, input),
            UnterminatedString,
          )
          if !string_repr_buf.is_empty() {
            add_literal(
              string_repr_buf.to_string(),
              start_pos,
              env.calc_offset(base~, input),
            )
          }
          break input
        }

        // CRLF
        (re"^" + RE_ASCII_LINE_BREAK, after=rest) => {
          let end_pos = env.calc_offset(base~, rest)
          if !end_with_newline {
            env.add_lexing_error(
              start=start_pos,
              end=end_pos,
              UnterminatedString,
            )
          }
          if !string_repr_buf.is_empty() {
            add_literal(string_repr_buf.to_string(), start_pos, end_pos)
          }
          // Need to back off to handle NEWLINE token
          break input
        }

        // Any other character
        (re"^(?:.)" as c, after=rest) => {
          env.register_surrogate_pair_for_char(c)
          string_repr_buf.write_char(c)
          continue rest, start_pos
        }
      }
    }
  }

  let rest = process_string(input, start_pos)
  if interps.length() == 0 {
    let interps : Array[InterpElem] = [
      InterpLit(repr="", loc={
        start: env.make_pos(start_pos),
        end: env.make_pos(env.calc_offset(base~, rest)),
      }),
    ]
    (rest, interps)
  } else {
    (rest, interps)
  }
}