///|
priv struct Lexer {
chars : Array[Char]
mut cursor : Int
}
///|
fn Lexer::new(source : String) -> Lexer {
{ chars: source.to_array(), cursor: 0 }
}
///|
fn Lexer::at_end(self : Lexer) -> Bool {
self.cursor >= self.chars.length()
}
///|
fn Lexer::peek(self : Lexer) -> Char? {
self.chars.get(self.cursor)
}
///|
fn Lexer::peek_next(self : Lexer) -> Char? {
self.chars.get(self.cursor + 1)
}
///|
fn Lexer::advance(self : Lexer) -> Char {
let ch = self.chars[self.cursor]
self.cursor = self.cursor + 1
ch
}
///|
fn Lexer::matches(self : Lexer, expected : Char) -> Bool {
if self.peek() == Some(expected) {
self.cursor = self.cursor + 1
true
} else {
false
}
}
///|
fn is_identifier_start(ch : Char) -> Bool {
ch.is_ascii_alphabetic() || ch == '_'
}
///|
fn is_identifier_continue(ch : Char) -> Bool {
is_identifier_start(ch) || ch.is_ascii_digit()
}
///|
fn Lexer::skip_space(self : Lexer) -> Unit {
while self.peek().map(fn(ch) { ch.is_ascii_whitespace() }).unwrap_or(false) {
self.cursor = self.cursor + 1
}
}
///|
fn Lexer::identifier(self : Lexer, start : Int) -> Token {
while self.peek().map(is_identifier_continue).unwrap_or(false) {
self.cursor = self.cursor + 1
}
let text = self.chars[start:self.cursor] |> String::from_array
let kind = match text {
"true" => True
"false" => False
"null" => Null
"in" => In
_ => Identifier(text)
}
Token::new(kind, start, self.cursor)
}
///|
fn Lexer::number(self : Lexer, start : Int) -> Token {
while self.peek().map(fn(ch) { ch.is_ascii_digit() }).unwrap_or(false) {
self.cursor = self.cursor + 1
}
if self.peek() == Some('.') &&
self.peek_next().map(fn(ch) { ch.is_ascii_digit() }).unwrap_or(false) {
self.cursor = self.cursor + 1
while self.peek().map(fn(ch) { ch.is_ascii_digit() }).unwrap_or(false) {
self.cursor = self.cursor + 1
}
}
if self.peek() == Some('e') || self.peek() == Some('E') {
let exponent_start = self.cursor
self.cursor = self.cursor + 1
if self.peek() == Some('+') || self.peek() == Some('-') {
self.cursor = self.cursor + 1
}
if self.peek().map(fn(ch) { ch.is_ascii_digit() }).unwrap_or(false) {
while self.peek().map(fn(ch) { ch.is_ascii_digit() }).unwrap_or(false) {
self.cursor = self.cursor + 1
}
} else {
self.cursor = exponent_start
}
}
let text = self.chars[start:self.cursor] |> String::from_array
Token::new(Number(text), start, self.cursor)
}
///|
fn Lexer::string_literal(self : Lexer, start : Int) -> Token raise RuleFailure {
let builder = StringBuilder()
while !self.at_end() {
let ch = self.advance()
if ch == '"' {
return Token::new(StringLiteral(builder.to_string()), start, self.cursor)
}
if ch == '\\' {
if self.at_end() {
break
}
let escaped = self.advance()
match escaped {
'"' => builder.write_char('"')
'\\' => builder.write_char('\\')
'/' => builder.write_char('/')
'b' => builder.write_char('\b')
'f' => builder.write_char('\f')
'n' => builder.write_char('\n')
'r' => builder.write_char('\r')
't' => builder.write_char('\t')
_ =>
raise RuleFailure(
Diagnostic::new(
Lex,
"L002",
"unsupported escape sequence",
Span::new(self.cursor - 2, self.cursor),
hint="Use one of: \\\" \\\\ \\/ \\b \\f \\n \\r \\t",
),
)
}
} else {
builder.write_char(ch)
}
}
raise RuleFailure(
Diagnostic::new(
Lex,
"L001",
"unterminated string literal",
Span::new(start, self.cursor),
hint="Add a closing double quote.",
),
)
}
///|
fn Lexer::next_token(self : Lexer) -> Token raise RuleFailure {
self.skip_space()
if self.at_end() {
return Token::new(Eof, self.cursor, self.cursor)
}
let start = self.cursor
let ch = self.advance()
if is_identifier_start(ch) {
return self.identifier(start)
}
if ch.is_ascii_digit() {
return self.number(start)
}
let kind = match ch {
'&' if self.matches('&') => TokenKind::And
'|' if self.matches('|') => TokenKind::Or
'!' if self.matches('=') => BangEqual
'!' => Bang
'=' if self.matches('=') => EqualEqual
'<' if self.matches('=') => LessEqual
'<' => Less
'>' if self.matches('=') => GreaterEqual
'>' => Greater
'+' => Plus
'-' => Minus
'*' => Star
'/' => Slash
'%' => Percent
'(' => LeftParen
')' => RightParen
'[' => LeftBracket
']' => RightBracket
',' => Comma
'.' => Dot
'?' if self.matches('.') => QuestionDot
'?' if self.matches('[') => QuestionLeftBracket
'"' => return self.string_literal(start)
_ =>
raise RuleFailure(
Diagnostic::new(
Lex,
"L003",
"unexpected character `\{ch}`",
Span::new(start, self.cursor),
),
)
}
Token::new(kind, start, self.cursor)
}
///|
fn lex(source : String) -> Array[Token] raise RuleFailure {
let lexer = Lexer::new(source)
let tokens = []
while true {
let token = lexer.next_token()
tokens.push(token)
if token.kind is Eof {
break
}
}
tokens
}