///|
pub suberror ParseError {
UnexpectedToken(Token)
LexError(Loc, String)
} derive(Eq)
///|
impl Show for ParseError with output(self, logger) -> Unit {
match self {
UnexpectedToken(tok) => logger.write_string("unexpected token: \{tok}")
LexError(loc, msg) => logger.write_string("\{loc}: \{msg}")
}
}
///|
priv struct Lexer {
chars : Array[Char]
mut pos : Int
mut line : Int
mut line_start : Int
}
///|
fn Lexer::new(input : String) -> Lexer {
{ chars: input.to_array(), pos: 0, line: 1, line_start: 0 }
}
///|
fn Lexer::current(self : Lexer) -> Char? {
if self.pos < self.chars.length() {
Some(self.chars[self.pos])
} else {
None
}
}
///|
fn Lexer::advance(self : Lexer) -> Char? {
let ch = self.current()
if ch is Some(_) {
self.pos += 1
}
ch
}
///|
fn Lexer::peek(self : Lexer, offset : Int) -> Char? {
let idx = self.pos + offset
if idx < self.chars.length() {
Some(self.chars[idx])
} else {
None
}
}
///|
fn Lexer::loc(self : Lexer, start_pos : Int) -> Loc {
let start_col = start_pos - self.line_start + 1
let end_col = self.pos - self.line_start + 1
{
start: { line: self.line, column: start_col },
end: { line: self.line, column: end_col },
}
}
///|
fn Lexer::chars_slice(self : Lexer, start : Int, end : Int) -> String {
let buf = StringBuilder::new()
for i in start.. Unit {
while true {
match self.current() {
Some(' ') | Some('\t') | Some('\f') | Some('\r') =>
self.advance() |> ignore
Some('\n') => {
self.advance() |> ignore
self.line = self.line + 1
self.line_start = self.pos
}
Some('/') =>
match self.peek(1) {
Some('/') => self.skip_comment()
_ => break
}
_ => break
}
}
}
///|
fn Lexer::skip_comment(self : Lexer) -> Unit {
// skip //
self.advance() |> ignore
self.advance() |> ignore
while !(self.current() is (Some('\n') | None)) {
self.advance() |> ignore
}
}
///|
fn Lexer::tokenize(self : Lexer) -> Array[Token] raise ParseError {
let tokens : Array[Token] = []
while true {
self.skip_whitespace()
if self.pos >= self.chars.length() {
break
}
let token = self.next_token()
tokens.push(token)
}
let eof_col = self.pos - self.line_start + 1
tokens.push(
Eof({
start: { line: self.line, column: eof_col },
end: { line: self.line, column: eof_col },
}),
)
tokens
}
///|
fn Lexer::next_token(self : Lexer) -> Token raise ParseError {
let start_pos = self.pos
let ch = self.advance()
match ch {
Some('[') => LBracket(self.loc(start_pos))
Some(']') => RBracket(self.loc(start_pos))
Some(',') => Comma(self.loc(start_pos))
Some(':') => Colon(self.loc(start_pos))
Some('{') => LBrace(self.loc(start_pos))
Some('}') => RBrace(self.loc(start_pos))
Some('=') => Equal(self.loc(start_pos))
Some('(') => LParen(self.loc(start_pos))
Some(')') => RParen(self.loc(start_pos))
Some(';') => Semi(self.loc(start_pos))
Some('"') => {
self.pos = start_pos
self.lex_string()
}
Some('@') => {
self.pos = start_pos
self.lex_pkg_name()
}
Some('-') =>
match self.current() {
Some(c) if c.is_ascii_digit() => {
self.pos = start_pos
self.lex_int()
}
_ => raise UnexpectedToken(LIdent(self.loc(start_pos), "-"))
}
Some(c) =>
if c.is_ascii_digit() {
self.pos = start_pos
self.lex_int()
} else if c.is_ascii_alphabetic() || c == '_' {
self.pos = start_pos
self.lex_ident()
} else {
raise LexError(self.loc(start_pos), "unexpected character '\{c}'")
}
None => raise LexError(self.loc(start_pos), "unexpected end of input")
}
}
///|
fn Lexer::lex_string(self : Lexer) -> Token raise ParseError {
let start_pos = self.pos
// skip opening "
self.advance() |> ignore
let buf = StringBuilder::new()
while self.pos < self.chars.length() && self.current() != Some('"') {
if self.current() == Some('\n') {
raise LexError(self.loc(start_pos), "unterminated string literal")
}
match self.advance() {
Some('\\') => {
let escaped = self.unescape()
buf.write_char(escaped)
}
Some(c) => buf.write_char(c)
None => raise LexError(self.loc(start_pos), "unterminated string literal")
}
}
if self.pos >= self.chars.length() {
raise LexError(self.loc(start_pos), "unterminated string literal")
}
// skip closing "
self.advance() |> ignore
let loc = self.loc(start_pos)
String(loc, buf.to_string())
}
///|
fn Lexer::unescape(self : Lexer) -> Char raise ParseError {
let err_loc : Loc = {
start: { line: self.line, column: 0 },
end: { line: self.line, column: 0 },
}
match self.advance() {
Some('n') => '\n'
Some('t') => '\t'
Some('r') => '\r'
Some('f') => '\u{0C}'
Some('b') => '\u{08}'
Some('\\') => '\\'
Some('"') => '"'
Some('/') => '/'
Some('u') => {
let buf = StringBuilder::new()
for _ in 0..<4 {
match self.advance() {
Some(c) => buf.write_char(c)
None =>
raise LexError(err_loc, "unexpected end of input in unicode escape")
}
}
let hex = buf.to_string()
let code = @string.parse_int(hex.to_string_view(), base=16) catch {
_ => raise LexError(err_loc, "invalid hex escape: \\u\{hex}")
}
match code.to_char() {
Some(c) => c
None => raise LexError(err_loc, "invalid unicode code point: \{hex}")
}
}
Some(c) => raise LexError(err_loc, "invalid escape sequence: \\\{c}")
None =>
raise LexError(err_loc, "unexpected end of input in escape sequence")
}
}
///|
fn Lexer::lex_int(self : Lexer) -> Token raise ParseError {
let start_pos = self.pos
let neg = if self.current() == Some('-') {
self.advance() |> ignore
true
} else {
false
}
let num_start = self.pos
while self.pos < self.chars.length() {
match self.current() {
Some(c) if c.is_ascii_digit() => self.advance() |> ignore
_ => break
}
}
let digits = self.chars_slice(num_start, self.pos)
let n = @string.parse_int(digits.to_string_view()) catch {
_ => raise LexError(self.loc(start_pos), "invalid integer literal")
}
let val = if neg { -n } else { n }
Int(self.loc(start_pos), val)
}
///|
fn Lexer::lex_ident(self : Lexer) -> Token {
let start_pos = self.pos
while self.pos < self.chars.length() {
match self.current() {
Some(c) if c.is_ascii_alphabetic() || c == '_' || c.is_ascii_digit() =>
self.advance() |> ignore
_ => break
}
}
let word = self.chars_slice(start_pos, self.pos)
let loc = self.loc(start_pos)
match word {
"true" => True(loc)
"false" => False(loc)
"for" => For(loc)
"as" => As(loc)
"import" => Import(loc)
_ => LIdent(loc, word)
}
}
///|
fn Lexer::lex_pkg_name(self : Lexer) -> Token {
let start_pos = self.pos
// skip @
self.advance() |> ignore
while self.pos < self.chars.length() {
match self.current() {
Some(c) if c.is_ascii_alphabetic() ||
c == '_' ||
c.is_ascii_digit() ||
c == '/' => self.advance() |> ignore
_ => break
}
}
let name = self.chars_slice(start_pos + 1, self.pos)
PkgName(self.loc(start_pos), name)
}
///|
fn tokenize(input : String) -> Array[Token] raise ParseError {
let lexer = Lexer::new(input)
lexer.tokenize()
}