///|
/// JSON tokens
enum Token {
TNull
TTrue
TFalse
TNumber(Double)
TString(String)
TLeftBrace // {
TRightBrace // }
TLeftBracket // [
TRightBracket // ]
TColon // :
TComma // ,
TEof
} derive(Show, Eq)
///|
/// Lexer state
struct Lexer {
input : String
mut pos : Int
mut line : Int
mut column : Int
}
///|
/// Create new lexer
fn Lexer::new(input : String) -> Lexer {
{ input, pos: 0, line: 1, column: 1 }
}
///|
/// Get current position
fn Lexer::position(self : Lexer) -> Position {
{ line: self.line, column: self.column }
}
///|
/// Peek current character
fn Lexer::peek(self : Lexer) -> Char? {
if self.pos >= self.input.length() {
None
} else {
self.input.get_char(self.pos)
}
}
///|
/// Peek at offset
fn Lexer::peek_at(self : Lexer, offset : Int) -> Char? {
let pos = self.pos + offset
if pos >= self.input.length() {
None
} else {
self.input.get_char(pos)
}
}
///|
/// Consume current character
fn Lexer::next(self : Lexer) -> Char? {
match self.peek() {
Some(ch) => {
self.pos += 1
if ch == '\n' {
self.line += 1
self.column = 1
} else {
self.column += 1
}
Some(ch)
}
None => None
}
}
///|
/// Skip whitespace
fn Lexer::skip_whitespace(self : Lexer) -> Unit {
while true {
match self.peek() {
Some(' ') | Some('\t') | Some('\r') | Some('\n') => {
self.next() |> ignore
continue
}
_ => break
}
}
}
///|
/// Check if character is digit
fn is_digit(ch : Char) -> Bool {
ch >= '0' && ch <= '9'
}
///|
/// Parse string with escape sequences
fn Lexer::parse_string(self : Lexer) -> String raise JsonError {
self.next() |> ignore // consume opening "
let buf = @buffer.new()
while true {
match self.next() {
Some('"') => break buf.to_string()
Some('\\') =>
match self.next() {
Some('"') => {
buf.write_char('"')
continue
}
Some('\\') => {
buf.write_char('\\')
continue
}
Some('/') => {
buf.write_char('/')
continue
}
Some('b') => {
buf.write_char('\b')
continue
}
Some('f') => {
buf.write_char('\u000C')
continue
}
Some('n') => {
buf.write_char('\n')
continue
}
Some('r') => {
buf.write_char('\r')
continue
}
Some('t') => {
buf.write_char('\t')
continue
}
Some('u') => {
// Unicode escape \uXXXX
let hex = self.parse_unicode_escape()
buf.write_char(hex)
continue
}
Some(ch) => {
let pos = self.position()
let msg = "Invalid escape: \\\{ch}"
raise JsonError::InvalidEscape(pos, msg)
}
None => raise JsonError::UnexpectedEof(self.position())
}
Some(ch) if ch >= '\u0000' && ch <= '\u001F' => {
// Control characters must be escaped
let pos = self.position()
raise JsonError::UnexpectedChar(pos, ch)
}
Some(ch) => {
buf.write_char(ch)
continue
}
None => raise JsonError::UnexpectedEof(self.position())
}
} else {
""
}
}
///|
/// Parse unicode escape sequence \uXXXX
fn Lexer::parse_unicode_escape(self : Lexer) -> Char raise JsonError {
let mut code = 0
for i = 0; i < 4; i = i + 1 {
match self.next() {
Some(ch) if ch >= '0' && ch <= '9' =>
code = code * 16 + (ch.to_int() - 48)
Some(ch) if ch >= 'a' && ch <= 'f' =>
code = code * 16 + (ch.to_int() - 97 + 10)
Some(ch) if ch >= 'A' && ch <= 'F' =>
code = code * 16 + (ch.to_int() - 65 + 10)
Some(ch) => {
let pos = self.position()
let msg = "Invalid hex digit: \{ch}"
raise JsonError::InvalidUnicode(pos, msg)
}
None => raise JsonError::UnexpectedEof(self.position())
}
}
code.unsafe_to_char()
}
///|
/// Parse number
fn Lexer::parse_number(self : Lexer) -> Double raise JsonError {
let buf = @buffer.new()
let pos = self.position()
// Optional minus
if self.peek() == Some('-') {
buf.write_char('-')
self.next() |> ignore
}
// Integer part
match self.peek() {
Some('0') => {
buf.write_char('0')
self.next() |> ignore
}
Some(ch) if ch >= '1' && ch <= '9' =>
while true {
match self.peek() {
Some(ch) if is_digit(ch) => {
buf.write_char(ch)
self.next() |> ignore
continue
}
_ => break
}
}
_ => {
let msg = "Expected digit"
raise JsonError::InvalidNumber(pos, msg)
}
}
// Fractional part
if self.peek() == Some('.') {
buf.write_char('.')
self.next() |> ignore
let mut has_digit = false
while true {
match self.peek() {
Some(ch) if is_digit(ch) => {
has_digit = true
buf.write_char(ch)
self.next() |> ignore
continue
}
_ => break
}
}
if not(has_digit) {
let msg = "Expected digit after decimal point"
raise JsonError::InvalidNumber(pos, msg)
}
}
// Exponent part
match self.peek() {
Some('e') | Some('E') => {
buf.write_char('e')
self.next() |> ignore
match self.peek() {
Some('+') | Some('-') => {
let ch = self.next().unwrap()
buf.write_char(ch)
}
_ => ()
}
let mut has_digit = false
while true {
match self.peek() {
Some(ch) if is_digit(ch) => {
has_digit = true
buf.write_char(ch)
self.next() |> ignore
continue
}
_ => break
}
}
if not(has_digit) {
let msg = "Expected digit in exponent"
raise JsonError::InvalidNumber(pos, msg)
}
}
_ => ()
}
let s = buf.to_string()
try @strconv.parse_double(s) catch {
_ => {
let msg = "Invalid number: \{s}"
raise JsonError::InvalidNumber(pos, msg)
}
} noraise {
n => n
}
}
///|
/// Get next token
pub fn Lexer::next_token(self : Lexer) -> Token raise JsonError {
self.skip_whitespace()
match self.peek() {
None => TEof
Some('{') => {
self.next() |> ignore
TLeftBrace
}
Some('}') => {
self.next() |> ignore
TRightBrace
}
Some('[') => {
self.next() |> ignore
TLeftBracket
}
Some(']') => {
self.next() |> ignore
TRightBracket
}
Some(':') => {
self.next() |> ignore
TColon
}
Some(',') => {
self.next() |> ignore
TComma
}
Some('"') => {
let s = self.parse_string()
TString(s)
}
Some('-') => {
let n = self.parse_number()
TNumber(n)
}
Some(ch) if is_digit(ch) => {
let n = self.parse_number()
TNumber(n)
}
Some('n') =>
if self.peek_at(1) == Some('u') &&
self.peek_at(2) == Some('l') &&
self.peek_at(3) == Some('l') {
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
TNull
} else {
let pos = self.position()
let ch = self.next().unwrap()
raise JsonError::UnexpectedChar(pos, ch)
}
Some('t') =>
if self.peek_at(1) == Some('r') &&
self.peek_at(2) == Some('u') &&
self.peek_at(3) == Some('e') {
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
TTrue
} else {
let pos = self.position()
let ch = self.next().unwrap()
raise JsonError::UnexpectedChar(pos, ch)
}
Some('f') =>
if self.peek_at(1) == Some('a') &&
self.peek_at(2) == Some('l') &&
self.peek_at(3) == Some('s') &&
self.peek_at(4) == Some('e') {
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
self.next() |> ignore
TFalse
} else {
let pos = self.position()
let ch = self.next().unwrap()
raise JsonError::UnexpectedChar(pos, ch)
}
Some(ch) => {
let pos = self.position()
raise JsonError::UnexpectedChar(pos, ch)
}
}
}