///|
/// Internal token categories. They are intentionally small: SQLSentinel
/// performs conservative structural checks rather than dialect-specific parsing.
priv enum TokenKind {
Word
StringLiteral
QuotedIdentifier
Number
Symbol
Directive
} derive(Eq)
///|
/// One lexical token with a 1-based source location.
priv struct Token {
kind : TokenKind
text : String
value : String
line : Int
column : Int
}
///|
fn is_word_start(ch : Char) -> Bool {
ch.is_ascii_alphabetic() || ch == '_'
}
///|
fn is_word_part(ch : Char) -> Bool {
ch.is_ascii_alphabetic() || ch.is_ascii_digit() || ch == '_' || ch == '$'
}
///|
fn is_identifier_quote(ch : Char) -> Bool {
ch == '"' || ch == '`' || ch == '['
}
///|
fn identifier_quote_end(ch : Char) -> Char {
match ch {
'[' => ']'
other => other
}
}
///|
fn read_quoted_identifier(chars : Array[Char], start : Int) -> (Int, String) {
let open = chars[start]
let close = identifier_quote_end(open)
let builder = StringBuilder()
let mut i = start + 1
while i < chars.length() {
let ch = chars[i]
if ch == close {
if close != ']' && i + 1 < chars.length() && chars[i + 1] == close {
builder.write_char(close)
i = i + 2
} else {
return (i + 1, builder.to_string())
}
} else {
builder.write_char(ch)
i = i + 1
}
}
(i, builder.to_string())
}
///|
fn read_string_literal(chars : Array[Char], start : Int) -> (Int, String) {
let builder = StringBuilder()
let mut i = start + 1
while i < chars.length() {
let ch = chars[i]
if ch == '\'' {
if i + 1 < chars.length() && chars[i + 1] == '\'' {
builder.write_char('\'')
i = i + 2
} else {
return (i + 1, builder.to_string())
}
} else if ch == '\\' && i + 1 < chars.length() {
builder.write_char(chars[i + 1])
i = i + 2
} else {
builder.write_char(ch)
i = i + 1
}
}
(i, builder.to_string())
}
///|
fn dollar_delimiter(chars : Array[Char], start : Int) -> String? {
let mut i = start + 1
while i < chars.length() {
let ch = chars[i]
if ch == '$' {
return Some(String::from_array(chars[start:i + 1]))
}
if !(ch.is_ascii_alphabetic() || ch.is_ascii_digit() || ch == '_') {
return None
}
i = i + 1
}
None
}
///|
fn read_dollar_string(
chars : Array[Char],
start : Int,
delimiter : String,
) -> (Int, String) {
let delimiter_chars = delimiter.to_array()
let body_start = start + delimiter_chars.length()
let mut i = body_start
while i + delimiter_chars.length() <= chars.length() {
let mut matches = true
for j in 0.. Token? {
let trimmed = content.trim()
if trimmed.to_lower().has_prefix("sqlsentinel") {
Some({
kind: Directive,
text: trimmed.to_owned(),
value: trimmed.to_lower().to_owned(),
line,
column,
})
} else {
None
}
}
///|
fn tokenize(source : String) -> Array[Token] {
let chars = source.to_array()
let tokens = []
let mut i = 0
let mut line = 1
let mut column = 1
while i < chars.length() {
let ch = chars[i]
if ch == '\n' {
line = line + 1
column = 1
i = i + 1
continue
}
if ch.is_whitespace() {
i = i + 1
column = column + 1
continue
}
if ch == '-' && i + 1 < chars.length() && chars[i + 1] == '-' {
let comment_line = line
let comment_column = column
let content = StringBuilder()
i = i + 2
column = column + 2
while i < chars.length() && chars[i] != '\n' {
content.write_char(chars[i])
i = i + 1
column = column + 1
}
match
directive_comment_token(
content.to_string(),
comment_line,
comment_column,
) {
Some(token) => tokens.push(token)
None => ()
}
continue
}
if ch == '#' && !(i + 1 < chars.length() && chars[i + 1] == '>') {
let comment_line = line
let comment_column = column
let content = StringBuilder()
i = i + 1
column = column + 1
while i < chars.length() && chars[i] != '\n' {
content.write_char(chars[i])
i = i + 1
column = column + 1
}
match
directive_comment_token(
content.to_string(),
comment_line,
comment_column,
) {
Some(token) => tokens.push(token)
None => ()
}
continue
}
if ch == '/' && i + 1 < chars.length() && chars[i + 1] == '*' {
let comment_line = line
let comment_column = column
let content_start = i + 2
let mut depth = 1
i = i + 2
column = column + 2
while i < chars.length() && depth > 0 {
if i + 1 < chars.length() && chars[i] == '/' && chars[i + 1] == '*' {
depth = depth + 1
i = i + 2
column = column + 2
} else if chars[i] == '*' &&
i + 1 < chars.length() &&
chars[i + 1] == '/' {
depth = depth - 1
i = i + 2
column = column + 2
} else if chars[i] == '\n' {
line = line + 1
column = 1
i = i + 1
} else {
i = i + 1
column = column + 1
}
}
let content_end = if depth == 0 && i >= 2 { i - 2 } else { i }
match
directive_comment_token(
String::from_array(chars[content_start:content_end]),
comment_line,
comment_column,
) {
Some(token) => tokens.push(token)
None => ()
}
continue
}
let token_line = line
let token_column = column
if is_word_start(ch) {
let start = i
while i < chars.length() && is_word_part(chars[i]) {
i = i + 1
column = column + 1
}
let text = String::from_array(chars[start:i])
tokens.push({
kind: Word,
text,
value: text.to_upper(),
line: token_line,
column: token_column,
})
continue
}
if ch == '\'' {
let (next, value) = read_string_literal(chars, i)
let length = next - i
i = next
column = column + length
tokens.push({
kind: StringLiteral,
text: "''",
value,
line: token_line,
column: token_column,
})
continue
}
if is_identifier_quote(ch) {
let (next, value) = read_quoted_identifier(chars, i)
let length = next - i
i = next
column = column + length
tokens.push({
kind: QuotedIdentifier,
text: String::from_array(chars[i - length:i]),
value,
line: token_line,
column: token_column,
})
continue
}
if ch == '$' {
match dollar_delimiter(chars, i) {
Some(delimiter) => {
let (next, value) = read_dollar_string(chars, i, delimiter)
let length = next - i
i = next
column = column + length
tokens.push({
kind: StringLiteral,
text: "$$",
value,
line: token_line,
column: token_column,
})
continue
}
None => ()
}
}
if ch.is_ascii_digit() {
let start = i
while i < chars.length() &&
(
chars[i].is_ascii_digit() ||
chars[i] == '.' ||
chars[i] == 'e' ||
chars[i] == 'E' ||
chars[i] == '+' ||
chars[i] == '-'
) {
i = i + 1
column = column + 1
}
let text = String::from_array(chars[start:i])
tokens.push({
kind: Number,
text,
value: text,
line: token_line,
column: token_column,
})
continue
}
let mut symbol = ch.to_string()
if i + 1 < chars.length() {
let pair = String::from_array(chars[i:i + 2])
match pair {
"<=" | ">=" | "<>" | "!=" | "::" | "->" | "=>" | "||" | "&&" | "#>" => {
symbol = pair
i = i + 1
column = column + 1
}
_ => ()
}
}
tokens.push({
kind: Symbol,
text: symbol,
value: symbol,
line: token_line,
column: token_column,
})
i = i + 1
column = column + 1
}
tokens
}
///|
fn split_statements(tokens : Array[Token]) -> Array[Array[Token]] {
let statements = []
let mut current = []
for token in tokens {
if token.kind == Directive {
continue
} else if token.kind == Symbol && token.text == ";" {
if !current.is_empty() {
statements.push(current)
current = []
}
} else {
current.push(token)
}
}
if !current.is_empty() {
statements.push(current)
}
statements
}
///|
fn token_is_word(token : Token, word : String) -> Bool {
token.kind == Word && token.value == word
}
///|
fn token_is_symbol(token : Token, symbol : String) -> Bool {
token.kind == Symbol && token.text == symbol
}