///|
/// Stable categories reported by the strict Query-string parser.
pub(all) enum QueryStringErrorKind {
EmptyQuery
UnexpectedToken(String)
UnsupportedSyntax(String)
UnterminatedPhrase
DanglingEscape
InvalidBoost(String)
InvalidFieldValue(String, String)
UnknownField(String)
InvalidDefaultField(Int)
FieldNotIndexed(String)
NoDefaultField
AnalysisFailure(String)
NegativeOnlyQuery
LimitExceeded(String, Int)
} derive(Eq, @debug.Debug)
///|
/// Query-string errors carry half-open UTF-8 byte offsets into the original
/// input. The offsets remain portable across native, JS, Wasm, and WasmGC.
pub(all) suberror QueryStringError {
Failure(QueryStringErrorKind, Int, Int)
} derive(Eq, @debug.Debug)
///|
fn query_string_error_message(kind : QueryStringErrorKind) -> String {
match kind {
EmptyQuery => "query string must not be empty"
UnexpectedToken(token) => "unexpected query token: \{token}"
UnsupportedSyntax(syntax) => "unsupported query syntax: \{syntax}"
UnterminatedPhrase => "unterminated quoted phrase"
DanglingEscape => "query string ends with an incomplete escape"
InvalidBoost(value) => "invalid query boost: \{value}"
InvalidFieldValue(field, value) =>
"invalid value for field \{field}: \{value}"
UnknownField(field) => "unknown field: \{field}"
InvalidDefaultField(field_id) => "invalid default field id: \{field_id}"
FieldNotIndexed(field) => "field is not indexed: \{field}"
NoDefaultField => "query string has no default field"
AnalysisFailure(message) => "query analysis failed: \{message}"
NegativeOnlyQuery => "negative-only query is not supported"
LimitExceeded(name, limit) => "query string exceeds \{name} limit: \{limit}"
}
}
///|
pub impl Show for QueryStringError with fn output(self, logger) {
match self {
Failure(kind, start_offset, end_offset) =>
logger.write_string(
"\{query_string_error_message(kind)} at bytes \{start_offset}..\{end_offset}",
)
}
}
///|
pub fn QueryStringError::kind(self : QueryStringError) -> QueryStringErrorKind {
match self {
Failure(kind, _, _) => kind
}
}
///|
pub fn QueryStringError::start_offset(self : QueryStringError) -> Int {
match self {
Failure(_, start_offset, _) => start_offset
}
}
///|
pub fn QueryStringError::end_offset(self : QueryStringError) -> Int {
match self {
Failure(_, _, end_offset) => end_offset
}
}
///|
fn[T] query_string_fail(
kind : QueryStringErrorKind,
start_offset : Int,
end_offset : Int,
) -> T raise QueryStringError {
raise QueryStringError::Failure(kind, start_offset, end_offset)
}
///|
priv enum QueryTokenKind {
Word(String)
Phrase(String)
RegexLiteral(String)
And
Or
Not
Plus
Minus
LeftParen
RightParen
Colon
Caret
Tilde
LeftBracket
RightBracket
LeftBrace
RightBrace
Star
End
} derive(Eq)
///|
priv struct QueryToken {
kind : QueryTokenKind
start_offset : Int
end_offset : Int
}
///|
priv struct QueryCharacter {
value : Char
start_offset : Int
end_offset : Int
}
///|
fn query_string_utf8_width(character : Char) -> Int {
let code = character.to_int()
if code <= 0x7F {
1
} else if code <= 0x7FF {
2
} else if code <= 0xFFFF {
3
} else {
4
}
}
///|
fn query_characters(text : String) -> Array[QueryCharacter] {
let characters : Array[QueryCharacter] = []
let mut offset = 0
for value in text {
let next_offset = offset + query_string_utf8_width(value)
characters.push({ value, start_offset: offset, end_offset: next_offset })
offset = next_offset
}
characters
}
///|
fn query_character_string(character : Char) -> String {
String::from_array([character])
}
///|
fn unsupported_query_character(character : Char) -> Bool {
character == '&' || character == '|' || character == '!'
}
///|
fn reserved_query_character(character : Char) -> Bool {
character == '"' ||
character == '\\' ||
character == '+' ||
character == '-' ||
character == '(' ||
character == ')' ||
character == ':' ||
character == '^' ||
character == '~' ||
character == '/' ||
character == '[' ||
character == ']' ||
character == '{' ||
character == '}' ||
unsupported_query_character(character)
}
///|
fn query_token_label(token : QueryToken) -> String {
match token.kind {
Word(value) => value
Phrase(value) => "\"\{value}\""
RegexLiteral(value) => "/\{value}/"
And => "AND"
Or => "OR"
Not => "NOT"
Plus => "+"
Minus => "-"
LeftParen => "("
RightParen => ")"
Colon => ":"
Caret => "^"
Tilde => "~"
LeftBracket => "["
RightBracket => "]"
LeftBrace => "{"
RightBrace => "}"
Star => "*"
End => "end of query"
}
}
///|
fn lex_query_string(text : String) -> Array[QueryToken] raise QueryStringError {
let characters = query_characters(text)
let total_bytes = if characters.length() == 0 {
0
} else {
characters[characters.length() - 1].end_offset
}
guard total_bytes <= 4096 else {
query_string_fail(LimitExceeded("input bytes", 4096), 0, total_bytes)
}
let tokens : Array[QueryToken] = []
let mut cursor = 0
while cursor < characters.length() {
let current = characters[cursor]
let character = current.value
if character.is_whitespace() {
cursor += 1
continue
}
if unsupported_query_character(character) {
query_string_fail(
UnsupportedSyntax(query_character_string(character)),
current.start_offset,
current.end_offset,
)
}
let simple_kind : QueryTokenKind? = match character {
'+' => Some(Plus)
'-' => Some(Minus)
'(' => Some(LeftParen)
')' => Some(RightParen)
':' => Some(Colon)
'^' => Some(Caret)
'[' => Some(LeftBracket)
']' => Some(RightBracket)
'{' => Some(LeftBrace)
'}' => Some(RightBrace)
'~' => Some(Tilde)
_ => None
}
match simple_kind {
Some(kind) => {
tokens.push({
kind,
start_offset: current.start_offset,
end_offset: current.end_offset,
})
cursor += 1
continue
}
None => ()
}
if character == '/' {
let start_offset = current.start_offset
let values : Array[Char] = []
cursor += 1
let mut closed = false
let mut end_offset = current.end_offset
while cursor < characters.length() {
let regex_character = characters[cursor]
if regex_character.value == '/' {
closed = true
end_offset = regex_character.end_offset
cursor += 1
break
}
if regex_character.value == '\\' && cursor + 1 < characters.length() {
let next = characters[cursor + 1]
if next.value != '/' {
values.push('\\')
}
values.push(next.value)
end_offset = next.end_offset
cursor += 2
continue
}
values.push(regex_character.value)
end_offset = regex_character.end_offset
cursor += 1
}
guard closed else {
query_string_fail(
UnsupportedSyntax("unterminated regular expression"),
start_offset,
total_bytes,
)
}
guard values.length() > 0 else {
query_string_fail(
UnexpectedToken("empty regex"),
start_offset,
end_offset,
)
}
tokens.push({
kind: RegexLiteral(String::from_array(values)),
start_offset,
end_offset,
})
continue
}
if character == '"' {
let start_offset = current.start_offset
let values : Array[Char] = []
cursor += 1
let mut closed = false
let mut end_offset = current.end_offset
while cursor < characters.length() {
let phrase_character = characters[cursor]
if phrase_character.value == '"' {
closed = true
end_offset = phrase_character.end_offset
cursor += 1
break
}
if phrase_character.value == '\\' {
guard cursor + 1 < characters.length() else {
query_string_fail(
DanglingEscape,
phrase_character.start_offset,
phrase_character.end_offset,
)
}
values.push(characters[cursor + 1].value)
end_offset = characters[cursor + 1].end_offset
cursor += 2
continue
}
values.push(phrase_character.value)
end_offset = phrase_character.end_offset
cursor += 1
}
guard closed else {
query_string_fail(UnterminatedPhrase, start_offset, total_bytes)
}
guard values.length() > 0 else {
query_string_fail(
UnexpectedToken("empty phrase"),
start_offset,
end_offset,
)
}
tokens.push({
kind: Phrase(String::from_array(values)),
start_offset,
end_offset,
})
continue
}
let start_offset = current.start_offset
let values : Array[Char] = []
let mut end_offset = current.end_offset
let mut escaped = false
while cursor < characters.length() {
let word_character = characters[cursor]
if word_character.value.is_whitespace() ||
(
reserved_query_character(word_character.value) &&
word_character.value != '\\'
) {
break
}
if word_character.value == '\\' {
guard cursor + 1 < characters.length() else {
query_string_fail(
DanglingEscape,
word_character.start_offset,
word_character.end_offset,
)
}
let escaped_character = characters[cursor + 1].value
if escaped_character == '*' || escaped_character == '?' {
values.push('\\')
}
values.push(escaped_character)
end_offset = characters[cursor + 1].end_offset
escaped = true
cursor += 2
continue
}
values.push(word_character.value)
end_offset = word_character.end_offset
cursor += 1
}
guard values.length() > 0 else {
query_string_fail(
UnexpectedToken(query_character_string(character)),
current.start_offset,
current.end_offset,
)
}
let value = String::from_array(values)
let kind = if !escaped {
match value {
"AND" => And
"OR" => Or
"NOT" => Not
_ => Word(value)
}
} else {
Word(value)
}
tokens.push({ kind, start_offset, end_offset })
}
tokens.push({ kind: End, start_offset: total_bytes, end_offset: total_bytes })
tokens
}
///|
pub(all) enum QueryLiteralMode {
Plain
PhraseMode
RegexMode
} derive(Eq, @debug.Debug)
///|
pub(all) enum UserQueryNodeKind {
Literal(String, QueryLiteralMode)
Range(String?, String?, Bool, Bool)
FieldScope(String, Int, Int, Int)
Conjunction(Array[Int])
Disjunction(Array[Int])
Required(Int)
Prohibited(Int)
Boost(Int, Double)
Proximity(Int, Int)
Group(Int)
}
///|
pub struct UserQueryNode {
kind : UserQueryNodeKind
start_offset : Int
end_offset : Int
}
///|
/// Schema-independent syntax tree. Its representation is intentionally opaque;
/// applications can parse once and bind it against a parser configuration.
pub struct UserQueryAst {
nodes : ReadOnlyArray[UserQueryNode]
root : Int
source_length : Int
}
///|
pub fn UserQueryAst::node_count(self : UserQueryAst) -> Int {
self.nodes.length()
}
///|
pub fn UserQueryAst::source_length(self : UserQueryAst) -> Int {
self.source_length
}
///|
priv struct QuerySyntaxParser {
tokens : ReadOnlyArray[QueryToken]
mut cursor : Int
nodes : Array[UserQueryNode]
conjunction_by_default : Bool
}
///|
fn QuerySyntaxParser::peek(self : QuerySyntaxParser) -> QueryToken {
self.tokens[self.cursor]
}
///|
fn QuerySyntaxParser::advance(self : QuerySyntaxParser) -> QueryToken {
let token = self.peek()
if self.cursor + 1 < self.tokens.length() {
self.cursor += 1
}
token
}
///|
fn QuerySyntaxParser::push_node(
self : QuerySyntaxParser,
kind : UserQueryNodeKind,
start_offset : Int,
end_offset : Int,
) -> Int raise QueryStringError {
guard self.nodes.length() < 512 else {
query_string_fail(LimitExceeded("AST nodes", 512), start_offset, end_offset)
}
let index = self.nodes.length()
self.nodes.push({ kind, start_offset, end_offset })
index
}
///|
fn token_starts_clause(token : QueryToken) -> Bool {
match token.kind {
Word(_)
| Phrase(_)
| RegexLiteral(_)
| Plus
| Minus
| Not
| LeftParen
| LeftBracket
| LeftBrace => true
_ => false
}
}
///|
fn parse_boost_value(token : QueryToken) -> Double raise QueryStringError {
let value = match token.kind {
Word(value) => value
_ =>
query_string_fail(
InvalidBoost(query_token_label(token)),
token.start_offset,
token.end_offset,
)
}
let characters = value.to_array()
let mut result = 0.0
let mut fractional_divisor = 0.0
let mut saw_digit = false
let mut saw_dot = false
for character in characters {
if character == '.' {
guard !saw_dot else {
query_string_fail(
InvalidBoost(value),
token.start_offset,
token.end_offset,
)
}
saw_dot = true
fractional_divisor = 10.0
continue
}
let code = character.to_int()
guard code >= 0x30 && code <= 0x39 else {
query_string_fail(
InvalidBoost(value),
token.start_offset,
token.end_offset,
)
}
saw_digit = true
let digit = (code - 0x30).to_double()
if saw_dot {
result += digit / fractional_divisor
fractional_divisor *= 10.0
} else {
result = result * 10.0 + digit
}
guard result <= 1000.0 else {
query_string_fail(
InvalidBoost(value),
token.start_offset,
token.end_offset,
)
}
}
guard saw_digit && result > 0.0 else {
query_string_fail(InvalidBoost(value), token.start_offset, token.end_offset)
}
result
}
///|
fn parse_proximity_value(token : QueryToken) -> Int raise QueryStringError {
let value = match token.kind {
Word(value) => value
_ =>
query_string_fail(
UnexpectedToken(query_token_label(token)),
token.start_offset,
token.end_offset,
)
}
let mut result = 0
guard value.length() > 0 else {
query_string_fail(
UnexpectedToken(value),
token.start_offset,
token.end_offset,
)
}
for character in value {
let code = character.to_int()
guard code >= 0x30 && code <= 0x39 else {
query_string_fail(
UnexpectedToken(value),
token.start_offset,
token.end_offset,
)
}
result = result * 10 + code - 0x30
guard result <= 100 else {
query_string_fail(
LimitExceeded("proximity", 100),
token.start_offset,
token.end_offset,
)
}
}
result
}
///|
fn QuerySyntaxParser::parse(
self : QuerySyntaxParser,
) -> UserQueryAst raise QueryStringError {
let first = self.peek()
guard first.kind != End else {
query_string_fail(EmptyQuery, first.start_offset, first.end_offset)
}
let root = self.parse_disjunction(0)
let trailing = self.peek()
if trailing.kind == Star {
query_string_fail(
UnsupportedSyntax("*"),
trailing.start_offset,
trailing.end_offset,
)
}
guard trailing.kind == End else {
query_string_fail(
UnexpectedToken(query_token_label(trailing)),
trailing.start_offset,
trailing.end_offset,
)
}
let frozen : Array[UserQueryNode] = []
for node in self.nodes {
frozen.push(node)
}
{
nodes: ReadOnlyArray::from_array(frozen),
root,
source_length: trailing.end_offset,
}
}
///|
fn QuerySyntaxParser::parse_disjunction(
self : QuerySyntaxParser,
depth : Int,
) -> Int raise QueryStringError {
let children : Array[Int] = [self.parse_conjunction(depth)]
while true {
let token = self.peek()
let explicit = token.kind == Or
let implicit = !self.conjunction_by_default && token_starts_clause(token)
if !explicit && !implicit {
break
}
if explicit {
ignore(self.advance())
let next = self.peek()
guard token_starts_clause(next) else {
query_string_fail(
UnexpectedToken(query_token_label(next)),
next.start_offset,
next.end_offset,
)
}
}
children.push(self.parse_conjunction(depth))
}
if children.length() == 1 {
children[0]
} else {
let first = self.nodes[children[0]]
let last = self.nodes[children[children.length() - 1]]
self.push_node(Disjunction(children), first.start_offset, last.end_offset)
}
}
///|
fn QuerySyntaxParser::parse_conjunction(
self : QuerySyntaxParser,
depth : Int,
) -> Int raise QueryStringError {
let children : Array[Int] = [self.parse_unary(depth)]
while true {
let token = self.peek()
let explicit = token.kind == And
let implicit = self.conjunction_by_default && token_starts_clause(token)
if !explicit && !implicit {
break
}
if explicit {
ignore(self.advance())
let next = self.peek()
guard token_starts_clause(next) else {
query_string_fail(
UnexpectedToken(query_token_label(next)),
next.start_offset,
next.end_offset,
)
}
}
children.push(self.parse_unary(depth))
}
if children.length() == 1 {
children[0]
} else {
let first = self.nodes[children[0]]
let last = self.nodes[children[children.length() - 1]]
self.push_node(Conjunction(children), first.start_offset, last.end_offset)
}
}
///|
fn QuerySyntaxParser::parse_unary(
self : QuerySyntaxParser,
depth : Int,
) -> Int raise QueryStringError {
let token = self.peek()
match token.kind {
Plus => {
guard depth < 32 else {
query_string_fail(
LimitExceeded("nesting depth", 32),
token.start_offset,
token.end_offset,
)
}
ignore(self.advance())
let child = self.parse_unary(depth + 1)
let child_node = self.nodes[child]
self.push_node(Required(child), token.start_offset, child_node.end_offset)
}
Minus | Not => {
guard depth < 32 else {
query_string_fail(
LimitExceeded("nesting depth", 32),
token.start_offset,
token.end_offset,
)
}
ignore(self.advance())
let child = self.parse_unary(depth + 1)
let child_node = self.nodes[child]
self.push_node(
Prohibited(child),
token.start_offset,
child_node.end_offset,
)
}
_ => self.parse_postfix(depth)
}
}
///|
fn QuerySyntaxParser::parse_postfix(
self : QuerySyntaxParser,
depth : Int,
) -> Int raise QueryStringError {
let mut node_index = self.parse_primary(depth)
if self.peek().kind == Tilde {
let tilde = self.advance()
let mut proximity_end = tilde.end_offset
let distance = match self.peek().kind {
Word(_) => {
let value_token = self.advance()
proximity_end = value_token.end_offset
parse_proximity_value(value_token)
}
_ => 2
}
let node = self.nodes[node_index]
node_index = self.push_node(
Proximity(node_index, distance),
node.start_offset,
proximity_end,
)
}
if self.peek().kind == Caret {
ignore(self.advance())
let value_token = self.advance()
let boost = parse_boost_value(value_token)
let node = self.nodes[node_index]
node_index = self.push_node(
Boost(node_index, boost),
node.start_offset,
value_token.end_offset,
)
}
node_index
}
///|
fn QuerySyntaxParser::parse_primary(
self : QuerySyntaxParser,
depth : Int,
) -> Int raise QueryStringError {
let token = self.advance()
match token.kind {
Word(value) =>
if self.peek().kind == Colon {
ignore(self.advance())
let target = self.peek()
guard depth < 32 else {
query_string_fail(
LimitExceeded("nesting depth", 32),
token.start_offset,
token.end_offset,
)
}
guard target.kind is Word(_) ||
target.kind is Phrase(_) ||
target.kind is RegexLiteral(_) ||
target.kind == LeftParen ||
target.kind == LeftBracket ||
target.kind == LeftBrace else {
query_string_fail(
UnexpectedToken(query_token_label(target)),
target.start_offset,
target.end_offset,
)
}
let child = self.parse_postfix(depth + 1)
let child_node = self.nodes[child]
self.push_node(
FieldScope(value, child, token.start_offset, token.end_offset),
token.start_offset,
child_node.end_offset,
)
} else {
self.push_node(
Literal(value, Plain),
token.start_offset,
token.end_offset,
)
}
Phrase(value) =>
self.push_node(
Literal(value, PhraseMode),
token.start_offset,
token.end_offset,
)
RegexLiteral(value) =>
self.push_node(
Literal(value, RegexMode),
token.start_offset,
token.end_offset,
)
LeftBracket | LeftBrace => {
let lower_inclusive = token.kind == LeftBracket
let lower_token = self.advance()
let lower = match lower_token.kind {
Word("*") => None
Word(value) | Phrase(value) => Some(value)
Star => None
_ =>
query_string_fail(
UnexpectedToken(query_token_label(lower_token)),
lower_token.start_offset,
lower_token.end_offset,
)
}
let separator = self.advance()
guard separator.kind == Word("TO") else {
query_string_fail(
UnexpectedToken(query_token_label(separator)),
separator.start_offset,
separator.end_offset,
)
}
let upper_token = self.advance()
let upper = match upper_token.kind {
Word("*") => None
Word(value) | Phrase(value) => Some(value)
Star => None
_ =>
query_string_fail(
UnexpectedToken(query_token_label(upper_token)),
upper_token.start_offset,
upper_token.end_offset,
)
}
let closing = self.advance()
let upper_inclusive = match closing.kind {
RightBracket => true
RightBrace => false
_ =>
query_string_fail(
UnexpectedToken(query_token_label(closing)),
closing.start_offset,
closing.end_offset,
)
}
self.push_node(
Range(lower, upper, lower_inclusive, upper_inclusive),
token.start_offset,
closing.end_offset,
)
}
Star =>
query_string_fail(
UnsupportedSyntax("*"),
token.start_offset,
token.end_offset,
)
LeftParen => {
guard depth < 32 else {
query_string_fail(
LimitExceeded("nesting depth", 32),
token.start_offset,
token.end_offset,
)
}
let child = self.parse_disjunction(depth + 1)
let closing = self.peek()
guard closing.kind == RightParen else {
query_string_fail(
UnexpectedToken(query_token_label(closing)),
closing.start_offset,
closing.end_offset,
)
}
ignore(self.advance())
self.push_node(Group(child), token.start_offset, closing.end_offset)
}
_ =>
query_string_fail(
UnexpectedToken(query_token_label(token)),
token.start_offset,
token.end_offset,
)
}
}
///|
fn parse_user_query_ast(
text : String,
conjunction_by_default : Bool,
) -> UserQueryAst raise QueryStringError {
let tokens = lex_query_string(text)
QuerySyntaxParser::{
tokens: ReadOnlyArray::from_array(tokens),
cursor: 0,
nodes: [],
conjunction_by_default,
}.parse()
}
///|
/// Strict human-query parser. Syntax parsing is Schema-independent; semantic
/// binding resolves fields and invokes each field's configured Analyzer.
pub struct QueryStringParser {
schema : Schema
tokenizers : TokenizerManager
default_fields : ReadOnlyArray[FieldId]
mut conjunction_by_default : Bool
}
///|
pub fn QueryStringParser::new(
schema : Schema,
tokenizers : TokenizerManager,
default_fields : Array[FieldId],
) -> QueryStringParser {
let fields : Array[FieldId] = []
for field in default_fields {
if fields.search_by(existing => existing == field) is None {
fields.push(field)
}
}
{
schema,
tokenizers: tokenizers.snapshot(),
default_fields: ReadOnlyArray::from_array(fields),
conjunction_by_default: false,
}
}
///|
pub fn QueryStringParser::set_conjunction_by_default(
self : QueryStringParser,
conjunction : Bool,
) -> Unit {
self.conjunction_by_default = conjunction
}
///|
pub fn QueryStringParser::parse_ast(
self : QueryStringParser,
text : String,
) -> UserQueryAst raise QueryStringError {
parse_user_query_ast(text, self.conjunction_by_default)
}
///|
priv struct QueryStringCompiler {
schema : Schema
parser : QueryParser
default_fields : ReadOnlyArray[FieldId]
ast : UserQueryAst
mut clause_count : Int
}
///|
fn QueryStringCompiler::resolve_field(
self : QueryStringCompiler,
name : String,
start_offset : Int,
end_offset : Int,
) -> FieldId raise QueryStringError {
let field_id = match self.schema.field(name) {
Some(field_id) => field_id
None => query_string_fail(UnknownField(name), start_offset, end_offset)
}
guard self.schema.is_indexed(field_id) else {
query_string_fail(FieldNotIndexed(name), start_offset, end_offset)
}
field_id
}
///|
fn QueryStringCompiler::compile_literal_for_field(
self : QueryStringCompiler,
field_id : FieldId,
text : String,
mode : QueryLiteralMode,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
let field_name = match self.schema.field_name(field_id) {
Some(name) => name
None =>
query_string_fail(
InvalidDefaultField(field_id.value),
start_offset,
end_offset,
)
}
guard self.schema.is_indexed(field_id) else {
query_string_fail(FieldNotIndexed(field_name), start_offset, end_offset)
}
let field_type = self.schema.field_type(field_id)
let (unescaped_text, has_wildcard, prefix_only) = query_pattern_details(text)
if mode == RegexMode {
guard field_type == Some(@schema.Text) ||
field_type == Some(@schema.Keyword) else {
query_string_fail(
UnsupportedSyntax("regex on typed field"),
start_offset,
end_offset,
)
}
return RegexQuery::new(field_id, text) catch {
_ =>
query_string_fail(
InvalidFieldValue(field_name, text),
start_offset,
end_offset,
)
}
}
if mode == Plain && has_wildcard {
if unescaped_text == "*" {
return ExistsQuery::new(field_id) as &Query
}
guard field_type == Some(@schema.Text) ||
field_type == Some(@schema.Keyword) else {
query_string_fail(
UnsupportedSyntax("wildcard on typed field"),
start_offset,
end_offset,
)
}
return if prefix_only {
let prefix_chars = unescaped_text.to_array()
ignore(prefix_chars.pop())
PrefixQuery::new(field_id, String::from_array(prefix_chars)) as &Query
} else {
WildcardQuery::new(field_id, text) as &Query
}
}
match field_type {
Some(@schema.Text) =>
if mode == PhraseMode {
self.parser.parse_phrase(field_id, text) catch {
error =>
query_string_fail(
AnalysisFailure(error.to_string()),
start_offset,
end_offset,
)
}
} else {
self.parser.parse_query(field_id, unescaped_text) catch {
error =>
query_string_fail(
AnalysisFailure(error.to_string()),
start_offset,
end_offset,
)
}
}
Some(_) if mode == Plain =>
ExactQuery::new(
field_id,
self.typed_value(field_id, unescaped_text, start_offset, end_offset),
)
as &Query
Some(_) =>
query_string_fail(
UnsupportedSyntax("phrase on typed field"),
start_offset,
end_offset,
)
None =>
query_string_fail(
InvalidDefaultField(field_id.value),
start_offset,
end_offset,
)
}
}
///|
fn QueryStringCompiler::compile_literal(
self : QueryStringCompiler,
text : String,
mode : QueryLiteralMode,
inherited_field : FieldId?,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
match inherited_field {
Some(field_id) =>
self.compile_literal_for_field(
field_id, text, mode, start_offset, end_offset,
)
None => {
guard self.default_fields.length() > 0 else {
query_string_fail(NoDefaultField, start_offset, end_offset)
}
let alternatives : Array[BooleanClause] = []
for field_id in self.default_fields {
alternatives.push(
BooleanClause::new(
Occur::Should,
self.compile_literal_for_field(
field_id, text, mode, start_offset, end_offset,
),
),
)
}
if alternatives.length() == 1 {
alternatives[0].query
} else {
self.clause_count += alternatives.length()
guard self.clause_count <= 256 else {
query_string_fail(
LimitExceeded("Boolean clauses", 256),
start_offset,
end_offset,
)
}
BooleanQuery::new(alternatives) as &Query
}
}
}
}
///|
fn QueryStringCompiler::compile_range_for_field(
self : QueryStringCompiler,
field_id : FieldId,
lower : String?,
upper : String?,
lower_inclusive : Bool,
upper_inclusive : Bool,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
let field_name = match self.schema.field_name(field_id) {
Some(name) => name
None =>
query_string_fail(
InvalidDefaultField(field_id.value),
start_offset,
end_offset,
)
}
guard self.schema.is_indexed(field_id) else {
query_string_fail(FieldNotIndexed(field_name), start_offset, end_offset)
}
if self.schema.field_type(field_id) == Some(@schema.Text) {
return TermRangeQuery::new(
field_id, lower, lower_inclusive, upper, upper_inclusive,
)
as &Query
}
let lower_value = match lower {
Some(value) =>
Some(self.typed_value(field_id, value, start_offset, end_offset))
None => None
}
let upper_value = match upper {
Some(value) =>
Some(self.typed_value(field_id, value, start_offset, end_offset))
None => None
}
RangeQuery::new(
field_id, lower_value, lower_inclusive, upper_value, upper_inclusive,
)
as &Query
}
///|
fn query_pattern_details(text : String) -> (String, Bool, Bool) {
let output : Array[Char] = []
let chars = text.to_array()
let mut escaped = false
let mut wildcard_count = 0
let mut final_unescaped_star = false
for index in 0.. 0,
wildcard_count == 1 && final_unescaped_star,
)
}
///|
fn QueryStringCompiler::compile_proximity_for_field(
self : QueryStringCompiler,
field_id : FieldId,
text : String,
mode : QueryLiteralMode,
distance : Int,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
let field_name = match self.schema.field_name(field_id) {
Some(name) => name
None =>
query_string_fail(
InvalidDefaultField(field_id.value),
start_offset,
end_offset,
)
}
guard self.schema.is_indexed(field_id) else {
query_string_fail(FieldNotIndexed(field_name), start_offset, end_offset)
}
match mode {
PhraseMode => {
guard self.schema.field_type(field_id) == Some(@schema.Text) else {
query_string_fail(
UnsupportedSyntax("phrase slop on typed field"),
start_offset,
end_offset,
)
}
self.parser.parse_phrase_with_slop(field_id, text, distance) catch {
error =>
query_string_fail(
AnalysisFailure(error.to_string()),
start_offset,
end_offset,
)
}
}
RegexMode =>
query_string_fail(
UnsupportedSyntax("regex proximity"),
start_offset,
end_offset,
)
Plain => {
let (literal, has_wildcard, _) = query_pattern_details(text)
guard !has_wildcard else {
query_string_fail(
UnsupportedSyntax("fuzzy wildcard"),
start_offset,
end_offset,
)
}
match self.schema.field_type(field_id) {
Some(@schema.Text) => {
let tokens = self.parser.analyze(field_id, literal) catch {
error =>
query_string_fail(
AnalysisFailure(error.to_string()),
start_offset,
end_offset,
)
}
guard tokens.length() > 0 else {
query_string_fail(
AnalysisFailure("empty fuzzy term"),
start_offset,
end_offset,
)
}
let clauses : Array[BooleanClause] = []
let terms : Array[String] = []
for token in tokens {
if terms.search_by(existing => existing == token.text) is None {
terms.push(token.text)
clauses.push(
BooleanClause::new(
Occur::Should,
FuzzyQuery::new(field_id, token.text, distance.min(2)),
),
)
}
}
if clauses.length() == 1 {
clauses[0].query
} else {
BooleanQuery::new(clauses) as &Query
}
}
Some(@schema.Keyword) =>
FuzzyQuery::new(field_id, literal, distance.min(2)) as &Query
_ =>
query_string_fail(
UnsupportedSyntax("fuzzy typed field"),
start_offset,
end_offset,
)
}
}
}
}
///|
fn QueryStringCompiler::compile_proximity(
self : QueryStringCompiler,
child : Int,
distance : Int,
inherited_field : FieldId?,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
let child_node = self.ast.nodes[child]
let (text, mode) = match child_node.kind {
Literal(text, mode) => (text, mode)
_ =>
query_string_fail(
UnsupportedSyntax("proximity on compound query"),
start_offset,
end_offset,
)
}
match inherited_field {
Some(field_id) =>
self.compile_proximity_for_field(
field_id, text, mode, distance, start_offset, end_offset,
)
None => {
guard self.default_fields.length() > 0 else {
query_string_fail(NoDefaultField, start_offset, end_offset)
}
let clauses : Array[BooleanClause] = []
for field_id in self.default_fields {
clauses.push(
BooleanClause::new(
Occur::Should,
self.compile_proximity_for_field(
field_id, text, mode, distance, start_offset, end_offset,
),
),
)
}
if clauses.length() == 1 {
clauses[0].query
} else {
BooleanQuery::new(clauses) as &Query
}
}
}
}
///|
fn QueryStringCompiler::compile_range(
self : QueryStringCompiler,
lower : String?,
upper : String?,
lower_inclusive : Bool,
upper_inclusive : Bool,
inherited_field : FieldId?,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
match inherited_field {
Some(field_id) =>
self.compile_range_for_field(
field_id, lower, upper, lower_inclusive, upper_inclusive, start_offset, end_offset,
)
None => {
guard self.default_fields.length() > 0 else {
query_string_fail(NoDefaultField, start_offset, end_offset)
}
let alternatives : Array[BooleanClause] = []
for field_id in self.default_fields {
alternatives.push(
BooleanClause::new(
Occur::Should,
self.compile_range_for_field(
field_id, lower, upper, lower_inclusive, upper_inclusive, start_offset,
end_offset,
),
),
)
}
if alternatives.length() == 1 {
alternatives[0].query
} else {
BooleanQuery::new(alternatives) as &Query
}
}
}
}
///|
fn QueryStringCompiler::compile_boolean(
self : QueryStringCompiler,
children : Array[Int],
default_occur : Occur,
inherited_field : FieldId?,
start_offset : Int,
end_offset : Int,
) -> &Query raise QueryStringError {
let clauses : Array[BooleanClause] = []
let mut has_positive = false
for child in children {
let (occur, target) = query_clause_target(self.ast, child, default_occur)
let query = self.compile_node(target, inherited_field)
clauses.push(BooleanClause::new(occur, query))
if occur != Occur::MustNot {
has_positive = true
}
}
guard has_positive else {
query_string_fail(NegativeOnlyQuery, start_offset, end_offset)
}
self.clause_count += clauses.length()
guard self.clause_count <= 256 else {
query_string_fail(
LimitExceeded("Boolean clauses", 256),
start_offset,
end_offset,
)
}
if clauses.length() == 1 {
clauses[0].query
} else {
BooleanQuery::new(clauses) as &Query
}
}
///|
fn query_clause_target(
ast : UserQueryAst,
node_index : Int,
default_occur : Occur,
) -> (Occur, Int) {
let node = ast.nodes[node_index]
match node.kind {
Required(target) => (Occur::Must, target)
Prohibited(target) => (Occur::MustNot, target)
Group(target) => {
let target_node = ast.nodes[target]
match target_node.kind {
Required(_) | Prohibited(_) =>
query_clause_target(ast, target, default_occur)
_ => (default_occur, node_index)
}
}
_ => (default_occur, node_index)
}
}
///|
fn QueryStringCompiler::compile_node(
self : QueryStringCompiler,
node_index : Int,
inherited_field : FieldId?,
) -> &Query raise QueryStringError {
let node = self.ast.nodes[node_index]
match node.kind {
Literal(text, mode) =>
self.compile_literal(
text,
mode,
inherited_field,
node.start_offset,
node.end_offset,
)
Range(lower, upper, lower_inclusive, upper_inclusive) =>
self.compile_range(
lower,
upper,
lower_inclusive,
upper_inclusive,
inherited_field,
node.start_offset,
node.end_offset,
)
FieldScope(name, child, field_start, field_end) => {
let field_id = self.resolve_field(name, field_start, field_end)
self.compile_node(child, Some(field_id))
}
Conjunction(children) =>
self.compile_boolean(
children,
Occur::Must,
inherited_field,
node.start_offset,
node.end_offset,
)
Disjunction(children) =>
self.compile_boolean(
children,
Occur::Should,
inherited_field,
node.start_offset,
node.end_offset,
)
Required(child) => self.compile_node(child, inherited_field)
Prohibited(_) =>
query_string_fail(NegativeOnlyQuery, node.start_offset, node.end_offset)
Boost(child, factor) =>
BoostQuery::new(self.compile_node(child, inherited_field), factor)
as &Query
Proximity(child, distance) =>
self.compile_proximity(
child,
distance,
inherited_field,
node.start_offset,
node.end_offset,
)
Group(child) => self.compile_node(child, inherited_field)
}
}
///|
pub fn QueryStringParser::build_query_from_ast(
self : QueryStringParser,
ast : UserQueryAst,
) -> &Query raise QueryStringError {
for field_id in self.default_fields {
let field_name = match self.schema.field_name(field_id) {
Some(name) => name
None => query_string_fail(InvalidDefaultField(field_id.value), 0, 0)
}
guard self.schema.is_indexed(field_id) else {
query_string_fail(FieldNotIndexed(field_name), 0, 0)
}
}
let leaf_parser = QueryParser::new(self.schema, self.tokenizers)
leaf_parser.set_conjunction_by_default(self.conjunction_by_default)
QueryStringCompiler::{
schema: self.schema,
parser: leaf_parser,
default_fields: self.default_fields,
ast,
clause_count: 0,
}.compile_node(ast.root, None)
}
///|
pub fn QueryStringParser::parse_query(
self : QueryStringParser,
text : String,
) -> &Query raise QueryStringError {
self.build_query_from_ast(self.parse_ast(text))
}
///|
pub struct LenientQueryResult {
query : &Query
warnings : ReadOnlyArray[String]
recovered : Bool
}
///|
pub fn LenientQueryResult::query(self : LenientQueryResult) -> &Query {
self.query
}
///|
pub fn LenientQueryResult::warnings(
self : LenientQueryResult,
) -> ReadOnlyArray[String] {
self.warnings
}
///|
pub fn LenientQueryResult::recovered(self : LenientQueryResult) -> Bool {
self.recovered
}
///|
fn lenient_query_text(text : String) -> String {
let output = StringBuilder()
let mut count = 0
for ch in text {
if count >= 1024 {
break
}
if ch == ':' ||
ch == '(' ||
ch == ')' ||
ch == '[' ||
ch == ']' ||
ch == '{' ||
ch == '}' ||
ch == '^' ||
ch == '~' ||
ch == '/' ||
ch == '"' ||
ch == '+' ||
ch == '-' ||
ch == '&' ||
ch == '|' ||
ch == '!' ||
ch == '*' ||
ch == '?' {
output.write_char(' ')
} else {
output.write_char(ch)
}
count += 1
}
output.to_string()
}
///|
/// Error-tolerant entry point. Strict parsing is attempted first; on failure,
/// structural punctuation is removed and the remaining text is analyzed over
/// the configured default fields. The original diagnostic is preserved.
pub fn QueryStringParser::parse_query_lenient(
self : QueryStringParser,
text : String,
) -> LenientQueryResult raise QueryStringError {
{ query: self.parse_query(text), warnings: [], recovered: false } catch {
strict_error => {
let fallback = lenient_query_text(text)
guard self.default_fields.length() > 0 else {
query_string_fail(NoDefaultField, 0, @utf8.encode(text).length())
}
let leaf_parser = QueryParser::new(self.schema, self.tokenizers)
leaf_parser.set_conjunction_by_default(self.conjunction_by_default)
let alternatives : Array[BooleanClause] = []
for field_id in self.default_fields {
let child = leaf_parser.parse_query(field_id, fallback) catch {
error =>
query_string_fail(
AnalysisFailure(Show::to_string(error)),
0,
@utf8.encode(text).length(),
)
}
alternatives.push(BooleanClause::new(Occur::Should, child))
}
let query : &Query = if alternatives.length() == 1 {
alternatives[0].query
} else {
BooleanQuery::new(alternatives) as &Query
}
{
query,
warnings: ReadOnlyArray::from_array([Show::to_string(strict_error)]),
recovered: true,
}
}
}
}