///|
/// Detected delimited-text dialect with a ready-to-use parser config.
pub(all) struct DialectGuess {
delimiter : Char
delimiter_name : String
confidence : Int
has_header : Bool
reason : String
config : ParseConfig
} derive(Eq)
///|
fn delimiter_label(delimiter : Char) -> String {
match delimiter {
',' => "comma"
'\t' => "tab"
'|' => "pipe"
';' => "semicolon"
_ => "custom"
}
}
///|
fn min_int(a : Int, b : Int) -> Int {
if a < b {
a
} else {
b
}
}
///|
fn clamp_percent(n : Int) -> Int {
if n < 0 {
0
} else if n > 100 {
100
} else {
n
}
}
///|
fn delimiter_counts(
input : String,
delimiter : Char,
quote : Char,
) -> Array[Int] {
let counts : Array[Int] = []
let mut count = 0
let mut has_content = false
let mut in_quotes = false
let mut i = 0
while i < input.length() {
let c = match input.get_char(i) {
Some(ch) => ch
None => '\n'
}
if c == quote {
has_content = true
if in_quotes &&
i + 1 < input.length() &&
input.get_char(i + 1) == Some(quote) {
i = i + 2
} else {
in_quotes = !in_quotes
i = i + 1
}
} else if !in_quotes && c == delimiter {
has_content = true
count = count + 1
i = i + 1
} else if !in_quotes && is_line_break(c) {
if has_content {
counts.push(count)
}
count = 0
has_content = false
i = i + 1
if c == '\r' && i < input.length() && input.get_char(i) == Some('\n') {
i = i + 1
}
} else {
if c != ' ' && c != '\t' {
has_content = true
}
i = i + 1
}
}
if has_content {
counts.push(count)
}
counts
}
///|
fn candidate_score(counts : Array[Int]) -> Int {
let mut first_positive = -1
let mut positive = 0
let mut consistent = 0
for count in counts {
if count > 0 {
positive = positive + 1
if first_positive < 0 {
first_positive = count
}
if count == first_positive {
consistent = consistent + 1
}
}
}
if first_positive < 0 {
0
} else {
positive * 20 + consistent * 20 + first_positive * 5
}
}
///|
fn is_ascii_alpha(c : Char) -> Bool {
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}
///|
fn has_alpha(text : String) -> Bool {
let mut i = 0
while i < text.length() {
match text.get_char(i) {
Some(c) => if is_ascii_alpha(c) { return true }
None => ()
}
i = i + 1
}
false
}
///|
fn looks_like_header_cell(text : String) -> Bool {
let s = text.trim().to_owned()
!s.is_empty() &&
has_alpha(s) &&
!is_integer_text(s) &&
!is_decimal_text(s) &&
!is_boolean_text(s)
}
///|
fn looks_like_data_cell(text : String) -> Bool {
let s = text.trim().to_owned()
is_integer_text(s) || is_decimal_text(s) || is_boolean_text(s)
}
///|
fn parse_rows_for_sniff(input : String, delimiter : Char) -> Table? {
let config : ParseConfig = {
delimiter,
quote: '"',
has_header: false,
trim_unquoted: true,
skip_empty_lines: true,
strict_columns: false,
comment: None,
}
match parse_with(input, config) {
Ok(table) => Some(table)
Err(_) => None
}
}
///|
fn guess_header(input : String, delimiter : Char) -> Bool {
match parse_rows_for_sniff(input, delimiter) {
Some(table) =>
if table.rows.length() < 2 {
false
} else {
let first = table.rows[0]
let second = table.rows[1]
let width = min_int(first.length(), second.length())
let mut header_like = 0
let mut data_like = 0
let mut i = 0
while i < width {
if looks_like_header_cell(first[i]) {
header_like = header_like + 1
}
if looks_like_data_cell(second[i]) {
data_like = data_like + 1
}
i = i + 1
}
header_like > 0 && header_like >= data_like
}
None => false
}
}
///|
fn config_from_guess(delimiter : Char, has_header : Bool) -> ParseConfig {
{
delimiter,
quote: '"',
has_header,
trim_unquoted: true,
skip_empty_lines: true,
strict_columns: true,
comment: None,
}
}
///|
/// Detect a likely delimiter and header mode from a text sample.
pub fn sniff(input : String) -> DialectGuess {
let candidates = [',', '\t', '|', ';']
let mut best_delimiter = ','
let mut best_score = -1
let mut best_lines = 0
for delimiter in candidates {
let counts = delimiter_counts(input, delimiter, '"')
let score = candidate_score(counts)
if score > best_score {
best_score = score
best_delimiter = delimiter
best_lines = counts.length()
}
}
let has_header = guess_header(input, best_delimiter)
let confidence = if best_score <= 0 { 0 } else { clamp_percent(best_score) }
let name = delimiter_label(best_delimiter)
let reason =
$|selected \{name} delimiter from \{best_lines} sampled line(s)
{
delimiter: best_delimiter,
delimiter_name: name,
confidence,
has_header,
reason,
config: config_from_guess(best_delimiter, has_header),
}
}
///|
/// Parse text after automatic delimiter/header detection.
pub fn parse_auto(input : String) -> Result[Table, ParseError] {
let guess = sniff(input)
parse_with(input, guess.config)
}
///|
/// Compact description for logs, CLIs, and review output.
pub fn DialectGuess::summary(self : DialectGuess) -> String {
let header = if self.has_header { "header" } else { "no-header" }
let text =
$|\{self.delimiter_name}, confidence=\{self.confidence}, \{header}
text
}