///|
/// Deterministic delimited-data parsing for instrument and audit exports.
pub struct DelimitedRow {
fields : Array[String]
line_number : Int
}
///|
pub fn delimited_row(fields : Array[String], line_number : Int) -> DelimitedRow {
{ fields, line_number }
}
///|
pub fn DelimitedRow::field(self : DelimitedRow, index : Int) -> String {
if index < 0 || index >= self.fields.length() {
""
} else {
self.fields[index]
}
}
///|
pub fn DelimitedRow::width(self : DelimitedRow) -> Int {
self.fields.length()
}
///|
pub fn DelimitedRow::valid(self : DelimitedRow) -> Bool {
self.line_number > 0
}
///|
pub struct DelimitedTable {
headers : Array[String]
rows : Array[DelimitedRow]
delimiter : String
}
///|
pub fn delimited_table(
headers : Array[String],
rows : Array[DelimitedRow],
delimiter : String,
) -> DelimitedTable {
{ headers, rows, delimiter }
}
///|
pub fn DelimitedTable::width(self : DelimitedTable) -> Int {
self.headers.length()
}
///|
pub fn DelimitedTable::valid_rows(self : DelimitedTable) -> Array[DelimitedRow] {
self.rows.filter(fn(r) { r.valid() && r.width() == self.width() })
}
///|
pub fn DelimitedTable::invalid_rows(
self : DelimitedTable,
) -> Array[DelimitedRow] {
self.rows.filter(fn(r) { !r.valid() || r.width() != self.width() })
}
///|
pub fn DelimitedTable::column(
self : DelimitedTable,
name : String,
) -> Array[String] {
let mut index = -1
for i = 0; i < self.headers.length(); i = i + 1 {
if self.headers[i] == name {
index = i
}
}
if index < 0 {
[]
} else {
self.valid_rows().map(fn(r) { r.field(index) })
}
}
///|
pub fn DelimitedTable::row_count(self : DelimitedTable) -> Int {
self.valid_rows().length()
}
///|
pub fn DelimitedTable::quality(self : DelimitedTable) -> Float {
if self.rows.length() == 0 {
1.0
} else {
Float::from_int(self.valid_rows().length()) /
Float::from_int(self.rows.length())
}
}
///|
pub fn split_delimited_line(
line : String,
delimiter : String,
line_number : Int,
) -> DelimitedRow {
let fields : Array[String] = []
for part in line.split(delimiter) {
fields.push(part.to_owned())
}
delimited_row(fields, line_number)
}
///|
pub fn parse_delimited(text : String, delimiter : String) -> DelimitedTable {
let lines : Array[String] = []
for line in text.split("\n") {
lines.push(line.to_owned())
}
if lines.length() == 0 {
delimited_table([], [], delimiter)
} else {
let headers = split_delimited_line(lines[0], delimiter, 1).fields
let rows : Array[DelimitedRow] = []
for index = 1; index < lines.length(); index = index + 1 {
if lines[index] != "" {
rows.push(split_delimited_line(lines[index], delimiter, index + 1))
}
}
delimited_table(headers, rows, delimiter)
}
}
///|
pub fn table_to_csv(data : DelimitedTable) -> String {
let lines : Array[String] = []
lines.push(data.headers.join(data.delimiter))
for row in data.rows {
lines.push(row.fields.join(data.delimiter))
}
lines.join("\n")
}
///|
pub struct ImportIssue {
line_number : Int
field : String
message : String
severity : String
}
///|
pub fn import_issue(
line_number : Int,
field : String,
message : String,
severity : String,
) -> ImportIssue {
{ line_number, field, message, severity }
}
///|
pub fn import_issues_table(issues : Array[ImportIssue]) -> ReportTable {
let rows : Array[Array[String]] = []
for issue in issues {
rows.push([
"\{issue.line_number}",
issue.field,
issue.message,
issue.severity,
])
}
table(["line", "field", "message", "severity"], rows)
}
///|
pub fn missing_field_issues(
data : DelimitedTable,
required : Array[String],
) -> Array[ImportIssue] {
let result : Array[ImportIssue] = []
for name in required {
if !data.headers.contains(name) {
result.push(import_issue(1, name, "missing header", "error"))
}
}
result
}
///|
pub fn duplicate_headers(headers : Array[String]) -> Array[String] {
let result : Array[String] = []
for i = 0; i < headers.length(); i = i + 1 {
for j = i + 1; j < headers.length(); j = j + 1 {
if headers[i] == headers[j] && !result.contains(headers[i]) {
result.push(headers[i])
}
}
}
result
}
///|
pub fn import_gate(
data : DelimitedTable,
minimum_quality : Float,
required : Array[String],
) -> Bool {
data.quality() >= minimum_quality &&
missing_field_issues(data, required).length() == 0 &&
duplicate_headers(data.headers).length() == 0
}