///|
/// Per-column profile inferred from parsed table values.
pub(all) struct ColumnProfile {
name : String
index : Int
empty_count : Int
non_empty_count : Int
integer_count : Int
decimal_count : Int
boolean_count : Int
inferred_kind : FieldKind
required : Bool
} derive(Eq)
///|
/// Whole-table data quality profile.
pub(all) struct TableProfile {
row_count : Int
column_count : Int
empty_cells : Int
quality_score : Int
columns : Array[ColumnProfile]
} derive(Eq)
///|
fn generated_column_name(index : Int) -> String {
let text =
$|column_\{index + 1}
text
}
///|
fn table_column_name(table : Table, index : Int) -> String {
if index < table.header.length() {
table.header[index]
} else {
generated_column_name(index)
}
}
///|
fn infer_kind_from_counts(
non_empty : Int,
integer_count : Int,
decimal_count : Int,
boolean_count : Int,
required : Bool,
) -> FieldKind {
if non_empty == 0 {
Text
} else if boolean_count == non_empty {
Boolean
} else if integer_count == non_empty {
Integer
} else if decimal_count == non_empty {
Decimal
} else if required {
NonEmpty
} else {
Text
}
}
///|
fn profile_column(table : Table, index : Int) -> ColumnProfile {
let mut empty_count = 0
let mut non_empty = 0
let mut integer_count = 0
let mut decimal_count = 0
let mut boolean_count = 0
let mut row = 0
while row < table.rows.length() {
let value = match table.rows[row].get(index) {
Some(v) => v
None => ""
}
let normalized = value.trim().to_owned()
if normalized.is_empty() {
empty_count = empty_count + 1
} else {
non_empty = non_empty + 1
if is_integer_text(normalized) {
integer_count = integer_count + 1
}
if is_decimal_text(normalized) {
decimal_count = decimal_count + 1
}
if is_boolean_text(normalized) {
boolean_count = boolean_count + 1
}
}
row = row + 1
}
let required = table.rows.length() > 0 && empty_count == 0
let kind = infer_kind_from_counts(
non_empty, integer_count, decimal_count, boolean_count, required,
)
{
name: table_column_name(table, index),
index,
empty_count,
non_empty_count: non_empty,
integer_count,
decimal_count,
boolean_count,
inferred_kind: kind,
required,
}
}
///|
fn quality_score(row_count : Int, column_count : Int, empty_cells : Int) -> Int {
let total = row_count * column_count
if total <= 0 {
100
} else {
clamp_percent(100 - empty_cells * 100 / total)
}
}
///|
/// Build a deterministic quality profile for a parsed table.
pub fn profile(table : Table) -> TableProfile {
let row_count = table.row_count()
let column_count = table.column_count()
let columns : Array[ColumnProfile] = []
let mut empty_cells = 0
let mut c = 0
while c < column_count {
let column = profile_column(table, c)
empty_cells = empty_cells + column.empty_count
columns.push(column)
c = c + 1
}
{
row_count,
column_count,
empty_cells,
quality_score: quality_score(row_count, column_count, empty_cells),
columns,
}
}
///|
/// Infer a closed schema from current table values.
pub fn infer_schema(table : Table) -> Schema {
let prof = profile(table)
let columns : Array[ColumnRule] = []
for column in prof.columns {
columns.push({
name: column.name,
kind: column.inferred_kind,
required: column.required,
})
}
closed_schema(columns)
}
///|
/// Stable string label for an inferred field kind.
pub fn ColumnProfile::kind_label(self : ColumnProfile) -> String {
kind_name(self.inferred_kind)
}
///|
/// Compact profile summary.
pub fn TableProfile::summary(self : TableProfile) -> String {
let text =
$|rows=\{self.row_count}, columns=\{self.column_count}, empty_cells=\{self.empty_cells}, quality=\{self.quality_score}
text
}