///|
/// Quality-control configuration for numeric telemetry.
pub struct QualityRule {
lower_bound : Double
upper_bound : Double
max_step : Double
flatline_window : Int
duplicate_window : Int
}
///|
/// One observation and its quality diagnostics.
pub struct QualityObservation {
index : Int
value : Double
score : Double
valid : Bool
range_violation : Bool
jump_violation : Bool
flatline_violation : Bool
}
///|
/// Aggregate quality diagnostics for a data batch.
pub struct QualityReport {
count : Int
valid_count : Int
missing_count : Int
range_violations : Int
jump_violations : Int
flatline_violations : Int
duplicate_fraction : Double
completeness : Double
quality_score : Double
}
///|
/// Stateful quality monitor for an incoming telemetry stream.
pub struct QualityMonitor {
rule : QualityRule
mut seen : Int
mut invalid : Int
mut violations : Int
mut last_value : Double
mut flatline_run : Int
mut initialized : Bool
}
///|
pub fn quality_default_rule() -> QualityRule {
{
lower_bound: -1.0e12,
upper_bound: 1.0e12,
max_step: 1.0e12,
flatline_window: 8,
duplicate_window: 8,
}
}
///|
pub fn quality_rule(
lower_bound : Double,
upper_bound : Double,
max_step : Double,
flatline_window : Int,
duplicate_window : Int,
) -> QualityRule {
let safe_flatline = if flatline_window < 2 { 2 } else { flatline_window }
let safe_duplicate = if duplicate_window < 2 { 2 } else { duplicate_window }
{
lower_bound,
upper_bound,
max_step: if max_step < 0.0 {
-max_step
} else {
max_step
},
flatline_window: safe_flatline,
duplicate_window: safe_duplicate,
}
}
///|
pub fn quality_rule_valid(rule : QualityRule) -> Bool {
rule.lower_bound <= rule.upper_bound &&
rule.max_step >= 0.0 &&
rule.flatline_window >= 2 &&
rule.duplicate_window >= 2
}
///|
pub fn quality_is_missing(value : Double) -> Bool {
value != value
}
///|
pub fn quality_in_range(value : Double, rule : QualityRule) -> Bool {
!quality_is_missing(value) &&
value >= rule.lower_bound &&
value <= rule.upper_bound
}
///|
pub fn quality_range_distance(value : Double, rule : QualityRule) -> Double {
if quality_is_missing(value) {
1.0
} else if value < rule.lower_bound {
rule.lower_bound - value
} else if value > rule.upper_bound {
value - rule.upper_bound
} else {
0.0
}
}
///|
pub fn quality_range_score(value : Double, rule : QualityRule) -> Double {
let distance = quality_range_distance(value, rule)
1.0 / (1.0 + distance)
}
///|
pub fn quality_jump_distance(previous : Double, current : Double) -> Double {
if quality_is_missing(previous) || quality_is_missing(current) {
0.0
} else {
abs_double(current - previous)
}
}
///|
pub fn quality_jump_violation(
previous : Double,
current : Double,
rule : QualityRule,
) -> Bool {
quality_jump_distance(previous, current) > rule.max_step
}
///|
pub fn quality_jump_score(
previous : Double,
current : Double,
rule : QualityRule,
) -> Double {
let distance = quality_jump_distance(previous, current)
if rule.max_step == 0.0 {
if distance == 0.0 {
1.0
} else {
0.0
}
} else if distance <= rule.max_step {
1.0
} else {
rule.max_step / distance
}
}
///|
pub fn quality_flatline_run(data : Array[Double], index : Int) -> Int {
if index < 0 || index >= data.length() {
return 0
}
let value = data[index]
let mut left = index
let mut right = index
while left > 0 && data[left - 1] == value {
left -= 1
}
while right + 1 < data.length() && data[right + 1] == value {
right += 1
}
right - left + 1
}
///|
pub fn quality_flatline_violation(
data : Array[Double],
index : Int,
rule : QualityRule,
) -> Bool {
quality_flatline_run(data, index) >= rule.flatline_window
}
///|
pub fn quality_flatline_score(
data : Array[Double],
index : Int,
rule : QualityRule,
) -> Double {
let run = quality_flatline_run(data, index)
if run < rule.flatline_window {
1.0
} else {
rule.flatline_window.to_double() / run.to_double()
}
}
///|
pub fn quality_duplicate_run(data : Array[Double], index : Int) -> Int {
if index < 0 || index >= data.length() {
return 0
}
let value = data[index]
let mut run = 1
let mut cursor = index - 1
while cursor >= 0 && data[cursor] == value {
run += 1
cursor -= 1
}
cursor = index + 1
while cursor < data.length() && data[cursor] == value {
run += 1
cursor += 1
}
run
}
///|
pub fn quality_duplicate_fraction_in_window(
data : Array[Double],
start : Int,
end : Int,
) -> Double {
if start >= end || start < 0 || end > data.length() {
return 0.0
}
let values = []
for index = start; index < end; index = index + 1 {
values.push(data[index])
}
duplicate_fraction(values)
}
///|
pub fn quality_observation(
data : Array[Double],
index : Int,
rule : QualityRule,
) -> QualityObservation {
let value = data[index]
let previous = if index == 0 { value } else { data[index - 1] }
let missing = quality_is_missing(value)
let range_bad = !quality_in_range(value, rule)
let jump_bad = index > 0 && quality_jump_violation(previous, value, rule)
let flatline_bad = quality_flatline_violation(data, index, rule)
let score = quality_observation_score(data, index, rule)
{
index,
value,
score,
valid: !missing && !range_bad && !jump_bad,
range_violation: range_bad,
jump_violation: jump_bad,
flatline_violation: flatline_bad,
}
}
///|
pub fn quality_observation_score(
data : Array[Double],
index : Int,
rule : QualityRule,
) -> Double {
if index < 0 || index >= data.length() {
return 0.0
}
let value = data[index]
let range_score = quality_range_score(value, rule)
let jump_score = if index == 0 {
1.0
} else {
quality_jump_score(data[index - 1], value, rule)
}
let flatline_score = quality_flatline_score(data, index, rule)
(range_score + jump_score + flatline_score) / 3.0
}
///|
pub fn quality_report(
data : Array[Double],
rule : QualityRule,
) -> QualityReport {
let mut missing_count = 0
let mut range_violations = 0
let mut jump_violations = 0
let mut flatline_violations = 0
let mut valid_count = 0
let mut total_score = 0.0
for index = 0; index < data.length(); index = index + 1 {
let observation = quality_observation(data, index, rule)
if quality_is_missing(observation.value) {
missing_count += 1
}
if observation.range_violation {
range_violations += 1
}
if observation.jump_violation {
jump_violations += 1
}
if observation.flatline_violation {
flatline_violations += 1
}
if observation.valid {
valid_count += 1
}
total_score += observation.score
}
let count = data.length()
let completeness = if count == 0 {
0.0
} else {
(count - missing_count).to_double() / count.to_double()
}
{
count,
valid_count,
missing_count,
range_violations,
jump_violations,
flatline_violations,
duplicate_fraction: duplicate_fraction(data),
completeness,
quality_score: if count == 0 {
0.0
} else {
total_score / count.to_double()
},
}
}
///|
pub fn quality_report_score(report : QualityReport) -> Double {
report.quality_score * report.completeness
}
///|
pub fn quality_report_lines(report : QualityReport) -> Array[String] {
[
"count=" + report.count.to_string(),
"valid_count=" + report.valid_count.to_string(),
"missing_count=" + report.missing_count.to_string(),
"range_violations=" + report.range_violations.to_string(),
"jump_violations=" + report.jump_violations.to_string(),
"flatline_violations=" + report.flatline_violations.to_string(),
"duplicate_fraction=" + report.duplicate_fraction.to_string(),
"completeness=" + report.completeness.to_string(),
"quality_score=" + report.quality_score.to_string(),
]
}
///|
pub fn quality_report_string(report : QualityReport) -> String {
quality_report_lines(report).join("\n")
}
///|
pub fn quality_scores(
data : Array[Double],
rule : QualityRule,
) -> Array[Double] {
let result = []
for index = 0; index < data.length(); index = index + 1 {
result.push(quality_observation_score(data, index, rule))
}
result
}
///|
pub fn quality_valid_flags(
data : Array[Double],
rule : QualityRule,
) -> Array[Bool] {
let result = []
for index = 0; index < data.length(); index = index + 1 {
result.push(quality_observation(data, index, rule).valid)
}
result
}
///|
pub fn quality_missing_indices(data : Array[Double]) -> Array[Int] {
let result = []
for index = 0; index < data.length(); index = index + 1 {
if quality_is_missing(data[index]) {
result.push(index)
}
}
result
}
///|
pub fn quality_range_violation_indices(
data : Array[Double],
rule : QualityRule,
) -> Array[Int] {
let result = []
for index = 0; index < data.length(); index = index + 1 {
if !quality_in_range(data[index], rule) {
result.push(index)
}
}
result
}
///|
pub fn quality_jump_violation_indices(
data : Array[Double],
rule : QualityRule,
) -> Array[Int] {
let result = []
for index = 1; index < data.length(); index = index + 1 {
if quality_jump_violation(data[index - 1], data[index], rule) {
result.push(index)
}
}
result
}
///|
pub fn quality_flatline_indices(
data : Array[Double],
rule : QualityRule,
) -> Array[Int] {
let result = []
for index = 0; index < data.length(); index = index + 1 {
if quality_flatline_violation(data, index, rule) {
result.push(index)
}
}
result
}
///|
pub fn quality_clean(data : Array[Double], rule : QualityRule) -> Array[Double] {
let result = []
let fallback = if data.length() == 0 { 0.0 } else { median(data) }
for index = 0; index < data.length(); index = index + 1 {
if quality_valid_flags(data, rule)[index] {
result.push(data[index])
} else if index == 0 {
result.push(fallback)
} else {
result.push(result[index - 1])
}
}
result
}
///|
pub fn quality_impute_forward(data : Array[Double]) -> Array[Double] {
let result = []
let mut last = 0.0
let mut initialized = false
for value in data {
if !quality_is_missing(value) {
last = value
initialized = true
result.push(value)
} else if initialized {
result.push(last)
} else {
result.push(0.0)
}
}
result
}
///|
pub fn quality_impute_median(data : Array[Double]) -> Array[Double] {
let valid = []
for value in data {
if !quality_is_missing(value) {
valid.push(value)
}
}
let fallback = if valid.length() == 0 { 0.0 } else { median(valid) }
let result = []
for value in data {
result.push(if quality_is_missing(value) { fallback } else { value })
}
result
}
///|
pub fn quality_impute_linear(data : Array[Double]) -> Array[Double] {
let result = quality_impute_forward(data)
if result.length() <= 1 {
return result
}
let mut next_known = result.length() - 1
for index = result.length() - 2; index >= 0; index = index - 1 {
if !quality_is_missing(data[index]) {
next_known = index
} else {
let left = if index == 0 { result[next_known] } else { result[index - 1] }
let right = result[next_known]
let span = if next_known == index { 1 } else { next_known - index }
result[index] = left + (right - left) / span.to_double()
}
}
result
}
///|
pub fn quality_clip(data : Array[Double], rule : QualityRule) -> Array[Double] {
let result = []
for value in data {
if quality_is_missing(value) {
result.push(value)
} else if value < rule.lower_bound {
result.push(rule.lower_bound)
} else if value > rule.upper_bound {
result.push(rule.upper_bound)
} else {
result.push(value)
}
}
result
}
///|
pub fn quality_sanitize(
data : Array[Double],
rule : QualityRule,
) -> Array[Double] {
quality_clip(quality_impute_median(data), rule)
}
///|
pub fn QualityMonitor::new(rule : QualityRule) -> QualityMonitor {
{
rule,
seen: 0,
invalid: 0,
violations: 0,
last_value: 0.0,
flatline_run: 0,
initialized: false,
}
}
///|
pub fn QualityMonitor::observe(
self : QualityMonitor,
value : Double,
) -> QualityObservation {
let index = self.seen
let previous = self.last_value
self.seen += 1
let missing = quality_is_missing(value)
let range_bad = !quality_in_range(value, self.rule)
let jump_bad = self.initialized &&
quality_jump_violation(previous, value, self.rule)
if missing || range_bad || jump_bad {
self.invalid += 1
}
if jump_bad || range_bad {
self.violations += 1
}
if self.initialized && value == previous {
self.flatline_run += 1
} else {
self.flatline_run = 1
}
self.last_value = value
self.initialized = true
{
index,
value,
score: if self.initialized {
quality_jump_score(previous, value, self.rule)
} else {
0.0
},
valid: !missing && !range_bad && !jump_bad,
range_violation: range_bad,
jump_violation: jump_bad,
flatline_violation: self.flatline_run >= self.rule.flatline_window,
}
}
///|
pub fn QualityMonitor::observe_many(
self : QualityMonitor,
data : Array[Double],
) -> Array[QualityObservation] {
let result = []
for value in data {
result.push(self.observe(value))
}
result
}
///|
pub fn QualityMonitor::reset(self : QualityMonitor) -> Unit {
self.seen = 0
self.invalid = 0
self.violations = 0
self.last_value = 0.0
self.flatline_run = 0
self.initialized = false
}
///|
pub fn QualityMonitor::seen(self : QualityMonitor) -> Int {
self.seen
}
///|
pub fn QualityMonitor::invalid(self : QualityMonitor) -> Int {
self.invalid
}
///|
pub fn QualityMonitor::violation_rate(self : QualityMonitor) -> Double {
if self.seen == 0 {
0.0
} else {
self.violations.to_double() / self.seen.to_double()
}
}
///|
pub fn QualityMonitor::completeness(self : QualityMonitor) -> Double {
if self.seen == 0 {
0.0
} else {
(self.seen - self.invalid).to_double() / self.seen.to_double()
}
}