///|
/// One named regression case for a compiled rule set.
pub(all) struct RuleTestCase {
name : String
input : Json
expected_passed : Bool
expected_failed_rules : Array[String]?
} derive(Eq, Debug, ToJson, FromJson)
///|
pub fn RuleTestCase::new(
name : String,
input : Json,
expected_passed : Bool,
expected_failed_rules? : Array[String],
) -> RuleTestCase {
{ name, input, expected_passed, expected_failed_rules }
}
///|
/// Defensive limits for externally supplied regression suites.
pub(all) struct RuleTestSuiteLimits {
max_cases : Int
max_name_length : Int
} derive(Eq, Debug, ToJson, FromJson)
///|
pub fn RuleTestSuiteLimits::default() -> RuleTestSuiteLimits {
{ max_cases: 1024, max_name_length: 128 }
}
///|
/// Validated collection of rule regression cases.
pub struct RuleTestSuite {
cases : Array[RuleTestCase]
}
///|
fn test_suite_diagnostic(
code : String,
message : String,
hint : String,
) -> Diagnostic {
Diagnostic::new(Configure, code, message, Span::new(0, 0), hint~)
}
///|
fn has_duplicate_strings(values : Array[String]) -> Bool {
for index, value in values {
for previous = 0; previous < index; previous = previous + 1 {
if values[previous] == value {
return true
}
}
}
false
}
///|
fn validate_test_suite_limits(
limits : RuleTestSuiteLimits,
) -> Result[Unit, Diagnostic] {
if limits.max_cases <= 0 || limits.max_name_length <= 0 {
Err(
test_suite_diagnostic(
"C041", "rule test suite limits must be positive integers", "Set max_cases and max_name_length to at least 1.",
),
)
} else {
Ok(())
}
}
///|
pub fn RuleTestSuite::new(
cases : Array[RuleTestCase],
limits? : RuleTestSuiteLimits = RuleTestSuiteLimits::default(),
) -> Result[RuleTestSuite, Array[Diagnostic]] {
match validate_test_suite_limits(limits) {
Err(diagnostic) => return Err([diagnostic])
Ok(_) => ()
}
if cases.length() > limits.max_cases {
return Err([
test_suite_diagnostic(
"C042",
"rule test suite contains \{cases.length()} cases, exceeding the limit of \{limits.max_cases}",
"Split the suite or explicitly raise max_cases after review.",
),
])
}
if cases.is_empty() {
return Err([
test_suite_diagnostic(
"C047", "rule test suite must contain at least one case", "Add a named case so an empty file cannot make CI pass accidentally.",
),
])
}
let diagnostics = []
let names = []
for index, case in cases {
if case.name.is_empty() {
diagnostics.push(
test_suite_diagnostic(
"C043",
"rule test case at index \{index} has an empty name",
"Give every case a stable, descriptive name.",
),
)
} else if case.name.char_length() > limits.max_name_length {
diagnostics.push(
test_suite_diagnostic(
"C043",
"rule test case `\{case.name}` exceeds the name length limit",
"Shorten the case name or explicitly raise max_name_length.",
),
)
} else if names.contains(case.name) {
diagnostics.push(
test_suite_diagnostic(
"C044",
"duplicate rule test case name `\{case.name}`",
"Use unique names so CI failures identify exactly one case.",
),
)
} else {
names.push(case.name)
}
match case.expected_failed_rules {
Some(failed_rules) => {
if case.expected_passed && !failed_rules.is_empty() {
diagnostics.push(
test_suite_diagnostic(
"C045",
"case `\{case.name}` expects success and failed rules at the same time",
"Set expected_failed_rules to an empty array or expected_passed to false.",
),
)
}
if has_duplicate_strings(failed_rules) {
diagnostics.push(
test_suite_diagnostic(
"C046",
"case `\{case.name}` contains duplicate expected failed rule names",
"List each expected failed rule exactly once.",
),
)
}
}
None => ()
}
}
if diagnostics.is_empty() {
Ok({ cases, })
} else {
Err(diagnostics)
}
}
///|
/// Decode and validate a JSON array of regression cases.
pub fn RuleTestSuite::from_json(
config : Json,
limits? : RuleTestSuiteLimits = RuleTestSuiteLimits::default(),
) -> Result[RuleTestSuite, Array[Diagnostic]] {
let cases : Array[RuleTestCase] = @json.from_json(config) catch {
error =>
return Err([
test_suite_diagnostic(
"C040",
"invalid rule test suite: \{error}",
"Expected a JSON array with name, input, expected_passed, and optional expected_failed_rules fields.",
),
])
}
RuleTestSuite::new(cases, limits~)
}
///|
pub fn RuleTestSuite::length(self : RuleTestSuite) -> Int {
self.cases.length()
}
///|
fn failed_rule_names(report : RuleReport) -> Array[String] {
report.outcomes
.filter(fn(outcome) { outcome.status == Failed })
.map(fn(outcome) { outcome.name })
}
///|
fn error_rule_names(report : RuleReport) -> Array[String] {
report.outcomes
.filter(fn(outcome) { outcome.status == EvaluationError })
.map(fn(outcome) { outcome.name })
}
///|
fn same_string_set(left : Array[String], right : Array[String]) -> Bool {
left.length() == right.length() &&
left.all(fn(value) { right.contains(value) })
}
///|
/// Result of one rule regression case.
pub(all) struct RuleTestCaseReport {
name : String
matched : Bool
expected_passed : Bool
actual_passed : Bool
expected_failed_rules : Array[String]?
actual_failed_rules : Array[String]
actual_error_rules : Array[String]
rule_report : RuleReport
} derive(Eq, Debug, ToJson, FromJson)
///|
/// Aggregate result for a complete rule regression suite.
pub(all) struct RuleTestSuiteReport {
passed : Bool
case_count : Int
matched_count : Int
mismatched_count : Int
cases : Array[RuleTestCaseReport]
} derive(Eq, Debug, ToJson, FromJson)
///|
pub fn RuleTestSuiteReport::to_json_string(
self : RuleTestSuiteReport,
indent? : Int = 2,
) -> String {
self.to_json().stringify(indent~)
}
///|
/// Run a validated regression suite while reusing the already compiled rules.
pub fn RuleSet::run_test_suite(
self : RuleSet,
suite : RuleTestSuite,
) -> RuleTestSuiteReport {
let reports = []
let mut matched_count = 0
for case in suite.cases {
let rule_report = self.evaluate(case.input)
let actual_failed_rules = failed_rule_names(rule_report)
let actual_error_rules = error_rule_names(rule_report)
let failure_names_match = match case.expected_failed_rules {
Some(expected) => same_string_set(expected, actual_failed_rules)
None => true
}
let matched = case.expected_passed == rule_report.passed &&
failure_names_match &&
actual_error_rules.is_empty()
if matched {
matched_count = matched_count + 1
}
reports.push({
name: case.name,
matched,
expected_passed: case.expected_passed,
actual_passed: rule_report.passed,
expected_failed_rules: case.expected_failed_rules,
actual_failed_rules,
actual_error_rules,
rule_report,
})
}
{
passed: matched_count == suite.cases.length(),
case_count: suite.cases.length(),
matched_count,
mismatched_count: suite.cases.length() - matched_count,
cases: reports,
}
}