///|
pub(all) struct Scenario {
name : String
environment : Environment
} derive(Debug, Eq)
///|
pub fn scenario(name : String, environment : Environment) -> Scenario {
{ name, environment }
}
///|
pub(all) struct ScenarioResult {
name : String
truth : Truth
trace : Trace
changed_from_baseline : Bool
} derive(Debug, Eq)
///|
pub(all) struct BatchResult {
policy_name : String
baseline : Truth
scenarios : Array[ScenarioResult]
} derive(Debug, Eq)
///|
/// Evaluate named environments as a deterministic regression suite.
pub fn Policy::evaluate_scenarios(
self : Policy,
scenarios : Array[Scenario],
) -> BatchResult {
let baseline = self.evaluate().truth
let results : Array[ScenarioResult] = []
for item in scenarios {
let evaluation = evaluate(self.goal, item.environment)
results.push({
name: item.name,
truth: evaluation.truth,
trace: evaluation.trace,
changed_from_baseline: evaluation.truth != baseline,
})
}
{ policy_name: self.name, baseline, scenarios: results }
}
///|
pub fn BatchResult::changed(self : BatchResult) -> Array[ScenarioResult] {
self.scenarios.filter(item => item.changed_from_baseline)
}
///|
pub fn BatchResult::to_markdown(self : BatchResult) -> String {
let out = StringBuilder::new()
out.write_string("# Scenario regression: \{self.policy_name}\n\n")
out.write_string("Baseline: **\{self.baseline.render()}**\n\n")
out.write_string("| scenario | outcome | changed |\n")
out.write_string("| --- | --- | --- |\n")
for item in self.scenarios {
out.write_string(
"| \{markdown_cell(item.name)} | \{item.truth.render()} | \{item.changed_from_baseline} |\n",
)
}
out.to_string()
}
///|
fn markdown_cell(value : String) -> String {
value.replace_all(old="|", new="\\|").replace_all(old="\n", new=" ")
}
///|
pub(all) struct EnvironmentChange {
name : String
before : Value?
after : Value?
} derive(Debug, Eq)
///|
/// Deterministic environment diff used when explaining scenario regressions.
pub fn diff_environments(
before : Environment,
after : Environment,
) -> Array[EnvironmentChange] {
let result : Array[EnvironmentChange] = []
for binding in before {
let next = environment_get(after, binding.name)
if next != Some(binding.value) {
result.push({
name: binding.name,
before: Some(binding.value),
after: next,
})
}
}
for binding in after {
if environment_get(before, binding.name) is None {
result.push({
name: binding.name,
before: None,
after: Some(binding.value),
})
}
}
result
}