///|
pub(all) enum Expectation {
Observe
Deliver(Array[Action])
FailWith(String)
} derive(Eq, Debug)
///|
pub(all) struct ReplayCase {
id : String
message : Message
expectation : Expectation
}
///|
pub(all) struct CaseResult {
id : String
plan : Plan?
error : Diagnostic?
expectation_met : Bool?
} derive(Eq, Debug)
///|
pub(all) struct ReplayLimits {
cases : Int
total_steps : Int
total_comparisons : Int
} derive(Eq, Debug)
///|
pub fn ReplayLimits::default() -> ReplayLimits {
{ cases: 1000, total_steps: 5000000, total_comparisons: 100000000 }
}
///|
pub(all) struct ReplayReport {
results : Array[CaseResult]
failed_expectations : Int
execution_errors : Int
charged_steps : Int
charged_comparisons : Int
} derive(Eq, Debug)
///|
fn equivalent_actions(left : Array[Action], right : Array[Action]) -> Bool {
left.all(a => right.any(b => same_delivery(a, b))) &&
right.all(a => left.any(b => same_delivery(a, b)))
}
///|
fn expectation_matches(
expectation : Expectation,
plan : Plan?,
error : Diagnostic?,
) -> Bool? {
match expectation {
Observe => None
Deliver(actions) =>
Some(
match plan {
Some(p) => equivalent_actions(p.actions, actions)
None => false
},
)
FailWith(code) =>
Some(
match error {
Some(d) => d.code == code
None => false
},
)
}
}
///|
pub fn Program::replay(
self : Program,
cases : Array[ReplayCase],
limits? : ReplayLimits = ReplayLimits::default(),
options? : ExecutionOptions = ExecutionOptions::default(),
) -> ReplayReport raise SieveError {
if limits.cases < 1 ||
limits.cases > 10000 ||
limits.total_steps < 1 ||
limits.total_steps > 100000000 ||
limits.total_comparisons < 1 ||
limits.total_comparisons > 1000000000 {
fail("replay.limits", "invalid replay limits", origin())
}
if cases.length() > limits.cases {
fail("replay.count", "too many replay cases", origin())
}
options.check()
let ids : Map[String, Bool] = Map([])
for item in cases {
if item.id.is_empty() ||
item.id.length() > 256 ||
!single_line(item.id) ||
ids.contains(item.id) {
fail(
"replay.id",
"case ids must be unique, bounded and nonempty",
origin(),
)
}
match item.expectation {
Deliver(actions) => {
if actions.length() > self.limits.actions {
fail("limit.actions", "expected action count exceeds limit", origin())
}
for action in actions {
match action {
FileInto(target) | Redirect(target) => {
if target.length() > self.limits.string_chars {
fail("limit.string", "expected destination too long", origin())
}
safe_destination(
target,
if action is Redirect(_) {
"redirect"
} else {
"fileinto"
},
origin(),
)
}
_ => ()
}
}
}
FailWith(code) =>
if code.is_empty() || code.length() > 256 {
fail(
"replay.expectation",
"expected error code must be bounded and nonempty",
origin(),
)
}
Observe => ()
}
ids[item.id] = true
}
let results : Array[CaseResult] = []
let mut steps = 0
let mut comparisons = 0
let mut failed = 0
let mut errors = 0
for item in cases {
let remaining_steps = limits.total_steps - steps
let remaining_comparisons = limits.total_comparisons - comparisons
let mut plan : Plan? = None
let mut error : Diagnostic? = None
if remaining_steps <= 0 || remaining_comparisons <= 0 {
error = Some({
code: "replay.budget",
message: "aggregate replay budget exhausted",
span: origin(),
})
} else {
let slice = {
..self.limits,
steps: if remaining_steps < self.limits.steps {
remaining_steps
} else {
self.limits.steps
},
comparisons: if remaining_comparisons < self.limits.comparisons {
remaining_comparisons
} else {
self.limits.comparisons
},
}
let bounded = { ..self, limits: slice }
try {
let result = bounded.run(item.message, options~)
steps += result.steps
comparisons += result.comparisons
plan = Some(result)
} catch {
SieveError(d) => {
error = Some(d)
steps += slice.steps
comparisons += slice.comparisons
}
}
}
let met = expectation_matches(item.expectation, plan, error)
if met == Some(false) {
failed += 1
}
if error is Some(_) {
errors += 1
}
results.push({ id: item.id, plan, error, expectation_met: met })
}
{
results,
failed_expectations: failed,
execution_errors: errors,
charged_steps: steps,
charged_comparisons: comparisons,
}
}
///|
pub(all) struct PolicyChange {
id : String
changed : Bool
removed : Array[Action]
added : Array[Action]
delivery_lost : Bool
new_redirect : Bool
before_error : String?
after_error : String?
} derive(Eq, Debug)
///|
pub fn compare_replays(
before : ReplayReport,
after : ReplayReport,
) -> Array[PolicyChange] raise SieveError {
if before.results.length() != after.results.length() {
fail(
"replay.alignment",
"reports must contain identical ordered case ids",
origin(),
)
}
let changes : Array[PolicyChange] = []
for i = 0; i < before.results.length(); i = i + 1 {
let left = before.results[i]
let right = after.results[i]
if left.id != right.id {
fail(
"replay.alignment",
"reports must contain identical ordered case ids",
origin(),
)
}
let a = match left.plan {
Some(p) => p.actions
None => []
}
let b = match right.plan {
Some(p) => p.actions
None => []
}
let removed = a.filter(x => !b.any(y => same_delivery(x, y)))
let added = b.filter(x => !a.any(y => same_delivery(x, y)))
let before_error = left.error.map(d => d.code)
let after_error = right.error.map(d => d.code)
let had_delivery = a.any(action => action != Discard)
let has_delivery = b.any(action => action != Discard)
changes.push({
id: left.id,
changed: !removed.is_empty() ||
!added.is_empty() ||
before_error != after_error,
removed,
added,
delivery_lost: had_delivery && !has_delivery,
new_redirect: added.any(action => {
match action {
Redirect(_) => true
_ => false
}
}),
before_error,
after_error,
})
}
changes
}
///|
pub fn ReplayReport::as_json(
self : ReplayReport,
redact_destinations? : Bool = false,
) -> Json {
Json::object({
"results": Json::array(
self.results.map(item => {
Json::object({
"id": Json::string(item.id),
"plan": match item.plan {
Some(p) => p.as_json(redact_destinations~)
None => Json::null()
},
"error": match item.error {
Some(d) => d.as_json()
None => Json::null()
},
"expectation_met": match item.expectation_met {
Some(v) => Json::boolean(v)
None => Json::null()
},
})
}),
),
"failed_expectations": Json::number(self.failed_expectations.to_double()),
"execution_errors": Json::number(self.execution_errors.to_double()),
"charged_steps": Json::number(self.charged_steps.to_double()),
"charged_comparisons": Json::number(self.charged_comparisons.to_double()),
})
}