///|
/// Fault package for M4-FAULT-RESOURCE.
///|
pub fn package_id() -> String {
"lockwire/fault"
}
///|
pub(all) enum FaultKind {
Delay(@core.Duration)
Drop
Corrupt(offset~ : Int, value~ : Byte)
Duplicate
Reorder
} derive(Eq, Debug)
///|
pub fn FaultKind::label(self : FaultKind) -> String {
match self {
Delay(_) => "delay"
Drop => "drop"
Corrupt(..) => "corrupt"
Duplicate => "dup"
Reorder => "reorder"
}
}
///|
pub fn FaultKind::code(self : FaultKind) -> Int {
match self {
Delay(_) => 1
Drop => 2
Corrupt(..) => 3
Duplicate => 4
Reorder => 5
}
}
///|
pub(all) enum FaultSchedule {
Always
Once
CountDown(Int)
AfterStep(Int)
AtStep(Int)
} derive(Eq, Debug)
///|
pub(all) struct FaultRule {
kind : FaultKind
schedule : FaultSchedule
mut fired_count : Int
} derive(Eq, Debug)
///|
pub fn FaultRule::new(kind : FaultKind, schedule : FaultSchedule) -> FaultRule {
{ kind, schedule, fired_count: 0 }
}
///|
pub fn FaultRule::should_fire(self : FaultRule, step : Int) -> Bool {
let fires = match self.schedule {
Always => true
Once => self.fired_count == 0
CountDown(n) => self.fired_count < n
AfterStep(n) => step >= n
AtStep(n) => step == n && self.fired_count == 0
}
if fires {
self.fired_count += 1
}
fires
}
///|
pub(all) struct FaultHit {
seed : Int
step : Int
kind : FaultKind
label : String
} derive(Eq, Debug)
///|
pub fn FaultHit::to_trace_event(
self : FaultHit,
event_id~ : Int,
vtime~ : @core.VTime,
clock_domain~ : String,
node_id~ : String,
medium_id~ : String,
backend~ : @core.BackendProfile,
) -> @trace.TraceEvent {
@trace.TraceEvent::make(
event_id~,
parent_id=None,
vtime~,
clock_domain~,
raw_ns=vtime.ns(),
node_id~,
medium_id~,
channel_id=None,
direction=@trace.Fault,
payload_digest=None,
rng_step=self.step,
seed=self.seed,
backend~,
label=self.label,
)
}
///|
pub(all) enum TopologyFaultKind {
Partition
Heal
} derive(Eq, Debug)
///|
pub fn TopologyFaultKind::label(self : TopologyFaultKind) -> String {
match self {
Partition => "partition"
Heal => "heal"
}
}
///|
pub fn TopologyFaultKind::code(self : TopologyFaultKind) -> Int {
match self {
Partition => 101
Heal => 102
}
}
///|
pub(all) struct TopologyFaultHit {
seed : Int
step : Int
kind : TopologyFaultKind
label : String
} derive(Eq, Debug)
///|
pub fn TopologyFaultHit::make(
seed~ : Int,
step~ : Int,
kind~ : TopologyFaultKind,
) -> TopologyFaultHit {
{ seed, step, kind, label: "fabric." + kind.label() }
}
///|
pub fn TopologyFaultHit::to_trace_event(
self : TopologyFaultHit,
event_id~ : Int,
vtime~ : @core.VTime,
clock_domain~ : String,
node_id~ : String,
medium_id~ : String,
backend~ : @core.BackendProfile,
) -> @trace.TraceEvent {
@trace.TraceEvent::make(
event_id~,
parent_id=None,
vtime~,
clock_domain~,
raw_ns=vtime.ns(),
node_id~,
medium_id~,
channel_id=None,
direction=topology_fault_direction(self.kind),
payload_digest=Some(self.kind.code()),
rng_step=self.step,
seed=self.seed,
backend~,
label=self.label,
)
}
///|
fn topology_fault_direction(kind : TopologyFaultKind) -> @trace.TraceDirection {
match kind {
Partition => @trace.Fault
Heal => @trace.Meta
}
}
///|
pub(all) struct FaultOutcome {
hit : FaultHit?
frames : Array[Bytes]
delayed_by : @core.Duration?
dropped : Bool
} derive(Eq, Debug)
///|
pub fn FaultOutcome::is_faulted(self : FaultOutcome) -> Bool {
self.hit is Some(_)
}
///|
pub(all) struct FaultReport {
seed : Int
total_steps : Int
hits : Array[FaultHit]
} derive(Eq, Debug)
///|
pub(all) struct BuggifyGate {
point_id : String
threshold_per_mille : Int
fault_kind : FaultKind
} derive(Eq, Debug)
///|
pub fn BuggifyGate::make(
point_id~ : String,
threshold_per_mille~ : Int,
fault_kind~ : FaultKind,
) -> BuggifyGate {
{ point_id, threshold_per_mille, fault_kind }
}
///|
pub fn BuggifyGate::should_trigger(self : BuggifyGate, seed : Int) -> Bool {
gate_score(seed, self.point_id) < clamp_per_mille(self.threshold_per_mille)
}
///|
pub(all) struct SwarmSeedPool {
id : String
seeds : Array[Int]
} derive(Eq, Debug)
///|
pub fn SwarmSeedPool::make(id~ : String, seeds~ : Array[Int]) -> SwarmSeedPool {
{ id, seeds }
}
///|
pub fn default_swarm_seed_pool() -> SwarmSeedPool {
SwarmSeedPool::make(id="lockwire.l4a.default", seeds=[7, 11, 23, 31, 42])
}
///|
pub fn default_buggify_gates() -> Array[BuggifyGate] {
[
BuggifyGate::make(
point_id="retry.cleanup.delay",
threshold_per_mille=650,
fault_kind=Delay(@core.Duration::from_ns(2L)),
),
BuggifyGate::make(
point_id="rx.drop.boundary",
threshold_per_mille=450,
fault_kind=Drop,
),
BuggifyGate::make(
point_id="payload.corrupt.branch",
threshold_per_mille=350,
fault_kind=Corrupt(offset=0, value=b'!'),
),
]
}
///|
pub(all) struct FaultCampaignCase {
seed : Int
point_id : String
fault_kind_label : String
fault_kind_code : Int
triggered : Bool
fault_hit_count : Int
digest : @core.SimDigest
failed : Bool
failure_summary : String
} derive(Eq, Debug)
///|
pub(all) struct FaultCampaignReport {
campaign_id : String
seed_pool_id : String
seed_count : Int
gate_count : Int
cases : Array[FaultCampaignCase]
triggered_case_count : Int
fault_hit_count : Int
failure_count : Int
first_failure_seed : Int?
first_failure_point_id : String
first_failure_summary : String
shrink_candidate_seed : Int?
shrink_status : String
digest : @core.SimDigest
real_hot_path : Bool
} derive(Eq, Debug)
///|
pub(all) struct FaultPointCoverage {
point_id : String
case_count : Int
triggered_count : Int
covered : Bool
} derive(Eq, Debug)
///|
pub(all) struct BranchLabelCoverage {
label : String
hit_count : Int
covered : Bool
} derive(Eq, Debug)
///|
pub(all) struct SeedFailureHistogramBucket {
seed : Int
case_count : Int
failure_count : Int
first_failure_point_id : String
first_failure_digest : @core.SimDigest
} derive(Eq, Debug)
///|
pub(all) struct FaultCampaignCoverageReport {
campaign_id : String
seed_pool_id : String
coverage_source : String
scenario_count : Int
case_count : Int
seed_count : Int
fault_point_count : Int
covered_fault_point_count : Int
fault_point_coverage_per_mille : Int
branch_label_count : Int
covered_branch_label_count : Int
branch_label_coverage_per_mille : Int
failing_seed_count : Int
first_failure_seed : Int?
first_failure_point_id : String
first_failure_digest : @core.SimDigest
digest : @core.SimDigest
fault_points : Array[FaultPointCoverage]
branch_labels : Array[BranchLabelCoverage]
seed_failure_histogram : Array[SeedFailureHistogramBucket]
compiler_coverage_evidence : Bool
external_fuzzing_evidence : Bool
real_hot_path : Bool
} derive(Eq, Debug)
///|
pub fn FaultCampaignReport::passes(self : FaultCampaignReport) -> Bool {
self.seed_count > 0 &&
self.gate_count > 0 &&
self.cases.length() == self.seed_count * self.gate_count &&
self.triggered_case_count > 0 &&
self.fault_hit_count == self.triggered_case_count &&
self.failure_count > 0 &&
self.first_failure_summary != "" &&
self.shrink_status == "placeholder-first-failure-seed-not-minimized" &&
!self.real_hot_path
}
///|
pub fn FaultCampaignReport::to_text(self : FaultCampaignReport) -> String {
let buf = StringBuilder::new()
buf.write_string("schema=lockwire.fault-campaign-report.v1\n")
buf.write_string("campaign_id=" + self.campaign_id + "\n")
buf.write_string("seed_pool_id=" + self.seed_pool_id + "\n")
buf.write_string("seed_count=" + self.seed_count.to_string() + "\n")
buf.write_string("gate_count=" + self.gate_count.to_string() + "\n")
buf.write_string(
"triggered_case_count=" + self.triggered_case_count.to_string() + "\n",
)
buf.write_string("fault_hit_count=" + self.fault_hit_count.to_string() + "\n")
buf.write_string("failure_count=" + self.failure_count.to_string() + "\n")
buf.write_string(
"first_failure_seed=" + int_option_text(self.first_failure_seed) + "\n",
)
buf.write_string(
"first_failure_point_id=" + self.first_failure_point_id + "\n",
)
buf.write_string("first_failure_summary=" + self.first_failure_summary + "\n")
buf.write_string(
"shrink_candidate_seed=" +
int_option_text(self.shrink_candidate_seed) +
"\n",
)
buf.write_string("shrink_status=" + self.shrink_status + "\n")
buf.write_string(
"digest_state=" + self.digest.state_digest.to_string() + "\n",
)
buf.write_string(
"digest_events=" + self.digest.event_count.to_string() + "\n",
)
buf.write_string("real_hot_path=" + bool_text(self.real_hot_path) + "\n")
for case in self.cases {
buf.write_string("case=" + case.to_text_line() + "\n")
}
buf.to_string()
}
///|
pub fn FaultCampaignCoverageReport::passes(
self : FaultCampaignCoverageReport,
) -> Bool {
self.coverage_source == "fault-campaign-case-labels" &&
self.scenario_count == self.case_count &&
self.case_count > 0 &&
self.seed_count > 0 &&
self.seed_failure_histogram.length() == self.seed_count &&
self.fault_points.length() == self.fault_point_count &&
self.branch_labels.length() == self.branch_label_count &&
self.covered_fault_point_count > 0 &&
self.covered_fault_point_count <= self.fault_point_count &&
self.covered_branch_label_count > 0 &&
self.covered_branch_label_count <= self.branch_label_count &&
self.failing_seed_count > 0 &&
self.first_failure_point_id != "" &&
self.first_failure_digest.event_count > 0 &&
self.digest.event_count > 0 &&
!self.compiler_coverage_evidence &&
!self.external_fuzzing_evidence &&
!self.real_hot_path
}
///|
pub fn FaultCampaignCoverageReport::to_text(
self : FaultCampaignCoverageReport,
) -> String {
let buf = StringBuilder::new()
buf.write_string("schema=lockwire.fault-campaign-coverage-report.v1\n")
buf.write_string("campaign_id=" + self.campaign_id + "\n")
buf.write_string("seed_pool_id=" + self.seed_pool_id + "\n")
buf.write_string("coverage_source=" + self.coverage_source + "\n")
buf.write_string("scenario_count=" + self.scenario_count.to_string() + "\n")
buf.write_string("case_count=" + self.case_count.to_string() + "\n")
buf.write_string("seed_count=" + self.seed_count.to_string() + "\n")
buf.write_string(
"fault_point_count=" + self.fault_point_count.to_string() + "\n",
)
buf.write_string(
"covered_fault_point_count=" +
self.covered_fault_point_count.to_string() +
"\n",
)
buf.write_string(
"fault_point_coverage_per_mille=" +
self.fault_point_coverage_per_mille.to_string() +
"\n",
)
buf.write_string(
"branch_label_count=" + self.branch_label_count.to_string() + "\n",
)
buf.write_string(
"covered_branch_label_count=" +
self.covered_branch_label_count.to_string() +
"\n",
)
buf.write_string(
"branch_label_coverage_per_mille=" +
self.branch_label_coverage_per_mille.to_string() +
"\n",
)
buf.write_string("failing_seed_count=" + self.failing_seed_count.to_string())
buf.write_char('\n')
buf.write_string(
"first_failure_seed=" + int_option_text(self.first_failure_seed) + "\n",
)
buf.write_string(
"first_failure_point_id=" + self.first_failure_point_id + "\n",
)
buf.write_string(
"first_failure_digest_state=" +
self.first_failure_digest.state_digest.to_string() +
"\n",
)
buf.write_string(
"digest_state=" + self.digest.state_digest.to_string() + "\n",
)
buf.write_string(
"digest_events=" + self.digest.event_count.to_string() + "\n",
)
buf.write_string(
"compiler_coverage_evidence=" +
bool_text(self.compiler_coverage_evidence) +
"\n",
)
buf.write_string(
"external_fuzzing_evidence=" +
bool_text(self.external_fuzzing_evidence) +
"\n",
)
buf.write_string("real_hot_path=" + bool_text(self.real_hot_path) + "\n")
for point in self.fault_points {
buf.write_string("fault_point=" + point.to_text_line() + "\n")
}
for branch in self.branch_labels {
buf.write_string("branch_label=" + branch.to_text_line() + "\n")
}
for bucket in self.seed_failure_histogram {
buf.write_string("seed_failure=" + bucket.to_text_line() + "\n")
}
buf.to_string()
}
///|
pub fn run_buggify_swarm_seed_pool(
seed_pool : SwarmSeedPool,
gates : Array[BuggifyGate],
) -> FaultCampaignReport {
let cases : Array[FaultCampaignCase] = []
for seed in seed_pool.seeds {
for gate in gates {
cases.push(run_buggify_case(seed, gate))
}
}
let mut digest = @core.SimDigest::empty(seed=seed_pool.seeds.length())
digest = mix_string(digest, seed_pool.id)
for case in cases {
digest = mix_case(digest, case)
}
let first_failure = find_first_failure(cases)
{
campaign_id: seed_pool.id + ".buggify-swarm",
seed_pool_id: seed_pool.id,
seed_count: seed_pool.seeds.length(),
gate_count: gates.length(),
cases,
triggered_case_count: count_triggered(cases),
fault_hit_count: count_fault_hits(cases),
failure_count: count_failures(cases),
first_failure_seed: first_failure_seed(first_failure),
first_failure_point_id: first_failure_point(first_failure),
first_failure_summary: first_failure_summary(first_failure),
shrink_candidate_seed: first_failure_seed(first_failure),
shrink_status: "placeholder-first-failure-seed-not-minimized",
digest,
real_hot_path: false,
}
}
///|
pub fn buggify_swarm_seed_pool_fixture() -> FaultCampaignReport {
run_buggify_swarm_seed_pool(
default_swarm_seed_pool(),
default_buggify_gates(),
)
}
///|
pub fn fault_campaign_coverage_report(
report : FaultCampaignReport,
) -> FaultCampaignCoverageReport {
let fault_points = fault_point_coverage(report.cases)
let branch_labels = branch_label_coverage(report.cases)
let seed_failure_histogram = seed_failure_histogram(report.cases)
let covered_fault_point_count = count_covered_fault_points(fault_points)
let covered_branch_label_count = count_covered_branch_labels(branch_labels)
let first_failure = find_first_failure(report.cases)
let first_failure_digest = first_failure_digest(first_failure)
let failing_seed_count = count_failing_seeds(seed_failure_histogram)
let digest = coverage_digest(
report, fault_points, branch_labels, seed_failure_histogram, first_failure_digest,
)
{
campaign_id: report.campaign_id,
seed_pool_id: report.seed_pool_id,
coverage_source: "fault-campaign-case-labels",
scenario_count: report.cases.length(),
case_count: report.cases.length(),
seed_count: report.seed_count,
fault_point_count: fault_points.length(),
covered_fault_point_count,
fault_point_coverage_per_mille: per_mille(
covered_fault_point_count,
fault_points.length(),
),
branch_label_count: branch_labels.length(),
covered_branch_label_count,
branch_label_coverage_per_mille: per_mille(
covered_branch_label_count,
branch_labels.length(),
),
failing_seed_count,
first_failure_seed: first_failure_seed(first_failure),
first_failure_point_id: first_failure_point(first_failure),
first_failure_digest,
digest,
fault_points,
branch_labels,
seed_failure_histogram,
compiler_coverage_evidence: false,
external_fuzzing_evidence: false,
real_hot_path: report.real_hot_path,
}
}
///|
pub fn buggify_swarm_coverage_report_fixture() -> FaultCampaignCoverageReport {
fault_campaign_coverage_report(buggify_swarm_seed_pool_fixture())
}
///|
pub fn buggify_swarm_trace_fixture(
report : FaultCampaignReport,
) -> @trace.TraceLog {
let log = @trace.TraceLog::new()
let mut event_id = 1
for case in report.cases {
if case.triggered {
let ns = Int64::from_int(event_id) * 100L
log.append(
@trace.TraceEvent::make(
event_id~,
parent_id=None,
vtime=@core.VTime::from_ns(ns),
clock_domain="sim",
raw_ns=ns,
node_id="buggify-swarm",
medium_id=report.campaign_id,
channel_id=None,
direction=@trace.Fault,
payload_digest=Some(case.fault_kind_code),
rng_step=case.seed,
seed=case.seed,
backend=@core.SimNative,
label="buggify." + case.point_id,
),
)
event_id += 1
}
}
log
}
///|
pub struct FaultPlan {
priv seed : Int
priv rules : Array[FaultRule]
priv hits : Array[FaultHit]
priv mut step : Int
}
///|
pub fn FaultPlan::new(seed~ : Int) -> FaultPlan {
{ seed, rules: [], hits: [], step: 0 }
}
///|
pub fn FaultPlan::from_seed(seed~ : Int) -> FaultPlan {
let plan = FaultPlan::new(seed~)
let target = (if seed < 0 { -seed } else { seed }) % 5
plan.add_rule(
FaultRule::new(Delay(@core.Duration::from_ns(1L)), AtStep(target)),
)
plan
}
///|
pub fn FaultPlan::add_rule(self : FaultPlan, rule : FaultRule) -> Unit {
self.rules.push(rule)
}
///|
pub fn FaultPlan::step(self : FaultPlan) -> Int {
self.step
}
///|
pub fn FaultPlan::hit_count(self : FaultPlan) -> Int {
self.hits.length()
}
///|
pub fn FaultPlan::report(self : FaultPlan) -> FaultReport {
{ seed: self.seed, total_steps: self.step, hits: self.hits.copy() }
}
///|
pub fn FaultPlan::apply(self : FaultPlan, payload : Bytes) -> FaultOutcome {
let kind = self.next_kind()
let outcome = match kind {
None => forward([payload])
Some(Delay(duration)) =>
faulted(
self.record_hit(Delay(duration)),
[payload],
Some(duration),
false,
)
Some(Drop) => faulted(self.record_hit(Drop), [], None, true)
Some(Corrupt(offset~, value~)) => {
let hit = self.record_hit(Corrupt(offset~, value~))
faulted(hit, [corrupt_payload(payload, offset, value)], None, false)
}
Some(Duplicate) =>
faulted(self.record_hit(Duplicate), [payload, payload], None, false)
Some(Reorder) => faulted(self.record_hit(Reorder), [payload], None, false)
}
self.step += 1
outcome
}
///|
pub fn FaultPlan::apply_pair(
self : FaultPlan,
first : Bytes,
second : Bytes,
) -> FaultOutcome {
let kind = self.next_kind()
let outcome = match kind {
Some(Reorder) =>
faulted(self.record_hit(Reorder), [second, first], None, false)
Some(Drop) => faulted(self.record_hit(Drop), [second], None, true)
Some(Delay(duration)) =>
faulted(
self.record_hit(Delay(duration)),
[first, second],
Some(duration),
false,
)
Some(Corrupt(offset~, value~)) => {
let hit = self.record_hit(Corrupt(offset~, value~))
faulted(hit, [corrupt_payload(first, offset, value), second], None, false)
}
Some(Duplicate) =>
faulted(self.record_hit(Duplicate), [first, first, second], None, false)
None => forward([first, second])
}
self.step += 1
outcome
}
///|
fn FaultPlan::next_kind(self : FaultPlan) -> FaultKind? {
for rule in self.rules {
if rule.should_fire(self.step) {
return Some(rule.kind)
}
}
None
}
///|
fn FaultPlan::record_hit(self : FaultPlan, kind : FaultKind) -> FaultHit {
let hit : FaultHit = {
seed: self.seed,
step: self.step,
kind,
label: kind.label(),
}
self.hits.push(hit)
hit
}
///|
fn forward(frames : Array[Bytes]) -> FaultOutcome {
{ hit: None, frames, delayed_by: None, dropped: false }
}
///|
fn faulted(
hit : FaultHit,
frames : Array[Bytes],
delayed_by : @core.Duration?,
dropped : Bool,
) -> FaultOutcome {
{ hit: Some(hit), frames, delayed_by, dropped }
}
///|
fn corrupt_payload(payload : Bytes, offset : Int, value : Byte) -> Bytes {
if offset < 0 || offset >= payload.length() {
return payload
}
let buf = FixedArray::make(payload.length(), b'\x00')
for i in 0.. FaultCampaignCase {
let triggered = gate.should_trigger(seed)
let plan = FaultPlan::new(seed~)
if triggered {
plan.add_rule(FaultRule::new(gate.fault_kind, Once))
}
let outcome = plan.apply(case_payload(seed, gate.point_id))
let report = plan.report()
let failed = outcome.dropped
let failure_summary = if failed {
"seed=" +
seed.to_string() +
"|point=" +
gate.point_id +
"|kind=" +
gate.fault_kind.label() +
"|step=0"
} else {
""
}
{
seed,
point_id: gate.point_id,
fault_kind_label: gate.fault_kind.label(),
fault_kind_code: gate.fault_kind.code(),
triggered,
fault_hit_count: report.hits.length(),
digest: case_digest(seed, gate, outcome),
failed,
failure_summary,
}
}
///|
fn case_payload(seed : Int, point_id : String) -> Bytes {
Bytes::from_array([
(abs_int(seed) % 251).to_byte(),
(point_id.length() % 251).to_byte(),
b'L',
b'4',
b'A',
])
}
///|
fn case_digest(
seed : Int,
gate : BuggifyGate,
outcome : FaultOutcome,
) -> @core.SimDigest {
let mut digest = @core.SimDigest::empty(seed~)
digest = mix_string(digest, gate.point_id)
digest = digest.mix(gate.fault_kind.code())
digest = digest.mix(if gate.should_trigger(seed) { 1 } else { 0 })
digest = digest.mix(if outcome.is_faulted() { 1 } else { 0 })
digest = digest.mix(if outcome.dropped { 1 } else { 0 })
digest = digest.mix(outcome.frames.length())
for frame in outcome.frames {
digest = mix_bytes(digest, frame)
}
digest
}
///|
fn mix_case(
digest : @core.SimDigest,
case : FaultCampaignCase,
) -> @core.SimDigest {
let mut out = digest
out = out.mix(case.seed)
out = mix_string(out, case.point_id)
out = mix_string(out, case.fault_kind_label)
out = out.mix(case.fault_kind_code)
out = out.mix(if case.triggered { 1 } else { 0 })
out = out.mix(case.fault_hit_count)
out = out.mix(if case.failed { 1 } else { 0 })
out = mix_string(out, case.failure_summary)
out = out.mix(case.digest.state_digest.to_int())
out.mix(case.digest.event_count)
}
///|
fn mix_bytes(digest : @core.SimDigest, bytes : Bytes) -> @core.SimDigest {
let mut out = digest.mix(bytes.length())
for i in 0.. @core.SimDigest {
let mut out = digest.mix(value.length())
for c in value {
out = out.mix(c.to_int())
}
out
}
///|
fn gate_score(seed : Int, point_id : String) -> Int {
let mut score = abs_int(seed) * 131 + point_id.length()
for c in point_id {
score = (score * 33 + c.to_int()) % 1000
}
score % 1000
}
///|
fn clamp_per_mille(value : Int) -> Int {
if value < 0 {
0
} else if value > 1000 {
1000
} else {
value
}
}
///|
fn abs_int(value : Int) -> Int {
if value < 0 {
-value
} else {
value
}
}
///|
fn count_triggered(cases : ArrayView[FaultCampaignCase]) -> Int {
count_cases(cases, fn(case) { case.triggered })
}
///|
fn count_fault_hits(cases : ArrayView[FaultCampaignCase]) -> Int {
let mut count = 0
for case in cases {
count += case.fault_hit_count
}
count
}
///|
fn count_failures(cases : ArrayView[FaultCampaignCase]) -> Int {
count_cases(cases, fn(case) { case.failed })
}
///|
fn count_covered_fault_points(points : ArrayView[FaultPointCoverage]) -> Int {
let mut count = 0
for point in points {
if point.covered {
count += 1
}
}
count
}
///|
fn count_covered_branch_labels(labels : ArrayView[BranchLabelCoverage]) -> Int {
let mut count = 0
for label in labels {
if label.covered {
count += 1
}
}
count
}
///|
fn count_failing_seeds(buckets : ArrayView[SeedFailureHistogramBucket]) -> Int {
let mut count = 0
for bucket in buckets {
if bucket.failure_count > 0 {
count += 1
}
}
count
}
///|
fn count_cases(
cases : ArrayView[FaultCampaignCase],
pred : (FaultCampaignCase) -> Bool,
) -> Int {
let mut count = 0
for case in cases {
if pred(case) {
count += 1
}
}
count
}
///|
fn find_first_failure(
cases : ArrayView[FaultCampaignCase],
) -> FaultCampaignCase? {
for case in cases {
if case.failed {
return Some(case)
}
}
None
}
///|
fn fault_point_coverage(
cases : ArrayView[FaultCampaignCase],
) -> Array[FaultPointCoverage] {
let point_ids : Array[String] = []
for case in cases {
push_unique_string(point_ids, case.point_id)
}
let points : Array[FaultPointCoverage] = []
for point_id in point_ids {
let mut case_count = 0
let mut triggered_count = 0
for case in cases {
if case.point_id == point_id {
case_count += 1
if case.triggered {
triggered_count += 1
}
}
}
points.push({
point_id,
case_count,
triggered_count,
covered: triggered_count > 0,
})
}
points
}
///|
fn branch_label_coverage(
cases : ArrayView[FaultCampaignCase],
) -> Array[BranchLabelCoverage] {
let required = required_branch_labels(cases)
let observed = observed_branch_labels(cases)
let out : Array[BranchLabelCoverage] = []
for label in required {
let hit_count = count_string(observed, label)
out.push({ label, hit_count, covered: hit_count > 0 })
}
out
}
///|
fn required_branch_labels(
cases : ArrayView[FaultCampaignCase],
) -> Array[String] {
let labels : Array[String] = [
"scenario.case", "gate.evaluate", "case.triggered", "case.not-triggered", "fault-hit.present",
"fault-hit.absent", "outcome.failed", "outcome.survived",
]
for case in cases {
push_unique_string(labels, "fault-kind." + case.fault_kind_label)
}
labels
}
///|
fn observed_branch_labels(
cases : ArrayView[FaultCampaignCase],
) -> Array[String] {
let labels : Array[String] = []
for case in cases {
labels.push("scenario.case")
labels.push("gate.evaluate")
labels.push("fault-kind." + case.fault_kind_label)
if case.triggered {
labels.push("case.triggered")
labels.push("fault-hit.present")
} else {
labels.push("case.not-triggered")
labels.push("fault-hit.absent")
}
if case.failed {
labels.push("outcome.failed")
} else {
labels.push("outcome.survived")
}
}
labels
}
///|
fn seed_failure_histogram(
cases : ArrayView[FaultCampaignCase],
) -> Array[SeedFailureHistogramBucket] {
let seeds : Array[Int] = []
for case in cases {
push_unique_int(seeds, case.seed)
}
let buckets : Array[SeedFailureHistogramBucket] = []
for seed in seeds {
let mut case_count = 0
let mut failure_count = 0
let mut first_failure : FaultCampaignCase? = None
for case in cases {
if case.seed == seed {
case_count += 1
if case.failed {
failure_count += 1
if first_failure is None {
first_failure = Some(case)
}
}
}
}
buckets.push({
seed,
case_count,
failure_count,
first_failure_point_id: first_failure_point(first_failure),
first_failure_digest: first_failure_digest(first_failure),
})
}
buckets
}
///|
fn first_failure_seed(case : FaultCampaignCase?) -> Int? {
match case {
Some(value) => Some(value.seed)
None => None
}
}
///|
fn first_failure_point(case : FaultCampaignCase?) -> String {
match case {
Some(value) => value.point_id
None => ""
}
}
///|
fn first_failure_summary(case : FaultCampaignCase?) -> String {
match case {
Some(value) => value.failure_summary
None => ""
}
}
///|
fn first_failure_digest(case : FaultCampaignCase?) -> @core.SimDigest {
match case {
Some(value) => value.digest
None => @core.SimDigest::empty(seed=0)
}
}
///|
fn FaultCampaignCase::to_text_line(self : FaultCampaignCase) -> String {
"seed=" +
self.seed.to_string() +
"|point=" +
self.point_id +
"|kind=" +
self.fault_kind_label +
"|triggered=" +
bool_text(self.triggered) +
"|fault_hit_count=" +
self.fault_hit_count.to_string() +
"|digest=" +
self.digest.state_digest.to_string() +
"|failed=" +
bool_text(self.failed) +
"|failure=" +
self.failure_summary
}
///|
fn FaultPointCoverage::to_text_line(self : FaultPointCoverage) -> String {
"point=" +
self.point_id +
"|case_count=" +
self.case_count.to_string() +
"|triggered_count=" +
self.triggered_count.to_string() +
"|covered=" +
bool_text(self.covered)
}
///|
fn BranchLabelCoverage::to_text_line(self : BranchLabelCoverage) -> String {
"label=" +
self.label +
"|hit_count=" +
self.hit_count.to_string() +
"|covered=" +
bool_text(self.covered)
}
///|
fn SeedFailureHistogramBucket::to_text_line(
self : SeedFailureHistogramBucket,
) -> String {
"seed=" +
self.seed.to_string() +
"|case_count=" +
self.case_count.to_string() +
"|failure_count=" +
self.failure_count.to_string() +
"|first_failure_point_id=" +
self.first_failure_point_id +
"|first_failure_digest=" +
self.first_failure_digest.state_digest.to_string()
}
///|
fn coverage_digest(
report : FaultCampaignReport,
points : ArrayView[FaultPointCoverage],
branches : ArrayView[BranchLabelCoverage],
histogram : ArrayView[SeedFailureHistogramBucket],
first_failure_digest : @core.SimDigest,
) -> @core.SimDigest {
let mut digest = @core.SimDigest::empty(seed=report.seed_count)
digest = mix_string(digest, report.campaign_id)
digest = mix_string(digest, report.seed_pool_id)
digest = digest.mix(report.cases.length())
for point in points {
digest = mix_string(digest, point.point_id)
digest = digest.mix(point.case_count)
digest = digest.mix(point.triggered_count)
digest = digest.mix(if point.covered { 1 } else { 0 })
}
for branch in branches {
digest = mix_string(digest, branch.label)
digest = digest.mix(branch.hit_count)
digest = digest.mix(if branch.covered { 1 } else { 0 })
}
for bucket in histogram {
digest = digest.mix(bucket.seed)
digest = digest.mix(bucket.case_count)
digest = digest.mix(bucket.failure_count)
digest = mix_string(digest, bucket.first_failure_point_id)
digest = digest.mix(bucket.first_failure_digest.state_digest.to_int())
}
digest.mix(first_failure_digest.state_digest.to_int())
}
///|
fn per_mille(part : Int, total : Int) -> Int {
if total <= 0 {
0
} else {
part * 1000 / total
}
}
///|
fn push_unique_string(values : Array[String], value : String) -> Unit {
if !contains_string(values, value) {
values.push(value)
}
}
///|
fn push_unique_int(values : Array[Int], value : Int) -> Unit {
if !contains_int(values, value) {
values.push(value)
}
}
///|
fn count_string(values : ArrayView[String], value : String) -> Int {
let mut count = 0
for item in values {
if item == value {
count += 1
}
}
count
}
///|
fn contains_string(values : ArrayView[String], value : String) -> Bool {
for item in values {
if item == value {
return true
}
}
false
}
///|
fn contains_int(values : ArrayView[Int], value : Int) -> Bool {
for item in values {
if item == value {
return true
}
}
false
}
///|
fn int_option_text(value : Int?) -> String {
match value {
Some(v) => v.to_string()
None => ""
}
}
///|
fn bool_text(value : Bool) -> String {
if value {
"true"
} else {
"false"
}
}