///|
/// Severity of an evidence-quality finding.
pub(all) enum EvidenceScoreLevel {
EvidenceError
EvidenceWarning
EvidenceNote
} derive(Eq, Debug)
///|
/// One quality issue found in a witness pack or source report.
pub(all) struct EvidenceScoreIssue {
code : String
level : EvidenceScoreLevel
case_id : String?
path : String
message : String
suggestion : String
} derive(Eq, Debug)
///|
/// Summary of whether generated witnesses are ready to be used as evidence.
pub(all) struct EvidenceScorecard {
contract_name : String
old_version : String
new_version : String
mode : CompatibilityMode
breaking_count : Int
covered_breaking_count : Int
case_count : Int
duplicate_case_ids : Int
duplicate_payloads : Int
sensitive_case_count : Int
weak_reason_count : Int
issues : Array[EvidenceScoreIssue]
} derive(Eq, Debug)
///|
pub fn EvidenceScoreLevel::render(self : EvidenceScoreLevel) -> String {
match self {
EvidenceError => "error"
EvidenceWarning => "warning"
EvidenceNote => "note"
}
}
///|
pub fn EvidenceScorecard::error_count(self : EvidenceScorecard) -> Int {
self.issues.count_if(issue => issue.level == EvidenceError)
}
///|
pub fn EvidenceScorecard::warning_count(self : EvidenceScorecard) -> Int {
self.issues.count_if(issue => issue.level == EvidenceWarning)
}
///|
pub fn EvidenceScorecard::note_count(self : EvidenceScorecard) -> Int {
self.issues.count_if(issue => issue.level == EvidenceNote)
}
///|
pub fn EvidenceScorecard::passed(self : EvidenceScorecard) -> Bool {
self.error_count() == 0
}
///|
/// Compact readiness state for humans and CI summaries.
pub fn EvidenceScorecard::readiness(self : EvidenceScorecard) -> String {
if self.error_count() > 0 {
"blocked"
} else if self.warning_count() > 0 || self.quality_score() < 90 {
"needs-review"
} else {
"ready"
}
}
///|
/// Explain why the current readiness label was selected.
pub fn EvidenceScorecard::readiness_reason(self : EvidenceScorecard) -> String {
let errors = self.error_count()
let warnings = self.warning_count()
if errors > 0 {
"blocking evidence defects must be fixed before the pack is reusable"
} else if warnings > 0 {
"warnings remain, so a maintainer should review the pack before sharing it"
} else if self.quality_score() < 90 {
"the score is below the recommended release-evidence threshold"
} else if self.note_count() > 0 {
"only non-blocking notes remain and the quality score is above threshold"
} else {
"no evidence-quality issues were detected"
}
}
///|
/// Suggest the next local action without turning EvoWitness into a submission checker.
pub fn EvidenceScorecard::next_action(self : EvidenceScorecard) -> String {
if self.error_count() > 0 {
"fix missing or malformed witness cases, then regenerate the scorecard"
} else if self.sensitive_case_count > 0 {
"review synthetic payloads for secret-like field names before publishing"
} else if self.weak_reason_count > 0 {
"expand weak witness reasons so the accept/reject boundary is clear"
} else if self.duplicate_case_ids > 0 {
"regenerate witness ids so downstream replay cases remain addressable"
} else if self.duplicate_payloads > 0 {
"optionally split shared payloads when reviewers need one payload per reason"
} else if self.warning_count() > 0 {
"inspect warning-level findings and decide whether they are acceptable"
} else {
"keep the witness pack with the release notes and replay it in CI"
}
}
///|
/// Score the pack as a release-review artifact. The score is deterministic and
/// intentionally simple so CI can set a threshold without hidden weights.
pub fn EvidenceScorecard::quality_score(self : EvidenceScorecard) -> Int {
let raw = 100 -
self.error_count() * 25 -
self.warning_count() * 8 -
self.note_count() * 2
if raw < 0 {
0
} else {
raw
}
}
///|
/// Score a full analysis report and its derived witness pack.
pub fn AnalysisReport::score_witness_evidence(
self : AnalysisReport,
) -> EvidenceScorecard {
let pack = self.to_witness_pack()
score_witness_pack_against_report(pack, self)
}
///|
/// Score a standalone witness pack when the original report is unavailable.
pub fn WitnessPack::score_standalone(self : WitnessPack) -> EvidenceScorecard {
score_witness_pack(self, self.cases.length(), self.cases.length())
}
///|
pub fn EvidenceScorecard::to_json(self : EvidenceScorecard) -> String {
let issues = self.issues.map(evidence_issue_to_json)
"{" +
"\"schemaVersion\":\"1.0\"," +
"\"kind\":\"evowitness-witness-scorecard\"," +
"\"contract\":" +
json_string(self.contract_name) +
"," +
"\"oldVersion\":" +
json_string(self.old_version) +
"," +
"\"newVersion\":" +
json_string(self.new_version) +
"," +
"\"mode\":" +
json_string(self.mode.render()) +
"," +
"\"passed\":" +
bool_json(self.passed()) +
"," +
"\"qualityScore\":" +
self.quality_score().to_string() +
"," +
"\"readiness\":" +
json_string(self.readiness()) +
"," +
"\"readinessReason\":" +
json_string(self.readiness_reason()) +
"," +
"\"nextAction\":" +
json_string(self.next_action()) +
"," +
"\"summary\":{" +
"\"breaking\":" +
self.breaking_count.to_string() +
"," +
"\"coveredBreaking\":" +
self.covered_breaking_count.to_string() +
"," +
"\"cases\":" +
self.case_count.to_string() +
"," +
"\"duplicateCaseIds\":" +
self.duplicate_case_ids.to_string() +
"," +
"\"duplicatePayloads\":" +
self.duplicate_payloads.to_string() +
"," +
"\"sensitiveCases\":" +
self.sensitive_case_count.to_string() +
"," +
"\"weakReasons\":" +
self.weak_reason_count.to_string() +
"," +
"\"errors\":" +
self.error_count().to_string() +
"," +
"\"warnings\":" +
self.warning_count().to_string() +
"," +
"\"notes\":" +
self.note_count().to_string() +
"}," +
"\"issues\":[" +
issues.join(",") +
"]}"
}
///|
pub fn EvidenceScorecard::to_markdown(self : EvidenceScorecard) -> String {
let out = StringBuilder()
out.write_string("# EvoWitness witness scorecard\n\n")
out.write_string(
"- Contract: `" + markdown_escape(self.contract_name) + "`\n",
)
out.write_string(
"- Versions: `" +
markdown_escape(self.old_version) +
"` -> `" +
markdown_escape(self.new_version) +
"`\n",
)
out.write_string("- Mode: `" + self.mode.render() + "`\n")
out.write_string(
"- Result: **" + (if self.passed() { "PASS" } else { "FAIL" }) + "**\n",
)
out.write_string(
"- Quality score: " + self.quality_score().to_string() + "\n",
)
out.write_string("- Readiness: `" + self.readiness() + "`\n")
out.write_string(
"- Readiness reason: " + markdown_escape(self.readiness_reason()) + "\n",
)
out.write_string(
"- Next action: " + markdown_escape(self.next_action()) + "\n",
)
out.write_string(
"- Breaking findings: " + self.breaking_count.to_string() + "\n",
)
out.write_string(
"- Covered by witnesses: " + self.covered_breaking_count.to_string() + "\n",
)
out.write_string("- Witness cases: " + self.case_count.to_string() + "\n\n")
out.write_string("| Metric | Count |\n")
out.write_string("|---|---|\n")
out.write_string(
"| Duplicate case ids | " + self.duplicate_case_ids.to_string() + " |\n",
)
out.write_string(
"| Duplicate payloads | " + self.duplicate_payloads.to_string() + " |\n",
)
out.write_string(
"| Sensitive cases | " + self.sensitive_case_count.to_string() + " |\n",
)
out.write_string(
"| Weak reasons | " + self.weak_reason_count.to_string() + " |\n",
)
out.write_string("| Errors | " + self.error_count().to_string() + " |\n")
out.write_string("| Warnings | " + self.warning_count().to_string() + " |\n")
out.write_string("| Notes | " + self.note_count().to_string() + " |\n\n")
if self.issues.is_empty() {
out.write_string("No evidence-quality issues were detected.\n")
return out.to_string()
}
out.write_string("| Level | Code | Case | Path | Message | Suggestion |\n")
out.write_string("|---|---|---|---|---|---|\n")
for issue in self.issues {
out.write_string("| " + issue.level.render() + " | `")
out.write_string(issue.code + "` | ")
out.write_string(markdown_escape(optional_case(issue.case_id)) + " | `")
out.write_string(markdown_escape(issue.path) + "` | ")
out.write_string(markdown_escape(issue.message) + " | ")
out.write_string(markdown_escape(issue.suggestion) + " |\n")
}
out.to_string()
}
///|
fn score_witness_pack_against_report(
pack : WitnessPack,
report : AnalysisReport,
) -> EvidenceScorecard {
let breaking_count = report.breaking_count()
let covered_breaking_count = report.changes.count_if(change => {
change.severity == Breaking && change.witness is Some(_)
})
let scorecard = score_witness_pack(
pack, breaking_count, covered_breaking_count,
)
let issues = scorecard.issues.copy()
for change in report.changes {
if change.severity == Breaking && change.witness is None {
issues.push(
evidence_issue(
"EVIDENCE_MISSING_WITNESS",
EvidenceError,
None,
change.path,
"breaking change '" + change.code + "' has no replayable witness",
"Construct a source-valid payload that the target contract rejects.",
),
)
}
}
rebuild_scorecard_with_issues(scorecard, issues)
}
///|
fn score_witness_pack(
pack : WitnessPack,
breaking_count : Int,
covered_breaking_count : Int,
) -> EvidenceScorecard {
let issues : Array[EvidenceScoreIssue] = []
let mut duplicate_case_ids = 0
let mut duplicate_payloads = 0
let mut sensitive_case_count = 0
let mut weak_reason_count = 0
if breaking_count > 0 && pack.cases.is_empty() {
issues.push(
evidence_issue(
"EVIDENCE_EMPTY_PACK",
EvidenceError,
None,
"$",
"the report has breaking findings but the witness pack is empty",
"Generate witness cases before using the report as acceptance evidence.",
),
)
}
if covered_breaking_count < breaking_count {
issues.push(
evidence_issue(
"EVIDENCE_INCOMPLETE_COVERAGE",
EvidenceError,
None,
"$",
"not every breaking finding is covered by a witness case",
"Every breaking rule should carry one accepted-by/rejected-by payload.",
),
)
}
let seen_ids : Array[String] = []
let seen_payloads : Array[String] = []
for index = 0; index < pack.cases.length(); index = index + 1 {
let case = pack.cases[index]
validate_case_shape(case, index + 1, issues)
if seen_ids.contains(case.id) {
duplicate_case_ids = duplicate_case_ids + 1
issues.push(
evidence_issue(
"EVIDENCE_DUPLICATE_CASE_ID",
EvidenceError,
Some(case.id),
case.path,
"case id is reused by more than one witness",
"Keep case ids unique so downstream tests can address failures.",
),
)
} else {
seen_ids.push(case.id)
}
let payload_key = case.accepted_by +
"->" +
case.rejected_by +
":" +
case.payload
if seen_payloads.contains(payload_key) {
duplicate_payloads = duplicate_payloads + 1
issues.push(
evidence_issue(
"EVIDENCE_SHARED_PAYLOAD",
EvidenceNote,
Some(case.id),
case.path,
"another case uses the same payload and contract direction",
"Shared payloads are allowed, but reviewers may want one reason per case.",
),
)
} else {
seen_payloads.push(payload_key)
}
if has_sensitive_signal(case) {
sensitive_case_count = sensitive_case_count + 1
issues.push(
evidence_issue(
"EVIDENCE_SENSITIVE_SIGNAL",
EvidenceWarning,
Some(case.id),
case.path,
"case path or payload resembles a secret-bearing field",
"Use synthetic values and review before uploading evidence externally.",
),
)
}
if has_weak_reason(case) {
weak_reason_count = weak_reason_count + 1
issues.push(
evidence_issue(
"EVIDENCE_WEAK_REASON",
EvidenceWarning,
Some(case.id),
case.path,
"case reason is too short to explain the compatibility failure",
"Include why the source accepts and why the target rejects the payload.",
),
)
}
}
{
contract_name: pack.contract_name,
old_version: pack.old_version,
new_version: pack.new_version,
mode: pack.mode,
breaking_count,
covered_breaking_count,
case_count: pack.cases.length(),
duplicate_case_ids,
duplicate_payloads,
sensitive_case_count,
weak_reason_count,
issues,
}
}
///|
fn validate_case_shape(
case : WitnessCase,
expected_index : Int,
issues : Array[EvidenceScoreIssue],
) -> Unit {
if !case.id.has_prefix("EW-") {
issues.push(
evidence_issue(
"EVIDENCE_CASE_ID_PREFIX",
EvidenceWarning,
Some(case.id),
case.path,
"case id does not use the expected EW- prefix",
"Use EvoWitness generated ids to keep report and replay output aligned.",
),
)
}
let expected_prefix = "EW-" + score_pad3(expected_index) + "-"
if case.id.has_prefix("EW-") && !case.id.has_prefix(expected_prefix) {
issues.push(
evidence_issue(
"EVIDENCE_CASE_ID_ORDER",
EvidenceNote,
Some(case.id),
case.path,
"case id ordinal does not match its pack position",
"Regenerate the witness pack after reordering findings.",
),
)
}
if case.code.is_empty() {
issues.push(
evidence_issue(
"EVIDENCE_EMPTY_CODE",
EvidenceError,
Some(case.id),
case.path,
"case has no rule code",
"Keep the original compatibility rule code on every witness case.",
),
)
}
if !case.path.has_prefix("$") {
issues.push(
evidence_issue(
"EVIDENCE_PATH_NOT_ABSOLUTE",
EvidenceWarning,
Some(case.id),
case.path,
"case path is not a stable contract path",
"Use '$' based paths such as '$.Order.id'.",
),
)
}
if case.payload.is_empty() {
issues.push(
evidence_issue(
"EVIDENCE_EMPTY_PAYLOAD",
EvidenceError,
Some(case.id),
case.path,
"case payload is empty",
"Store a syntactically valid JSON payload with every witness case.",
),
)
}
if case.accepted_by.is_empty() || case.rejected_by.is_empty() {
issues.push(
evidence_issue(
"EVIDENCE_MISSING_SIDE",
EvidenceError,
Some(case.id),
case.path,
"case is missing an accepted-by or rejected-by side",
"Record both contract versions so the assertion can be replayed.",
),
)
} else if case.accepted_by == case.rejected_by {
issues.push(
evidence_issue(
"EVIDENCE_SAME_SIDE",
EvidenceError,
Some(case.id),
case.path,
"case accepts and rejects against the same contract version",
"A counterexample must compare two different contract versions.",
),
)
}
if !case.assertion.contains(case.accepted_by) ||
!case.assertion.contains(case.rejected_by) {
issues.push(
evidence_issue(
"EVIDENCE_ASSERTION_MISMATCH",
EvidenceWarning,
Some(case.id),
case.path,
"case assertion does not mention both replay sides",
"Regenerate the assertion from accepted_by and rejected_by.",
),
)
}
}
///|
fn evidence_issue(
code : String,
level : EvidenceScoreLevel,
case_id : String?,
path : String,
message : String,
suggestion : String,
) -> EvidenceScoreIssue {
{ code, level, case_id, path, message, suggestion }
}
///|
fn rebuild_scorecard_with_issues(
scorecard : EvidenceScorecard,
issues : Array[EvidenceScoreIssue],
) -> EvidenceScorecard {
{
contract_name: scorecard.contract_name,
old_version: scorecard.old_version,
new_version: scorecard.new_version,
mode: scorecard.mode,
breaking_count: scorecard.breaking_count,
covered_breaking_count: scorecard.covered_breaking_count,
case_count: scorecard.case_count,
duplicate_case_ids: scorecard.duplicate_case_ids,
duplicate_payloads: scorecard.duplicate_payloads,
sensitive_case_count: scorecard.sensitive_case_count,
weak_reason_count: scorecard.weak_reason_count,
issues,
}
}
///|
fn evidence_issue_to_json(issue : EvidenceScoreIssue) -> String {
"{" +
"\"code\":" +
json_string(issue.code) +
"," +
"\"level\":" +
json_string(issue.level.render()) +
"," +
"\"caseId\":" +
optional_json_string(issue.case_id) +
"," +
"\"path\":" +
json_string(issue.path) +
"," +
"\"message\":" +
json_string(issue.message) +
"," +
"\"suggestion\":" +
json_string(issue.suggestion) +
"}"
}
///|
fn optional_case(value : String?) -> String {
match value {
Some(text) => "`" + text + "`"
None => "-"
}
}
///|
fn has_sensitive_signal(case : WitnessCase) -> Bool {
let haystack = (case.path + " " + case.payload).to_lower()
contains_sensitive_word(haystack)
}
///|
fn contains_sensitive_word(value : String) -> Bool {
value.contains("password") ||
value.contains("passwd") ||
value.contains("secret") ||
value.contains("token") ||
value.contains("credential") ||
value.contains("apikey") ||
value.contains("api_key") ||
value.contains("authorization")
}
///|
fn has_weak_reason(case : WitnessCase) -> Bool {
case.reason.length() < 24 ||
(
!case.reason.contains("source") &&
!case.reason.contains("receiver") &&
!case.reason.contains("target")
)
}
///|
fn score_pad3(value : Int) -> String {
if value < 10 {
"00" + value.to_string()
} else if value < 100 {
"0" + value.to_string()
} else {
value.to_string()
}
}