///|
/// Severity used by CI policy checks and human-facing audit output.
pub(all) enum TapSeverity {
SeverityInfo
SeverityWarning
SeverityError
} derive(Debug, Eq)
///|
/// Stable metadata for one known TapTrail issue code.
pub(all) struct TapIssueMeta {
code : String
severity : TapSeverity
category : String
title : String
remediation : String
} derive(Debug, Eq)
///|
/// Aggregated issue counts by severity.
pub(all) struct TapSeverityCounts {
infos : Int
warnings : Int
errors : Int
} derive(Debug, Eq)
///|
/// A configurable CI policy for TAP reports.
pub(all) struct TapPolicy {
name : String
require_version : Bool
require_plan : Bool
require_plan_match : Bool
require_monotonic_numbers : Bool
require_numbered_points : Bool
allow_bailout : Bool
allow_unknown_lines : Bool
allow_todo : Bool
allow_skip : Bool
max_failures : Int
max_parse_issues : Int
min_total : Int
min_passed : Int
} derive(Debug, Eq)
///|
/// Result of applying a policy to a report.
pub(all) struct TapPolicyResult {
policy_name : String
ok : Bool
score : Int
errors : Int
warnings : Int
infos : Int
messages : Array[String]
} derive(Debug, Eq)
///|
/// Policy suited for normal development branches.
pub fn default_policy() -> TapPolicy {
TapPolicy::{
name: "default",
require_version: true,
require_plan: true,
require_plan_match: true,
require_monotonic_numbers: true,
require_numbered_points: false,
allow_bailout: false,
allow_unknown_lines: false,
allow_todo: true,
allow_skip: true,
max_failures: 0,
max_parse_issues: 0,
min_total: 0,
min_passed: 0,
}
}
///|
/// Stricter policy for package release gates.
pub fn release_policy() -> TapPolicy {
TapPolicy::{
name: "release",
require_version: true,
require_plan: true,
require_plan_match: true,
require_monotonic_numbers: true,
require_numbered_points: true,
allow_bailout: false,
allow_unknown_lines: false,
allow_todo: true,
allow_skip: true,
max_failures: 0,
max_parse_issues: 0,
min_total: 1,
min_passed: 1,
}
}
///|
/// Policy for strict conformance checks.
pub fn strict_policy() -> TapPolicy {
TapPolicy::{
name: "strict",
require_version: true,
require_plan: true,
require_plan_match: true,
require_monotonic_numbers: true,
require_numbered_points: true,
allow_bailout: false,
allow_unknown_lines: false,
allow_todo: false,
allow_skip: false,
max_failures: 0,
max_parse_issues: 0,
min_total: 1,
min_passed: 1,
}
}
///|
/// Policy for exploratory runs where diagnostics are useful but failures should
/// still be visible.
pub fn audit_policy() -> TapPolicy {
TapPolicy::{
name: "audit",
require_version: true,
require_plan: true,
require_plan_match: true,
require_monotonic_numbers: true,
require_numbered_points: false,
allow_bailout: false,
allow_unknown_lines: true,
allow_todo: true,
allow_skip: true,
max_failures: 0,
max_parse_issues: 10,
min_total: 0,
min_passed: 0,
}
}
///|
/// Return stable metadata for a known issue code.
pub fn issue_meta(code : String) -> TapIssueMeta {
if code == "missing-version" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "format",
title: "Missing TAP version",
remediation: "Add `TAP version 13` as the first logical TAP line.",
}
} else if code == "unsupported-version" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "format",
title: "Unsupported TAP version",
remediation: "Emit TAP13 output or convert the stream before parsing.",
}
} else if code == "missing-plan" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "plan",
title: "Missing TAP plan",
remediation: "Emit a `1..N` plan line so CI can verify test count.",
}
} else if code == "invalid-plan" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "plan",
title: "Invalid TAP plan",
remediation: "Use a numeric plan such as `1..3` or `1..0 # SKIP reason`.",
}
} else if code == "plan-start" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "plan",
title: "Plan does not start at one",
remediation: "Use `1..N`; TapTrail treats other starts as release risk.",
}
} else if code == "plan-count" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "plan",
title: "Plan count mismatch",
remediation: "Make the number of emitted test points match the plan.",
}
} else if code == "invalid-point" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "point",
title: "Invalid test point",
remediation: "Emit `ok N - name` or `not ok N - name` lines.",
}
} else if code == "failed-point" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "result",
title: "Failing test point",
remediation: "Fix the failing test or mark known future work as TODO.",
}
} else if code == "duplicate-number" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "point",
title: "Duplicate test number",
remediation: "Ensure each numbered test point uses a unique number.",
}
} else if code == "non-monotonic" {
TapIssueMeta::{
code,
severity: SeverityWarning,
category: "point",
title: "Non-monotonic numbering",
remediation: "Emit numbered test points in strictly increasing order.",
}
} else if code == "bailout" {
TapIssueMeta::{
code,
severity: SeverityError,
category: "runtime",
title: "Bail out detected",
remediation: "Investigate the harness crash or explicit abort reason.",
}
} else if code == "unknown-line" {
TapIssueMeta::{
code,
severity: SeverityWarning,
category: "format",
title: "Unknown TAP line",
remediation: "Remove noise or prefix diagnostic comments with `#`.",
}
} else if code == "unterminated-yaml" {
TapIssueMeta::{
code,
severity: SeverityWarning,
category: "diagnostic",
title: "Unterminated YAMLish block",
remediation: "Close YAMLish diagnostic blocks with an indented `...` line.",
}
} else {
TapIssueMeta::{
code,
severity: SeverityWarning,
category: "unknown",
title: "Uncataloged issue",
remediation: "Inspect the TAP stream and update the issue catalog if needed.",
}
}
}
///|
/// Turn a severity enum into a short stable label.
pub fn severity_label(severity : TapSeverity) -> String {
match severity {
SeverityInfo => "info"
SeverityWarning => "warning"
SeverityError => "error"
}
}
///|
/// Numeric order used by external tools that need sortable severity.
pub fn severity_rank(severity : TapSeverity) -> Int {
match severity {
SeverityInfo => 1
SeverityWarning => 2
SeverityError => 3
}
}
///|
/// Return true if the issue should block a normal CI gate.
pub fn issue_is_blocking(code : String) -> Bool {
issue_meta(code).severity == SeverityError
}
///|
/// Return true if the issue came from parser or TAP shape concerns.
pub fn issue_is_format_related(code : String) -> Bool {
let category = issue_meta(code).category
category == "format" || category == "plan" || category == "point"
}
///|
/// Return true if the issue is about runtime execution outcome.
pub fn issue_is_runtime_related(code : String) -> Bool {
let category = issue_meta(code).category
category == "runtime" || category == "result"
}
///|
/// Count report issues by severity.
pub fn count_issue_severities(report : TapReport) -> TapSeverityCounts {
let mut infos = 0
let mut warnings = 0
let mut errors = 0
for item in report.issues {
match issue_meta(item.code).severity {
SeverityInfo => infos += 1
SeverityWarning => warnings += 1
SeverityError => errors += 1
}
}
TapSeverityCounts::{ infos, warnings, errors }
}
///|
/// Count non-TODO failing test points.
pub fn count_blocking_failures(report : TapReport) -> Int {
let mut count = 0
for point in report.document.points {
if !point.ok {
match point.directive {
Todo(_) => ()
_ => count += 1
}
}
}
count
}
///|
/// Count test points without explicit test numbers.
pub fn count_unnumbered_points(report : TapReport) -> Int {
let mut count = 0
for point in report.document.points {
if point.number == 0 {
count += 1
}
}
count
}
///|
/// Count test points with TODO directives.
pub fn count_todo_points(report : TapReport) -> Int {
let mut count = 0
for point in report.document.points {
match point.directive {
Todo(_) => count += 1
_ => ()
}
}
count
}
///|
/// Count test points with SKIP directives.
pub fn count_skip_points(report : TapReport) -> Int {
let mut count = 0
for point in report.document.points {
match point.directive {
Skip(_) => count += 1
_ => ()
}
}
count
}
///|
/// Count parse issues before semantic validation issues are added.
pub fn count_parse_issues(report : TapReport) -> Int {
report.document.parse_issues.length()
}
///|
/// Calculate a simple 0-100 health score for dashboards.
pub fn health_score(report : TapReport) -> Int {
let counts = count_issue_severities(report)
let mut score = 100
score -= counts.errors * 25
score -= counts.warnings * 10
score -= counts.infos * 2
if report.summary.total == 0 {
score -= 5
}
if report.summary.todo > 0 {
score -= report.summary.todo * 3
}
if report.summary.skipped > 0 {
score -= report.summary.skipped * 2
}
clamp_score(score)
}
///|
/// Evaluate one report with a named policy.
pub fn evaluate_policy(
report : TapReport,
policy : TapPolicy,
) -> TapPolicyResult {
let messages : Array[String] = []
let mut errors = 0
let mut warnings = 0
let mut infos = 0
for item in report.issues {
let meta = issue_meta(item.code)
let allowed = issue_allowed_by_policy(item.code, policy)
if allowed {
infos += 1
messages.push("allowed \{meta.category}/\{item.code}: \{item.message}")
} else {
match meta.severity {
SeverityError => errors += 1
SeverityWarning => warnings += 1
SeverityInfo => infos += 1
}
messages.push(
"\{severity_label(meta.severity)} \{meta.category}/\{item.code}: \{item.message}",
)
}
}
if policy.require_version && !report.document.has_version {
errors += 1
messages.push("policy requires a TAP version line")
}
if policy.require_plan && !report.document.has_plan {
errors += 1
messages.push("policy requires a TAP plan line")
}
if policy.require_plan_match && report.document.has_plan {
if report.summary.planned != report.summary.total {
errors += 1
messages.push("policy requires plan count to match emitted points")
}
}
if policy.require_numbered_points {
let unnumbered = count_unnumbered_points(report)
if unnumbered > 0 {
errors += 1
messages.push("policy requires numbered points; found \{unnumbered}")
}
}
if !policy.allow_todo {
let todo = count_todo_points(report)
if todo > 0 {
errors += 1
messages.push("policy disallows TODO points; found \{todo}")
}
}
if !policy.allow_skip {
let skipped = count_skip_points(report)
if skipped > 0 {
errors += 1
messages.push("policy disallows skipped points; found \{skipped}")
}
}
let failures = count_blocking_failures(report)
if failures > policy.max_failures {
errors += 1
messages.push(
"blocking failures \{failures} exceed limit \{policy.max_failures}",
)
}
let parse_issues = count_parse_issues(report)
if parse_issues > policy.max_parse_issues {
errors += 1
messages.push(
"parse issues \{parse_issues} exceed limit \{policy.max_parse_issues}",
)
}
if report.summary.total < policy.min_total {
errors += 1
messages.push(
"total tests \{report.summary.total} below minimum \{policy.min_total}",
)
}
if report.summary.passed < policy.min_passed {
errors += 1
messages.push(
"passed tests \{report.summary.passed} below minimum \{policy.min_passed}",
)
}
if messages.is_empty() {
messages.push("policy \{policy.name} passed without findings")
}
let score = policy_score(report, errors, warnings, infos)
TapPolicyResult::{
policy_name: policy.name,
ok: errors == 0,
score,
errors,
warnings,
infos,
messages,
}
}
///|
/// Return true when a policy accepts a report.
pub fn policy_passes(report : TapReport, policy : TapPolicy) -> Bool {
evaluate_policy(report, policy).ok
}
///|
/// Render one policy result as Markdown.
pub fn policy_result_to_markdown(result : TapPolicyResult) -> String {
let sb = StringBuilder()
sb.write_string("## Policy: ")
sb.write_string(result.policy_name)
sb.write_string("\n\n")
sb.write_string("- status: ")
sb.write_string(if result.ok { "ok" } else { "failed" })
sb.write_string("\n")
sb.write_string("- score: \{result.score}\n")
sb.write_string("- errors: \{result.errors}\n")
sb.write_string("- warnings: \{result.warnings}\n")
sb.write_string("- infos: \{result.infos}\n")
sb.write_string("\n")
for message in result.messages {
sb.write_string("- ")
sb.write_string(message)
sb.write_string("\n")
}
sb.to_string()
}
///|
/// Render all known issue metadata as Markdown.
pub fn issue_catalog_markdown() -> String {
let codes = known_issue_codes()
let sb = StringBuilder()
sb.write_string("# TapTrail Issue Catalog\n\n")
sb.write_string("| Code | Severity | Category | Title | Remediation |\n")
sb.write_string("| --- | --- | --- | --- | --- |\n")
for code in codes {
let meta = issue_meta(code)
sb.write_string("| `")
sb.write_string(meta.code)
sb.write_string("` | ")
sb.write_string(severity_label(meta.severity))
sb.write_string(" | ")
sb.write_string(meta.category)
sb.write_string(" | ")
sb.write_string(meta.title)
sb.write_string(" | ")
sb.write_string(meta.remediation)
sb.write_string(" |\n")
}
sb.to_string()
}
///|
/// Return issue codes currently produced by TapTrail validators.
pub fn known_issue_codes() -> Array[String] {
[
"missing-version", "unsupported-version", "missing-plan", "invalid-plan", "plan-start",
"plan-count", "invalid-point", "failed-point", "duplicate-number", "non-monotonic",
"bailout", "unknown-line", "unterminated-yaml",
]
}
///|
/// Render a short one-line status for terminals.
pub fn policy_status_line(result : TapPolicyResult) -> String {
"\{result.policy_name}: \{if result.ok { "ok" } else { "failed" }} score=\{result.score} errors=\{result.errors} warnings=\{result.warnings}"
}
///|
/// Return a textual grade for a score.
pub fn score_grade(score : Int) -> String {
if score >= 95 {
"excellent"
} else if score >= 80 {
"good"
} else if score >= 60 {
"needs-work"
} else {
"risk"
}
}
///|
/// Render policy settings so release notes can capture the gate definition.
pub fn policy_to_markdown(policy : TapPolicy) -> String {
let sb = StringBuilder()
sb.write_string("## Policy Settings\n\n")
sb.write_string("- name: \{policy.name}\n")
sb.write_string("- require_version: \{policy.require_version}\n")
sb.write_string("- require_plan: \{policy.require_plan}\n")
sb.write_string("- require_plan_match: \{policy.require_plan_match}\n")
sb.write_string(
"- require_monotonic_numbers: \{policy.require_monotonic_numbers}\n",
)
sb.write_string(
"- require_numbered_points: \{policy.require_numbered_points}\n",
)
sb.write_string("- allow_bailout: \{policy.allow_bailout}\n")
sb.write_string("- allow_unknown_lines: \{policy.allow_unknown_lines}\n")
sb.write_string("- allow_todo: \{policy.allow_todo}\n")
sb.write_string("- allow_skip: \{policy.allow_skip}\n")
sb.write_string("- max_failures: \{policy.max_failures}\n")
sb.write_string("- max_parse_issues: \{policy.max_parse_issues}\n")
sb.write_string("- min_total: \{policy.min_total}\n")
sb.write_string("- min_passed: \{policy.min_passed}\n")
sb.to_string()
}
///|
/// Convert a report and policy into a compact audit record.
pub fn policy_audit_markdown(report : TapReport, policy : TapPolicy) -> String {
let result = evaluate_policy(report, policy)
let sb = StringBuilder()
sb.write_string(to_markdown(report))
sb.write_string("\n")
sb.write_string(policy_result_to_markdown(result))
sb.to_string()
}
///|
/// Return true if a code is accepted by policy switches.
fn issue_allowed_by_policy(code : String, policy : TapPolicy) -> Bool {
if code == "unknown-line" {
policy.allow_unknown_lines
} else if code == "bailout" {
policy.allow_bailout
} else if code == "non-monotonic" {
!policy.require_monotonic_numbers
} else if code == "plan-count" {
!policy.require_plan_match
} else if code == "missing-version" || code == "unsupported-version" {
!policy.require_version
} else if code == "missing-plan" ||
code == "invalid-plan" ||
code == "plan-start" {
!policy.require_plan
} else {
false
}
}
///|
fn policy_score(
report : TapReport,
errors : Int,
warnings : Int,
infos : Int,
) -> Int {
let mut score = health_score(report)
score -= errors * 10
score -= warnings * 5
score -= infos
clamp_score(score)
}
///|
fn clamp_score(score : Int) -> Int {
if score < 0 {
0
} else if score > 100 {
100
} else {
score
}
}