///|
pub(all) enum TargetImportance {
ImportanceLow
ImportanceNormal
ImportanceHigh
ImportanceCritical
} derive(Eq, Debug)
///|
pub(all) enum ExpectationStatus {
ExpectationNotSet
ExpectationMatched
ExpectationMismatch
} derive(Eq, Debug)
///|
pub(all) enum CoverageBand {
CoverageEmpty
CoverageWeak
CoverageFair
CoverageStrong
} derive(Eq, Debug)
///|
pub(all) struct AuditTarget {
label : String
user_agent : String
url_or_path : String
expected_allowed : Bool?
importance : TargetImportance
note : String
} derive(Eq, Debug)
///|
pub(all) struct TargetAuditResult {
target : AuditTarget
decision : AccessDecision
snapshot : PathSnapshot
expectation_status : ExpectationStatus
matched_expected : Bool
importance_score : Int
risk_points : Int
tags : Array[String]
} derive(Eq, Debug)
///|
pub(all) struct SiteAuditSummary {
total_targets : Int
allowed_targets : Int
blocked_targets : Int
expected_targets : Int
matched_expectations : Int
mismatched_expectations : Int
critical_targets : Int
critical_mismatches : Int
high_risk_targets : Int
normalized_targets : Int
query_targets : Int
fragment_targets : Int
unique_agents : Int
coverage_band : CoverageBand
score : Int
notes : Array[String]
} derive(Eq, Debug)
///|
pub(all) struct SiteAuditReport {
policy : RobotsPolicy
targets : Array[AuditTarget]
results : Array[TargetAuditResult]
summary : SiteAuditSummary
rule_profile : PolicyProfile
} derive(Eq, Debug)
///|
pub(all) struct AgentMatrixRow {
user_agent : String
path : String
allowed : Bool
reason : String
matched_group : String
matched_pattern : String
} derive(Eq, Debug)
///|
pub(all) struct AgentMatrix {
rows : Array[AgentMatrixRow]
agent_count : Int
path_count : Int
allowed_count : Int
blocked_count : Int
} derive(Eq, Debug)
///|
pub fn TargetImportance::label(self : TargetImportance) -> String {
match self {
ImportanceLow => "low"
ImportanceNormal => "normal"
ImportanceHigh => "high"
ImportanceCritical => "critical"
}
}
///|
pub fn TargetImportance::score(self : TargetImportance) -> Int {
match self {
ImportanceLow => 1
ImportanceNormal => 2
ImportanceHigh => 4
ImportanceCritical => 8
}
}
///|
pub fn ExpectationStatus::label(self : ExpectationStatus) -> String {
match self {
ExpectationNotSet => "not-set"
ExpectationMatched => "matched"
ExpectationMismatch => "mismatch"
}
}
///|
pub fn CoverageBand::label(self : CoverageBand) -> String {
match self {
CoverageEmpty => "empty"
CoverageWeak => "weak"
CoverageFair => "fair"
CoverageStrong => "strong"
}
}
///|
pub fn target(
label : StringView,
user_agent : StringView,
url_or_path : StringView,
) -> AuditTarget {
{
label: label.to_owned(),
user_agent: user_agent.to_owned(),
url_or_path: url_or_path.to_owned(),
expected_allowed: None,
importance: ImportanceNormal,
note: "",
}
}
///|
pub fn expected_target(
label : StringView,
user_agent : StringView,
url_or_path : StringView,
expected_allowed : Bool,
importance : TargetImportance,
note : StringView,
) -> AuditTarget {
{
label: label.to_owned(),
user_agent: user_agent.to_owned(),
url_or_path: url_or_path.to_owned(),
expected_allowed: Some(expected_allowed),
importance,
note: note.to_owned(),
}
}
///|
pub fn public_target(
label : StringView,
url_or_path : StringView,
) -> AuditTarget {
expected_target(
label,
"*",
url_or_path,
true,
ImportanceNormal,
"public path should remain crawlable",
)
}
///|
pub fn private_target(
label : StringView,
url_or_path : StringView,
) -> AuditTarget {
expected_target(
label,
"*",
url_or_path,
false,
ImportanceHigh,
"private path should be blocked",
)
}
///|
pub fn critical_private_target(
label : StringView,
user_agent : StringView,
url_or_path : StringView,
) -> AuditTarget {
expected_target(
label,
user_agent,
url_or_path,
false,
ImportanceCritical,
"critical private path",
)
}
///|
pub fn AuditTarget::with_expected(
self : AuditTarget,
expected_allowed : Bool,
) -> AuditTarget {
{
label: self.label,
user_agent: self.user_agent,
url_or_path: self.url_or_path,
expected_allowed: Some(expected_allowed),
importance: self.importance,
note: self.note,
}
}
///|
pub fn AuditTarget::with_importance(
self : AuditTarget,
importance : TargetImportance,
) -> AuditTarget {
{
label: self.label,
user_agent: self.user_agent,
url_or_path: self.url_or_path,
expected_allowed: self.expected_allowed,
importance,
note: self.note,
}
}
///|
pub fn AuditTarget::with_note(
self : AuditTarget,
note : StringView,
) -> AuditTarget {
{
label: self.label,
user_agent: self.user_agent,
url_or_path: self.url_or_path,
expected_allowed: self.expected_allowed,
importance: self.importance,
note: note.to_owned(),
}
}
///|
pub fn AuditTarget::normalized_path(self : AuditTarget) -> String {
normalize_target(self.url_or_path).normalized
}
///|
pub fn AuditTarget::to_line(self : AuditTarget) -> String {
self.label +
" " +
self.user_agent +
" " +
self.url_or_path +
" expected=" +
optional_bool_label(self.expected_allowed) +
" importance=" +
self.importance.label()
}
///|
pub fn audit_target(
policy : RobotsPolicy,
target : AuditTarget,
) -> TargetAuditResult {
let snapshot = normalize_target(target.url_or_path)
let decision = policy.decide(target.user_agent, snapshot.normalized)
let expectation_status = match target.expected_allowed {
Some(expected) =>
if expected == decision.allowed {
ExpectationMatched
} else {
ExpectationMismatch
}
None => ExpectationNotSet
}
let matched_expected = expectation_status != ExpectationMismatch
let importance_score = target.importance.score()
let risk_points = target_risk_points(
target, decision, snapshot, expectation_status,
)
{
target,
decision,
snapshot,
expectation_status,
matched_expected,
importance_score,
risk_points,
tags: target_tags(
target, decision, snapshot, expectation_status, risk_points,
),
}
}
///|
pub fn audit_targets(
policy : RobotsPolicy,
targets : Array[AuditTarget],
) -> SiteAuditReport {
let results : Array[TargetAuditResult] = []
for item in targets {
results.push(audit_target(policy, item))
}
{
policy,
targets,
results,
summary: summarize_site_results(results),
rule_profile: profile_policy(policy),
}
}
///|
pub fn RobotsPolicy::audit_targets(
self : RobotsPolicy,
targets : Array[AuditTarget],
) -> SiteAuditReport {
audit_targets(self, targets)
}
///|
pub fn summarize_site_results(
results : Array[TargetAuditResult],
) -> SiteAuditSummary {
let notes : Array[String] = []
let mut total_targets = 0
let mut allowed_targets = 0
let mut blocked_targets = 0
let mut expected_targets = 0
let mut matched_expectations = 0
let mut mismatched_expectations = 0
let mut critical_targets = 0
let mut critical_mismatches = 0
let mut high_risk_targets = 0
let mut normalized_targets = 0
let mut query_targets = 0
let mut fragment_targets = 0
let mut risk_points = 0
let agents : Array[String] = []
for result in results {
total_targets = total_targets + 1
if result.decision.allowed {
allowed_targets = allowed_targets + 1
} else {
blocked_targets = blocked_targets + 1
}
if result.target.expected_allowed is Some(_) {
expected_targets = expected_targets + 1
}
if result.expectation_status == ExpectationMatched {
matched_expectations = matched_expectations + 1
}
if result.expectation_status == ExpectationMismatch {
mismatched_expectations = mismatched_expectations + 1
}
if result.target.importance == ImportanceCritical {
critical_targets = critical_targets + 1
if result.expectation_status == ExpectationMismatch {
critical_mismatches = critical_mismatches + 1
}
}
if result.risk_points >= 8 {
high_risk_targets = high_risk_targets + 1
}
if result.snapshot.collapsed_slashes ||
result.snapshot.removed_dot_segments ||
result.snapshot.root_fallback {
normalized_targets = normalized_targets + 1
}
if result.snapshot.had_query {
query_targets = query_targets + 1
}
if result.snapshot.had_fragment {
fragment_targets = fragment_targets + 1
}
risk_points = risk_points + result.risk_points
add_unique(agents, result.target.user_agent)
}
let coverage_band = classify_coverage(
total_targets, expected_targets, critical_targets,
)
let score = score_site_audit(
total_targets, matched_expectations, mismatched_expectations, critical_mismatches,
risk_points,
)
if total_targets == 0 {
notes.push("no audit targets were provided")
}
if expected_targets == 0 {
notes.push("no expected outcomes were configured")
}
if mismatched_expectations > 0 {
notes.push("some targets did not match the configured expectation")
}
if critical_mismatches > 0 {
notes.push("critical target mismatch requires review")
}
if query_targets > 0 {
notes.push("targets include query strings")
}
if normalized_targets > 0 {
notes.push("some targets required path normalization")
}
{
total_targets,
allowed_targets,
blocked_targets,
expected_targets,
matched_expectations,
mismatched_expectations,
critical_targets,
critical_mismatches,
high_risk_targets,
normalized_targets,
query_targets,
fragment_targets,
unique_agents: agents.length(),
coverage_band,
score,
notes,
}
}
///|
pub fn SiteAuditSummary::pass_rate_percent(self : SiteAuditSummary) -> Int {
if self.expected_targets == 0 {
0
} else {
self.matched_expectations * 100 / self.expected_targets
}
}
///|
pub fn SiteAuditSummary::is_clean(self : SiteAuditSummary) -> Bool {
self.mismatched_expectations == 0 && self.critical_mismatches == 0
}
///|
pub fn SiteAuditSummary::to_line(self : SiteAuditSummary) -> String {
"targets=" +
self.total_targets.to_string() +
" allowed=" +
self.allowed_targets.to_string() +
" blocked=" +
self.blocked_targets.to_string() +
" expected=" +
self.expected_targets.to_string() +
" matched=" +
self.matched_expectations.to_string() +
" mismatched=" +
self.mismatched_expectations.to_string() +
" score=" +
self.score.to_string()
}
///|
pub fn SiteAuditSummary::to_markdown(self : SiteAuditSummary) -> String {
let lines : Array[String] = []
lines.push("## Target Summary")
lines.push("")
lines.push("- Total targets: " + self.total_targets.to_string())
lines.push("- Allowed targets: " + self.allowed_targets.to_string())
lines.push("- Blocked targets: " + self.blocked_targets.to_string())
lines.push("- Expected targets: " + self.expected_targets.to_string())
lines.push("- Matched expectations: " + self.matched_expectations.to_string())
lines.push(
"- Mismatched expectations: " + self.mismatched_expectations.to_string(),
)
lines.push("- Critical targets: " + self.critical_targets.to_string())
lines.push("- Critical mismatches: " + self.critical_mismatches.to_string())
lines.push("- High risk targets: " + self.high_risk_targets.to_string())
lines.push("- Normalized targets: " + self.normalized_targets.to_string())
lines.push("- Query targets: " + self.query_targets.to_string())
lines.push("- Fragment targets: " + self.fragment_targets.to_string())
lines.push("- Unique agents: " + self.unique_agents.to_string())
lines.push("- Coverage band: " + self.coverage_band.label())
lines.push("- Pass rate: " + self.pass_rate_percent().to_string() + "%")
lines.push("- Score: " + self.score.to_string())
if self.notes.is_empty() {
lines.push("- Notes: none")
} else {
lines.push("- Notes: " + self.notes.join("; "))
}
lines.join("\n")
}
///|
pub fn TargetAuditResult::status_label(self : TargetAuditResult) -> String {
if self.decision.allowed {
"ALLOW"
} else {
"BLOCK"
}
}
///|
pub fn TargetAuditResult::expectation_label(self : TargetAuditResult) -> String {
self.expectation_status.label()
}
///|
pub fn TargetAuditResult::to_line(self : TargetAuditResult) -> String {
self.status_label() +
" " +
self.target.label +
" " +
self.snapshot.normalized +
" " +
self.expectation_label() +
" risk=" +
self.risk_points.to_string()
}
///|
pub fn TargetAuditResult::to_markdown_row(self : TargetAuditResult) -> String {
"| " +
self.target.label.replace_all(old="|", new="\\|") +
" | " +
self.target.user_agent.replace_all(old="|", new="\\|") +
" | " +
self.snapshot.normalized.replace_all(old="|", new="\\|") +
" | " +
self.status_label() +
" | " +
optional_bool_label(self.target.expected_allowed) +
" | " +
self.expectation_label() +
" | " +
self.risk_points.to_string() +
" |"
}
///|
pub fn TargetAuditResult::to_csv_row(self : TargetAuditResult) -> String {
csv_cell(self.target.label) +
"," +
csv_cell(self.target.user_agent) +
"," +
csv_cell(self.snapshot.normalized) +
"," +
csv_cell(self.status_label()) +
"," +
csv_cell(optional_bool_label(self.target.expected_allowed)) +
"," +
csv_cell(self.expectation_label()) +
"," +
csv_cell(self.risk_points.to_string()) +
"," +
csv_cell(self.decision.reason)
}
///|
pub fn SiteAuditReport::to_markdown(self : SiteAuditReport) -> String {
let lines : Array[String] = []
lines.push("# RoboPolicy Site Audit")
lines.push("")
lines.push(self.summary.to_markdown())
lines.push("")
lines.push("## Target Results")
lines.push("")
lines.push("| Label | Agent | Path | Decision | Expected | Status | Risk |")
lines.push("| --- | --- | --- | --- | --- | --- | --- |")
for result in self.results {
lines.push(result.to_markdown_row())
}
lines.push("")
lines.push("## Rule Profile")
lines.push("")
lines.push("- " + self.rule_profile.to_line())
lines.join("\n")
}
///|
pub fn SiteAuditReport::to_csv(self : SiteAuditReport) -> String {
let lines : Array[String] = []
lines.push("label,user_agent,path,decision,expected,status,risk,reason")
for result in self.results {
lines.push(result.to_csv_row())
}
lines.join("\n")
}
///|
pub fn SiteAuditReport::failed_results(
self : SiteAuditReport,
) -> Array[TargetAuditResult] {
let failed : Array[TargetAuditResult] = []
for result in self.results {
if result.expectation_status == ExpectationMismatch {
failed.push(result)
}
}
failed
}
///|
pub fn SiteAuditReport::critical_results(
self : SiteAuditReport,
) -> Array[TargetAuditResult] {
let critical : Array[TargetAuditResult] = []
for result in self.results {
if result.target.importance == ImportanceCritical {
critical.push(result)
}
}
critical
}
///|
pub fn SiteAuditReport::allowed_results(
self : SiteAuditReport,
) -> Array[TargetAuditResult] {
let allowed : Array[TargetAuditResult] = []
for result in self.results {
if result.decision.allowed {
allowed.push(result)
}
}
allowed
}
///|
pub fn SiteAuditReport::blocked_results(
self : SiteAuditReport,
) -> Array[TargetAuditResult] {
let blocked : Array[TargetAuditResult] = []
for result in self.results {
if !result.decision.allowed {
blocked.push(result)
}
}
blocked
}
///|
pub fn SiteAuditReport::is_clean(self : SiteAuditReport) -> Bool {
self.summary.is_clean()
}
///|
pub fn build_agent_matrix(
policy : RobotsPolicy,
user_agents : Array[String],
paths : Array[String],
) -> AgentMatrix {
let rows : Array[AgentMatrixRow] = []
let mut allowed_count = 0
let mut blocked_count = 0
for agent in user_agents {
for path in paths {
let decision = policy.decide(agent, path)
if decision.allowed {
allowed_count = allowed_count + 1
} else {
blocked_count = blocked_count + 1
}
rows.push({
user_agent: agent,
path,
allowed: decision.allowed,
reason: decision.reason,
matched_group: decision.matched_group,
matched_pattern: matched_pattern_label(decision.matched_rule),
})
}
}
{
rows,
agent_count: user_agents.length(),
path_count: paths.length(),
allowed_count,
blocked_count,
}
}
///|
pub fn AgentMatrix::to_markdown(self : AgentMatrix) -> String {
let lines : Array[String] = []
lines.push("# RoboPolicy Agent Matrix")
lines.push("")
lines.push("- Agents: " + self.agent_count.to_string())
lines.push("- Paths: " + self.path_count.to_string())
lines.push("- Allowed: " + self.allowed_count.to_string())
lines.push("- Blocked: " + self.blocked_count.to_string())
lines.push("")
lines.push("| Agent | Path | Decision | Group | Rule | Reason |")
lines.push("| --- | --- | --- | --- | --- | --- |")
for row in self.rows {
lines.push(row.to_markdown_row())
}
lines.join("\n")
}
///|
pub fn AgentMatrixRow::to_markdown_row(self : AgentMatrixRow) -> String {
"| " +
self.user_agent.replace_all(old="|", new="\\|") +
" | " +
self.path.replace_all(old="|", new="\\|") +
" | " +
(if self.allowed { "ALLOW" } else { "BLOCK" }) +
" | " +
self.matched_group.replace_all(old="|", new="\\|") +
" | " +
self.matched_pattern.replace_all(old="|", new="\\|") +
" | " +
self.reason.replace_all(old="|", new="\\|") +
" |"
}
///|
pub fn sample_targets() -> Array[AuditTarget] {
[
expected_target("home", "*", "/", true, ImportanceHigh, "home page"),
expected_target(
"admin",
"*",
"/admin/panel",
false,
ImportanceCritical,
"admin area",
),
expected_target(
"public-json",
"*",
"/tmp/public.json",
true,
ImportanceNormal,
"allowed json",
),
expected_target(
"hidden-json",
"*",
"/tmp/hidden.json",
false,
ImportanceHigh,
"hidden json",
),
expected_target(
"research-private",
"ResearchBot",
"/private/data.json",
false,
ImportanceCritical,
"research bot private data",
),
expected_target(
"research-summary",
"ResearchBot",
"/private/summary.html",
true,
ImportanceHigh,
"research summary",
),
]
}
///|
pub fn sample_site_audit_markdown() -> String {
audit_targets(parse(sample_robots()), sample_targets()).to_markdown()
}
///|
fn target_risk_points(
target : AuditTarget,
decision : AccessDecision,
snapshot : PathSnapshot,
expectation_status : ExpectationStatus,
) -> Int {
let mut risk = 0
if expectation_status == ExpectationMismatch {
risk = risk + target.importance.score() * 2
}
if target.importance == ImportanceCritical &&
expectation_status == ExpectationMismatch {
risk = risk + 8
}
if decision.allowed && target.expected_allowed is Some(false) {
risk = risk + 6
}
if !decision.allowed && target.expected_allowed is Some(true) {
risk = risk + 3
}
if snapshot.removed_dot_segments {
risk = risk + 2
}
if snapshot.collapsed_slashes {
risk = risk + 1
}
if snapshot.had_query {
risk = risk + 1
}
if snapshot.had_fragment {
risk = risk + 1
}
risk
}
///|
fn target_tags(
target : AuditTarget,
decision : AccessDecision,
snapshot : PathSnapshot,
expectation_status : ExpectationStatus,
risk_points : Int,
) -> Array[String] {
let tags : Array[String] = []
tags.push(target.importance.label())
if decision.allowed {
tags.push("allowed")
} else {
tags.push("blocked")
}
match expectation_status {
ExpectationNotSet => tags.push("no-expectation")
ExpectationMatched => tags.push("expected")
ExpectationMismatch => tags.push("mismatch")
}
if snapshot.had_query {
tags.push("query")
}
if snapshot.had_fragment {
tags.push("fragment")
}
if snapshot.collapsed_slashes {
tags.push("collapsed-slashes")
}
if snapshot.removed_dot_segments {
tags.push("dot-segments")
}
if risk_points >= 8 {
tags.push("high-risk")
} else if risk_points > 0 {
tags.push("review")
}
tags
}
///|
fn classify_coverage(
total_targets : Int,
expected_targets : Int,
critical_targets : Int,
) -> CoverageBand {
if total_targets == 0 {
CoverageEmpty
} else if expected_targets < 3 {
CoverageWeak
} else if expected_targets >= 6 && critical_targets >= 2 {
CoverageStrong
} else {
CoverageFair
}
}
///|
fn score_site_audit(
total_targets : Int,
matched_expectations : Int,
mismatched_expectations : Int,
critical_mismatches : Int,
risk_points : Int,
) -> Int {
let base = 50 + matched_expectations * 8 + total_targets * 2
let penalty = mismatched_expectations * 10 +
critical_mismatches * 20 +
risk_points
clamp_score(base - penalty)
}
///|
fn clamp_score(score : Int) -> Int {
if score < 0 {
0
} else if score > 100 {
100
} else {
score
}
}
///|
fn add_unique(items : Array[String], value : String) -> Unit {
if !items.contains(value) {
items.push(value)
}
}
///|
fn matched_pattern_label(rule : Rule?) -> String {
match rule {
Some(r) => r.pattern
None => ""
}
}
///|
fn optional_bool_label(value : Bool?) -> String {
match value {
Some(true) => "true"
Some(false) => "false"
None => "unset"
}
}
///|
fn csv_cell(value : String) -> String {
"\"" + value.replace_all(old="\"", new="\"\"") + "\""
}