// Lightweight advisory audit for parsed robots.txt files.
///|
pub(all) enum AuditSeverity {
Info
Warning
} derive(Eq, Debug)
///|
pub(all) enum AuditKind {
EmptyUserAgent
RuleBeforeFirstGroup
EmptyRule
DuplicateRule
DuplicateUserAgent
UnknownRecord
VeryBroadDisallow
UnusedGroup
SuspiciousPattern
} derive(Eq, Debug)
///|
pub struct AuditIssue {
severity : AuditSeverity
kind : AuditKind
line : Int
message : String
} derive(Eq, Debug)
///|
pub fn AuditIssue::severity(self : AuditIssue) -> AuditSeverity {
self.severity
}
///|
pub fn AuditIssue::kind(self : AuditIssue) -> AuditKind {
self.kind
}
///|
pub fn AuditIssue::line(self : AuditIssue) -> Int {
self.line
}
///|
pub fn AuditIssue::message(self : AuditIssue) -> String {
self.message
}
///|
pub fn AuditSeverity::label(self : AuditSeverity) -> String {
match self {
Info => "info"
Warning => "warning"
}
}
///|
pub fn AuditKind::label(self : AuditKind) -> String {
match self {
EmptyUserAgent => "EmptyUserAgent"
RuleBeforeFirstGroup => "RuleBeforeFirstGroup"
EmptyRule => "EmptyRule"
DuplicateRule => "DuplicateRule"
DuplicateUserAgent => "DuplicateUserAgent"
UnknownRecord => "UnknownRecord"
VeryBroadDisallow => "VeryBroadDisallow"
UnusedGroup => "UnusedGroup"
SuspiciousPattern => "SuspiciousPattern"
}
}
///|
fn audit_issue(
severity : AuditSeverity,
kind : AuditKind,
line : Int,
message : String,
) -> AuditIssue {
{ severity, kind, line, message }
}
///|
fn rule_key(rule : RobotsRule) -> String {
rule.kind().label() + ":" + rule.pattern()
}
///|
pub fn audit_robots(file : RobotsFile) -> Array[AuditIssue] {
let issues : Array[AuditIssue] = []
let seen_agent_groups : Array[String] = []
for group in file.groups() {
let group_key = join_strings(group.user_agents(), ",")
if !push_unique_string(seen_agent_groups, group_key) {
issues.push(
audit_issue(
Info,
UnusedGroup,
group.start_line(),
"another group has the same user-agent set",
),
)
}
let seen_agents : Array[String] = []
for agent in group.user_agents() {
if !push_unique_string(seen_agents, agent) {
issues.push(
audit_issue(
Info,
DuplicateUserAgent,
group.start_line(),
"duplicate user-agent '\{agent}'",
),
)
}
}
let seen_rules : Array[String] = []
for rule in group.rules() {
if rule.is_empty() {
issues.push(
audit_issue(
Info,
EmptyRule,
rule.line(),
"empty allow/disallow rule has no effect",
),
)
}
if rule.kind() == Disallow && rule.pattern() == "/" {
issues.push(
audit_issue(
Warning,
VeryBroadDisallow,
rule.line(),
"Disallow: / blocks all paths for this group",
),
)
}
if !push_unique_string(seen_rules, rule_key(rule)) {
issues.push(
audit_issue(
Info,
DuplicateRule,
rule.line(),
"duplicate \{rule.kind().label()} rule '\{rule.pattern()}'",
),
)
}
if rule.pattern().contains("**") || count_char(rule.pattern(), '$') > 1 {
issues.push(
audit_issue(
Info,
SuspiciousPattern,
rule.line(),
"pattern contains unusual wildcard or end-marker use",
),
)
}
}
}
for record in file.other_records() {
if !record.name().equal_ignore_ascii_case("sitemap") {
issues.push(
audit_issue(
Info,
UnknownRecord,
record.line(),
"unknown record '\{record.name()}' was preserved",
),
)
}
}
issues
}
///|
pub fn audit_to_text(issues : Array[AuditIssue]) -> String {
let sb = StringBuilder::new()
for issue in issues {
sb.write_string(issue.severity().label())
sb.write_string(" ")
sb.write_string(issue.kind().label())
sb.write_string(" line ")
sb.write_string("\{issue.line()}: ")
sb.write_string(issue.message())
sb.write_char('\n')
}
sb.to_string()
}