///|
pub fn shadowed_rules(group : Group) -> Array[String] {
let notes : Array[String] = []
for i = 0; i < group.rules.length(); i = i + 1 {
for j = i + 1; j < group.rules.length(); j = j + 1 {
let earlier = group.rules[i]
let later = group.rules[j]
if earlier.kind == later.kind && later.pattern.has_prefix(earlier.pattern) {
notes.push(
later.kind +
":" +
later.pattern +
" refines " +
earlier.kind +
":" +
earlier.pattern,
)
} else if earlier.pattern == later.pattern && earlier.kind != later.kind {
notes.push(
"conflicting exact rules for " +
earlier.pattern +
": " +
earlier.kind +
"/" +
later.kind,
)
}
}
}
notes
}
///|
pub fn audit(text : String) -> Array[String] {
let robots = parse(text)
let notes : Array[String] = []
if robots.groups.is_empty() {
notes.push("no groups")
}
if has_global_block(robots) {
notes.push("global wildcard blocks all paths")
}
for index = 0; index < robots.groups.length(); index = index + 1 {
let group = robots.groups[index]
if group.agents.is_empty() {
notes.push("group " + index.to_string() + " has no agents")
}
if group.rules.is_empty() {
notes.push("group " + index.to_string() + " has no path rules")
}
for note in shadowed_rules(group) {
notes.push("group " + index.to_string() + ": " + note)
}
}
notes
}
///|
pub fn audit_report(text : String) -> String {
let notes = audit(text)
if notes.is_empty() {
"ok"
} else {
join_lines(notes)
}
}
///|
pub fn risky_patterns(robots : Robots) -> Array[String] {
let result : Array[String] = []
for group in robots.groups {
for rule in group.rules {
if rule.pattern == "/" && is_disallow(rule) {
result.push("full-block")
} else if rule.pattern.find("*") is Some(_) && rule.pattern.length() <= 3 {
result.push("broad-wildcard:" + rule.pattern)
} else if rule.pattern.find("..") is Some(_) {
result.push("parent-segment:" + rule.pattern)
}
}
}
result
}
///|
pub fn compliance_report(text : String) -> String {
let lines : Array[String] = []
let parse = validation_report(text)
let lint = lint_report(text)
let audit = audit_report(text)
lines.push("validation=" + parse)
lines.push("lint=" + lint)
lines.push("audit=" + audit)
lines.push(stats_line(stats(text)))
join_lines(lines)
}
///|
pub fn rule_inventory(robots : Robots) -> Array[String] {
let lines : Array[String] = []
for group in robots.groups {
let agents = join_with(group.agents, ",")
for rule in group.rules {
lines.push(agents + " " + rule.kind + " " + rule.pattern)
}
}
lines
}
///|
pub fn inventory_report(text : String) -> String {
let lines = rule_inventory(parse(text))
if lines.is_empty() {
"empty"
} else {
join_lines(lines)
}
}