///|
pub struct RobotsStats {
groups : Int
agents : Int
allows : Int
disallows : Int
crawl_delays : Int
sitemaps : Int
hosts : Int
} derive(Debug, Eq)
///|
pub fn stats(text : String) -> RobotsStats {
stats_of(parse(text))
}
///|
pub fn stats_of(robots : Robots) -> RobotsStats {
let mut agents = 0
let mut allows = 0
let mut disallows = 0
let mut delays = 0
for group in robots.groups {
agents += group.agents.length()
if group.crawl_delay is Some(_) {
delays += 1
}
for rule in group.rules {
if is_allow(rule) {
allows += 1
} else {
disallows += 1
}
}
}
{
groups: robots.groups.length(),
agents,
allows,
disallows,
crawl_delays: delays,
sitemaps: robots.sitemaps.length(),
hosts: robots.hosts.length(),
}
}
///|
pub fn stats_line(s : RobotsStats) -> String {
"groups=" +
s.groups.to_string() +
" agents=" +
s.agents.to_string() +
" allow=" +
s.allows.to_string() +
" disallow=" +
s.disallows.to_string() +
" delays=" +
s.crawl_delays.to_string() +
" sitemaps=" +
s.sitemaps.to_string() +
" hosts=" +
s.hosts.to_string()
}
///|
pub fn stats_table(text : String) -> String {
let s = stats(text)
join_lines([
"metric,value",
"groups," + s.groups.to_string(),
"agents," + s.agents.to_string(),
"allow," + s.allows.to_string(),
"disallow," + s.disallows.to_string(),
"crawl-delay," + s.crawl_delays.to_string(),
"sitemaps," + s.sitemaps.to_string(),
"hosts," + s.hosts.to_string(),
])
}
///|
pub fn has_global_block(robots : Robots) -> Bool {
for group in robots.groups {
if contains_string(group.agents, "*") {
for rule in group.rules {
if is_disallow(rule) && rule.pattern == "/" {
return true
}
}
}
}
false
}
///|
pub fn group_rule_count(group : Group, kind : String) -> Int {
let mut total = 0
for rule in group.rules {
if rule.kind == kind {
total += 1
}
}
total
}
///|
pub fn largest_group(robots : Robots) -> Group? {
let mut best : Group? = None
let mut size = -1
for group in robots.groups {
if group.rules.length() > size {
best = Some(group)
size = group.rules.length()
}
}
best
}
///|
pub fn largest_group_summary(text : String) -> String {
match largest_group(parse(text)) {
Some(group) => group_summary(group)
None => "none"
}
}