///|
pub struct Rule {
kind : String
pattern : String
} derive(Debug, Eq)
///|
pub struct Group {
agents : Array[String]
rules : Array[Rule]
crawl_delay : Int?
} derive(Debug, Eq)
///|
pub struct Robots {
groups : Array[Group]
sitemaps : Array[String]
hosts : Array[String]
} derive(Debug, Eq)
///|
pub struct Decision {
allowed : Bool
agent : String
path : String
matched_kind : String
matched_pattern : String
} derive(Debug, Eq)
///|
pub fn empty_robots() -> Robots {
{ groups: [], sitemaps: [], hosts: [] }
}
///|
pub fn allow_rule(pattern : String) -> Rule {
{ kind: "allow", pattern }
}
///|
pub fn disallow_rule(pattern : String) -> Rule {
{ kind: "disallow", pattern }
}
///|
pub fn new_group(agent : String) -> Group {
{ agents: [lower_ascii(agent)], rules: [], crawl_delay: None }
}
///|
pub fn group_with(agents : Array[String], rules : Array[Rule]) -> Group {
let normalized : Array[String] = []
for agent in agents {
normalized.push(lower_ascii(agent))
}
{ agents: normalized, rules, crawl_delay: None }
}
///|
pub fn is_allow(rule : Rule) -> Bool {
rule.kind == "allow"
}
///|
pub fn is_disallow(rule : Rule) -> Bool {
rule.kind == "disallow"
}
///|
pub fn rule_weight(rule : Rule) -> Int {
rule.pattern.length()
}
///|
pub fn group_matches_agent(group : Group, agent : String) -> Bool {
let name = lower_ascii(agent)
for item in group.agents {
if item == "*" || name.find(item) is Some(_) {
return true
}
}
false
}
///|
pub fn group_agent_score(group : Group, agent : String) -> Int {
let name = lower_ascii(agent)
let mut best = -1
for item in group.agents {
if item == "*" {
if best < 0 {
best = 0
}
} else if name.find(item) is Some(_) && item.length() > best {
best = item.length()
}
}
best
}