// User-agent group selection and path access decisions.
///|
fn group_has_agent(group : RobotsGroup, agent : String) -> Bool {
let expected = lower_ascii(agent)
for group_agent in group.user_agents() {
if lower_ascii(group_agent) == expected {
return true
}
}
false
}
///|
fn has_exact_agent_group(file : RobotsFile, product_token : String) -> Bool {
for group in file.groups() {
if group_has_agent(group, product_token) {
return true
}
}
false
}
///|
fn group_applies(
group : RobotsGroup,
product_token : String,
use_wildcard : Bool,
) -> Bool {
if use_wildcard {
group_has_agent(group, "*")
} else {
group_has_agent(group, product_token)
}
}
///|
pub fn rules_for(
file : RobotsFile,
product_token : String,
) -> Array[RobotsRule] {
let result : Array[RobotsRule] = []
let use_wildcard = !has_exact_agent_group(file, product_token)
for group in file.groups() {
if group_applies(group, product_token, use_wildcard) {
for rule in group.rules() {
result.push(rule)
}
}
}
result
}
///|
pub fn is_allowed(
file : RobotsFile,
product_token : String,
uri_path : String,
) -> Result[AccessDecision, RobotsError] {
evaluate(file, product_token, uri_path)
}
///|
pub fn evaluate(
file : RobotsFile,
product_token : String,
uri_path : String,
) -> Result[AccessDecision, RobotsError] {
match normalize_path_for_match(uri_path) {
Err(e) => Err(e)
Ok(normalized_path) => {
if normalized_path == "/robots.txt" {
return Ok(
access_decision(
true,
product_token,
uri_path,
normalized_path,
None,
0,
0,
"RFC 9309 allows the robots.txt resource itself",
),
)
}
let has_exact = has_exact_agent_group(file, product_token)
let use_wildcard = !has_exact
let mut has_applicable_group = false
for group in file.groups() {
if group_applies(group, product_token, use_wildcard) {
has_applicable_group = true
}
}
if !has_applicable_group {
return Ok(
access_decision(
true,
product_token,
uri_path,
normalized_path,
None,
0,
0,
"no matching user-agent group",
),
)
}
let mut matched_groups = 0
let mut best_rule : RobotsRule? = None
let mut best_len = -1
let mut best_allow = false
for group in file.groups() {
if group_applies(group, product_token, use_wildcard) {
matched_groups = matched_groups + 1
for rule in group.rules() {
if rule.pattern().length() > 0 &&
pattern_matches(rule.pattern(), normalized_path) {
let len = pattern_specificity(rule.pattern())
let allow = rule.kind() == Allow
if len > best_len || (len == best_len && allow && !best_allow) {
best_rule = Some(rule)
best_len = len
best_allow = allow
}
}
}
}
}
match best_rule {
None =>
Ok(
access_decision(
true,
product_token,
uri_path,
normalized_path,
None,
0,
matched_groups,
"matching group has no matching rule",
),
)
Some(rule) =>
Ok(
access_decision(
rule.kind() == Allow,
product_token,
uri_path,
normalized_path,
Some(rule),
best_len,
matched_groups,
"longest matching rule",
),
)
}
}
}
}
///|
pub fn can_fetch(
file : RobotsFile,
product_token : String,
uri_path : String,
) -> Bool {
match evaluate(file, product_token, uri_path) {
Ok(d) => d.allowed()
Err(_) => false
}
}