// Core semantic model for RFC 9309 robots.txt files.
///|
pub(all) enum RuleKind {
Allow
Disallow
} derive(Eq, Debug)
///|
pub fn RuleKind::label(self : RuleKind) -> String {
match self {
Allow => "allow"
Disallow => "disallow"
}
}
///|
pub fn RuleKind::directive_name(self : RuleKind) -> String {
match self {
Allow => "Allow"
Disallow => "Disallow"
}
}
///|
pub fn RuleKind::json_name(self : RuleKind) -> String {
match self {
Allow => "allow"
Disallow => "disallow"
}
}
///|
pub struct RobotsRule {
kind : RuleKind
pattern : String
line : Int
byte_offset : Int
} derive(Eq, Debug)
///|
pub fn robots_rule(
kind : RuleKind,
pattern : String,
line : Int,
byte_offset : Int,
) -> RobotsRule {
{ kind, pattern, line, byte_offset }
}
///|
pub fn RobotsRule::kind(self : RobotsRule) -> RuleKind {
self.kind
}
///|
pub fn RobotsRule::pattern(self : RobotsRule) -> String {
self.pattern
}
///|
pub fn RobotsRule::line(self : RobotsRule) -> Int {
self.line
}
///|
pub fn RobotsRule::byte_offset(self : RobotsRule) -> Int {
self.byte_offset
}
///|
pub fn RobotsRule::is_empty(self : RobotsRule) -> Bool {
self.pattern.length() == 0
}
///|
pub struct RobotsGroup {
user_agents : Array[String]
rules : Array[RobotsRule]
start_line : Int
end_line : Int
} derive(Eq, Debug)
///|
pub fn robots_group(
user_agents : Array[String],
rules : Array[RobotsRule],
start_line : Int,
end_line : Int,
) -> RobotsGroup {
{ user_agents, rules, start_line, end_line }
}
///|
pub fn RobotsGroup::user_agents(self : RobotsGroup) -> Array[String] {
self.user_agents
}
///|
pub fn RobotsGroup::rules(self : RobotsGroup) -> Array[RobotsRule] {
self.rules
}
///|
pub fn RobotsGroup::start_line(self : RobotsGroup) -> Int {
self.start_line
}
///|
pub fn RobotsGroup::end_line(self : RobotsGroup) -> Int {
self.end_line
}
///|
pub fn RobotsGroup::agent_count(self : RobotsGroup) -> Int {
self.user_agents.length()
}
///|
pub fn RobotsGroup::rule_count(self : RobotsGroup) -> Int {
self.rules.length()
}
///|
pub struct OtherRecord {
name : String
value : String
line : Int
byte_offset : Int
} derive(Eq, Debug)
///|
pub fn other_record(
name : String,
value : String,
line : Int,
byte_offset : Int,
) -> OtherRecord {
{ name, value, line, byte_offset }
}
///|
pub fn OtherRecord::name(self : OtherRecord) -> String {
self.name
}
///|
pub fn OtherRecord::value(self : OtherRecord) -> String {
self.value
}
///|
pub fn OtherRecord::line(self : OtherRecord) -> Int {
self.line
}
///|
pub fn OtherRecord::byte_offset(self : OtherRecord) -> Int {
self.byte_offset
}
///|
pub fn OtherRecord::canonical_name(self : OtherRecord) -> String {
if self.name.equal_ignore_ascii_case("sitemap") {
"Sitemap"
} else {
self.name
}
}
///|
pub struct RobotsFile {
groups : Array[RobotsGroup]
other_records : Array[OtherRecord]
comments : Int
blank_lines : Int
source_lines : Int
} derive(Eq, Debug)
///|
pub fn robots_file(
groups : Array[RobotsGroup],
other_records : Array[OtherRecord],
comments : Int,
blank_lines : Int,
source_lines : Int,
) -> RobotsFile {
{ groups, other_records, comments, blank_lines, source_lines }
}
///|
pub fn empty_robots_file() -> RobotsFile {
{
groups: Array::new(),
other_records: Array::new(),
comments: 0,
blank_lines: 0,
source_lines: 0,
}
}
///|
pub fn RobotsFile::groups(self : RobotsFile) -> Array[RobotsGroup] {
self.groups
}
///|
pub fn RobotsFile::other_records(self : RobotsFile) -> Array[OtherRecord] {
self.other_records
}
///|
pub fn RobotsFile::comments(self : RobotsFile) -> Int {
self.comments
}
///|
pub fn RobotsFile::blank_lines(self : RobotsFile) -> Int {
self.blank_lines
}
///|
pub fn RobotsFile::source_lines(self : RobotsFile) -> Int {
self.source_lines
}
///|
pub fn RobotsFile::group_count(self : RobotsFile) -> Int {
self.groups.length()
}
///|
pub fn RobotsFile::other_record_count(self : RobotsFile) -> Int {
self.other_records.length()
}
///|
pub fn RobotsFile::rule_count(self : RobotsFile) -> Int {
let mut n = 0
for g in self.groups {
n = n + g.rules.length()
}
n
}
///|
pub fn RobotsFile::user_agent_count(self : RobotsFile) -> Int {
let mut n = 0
for g in self.groups {
n = n + g.user_agents.length()
}
n
}
///|
pub struct RobotsStats {
groups : Int
user_agents : Int
rules : Int
allows : Int
disallows : Int
other_records : Int
comments : Int
blank_lines : Int
source_lines : Int
} derive(Eq, Debug)
///|
pub fn RobotsStats::groups(self : RobotsStats) -> Int {
self.groups
}
///|
pub fn RobotsStats::user_agents(self : RobotsStats) -> Int {
self.user_agents
}
///|
pub fn RobotsStats::rules(self : RobotsStats) -> Int {
self.rules
}
///|
pub fn RobotsStats::allows(self : RobotsStats) -> Int {
self.allows
}
///|
pub fn RobotsStats::disallows(self : RobotsStats) -> Int {
self.disallows
}
///|
pub fn RobotsStats::other_records(self : RobotsStats) -> Int {
self.other_records
}
///|
pub fn RobotsStats::comments(self : RobotsStats) -> Int {
self.comments
}
///|
pub fn RobotsStats::blank_lines(self : RobotsStats) -> Int {
self.blank_lines
}
///|
pub fn RobotsStats::source_lines(self : RobotsStats) -> Int {
self.source_lines
}
///|
pub fn RobotsFile::stats(self : RobotsFile) -> RobotsStats {
let mut rules = 0
let mut allows = 0
let mut disallows = 0
let mut agents = 0
for g in self.groups {
agents = agents + g.user_agents.length()
for r in g.rules {
rules = rules + 1
if r.kind == Allow {
allows = allows + 1
} else {
disallows = disallows + 1
}
}
}
{
groups: self.groups.length(),
user_agents: agents,
rules,
allows,
disallows,
other_records: self.other_records.length(),
comments: self.comments,
blank_lines: self.blank_lines,
source_lines: self.source_lines,
}
}
///|
pub struct AccessDecision {
allowed : Bool
product_token : String
uri_path : String
normalized_path : String
matched_rule : RobotsRule?
matched_length : Int
matched_groups : Int
reason : String
} derive(Eq, Debug)
///|
pub fn access_decision(
allowed : Bool,
product_token : String,
uri_path : String,
normalized_path : String,
matched_rule : RobotsRule?,
matched_length : Int,
matched_groups : Int,
reason : String,
) -> AccessDecision {
{
allowed,
product_token,
uri_path,
normalized_path,
matched_rule,
matched_length,
matched_groups,
reason,
}
}
///|
pub fn AccessDecision::allowed(self : AccessDecision) -> Bool {
self.allowed
}
///|
pub fn AccessDecision::product_token(self : AccessDecision) -> String {
self.product_token
}
///|
pub fn AccessDecision::uri_path(self : AccessDecision) -> String {
self.uri_path
}
///|
pub fn AccessDecision::normalized_path(self : AccessDecision) -> String {
self.normalized_path
}
///|
pub fn AccessDecision::matched_rule(self : AccessDecision) -> RobotsRule? {
self.matched_rule
}
///|
pub fn AccessDecision::matched_length(self : AccessDecision) -> Int {
self.matched_length
}
///|
pub fn AccessDecision::matched_groups(self : AccessDecision) -> Int {
self.matched_groups
}
///|
pub fn AccessDecision::reason(self : AccessDecision) -> String {
self.reason
}
///|
pub fn AccessDecision::summary(self : AccessDecision) -> String {
match self.matched_rule {
None => {
let verdict = if self.allowed { "allowed" } else { "disallowed" }
"\{verdict} by default: \{self.reason}"
}
Some(r) => {
let verdict = if self.allowed { "allowed" } else { "disallowed" }
"\{verdict} by \{r.kind.label()} '\{r.pattern}' at line \{r.line}"
}
}
}
///|
pub fn library_version() -> String {
"0.1.1"
}