///|
test "parser accepts a minimal group" {
let f = unwrap_file(parse_robots("User-agent: *\nDisallow: /private/\n"))
assert_int_eq(f.group_count(), 1)
assert_int_eq(f.rule_count(), 1)
}
///|
test "parser accepts final line without newline" {
let f = unwrap_file(parse_robots("User-agent: *\nDisallow: /private/"))
assert_int_eq(f.group_count(), 1)
assert_int_eq(f.rule_count(), 1)
}
///|
test "parser accepts allow directives" {
let f = unwrap_file(parse_robots("User-agent: bot\nAllow: /public/\n"))
assert_true(f.groups()[0].rules()[0].kind() == Allow)
}
///|
test "parser accepts disallow directives" {
let f = unwrap_file(parse_robots("User-agent: bot\nDisallow: /tmp\n"))
assert_true(f.groups()[0].rules()[0].kind() == Disallow)
}
///|
test "parser lowercases user-agent tokens" {
let f = unwrap_file(parse_robots("User-agent: ExampleBot\nDisallow: /\n"))
assert_str_eq(f.groups()[0].user_agents()[0], "examplebot")
}
///|
test "parser handles uppercase field names" {
let f = unwrap_file(
parse_robots("USER-AGENT: bot\nDISALLOW: /\nALLOW: /ok\n"),
)
assert_int_eq(f.rule_count(), 2)
}
///|
test "parser trims whitespace around names and values" {
let f = unwrap_file(parse_robots(" User-agent : bot \n Disallow : /x \n"))
assert_str_eq(f.groups()[0].rules()[0].pattern(), "/x")
}
///|
test "parser strips full-line comments" {
let f = unwrap_file(parse_robots("# note\nUser-agent: *\nDisallow: /x\n"))
assert_int_eq(f.comments(), 1)
assert_int_eq(f.group_count(), 1)
}
///|
test "parser strips inline comments" {
let f = unwrap_file(
parse_robots("User-agent: * # all\nDisallow: /x # secret\n"),
)
assert_str_eq(f.groups()[0].rules()[0].pattern(), "/x")
assert_int_eq(f.comments(), 2)
}
///|
test "parser counts blank lines" {
let f = unwrap_file(parse_robots("User-agent: *\n\nDisallow: /\n"))
assert_int_eq(f.blank_lines(), 1)
}
///|
test "parser accepts CRLF newlines" {
let f = unwrap_file(parse_robots("User-agent: *\r\nDisallow: /\r\n"))
assert_str_eq(f.groups()[0].rules()[0].pattern(), "/")
}
///|
test "parser accepts CR before LF in mixed input" {
let f = unwrap_file(
parse_robots("User-agent: bot\r\nAllow: /a\nDisallow: /b\r\n"),
)
assert_int_eq(f.rule_count(), 2)
}
///|
test "parser accepts UTF-8 pattern text" {
let f = unwrap_file(parse_robots("User-agent: *\nDisallow: /私人/\n"))
assert_str_eq(f.groups()[0].rules()[0].pattern(), "/私人/")
}
///|
test "parser strips UTF-8 BOM" {
let f = unwrap_file(parse_robots("\u{FEFF}User-agent: *\nDisallow: /\n"))
assert_int_eq(f.group_count(), 1)
}
///|
test "parser preserves sitemap as other record" {
let f = unwrap_file(
parse_robots("User-agent: *\nDisallow: /x\nSitemap: https://e.test/s.xml\n"),
)
assert_int_eq(f.other_record_count(), 1)
assert_true(is_sitemap(f.other_records()[0]))
}
///|
test "parser preserves unknown records" {
let f = unwrap_file(
parse_robots("User-agent: *\nDisallow: /x\nCrawl-delay: 5\n"),
)
assert_str_eq(f.other_records()[0].name(), "Crawl-delay")
}
///|
test "parser starts a new group after rules" {
let f = unwrap_file(
parse_robots("User-agent: a\nDisallow: /a\nUser-agent: b\nAllow: /b\n"),
)
assert_int_eq(f.group_count(), 2)
}
///|
test "parser merges adjacent user-agent lines into same group" {
let f = unwrap_file(
parse_robots("User-agent: a\nUser-agent: b\nDisallow: /\n"),
)
assert_int_eq(f.group_count(), 1)
assert_int_eq(f.groups()[0].agent_count(), 2)
}
///|
test "parser rejects missing colon" {
expect_err_kind(parse_robots("User-agent *\n"), MissingColon)
}
///|
test "parser rejects empty user-agent" {
expect_err_kind(parse_robots("User-agent:\n"), EmptyUserAgent)
}
///|
test "parser rejects invalid user-agent token" {
expect_err_kind(parse_robots("User-agent: bad token\n"), InvalidUserAgent)
}
///|
test "parser rejects rule before first group" {
expect_err_kind(parse_robots("Disallow: /\n"), RuleBeforeFirstGroup)
}
///|
test "parser rejects malformed percent in rule" {
expect_err_kind(
parse_robots("User-agent: *\nDisallow: /x%ZZ\n"),
InvalidPercentEncoding,
)
}
///|
test "parser rejects illegal control characters" {
expect_err_kind(
parse_robots("User-agent: *\nDisallow: /x\u{0001}\n"),
ControlCharacter,
)
}
///|
test "parser reports bounded error context" {
let long = "User-agent " +
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
match parse_robots(long) {
Ok(_) => fail("expected error")
Err(e) => assert_true(e.context().length() <= max_context_bytes() + 3)
}
}