///|
pub struct CrawlPolicy {
  agent : String
  crawl_delay : Int?
  sitemaps : Array[String]
  hosts : Array[String]
  group_agents : Array[String]
} derive(Debug, Eq)

///|
pub fn policy_for(text : String, agent : String) -> CrawlPolicy {
  policy_from(parse(text), agent)
}

///|
pub fn policy_from(robots : Robots, agent : String) -> CrawlPolicy {
  match best_group(robots, agent) {
    Some(group) =>
      {
        agent,
        crawl_delay: group.crawl_delay,
        sitemaps: robots.sitemaps,
        hosts: robots.hosts,
        group_agents: group.agents,
      }
    None =>
      {
        agent,
        crawl_delay: None,
        sitemaps: robots.sitemaps,
        hosts: robots.hosts,
        group_agents: [],
      }
  }
}

///|
pub fn policy_summary(policy : CrawlPolicy) -> String {
  "agent=" +
  policy.agent +
  " delay=" +
  delay_text(policy.crawl_delay) +
  " group=" +
  join_with(policy.group_agents, ",") +
  " sitemaps=" +
  policy.sitemaps.length().to_string()
}

///|
pub fn allowed_paths(
  text : String,
  agent : String,
  paths : Array[String],
) -> Array[String] {
  let robots = parse(text)
  let result : Array[String] = []
  for path in paths {
    if decide(robots, agent, path).allowed {
      result.push(normalize_path(path))
    }
  }
  result
}

///|
pub fn disallowed_paths(
  text : String,
  agent : String,
  paths : Array[String],
) -> Array[String] {
  let robots = parse(text)
  let result : Array[String] = []
  for path in paths {
    if !decide(robots, agent, path).allowed {
      result.push(normalize_path(path))
    }
  }
  result
}

///|
pub fn filter_urls(
  text : String,
  agent : String,
  urls : Array[String],
) -> Array[String] {
  let robots = parse(text)
  let result : Array[String] = []
  for url in urls {
    if decide_url(robots, agent, url).allowed &&
      host_matches_robots(robots, url) {
      result.push(url)
    }
  }
  result
}

///|
pub fn blocked_report(
  text : String,
  agent : String,
  urls : Array[String],
) -> String {
  let robots = parse(text)
  let lines : Array[String] = []
  for url in urls {
    let decision = decide_url(robots, agent, url)
    if !decision.allowed {
      lines.push(url + " blocked by " + decision.matched_pattern)
    } else if !host_matches_robots(robots, url) {
      lines.push(url + " blocked by host")
    }
  }
  if lines.is_empty() {
    "ok"
  } else {
    join_lines(lines)
  }
}

///|
pub fn crawl_delay_or(text : String, agent : String, fallback : Int) -> Int {
  match policy_for(text, agent).crawl_delay {
    Some(delay) => delay
    None => fallback
  }
}