///|
pub struct FrontierItem {
  url : String
  path : String
  allowed : Bool
  reason : String
} derive(Debug, Eq)

///|
pub fn frontier(
  text : String,
  agent : String,
  urls : Array[String],
) -> Array[FrontierItem] {
  let robots = parse(text)
  let items : Array[FrontierItem] = []
  for url in dedupe_urls(urls) {
    let decision = decide_url(robots, agent, url)
    let host_ok = host_matches_robots(robots, url)
    let ok = decision.allowed && host_ok
    let reason = if !host_ok {
      "host"
    } else if decision.allowed {
      "allow"
    } else {
      decision.matched_pattern
    }
    items.push({ url, path: url_path(url), allowed: ok, reason })
  }
  items
}

///|
pub fn dedupe_urls(urls : Array[String]) -> Array[String] {
  let result : Array[String] = []
  for url in urls {
    if !contains_string(result, url) {
      result.push(url)
    }
  }
  result
}

///|
pub fn frontier_allowed(items : Array[FrontierItem]) -> Array[String] {
  let urls : Array[String] = []
  for item in items {
    if item.allowed {
      urls.push(item.url)
    }
  }
  urls
}

///|
pub fn frontier_blocked(items : Array[FrontierItem]) -> Array[String] {
  let urls : Array[String] = []
  for item in items {
    if !item.allowed {
      urls.push(item.url)
    }
  }
  urls
}

///|
pub fn frontier_report(items : Array[FrontierItem]) -> String {
  let lines : Array[String] = []
  for item in items {
    let state = if item.allowed { "allow" } else { "block" }
    lines.push(state + " " + item.url + " " + item.reason)
  }
  join_lines(lines)
}

///|
pub fn next_batch(
  text : String,
  agent : String,
  urls : Array[String],
  limit : Int,
) -> Array[String] {
  let result : Array[String] = []
  let items = frontier(text, agent, urls)
  for item in items {
    if item.allowed && result.length() < limit {
      result.push(item.url)
    }
  }
  result
}

///|
pub fn group_urls_by_host(urls : Array[String]) -> Array[String] {
  let lines : Array[String] = []
  let hosts : Array[String] = []
  for url in urls {
    match url_host(url) {
      Some(host) if !contains_string_case_insensitive(hosts, host) =>
        hosts.push(host)
      _ => ()
    }
  }
  for host in hosts {
    let mut count = 0
    for url in urls {
      match url_host(url) {
        Some(found) if lower_ascii(found) == lower_ascii(host) => count += 1
        _ => ()
      }
    }
    lines.push(host + "=" + count.to_string())
  }
  lines
}