///|
pub fn robots_url(site_url : String) -> String {
  let scheme = url_scheme_or(site_url, "https")
  let host = match url_host(site_url) {
    Some(found) => found
    None => trim_ascii(site_url)
  }
  scheme + "://" + host + "/robots.txt"
}

///|
pub fn url_scheme_or(url : String, fallback : String) -> String {
  match url.find("://") {
    Some(index) if index > 0 =>
      lower_ascii(url.unsafe_substring(start=0, end=index))
    _ => fallback
  }
}

///|
pub fn origin(url : String) -> String? {
  match url_host(url) {
    Some(host) => Some(url_scheme_or(url, "https") + "://" + host)
    None => None
  }
}

///|
pub fn origin_or(url : String, fallback : String) -> String {
  match origin(url) {
    Some(value) => value
    None => fallback
  }
}

///|
pub fn robots_urls_for(urls : Array[String]) -> Array[String] {
  let result : Array[String] = []
  for url in urls {
    let robots = robots_url(url)
    if !contains_string(result, robots) {
      result.push(robots)
    }
  }
  result
}

///|
pub fn same_origin_url(left : String, right : String) -> Bool {
  origin(left) == origin(right)
}

///|
pub fn attach_path(base_url : String, path : String) -> String {
  origin_or(base_url, "") + normalize_path(path)
}

///|
pub fn robots_source_report(urls : Array[String]) -> String {
  let lines : Array[String] = []
  for robots in robots_urls_for(urls) {
    lines.push(robots)
  }
  join_lines(lines)
}