///|
pub fn merge_groups_by_agent(robots : Robots) -> Robots {
  let agents : Array[String] = []
  for group in robots.groups {
    for agent in group.agents {
      if !contains_string(agents, agent) {
        agents.push(agent)
      }
    }
  }
  let groups : Array[Group] = []
  for agent in agents {
    let rules : Array[Rule] = []
    let mut delay : Int? = None
    for group in robots.groups {
      if contains_string(group.agents, agent) {
        for rule in group.rules {
          if !contains_rule(rules, rule) {
            rules.push(rule)
          }
        }
        if delay is None {
          delay = group.crawl_delay
        }
      }
    }
    groups.push({ agents: [agent], rules, crawl_delay: delay })
  }
  { groups, sitemaps: robots.sitemaps, hosts: robots.hosts }
}

///|
pub fn contains_rule(rules : Array[Rule], target : Rule) -> Bool {
  for rule in rules {
    if rule.kind == target.kind && rule.pattern == target.pattern {
      return true
    }
  }
  false
}

///|
pub fn remove_duplicate_rules(group : Group) -> Group {
  let rules : Array[Rule] = []
  for rule in group.rules {
    if !contains_rule(rules, rule) {
      rules.push(rule)
    }
  }
  { agents: group.agents, rules, crawl_delay: group.crawl_delay }
}

///|
pub fn normalize_robots(robots : Robots) -> Robots {
  let groups : Array[Group] = []
  for group in merge_groups_by_agent(robots).groups {
    groups.push(remove_duplicate_rules(group))
  }
  {
    groups,
    sitemaps: dedupe_urls(robots.sitemaps),
    hosts: unique_lower(robots.hosts),
  }
}

///|
pub fn unique_lower(values : Array[String]) -> Array[String] {
  let result : Array[String] = []
  for value in values {
    let lower = lower_ascii(value)
    if !contains_string(result, lower) {
      result.push(lower)
    }
  }
  result
}

///|
pub fn append_sitemap(robots : Robots, sitemap : String) -> Robots {
  let sitemaps : Array[String] = []
  for item in robots.sitemaps {
    sitemaps.push(item)
  }
  if !contains_string(sitemaps, sitemap) {
    sitemaps.push(sitemap)
  }
  { groups: robots.groups, sitemaps, hosts: robots.hosts }
}

///|
pub fn append_host(robots : Robots, host : String) -> Robots {
  let hosts : Array[String] = []
  for item in robots.hosts {
    hosts.push(item)
  }
  if !contains_string_case_insensitive(hosts, host) {
    hosts.push(host)
  }
  { groups: robots.groups, sitemaps: robots.sitemaps, hosts }
}

///|
pub fn invert_group(group : Group) -> Group {
  let rules : Array[Rule] = []
  for rule in group.rules {
    if is_allow(rule) {
      rules.push(disallow_rule(rule.pattern))
    } else {
      rules.push(allow_rule(rule.pattern))
    }
  }
  { agents: group.agents, rules, crawl_delay: group.crawl_delay }
}

///|
pub fn rules_for_kind(group : Group, kind : String) -> Array[String] {
  let result : Array[String] = []
  for rule in group.rules {
    if rule.kind == kind {
      result.push(rule.pattern)
    }
  }
  result
}

///|
pub fn remove_rules_with_prefix(group : Group, prefix : String) -> Group {
  let rules : Array[Rule] = []
  for rule in group.rules {
    if !rule.pattern.has_prefix(prefix) {
      rules.push(rule)
    }
  }
  { agents: group.agents, rules, crawl_delay: group.crawl_delay }
}

///|
pub fn transform_report(text : String) -> String {
  let before = stats(text)
  let after = stats_of(normalize_robots(parse(text)))
  "before " + stats_line(before) + "\nafter " + stats_line(after)
}