///|
pub enum GainScheme {
  Linear
  Exp2
} derive(Debug, Eq, ToJson)

///|
pub struct JudgedDoc {
  query_id : String
  doc_id : String
  relevance : Int
} derive(Debug, Eq, ToJson)

///|
pub struct RetrievedDoc {
  query_id : String
  doc_id : String
  score : Double
} derive(Debug, Eq, ToJson)

///|
pub struct EvalConfig {
  cutoffs : Array[Int]
  relevant_threshold : Int
  gain_scheme : GainScheme
  missing_relevance : Int
} derive(Debug, ToJson)

///|
pub struct QueryEvaluation {
  query_id : String
  relevant_total : Int
  retrieved_total : Int
  metrics : Map[String, Double]
} derive(Debug, ToJson)

///|
pub struct AggregateMetric {
  name : String
  mean : Double
  min : Double
  max : Double
} derive(Debug, ToJson)

///|
pub struct BenchmarkReport {
  cutoffs : Array[Int]
  relevant_threshold : Int
  query_count : Int
  queries : Array[QueryEvaluation]
  summary : Array[AggregateMetric]
} derive(Debug, ToJson)

///|
pub struct CandidatePool {
  query_id : String
  doc_ids : Array[String]
} derive(Debug, ToJson)

///|
pub enum NegativeStrategy {
  Tail(Int)
  HardWindow(Int)
  Stride(Int)
} derive(Debug, Eq, ToJson)

///|
pub struct NegativeSampleConfig {
  per_query : Int
  relevant_threshold : Int
  skip_judged : Bool
  strategy : NegativeStrategy
} derive(Debug, ToJson)

///|
pub struct NegativeSample {
  query_id : String
  doc_id : String
  source_rank : Int
  strategy : String
} derive(Debug, Eq, ToJson)

///|
pub enum ValidationLevel {
  Warning
  Error
} derive(Debug, Eq, ToJson)

///|
pub struct ValidationIssue {
  code : String
  level : ValidationLevel
  message : String
  query_id : String
  doc_id : String
} derive(Debug, Eq, ToJson)

///|
pub struct ValidationSummary {
  error_count : Int
  warning_count : Int
  query_count : Int
  judged_count : Int
  retrieved_count : Int
  issues : Array[ValidationIssue]
} derive(Debug, ToJson)

///|
pub fn ValidationSummary::is_valid(self : ValidationSummary) -> Bool {
  self.error_count == 0
}

///|
pub struct QueryComparison {
  query_id : String
  baseline : Double
  candidate : Double
  delta : Double
  outcome : String
  overlap : Int
} derive(Debug, ToJson)

///|
pub struct RunComparison {
  cutoff : Int
  query_count : Int
  wins : Int
  losses : Int
  ties : Int
  mean_delta : Double
  queries : Array[QueryComparison]
} derive(Debug, ToJson)

///|
pub struct DatasetProfile {
  query_count : Int
  judged_count : Int
  retrieved_count : Int
  relevant_count : Int
  run_query_coverage : Double
  qrels_query_coverage : Double
  mean_run_length : Double
  mean_score : Double
  score_stddev : Double
  unjudged_retrievals : Int
  empty_query_count : Int
  duplicate_query_count : Int
} derive(Debug, ToJson)

///|
pub fn DatasetProfile::is_usable(self : DatasetProfile) -> Bool {
  self.query_count > 0 && self.judged_count > 0
}

///|
pub struct CandidatePoolProfile {
  pool_count : Int
  total_candidate_count : Int
  unique_candidate_count : Int
  duplicate_candidate_count : Int
  mean_pool_size : Double
  min_pool_size : Int
  max_pool_size : Int
} derive(Debug, ToJson)

///|
pub struct QueryBucket {
  query_id : String
  label : String
  score : Double
} derive(Debug, ToJson)

///|
pub struct RankPoint {
  rank : Int
  precision : Double
  recall : Double
} derive(Debug, ToJson)

///|
pub struct MetricDistribution {
  count : Int
  mean : Double
  min : Double
  max : Double
  median : Double
  p25 : Double
  p75 : Double
  stddev : Double
} derive(Debug, ToJson)

///|
pub struct TsvScanStats {
  total_lines : Int
  blank_lines : Int
  comment_lines : Int
  data_lines : Int
} derive(Debug, Eq, ToJson)

///|
pub struct QueryPartition {
  partition : Int
  query_ids : Array[String]
  qrels : Array[JudgedDoc]
} derive(Debug, ToJson)

///|
pub struct BenchmarkCase {
  name : String
  qrels : Array[JudgedDoc]
  run : Array[RetrievedDoc]
} derive(Debug, ToJson)

///|
pub fn BenchmarkCase::new(
  name~ : String,
  qrels~ : Array[JudgedDoc],
  run~ : Array[RetrievedDoc],
) -> BenchmarkCase {
  { name, qrels, run }
}

///|
pub struct BenchmarkResult {
  name : String
  report : BenchmarkReport
  validation : ValidationSummary
} derive(Debug, ToJson)

///|
pub struct QueryStatistic {
  query_id : String
  judged_count : Int
  relevant_count : Int
  retrieved_count : Int
  unjudged_count : Int
  mean_score : Double
} derive(Debug, ToJson)

///|
pub struct DocumentStatistic {
  doc_id : String
  query_frequency : Int
  relevant_query_count : Int
  maximum_relevance : Int
} derive(Debug, ToJson)

///|
pub struct RunQuality {
  query_count : Int
  row_count : Int
  unique_row_count : Int
  duplicate_rate : Double
  unjudged_rate : Double
  score_monotonicity : Double
  finite_score_rate : Double
} derive(Debug, ToJson)

///|
pub struct BenchmarkManifest {
  name : String
  version : String
  license : String
  query_count : Int
  qrels_rows : Int
  run_rows : Int
  relevant_documents : Int
  candidate_coverage : Double
} derive(Debug, ToJson)

///|
pub fn BenchmarkManifest::new(
  name~ : String,
  version~ : String,
  license~ : String,
  qrels~ : Array[JudgedDoc],
  run~ : Array[RetrievedDoc],
) -> BenchmarkManifest {
  let profile = profile_dataset(qrels, run)
  {
    name,
    version,
    license,
    query_count: profile.query_count,
    qrels_rows: qrels.length(),
    run_rows: run.length(),
    relevant_documents: profile.relevant_count,
    candidate_coverage: 0.0,
  }
}

///|
pub struct EvaluationPlan {
  name : String
  cutoffs : Array[Int]
  threshold : Int
  gain : GainScheme
} derive(Debug, ToJson)

///|
pub fn EvaluationPlan::new(
  name~ : String,
  cutoffs~ : Array[Int],
  threshold~ : Int,
  gain~ : GainScheme,
) -> EvaluationPlan {
  { name, cutoffs, threshold, gain }
}

///|
pub struct PlanResult {
  name : String
  report : BenchmarkReport
  validation : ValidationSummary
} derive(Debug, ToJson)

///|
pub struct ThresholdEvaluation {
  threshold : Int
  cutoff : Int
  precision : Double
  recall : Double
  f1 : Double
  ndcg : Double
} derive(Debug, ToJson)

///|
pub struct CorpusProfile {
  query_count : Int
  qrels_rows : Int
  run_rows : Int
  unique_document_count : Int
  relevant_document_count : Int
  max_relevance : Int
  unjudged_retrievals : Int
  mean_judgments_per_query : Double
  mean_run_length : Double
  mean_score : Double
  score_stddev : Double
} derive(Debug, ToJson)

///|
pub struct TraceStep {
  rank : Int
  doc_id : String
  score : Double
  relevance : Int
  judged : Bool
  hits : Int
  precision : Double
  recall : Double
} derive(Debug, ToJson)

///|
pub fn JudgedDoc::new(
  query_id~ : String,
  doc_id~ : String,
  relevance~ : Int,
) -> JudgedDoc {
  { query_id, doc_id, relevance }
}

///|
pub fn RetrievedDoc::new(
  query_id~ : String,
  doc_id~ : String,
  score~ : Double,
) -> RetrievedDoc {
  { query_id, doc_id, score }
}

///|
pub fn CandidatePool::new(
  query_id~ : String,
  doc_ids~ : Array[String],
) -> CandidatePool {
  { query_id, doc_ids }
}