///|
pub(all) enum ProviderFailureCheck {
ErrorNormalized
RoutineFailedBeforeModelCall
PartialRecordPreserved
RetryDispositionRecorded
} derive(Debug, Eq, ToJson, FromJson)
///|
pub(all) enum ProviderFailureStatus {
Pass
NeedsWork
} derive(Debug, Eq, ToJson, FromJson)
///|
pub(all) struct ProviderFailureInput {
result : @orchestration.TwoStageRoutineResult
provider_error : @model.ProviderError
} derive(Debug, Eq, ToJson, FromJson)
///|
pub(all) struct ProviderFailureFinding {
run_id : @domain.RunId
check : ProviderFailureCheck
status : ProviderFailureStatus
path : String
message : String
} derive(Debug, Eq, ToJson, FromJson)
///|
pub(all) struct ProviderFailureReport {
run_id : @domain.RunId
ready : Bool
error_kind : @model.ProviderErrorKind
retryable : Bool
artifact_count : Int
findings : Array[ProviderFailureFinding]
summary : String
} derive(Debug, Eq, ToJson, FromJson)
///|
pub fn provider_failure_input(
result : @orchestration.TwoStageRoutineResult,
provider_error : @model.ProviderError,
) -> ProviderFailureInput {
{ result, provider_error }
}
///|
pub fn provider_failure_finding(
run_id : @domain.RunId,
check : ProviderFailureCheck,
status : ProviderFailureStatus,
path : String,
message : String,
) -> ProviderFailureFinding {
{ run_id, check, status, path, message }
}
///|
fn normalized_finding(input : ProviderFailureInput) -> ProviderFailureFinding {
let error = input.provider_error
if error.normalized_message != "" && error.kind != ProviderUnknown {
provider_failure_finding(
input.result.request.run_id,
ErrorNormalized,
Pass,
"provider-error",
error.normalized_message,
)
} else {
provider_failure_finding(
input.result.request.run_id,
ErrorNormalized,
NeedsWork,
"provider-error",
"provider error is unclassified",
)
}
}
///|
fn routine_failure_finding(
input : ProviderFailureInput,
) -> ProviderFailureFinding {
let result = input.result
if result.outcome is Failed &&
result.failure_kind is Some(GatewayUnavailable) &&
result.stage1_model_request is None &&
result.stage2_model_request is None {
provider_failure_finding(
result.request.run_id,
RoutineFailedBeforeModelCall,
Pass,
"two-stage-routine-result",
"routine fails at provider gateway boundary without fabricating model-call evidence",
)
} else {
provider_failure_finding(
result.request.run_id,
RoutineFailedBeforeModelCall,
NeedsWork,
"two-stage-routine-result",
"routine did not stop cleanly at the provider gateway boundary",
)
}
}
///|
fn partial_record_finding(
input : ProviderFailureInput,
) -> ProviderFailureFinding {
let result = input.result
let has_record = result.artifact_paths.contains(
"records/analyses/\{result.request.run_id}.json",
)
if result.partial_record_available && has_record {
provider_failure_finding(
result.request.run_id,
PartialRecordPreserved,
Pass,
"two-stage-routine-result.artifact_paths",
"provider failure preserves a partial analysis record path",
)
} else {
provider_failure_finding(
result.request.run_id,
PartialRecordPreserved,
NeedsWork,
"two-stage-routine-result.artifact_paths",
"provider failure does not expose a partial analysis record path",
)
}
}
///|
fn retry_disposition_finding(
input : ProviderFailureInput,
) -> ProviderFailureFinding {
if input.provider_error.retryable {
provider_failure_finding(
input.result.request.run_id,
RetryDispositionRecorded,
Pass,
"provider-error.retryable",
"provider failure is retryable through a bounded MoonClaw retry path",
)
} else {
provider_failure_finding(
input.result.request.run_id,
RetryDispositionRecorded,
Pass,
"provider-error.retryable",
"provider failure is non-retryable and should move to operator review",
)
}
}
///|
fn report_ready(findings : Array[ProviderFailureFinding]) -> Bool {
findings.all(fn(finding) { finding.status is Pass })
}
///|
fn provider_error_label(kind : @model.ProviderErrorKind) -> String {
match kind {
ProviderAuthFailed => "provider_auth_failed"
ProviderQuotaExhausted => "provider_quota_exhausted"
ProviderRateLimited => "provider_rate_limited"
ProviderUnavailable => "provider_unavailable"
ProviderBadRequest => "provider_bad_request"
ProviderUnknown => "provider_unknown"
}
}
///|
fn report_summary(report : ProviderFailureReport) -> String {
let pass_count = report.findings.fold(init=0, fn(count, finding) {
if finding.status is Pass {
count + 1
} else {
count
}
})
if report.ready {
"provider failure report passed \{pass_count}/\{report.findings.length()} checks for \{provider_error_label(report.error_kind)}"
} else {
"provider failure report has \{report.findings.length() - pass_count} check(s) needing work"
}
}
///|
pub fn prepare_provider_failure_report(
input : ProviderFailureInput,
) -> ProviderFailureReport {
let findings = [
normalized_finding(input),
routine_failure_finding(input),
partial_record_finding(input),
retry_disposition_finding(input),
]
let report : ProviderFailureReport = {
run_id: input.result.request.run_id,
ready: report_ready(findings),
error_kind: input.provider_error.kind,
retryable: input.provider_error.retryable,
artifact_count: input.result.artifact_paths.length(),
findings,
summary: "",
}
{ ..report, summary: report_summary(report) }
}