///|
/// Aggregated timing statistics for stopwatch-style benchmark samples.
///
/// The package keeps the data model intentionally small: callers can feed
/// elapsed nanoseconds from any clock source and use the derived summary in CLI
/// tools, test reports, CI logs, or documentation examples.
pub(all) struct Summary {
count : Int
total_ns : Int
min_ns : Int
max_ns : Int
mean_ns : Int
} derive(Debug, Eq)
///|
/// Simple performance budget for CI or documentation smoke checks.
pub(all) struct Budget {
max_mean_ns : Int
max_spread_ppm : Int
} derive(Debug, Eq)
///|
/// Sample cleanup settings for noisy clocks and imported benchmark logs.
///
/// `max_ns <= 0` means "no upper bound". Set `drop_zero` when a coarse clock
/// can occasionally round very short samples to `0 ns`.
pub(all) struct SampleFilter {
min_ns : Int
max_ns : Int
drop_zero : Bool
} derive(Debug, Eq)
///|
/// Difference between a current summary and a baseline summary.
///
/// `delta_ppm` is measured against the baseline mean. Positive values are
/// slower, negative values are faster.
pub(all) struct Comparison {
baseline_mean_ns : Int
current_mean_ns : Int
delta_ns : Int
delta_ppm : Int
faster : Bool
slower : Bool
} derive(Debug, Eq)
///|
/// Raw benchmark samples plus the derived aggregate summary.
///
/// Use `SampleSet` when a caller needs percentiles, variance, CSV/JSON reports,
/// or later export of the original elapsed samples.
pub(all) struct SampleSet {
samples : Array[Int]
summary : Summary
} derive(Debug, Eq)
///|
/// Distribution statistics derived from all recorded samples.
pub(all) struct Distribution {
p50_ns : Int
p90_ns : Int
p95_ns : Int
p99_ns : Int
variance_ns2 : Double
std_dev_ns : Double
} derive(Debug)
///|
/// Compact quality signal for a sample set.
///
/// The fields are expressed in parts per million so CI thresholds can stay
/// integer based. `100_000ppm` means 10 percent.
pub(all) struct Quality {
sample_count : Int
spread_ppm : Int
std_dev_ppm : Int
stable : Bool
} derive(Debug, Eq)
///|
/// Clock adapter used by benchmark helpers.
///
/// Production tools can pass a platform-specific monotonic clock. Tests can
/// pass a deterministic clock so examples do not depend on wall time.
pub(all) struct Clock {
name : String
now_ns : () -> Int
}
///|
/// Return an empty summary.
pub fn empty() -> Summary {
{ count: 0, total_ns: 0, min_ns: 0, max_ns: 0, mean_ns: 0 }
}
///|
/// Start a summary with one elapsed sample in nanoseconds.
pub fn singleton(elapsed_ns : Int) -> Summary {
{
count: 1,
total_ns: elapsed_ns,
min_ns: elapsed_ns,
max_ns: elapsed_ns,
mean_ns: elapsed_ns,
}
}
///|
/// Return an empty sample set.
pub fn sample_set() -> SampleSet {
{ samples: [], summary: empty() }
}
///|
/// Build a sample set from elapsed nanosecond values.
pub fn sample_set_from(samples : Array[Int]) -> SampleSet {
let mut set = sample_set()
for sample in samples {
set = set.add_sample(sample)
}
set
}
///|
/// Keep only samples that match the cleanup filter.
///
/// This is useful when imported data contains warm-up artifacts, known outlier
/// bounds, or `0 ns` values caused by a coarse demo clock.
pub fn filter_samples(set : SampleSet, filter : SampleFilter) -> SampleSet {
let mut filtered = sample_set()
for sample in set.samples {
let keep_zero = !(filter.drop_zero && sample == 0)
let keep_min = sample >= filter.min_ns
let keep_max = filter.max_ns <= 0 || sample <= filter.max_ns
if keep_zero && keep_min && keep_max {
filtered = filtered.add_sample(sample)
}
}
filtered
}
///|
/// Drop the same number of sorted samples from the low and high ends.
///
/// If the requested trim would remove every sample, an empty set is returned.
pub fn trim_samples(set : SampleSet, drop_each_side : Int) -> SampleSet {
let count = set.samples.length()
let drop = if drop_each_side < 0 { 0 } else { drop_each_side }
if count == 0 || drop * 2 >= count {
sample_set()
} else {
let sorted = set.samples.copy()
sorted.sort()
let mut trimmed = sample_set()
for index in drop..<(count - drop) {
trimmed = trimmed.add_sample(sorted[index])
}
trimmed
}
}
///|
/// Drop the first N samples while preserving the original order of the rest.
///
/// This is intended for benchmark warm-up samples, for example the first run
/// after loading a CLI, parser, or runtime cache.
pub fn drop_warmup(set : SampleSet, warmup_count : Int) -> SampleSet {
let count = set.samples.length()
let start = if warmup_count < 0 {
0
} else if warmup_count > count {
count
} else {
warmup_count
}
let mut result = sample_set()
for index in start.. SampleSet {
let mut result = sample_set()
for sample in set.samples {
let clamped_low = if sample < min_ns { min_ns } else { sample }
let clamped = if max_ns > 0 && clamped_low > max_ns {
max_ns
} else {
clamped_low
}
result = result.add_sample(clamped)
}
result
}
///|
/// Add one elapsed sample and update aggregate statistics.
pub fn SampleSet::add_sample(set : Self, elapsed_ns : Int) -> SampleSet {
let samples = set.samples.copy()
samples.push(elapsed_ns)
{ samples, summary: set.summary.add_sample(elapsed_ns) }
}
///|
/// Return the system clock adapter used by the default benchmark helpers.
///
/// `env.now()` currently exposes milliseconds, so the adapter converts the
/// value to nanoseconds for a single public time unit.
pub fn system_clock() -> Clock {
{ name: "env.now-ms", now_ns: () => @env.now().to_int() * 1_000_000 }
}
///|
/// Measure one execution of `run` with a caller-provided clock.
pub fn measure_with_clock(clock : Clock, run : () -> Unit) -> Int {
let start = (clock.now_ns)()
run()
let finish = (clock.now_ns)()
if finish > start {
finish - start
} else {
0
}
}
///|
/// Execute `run` repeatedly and keep every measured elapsed sample.
pub fn benchmark_with_clock(
iterations : Int,
clock : Clock,
run : () -> Unit,
) -> SampleSet {
let mut set = sample_set()
let count = if iterations < 0 { 0 } else { iterations }
for _ in 0.. Unit) -> Int {
measure_with_clock(system_clock(), run)
}
///|
/// Execute `run` repeatedly and aggregate the measured elapsed times.
pub fn benchmark(iterations : Int, run : () -> Unit) -> Summary {
benchmark_with_clock(iterations, system_clock(), run).summary
}
///|
/// Return true when the summary contains at least one sample.
pub fn has_samples(summary : Summary) -> Bool {
summary.count > 0
}
///|
/// Add one elapsed sample to an existing summary.
pub fn Summary::add_sample(summary : Self, elapsed_ns : Int) -> Summary {
if summary.count == 0 {
singleton(elapsed_ns)
} else {
let count = summary.count + 1
let total_ns = summary.total_ns + elapsed_ns
let min_ns = if elapsed_ns < summary.min_ns {
elapsed_ns
} else {
summary.min_ns
}
let max_ns = if elapsed_ns > summary.max_ns {
elapsed_ns
} else {
summary.max_ns
}
{ count, total_ns, min_ns, max_ns, mean_ns: total_ns / count }
}
}
///|
/// Merge two summaries. Empty summaries are identity values.
pub fn merge(left : Summary, right : Summary) -> Summary {
if left.count == 0 {
right
} else if right.count == 0 {
left
} else {
let count = left.count + right.count
let total_ns = left.total_ns + right.total_ns
let min_ns = if left.min_ns < right.min_ns {
left.min_ns
} else {
right.min_ns
}
let max_ns = if left.max_ns > right.max_ns {
left.max_ns
} else {
right.max_ns
}
{ count, total_ns, min_ns, max_ns, mean_ns: total_ns / count }
}
}
///|
/// Return a percentile in nanoseconds using linear interpolation.
///
/// `pct` is clamped into `[0, 100]`. Empty sample sets return `0`.
pub fn percentile_ns(set : SampleSet, pct : Int) -> Int {
let count = set.samples.length()
if count == 0 {
0
} else if count == 1 {
set.samples[0]
} else {
let sorted = set.samples.copy()
sorted.sort()
let clamped = if pct < 0 { 0 } else if pct > 100 { 100 } else { pct }
let rank = clamped * (count - 1)
let lower = rank / 100
let upper = if lower + 1 < count { lower + 1 } else { lower }
let fraction = rank % 100
sorted[lower] +
interpolate_delta_ns(sorted[upper] - sorted[lower], fraction)
}
}
///|
fn interpolate_delta_ns(delta : Int, fraction : Int) -> Int {
delta / 100 * fraction + delta % 100 * fraction / 100
}
///|
/// Sample variance in square nanoseconds.
pub fn variance_ns2(set : SampleSet) -> Double {
if set.summary.count < 2 {
0.0
} else {
let mean = set.summary.total_ns.to_double() / set.summary.count.to_double()
let mut total = 0.0
for sample in set.samples {
let delta = sample.to_double() - mean
total = total + delta * delta
}
total / (set.summary.count - 1).to_double()
}
}
///|
/// Sample standard deviation in nanoseconds.
pub fn std_dev_ns(set : SampleSet) -> Double {
variance_ns2(set).sqrt()
}
///|
/// Compute common latency percentiles and variance for a sample set.
pub fn distribution(set : SampleSet) -> Distribution {
{
p50_ns: percentile_ns(set, 50),
p90_ns: percentile_ns(set, 90),
p95_ns: percentile_ns(set, 95),
p99_ns: percentile_ns(set, 99),
variance_ns2: variance_ns2(set),
std_dev_ns: std_dev_ns(set),
}
}
///|
/// Min/max spread relative to the mean, expressed in parts per million.
pub fn relative_spread_ppm(summary : Summary) -> Int {
if summary.count < 2 || summary.mean_ns <= 0 {
0
} else {
(spread_ns(summary).to_double() * 1_000_000.0 / summary.mean_ns.to_double()).to_int()
}
}
///|
/// Sample standard deviation relative to the mean, expressed in ppm.
pub fn std_dev_ppm(set : SampleSet) -> Int {
if set.summary.count < 2 || set.summary.mean_ns <= 0 {
0
} else {
(std_dev_ns(set) * 1_000_000.0 / set.summary.mean_ns.to_double()).to_int()
}
}
///|
/// Summarize whether a sample set is stable enough for a tolerance threshold.
pub fn quality(set : SampleSet, tolerance_ppm : Int) -> Quality {
let spread = relative_spread_ppm(set.summary)
{
sample_count: set.summary.count,
spread_ppm: spread,
std_dev_ppm: std_dev_ppm(set),
stable: set.summary.count > 1 && spread <= tolerance_ppm,
}
}
///|
/// Difference between the slowest and fastest samples in nanoseconds.
pub fn spread_ns(summary : Summary) -> Int {
if summary.count < 2 {
0
} else {
summary.max_ns - summary.min_ns
}
}
///|
/// Integer operations per second, using nanoseconds as the time base.
pub fn samples_per_second(summary : Summary) -> Int {
if summary.count == 0 || summary.total_ns <= 0 {
0
} else {
let whole = 1_000_000_000 / summary.total_ns
let remain = 1_000_000_000 % summary.total_ns
whole * summary.count + remain * summary.count / summary.total_ns
}
}
///|
/// Return true when the min/max spread is inside a parts-per-million tolerance.
///
/// A tolerance of `100_000` means 10 percent.
pub fn is_stable(summary : Summary, tolerance_ppm : Int) -> Bool {
if summary.count < 2 || summary.mean_ns <= 0 {
false
} else {
let spread = spread_ns(summary)
let whole = summary.mean_ns / 1_000_000
let remain = summary.mean_ns % 1_000_000
let allowed = whole * tolerance_ppm +
remain / 1_000 * tolerance_ppm / 1_000 +
remain % 1_000 * tolerance_ppm / 1_000_000
spread <= allowed
}
}
///|
/// Return true when a summary is non-empty and fits the given performance
/// budget.
pub fn meets_budget(summary : Summary, budget : Budget) -> Bool {
has_samples(summary) &&
summary.mean_ns <= budget.max_mean_ns &&
is_stable(summary, budget.max_spread_ppm)
}
///|
/// Produce a compact pass/fail budget report for CI logs.
pub fn budget_report(summary : Summary, budget : Budget) -> String {
let status = if meets_budget(summary, budget) { "pass" } else { "fail" }
"\{status}: mean<=\{format_ns(budget.max_mean_ns)}, spread<=\{budget.max_spread_ppm}ppm, \{describe(summary)}"
}
///|
/// Compare current benchmark results against a baseline summary.
pub fn compare_to_baseline(current : Summary, baseline : Summary) -> Comparison {
let delta_ns = current.mean_ns - baseline.mean_ns
let delta_ppm = if baseline.mean_ns <= 0 {
0
} else {
(delta_ns.to_double() * 1_000_000.0 / baseline.mean_ns.to_double()).to_int()
}
{
baseline_mean_ns: baseline.mean_ns,
current_mean_ns: current.mean_ns,
delta_ns,
delta_ppm,
faster: delta_ns < 0,
slower: delta_ns > 0,
}
}
///|
/// Return true when the current mean is not slower than the tolerated baseline.
pub fn within_regression_budget(
current : Summary,
baseline : Summary,
tolerance_ppm : Int,
) -> Bool {
has_samples(current) &&
has_samples(baseline) &&
baseline.mean_ns > 0 &&
compare_to_baseline(current, baseline).delta_ppm <= tolerance_ppm
}
///|
/// Produce a compact pass/fail regression report for release checks.
pub fn regression_report(
current : Summary,
baseline : Summary,
tolerance_ppm : Int,
) -> String {
let comparison = compare_to_baseline(current, baseline)
let status = if within_regression_budget(current, baseline, tolerance_ppm) {
"pass"
} else {
"fail"
}
"\{status}: baseline=\{format_ns(comparison.baseline_mean_ns)}, current=\{format_ns(comparison.current_mean_ns)}, delta=\{format_ns(comparison.delta_ns)} (\{comparison.delta_ppm}ppm), tolerance=\{tolerance_ppm}ppm"
}
///|
/// CSV header for summary rows.
pub fn summary_csv_header() -> String {
"count,total_ns,min_ns,max_ns,mean_ns,spread_ns,throughput_per_second"
}
///|
/// Machine-readable CSV row for a summary.
pub fn summary_csv(summary : Summary) -> String {
"\{summary.count},\{summary.total_ns},\{summary.min_ns},\{summary.max_ns},\{summary.mean_ns},\{spread_ns(summary)},\{samples_per_second(summary)}"
}
///|
/// Machine-readable JSON report for a summary.
pub fn summary_json(summary : Summary) -> String {
"{\"count\":\{summary.count},\"total_ns\":\{summary.total_ns},\"min_ns\":\{summary.min_ns},\"max_ns\":\{summary.max_ns},\"mean_ns\":\{summary.mean_ns},\"spread_ns\":\{spread_ns(summary)},\"throughput_per_second\":\{samples_per_second(summary)}}"
}
///|
/// Machine-readable JSON report for common distribution statistics.
pub fn distribution_json(distribution : Distribution) -> String {
"{\"p50_ns\":\{distribution.p50_ns},\"p90_ns\":\{distribution.p90_ns},\"p95_ns\":\{distribution.p95_ns},\"p99_ns\":\{distribution.p99_ns},\"variance_ns2\":\{distribution.variance_ns2},\"std_dev_ns\":\{distribution.std_dev_ns}}"
}
///|
/// Machine-readable JSON report including summary and distribution fields.
pub fn sample_set_json(set : SampleSet) -> String {
"{\"summary\":\{summary_json(set.summary)},\"distribution\":\{distribution_json(distribution(set))}}"
}
///|
/// Markdown table header and row for a summary.
pub fn summary_markdown(summary : Summary) -> String {
"|count|mean|min|max|spread|throughput/s|\n|---:|---:|---:|---:|---:|---:|\n|\{summary.count}|\{format_ns(summary.mean_ns)}|\{format_ns(summary.min_ns)}|\{format_ns(summary.max_ns)}|\{format_ns(spread_ns(summary))}|\{samples_per_second(summary)}|"
}
///|
/// Markdown report including summary and percentile fields.
pub fn sample_set_markdown(set : SampleSet) -> String {
let dist = distribution(set)
"\{summary_markdown(set.summary)}\n\np50=\{format_ns(dist.p50_ns)}, p95=\{format_ns(dist.p95_ns)}, p99=\{format_ns(dist.p99_ns)}"
}
///|
/// Human-readable quality report for CI logs and benchmark notes.
pub fn quality_report(set : SampleSet, tolerance_ppm : Int) -> String {
let q = quality(set, tolerance_ppm)
let status = if q.stable { "stable" } else { "noisy" }
"\{status}: samples=\{q.sample_count}, spread=\{q.spread_ppm}ppm, stddev=\{q.std_dev_ppm}ppm, tolerance=\{tolerance_ppm}ppm"
}
///|
/// Format a nanosecond duration into a compact human-readable unit.
pub fn format_ns(ns : Int) -> String {
if ns < 0 {
"-\{format_ns(0 - ns)}"
} else if ns >= 1_000_000 {
"\{ns / 1_000_000} ms"
} else if ns >= 1_000 {
"\{ns / 1_000} us"
} else {
"\{ns} ns"
}
}
///|
/// Produce a one-line report suitable for CLI output and CI logs.
pub fn describe(summary : Summary) -> String {
if summary.count == 0 {
"count=0"
} else {
"count=\{summary.count}, mean=\{format_ns(summary.mean_ns)}, min=\{format_ns(summary.min_ns)}, max=\{format_ns(summary.max_ns)}, throughput=\{samples_per_second(summary)}/s"
}
}