#!/usr/bin/env bash
# =============================================================================
#  test-chrony.sh - measure Chrony NTP server performance, one metric at a
#                   time, and save an easy-to-read report
# =============================================================================
#
#  USAGE (as root, after ./start-chrony.sh)
#      ./test-chrony.sh                     run every test (about 2 minutes)
#      ./test-chrony.sh throughput          run one test
#      ./test-chrony.sh latency loss        run several tests
#      ./test-chrony.sh --list              show the test names
#
#      DURATION=30 LOAD_RATE=5000 ./test-chrony.sh latency
#                                           override settings.conf for one run
#
#  THE TESTS  (explained in detail in PERFORMANCE-METRICS.md)
#      startup      time from "start" until the server answers with good time
#      quality      what the server tells clients about its time (stratum, leap)
#      throughput   the most NTP requests answered per second
#      latency      time one answer takes, at a steady realistic load
#      loss         requests not answered (or answered badly) at that load
#      accuracy     time error a client sees (offset), at that load
#      clients      how many different clients can be served, still fast
#      ratelimit    a flooding client is slowed down, normal clients are not
#      cpu          CPU used by chronyd per answered request
#      memory       memory used by chronyd per client it remembers
#
#  OUTPUT
#      results/<date>-<time>/report.md    the human-readable report
#      results/<date>-<time>/summary.csv  one line per test (for spreadsheets)
#      results/<date>-<time>/raw/         unmodified measurement output
#      results/latest                     link to the newest result folder
#
#  EXIT CODE
#      0 = every test passed,  1 = at least one FAIL,  2 = could not run
#
#  The load comes from lib/ntpload.py (Python 3, standard library only). It
#  sends real NTP requests over UDP to the test chronyd on 127.0.0.1, from
#  many different 127.x.y.z addresses. No internet access is needed.
# =============================================================================

set -euo pipefail
source "$(dirname -- "${BASH_SOURCE[0]}")/lib/common.sh"

ALL_TESTS=(startup quality throughput latency loss accuracy clients ratelimit cpu memory)

GENERATOR_BUSY_PCT=90                # a sender above this = it may be the limit

# Each test sends from its own range of client addresses.
CLIENTS_STEADY="127.10.0.1"
CLIENTS_THROUGHPUT="127.11.0.1"
CLIENTS_STEPS="127.20.0.1"
CLIENTS_FLOOD="127.30.0.1"
CLIENTS_NORMAL="127.31.0.1"
CLIENTS_MEMORY="127.40.0.1"

# Filled in by the functions below.
RESULT_DIR=""
RAW_DIR=""
DETAILS_FILE=""
SUMMARY_ROWS=()     # "Test|Measured|Target|Verdict"
FAIL_COUNT=0
GENERATOR_LIMITED_TESTS=()
RATELIMIT_ACTIVE=no


# =============================================================================
#  PART 1 - small helpers
# =============================================================================

# Floating-point math and comparisons (bash itself only knows whole numbers).
calc()          { awk "BEGIN { printf \"%.1f\", $* }"; }
calc2()         { awk "BEGIN { printf \"%.2f\", $* }"; }
calc3()         { awk "BEGIN { printf \"%.3f\", $* }"; }
is_at_most()    { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a <= b) }'; }
is_at_least()   { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a >= b) }'; }
now_seconds()   { date +%s.%N; }

# 1234567.8 -> 1,234,568   (easier to read in the report)
pretty() {
    printf '%.0f' "$1" | sed ':again; s/\B[0-9]\{3\}\>/,&/; t again'
}

# Text padded with spaces to a width, for the summary table. printf counts
# bytes, and "µ" is 2 bytes but 1 character, so widen the field for each one.
cell() {
    local text="$1" width="$2" extra
    extra="$(grep -o 'µ' <<< "$text" | wc -l)"
    printf "%-$(( width + extra ))s" "$text"
}

# Turn a comparison into the words PASS / FAIL.
verdict_at_most()  { if is_at_most  "$1" "$2"; then echo PASS; else echo FAIL; fi; }
verdict_at_least() { if is_at_least "$1" "$2"; then echo PASS; else echo FAIL; fi; }

# Remember one line for the summary table at the top of the report.
#   record_result "Test name" "measured value" "target" PASS|FAIL|INFO
record_result() {
    local name="$1" measured="$2" target="$3" verdict="$4"
    SUMMARY_ROWS+=("$name|$measured|$target|$verdict")
    [[ $verdict == FAIL ]] && FAIL_COUNT=$(( FAIL_COUNT + 1 ))

    local colour='1;32'                       # green  = PASS
    [[ $verdict == FAIL ]] && colour='1;31'   # red    = FAIL
    [[ $verdict == INFO ]] && colour='1;36'   # cyan   = INFO
    printf "    => \033[${colour}m%-4s\033[0m  %s   (target: %s)\n" "$verdict" "$measured" "$target"
}

# Append Markdown text (read from stdin) to the "details" part of the report.
add_details() {
    cat >> "$DETAILS_FILE"
}

section_title() {
    echo
    printf '\033[1m==> %s\033[0m\n' "$*"
}


# =============================================================================
#  PART 2 - server-side counters
# =============================================================================

# One number from "chronyc serverstats", e.g.  server_stat "NTP packets dropped"
server_stat() {
    test_chronyc serverstats 2>/dev/null |
        awk -F': *' -v label="$1" '$1 ~ "^" label { print $2 + 0; found = 1 } END { if (!found) print 0 }'
}

# Packets the kernel threw away because chronyd's receive buffer was full:
# last column ("drops") of chronyd's line in /proc/net/udp.
server_socket_drops() {
    local port_hex
    port_hex="$(printf '%04X' "$NTP_PORT")"
    awk -v port=":$port_hex" 'NR > 1 && $2 ~ port "$" { drops += $NF } END { print drops + 0 }' /proc/net/udp
}

# Number of clients chronyd remembers right now ("chronyc clients", without
# its two header lines).
remembered_clients() {
    test_chronyc clients 2>/dev/null | tail -n +3 | wc -l
}


# =============================================================================
#  PART 3 - running the load generator and reading its results
# =============================================================================
#
#  run_load <name> <steady|max> [ntpload.py options ...]
#
#  Runs lib/ntpload.py once and saves raw/<name>.txt: the generator's own
#  "key = value" lines, followed by what chronyd did during the run:
#      chronyd_cpu_seconds       CPU time chronyd used
#      server_ntp_received       NTP requests chronyd received
#      server_ntp_dropped        requests chronyd ignored on purpose (rate limit)
#      server_socket_drops       requests lost because chronyd was too busy
#      server_records_dropped    client records replaced because the memory was full
#
#  A run that already happened in this test session is not repeated (the
#  "loss" test re-uses the "latency" run, for example).
#
#  Read single values afterwards with:  value <key>
#  Example:  run_load steady steady --rate 1000 --duration 10
#            value response_ms_p99        ->  0.036
# -----------------------------------------------------------------------------
LOAD_OUTPUT=""

run_load() {
    local name="$1" mode="$2"
    shift 2
    LOAD_OUTPUT="$RAW_DIR/$name.txt"
    [[ -s $LOAD_OUTPUT ]] && return 0             # already measured

    printf '    %-22s ' "$name"
    local cpu_before received_before dropped_before socket_before records_before
    cpu_before="$(chronyd_cpu_seconds)"
    received_before="$(server_stat 'NTP packets received')"
    dropped_before="$(server_stat 'NTP packets dropped')"
    records_before="$(server_stat 'Client log records dropped')"
    socket_before="$(server_socket_drops)"

    if ! python3 "$NTP_LOAD" "$mode" --server "$LISTEN_ADDRESS" --port "$NTP_PORT" "$@" \
            > "$LOAD_OUTPUT.part" 2> "$RAW_DIR/$name-errors.txt"; then
        echo "failed"
        die "The load generator failed. See $RAW_DIR/$name-errors.txt"
    fi
    [[ -s $RAW_DIR/$name-errors.txt ]] || rm -f "$RAW_DIR/$name-errors.txt"

    {
        cat "$LOAD_OUTPUT.part"
        printf '%-28s = %s\n' \
            chronyd_cpu_seconds    "$(calc3 "$(chronyd_cpu_seconds) - $cpu_before")" \
            server_ntp_received    "$(( $(server_stat 'NTP packets received') - received_before ))" \
            server_ntp_dropped     "$(( $(server_stat 'NTP packets dropped') - dropped_before ))" \
            server_socket_drops    "$(( $(server_socket_drops) - socket_before ))" \
            server_records_dropped "$(( $(server_stat 'Client log records dropped') - records_before ))"
    } > "$LOAD_OUTPUT"
    rm -f "$LOAD_OUTPUT.part"

    if [[ $mode == max ]]; then
        printf '%10s answers/s   %s%% lost\n' "$(pretty "$(value answered_per_s)")" "$(calc2 "$(value lost_pct)")"
        check_sender_load_max "$name"
    else
        printf '%8s requests/s   %s%% failed   p99 %s ms\n' \
            "$(pretty "$(value achieved_rate)")" "$(calc2 "$(value failed_pct)")" "$(value response_ms_p99)"
        check_sender_load_steady "$name"
    fi
}

# One value from the last run output.  value <key>  (missing -> "0")
value() {
    awk -v key="$1" '$1 == key { print $3; found = 1 } END { if (!found) print 0 }' "$LOAD_OUTPUT"
}

# The same, from another saved run:  value_of <name> <key>
value_of() {
    awk -v key="$2" '$1 == key { print $3; found = 1 } END { if (!found) print 0 }' "$RAW_DIR/$1.txt"
}

# Note it when the load generator itself was the limit: then the result
# shows the limit of the generator, not of chronyd.
check_sender_load_max() {
    if is_at_least "$(value generator_cpu_pct_avg)" "$GENERATOR_BUSY_PCT"; then
        GENERATOR_LIMITED_TESTS+=("$1")
        warn "The senders were ${GENERATOR_BUSY_PCT}%+ busy in '$1'; chronyd may be faster than measured."
    fi
}

check_sender_load_steady() {
    if is_at_most "$(value achieved_rate)" "$(calc "$(value target_rate) * 0.95")" ||
       (( $(value generator_socket_drops) > 0 )); then
        GENERATOR_LIMITED_TESTS+=("$1")
        warn "The load generator could not keep up in '$1' (rate $(value achieved_rate) of $(value target_rate)/s, $(value generator_socket_drops) of its own packets dropped)."
    fi
}

# The steady load shared by the latency, loss, accuracy and cpu tests.
run_steady_load() {
    run_load steady steady --rate "$LOAD_RATE" --duration "$DURATION" \
        --clients "$LOAD_CLIENTS" --first-client "$CLIENTS_STEADY"
}

# The full-speed load shared by the throughput and cpu tests.
run_full_load() {
    local workers="$THROUGHPUT_WORKERS"
    if [[ $workers == auto ]]; then
        workers=2
        (( $(nproc) < 3 )) && workers=1
    fi
    run_load throughput max --duration "$DURATION" --workers "$workers" \
        --window "$THROUGHPUT_WINDOW" --first-client "$CLIENTS_THROUGHPUT"
}


# =============================================================================
#  PART 4 - the tests, one function per metric
# =============================================================================

# -----------------------------------------------------------------------------
test_startup() {
    section_title "Startup time  ($STARTUP_ROUNDS restart(s))"

    local round answer_ms=() synced_ms=() command_ms=()
    for (( round = 1; round <= STARTUP_ROUNDS; round++ )); do
        chronyd_stop

        # 1) Start the "waiter" first: it asks the (stopped) server every
        #    2 ms. Starting Python takes a moment, so it gets a head start.
        local waiter_output="$RAW_DIR/startup-round-$round.txt"
        python3 "$NTP_LOAD" wait --server "$LISTEN_ADDRESS" --port "$NTP_PORT" \
            --timeout 60 > "$waiter_output" &
        local waiter_pid=$!
        sleep 0.5

        # 2) Start chronyd and note the time.
        local start_time command_done
        start_time="$(now_seconds)"
        chronyd_start
        command_done="$(now_seconds)"

        # 3) The waiter prints when the first answer, and the first answer
        #    with synchronized time, arrived.
        wait "$waiter_pid" || die "The test server did not answer within 60 s after a restart."
        local answer_epoch synced_epoch
        answer_epoch="$(awk '$1 == "answer_epoch" { print $3 }' "$waiter_output")"
        synced_epoch="$(awk '$1 == "synced_epoch" { print $3 }' "$waiter_output")"

        command_ms+=("$(calc "($command_done - $start_time) * 1000")")
        answer_ms+=("$(calc "($answer_epoch - $start_time) * 1000")")
        synced_ms+=("$(calc "($synced_epoch - $start_time) * 1000")")
        printf '    restart %d: answers after %s ms, with synchronized time after %s ms\n' \
            "$round" "${answer_ms[-1]}" "${synced_ms[-1]}"
    done

    local average_answer average_synced
    average_answer="$(printf '%s\n' "${answer_ms[@]}" | awk '{ s += $1 } END { printf "%.1f", s / NR }')"
    average_synced="$(printf '%s\n' "${synced_ms[@]}" | awk '{ s += $1 } END { printf "%.1f", s / NR }')"

    record_result "Startup time" "$average_synced ms average (answers after $average_answer ms)" \
        "<= $TARGET_STARTUP_MAX_MS ms" "$(verdict_at_most "$average_synced" "$TARGET_STARTUP_MAX_MS")"

    add_details <<EOF
## Startup time

The test server was stopped and started $STARTUP_ROUNDS time(s). A small program asked
for the time every 2 ms, from before the start command until the answer said
"synchronized". Three times were measured from the start command:

- **Start command returned**: \`$(has_systemd && echo "systemctl start $TEST_SERVICE" || echo chronyd)\` came back.
- **First answer**: the first NTP answer of any kind.
- **Synchronized answer**: the first answer that clients can use (leap status
  is not "not synchronized", stratum is not 0). Clients ignore the server before that.

| Round | Start command returned (ms) | First answer (ms) | Synchronized answer (ms) |
|------:|----------------------------:|------------------:|-------------------------:|
$(for i in "${!answer_ms[@]}"; do printf '| %5d | %27s | %17s | %24s |\n' $(( i + 1 )) "${command_ms[$i]}" "${answer_ms[$i]}" "${synced_ms[$i]}"; done)

Average until a synchronized answer: **$average_synced ms**.

With \`local stratum $LOCAL_STRATUM\` (no upstream servers, as in an offline network) the
server's time is usable as soon as it answers. A server with real upstream
servers must first measure them, which takes one or more minutes
(PERFORMANCE-METRICS.md, section 4.1).

EOF
}

# -----------------------------------------------------------------------------
test_quality() {
    section_title "Served time quality  (one request, every field checked)"

    local answer_file="$RAW_DIR/quality.txt"
    ntp_probe > "$answer_file" || die "The test server did not answer."
    test_chronyc tracking > "$RAW_DIR/quality-chronyc-tracking.txt" 2>&1 || true

    field() { awk -v key="$1" '$1 == key { print $3 }' "$answer_file"; }
    local leap stratum leap_text
    leap="$(field leap)"
    stratum="$(field stratum)"
    leap_text="$(awk -F' = ' '$1 ~ /^leap_text/ { print $2 }' "$answer_file")"

    local verdict=PASS
    (( leap == 3 || stratum == 0 || stratum > TARGET_MAX_STRATUM )) && verdict=FAIL

    record_result "Served time quality" "stratum $stratum, leap status '$leap_text'" \
        "stratum 1-$TARGET_MAX_STRATUM, synchronized" "$verdict"

    add_details <<EOF
## Served time quality

Every NTP answer tells the client how good the server's time is. Clients use
these fields to choose between servers, and they ignore a server that says it
is not synchronized. One request was sent and every field checked:

| Field | Value | Meaning |
|---|---|---|
| Leap status | $leap ($leap_text) | 3 = "not synchronized": clients ignore the server |
| Stratum | $stratum | steps from a reference clock; 16 or 0 = unusable |
| Reference ID | $(field reference_id) | what the server follows (127.127.1.1 = its own clock, "local") |
| Root delay | $(field root_delay_ms) ms | round-trip time to the reference clock |
| Root dispersion | $(field root_dispersion_ms) ms | the server's own estimate of its maximum error |
| Precision | $(field precision_us) µs | how finely the server can read its clock |
| Reference updated | $(field reference_age_s) s ago | when the server last updated its time |

The client's maximum error is about root delay / 2 + root dispersion. Here
the server follows its own clock, so both are near 0: the server is only as
right as this machine's clock. In an offline network, set that clock
correctly (and keep it correct) - see PERFORMANCE-METRICS.md, section 4.2.

The server's own view (\`chronyc tracking\`): \`raw/quality-chronyc-tracking.txt\`

EOF
}

# -----------------------------------------------------------------------------
test_throughput() {
    section_title "Throughput  (as many requests as chronyd can answer, $DURATION s)"
    run_full_load

    local rate verdict chronyd_cpu_pct
    rate="$(value answered_per_s)"
    verdict="$(verdict_at_least "$rate" "$TARGET_THROUGHPUT_MIN_PER_S")"
    chronyd_cpu_pct="$(calc "$(value chronyd_cpu_seconds) / $DURATION * 100")"

    record_result "Throughput" "$(pretty "$rate") answers/s" \
        ">= $(pretty "$TARGET_THROUGHPUT_MIN_PER_S") answers/s" "$verdict"

    add_details <<EOF
## Throughput

$(value workers) sender process(es), each keeping $(value window) requests in flight, sent
requests for $DURATION seconds as fast as chronyd answered them. Each answer lets
a sender send one more request, so the server always has work waiting.

| | |
|---|---|
| Answers per second | **$(pretty "$rate")** |
| Requests sent / answered | $(pretty "$(value sent)") / $(pretty "$(value answered)") |
| Not answered | $(value lost_pct)% (under full load a few are normal: see the "loss" test for normal load) |
| Lost in chronyd's receive buffer (server too busy) | $(pretty "$(value server_socket_drops)") |
| chronyd CPU during the run | $chronyd_cpu_pct% of one CPU core (chronyd uses one thread) |
| Senders' CPU (average / highest) | $(calc "$(value generator_cpu_pct_avg)")% / $(calc "$(value generator_cpu_pct_max)")% |

What this number means: an NTP client asks about once every 64 to 1024
seconds. At this rate one server could serve about
**$(pretty "$(calc "$rate * 64")") clients polling every 64 s** (PERFORMANCE-METRICS.md 4.3).

EOF
}

# -----------------------------------------------------------------------------
test_latency() {
    section_title "Response time  ($LOAD_RATE requests/s from $LOAD_CLIENTS clients, $DURATION s)"
    run_steady_load

    local p95 p99 verdict
    p95="$(value response_ms_p95)"
    p99="$(value response_ms_p99)"
    verdict=PASS
    is_at_most "$p95" "$TARGET_LATENCY_P95_MAX_MS" || verdict=FAIL
    is_at_most "$p99" "$TARGET_LATENCY_P99_MAX_MS" || verdict=FAIL

    record_result "Response time" "p95 $p95 ms, p99 $p99 ms" \
        "p95 <= $TARGET_LATENCY_P95_MAX_MS ms, p99 <= $TARGET_LATENCY_P99_MAX_MS ms" "$verdict"

    add_details <<EOF
## Response time

A steady load of $LOAD_RATE requests per second (at random moments, like independent
clients) from $LOAD_CLIENTS different client addresses, for $DURATION seconds.
The time of every answer was measured: from sending the request until the
answer arrived (kernel receive time).

| Percentile | Response time (ms) | Meaning |
|---|---:|---|
| average | $(value response_ms_avg) | |
| p50 (median) | $(value response_ms_p50) | half of the answers were faster |
| p90 | $(value response_ms_p90) | |
| p95 | **$p95** | 95 of 100 answers were faster |
| p99 | **$p99** | 99 of 100 answers were faster |
| p99.9 | $(value response_ms_p999) | |
| slowest | $(value response_ms_max) | |

Time spent **inside chronyd** (from the answer's own receive and transmit
timestamps): median $(value processing_us_p50) µs, p99 $(value processing_us_p99) µs, slowest $(value processing_us_max) µs.

For NTP, a stable response time matters more than a short one: clients assume
the request and the answer took equally long, so any difference between the two
becomes a time error (see "accuracy").

EOF
}

# -----------------------------------------------------------------------------
test_loss() {
    section_title "Loss  (the same steady load as 'latency')"
    run_steady_load

    local failed verdict
    failed="$(value failed_pct)"
    verdict="$(verdict_at_most "$failed" "$TARGET_LOSS_MAX_PCT")"

    record_result "Loss" "$(calc3 "$failed")% ($(value lost) lost, $(value kiss_of_death) refused, $(value unsynchronized) unsynchronized)" \
        "<= $TARGET_LOSS_MAX_PCT%" "$verdict"

    add_details <<EOF
## Loss

Of the $(pretty "$(value sent)") requests of the steady load, these did not get a usable answer:

| | Count |
|---|---:|
| No answer within 1 s (lost) | $(value lost) |
| "Kiss-o'-Death" answers (server refused: $(value kiss_codes)) | $(value kiss_of_death) |
| Answers saying "not synchronized" | $(value unsynchronized) |
| **Total failed** | **$(calc3 "$failed")%** |
| Answered correctly | $(value answered_pct)% |

Where requests can be lost (a real network adds its own losses):

| Place | Count |
|---|---:|
| chronyd's receive buffer overflowed (chronyd too busy) | $(value server_socket_drops) |
| chronyd ignored them on purpose (rate limiting, access rules) | $(value server_ntp_dropped) |
| the load generator's own receive buffer overflowed | $(value generator_socket_drops) |

EOF
}

# -----------------------------------------------------------------------------
test_accuracy() {
    section_title "Accuracy  (time offset seen by clients, same steady load)"
    run_steady_load

    local p99 verdict
    p99="$(value offset_us_abs_p99)"
    verdict="$(verdict_at_most "$p99" "$TARGET_OFFSET_P99_MAX_US")"

    record_result "Accuracy (offset)" "p99 $p99 µs, median $(value offset_us_median) µs" \
        "p99 <= $TARGET_OFFSET_P99_MAX_US µs" "$verdict"

    add_details <<EOF
## Accuracy (time offset seen by clients)

For every answer of the steady load the client computed the **offset**, the
difference between the server's time and its own, the same way every NTP client
does:  offset = ((T2 - T1) + (T3 - T4)) / 2.

Client and server run on the **same machine and read the same clock**, so the
true offset is exactly 0. Everything measured is error added by the server
and by the way it is reached: when a timestamp is taken, and the difference
between the request's and the answer's travel time.

| | Offset (µs) |
|---|---:|
| smallest | $(value offset_us_min) |
| median | $(value offset_us_median) |
| largest | $(value offset_us_max) |
| p95 of the absolute value | $(value offset_us_abs_p95) |
| **p99 of the absolute value** | **$p99** |

A client on a real network also gets the network's asymmetry (up to half the
round-trip time). Typical good results: tens of µs in a LAN, 1 ms or less across
routers (PERFORMANCE-METRICS.md 4.6).

EOF
}

# -----------------------------------------------------------------------------
test_clients() {
    section_title "Many clients  ($(pretty "$CLIENT_STEPS_RATE") requests/s, spread over more and more clients)"

    # "best" = the highest step reached before the first step that was not ok.
    local step best=0 all_ok_so_far=yes table="" step_verdict
    for step in $CLIENT_STEPS; do
        run_load "clients-$step" steady --rate "$CLIENT_STEPS_RATE" --duration "$DURATION" \
            --clients "$step" --first-client "$CLIENTS_STEPS"

        step_verdict=ok
        is_at_most "$(value failed_pct)" "$TARGET_LOSS_MAX_PCT"            || step_verdict="too many failed"
        is_at_most "$(value response_ms_p99)" "$TARGET_LATENCY_P99_MAX_MS" || step_verdict="too slow"
        if [[ $step_verdict == ok && $all_ok_so_far == yes ]]; then
            best="$step"
        else
            all_ok_so_far=no
        fi

        table+="| $(pretty "$step") | $(value response_ms_p50) | $(value response_ms_p99) | $(calc3 "$(value failed_pct)") | $(pretty "$(value server_records_dropped)") | $step_verdict |"$'\n'
    done

    local remembered
    remembered="$(remembered_clients)"
    record_result "Many clients" "$(pretty "$best") clients (highest step that was fast and complete)" \
        ">= $(pretty "$TARGET_CLIENTS_MIN") clients" "$(verdict_at_least "$best" "$TARGET_CLIENTS_MIN")"

    add_details <<EOF
## Many clients

The same load of $(pretty "$CLIENT_STEPS_RATE") requests per second for $DURATION seconds, each time spread
over more different client addresses. chronyd keeps a record for every client
(for rate limiting and \`chronyc clients\`), so more clients means more
bookkeeping. A step is "ok" when no request failed and p99 <= $TARGET_LATENCY_P99_MAX_MS ms.

| Clients | p50 (ms) | p99 (ms) | Failed (%) | Client records replaced (memory full) | Step |
|---:|---:|---:|---:|---:|---|
$table
Highest step that was ok: **$(pretty "$best") clients**.

chronyd remembers at most as many clients as fit in \`clientloglimit\`
($(pretty "$CLIENT_LOG_LIMIT") bytes here); it now remembers $(pretty "$remembered"). Clients it cannot
remember are still answered, but when the memory is full every new client
replaces an old record, so rate limiting cannot follow them reliably
(PERFORMANCE-METRICS.md 4.7 and 4.10).

EOF
}

# -----------------------------------------------------------------------------
restore_normal_config() {
    if [[ $RATELIMIT_ACTIVE == yes ]]; then
        write_test_config
        chronyd_restart
        wait_until_ntp_answers 30 || warn "The test server did not come back after the ratelimit test."
        RATELIMIT_ACTIVE=no
    fi
}

test_ratelimit() {
    section_title "Rate limiting  (1 client floods with $FLOOD_RATE requests/s, $NORMAL_CLIENTS normal clients)"

    # 1) Restart the test server with rate limiting switched on.
    log "Restarting the test server with:  $RATELIMIT_LINE"
    RATELIMIT_ACTIVE=yes
    write_test_config "$RATELIMIT_LINE"
    chronyd_restart
    wait_until_ntp_answers 30 || die "The test server did not answer with rate limiting on."

    # 2) The flooding client and the normal clients, at the same time.
    local dropped_before
    dropped_before="$(server_stat 'NTP packets dropped')"
    python3 "$NTP_LOAD" steady --server "$LISTEN_ADDRESS" --port "$NTP_PORT" \
        --rate "$FLOOD_RATE" --duration "$DURATION" --clients 1 --even \
        --first-client "$CLIENTS_FLOOD" > "$RAW_DIR/ratelimit-flood.txt" &
    local flood_pid=$!
    python3 "$NTP_LOAD" steady --server "$LISTEN_ADDRESS" --port "$NTP_PORT" \
        --rate "$NORMAL_RATE" --duration "$DURATION" --clients "$NORMAL_CLIENTS" \
        --first-client "$CLIENTS_NORMAL" > "$RAW_DIR/ratelimit-normal.txt"
    wait "$flood_pid"
    local dropped=$(( $(server_stat 'NTP packets dropped') - dropped_before ))

    # 3) Back to the normal configuration (no rate limiting).
    log "Restarting the test server without rate limiting"
    restore_normal_config

    local flood_pct normal_pct verdict=PASS
    flood_pct="$(value_of ratelimit-flood answered_pct)"
    normal_pct="$(value_of ratelimit-normal answered_pct)"
    is_at_least "$normal_pct" "$TARGET_RATELIMIT_NORMAL_MIN_PCT" || verdict=FAIL
    is_at_most  "$flood_pct"  "$TARGET_RATELIMIT_FLOOD_MAX_PCT"  || verdict=FAIL
    printf '    flooding client answered: %s%%   normal clients answered: %s%%\n' "$(calc "$flood_pct")" "$(calc "$normal_pct")"

    record_result "Rate limiting" "flooder $(calc "$flood_pct")% answered, normal clients $(calc "$normal_pct")%" \
        "flooder <= $TARGET_RATELIMIT_FLOOD_MAX_PCT%, normal >= $TARGET_RATELIMIT_NORMAL_MIN_PCT%" "$verdict"

    add_details <<EOF
## Rate limiting

The test server was restarted with \`$RATELIMIT_LINE\` (for this test only).
Then, for $DURATION seconds and at the same time:

- **one flooding client** sent $FLOOD_RATE requests per second (a broken or hostile
  client, or a denial-of-service attempt), and
- **$NORMAL_CLIENTS normal clients** sent $NORMAL_RATE requests per second together (each
  about one request per $(calc "$NORMAL_CLIENTS / $NORMAL_RATE") s).

| | Sent | Answered | Answered (%) |
|---|---:|---:|---:|
| Flooding client | $(pretty "$(value_of ratelimit-flood sent)") | $(pretty "$(value_of ratelimit-flood answered_good)") | **$(calc "$flood_pct")** |
| Normal clients | $(pretty "$(value_of ratelimit-normal sent)") | $(pretty "$(value_of ratelimit-normal answered_good)") | **$(calc "$normal_pct")** |

chronyd ignored $(pretty "$dropped") requests on purpose. Good rate limiting answers
**all** normal clients and only a small part of the flood. With \`leak 2\`, about
1 in 4 requests over the limit is still answered on purpose: a client whose
address is being faked by an attacker is then slowed down, but not cut off
(PERFORMANCE-METRICS.md 4.8).

Raw results: \`raw/ratelimit-flood.txt\`, \`raw/ratelimit-normal.txt\`

EOF
}

# -----------------------------------------------------------------------------
test_cpu() {
    section_title "CPU per request  (re-uses the 'throughput' and 'latency' runs)"
    run_full_load
    local full_cpu full_answers
    full_cpu="$(value chronyd_cpu_seconds)"
    full_answers="$(value answered)"
    run_steady_load
    local steady_cpu steady_answers
    steady_cpu="$(value chronyd_cpu_seconds)"
    steady_answers="$(value answered_good)"

    if [[ $full_cpu == n/a ]] || (( full_answers == 0 )); then
        record_result "CPU per request" "not measured" "<= $TARGET_CPU_MAX_US_PER_REQUEST µs" FAIL
        return
    fi

    local full_us steady_us steady_pct
    full_us="$(calc2 "$full_cpu * 1000000 / $full_answers")"
    steady_us="$(calc2 "$steady_cpu * 1000000 / ($steady_answers + 0.000001)")"
    steady_pct="$(calc2 "$steady_cpu / $DURATION * 100")"

    record_result "CPU per request" "$full_us µs at full load ($steady_pct% CPU at $LOAD_RATE/s)" \
        "<= $TARGET_CPU_MAX_US_PER_REQUEST µs" "$(verdict_at_most "$full_us" "$TARGET_CPU_MAX_US_PER_REQUEST")"

    add_details <<EOF
## CPU per request

chronyd's CPU time (user + system, from /proc/<pid>/stat) was read before
and after each load, and divided by the number of answers.

| Load | Answers | chronyd CPU (s) | CPU per answer (µs) | CPU use |
|---|---:|---:|---:|---:|
| Full speed ("throughput") | $(pretty "$full_answers") | $full_cpu | **$full_us** | $(calc "$full_cpu / $DURATION * 100")% of one core |
| Steady $LOAD_RATE/s ("latency") | $(pretty "$steady_answers") | $steady_cpu | $steady_us | $steady_pct% of one core |

At a low rate every request also pays for chronyd waking up, so the cost per
answer is higher than at full speed. The CPU needed for your own clients is
about: clients / poll interval x CPU per answer (PERFORMANCE-METRICS.md 4.9).

EOF
}

# -----------------------------------------------------------------------------
test_memory() {
    section_title "Memory per client  ($(pretty "$MEMORY_CLIENTS") new clients, one request each)"

    # Start from an empty client list.
    chronyd_restart
    wait_until_ntp_answers 30 || die "The test server did not answer after the restart."
    sleep 1
    local rss_before
    rss_before="$(chronyd_rss_kb)"

    run_load memory steady --rate 5000 --count "$MEMORY_CLIENTS" \
        --clients "$MEMORY_CLIENTS" --first-client "$CLIENTS_MEMORY"

    local rss_after remembered not_remembered
    rss_after="$(chronyd_rss_kb)"
    remembered="$(remembered_clients)"
    not_remembered="$(value server_records_dropped)"

    local growth_kb=$(( rss_after - rss_before ))
    local bytes_per_client verdict
    if (( remembered > 0 )); then
        bytes_per_client="$(calc "$growth_kb * 1024 / $remembered")"
        verdict="$(verdict_at_most "$bytes_per_client" "$TARGET_MEMORY_MAX_BYTES_PER_CLIENT")"
    else
        bytes_per_client="n/a"
        verdict=FAIL
    fi

    record_result "Memory per client" "$bytes_per_client bytes ($(pretty "$remembered") clients remembered)" \
        "<= $TARGET_MEMORY_MAX_BYTES_PER_CLIENT bytes" "$verdict"
    record_result "Memory at start" "$(calc "$rss_before / 1024") MB" "-" INFO

    add_details <<EOF
## Memory per client

The test server was restarted (empty client list), then $(pretty "$MEMORY_CLIENTS") new clients each
sent one request. chronyd's resident memory (VmRSS) was read before and after.

| | |
|---|---|
| Memory after start | $(calc "$rss_before / 1024") MB |
| Memory after $(pretty "$MEMORY_CLIENTS") clients | $(calc "$rss_after / 1024") MB (+$(pretty "$growth_kb") KB) |
| Clients remembered (\`chronyc clients\`) | $(pretty "$remembered") |
| Client records replaced (memory full) | $(pretty "$not_remembered") |
| **Memory per remembered client** | **$bytes_per_client bytes** |
| Limit (\`clientloglimit\`) | $(pretty "$CLIENT_LOG_LIMIT") bytes |

chronyd never uses more memory for clients than \`clientloglimit\`, however
many clients there are: memory cannot run out. When the limit is reached,
older records are reused. The table reserves 128 bytes per client, so a
server that must rate-limit N clients needs \`clientloglimit\` of about N x 128
(PERFORMANCE-METRICS.md 4.10).

EOF
}


# =============================================================================
#  PART 5 - preparing, and writing the final report
# =============================================================================

# Stop with exit code 2 ("could not run") and a clear message.
cannot_run() {
    printf '\033[1;31m[ERROR ]\033[0m %s\n' "$*" >&2
    exit 2
}

check_ready_to_test() {
    [[ $EUID -eq 0 ]]              || cannot_run "Please run as root, for example:  sudo $0"
    command -v python3 >/dev/null  || cannot_run "'python3' not found."
    command -v chronyc >/dev/null  || cannot_run "'chronyc' not found. Run ./start-chrony.sh first."
    test_chronyd_is_running        || cannot_run "The test chronyd is not running. Run ./start-chrony.sh first."
    ntp_answers                    || cannot_run "The test chronyd does not answer on $LISTEN_ADDRESS:$NTP_PORT. Run ./start-chrony.sh first."
    test_chronyc serverstats >/dev/null 2>&1 ||
        cannot_run "chronyc cannot reach the test chronyd through $TEST_SOCKET. Run ./start-chrony.sh again."
}

prepare_result_folder() {
    RESULT_DIR="$KIT_DIR/results/$(date +%Y%m%d-%H%M%S)"
    RAW_DIR="$RESULT_DIR/raw"
    DETAILS_FILE="$RAW_DIR/.details.md"
    mkdir -p "$RAW_DIR"
    : > "$DETAILS_FILE"
    ln -sfn "$(basename "$RESULT_DIR")" "$KIT_DIR/results/latest"
}

# Called when the script ends, also after Ctrl-C or an error: stop the load
# generators and switch rate limiting off again.
finish() {
    kill $(jobs -p) 2>/dev/null || true
    restore_normal_config
}

write_report() {
    local tests_run="$1" started="$2" finished="$3"
    local report="$RESULT_DIR/report.md"
    local os_name selinux cpu_model memory_total

    os_name="$(. /etc/os-release && echo "$PRETTY_NAME")"
    selinux="$(getenforce 2>/dev/null || echo 'not available')"
    cpu_model="$(awk -F': ' '/^model name/ { print $2; exit }' /proc/cpuinfo)"
    memory_total="$(awk '/^MemTotal/ { printf "%.1f GB", $2 / 1048576 }' /proc/meminfo)"

    local overall="ALL PASSED"
    (( FAIL_COUNT > 0 )) && overall="$FAIL_COUNT TEST(S) FAILED"

    {
        echo "# Chrony NTP Server Performance Test Report"
        echo
        echo "**Result: $overall**"
        echo
        echo "| | |"
        echo "|---|---|"
        echo "| Test started  | $started |"
        echo "| Test finished | $finished |"
        echo "| Host | $(hostname) |"
        echo "| Operating system | $os_name (kernel $(uname -r)) |"
        echo "| CPU | $(nproc) x $cpu_model |"
        echo "| Memory | $memory_total |"
        echo "| NTP server | $(chrony_version), test instance \`$TEST_SERVICE\` (chronyd $CHRONYD_OPTIONS -x) |"
        echo "| Answers on | $LISTEN_ADDRESS, UDP port $NTP_PORT |"
        echo "| Time source | this machine's clock, \`local stratum $LOCAL_STRATUM\` |"
        echo "| clientloglimit | $(pretty "$CLIENT_LOG_LIMIT") bytes |"
        echo "| SELinux | $selinux |"
        echo "| Tests run | $tests_run |"
        echo "| Load per run | $DURATION s; steady load $LOAD_RATE requests/s from $LOAD_CLIENTS clients |"
        echo "| Load generator | lib/ntpload.py, $(python3 --version 2>&1) |"
        echo
        echo "## Summary"
        echo
        echo "PASS = target met, FAIL = target missed, INFO = measured only (no target)."
        echo "Targets are set in \`settings.conf\`. What each metric means: \`PERFORMANCE-METRICS.md\`."
        echo
        printf '| %-22s | %-62s | %-44s | %-7s |\n' "Test" "Measured" "Target" "Verdict"
        printf '|%s|%s|%s|%s|\n' "$(printf -- '-%.0s' {1..24})" "$(printf -- '-%.0s' {1..64})" \
                                 "$(printf -- '-%.0s' {1..46})" "$(printf -- '-%.0s' {1..9})"
        local row name measured target verdict
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            [[ $verdict == FAIL ]] && verdict="**FAIL**"
            printf '| %s | %s | %s | %-7s |\n' "$(cell "$name" 22)" "$(cell "$measured" 62)" \
                                               "$(cell "$target" 44)" "$verdict"
        done
        echo
        if (( ${#GENERATOR_LIMITED_TESTS[@]} > 0 )); then
            echo "> **Note:** the load generator itself was (nearly) the limit in:"
            echo "> ${GENERATOR_LIMITED_TESTS[*]}."
            echo "> In those runs chronyd may be faster than the numbers show."
            echo
        fi
        echo "> **Remember:** clients and server run on the same machine and talk over"
        echo "> 127.0.0.1. The results show what chronyd can do without a network;"
        echo "> real clients also pay the network's delay, loss and asymmetry."
        echo
        echo "# Details"
        echo
        cat "$DETAILS_FILE"
        echo "## Files in this folder"
        echo
        echo "- \`report.md\` - this report"
        echo "- \`summary.csv\` - the summary table, for spreadsheets"
        echo "- \`raw/\` - every value the load generator measured (\"key = value\" lines),"
        echo "  with chronyd's own counters for the same run"
    } > "$report"

    # The same summary as CSV.
    {
        echo "test,measured,target,verdict"
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            printf '"%s","%s","%s","%s"\n' "$name" "$measured" "$target" "$verdict"
        done
    } > "$RESULT_DIR/summary.csv"

    rm -f "$DETAILS_FILE"
}

print_usage() {
    # Print the comment block at the top of this file.
    sed -n '3,/^set -euo pipefail/{/^#/p}' "$0" | sed 's/^# \{0,1\}//'
}


# =============================================================================
#  MAIN
# =============================================================================
main() {
    case "${1:-}" in
        -h|--help) print_usage; exit 0 ;;
        --list)    printf '%s\n' "${ALL_TESTS[@]}"; exit 0 ;;
    esac

    # Which tests to run: the names given, or all of them.
    local tests=("$@")
    if (( ${#tests[@]} == 0 )) || [[ ${tests[0]} == all ]]; then
        tests=("${ALL_TESTS[@]}")
    fi
    local test_name
    for test_name in "${tests[@]}"; do
        if [[ " ${ALL_TESTS[*]} " != *" $test_name "* ]]; then
            echo "Unknown test: '$test_name'. Valid tests: ${ALL_TESTS[*]}" >&2
            exit 2
        fi
    done

    check_ready_to_test
    prepare_result_folder
    trap finish EXIT

    local started finished
    started="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    log "Testing the Chrony test server on $LISTEN_ADDRESS:$NTP_PORT - tests: ${tests[*]}"
    log "Results folder: $RESULT_DIR"

    for test_name in "${tests[@]}"; do
        "test_$test_name"
    done

    finished="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    write_report "${tests[*]}" "$started" "$finished"

    echo
    sed -n '/^## Summary/,/^# Details/p' "$RESULT_DIR/report.md" | sed '$d'
    echo
    ok "Report saved: $RESULT_DIR/report.md"

    # Exit code: 0 when nothing failed, otherwise 1.
    if (( FAIL_COUNT > 0 )); then
        exit 1
    fi
}

main "$@"