#!/usr/bin/env bash
# =============================================================================
#  test-dhcp.sh - measure DHCP server (dhcpd) performance, one metric at a
#                 time, and save an easy-to-read report
# =============================================================================
#
#  USAGE (as root, after ./start-dhcp.sh)
#      ./test-dhcp.sh                    run every test (about 3 minutes)
#      ./test-dhcp.sh throughput         run one test
#      ./test-dhcp.sh latency loss       run several tests
#      ./test-dhcp.sh --list             show the test names
#
#      DURATION=30 LOAD_RATE=500 ./test-dhcp.sh latency
#                                        override settings.conf for one run
#
#  THE TESTS  (explained in detail in PERFORMANCE-METRICS.md)
#      startup      time from "start daemon" until the first OFFER is sent
#      throughput   new leases handed out per second, at full speed
#      latency      time a client waits for its lease (average, percentiles)
#      loss         clients that got no lease at a normal, steady load
#      scaling      how the server copes as the client rate rises (capacity)
#      renew        lease renewals handled per second
#      cpu          CPU used by dhcpd at a steady load
#      memory       memory used by dhcpd, idle and under load
#      leasedb      how fast the lease database file grows
#
#  OUTPUT
#      results/<date>-<time>/report.md    the human-readable report
#      results/<date>-<time>/summary.csv  one line per test (for spreadsheets)
#      results/<date>-<time>/raw/         unmodified measurement output
#      results/latest                     link to the newest result folder
#
#  EXIT CODE
#      0 = every test passed,  1 = at least one FAIL,  2 = could not run
#
#  Load is generated by lib/dhcp-load.py, which simulates many DHCP clients
#  on the private test network. No internet access is needed.
# =============================================================================

set -euo pipefail
source "$(dirname -- "${BASH_SOURCE[0]}")/lib/common.sh"

ALL_TESTS=(startup throughput latency loss scaling renew cpu memory leasedb)

SAMPLE_INTERVAL_SECONDS=1       # how often CPU / memory / lease file are sampled
CLOCK_TICKS_PER_SECOND="$(getconf CLK_TCK)"
GENERATOR_BUSY_PCT=90           # load generator above this = it may be the limit

# Filled in by the functions below.
RESULT_DIR=""
RAW_DIR=""
DETAILS_FILE=""
SUMMARY_ROWS=()     # "Test|Measured|Target|Verdict"
FAIL_COUNT=0
GENERATOR_LIMITED_TESTS=()


# =============================================================================
#  PART 1 - small helpers
# =============================================================================

# Floating-point math and comparisons (bash itself only knows whole numbers).
calc()          { awk "BEGIN { printf \"%.1f\", $* }"; }
calc2()         { awk "BEGIN { printf \"%.2f\", $* }"; }
is_at_most()    { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a <= b) }'; }
is_at_least()   { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a >= b) }'; }
now_seconds()   { date +%s.%N; }

# Turn a comparison into the words PASS / FAIL.
verdict_at_most()  { if is_at_most  "$1" "$2"; then echo PASS; else echo FAIL; fi; }
verdict_at_least() { if is_at_least "$1" "$2"; then echo PASS; else echo FAIL; fi; }

# Remember one line for the summary table at the top of the report.
#   record_result "Test name" "measured value" "target" PASS|FAIL|INFO
record_result() {
    local name="$1" measured="$2" target="$3" verdict="$4"
    SUMMARY_ROWS+=("$name|$measured|$target|$verdict")
    [[ $verdict == FAIL ]] && FAIL_COUNT=$(( FAIL_COUNT + 1 ))

    local colour='1;32'                       # green  = PASS
    [[ $verdict == FAIL ]] && colour='1;31'   # red    = FAIL
    [[ $verdict == INFO ]] && colour='1;36'   # cyan   = INFO
    printf "    => \033[${colour}m%-4s\033[0m  %s   (target: %s)\n" "$verdict" "$measured" "$target"
}

# Append Markdown text (read from stdin) to the "details" part of the report.
add_details() {
    cat >> "$DETAILS_FILE"
}

section_title() {
    echo
    printf '\033[1m==> %s\033[0m\n' "$*"
}


# =============================================================================
#  PART 2 - running the load generator and reading its output
# =============================================================================
#
#  run_load <name> <mode> [generator options ...]
#
#  Runs lib/dhcp-load.py once and saves:
#      raw/<name>.txt              all results, as "key = value" lines
#      raw/<name>-percentiles.csv  latency for every percentile 1..100
#
#  Read single values afterwards with:  value <key>
#  Example:  run_load throughput dora --outstanding 50 --duration 10
#            value rate_per_s          ->  12157.200
# -----------------------------------------------------------------------------
LOAD_OUTPUT=""

run_load() {
    local name="$1"
    shift
    LOAD_OUTPUT="$RAW_DIR/$name.txt"

    printf '    %-28s ' "$name"
    if ! run_load_generator "$@" \
            --output "$LOAD_OUTPUT" \
            --percentiles "$RAW_DIR/$name-percentiles.csv" \
            2> "$RAW_DIR/$name-errors.txt"; then
        echo "failed"
        die "The load generator failed. See $RAW_DIR/$name-errors.txt"
    fi
    [[ -s $RAW_DIR/$name-errors.txt ]] || rm -f "$RAW_DIR/$name-errors.txt"

    printf '%10.0f per second, %s%% failed\n' "$(value rate_per_s)" "$(calc2 "$(value failed_pct)")"

    # Note it when the load generator itself was almost fully busy: then the
    # result shows the limit of the generator, not of the DHCP server.
    if is_at_least "$(value generator_cpu_pct)" "$GENERATOR_BUSY_PCT"; then
        GENERATOR_LIMITED_TESTS+=("$name")
        warn "The load generator was ${GENERATOR_BUSY_PCT}%+ busy in '$name'; the server may be faster than measured."
    fi
}

# One value from the last run_load output.  value <key>  (missing -> "0")
value() {
    awk -v key="$1" '$1 == key { print $3; found = 1 } END { if (!found) print 0 }' "$LOAD_OUTPUT"
}

# A latency value, rounded to 2 decimals.
ms() {
    calc2 "$(value "$1")"
}


# =============================================================================
#  PART 3 - measuring dhcpd itself (CPU, memory, disk)
# =============================================================================

# CPU time (seconds) that the dhcpd process has used so far.
# Fields 14 and 15 of /proc/<pid>/stat are user and system CPU ticks.
dhcpd_cpu_seconds() {
    local pid
    pid="$(dhcp_main_pid)"
    awk -v hz="$CLOCK_TICKS_PER_SECOND" '{ printf "%.3f", ($14 + $15) / hz }' "/proc/$pid/stat"
}

# Memory in RAM ("resident set size") of dhcpd, in MB.
dhcpd_memory_mb() {
    local pid
    pid="$(dhcp_main_pid)"
    awk '/^VmRSS:/ { printf "%.1f", $2 / 1024 }' "/proc/$pid/status"
}

# Bytes dhcpd has written to storage so far (from /proc/<pid>/io).
dhcpd_bytes_written() {
    local pid
    pid="$(dhcp_main_pid)"
    awk '/^write_bytes:/ { print $2 }' "/proc/$pid/io" 2>/dev/null || echo 0
}

lease_file_bytes() {
    stat -c %s "$TEST_LEASES"
}

# CPU seconds used so far by the logging daemons. dhcpd logs every DHCP
# message to syslog, so journald / rsyslogd also work while dhcpd is busy.
logging_cpu_seconds() {
    local pid total=0 ticks
    for pid in $(pgrep -x 'systemd-journal|rsyslogd' || true); do
        ticks="$(awk '{ print $14 + $15 }' "/proc/$pid/stat" 2>/dev/null)" || continue
        total=$(( total + ${ticks:-0} ))
    done
    calc2 "$total / $CLOCK_TICKS_PER_SECOND"
}

# UDP packets dropped on the CLIENT side (in the test namespace) because the
# load generator's receive buffer was full. Should always stay 0.
client_side_drops() {
    ip netns exec "$TEST_NETNS" awk '/^Udp:/ { if (++n == 2) print $6 }' /proc/net/snmp
}

# Runs in the background: writes one CSV line per second until killed.
sampler_loop() {
    local csv_file="$1"
    local start previous_time previous_cpu now cpu

    echo "elapsed_s,dhcpd_cpu_percent_of_one_core,dhcpd_memory_mb,lease_file_mb" > "$csv_file"
    start="$(now_seconds)"
    previous_time="$start"
    previous_cpu="$(dhcpd_cpu_seconds)"

    while true; do
        sleep "$SAMPLE_INTERVAL_SECONDS"
        now="$(now_seconds)"
        cpu="$(dhcpd_cpu_seconds)"
        printf '%s,%s,%s,%s\n' \
            "$(calc "$now - $start")" \
            "$(calc "($cpu - $previous_cpu) / ($now - $previous_time) * 100")" \
            "$(dhcpd_memory_mb)" \
            "$(calc2 "$(lease_file_bytes) / 1048576")" >> "$csv_file"
        previous_time="$now"
        previous_cpu="$cpu"
    done
}

# -----------------------------------------------------------------------------
#  run_steady_load
#
#  One run at a steady LOAD_RATE new clients per second, while dhcpd's CPU,
#  memory and lease file are sampled every second. The latency, loss, cpu,
#  memory and leasedb tests all read from this same run, so when you run
#  more than one of them the load is generated only once.
#
#  Sets: STEADY_OUTPUT (generator results file)
#        STEADY_CPU_AVG_PCT  STEADY_CPU_PEAK_PCT  STEADY_CPU_SECONDS
#        STEADY_MEM_IDLE_MB  STEADY_MEM_PEAK_MB
#        STEADY_LEASE_BYTES_BEFORE  STEADY_LEASE_BYTES_AFTER  STEADY_DISK_WRITTEN
#        STEADY_LOG_CPU_SECONDS  STEADY_CLIENT_DROPS  STEADY_LOG_WARNINGS
#        STEADY_SECONDS
# -----------------------------------------------------------------------------
STEADY_LOAD_DONE=no

run_steady_load() {
    [[ $STEADY_LOAD_DONE == yes ]] && { LOAD_OUTPUT="$STEADY_OUTPUT"; return; }
    local samples="$RAW_DIR/resource-samples.csv"

    # Values before the load starts.
    STEADY_MEM_IDLE_MB="$(dhcpd_memory_mb)"
    STEADY_LEASE_BYTES_BEFORE="$(lease_file_bytes)"
    local disk_before cpu_before time_before log_cpu_before drops_before since_epoch
    disk_before="$(dhcpd_bytes_written)"
    cpu_before="$(dhcpd_cpu_seconds)"
    log_cpu_before="$(logging_cpu_seconds)"
    drops_before="$(client_side_drops)"
    time_before="$(now_seconds)"
    since_epoch="$(date +%s)"

    sampler_loop "$samples" &
    local sampler_pid=$!

    run_load "steady-load-${LOAD_RATE}-per-s" dora --rate "$LOAD_RATE" --duration "$DURATION"
    STEADY_OUTPUT="$LOAD_OUTPUT"

    local cpu_after time_after
    cpu_after="$(dhcpd_cpu_seconds)"
    time_after="$(now_seconds)"
    kill "$sampler_pid" 2>/dev/null || true
    wait "$sampler_pid" 2>/dev/null || true

    STEADY_SECONDS="$(calc2 "$time_after - $time_before")"
    STEADY_CPU_SECONDS="$(calc2 "$cpu_after - $cpu_before")"
    STEADY_CPU_AVG_PCT="$(calc "$STEADY_CPU_SECONDS / $STEADY_SECONDS * 100")"
    STEADY_CPU_PEAK_PCT="$(awk -F, 'NR > 1 && $2 > m { m = $2 } END { printf "%.1f", m }' "$samples")"
    STEADY_MEM_PEAK_MB="$(awk -F, -v m="$STEADY_MEM_IDLE_MB" 'NR > 1 && $3 > m { m = $3 } END { printf "%.1f", m }' "$samples")"
    STEADY_LEASE_BYTES_AFTER="$(lease_file_bytes)"
    STEADY_DISK_WRITTEN=$(( $(dhcpd_bytes_written) - disk_before ))
    STEADY_LOG_CPU_SECONDS="$(calc2 "$(logging_cpu_seconds) - $log_cpu_before")"
    STEADY_CLIENT_DROPS=$(( $(client_side_drops) - drops_before ))

    # Warnings dhcpd logged during the run (only available with systemd).
    STEADY_LOG_WARNINGS="not available (no systemd journal)"
    if has_systemd; then
        journalctl -u "$TEST_SERVICE" --since "@$since_epoch" --no-pager -q 2>/dev/null |
            grep -iE 'no free leases|error|unable|fail|drop' > "$RAW_DIR/steady-load-dhcpd-warnings.txt" || true
        STEADY_LOG_WARNINGS="$(wc -l < "$RAW_DIR/steady-load-dhcpd-warnings.txt")"
    fi

    STEADY_LOAD_DONE=yes
}


# =============================================================================
#  PART 4 - the tests, one function per metric
# =============================================================================

# -----------------------------------------------------------------------------
test_startup() {
    section_title "Startup time  ($STARTUP_ROUNDS restarts)"
    local lease_file_size
    lease_file_size="$(du -h "$TEST_LEASES" | cut -f1)"
    local round start_time offer_time elapsed_ms times=()

    for (( round = 1; round <= STARTUP_ROUNDS; round++ )); do
        dhcp_stop

        # Start the probe BEFORE the daemon: it sends a DISCOVER every 5 ms
        # and notes the exact moment the first OFFER arrives.
        local probe_output="$RAW_DIR/startup-round-$round.txt"
        local ready_file="$RAW_DIR/.probe-ready"
        rm -f "$ready_file"
        run_load_generator probe --timeout 60 --ready-file "$ready_file" \
            --output "$probe_output" 2>/dev/null &
        local probe_pid=$!
        while [[ ! -e $ready_file ]] && kill -0 "$probe_pid" 2>/dev/null; do
            sleep 0.01
        done
        rm -f "$ready_file"

        start_time="$(now_seconds)"
        dhcp_start
        if ! wait "$probe_pid"; then
            die "The DHCP server did not answer within 60 s after restart."
        fi

        offer_time="$(awk '$1 == "first_offer_epoch" { print $3 }' "$probe_output")"
        elapsed_ms="$(calc "($offer_time - $start_time) * 1000")"
        times+=("$elapsed_ms")
        printf '    restart %d: %s ms\n' "$round" "$elapsed_ms"
    done

    local average fastest slowest
    average="$(printf '%s\n' "${times[@]}" | awk '{ s += $1 } END { printf "%.1f", s / NR }')"
    fastest="$(printf '%s\n' "${times[@]}" | sort -n | head -1)"
    slowest="$(printf '%s\n' "${times[@]}" | sort -n | tail -1)"

    record_result "Startup time" "$average ms average" "<= $TARGET_STARTUP_MAX_MS ms" \
        "$(verdict_at_most "$average" "$TARGET_STARTUP_MAX_MS")"

    add_details <<EOF
## Startup time

The DHCP server was stopped and started $STARTUP_ROUNDS times. Each time was
measured from the start command until the first client received an OFFER.
The lease database was $lease_file_size at the time (dhcpd reads all of it
at startup, so a bigger file means a slower start).

| Round | Time (ms) |
|------:|----------:|
$(for i in "${!times[@]}"; do printf '| %5d | %9s |\n' $(( i + 1 )) "${times[$i]}"; done)

Average **$average ms**, fastest $fastest ms, slowest $slowest ms.

EOF
}

# -----------------------------------------------------------------------------
test_throughput() {
    section_title "Throughput  ($OUTSTANDING clients at a time, full speed)"
    run_load "throughput" dora --outstanding "$OUTSTANDING" --duration "$DURATION"

    local rate
    rate="$(calc "$(value rate_per_s)")"

    record_result "Throughput" "$rate new leases/s" \
        ">= $TARGET_THROUGHPUT_MIN_LPS leases/s" \
        "$(verdict_at_least "$rate" "$TARGET_THROUGHPUT_MIN_LPS")"

    add_details <<EOF
## Throughput

$OUTSTANDING simulated clients asked for a lease at the same time, for
$DURATION seconds. Each client started again (with the next MAC address)
as soon as it had its lease. Every lease is one full
DISCOVER -> OFFER -> REQUEST -> ACK exchange.

| Item | Value |
|---|---:|
| New leases per second | **$rate** |
| Leases handed out | $(value exchanges_completed) |
| Failed exchanges | $(value exchanges_failed) |
| Average time per lease | $(ms dora_avg_ms) ms |
| Load generator busy (% of one core) | $(calc "$(value generator_cpu_pct)")% |

Raw output: \`raw/throughput.txt\`

EOF
}

# -----------------------------------------------------------------------------
test_latency() {
    section_title "Latency  ($LOAD_RATE new clients per second)"
    run_steady_load

    local verdict=PASS
    is_at_most "$(value dora_p95_ms)" "$TARGET_LATENCY_P95_MAX_MS" || verdict=FAIL
    is_at_most "$(value dora_p99_ms)" "$TARGET_LATENCY_P99_MAX_MS" || verdict=FAIL

    record_result "Latency (p95 / p99)" "$(ms dora_p95_ms) ms / $(ms dora_p99_ms) ms" \
        "<= $TARGET_LATENCY_P95_MAX_MS ms / <= $TARGET_LATENCY_P99_MAX_MS ms" "$verdict"

    add_details <<EOF
## Latency (time to get a lease)

$LOAD_RATE new clients per second for $DURATION seconds. "p95 = 3 ms" means
95 of every 100 clients had their lease within 3 ms.

| Statistic | DISCOVER -> OFFER (ms) | REQUEST -> ACK (ms) | Whole lease (ms) |
|---|---:|---:|---:|
| Average      | $(ms discover_offer_avg_ms) | $(ms request_ack_avg_ms) | $(ms dora_avg_ms) |
| p50 (median) | $(ms discover_offer_p50_ms) | $(ms request_ack_p50_ms) | $(ms dora_p50_ms) |
| p90          | $(ms discover_offer_p90_ms) | $(ms request_ack_p90_ms) | $(ms dora_p90_ms) |
| p95          | $(ms discover_offer_p95_ms) | $(ms request_ack_p95_ms) | **$(ms dora_p95_ms)** |
| p99          | $(ms discover_offer_p99_ms) | $(ms request_ack_p99_ms) | **$(ms dora_p99_ms)** |
| Slowest      | $(ms discover_offer_max_ms) | $(ms request_ack_max_ms) | $(ms dora_max_ms) |

"Whole lease" is the time from the client's DISCOVER to the server's ACK,
the time a real client waits before it can use the network. Before it sends
the ACK, dhcpd writes the lease to disk, so on a slow disk REQUEST -> ACK is
clearly the slower half.

Every percentile from 1 to 100: \`raw/steady-load-${LOAD_RATE}-per-s-percentiles.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_loss() {
    section_title "Loss  ($LOAD_RATE new clients per second)"
    run_steady_load

    local failed_pct
    failed_pct="$(awk -v x="$(value failed_pct)" 'BEGIN { printf "%.3f", x }')"

    record_result "Loss (failed leases)" "$failed_pct% ($(value exchanges_failed) of $(value exchanges_started))" \
        "<= $TARGET_LOSS_MAX_PCT%" \
        "$(verdict_at_most "$failed_pct" "$TARGET_LOSS_MAX_PCT")"

    add_details <<EOF
## Loss (clients without a lease)

$LOAD_RATE new clients per second for $DURATION seconds. A client that gets no
answer within $EXCHANGE_TIMEOUT s, or gets a "no" (NAK), counts as failed.
The test does not re-send, so every lost packet shows up here. (A real client
re-sends after a few seconds, but its user notices the delay.)

| Item | Value |
|---|---:|
| Clients started | $(value exchanges_started) |
| Clients with a lease | $(value exchanges_completed) |
| DISCOVER without OFFER | $(value lost_discover) |
| REQUEST without ACK | $(value lost_request) |
| NAK answers ("no") | $(value naks) |
| **Failed** | **$failed_pct%** |
| Answers dropped on the client side (should be 0) | $STEADY_CLIENT_DROPS |
| Warnings in the dhcpd log during the test | $STEADY_LOG_WARNINGS |

EOF
}

# -----------------------------------------------------------------------------
test_scaling() {
    section_title "Scaling  (new clients per second: $RATE_LEVELS)"
    local rate table="" capacity=0 all_kept_up=yes

    for rate in $RATE_LEVELS; do
        run_load "scaling-${rate}-per-s" dora --rate "$rate" --duration "$DURATION"

        local achieved failed kept_up=yes
        achieved="$(calc "$(value rate_per_s)")"
        failed="$(calc2 "$(value failed_pct)")"

        # A step "keeps up" when almost every client got a lease, and the
        # leases were handed out at (nearly) the rate they were asked for.
        is_at_most "$failed" "$SCALING_LOSS_LIMIT_PCT" || kept_up=no
        is_at_least "$achieved" "$(calc "$rate * 0.95")" || kept_up=no

        if [[ $kept_up == yes && $all_kept_up == yes ]]; then
            capacity="$rate"
        else
            all_kept_up=no
        fi

        table+="$(printf '| %8s | %8s | %7s | %8s | %8s | %9s | %-7s |' \
            "$rate" "$achieved" "$failed" "$(ms dora_p95_ms)" "$(ms dora_p99_ms)" \
            "$(calc "$(value generator_cpu_pct)")" "$kept_up")"$'\n'
    done

    local measured="kept up to $capacity new clients/s"
    [[ $all_kept_up == yes ]] && measured="kept up at every step (>= $capacity/s)"
    [[ $capacity == 0 ]] && measured="did not keep up at any step"

    record_result "Scaling (capacity)" "$measured" \
        ">= $TARGET_CAPACITY_MIN_LPS/s" \
        "$(verdict_at_least "$capacity" "$TARGET_CAPACITY_MIN_LPS")"

    add_details <<EOF
## Scaling (capacity)

The number of new clients per second was raised step by step, $DURATION s per
step. A step "keeps up" when at most $SCALING_LOSS_LIMIT_PCT% of the clients failed
and at least 95% of the asked-for rate was served.

| Asked /s | Served /s | Failed % | p95 (ms) | p99 (ms) | Generator % | Kept up |
|---------:|----------:|---------:|---------:|---------:|------------:|:--------|
${table}
Capacity: **$measured**.
$(if [[ $all_kept_up == yes ]]; then
    echo
    echo "The server kept up with every step. To find its real limit, add higher"
    echo "rates, for example: RATE_LEVELS=\"$RATE_LEVELS 32000\" ./test-dhcp.sh scaling"
  fi)

"Generator %" is how busy the load generator was (% of one CPU core). Near
100% means the generator, not the server, set the limit at that step.

EOF
}

# -----------------------------------------------------------------------------
test_renew() {
    section_title "Renewals  ($RENEW_CLIENTS clients, $OUTSTANDING at a time)"
    run_load "renew" renew --clients "$RENEW_CLIENTS" --outstanding "$OUTSTANDING" --duration "$DURATION"

    local rate
    rate="$(calc "$(value rate_per_s)")"

    record_result "Renewal rate" "$rate renewals/s" \
        ">= $TARGET_RENEW_MIN_RPS renewals/s" \
        "$(verdict_at_least "$rate" "$TARGET_RENEW_MIN_RPS")"

    add_details <<EOF
## Renewal rate

First $RENEW_CLIENTS clients got a lease (not measured). Then those clients
renewed their leases (REQUEST -> ACK) as fast as possible for $DURATION
seconds, $OUTSTANDING at a time. On a normal network, most DHCP traffic is
renewals: every client renews at half of its lease time.

| Item | Value |
|---|---:|
| Renewals per second | **$rate** |
| Renewals done | $(value exchanges_completed) |
| Failed renewals | $(value exchanges_failed) |
| Average time per renewal | $(ms renew_avg_ms) ms |
| p95 / p99 | $(ms renew_p95_ms) ms / $(ms renew_p99_ms) ms |
| Load generator busy (% of one core) | $(calc "$(value generator_cpu_pct)")% |

EOF
}

# -----------------------------------------------------------------------------
test_cpu() {
    section_title "CPU usage  ($LOAD_RATE new clients per second)"
    run_steady_load

    local leases ms_per_1000
    leases="$(value exchanges_completed)"
    ms_per_1000="$(awk -v cpu="$STEADY_CPU_SECONDS" -v n="$leases" \
        'BEGIN { printf "%.1f", (n > 0 ? cpu * 1000 / n * 1000 : 0) }')"

    record_result "CPU usage" "$STEADY_CPU_AVG_PCT% of one core" \
        "<= $TARGET_CPU_MAX_PCT%" \
        "$(verdict_at_most "$STEADY_CPU_AVG_PCT" "$TARGET_CPU_MAX_PCT")"

    add_details <<EOF
## CPU usage

CPU used by the dhcpd process while it handed out $LOAD_RATE leases per second.
dhcpd is a single process with a single thread, so it can never use more
than one CPU core: **100% here means dhcpd is at its limit**, no matter how
many cores the machine has.

| Item | Value |
|---|---:|
| Average CPU, % of one core | **$STEADY_CPU_AVG_PCT%** |
| Busiest second, % of one core | $STEADY_CPU_PEAK_PCT% |
| CPU time per 1000 leases | $ms_per_1000 ms |
| CPU used by logging (journald + rsyslogd) in the same time | $STEADY_LOG_CPU_SECONDS s |

dhcpd logs every DHCP message to syslog. That logging work is done by
journald / rsyslogd, not dhcpd, and is shown separately above.

Per-second samples: \`raw/resource-samples.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_memory() {
    section_title "Memory usage  ($LOAD_RATE new clients per second)"
    run_steady_load

    record_result "Memory usage" "$STEADY_MEM_PEAK_MB MB peak (idle $STEADY_MEM_IDLE_MB MB)" \
        "<= $TARGET_MEMORY_MAX_MB MB" \
        "$(verdict_at_most "$STEADY_MEM_PEAK_MB" "$TARGET_MEMORY_MAX_MB")"

    add_details <<EOF
## Memory usage

Memory in RAM (resident set size) of the dhcpd process.

| Item | Idle | Under load (peak) |
|---|---:|---:|
| Memory (MB) | $STEADY_MEM_IDLE_MB | **$STEADY_MEM_PEAK_MB** |

dhcpd creates a record in memory for every address of the pool when it
starts ($POOL_START - $POOL_END here), so memory depends mostly on the
pool size and hardly on the load.

EOF
}

# -----------------------------------------------------------------------------
test_leasedb() {
    section_title "Lease database growth  ($LOAD_RATE new clients per second)"
    run_steady_load

    local leases growth bytes_per_lease mb_per_hour
    leases="$(value exchanges_completed)"
    growth=$(( STEADY_LEASE_BYTES_AFTER - STEADY_LEASE_BYTES_BEFORE ))
    bytes_per_lease="$(awk -v g="$growth" -v n="$leases" 'BEGIN { printf "%.0f", (n > 0 ? g / n : 0) }')"
    mb_per_hour="$(calc "$bytes_per_lease * $LOAD_RATE * 3600 / 1048576")"

    record_result "Lease file growth" "$bytes_per_lease bytes per lease ($mb_per_hour MB/hour)" \
        "information only" INFO

    add_details <<EOF
## Lease database growth

dhcpd keeps every lease in a text file, $TEST_LEASES. It adds a new record
to the end of the file for every lease it grants or renews, and flushes it to
disk before it answers the client. So disk speed limits how fast dhcpd works.
dhcpd rewrites the file (keeping only the newest record per address) when it
starts, and after that once an hour.

| Item | Value |
|---|---:|
| Lease file before the run | $(calc "$STEADY_LEASE_BYTES_BEFORE / 1048576") MB |
| Lease file after the run | $(calc "$STEADY_LEASE_BYTES_AFTER / 1048576") MB |
| Leases granted in the run | $leases |
| **Bytes added per lease** | **$bytes_per_lease** |
| Growth at $LOAD_RATE leases/s | $mb_per_hour MB per hour |
| Bytes dhcpd wrote to storage (from /proc) | $(calc "$STEADY_DISK_WRITTEN / 1048576") MB |

The test lease file is emptied at the start of every run
(RESET_LEASES_BEFORE_TEST=$RESET_LEASES_BEFORE_TEST in settings.conf).

EOF
}


# =============================================================================
#  PART 5 - preparing, and writing the final report
# =============================================================================

# Stop with exit code 2 ("could not run") and a clear message.
cannot_run() {
    printf '\033[1;31m[ERROR ]\033[0m %s\n' "$*" >&2
    exit 2
}

check_ready_to_test() {
    [[ $EUID -eq 0 ]]        || cannot_run "Please run as root, for example:  sudo $0"
    command -v python3 >/dev/null || cannot_run "'python3' not found."
    test_network_exists      || cannot_run "The test network does not exist. Run ./start-dhcp.sh first."
    dhcp_is_running          || cannot_run "The test DHCP server is not running. Run ./start-dhcp.sh first."
    dhcp_is_serving          || cannot_run "The test DHCP server does not answer. Run ./start-dhcp.sh first."
}

prepare_result_folder() {
    RESULT_DIR="$KIT_DIR/results/$(date +%Y%m%d-%H%M%S)"
    RAW_DIR="$RESULT_DIR/raw"
    DETAILS_FILE="$RAW_DIR/.details.md"
    mkdir -p "$RAW_DIR"
    : > "$DETAILS_FILE"
    ln -sfn "$(basename "$RESULT_DIR")" "$KIT_DIR/results/latest"
}

# A short unmeasured run, so the first real test does not start "cold".
warm_up() {
    log "Warm-up: 2 seconds of light load (not measured)"
    run_load_generator dora --rate 100 --duration 2 --output "$RAW_DIR/warm-up.txt" >/dev/null 2>&1 || true
}

write_report() {
    local tests_run="$1" started="$2" finished="$3"
    local report="$RESULT_DIR/report.md"
    local os_name selinux cpu_model memory_total

    os_name="$(. /etc/os-release && echo "$PRETTY_NAME")"
    selinux="$(getenforce 2>/dev/null || echo 'not available')"
    cpu_model="$(awk -F': ' '/^model name/ { print $2; exit }' /proc/cpuinfo)"
    memory_total="$(awk '/^MemTotal/ { printf "%.1f GB", $2 / 1048576 }' /proc/meminfo)"

    local overall="ALL PASSED"
    (( FAIL_COUNT > 0 )) && overall="$FAIL_COUNT TEST(S) FAILED"

    {
        echo "# DHCP Server Performance Test Report"
        echo
        echo "**Result: $overall**"
        echo
        echo "| | |"
        echo "|---|---|"
        echo "| Test started  | $started |"
        echo "| Test finished | $finished |"
        echo "| Host | $(hostname) |"
        echo "| Operating system | $os_name (kernel $(uname -r)) |"
        echo "| CPU | $(nproc) x $cpu_model |"
        echo "| Memory | $memory_total |"
        echo "| DHCP server | $(dhcpd_version) ($(rpm -q dhcp-server 2>/dev/null || echo 'dhcpd')) |"
        echo "| SELinux | $selinux |"
        echo "| Test network | $SERVER_IFACE $SERVER_IP/$PREFIX_LENGTH, pool $POOL_START - $POOL_END |"
        echo "| Lease file | $TEST_LEASES on $(df --output=source,fstype "$TEST_LEASES" | tail -1 | awk '{ print $1 " (" $2 ")" }') |"
        echo "| Tests run | $tests_run |"
        echo "| Load per run | $DURATION s; steady load $LOAD_RATE new clients/s; full speed $OUTSTANDING clients at a time |"
        echo
        echo "## Summary"
        echo
        echo "PASS = target met, FAIL = target missed, INFO = measured only (no target)."
        echo "Targets are set in \`settings.conf\`. What each metric means: \`PERFORMANCE-METRICS.md\`."
        echo
        printf '| %-22s | %-46s | %-30s | %-7s |\n' "Test" "Measured" "Target" "Verdict"
        printf '|%s|%s|%s|%s|\n' "$(printf -- '-%.0s' {1..24})" "$(printf -- '-%.0s' {1..48})" \
                                 "$(printf -- '-%.0s' {1..32})" "$(printf -- '-%.0s' {1..9})"
        local row name measured target verdict
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            [[ $verdict == FAIL ]] && verdict="**FAIL**"
            printf '| %-22s | %-46s | %-30s | %-7s |\n' "$name" "$measured" "$target" "$verdict"
        done
        echo
        if (( ${#GENERATOR_LIMITED_TESTS[@]} > 0 )); then
            echo "> **Note:** the load generator was at least ${GENERATOR_BUSY_PCT}% busy in:"
            echo "> ${GENERATOR_LIMITED_TESTS[*]}."
            echo "> In those runs the DHCP server may be faster than the numbers show."
            echo
        fi
        echo "# Details"
        echo
        cat "$DETAILS_FILE"
        echo "## Files in this folder"
        echo
        echo "- \`report.md\` - this report"
        echo "- \`summary.csv\` - the summary table, for spreadsheets"
        echo "- \`raw/\` - full output of every load run (\"key = value\" lines),"
        echo "  latency percentile tables and per-second resource samples"
    } > "$report"

    # The same summary as CSV.
    {
        echo "test,measured,target,verdict"
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            printf '"%s","%s","%s","%s"\n' "$name" "$measured" "$target" "$verdict"
        done
    } > "$RESULT_DIR/summary.csv"

    rm -f "$DETAILS_FILE"
}

print_usage() {
    # Print the comment block at the top of this file.
    sed -n '3,/^set -euo pipefail/{/^#/p}' "$0" | sed 's/^# \{0,1\}//'
}


# =============================================================================
#  MAIN
# =============================================================================
main() {
    case "${1:-}" in
        -h|--help) print_usage; exit 0 ;;
        --list)    printf '%s\n' "${ALL_TESTS[@]}"; exit 0 ;;
    esac

    # Which tests to run: the names given, or all of them.
    local tests=("$@")
    if (( ${#tests[@]} == 0 )) || [[ ${tests[0]} == all ]]; then
        tests=("${ALL_TESTS[@]}")
    fi
    local test_name
    for test_name in "${tests[@]}"; do
        if [[ " ${ALL_TESTS[*]} " != *" $test_name "* ]]; then
            echo "Unknown test: '$test_name'. Valid tests: ${ALL_TESTS[*]}" >&2
            exit 2
        fi
    done

    check_ready_to_test
    prepare_result_folder

    # Always stop background helpers if the script is interrupted.
    trap 'kill $(jobs -p) 2>/dev/null || true' EXIT

    local started finished
    started="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    log "Testing the DHCP server on $SERVER_IFACE ($SERVER_IP) - tests: ${tests[*]}"
    log "Results folder: $RESULT_DIR"
    if [[ $RESET_LEASES_BEFORE_TEST == yes ]]; then
        log "Emptying the test lease file first (RESET_LEASES_BEFORE_TEST=yes)"
        empty_lease_database
    fi
    warm_up

    for test_name in "${tests[@]}"; do
        "test_$test_name"
    done

    finished="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    write_report "${tests[*]}" "$started" "$finished"

    echo
    sed -n '/^## Summary/,/^# Details/p' "$RESULT_DIR/report.md" | sed '$d'
    echo
    ok "Report saved: $RESULT_DIR/report.md"

    # Exit code: 0 when nothing failed, otherwise 1.
    if (( FAIL_COUNT > 0 )); then
        exit 1
    fi
}

main "$@"