#!/usr/bin/env bash
# =============================================================================
#  test-nginx.sh - measure Nginx performance, one metric at a time,
#                  and save an easy-to-read report
# =============================================================================
#
#  USAGE (as root, after ./start-nginx.sh)
#      ./test-nginx.sh                    run every test (about 4 minutes)
#      ./test-nginx.sh throughput         run one test
#      ./test-nginx.sh latency errors     run several tests
#      ./test-nginx.sh --list             show the test names
#
#      DURATION=30 CONCURRENCY=100 ./test-nginx.sh throughput
#                                         override settings.conf for one run
#
#  THE TESTS  (explained in detail in PERFORMANCE-METRICS.md)
#      startup      time from "start daemon" until the first page is served
#      throughput   requests served per second
#      latency      response time per request (average and percentiles)
#      concurrency  how throughput and latency change as clients increase
#      keepalive    gain from reusing TCP connections
#      transfer     download speed of a large file (MB/s)
#      compression  gzip: how much smaller answers get, and what it costs
#      proxy        speed when Nginx forwards requests to a backend server
#      reload       failed requests while the configuration is reloaded
#      errors       failed requests while Nginx is overloaded
#      connections  open connections compared with the configured maximum
#      cpu          CPU used by Nginx under load
#      memory       memory used by Nginx, idle and under load
#
#  OUTPUT
#      results/<date>-<time>/report.md    the human-readable report
#      results/<date>-<time>/summary.csv  one line per test (for spreadsheets)
#      results/<date>-<time>/raw/         unmodified tool output
#      results/latest                     link to the newest result folder
#
#  EXIT CODE
#      0 = every test passed,  1 = at least one FAIL,  2 = could not run
#
#  Load is generated by ApacheBench ("ab", from the httpd-tools RPM).
#  All traffic stays on this machine; no internet access is needed.
# =============================================================================

set -euo pipefail
source "$(dirname -- "${BASH_SOURCE[0]}")/lib/common.sh"

# When the test cannot run at all, exit with code 2 (a FAIL result gives 1).
die() { printf '\033[1;31m[ERROR ]\033[0m %s\n' "$*" >&2; exit 2; }

ALL_TESTS=(startup throughput latency concurrency keepalive transfer compression
           proxy reload errors connections cpu memory)

SMALL_PAGE_URL="$BASE_URL/perf-test/small.html"     # 1 KB page
TEXT_PAGE_URL="$BASE_URL/perf-test/text.html"       # about 100 KB of HTML
LARGE_FILE_URL="$BASE_URL/perf-test/large.bin"      # 1 MB file
PROXY_PAGE_URL="$BASE_URL/perf-proxy/small.html"    # 1 KB page, through the proxy

SAMPLE_INTERVAL_SECONDS=1       # how often CPU / memory / connections are sampled
CPU_COUNT="$(nproc)"

# "ab" needs a request limit; it stops at DURATION or at this count,
# whichever comes first. 200,000 per second is far above what one host serves.
MAX_REQUESTS_PER_RUN=$(( DURATION * 200000 ))

# Filled in by the functions below.
RESULT_DIR=""
RAW_DIR=""
AB_DIR=""           # raw/ab-processes: the output of every single ab process
DETAILS_FILE=""
SUMMARY_ROWS=()     # "Test|Measured|Target|Verdict"
FAIL_COUNT=0


# =============================================================================
#  PART 1 - small helpers
# =============================================================================

# Floating-point math and comparisons (bash itself only knows whole numbers).
calc()          { awk "BEGIN { printf \"%.1f\", $* }"; }
is_at_most()    { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a <= b) }'; }
is_at_least()   { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a >= b) }'; }
now_seconds()   { date +%s.%N; }
count_words()   { echo $#; }      # count_words $list  ->  number of items in the list

# Round a number and add thousands separators: 357788.7 -> 357,789
thousands() {
    awk -v number="$1" 'BEGIN {
        digits = sprintf("%.0f", number); result = ""
        while (length(digits) > 3) {
            result = "," substr(digits, length(digits) - 2) result
            digits = substr(digits, 1, length(digits) - 3)
        }
        print digits result
    }'
}

# Turn a comparison into the words PASS / FAIL.
verdict_at_most()  { if is_at_most  "$1" "$2"; then echo PASS; else echo FAIL; fi; }
verdict_at_least() { if is_at_least "$1" "$2"; then echo PASS; else echo FAIL; fi; }

# Remember one line for the summary table at the top of the report.
#   record_result "Test name" "measured value" "target" PASS|FAIL|INFO
record_result() {
    local name="$1" measured="$2" target="$3" verdict="$4"
    SUMMARY_ROWS+=("$name|$measured|$target|$verdict")
    [[ $verdict == FAIL ]] && FAIL_COUNT=$(( FAIL_COUNT + 1 ))

    local colour='1;32'                       # green  = PASS
    [[ $verdict == FAIL ]] && colour='1;31'   # red    = FAIL
    [[ $verdict == INFO ]] && colour='1;36'   # cyan   = INFO
    printf "    => \033[${colour}m%-4s\033[0m  %s   (target: %s)\n" "$verdict" "$measured" "$target"
}

# Append Markdown text (read from stdin) to the "details" part of the report.
add_details() {
    cat >> "$DETAILS_FILE"
}

section_title() {
    echo
    printf '\033[1m==> %s\033[0m\n' "$*"
}


# =============================================================================
#  PART 2 - running ApacheBench and reading its output
# =============================================================================
#
#  run_ab <name> <url> <clients> <keepalive: yes|no> [seconds] [extra header]
#
#  Why several ab processes?
#  One ab process uses a single CPU core and cannot send much more than about
#  50,000 requests/s, while Nginx on a multi-core server can serve far more.
#  So the clients are split over AB_PROCESSES ab processes running side by
#  side, and their results are added up. Example: 50 clients with 4 processes
#  = 13 + 13 + 12 + 12 clients.
#
#  Saves, for each ab process i:
#      raw/ab-processes/<name>-ab<i>.txt              the full ab output
#      raw/ab-processes/<name>-ab<i>-percentiles.csv  its response time per percentile
#  and for the whole run:
#      raw/<name>-percentiles.csv            all processes merged, 0..99
#
#  Sets these variables for the caller (totals over all ab processes):
#      AB_COMPLETE  AB_FAILED  AB_FAILED_DETAIL  AB_NON_2XX  AB_SERVER_CLOSES
#      AB_RPS  AB_TRANSFER_KBPS  AB_MEAN_MS  AB_DOC_BYTES  AB_PROCESSES_USED
#      AB_P50_MS  AB_P90_MS  AB_P95_MS  AB_P99_MS  AB_MAX_MS
#
#  run_ab is made of two halves, ab_launch and ab_read_results, so that the
#  "reload" test can run the load in the background and reload meanwhile.
#
#  A note on "failed requests" in keep-alive mode:
#  A server sometimes answers a request completely but then closes the
#  keep-alive connection (the client simply reconnects). ApacheBench wrongly
#  counts each such answer as a "Length" failure. We count those closures
#  separately in AB_SERVER_CLOSES, and AB_FAILED holds only real failures.
# -----------------------------------------------------------------------------
run_ab() {
    local name="$1" url="$2" clients="$3" keepalive="$4" seconds="${5:-$DURATION}"
    printf '    %-34s %4s clients, keep-alive %-3s, %ss ... ' \
        "$name" "$clients" "$keepalive" "$seconds"
    ab_launch "$@"
    ab_read_results "$name" "$keepalive"
    printf '%10.0f req/s\n' "$AB_RPS"
}

# How many ab processes to use for <clients> clients.
ab_process_count() {
    local clients="$1" processes="$AB_PROCESSES"
    if [[ $processes == auto ]]; then
        processes=$(( CPU_COUNT / 2 ))         # half of the CPUs ...
        (( processes > 8 )) && processes=8     # ... but at most 8
    fi
    (( processes < 1 )) && processes=1
    (( processes > clients )) && processes="$clients"   # at least 1 client each
    echo "$processes"
}

# Start the ab processes, split the clients between them, wait for all.
ab_launch() {
    local name="$1" url="$2" clients="$3" keepalive="$4" seconds="${5:-$DURATION}"
    local extra_header="${6:-}"

    local processes
    processes="$(ab_process_count "$clients")"
    rm -f "$AB_DIR/$name"-ab*.txt "$AB_DIR/$name"-ab*-percentiles.csv

    local i my_clients pids=()
    for (( i = 1; i <= processes; i++ )); do
        # Share the clients out evenly; the first ones get any remainder.
        my_clients=$(( clients / processes ))
        (( i <= clients % processes )) && my_clients=$(( my_clients + 1 ))

        local options=(
            -t "$seconds"                   # run for this many seconds ...
            -n "$MAX_REQUESTS_PER_RUN"      # ... or until this many requests
            -c "$my_clients"                # simultaneous clients
            -r                              # keep going after socket errors
            -s 30                           # per-request timeout, seconds
            -e "$AB_DIR/$name-ab$i-percentiles.csv"   # save the percentile table
        )
        [[ $keepalive == yes ]] && options+=(-k)
        [[ -n $extra_header ]]  && options+=(-H "$extra_header")

        ab "${options[@]}" "$url" > "$AB_DIR/$name-ab$i.txt" 2>&1 &
        pids+=($!)
    done

    # ab returns non-zero on some socket errors; we still read its report.
    wait "${pids[@]}" || true
}

# Read one ab output file. Prints 12 numbers on one line:
#   complete non_2xx keepalive_answers requests_per_s kbytes_per_s mean_ms
#   max_ms connect_errors receive_errors length_errors exceptions document_bytes
# or the word "none" when ab produced no result.
ab_parse_file() {
    awk '
        /^Complete requests:/                 { complete  = $3 }
        /^Non-2xx responses:/                 { non2xx    = $3 }
        /^Keep-Alive requests:/               { keepalive = $3 }
        /^Requests per second:/               { rps       = $4 }
        /^Transfer rate:/                     { kbps      = $3 }
        /^Time per request:/ && mean == ""    { mean      = $4 }   # the first one is per client
        /^Document Length:/                   { doc       = $3 }
        /^ *100%/                             { max       = $2 }
        /^Failed requests:/ {
            # ab prints the causes on the next line (only when failures > 0):
            #    (Connect: 0, Receive: 0, Length: 23, Exceptions: 0)
            getline
            if ($0 ~ /^ *\(Connect:/) {
                gsub(/[^0-9 ]/, "")
                connect = $1; receive = $2; lengtherr = $3; exceptions = $4
            }
        }
        END {
            if (rps == "" || complete == "") { print "none"; exit }
            print complete, non2xx + 0, keepalive + 0, rps, kbps + 0, mean + 0,
                  max + 0, connect + 0, receive + 0, lengtherr + 0, exceptions + 0, doc + 0
        }' "$1"
}

# Merge the percentile tables of all ab processes into one table.
# Each row of a process's table stands for 1% of that process's requests,
# so it is weighted by the number of requests that process completed.
merge_percentiles() {
    local name="$1" pool="$RAW_DIR/.percentile-pool"
    local i file complete
    : > "$pool"
    for (( i = 1; i <= AB_PROCESSES_USED; i++ )); do
        file="$AB_DIR/$name-ab$i-percentiles.csv"
        complete="$(awk '/^Complete requests:/ {print $3}' "$AB_DIR/$name-ab$i.txt")"
        [[ -r $file ]] || continue
        # Output: "<time in ms> <weight>"
        awk -F, -v weight="${complete:-0}" 'NR > 1 { print $2, weight / 100 }' "$file" >> "$pool"
    done

    # Sort by time, then walk up until the running weight reaches each percentile.
    sort -g "$pool" | awk '
        { time[NR] = $1; weight[NR] = $2; total += $2 }
        END {
            print "Percentage served,Time in ms"
            row = 1; running = weight[1]
            for (p = 0; p < 100; p++) {
                while (running < total * p / 100 && row < NR) {
                    row++; running += weight[row]
                }
                printf "%d,%.3f\n", p, time[row]
            }
        }' > "$RAW_DIR/$name-percentiles.csv"
    rm -f "$pool"
}

# Read the output of all ab processes of test <name> into the AB_* variables.
ab_read_results() {
    local name="$1" keepalive="$2"
    local file line
    local complete non2xx keepalive_answers rps kbps mean max
    local connect receive length_errors exceptions doc_bytes

    AB_PROCESSES_USED=0
    AB_COMPLETE=0 AB_NON_2XX=0 AB_SERVER_CLOSES=0 AB_RPS=0 AB_TRANSFER_KBPS=0
    AB_MAX_MS=0 AB_DOC_BYTES=0
    local total_connect=0 total_receive=0 total_length=0 total_exceptions=0
    local weighted_mean_sum=0

    for file in "$AB_DIR/$name"-ab*.txt; do
        line="$(ab_parse_file "$file")"
        if [[ $line == none ]]; then
            echo "no result"
            die "ApacheBench did not produce a result. See $file"
        fi
        read -r complete non2xx keepalive_answers rps kbps mean max \
                connect receive length_errors exceptions doc_bytes <<< "$line"

        AB_PROCESSES_USED=$(( AB_PROCESSES_USED + 1 ))
        AB_COMPLETE=$(( AB_COMPLETE + complete ))
        AB_NON_2XX=$(( AB_NON_2XX + non2xx ))
        AB_RPS="$(calc "$AB_RPS + $rps")"
        AB_TRANSFER_KBPS="$(calc "$AB_TRANSFER_KBPS + $kbps")"
        weighted_mean_sum="$(calc "$weighted_mean_sum + $mean * $complete")"
        is_at_least "$max" "$AB_MAX_MS" && AB_MAX_MS="$max"
        AB_DOC_BYTES="$doc_bytes"

        # Answers that came back without keep-alive = connections Nginx closed.
        local closes=0
        [[ $keepalive == yes ]] && closes=$(( complete - keepalive_answers ))
        AB_SERVER_CLOSES=$(( AB_SERVER_CLOSES + closes ))

        # Length errors caused by those closures are not real failures.
        local real_length_errors=$(( length_errors - closes ))
        (( real_length_errors < 0 )) && real_length_errors=0

        total_connect=$(( total_connect + connect ))
        total_receive=$(( total_receive + receive ))
        total_length=$(( total_length + real_length_errors ))
        total_exceptions=$(( total_exceptions + exceptions ))
    done

    AB_FAILED=$(( total_connect + total_receive + total_length + total_exceptions ))
    AB_FAILED_DETAIL="connect $total_connect, receive $total_receive, wrong length $total_length, exceptions $total_exceptions"

    # Average time per request, weighted by how many requests each process did.
    AB_MEAN_MS="$(awk -v sum="$weighted_mean_sum" -v n="$AB_COMPLETE" \
                  'BEGIN { printf "%.3f", (n > 0 ? sum / n : 0) }')"

    merge_percentiles "$name"
    local percentiles="$RAW_DIR/$name-percentiles.csv"
    AB_P50_MS="$(awk -F, '$1 == 50 {printf "%.2f", $2}' "$percentiles")"
    AB_P90_MS="$(awk -F, '$1 == 90 {printf "%.2f", $2}' "$percentiles")"
    AB_P95_MS="$(awk -F, '$1 == 95 {printf "%.2f", $2}' "$percentiles")"
    AB_P99_MS="$(awk -F, '$1 == 99 {printf "%.2f", $2}' "$percentiles")"
}

# =============================================================================
#  PART 3 - sampling Nginx's CPU, memory and connections while load runs
# =============================================================================

# Total CPU time (seconds) that all Nginx processes have used so far.
# With systemd the cgroup counter is exact, even for processes that exit.
# Without systemd we add up the counters of the nginx processes alive now.
nginx_cpu_seconds() {
    local cgroup cpu_stat
    if has_systemd; then
        cgroup="$(systemctl show --property ControlGroup --value nginx 2>/dev/null)"
        cpu_stat="/sys/fs/cgroup${cgroup}/cpu.stat"
        if [[ -n $cgroup && -r $cpu_stat ]]; then
            awk '/^usage_usec/ {printf "%.3f", $2 / 1000000}' "$cpu_stat"
            return
        fi
    fi

    local ticks_per_second pid total_ticks=0 ticks
    ticks_per_second="$(getconf CLK_TCK)"
    for pid in $(nginx_pids); do
        # Fields 14 and 15 of /proc/<pid>/stat = user and system CPU ticks.
        ticks="$(awk '{print $14 + $15}' "/proc/$pid/stat" 2>/dev/null)" || continue
        total_ticks=$(( total_ticks + ${ticks:-0} ))
    done
    calc "$total_ticks / $ticks_per_second"
}

# Memory used by all Nginx processes, in MB.
# PSS ("proportional set size") shares memory used by several processes fairly
# between them, so adding the PSS of all processes gives the true total.
nginx_memory_mb() {
    local pid total_kb=0 kb
    for pid in $(nginx_pids); do
        kb="$(awk '/^Pss:/ {print $2}' "/proc/$pid/smaps_rollup" 2>/dev/null)" || continue
        total_kb=$(( total_kb + ${kb:-0} ))
    done
    calc "$total_kb / 1024"
}

nginx_process_count() {
    nginx_pids | wc -l
}

# Nginx's status page (stub_status) looks like this:
#     Active connections: 291
#     server accepts handled requests
#      16630948 16630948 31070465
#     Reading: 6 Writing: 179 Waiting: 106
#
# Prints: active reading writing waiting   ("?" when the page does not answer)
nginx_connections() {
    curl --noproxy '*' --silent --max-time 1 "$STATUS_URL" 2>/dev/null |
        awk '/^Active connections:/ {active = $3}
             /^Reading:/            {reading = $2; writing = $4; waiting = $6}
             END {
                 if (active == "") print "? ? ? ?"
                 else              print active, reading, writing, waiting
             }'
}

# Prints: accepted handled requests   (totals since Nginx started)
nginx_counters() {
    curl --noproxy '*' --silent --max-time 2 "$STATUS_URL" 2>/dev/null |
        awk 'NR == 3 {print $1, $2, $3}'
}

# Runs in the background: writes one CSV line per second until killed.
sampler_loop() {
    local csv_file="$1"
    local start previous_time previous_cpu now cpu cpu_percent
    local active reading writing waiting

    # This loop is stopped with "kill", often in the middle of a command.
    # That is expected, so do not report it as an error.
    trap - ERR
    set +e

    echo "elapsed_s,processes,memory_mb,cpu_percent_of_all_cpus,active_connections,reading,writing,waiting" > "$csv_file"
    start="$(now_seconds)"
    previous_time="$start"
    previous_cpu="$(nginx_cpu_seconds)"

    while true; do
        sleep "$SAMPLE_INTERVAL_SECONDS"
        now="$(now_seconds)"
        cpu="$(nginx_cpu_seconds)"
        cpu_percent="$(calc "($cpu - $previous_cpu) / ($now - $previous_time) / $CPU_COUNT * 100")"
        read -r active reading writing waiting < <(nginx_connections)

        printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \
            "$(calc "$now - $start")" "$(nginx_process_count)" "$(nginx_memory_mb)" \
            "$cpu_percent" "$active" "$reading" "$writing" "$waiting" >> "$csv_file"

        previous_time="$now"
        previous_cpu="$cpu"
    done
}

# -----------------------------------------------------------------------------
#  run_sampled_ab <name> <url> <clients> <keepalive: yes|no>
#
#  One load run while CPU, memory and connections are sampled every second
#  into raw/<name>-samples.csv.
#
#  Sets: LOAD_CPU_AVG_PCT  LOAD_CPU_PEAK_PCT  LOAD_CPU_SECONDS
#        LOAD_MEM_IDLE_MB  LOAD_MEM_PEAK_MB   LOAD_PROCS_IDLE  LOAD_PROCS_PEAK
#        LOAD_ACTIVE_PEAK  LOAD_WAITING_PEAK  LOAD_DROPPED
#        LOAD_REQUESTS     LOAD_RPS           (and all AB_* variables)
# -----------------------------------------------------------------------------
run_sampled_ab() {
    local name="$1" url="$2" clients="$3" keepalive="$4"
    local samples="$RAW_DIR/$name-samples.csv"

    # Values while Nginx is idle, before the load starts.
    LOAD_MEM_IDLE_MB="$(nginx_memory_mb)"
    LOAD_PROCS_IDLE="$(nginx_process_count)"

    local accepted_before handled_before accepted_after handled_after unused
    read -r accepted_before handled_before unused < <(nginx_counters)

    sampler_loop "$samples" &
    local sampler_pid=$!

    local cpu_before time_before cpu_after time_after
    cpu_before="$(nginx_cpu_seconds)"
    time_before="$(now_seconds)"

    run_ab "$name" "$url" "$clients" "$keepalive"

    cpu_after="$(nginx_cpu_seconds)"
    time_after="$(now_seconds)"
    kill "$sampler_pid" 2>/dev/null || true
    wait "$sampler_pid" 2>/dev/null || true

    read -r accepted_after handled_after unused < <(nginx_counters)

    LOAD_REQUESTS="$AB_COMPLETE"
    LOAD_RPS="$AB_RPS"
    LOAD_CPU_SECONDS="$(calc "$cpu_after - $cpu_before")"
    LOAD_CPU_AVG_PCT="$(calc "$LOAD_CPU_SECONDS / ($time_after - $time_before) / $CPU_COUNT * 100")"

    # Connections accepted but not handled = dropped, because a worker had
    # reached worker_connections (or the open-files limit).
    LOAD_DROPPED=$(( (${accepted_after:-0} - ${accepted_before:-0}) - (${handled_after:-0} - ${handled_before:-0}) ))

    # Column numbers: 2=processes 3=memory_mb 4=cpu% 5=active 8=waiting
    LOAD_CPU_PEAK_PCT="$(awk -F, 'NR > 1 && $4 > m {m = $4} END {printf "%.1f", m}' "$samples")"
    LOAD_MEM_PEAK_MB="$(awk  -F, -v m="$LOAD_MEM_IDLE_MB" 'NR > 1 && $3 > m {m = $3} END {printf "%.1f", m}' "$samples")"
    LOAD_PROCS_PEAK="$(awk   -F, -v m="$LOAD_PROCS_IDLE"  'NR > 1 && $2 > m {m = $2} END {print m}' "$samples")"
    LOAD_ACTIVE_PEAK="$(awk  -F, 'NR > 1 && $5 != "?" && $5 > m {m = $5} END {print m + 0}' "$samples")"
    LOAD_WAITING_PEAK="$(awk -F, 'NR > 1 && $8 != "?" && $8 > m {m = $8} END {print m + 0}' "$samples")"
}

# The cpu and memory tests share one sampled run (1 KB page, CONCURRENCY
# clients, keep-alive), so when both are chosen the load is generated only once.
RESOURCE_LOAD_DONE=no

run_resource_load() {
    [[ $RESOURCE_LOAD_DONE == yes ]] && return
    run_sampled_ab "resource-load" "$SMALL_PAGE_URL" "$CONCURRENCY" yes
    RESOURCE_LOAD_DONE=yes
}


# =============================================================================
#  PART 4 - the tests, one function per metric
# =============================================================================

# -----------------------------------------------------------------------------
test_startup() {
    section_title "Startup time  ($STARTUP_ROUNDS restarts)"
    local round start_time end_time elapsed_ms times=()

    for (( round = 1; round <= STARTUP_ROUNDS; round++ )); do
        nginx_stop
        start_time="$(now_seconds)"
        nginx_start
        if ! wait_until_serving 60; then
            die "Nginx did not come back after restart. See $NGINX_ERROR_LOG"
        fi
        end_time="$(now_seconds)"
        elapsed_ms="$(calc "($end_time - $start_time) * 1000")"
        times+=("$elapsed_ms")
        printf '    restart %d: %s ms\n' "$round" "$elapsed_ms"
    done

    local average fastest slowest
    average="$(printf '%s\n' "${times[@]}" | awk '{s += $1} END {printf "%.1f", s / NR}')"
    fastest="$(printf '%s\n' "${times[@]}" | sort -n | head -1)"
    slowest="$(printf '%s\n' "${times[@]}" | sort -n | tail -1)"

    record_result "Startup time" "$average ms average" "<= $TARGET_STARTUP_MAX_MS ms" \
        "$(verdict_at_most "$average" "$TARGET_STARTUP_MAX_MS")"

    add_details <<EOF
## Startup time

Nginx was stopped and started $STARTUP_ROUNDS times. Each time was measured
from the start command until the first test page was served.

| Round | Time (ms) |
|------:|----------:|
$(for i in "${!times[@]}"; do printf '| %5d | %9s |\n' $(( i + 1 )) "${times[$i]}"; done)

Average **$average ms**, fastest $fastest ms, slowest $slowest ms.

EOF
}

# -----------------------------------------------------------------------------
test_throughput() {
    section_title "Throughput  (1 KB page, $CONCURRENCY clients, keep-alive)"
    run_ab "throughput" "$SMALL_PAGE_URL" "$CONCURRENCY" yes

    record_result "Throughput" "$(thousands "$AB_RPS") requests/s" \
        ">= $TARGET_THROUGHPUT_MIN_RPS requests/s" \
        "$(verdict_at_least "$AB_RPS" "$TARGET_THROUGHPUT_MIN_RPS")"

    add_details <<EOF
## Throughput

$CONCURRENCY clients requested a 1 KB page as fast as possible for $DURATION seconds.

| Item | Value |
|---|---:|
| Requests per second | **$(thousands "$AB_RPS")** |
| Requests completed | $(thousands "$AB_COMPLETE") |
| Failed requests | $AB_FAILED |
| Keep-alive connections closed by Nginx | $AB_SERVER_CLOSES |
| Average time per request | $AB_MEAN_MS ms |
| Load generator processes (ab) | $AB_PROCESSES_USED |

Raw output of each ab process: \`raw/ab-processes/throughput-ab*.txt\`

EOF
}

# -----------------------------------------------------------------------------
test_latency() {
    section_title "Latency  (1 KB page, $CONCURRENCY clients, keep-alive)"
    run_ab "latency" "$SMALL_PAGE_URL" "$CONCURRENCY" yes

    local verdict=PASS
    is_at_most "$AB_P95_MS" "$TARGET_LATENCY_P95_MAX_MS" || verdict=FAIL
    is_at_most "$AB_P99_MS" "$TARGET_LATENCY_P99_MAX_MS" || verdict=FAIL

    record_result "Latency (p95 / p99)" "$AB_P95_MS ms / $AB_P99_MS ms" \
        "<= $TARGET_LATENCY_P95_MAX_MS ms / <= $TARGET_LATENCY_P99_MAX_MS ms" "$verdict"

    add_details <<EOF
## Latency (response time)

How long one request took, from sending it to receiving the full answer,
with $CONCURRENCY clients active. "p95 = 3 ms" means 95 of every 100 requests
finished within 3 ms.

| Statistic | Time (ms) |
|---|---:|
| Average | $AB_MEAN_MS |
| p50 (median) | $AB_P50_MS |
| p90 | $AB_P90_MS |
| p95 | **$AB_P95_MS** |
| p99 | **$AB_P99_MS** |
| Slowest request | $AB_MAX_MS |

Every percentile from 0 to 99: \`raw/latency-percentiles.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_concurrency() {
    section_title "Concurrency scaling  (1 KB page, clients: $CONCURRENCY_LEVELS)"
    local clients table="" peak_rps=0 peak_clients=0 last_rps=0 last_clients=0

    for clients in $CONCURRENCY_LEVELS; do
        run_ab "concurrency-${clients}-clients" "$SMALL_PAGE_URL" "$clients" yes
        table+="$(printf '| %7s | %10s | %8s | %8s | %8s | %6s |' \
            "$clients" "$(thousands "$AB_RPS")" "$AB_MEAN_MS" "$AB_P95_MS" \
            "$AB_P99_MS" "$AB_FAILED")"$'\n'

        if is_at_least "$AB_RPS" "$peak_rps"; then
            peak_rps="$AB_RPS"
            peak_clients="$clients"
        fi
        last_rps="$AB_RPS"
        last_clients="$clients"
    done

    local kept_pct
    kept_pct="$(calc "$last_rps / $peak_rps * 100")"

    record_result "Concurrency scaling" \
        "$kept_pct% of peak kept at $last_clients clients" \
        ">= $TARGET_SCALING_MIN_PCT% of peak" \
        "$(verdict_at_least "$kept_pct" "$TARGET_SCALING_MIN_PCT")"

    add_details <<EOF
## Concurrency scaling

The same test at several numbers of simultaneous clients. Healthy behaviour:
requests/second rises and then levels off; it should not collapse at the
highest level.

| Clients | Requests/s | Avg (ms) | p95 (ms) | p99 (ms) | Failed |
|--------:|-----------:|---------:|---------:|---------:|-------:|
${table}
Peak: **$(thousands "$peak_rps") requests/s at $peak_clients clients**.
At $last_clients clients Nginx still delivered **$kept_pct%** of that peak.

EOF
}

# -----------------------------------------------------------------------------
test_keepalive() {
    section_title "Keep-alive effect  (1 KB page, $CONCURRENCY clients)"

    run_ab "keepalive-on" "$SMALL_PAGE_URL" "$CONCURRENCY" yes
    local on_rps="$AB_RPS" on_mean="$AB_MEAN_MS" on_p95="$AB_P95_MS"

    run_ab "keepalive-off" "$SMALL_PAGE_URL" "$CONCURRENCY" no
    local off_rps="$AB_RPS" off_mean="$AB_MEAN_MS" off_p95="$AB_P95_MS"

    local gain
    gain="$(calc "$on_rps / $off_rps")"

    record_result "Keep-alive gain" "${gain}x more requests/s with keep-alive" \
        "information only" INFO

    add_details <<EOF
## Keep-alive effect

With keep-alive, a client sends many requests over one TCP connection.
Without it, every request opens and closes a new connection.

| Mode | Requests/s | Avg (ms) | p95 (ms) |
|---|---:|---:|---:|
| Keep-alive ON  | $(thousands "$on_rps")  | $on_mean  | $on_p95  |
| Keep-alive OFF | $(thousands "$off_rps") | $off_mean | $off_p95 |

Keep-alive served **${gain}x** as many requests per second.

EOF
}

# -----------------------------------------------------------------------------
test_transfer() {
    section_title "Transfer rate  (1 MB file, $CONCURRENCY clients, keep-alive)"
    run_ab "transfer" "$LARGE_FILE_URL" "$CONCURRENCY" yes

    local mb_per_second
    mb_per_second="$(calc "$AB_TRANSFER_KBPS / 1024")"

    record_result "Transfer rate" "$(thousands "$mb_per_second") MB/s" \
        ">= $TARGET_TRANSFER_MIN_MBPS MB/s" \
        "$(verdict_at_least "$mb_per_second" "$TARGET_TRANSFER_MIN_MBPS")"

    add_details <<EOF
## Transfer rate (bandwidth)

$CONCURRENCY clients downloaded a 1 MB file repeatedly for $DURATION seconds.

| Item | Value |
|---|---:|
| Data sent per second | **$mb_per_second MB/s** ($(calc "$mb_per_second * 8") Mbit/s) |
| Files downloaded per second | $(thousands "$AB_RPS") |
| Files downloaded in total | $AB_COMPLETE |
| Average time per download | $AB_MEAN_MS ms |

Over localhost the network is not a limit, so this shows how fast Nginx
itself can push data. Over a real network, the network card is usually the limit.

EOF
}

# -----------------------------------------------------------------------------
test_compression() {
    section_title "Compression (gzip level $GZIP_LEVEL)  (100 KB HTML page, $CONCURRENCY clients, keep-alive)"

    run_ab "compression-off" "$TEXT_PAGE_URL" "$CONCURRENCY" yes
    local plain_rps="$AB_RPS" plain_bytes="$AB_DOC_BYTES" plain_p95="$AB_P95_MS"
    local plain_mbps
    plain_mbps="$(calc "$AB_TRANSFER_KBPS / 1024")"

    run_ab "compression-gzip" "$TEXT_PAGE_URL" "$CONCURRENCY" yes "$DURATION" "Accept-Encoding: gzip"
    local gzip_rps="$AB_RPS" gzip_bytes="$AB_DOC_BYTES" gzip_p95="$AB_P95_MS"
    local gzip_mbps
    gzip_mbps="$(calc "$AB_TRANSFER_KBPS / 1024")"

    if (( gzip_bytes >= plain_bytes )); then
        record_result "Compression (gzip)" "answers were NOT compressed" \
            "gzip active" FAIL
        add_details <<EOF
## Compression (gzip)

The page was sent uncompressed ($gzip_bytes bytes) even though the client asked
for gzip. Check the "gzip" lines in $NGINX_CONF (./start-nginx.sh writes them).

EOF
        return
    fi

    local ratio speed_pct
    ratio="$(calc "$plain_bytes / $gzip_bytes")"
    speed_pct="$(calc "$gzip_rps / $plain_rps * 100")"

    record_result "Compression (gzip)" "${ratio}x smaller, $speed_pct% of uncompressed req/s" \
        "information only" INFO

    add_details <<EOF
## Compression (gzip)

$CONCURRENCY clients requested the same 100 KB HTML page, first without and then
with "Accept-Encoding: gzip". Compression makes answers smaller (less network
traffic, faster for users on slow links) but costs Nginx CPU time.

| Item | Uncompressed | gzip level $GZIP_LEVEL |
|---|---:|---:|
| Size of one answer | $plain_bytes bytes | **$gzip_bytes bytes** |
| Requests per second | $(thousands "$plain_rps") | **$(thousands "$gzip_rps")** |
| p95 response time | $plain_p95 ms | $gzip_p95 ms |
| Data actually sent | $plain_mbps MB/s | $gzip_mbps MB/s |

Answers were **${ratio}x smaller**. With compression Nginx served **$speed_pct%**
of the uncompressed request rate: the difference is the CPU cost of gzip.

EOF
}

# -----------------------------------------------------------------------------
test_proxy() {
    section_title "Reverse proxy  (1 KB page, $CONCURRENCY clients, keep-alive)"

    if ! curl --noproxy '*' -sf --max-time 5 "$PROXY_HEALTH_URL" >/dev/null; then
        record_result "Reverse proxy" "proxy does not work" \
            ">= $TARGET_PROXY_MIN_RPS requests/s" FAIL
        add_details <<EOF
## Reverse proxy

$PROXY_HEALTH_URL did not answer. The usual cause on RHEL is SELinux:
the boolean \`httpd_can_network_relay\` must be on (./start-nginx.sh switches it
on until the next reboot). Check \`getsebool httpd_can_network_relay\`,
\`ausearch -m AVC -ts recent\` and $NGINX_ERROR_LOG.

EOF
        return
    fi

    run_ab "proxy-direct" "$SMALL_PAGE_URL" "$CONCURRENCY" yes
    local direct_rps="$AB_RPS" direct_p50="$AB_P50_MS" direct_p95="$AB_P95_MS"

    run_ab "proxy-through-nginx" "$PROXY_PAGE_URL" "$CONCURRENCY" yes
    local proxy_rps="$AB_RPS" proxy_p50="$AB_P50_MS" proxy_p95="$AB_P95_MS"
    local proxy_bad=$(( AB_FAILED + AB_NON_2XX ))

    local kept_pct added_ms
    kept_pct="$(calc "$proxy_rps / $direct_rps * 100")"
    added_ms="$(awk -v a="$proxy_p50" -v b="$direct_p50" 'BEGIN { printf "%.2f", a - b }')"

    local verdict
    verdict="$(verdict_at_least "$proxy_rps" "$TARGET_PROXY_MIN_RPS")"
    (( proxy_bad > 0 )) && verdict=FAIL

    record_result "Reverse proxy" "$(thousands "$proxy_rps") requests/s ($kept_pct% of direct)" \
        ">= $TARGET_PROXY_MIN_RPS requests/s" "$verdict"

    add_details <<EOF
## Reverse proxy

Nginx is often placed in front of an application server and forwards
("proxies") each request to it. Here the same 1 KB page was requested
directly, and then through Nginx's proxy to a backend on 127.0.0.1:$BACKEND_PORT
(with $UPSTREAM_KEEPALIVE reusable backend connections per worker).

| Item | Direct | Through the proxy |
|---|---:|---:|
| Requests per second | $(thousands "$direct_rps") | **$(thousands "$proxy_rps")** |
| p50 response time | $direct_p50 ms | $proxy_p50 ms |
| p95 response time | $direct_p95 ms | $proxy_p95 ms |
| Failed or non-2xx answers | - | $proxy_bad |

Through the proxy Nginx delivered **$kept_pct%** of the direct request rate,
and each request took about **$added_ms ms** longer (median). A proxied request
is handled twice (by the front end and by the backend), so it costs Nginx about
twice the work. While Nginx has spare CPU the two rates can be close; the extra
time per request is then the clearer number. With a real application the
backend is usually much slower than Nginx.
$( (( proxy_bad > 0 )) && echo "
**$proxy_bad requests failed.** 502 Bad Gateway answers usually mean the
backend cannot be reached; check $NGINX_ERROR_LOG.")

EOF
}

# -----------------------------------------------------------------------------
#  One configuration reload. Sets RELOAD_MS to the time until every old
#  worker has exited and the new workers are running, or "timeout".
measure_one_reload() {
    local old_workers expected_count current_workers still_old start_time
    old_workers="$(nginx_worker_pids | sort)"
    expected_count="$(count_words $old_workers)"

    start_time="$(now_seconds)"
    nginx_reload

    local deadline=$(( SECONDS + 30 ))
    while (( SECONDS < deadline )); do
        current_workers="$(nginx_worker_pids | sort)"
        still_old="$(comm -12 <(printf '%s\n' "$old_workers") <(printf '%s\n' "$current_workers"))"
        if [[ -z $still_old ]] && (( $(count_words $current_workers) >= expected_count )); then
            RELOAD_MS="$(calc "($(now_seconds) - $start_time) * 1000")"
            return
        fi
        sleep 0.01
    done
    RELOAD_MS="timeout"
}

test_reload() {
    section_title "Reload under load  ($RELOAD_COUNT reloads, $CONCURRENCY clients, no keep-alive)"
    local name="reload-under-load"

    # Start the load in the background, then reload while it runs.
    printf '    %-34s %4s clients, keep-alive no , %ss (running in the background)\n' \
        "$name" "$CONCURRENCY" "$DURATION"
    ab_launch "$name" "$SMALL_PAGE_URL" "$CONCURRENCY" no &
    local ab_pid=$!

    local gap round times=() timeouts=0
    gap="$(calc "$DURATION / ($RELOAD_COUNT + 1)")"

    for (( round = 1; round <= RELOAD_COUNT; round++ )); do
        sleep "$gap"
        measure_one_reload
        times+=("$RELOAD_MS")
        [[ $RELOAD_MS == timeout ]] && timeouts=$(( timeouts + 1 ))
        printf '    reload %d: %s ms\n' "$round" "$RELOAD_MS"
    done

    wait "$ab_pid" || true
    ab_read_results "$name" no

    local bad_requests=$(( AB_FAILED + AB_NON_2XX ))
    local average
    average="$(printf '%s\n' "${times[@]}" | awk '$1 != "timeout" {s += $1; n++}
                                                   END {printf "%.1f", (n ? s / n : 0)}')"

    local verdict
    verdict="$(verdict_at_most "$bad_requests" "$TARGET_RELOAD_MAX_FAILED")"
    (( timeouts > 0 )) && verdict=FAIL

    record_result "Reload under load" "$bad_requests failed of $(thousands "$AB_COMPLETE"), reload $average ms" \
        "<= $TARGET_RELOAD_MAX_FAILED failed" "$verdict"

    add_details <<EOF
## Configuration reload under load

A reload ("systemctl reload nginx") makes Nginx read its configuration again
without stopping: new workers start, and the old workers finish their current
requests before they exit. It should never break a request. Nginx was reloaded
$RELOAD_COUNT times while $CONCURRENCY clients sent requests for $DURATION seconds.

| Reload | Time until only new workers run (ms) |
|------:|------------------------------------:|
$(for i in "${!times[@]}"; do printf '| %5d | %35s |\n' $(( i + 1 )) "${times[$i]}"; done)

| Item | Value |
|---|---:|
| Requests completed during the test | $(thousands "$AB_COMPLETE") |
| Failed requests | $AB_FAILED |
| &nbsp;&nbsp;broken down as | $AB_FAILED_DETAIL |
| Non-2xx answers | $AB_NON_2XX |
| **Requests broken by the reloads** | **$bad_requests** |
| Average reload time | $average ms |
$( (( timeouts > 0 )) && echo "
**$timeouts reload(s) did not finish within 30 s** (old workers kept running).")

EOF
}

# -----------------------------------------------------------------------------
test_errors() {
    section_title "Error rate under overload  ($ERROR_TEST_CONCURRENCY clients, no keep-alive)"

    local log_lines_before=0
    [[ -r $NGINX_ERROR_LOG ]] && log_lines_before="$(wc -l < "$NGINX_ERROR_LOG")"

    run_ab "errors-overload" "$SMALL_PAGE_URL" "$ERROR_TEST_CONCURRENCY" no

    local bad_requests error_rate new_log_lines="" new_log_count=0
    bad_requests=$(( AB_FAILED + AB_NON_2XX ))
    error_rate="$(awk -v bad="$bad_requests" -v all="$AB_COMPLETE" \
        'BEGIN { printf "%.3f", (all > 0 ? bad / all * 100 : 100) }')"

    if [[ -r $NGINX_ERROR_LOG ]]; then
        new_log_lines="$(tail -n +"$(( log_lines_before + 1 ))" "$NGINX_ERROR_LOG" |
                         grep -E '\[(error|crit|alert|emerg)\]' || true)"
        [[ -n $new_log_lines ]] && new_log_count="$(printf '%s\n' "$new_log_lines" | wc -l)"
        printf '%s\n' "$new_log_lines" > "$RAW_DIR/errors-new-error.log-lines.txt"
    fi

    record_result "Error rate" "$error_rate% ($bad_requests of $(thousands "$AB_COMPLETE"))" \
        "<= $TARGET_ERROR_RATE_MAX_PCT%" \
        "$(verdict_at_most "$error_rate" "$TARGET_ERROR_RATE_MAX_PCT")"

    add_details <<EOF
## Error rate under overload

$ERROR_TEST_CONCURRENCY clients, each opening a new connection for every request,
for $DURATION seconds. This pushes Nginx harder than normal traffic.

| Item | Value |
|---|---:|
| Requests completed | $(thousands "$AB_COMPLETE") |
| Failed requests (network / wrong length) | $AB_FAILED |
| &nbsp;&nbsp;broken down as | $AB_FAILED_DETAIL |
| Non-2xx answers (HTTP errors such as 503) | $AB_NON_2XX |
| **Error rate** | **$error_rate%** |
| New serious lines in $NGINX_ERROR_LOG | $new_log_count |

$(if [[ -n $new_log_lines ]]; then
    echo "First lines from the error log (all in \`raw/errors-new-error.log-lines.txt\`):"
    echo
    echo '```'
    printf '%s\n' "$new_log_lines" | head -10
    echo '```'
  fi)

EOF
}

# -----------------------------------------------------------------------------
test_connections() {
    section_title "Connections  ($CONNECTIONS_TEST_CLIENTS clients, keep-alive)"
    run_sampled_ab "connections-load" "$SMALL_PAGE_URL" "$CONNECTIONS_TEST_CLIENTS" yes

    local worker_count capacity used_pct
    worker_count="$(nginx_worker_pids | wc -l)"
    capacity=$(( worker_count * WORKER_CONNECTIONS ))
    used_pct="$(calc "$LOAD_ACTIVE_PEAK / $capacity * 100")"

    local verdict
    verdict="$(verdict_at_most "$used_pct" "$TARGET_CONNECTIONS_MAX_PCT")"
    (( LOAD_DROPPED > 0 || AB_FAILED > 0 )) && verdict=FAIL

    record_result "Connections" "$LOAD_ACTIVE_PEAK of $(thousands "$capacity") open ($used_pct%), $LOAD_DROPPED dropped" \
        "<= $TARGET_CONNECTIONS_MAX_PCT%, 0 dropped" "$verdict"

    add_details <<EOF
## Connections

$CONNECTIONS_TEST_CLIENTS clients kept their connections open (keep-alive) for
$DURATION seconds. Nginx's status page (/nginx-status) was read once per second.
Nginx can hold at most $worker_count workers x $WORKER_CONNECTIONS
worker_connections = **$capacity** connections.

| Item | Value |
|---|---:|
| Most open connections at one time | **$LOAD_ACTIVE_PEAK** |
| &nbsp;&nbsp;of which idle, waiting for the next request | $LOAD_WAITING_PEAK |
| Maximum possible connections | $capacity |
| Peak usage | **$used_pct%** |
| Connections dropped (accepted but not handled) | $LOAD_DROPPED |
| Failed requests | $AB_FAILED |
| Requests per second during the test | $(thousands "$AB_RPS") |

Dropped connections mean a worker ran out of worker_connections or of open
files. Per-second samples: \`raw/connections-load-samples.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_cpu() {
    section_title "CPU usage under load  ($CONCURRENCY clients, $CPU_COUNT CPUs)"
    run_resource_load

    local ms_per_1000
    ms_per_1000="$(calc "$LOAD_CPU_SECONDS * 1000 / $LOAD_REQUESTS * 1000")"

    record_result "CPU usage" "$LOAD_CPU_AVG_PCT% average of all CPUs" \
        "<= $TARGET_CPU_MAX_PCT%" \
        "$(verdict_at_most "$LOAD_CPU_AVG_PCT" "$TARGET_CPU_MAX_PCT")"

    add_details <<EOF
## CPU usage

CPU used by all Nginx processes while serving $(thousands "$LOAD_RPS") requests/s.
100% means every one of the $CPU_COUNT CPUs was fully busy with Nginx.
(The load generator "ab" runs on the same machine and uses CPU too.)

| Item | Value |
|---|---:|
| Average CPU, % of all CPUs | **$LOAD_CPU_AVG_PCT%** |
| Busiest second, % of all CPUs | $LOAD_CPU_PEAK_PCT% |
| Same average, in CPU cores | $(calc "$LOAD_CPU_AVG_PCT * $CPU_COUNT / 100") cores |
| CPU time per 1000 requests | $ms_per_1000 ms |

Per-second samples: \`raw/resource-load-samples.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_memory() {
    section_title "Memory usage  ($CONCURRENCY clients)"
    run_resource_load

    record_result "Memory usage" "$LOAD_MEM_PEAK_MB MB peak (idle $LOAD_MEM_IDLE_MB MB)" \
        "<= $TARGET_MEMORY_MAX_MB MB" \
        "$(verdict_at_most "$LOAD_MEM_PEAK_MB" "$TARGET_MEMORY_MAX_MB")"

    add_details <<EOF
## Memory usage

Memory of all Nginx processes together (PSS: memory shared between
processes is counted once, so this is the real total).

| Item | Idle | Under load (peak) |
|---|---:|---:|
| Memory (MB) | $LOAD_MEM_IDLE_MB | **$LOAD_MEM_PEAK_MB** |
| Nginx processes (1 master + workers) | $LOAD_PROCS_IDLE | $LOAD_PROCS_PEAK |
| Memory per process (MB) | $(calc "$LOAD_MEM_IDLE_MB / $LOAD_PROCS_IDLE") | $(calc "$LOAD_MEM_PEAK_MB / $LOAD_PROCS_PEAK") |

Per-second samples: \`raw/resource-load-samples.csv\`

EOF
}


# =============================================================================
#  PART 5 - preparing, and writing the final report
# =============================================================================

check_ready_to_test() {
    require_root "$@"
    command -v ab   >/dev/null || die "'ab' not found. Install httpd-tools (./start-nginx.sh does this)."
    command -v curl >/dev/null || die "'curl' not found."
    nginx_is_serving || die "Nginx is not serving $HEALTH_URL. Run ./start-nginx.sh first."
    curl --noproxy '*' -sf --max-time 2 "$STATUS_URL" | grep -q '^Active connections' ||
        die "Status page $STATUS_URL not available. Run ./start-nginx.sh first."

    # ab needs one open file per client; the default limit (1024) is too low
    # for the tests with 1000 clients.
    ulimit -n "$OPEN_FILES_LIMIT" 2>/dev/null ||
        warn "Could not raise the open-files limit; tests with many clients may fail."
}

prepare_result_folder() {
    RESULT_DIR="$KIT_DIR/results/$(date +%Y%m%d-%H%M%S)"
    RAW_DIR="$RESULT_DIR/raw"
    AB_DIR="$RAW_DIR/ab-processes"
    DETAILS_FILE="$RAW_DIR/.details.md"
    mkdir -p "$RAW_DIR" "$AB_DIR"
    : > "$DETAILS_FILE"
    ln -sfn "$(basename "$RESULT_DIR")" "$KIT_DIR/results/latest"
}

# A short unmeasured run, so the first real test does not start "cold".
warm_up() {
    log "Warm-up: 3 seconds of light load (not measured)"
    ab -k -t 3 -n 1000000 -c 10 "$SMALL_PAGE_URL" > "$RAW_DIR/warm-up.txt" 2>&1 || true
}

write_report() {
    local tests_run="$1" started="$2" finished="$3"
    local report="$RESULT_DIR/report.md"
    local nginx_version os_name selinux cpu_model memory_total worker_count

    nginx_version="$("$NGINX_BIN" -v 2>&1 | awk -F'/' '{print $2}')"
    os_name="$(. /etc/os-release && echo "$PRETTY_NAME")"
    selinux="$(getenforce 2>/dev/null || echo 'not available')"
    cpu_model="$(awk -F': ' '/^model name/ {print $2; exit}' /proc/cpuinfo)"
    memory_total="$(awk '/^MemTotal/ {printf "%.1f GB", $2 / 1048576}' /proc/meminfo)"
    worker_count="$(nginx_worker_pids | wc -l)"

    local overall="ALL PASSED"
    (( FAIL_COUNT > 0 )) && overall="$FAIL_COUNT TEST(S) FAILED"

    {
        echo "# Nginx Performance Test Report"
        echo
        echo "**Result: $overall**"
        echo
        echo "| | |"
        echo "|---|---|"
        echo "| Test started  | $started |"
        echo "| Test finished | $finished |"
        echo "| Host | $(hostname) |"
        echo "| Operating system | $os_name (kernel $(uname -r)) |"
        echo "| CPU | $CPU_COUNT x $cpu_model |"
        echo "| Memory | $memory_total |"
        echo "| Nginx | $nginx_version, $worker_count workers x $WORKER_CONNECTIONS connections |"
        echo "| SELinux | $selinux |"
        echo "| Target URL | $BASE_URL |"
        echo "| Tests run | $tests_run |"
        echo "| Load per run | $DURATION s, $CONCURRENCY clients (unless the test says otherwise) |"
        echo "| Load generator | ApacheBench (ab), up to $(ab_process_count 1000) processes side by side, on the same machine |"
        echo
        echo "## Summary"
        echo
        echo "PASS = target met, FAIL = target missed, INFO = measured only (no target)."
        echo "Targets are set in \`settings.conf\`. What each metric means: \`PERFORMANCE-METRICS.md\`."
        echo
        printf '| %-22s | %-50s | %-30s | %-7s |\n' "Test" "Measured" "Target" "Verdict"
        printf '|%s|%s|%s|%s|\n' "$(printf -- '-%.0s' {1..24})" "$(printf -- '-%.0s' {1..52})" \
                                 "$(printf -- '-%.0s' {1..32})" "$(printf -- '-%.0s' {1..9})"
        local row name measured target verdict
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            [[ $verdict == FAIL ]] && verdict="**FAIL**"
            printf '| %-22s | %-50s | %-30s | %-7s |\n' "$name" "$measured" "$target" "$verdict"
        done
        echo
        echo "# Details"
        echo
        cat "$DETAILS_FILE"
        echo "## Files in this folder"
        echo
        echo "- \`report.md\` - this report"
        echo "- \`summary.csv\` - the summary table, for spreadsheets"
        echo "- \`raw/\` - merged percentile tables and per-second resource samples"
        echo "- \`raw/ab-processes/\` - unmodified output of every ApacheBench process"
    } > "$report"

    # The same summary as CSV.
    {
        echo "test,measured,target,verdict"
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            printf '"%s","%s","%s","%s"\n' "$name" "$measured" "$target" "$verdict"
        done
    } > "$RESULT_DIR/summary.csv"

    rm -f "$DETAILS_FILE"
}

print_usage() {
    # Print the comment block at the top of this file.
    sed -n '3,/^set -euo pipefail/{/^#/p}' "$0" | sed 's/^# \{0,1\}//'
}


# =============================================================================
#  MAIN
# =============================================================================
main() {
    case "${1:-}" in
        -h|--help) print_usage; exit 0 ;;
        --list)    printf '%s\n' "${ALL_TESTS[@]}"; exit 0 ;;
    esac

    # Which tests to run: the names given, or all of them.
    local tests=("$@")
    if (( ${#tests[@]} == 0 )) || [[ ${tests[0]} == all ]]; then
        tests=("${ALL_TESTS[@]}")
    fi
    local test_name
    for test_name in "${tests[@]}"; do
        if [[ " ${ALL_TESTS[*]} " != *" $test_name "* ]]; then
            echo "Unknown test: '$test_name'. Valid tests: ${ALL_TESTS[*]}" >&2
            exit 2
        fi
    done

    check_ready_to_test
    prepare_result_folder

    # Always stop background jobs (sampler, ab) if the script is interrupted.
    trap 'kill $(jobs -p) 2>/dev/null || true' EXIT

    local started finished
    started="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    log "Testing $BASE_URL - tests: ${tests[*]}"
    log "Results folder: $RESULT_DIR"
    warm_up

    for test_name in "${tests[@]}"; do
        "test_$test_name"
    done

    finished="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    write_report "${tests[*]}" "$started" "$finished"

    echo
    sed -n '/^## Summary/,/^# Details/p' "$RESULT_DIR/report.md" | sed '$d'
    echo
    ok "Report saved: $RESULT_DIR/report.md"

    # Exit code: 0 when nothing failed, otherwise 1.
    if (( FAIL_COUNT > 0 )); then
        exit 1
    fi
}

main "$@"