#!/usr/bin/env bash
# =============================================================================
#  test-tomcat.sh - measure Tomcat performance, one metric at a time,
#                   and save an easy-to-read report
# =============================================================================
#
#  USAGE (as root, after ./start-tomcat.sh)
#      ./test-tomcat.sh                    run every test (about 4 minutes)
#      ./test-tomcat.sh dynamic            run one test
#      ./test-tomcat.sh latency errors     run several tests
#      ./test-tomcat.sh --list             show the test names
#
#      DURATION=30 CONCURRENCY=100 ./test-tomcat.sh dynamic
#                                          override settings.conf for one run
#
#  THE TESTS  (explained in detail in PERFORMANCE-METRICS.md)
#      startup      time from "start Tomcat" until the first page is served
#      static       requests/s for a static file (1 KB HTML page)
#      dynamic      requests/s for a dynamic page (JSP, Java code runs)
#      latency      response time per request (average and percentiles)
#      concurrency  how throughput and latency change as clients increase
#      keepalive    gain from reusing TCP connections
#      transfer     download speed of a large file (MB/s)
#      compression  gzip: how much smaller answers get, and what it costs
#      threads      worker thread pool: slow requests, pool full, queueing
#      sessions     how fast user sessions are created, memory per session
#      errors       failed requests while Tomcat is overloaded
#      connections  open connections compared with the configured maximum
#      gc           Java garbage collection: pause times and overhead
#      cpu          CPU used by Tomcat under load
#      memory       memory used by Tomcat (process and Java heap)
#
#  OUTPUT
#      results/<date>-<time>/report.md    the human-readable report
#      results/<date>-<time>/summary.csv  one line per test (for spreadsheets)
#      results/<date>-<time>/raw/         unmodified tool output
#      results/latest                     link to the newest result folder
#
#  EXIT CODE
#      0 = every test passed,  1 = at least one FAIL,  2 = could not run
#
#  Load is generated by ApacheBench ("ab", from the httpd-tools RPM).
#  All traffic stays on this machine; no internet access is needed.
# =============================================================================

set -euo pipefail
source "$(dirname -- "${BASH_SOURCE[0]}")/lib/common.sh"

# When the test cannot run at all, exit with code 2 (a FAIL result gives 1).
die() { printf '\033[1;31m[ERROR ]\033[0m %s\n' "$*" >&2; exit 2; }

ALL_TESTS=(startup static dynamic latency concurrency keepalive transfer compression
           threads sessions errors connections gc cpu memory)

SMALL_PAGE_URL="$BASE_URL/perf-test/small.html"     # 1 KB static page
DYNAMIC_PAGE_URL="$BASE_URL/perf-test/hello.jsp"    # small JSP page
TEXT_PAGE_URL="$BASE_URL/perf-test/text.html"       # about 40 KB of HTML
LARGE_FILE_URL="$BASE_URL/perf-test/large.bin"      # 1 MB file
SLOW_PAGE_URL="$BASE_URL/perf-test/slow.jsp?ms=$SLOW_REQUEST_MS"
SESSION_PAGE_URL="$BASE_URL/perf-test/session.jsp?bytes=$SESSION_DATA_BYTES"

SAMPLE_INTERVAL_SECONDS=1       # how often CPU / memory / threads are sampled
CPU_COUNT="$(nproc)"

# "ab" needs a request limit; it stops at DURATION or at this count,
# whichever comes first. 200,000 per second is far above what one host serves.
MAX_REQUESTS_PER_RUN=$(( DURATION * 200000 ))

# Filled in by the functions below.
RESULT_DIR=""
RAW_DIR=""
AB_DIR=""           # raw/ab-processes: the output of every single ab process
DETAILS_FILE=""
SUMMARY_ROWS=()     # "Test|Measured|Target|Verdict"
FAIL_COUNT=0


# =============================================================================
#  PART 1 - small helpers
# =============================================================================

# Floating-point math and comparisons (bash itself only knows whole numbers).
calc()          { awk "BEGIN { printf \"%.1f\", $* }"; }
calc2()         { awk "BEGIN { printf \"%.2f\", $* }"; }
is_at_most()    { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a <= b) }'; }
is_at_least()   { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a >= b) }'; }
now_seconds()   { date +%s.%N; }

# Round a number and add thousands separators: 357788.7 -> 357,789
thousands() {
    awk -v number="$1" 'BEGIN {
        digits = sprintf("%.0f", number); result = ""
        while (length(digits) > 3) {
            result = "," substr(digits, length(digits) - 2) result
            digits = substr(digits, 1, length(digits) - 3)
        }
        print digits result
    }'
}

# Turn a comparison into the words PASS / FAIL.
verdict_at_most()  { if is_at_most  "$1" "$2"; then echo PASS; else echo FAIL; fi; }
verdict_at_least() { if is_at_least "$1" "$2"; then echo PASS; else echo FAIL; fi; }

# Remember one line for the summary table at the top of the report.
#   record_result "Test name" "measured value" "target" PASS|FAIL|INFO
record_result() {
    local name="$1" measured="$2" target="$3" verdict="$4"
    SUMMARY_ROWS+=("$name|$measured|$target|$verdict")
    [[ $verdict == FAIL ]] && FAIL_COUNT=$(( FAIL_COUNT + 1 ))

    local colour='1;32'                       # green  = PASS
    [[ $verdict == FAIL ]] && colour='1;31'   # red    = FAIL
    [[ $verdict == INFO ]] && colour='1;36'   # cyan   = INFO
    printf "    => \033[${colour}m%-4s\033[0m  %s   (target: %s)\n" "$verdict" "$measured" "$target"
}

# Append Markdown text (read from stdin) to the "details" part of the report.
add_details() {
    cat >> "$DETAILS_FILE"
}

section_title() {
    echo
    printf '\033[1m==> %s\033[0m\n' "$*"
}


# =============================================================================
#  PART 2 - running ApacheBench and reading its output
# =============================================================================
#
#  run_ab <name> <url> <clients> <keepalive: yes|no> [seconds] [extra header]
#
#  Why several ab processes?
#  One ab process uses a single CPU core and cannot send much more than about
#  50,000 requests/s. So the clients are split over AB_PROCESSES ab processes
#  running side by side, and their results are added up. Example: 50 clients
#  with 4 processes = 13 + 13 + 12 + 12 clients.
#
#  Set AB_REQUEST_LIMIT to stop after that many requests in total (used by
#  the "sessions" test); otherwise each run lasts [seconds].
#
#  Saves, for each ab process i:
#      raw/ab-processes/<name>-ab<i>.txt              the full ab output
#      raw/ab-processes/<name>-ab<i>-percentiles.csv  its response time per percentile
#  and for the whole run:
#      raw/<name>-percentiles.csv            all processes merged, 0..99
#
#  Sets these variables for the caller (totals over all ab processes):
#      AB_COMPLETE  AB_FAILED  AB_FAILED_DETAIL  AB_NON_2XX  AB_SERVER_CLOSES
#      AB_RPS  AB_TRANSFER_KBPS  AB_MEAN_MS  AB_DOC_BYTES  AB_PROCESSES_USED
#      AB_P50_MS  AB_P90_MS  AB_P95_MS  AB_P99_MS  AB_MAX_MS
#
#  run_ab is made of two halves, ab_launch and ab_read_results, so that a
#  test can run the load in the background and measure something meanwhile.
#
#  A note on "failed requests" in keep-alive mode:
#  A server sometimes answers a request completely but then closes the
#  keep-alive connection (the client simply reconnects). ApacheBench wrongly
#  counts each such answer as a "Length" failure. We count those closures
#  separately in AB_SERVER_CLOSES, and AB_FAILED holds only real failures.
# -----------------------------------------------------------------------------
AB_REQUEST_LIMIT=""

run_ab() {
    local name="$1" url="$2" clients="$3" keepalive="$4" seconds="${5:-$DURATION}"
    if [[ -n $AB_REQUEST_LIMIT ]]; then
        printf '    %-34s %4s clients, keep-alive %-3s, %s requests ... ' \
            "$name" "$clients" "$keepalive" "$AB_REQUEST_LIMIT"
    else
        printf '    %-34s %4s clients, keep-alive %-3s, %ss ... ' \
            "$name" "$clients" "$keepalive" "$seconds"
    fi
    ab_launch "$@"
    ab_read_results "$name" "$keepalive"
    printf '%10.0f req/s\n' "$AB_RPS"
}

# How many ab processes to use for <clients> clients.
ab_process_count() {
    local clients="$1" processes="$AB_PROCESSES"
    if [[ $processes == auto ]]; then
        processes=$(( CPU_COUNT / 2 ))         # half of the CPUs ...
        (( processes > 8 )) && processes=8     # ... but at most 8
    fi
    (( processes < 1 )) && processes=1
    (( processes > clients )) && processes="$clients"   # at least 1 client each
    echo "$processes"
}

# Start the ab processes, split the clients between them, wait for all.
ab_launch() {
    local name="$1" url="$2" clients="$3" keepalive="$4" seconds="${5:-$DURATION}"
    local extra_header="${6:-}"

    local processes
    processes="$(ab_process_count "$clients")"
    rm -f "$AB_DIR/$name"-ab*.txt "$AB_DIR/$name"-ab*-percentiles.csv

    local i my_clients my_requests pids=()
    for (( i = 1; i <= processes; i++ )); do
        # Share the clients out evenly; the first ones get any remainder.
        my_clients=$(( clients / processes ))
        (( i <= clients % processes )) && my_clients=$(( my_clients + 1 ))

        # The same for the request limit, when one is set.
        my_requests="$MAX_REQUESTS_PER_RUN"
        if [[ -n $AB_REQUEST_LIMIT ]]; then
            my_requests=$(( AB_REQUEST_LIMIT / processes ))
            (( i <= AB_REQUEST_LIMIT % processes )) && my_requests=$(( my_requests + 1 ))
        fi

        local options=(
            -t "$seconds"                   # run for this many seconds ...
            -n "$my_requests"               # ... or until this many requests
            -c "$my_clients"                # simultaneous clients
            -r                              # keep going after socket errors
            -s 30                           # per-request timeout, seconds
            -e "$AB_DIR/$name-ab$i-percentiles.csv"   # save the percentile table
        )
        [[ $keepalive == yes ]] && options+=(-k)
        [[ -n $extra_header ]]  && options+=(-H "$extra_header")

        ab "${options[@]}" "$url" > "$AB_DIR/$name-ab$i.txt" 2>&1 &
        pids+=($!)
    done

    # ab returns non-zero on some socket errors; we still read its report.
    wait "${pids[@]}" || true
}

# Read one ab output file. Prints 12 numbers on one line:
#   complete non_2xx keepalive_answers requests_per_s kbytes_per_s mean_ms
#   max_ms connect_errors receive_errors length_errors exceptions document_bytes
# or the word "none" when ab produced no result.
ab_parse_file() {
    awk '
        /^Complete requests:/                 { complete  = $3 }
        /^Non-2xx responses:/                 { non2xx    = $3 }
        /^Keep-Alive requests:/               { keepalive = $3 }
        /^Requests per second:/               { rps       = $4 }
        /^Transfer rate:/                     { kbps      = $3 }
        /^Time per request:/ && mean == ""    { mean      = $4 }   # the first one is per client
        /^Document Length:/                   { doc       = $3 }
        /^ *100%/                             { max       = $2 }
        /^ *\(Connect:/ {
            # Below "Failed requests:" ab prints the causes (only when failures > 0):
            #    (Connect: 0, Receive: 0, Length: 23, Exceptions: 0)
            gsub(/[^0-9 ]/, "")
            connect = $1; receive = $2; lengtherr = $3; exceptions = $4
        }
        END {
            if (rps == "" || complete == "") { print "none"; exit }
            print complete, non2xx + 0, keepalive + 0, rps, kbps + 0, mean + 0,
                  max + 0, connect + 0, receive + 0, lengtherr + 0, exceptions + 0, doc + 0
        }' "$1"
}

# Merge the percentile tables of all ab processes into one table.
# Each row of a process's table stands for 1% of that process's requests,
# so it is weighted by the number of requests that process completed.
merge_percentiles() {
    local name="$1" pool="$RAW_DIR/.percentile-pool"
    local i file complete
    : > "$pool"
    for (( i = 1; i <= AB_PROCESSES_USED; i++ )); do
        file="$AB_DIR/$name-ab$i-percentiles.csv"
        complete="$(awk '/^Complete requests:/ {print $3}' "$AB_DIR/$name-ab$i.txt")"
        [[ -r $file ]] || continue
        # Output: "<time in ms> <weight>"
        awk -F, -v weight="${complete:-0}" 'NR > 1 { print $2, weight / 100 }' "$file" >> "$pool"
    done

    # Sort by time, then walk up until the running weight reaches each percentile.
    sort -g "$pool" | awk '
        { time[NR] = $1; weight[NR] = $2; total += $2 }
        END {
            print "Percentage served,Time in ms"
            row = 1; running = weight[1]
            for (p = 0; p < 100; p++) {
                while (running < total * p / 100 && row < NR) {
                    row++; running += weight[row]
                }
                printf "%d,%.3f\n", p, time[row]
            }
        }' > "$RAW_DIR/$name-percentiles.csv"
    rm -f "$pool"
}

# Read the output of all ab processes of test <name> into the AB_* variables.
ab_read_results() {
    local name="$1" keepalive="$2"
    local file line
    local complete non2xx keepalive_answers rps kbps mean max
    local connect receive length_errors exceptions doc_bytes

    AB_PROCESSES_USED=0
    AB_COMPLETE=0 AB_NON_2XX=0 AB_SERVER_CLOSES=0 AB_RPS=0 AB_TRANSFER_KBPS=0
    AB_MAX_MS=0 AB_DOC_BYTES=0
    local total_connect=0 total_receive=0 total_length=0 total_exceptions=0
    local weighted_mean_sum=0

    for file in "$AB_DIR/$name"-ab*.txt; do
        line="$(ab_parse_file "$file")"
        if [[ $line == none ]]; then
            echo "no result"
            die "ApacheBench did not produce a result. See $file"
        fi
        read -r complete non2xx keepalive_answers rps kbps mean max \
                connect receive length_errors exceptions doc_bytes <<< "$line"

        AB_PROCESSES_USED=$(( AB_PROCESSES_USED + 1 ))
        AB_COMPLETE=$(( AB_COMPLETE + complete ))
        AB_NON_2XX=$(( AB_NON_2XX + non2xx ))
        AB_RPS="$(calc "$AB_RPS + $rps")"
        AB_TRANSFER_KBPS="$(calc "$AB_TRANSFER_KBPS + $kbps")"
        weighted_mean_sum="$(calc "$weighted_mean_sum + $mean * $complete")"
        is_at_least "$max" "$AB_MAX_MS" && AB_MAX_MS="$max"
        AB_DOC_BYTES="$doc_bytes"

        # Answers that came back without keep-alive = connections Tomcat closed.
        local closes=0
        [[ $keepalive == yes ]] && closes=$(( complete - keepalive_answers ))
        AB_SERVER_CLOSES=$(( AB_SERVER_CLOSES + closes ))

        # Length errors caused by those closures are not real failures.
        local real_length_errors=$(( length_errors - closes ))
        (( real_length_errors < 0 )) && real_length_errors=0

        total_connect=$(( total_connect + connect ))
        total_receive=$(( total_receive + receive ))
        total_length=$(( total_length + real_length_errors ))
        total_exceptions=$(( total_exceptions + exceptions ))
    done

    AB_FAILED=$(( total_connect + total_receive + total_length + total_exceptions ))
    AB_FAILED_DETAIL="connect $total_connect, receive $total_receive, wrong length $total_length, exceptions $total_exceptions"

    # Average time per request, weighted by how many requests each process did.
    AB_MEAN_MS="$(awk -v sum="$weighted_mean_sum" -v n="$AB_COMPLETE" \
                  'BEGIN { printf "%.3f", (n > 0 ? sum / n : 0) }')"

    merge_percentiles "$name"
    local percentiles="$RAW_DIR/$name-percentiles.csv"
    AB_P50_MS="$(awk -F, '$1 == 50 {printf "%.2f", $2}' "$percentiles")"
    AB_P90_MS="$(awk -F, '$1 == 90 {printf "%.2f", $2}' "$percentiles")"
    AB_P95_MS="$(awk -F, '$1 == 95 {printf "%.2f", $2}' "$percentiles")"
    AB_P99_MS="$(awk -F, '$1 == 99 {printf "%.2f", $2}' "$percentiles")"
}


# =============================================================================
#  PART 3 - measuring Tomcat itself while load runs
# =============================================================================

# Total CPU time (seconds) that the Tomcat process has used so far.
# With systemd the cgroup counter of tomcat@perftest.service is used.
# Without systemd we read the counters of the Java process (all its threads).
tomcat_cpu_seconds() {
    local cgroup cpu_stat
    if has_systemd; then
        cgroup="$(systemctl show --property ControlGroup --value "$SERVICE_NAME" 2>/dev/null)"
        cpu_stat="/sys/fs/cgroup${cgroup}/cpu.stat"
        if [[ -n $cgroup && -r $cpu_stat ]]; then
            awk '/^usage_usec/ {printf "%.3f", $2 / 1000000}' "$cpu_stat"
            return
        fi
    fi

    local pid ticks
    pid="$(tomcat_pid)"
    # Fields 14 and 15 of /proc/<pid>/stat = user and system CPU ticks.
    ticks="$(awk '{print $14 + $15}' "/proc/$pid/stat" 2>/dev/null)" || ticks=0
    calc "${ticks:-0} / $(getconf CLK_TCK)"
}

# Memory of the Tomcat process, in MB. PSS ("proportional set size") counts
# memory shared with other processes (such as libraries) fairly.
# This includes the Java heap, Java's own code and all thread stacks.
tomcat_memory_mb() {
    local pid kb
    pid="$(tomcat_pid)"
    kb="$(awk '/^Pss:/ {print $2}' "/proc/$pid/smaps_rollup" 2>/dev/null)" || kb=0
    calc "${kb:-0} / 1024"
}

# GC pauses written to logs/gc.log between two moments of the Java VM's
# uptime (in seconds). A line looks like:
#   [12.345s][info][gc] GC(12) Pause Young (Normal) (G1 Evacuation Pause) 60M->12M(1024M) 2.345ms
# Prints: number_of_pauses  total_pause_ms  longest_pause_ms
gc_pauses_between() {
    local from_s="$1" to_s="$2"
    if [[ ! -r $GC_LOG ]]; then
        echo "0 0 0"
        return
    fi
    awk -v from="$from_s" -v to="$to_s" '
        / Pause / && $NF ~ /ms$/ && match($0, /^\[[0-9.]+s\]/) {
            uptime = substr($0, 2, RLENGTH - 3) + 0      # "[12.345s]" -> 12.345
            if (uptime < from || uptime > to) next
            pause = $NF; sub(/ms$/, "", pause); pause += 0
            count++; total += pause
            if (pause > longest) longest = pause
        }
        END { printf "%d %.1f %.1f\n", count, total, longest }' "$GC_LOG"
}

# Runs in the background: writes one CSV line per second until killed.
sampler_loop() {
    local csv_file="$1"
    local start previous_time previous_cpu now cpu cpu_percent status

    # This loop is stopped with "kill", often in the middle of a command.
    # That is expected, so do not report it as an error.
    trap - ERR
    set +e

    echo "elapsed_s,memory_mb,cpu_percent_of_all_cpus,heap_used_mb,threads_busy,threads_total,connections_open,sessions_active" > "$csv_file"
    start="$(now_seconds)"
    previous_time="$start"
    previous_cpu="$(tomcat_cpu_seconds)"

    while true; do
        sleep "$SAMPLE_INTERVAL_SECONDS"
        now="$(now_seconds)"
        cpu="$(tomcat_cpu_seconds)"
        cpu_percent="$(calc "($cpu - $previous_cpu) / ($now - $previous_time) / $CPU_COUNT * 100")"
        status="$(curl --noproxy '*' --silent --max-time 2 "$STATUS_URL" 2>/dev/null)"

        printf '%s,%s,%s,%s,%s,%s,%s,%s\n' \
            "$(calc "$now - $start")" "$(tomcat_memory_mb)" "$cpu_percent" \
            "$(status_value "$status" heap_used_mb)" \
            "$(status_value "$status" threads_busy)" \
            "$(status_value "$status" threads_total)" \
            "$(status_value "$status" connections_open)" \
            "$(status_value "$status" sessions_active)" >> "$csv_file"

        previous_time="$now"
        previous_cpu="$cpu"
    done
}

# Largest value in column <n> of a samples file ("?" = no answer, skipped).
column_peak() {
    local file="$1" column="$2" start_value="${3:-0}"
    awk -F, -v c="$column" -v m="$start_value" \
        'NR > 1 && $c != "?" && $c + 0 > m + 0 {m = $c} END {print m}' "$file"
}

# -----------------------------------------------------------------------------
#  run_sampled_ab <name> <url> <clients> <keepalive: yes|no>
#
#  One load run while Tomcat is measured every second into
#  raw/<name>-samples.csv, and its counters are read before and after.
#
#  Sets: LOAD_CPU_AVG_PCT  LOAD_CPU_PEAK_PCT  LOAD_CPU_SECONDS
#        LOAD_MEM_IDLE_MB  LOAD_MEM_PEAK_MB   LOAD_HEAP_IDLE_MB  LOAD_HEAP_PEAK_MB
#        LOAD_BUSY_PEAK    LOAD_THREADS_PEAK  LOAD_CONN_PEAK
#        LOAD_GC_COUNT     LOAD_GC_PAUSES     LOAD_GC_PAUSE_TOTAL_MS  LOAD_GC_PAUSE_MAX_MS
#        LOAD_WINDOW_MS    LOAD_TOMCAT_ERRORS LOAD_REQUESTS  LOAD_RPS
#        (and all AB_* variables)
# -----------------------------------------------------------------------------
run_sampled_ab() {
    local name="$1" url="$2" clients="$3" keepalive="$4"
    local samples="$RAW_DIR/$name-samples.csv"

    # Values while Tomcat is idle, before the load starts.
    local before after
    before="$(tomcat_status)"
    LOAD_MEM_IDLE_MB="$(tomcat_memory_mb)"
    LOAD_HEAP_IDLE_MB="$(status_value "$before" heap_used_mb)"

    sampler_loop "$samples" &
    local sampler_pid=$!

    local cpu_before time_before cpu_after time_after
    cpu_before="$(tomcat_cpu_seconds)"
    time_before="$(now_seconds)"

    run_ab "$name" "$url" "$clients" "$keepalive"

    cpu_after="$(tomcat_cpu_seconds)"
    time_after="$(now_seconds)"
    kill "$sampler_pid" 2>/dev/null || true
    wait "$sampler_pid" 2>/dev/null || true
    after="$(tomcat_status)"
    printf '%s\n' "$before" > "$RAW_DIR/$name-status-before.txt"
    printf '%s\n' "$after"  > "$RAW_DIR/$name-status-after.txt"

    LOAD_REQUESTS="$AB_COMPLETE"
    LOAD_RPS="$AB_RPS"
    LOAD_CPU_SECONDS="$(calc "$cpu_after - $cpu_before")"
    LOAD_CPU_AVG_PCT="$(calc "$LOAD_CPU_SECONDS / ($time_after - $time_before) / $CPU_COUNT * 100")"

    # Columns: 2=memory_mb 3=cpu% 4=heap_used_mb 5=threads_busy 6=threads_total 7=connections_open
    LOAD_CPU_PEAK_PCT="$(column_peak "$samples" 3)"
    LOAD_MEM_PEAK_MB="$(column_peak "$samples" 2 "$LOAD_MEM_IDLE_MB")"
    LOAD_HEAP_PEAK_MB="$(column_peak "$samples" 4 "$LOAD_HEAP_IDLE_MB")"
    LOAD_BUSY_PEAK="$(column_peak "$samples" 5)"
    LOAD_THREADS_PEAK="$(column_peak "$samples" 6)"
    LOAD_CONN_PEAK="$(column_peak "$samples" 7)"

    # Counters from Tomcat and Java: difference between after and before.
    LOAD_TOMCAT_ERRORS=$(( $(status_value "$after" requests_error) - $(status_value "$before" requests_error) ))
    LOAD_GC_COUNT=$(( $(status_value "$after" gc_count) - $(status_value "$before" gc_count) ))

    # GC pauses from gc.log, within the time the load ran.
    local uptime_before_ms uptime_after_ms
    uptime_before_ms="$(status_value "$before" jvm_uptime_ms)"
    uptime_after_ms="$(status_value "$after" jvm_uptime_ms)"
    LOAD_WINDOW_MS=$(( uptime_after_ms - uptime_before_ms ))
    read -r LOAD_GC_PAUSES LOAD_GC_PAUSE_TOTAL_MS LOAD_GC_PAUSE_MAX_MS < <(
        gc_pauses_between "$(calc2 "$uptime_before_ms / 1000")" "$(calc2 "$uptime_after_ms / 1000")")
}

# The gc, cpu and memory tests share one sampled run (JSP page, CONCURRENCY
# clients, keep-alive), so when several are chosen the load runs only once.
RESOURCE_LOAD_DONE=no

run_resource_load() {
    [[ $RESOURCE_LOAD_DONE == yes ]] && return
    run_sampled_ab "resource-load" "$DYNAMIC_PAGE_URL" "$CONCURRENCY" yes
    RESOURCE_LOAD_DONE=yes
}


# =============================================================================
#  PART 4 - the tests, one function per metric
# =============================================================================

# -----------------------------------------------------------------------------
test_startup() {
    section_title "Startup time  ($STARTUP_ROUNDS restarts)"
    local round start_time ready_time jsp_time
    local ready_ms jsp_ms ready_times=() jsp_times=()

    for (( round = 1; round <= STARTUP_ROUNDS; round++ )); do
        tomcat_stop
        start_time="$(now_seconds)"
        tomcat_start
        if ! wait_until_serving 120; then
            die "Tomcat did not come back after restart. See $LOG_DIR/"
        fi
        ready_time="$(now_seconds)"

        # The first dynamic page after a start: Java loads the compiled JSP.
        curl --noproxy '*' -sf --max-time 60 -o /dev/null "$DYNAMIC_PAGE_URL" ||
            die "The dynamic page $DYNAMIC_PAGE_URL failed after restart."
        jsp_time="$(now_seconds)"

        ready_ms="$(calc "($ready_time - $start_time) * 1000")"
        jsp_ms="$(calc "($jsp_time - $ready_time) * 1000")"
        ready_times+=("$ready_ms")
        jsp_times+=("$jsp_ms")
        printf '    restart %d: ready after %s ms, first JSP page took %s ms more\n' \
            "$round" "$ready_ms" "$jsp_ms"
    done

    local average fastest slowest jsp_average
    average="$(printf '%s\n' "${ready_times[@]}" | awk '{s += $1} END {printf "%.1f", s / NR}')"
    fastest="$(printf '%s\n' "${ready_times[@]}" | sort -n | head -1)"
    slowest="$(printf '%s\n' "${ready_times[@]}" | sort -n | tail -1)"
    jsp_average="$(printf '%s\n' "${jsp_times[@]}" | awk '{s += $1} END {printf "%.1f", s / NR}')"

    record_result "Startup time" "$average ms average" "<= $TARGET_STARTUP_MAX_MS ms" \
        "$(verdict_at_most "$average" "$TARGET_STARTUP_MAX_MS")"

    add_details <<EOF
## Startup time

Tomcat was stopped and started $STARTUP_ROUNDS times. "Ready" is the time from
the start command until the first page (a static file) was served. Right
after that, the first dynamic page (hello.jsp) was requested once more.

| Round | Ready (ms) | First JSP page, extra (ms) |
|------:|-----------:|---------------------------:|
$(for i in "${!ready_times[@]}"; do printf '| %5d | %10s | %26s |\n' $(( i + 1 )) "${ready_times[$i]}" "${jsp_times[$i]}"; done)

Average **$average ms** until ready (fastest $fastest ms, slowest $slowest ms).
The first JSP request after a start took on average $jsp_average ms extra, because
Java loads and prepares the page's code on first use. (The JSP was already
compiled by start-tomcat.sh; a JSP that was never compiled takes seconds.)

EOF
}

# -----------------------------------------------------------------------------
test_static() {
    section_title "Static page throughput  (1 KB file, $CONCURRENCY clients, keep-alive)"
    run_ab "static" "$SMALL_PAGE_URL" "$CONCURRENCY" yes

    STATIC_RPS="$AB_RPS"      # remembered for the "dynamic" test
    record_result "Static throughput" "$(thousands "$AB_RPS") requests/s" \
        ">= $TARGET_STATIC_MIN_RPS requests/s" \
        "$(verdict_at_least "$AB_RPS" "$TARGET_STATIC_MIN_RPS")"

    add_details <<EOF
## Static page throughput

$CONCURRENCY clients requested a 1 KB HTML file as fast as possible for
$DURATION seconds. Tomcat's DefaultServlet serves it from its memory cache.

| Item | Value |
|---|---:|
| Requests per second | **$(thousands "$AB_RPS")** |
| Requests completed | $(thousands "$AB_COMPLETE") |
| Failed requests | $AB_FAILED |
| Keep-alive connections closed by Tomcat | $AB_SERVER_CLOSES |
| Average time per request | $AB_MEAN_MS ms |
| Load generator processes (ab) | $AB_PROCESSES_USED |

Raw output of each ab process: \`raw/ab-processes/static-ab*.txt\`

EOF
}

# -----------------------------------------------------------------------------
STATIC_RPS=""

test_dynamic() {
    section_title "Dynamic page throughput  (JSP page, $CONCURRENCY clients, keep-alive)"
    run_ab "dynamic" "$DYNAMIC_PAGE_URL" "$CONCURRENCY" yes

    local compared=""
    if [[ -n $STATIC_RPS ]]; then
        compared="The static 1 KB file (test \"static\") reached $(thousands "$STATIC_RPS") requests/s,
so the dynamic page runs at **$(calc "$AB_RPS / $STATIC_RPS * 100")%** of the static rate."
    fi

    record_result "Dynamic throughput" "$(thousands "$AB_RPS") requests/s" \
        ">= $TARGET_DYNAMIC_MIN_RPS requests/s" \
        "$(verdict_at_least "$AB_RPS" "$TARGET_DYNAMIC_MIN_RPS")"

    add_details <<EOF
## Dynamic page throughput

$CONCURRENCY clients requested hello.jsp as fast as possible for $DURATION seconds.
For every request Tomcat runs the page's Java code and builds a new page of
$AB_DOC_BYTES bytes. This is the typical work of an application server.

| Item | Value |
|---|---:|
| Requests per second | **$(thousands "$AB_RPS")** |
| Requests completed | $(thousands "$AB_COMPLETE") |
| Failed requests | $AB_FAILED |
| Non-2xx answers | $AB_NON_2XX |
| Average time per request | $AB_MEAN_MS ms |

$compared

EOF
}

# -----------------------------------------------------------------------------
test_latency() {
    section_title "Latency  (JSP page, $CONCURRENCY clients, keep-alive)"
    run_ab "latency" "$DYNAMIC_PAGE_URL" "$CONCURRENCY" yes

    local verdict=PASS
    is_at_most "$AB_P95_MS" "$TARGET_LATENCY_P95_MAX_MS" || verdict=FAIL
    is_at_most "$AB_P99_MS" "$TARGET_LATENCY_P99_MAX_MS" || verdict=FAIL

    record_result "Latency (p95 / p99)" "$AB_P95_MS ms / $AB_P99_MS ms" \
        "<= $TARGET_LATENCY_P95_MAX_MS ms / <= $TARGET_LATENCY_P99_MAX_MS ms" "$verdict"

    add_details <<EOF
## Latency (response time)

How long one request for the dynamic page took, from sending it to receiving
the full answer, with $CONCURRENCY clients active. "p95 = 3 ms" means 95 of every
100 requests finished within 3 ms.

| Statistic | Time (ms) |
|---|---:|
| Average | $AB_MEAN_MS |
| p50 (median) | $AB_P50_MS |
| p90 | $AB_P90_MS |
| p95 | **$AB_P95_MS** |
| p99 | **$AB_P99_MS** |
| Slowest request | $AB_MAX_MS |

A large gap between p50 and p99 in a Java server is often caused by garbage
collection pauses (see the "gc" test). Every percentile from 0 to 99:
\`raw/latency-percentiles.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_concurrency() {
    section_title "Concurrency scaling  (JSP page, clients: $CONCURRENCY_LEVELS)"
    local clients table="" peak_rps=0 peak_clients=0 last_rps=0 last_clients=0

    for clients in $CONCURRENCY_LEVELS; do
        run_ab "concurrency-${clients}-clients" "$DYNAMIC_PAGE_URL" "$clients" yes
        table+="$(printf '| %7s | %10s | %8s | %8s | %8s | %6s |' \
            "$clients" "$(thousands "$AB_RPS")" "$AB_MEAN_MS" "$AB_P95_MS" \
            "$AB_P99_MS" "$AB_FAILED")"$'\n'

        if is_at_least "$AB_RPS" "$peak_rps"; then
            peak_rps="$AB_RPS"
            peak_clients="$clients"
        fi
        last_rps="$AB_RPS"
        last_clients="$clients"
    done

    local kept_pct
    kept_pct="$(calc "$last_rps / $peak_rps * 100")"

    record_result "Concurrency scaling" \
        "$kept_pct% of peak kept at $last_clients clients" \
        ">= $TARGET_SCALING_MIN_PCT% of peak" \
        "$(verdict_at_least "$kept_pct" "$TARGET_SCALING_MIN_PCT")"

    add_details <<EOF
## Concurrency scaling

The dynamic-page test at several numbers of simultaneous clients. Healthy
behaviour: requests/second rises and then levels off; it should not collapse
at the highest level. Above $MAX_THREADS clients (maxThreads) some requests
must wait for a free worker thread, so the response time grows.

| Clients | Requests/s | Avg (ms) | p95 (ms) | p99 (ms) | Failed |
|--------:|-----------:|---------:|---------:|---------:|-------:|
${table}
Peak: **$(thousands "$peak_rps") requests/s at $peak_clients clients**.
At $last_clients clients Tomcat still delivered **$kept_pct%** of that peak.

EOF
}

# -----------------------------------------------------------------------------
test_keepalive() {
    section_title "Keep-alive effect  (JSP page, $CONCURRENCY clients)"

    run_ab "keepalive-on" "$DYNAMIC_PAGE_URL" "$CONCURRENCY" yes
    local on_rps="$AB_RPS" on_mean="$AB_MEAN_MS" on_p95="$AB_P95_MS"

    run_ab "keepalive-off" "$DYNAMIC_PAGE_URL" "$CONCURRENCY" no
    local off_rps="$AB_RPS" off_mean="$AB_MEAN_MS" off_p95="$AB_P95_MS"

    local gain
    gain="$(calc "$on_rps / $off_rps")"

    record_result "Keep-alive gain" "${gain}x more requests/s with keep-alive" \
        "information only" INFO

    add_details <<EOF
## Keep-alive effect

With keep-alive, a client sends many requests over one TCP connection.
Without it, every request opens and closes a new connection.

| Mode | Requests/s | Avg (ms) | p95 (ms) |
|---|---:|---:|---:|
| Keep-alive ON  | $(thousands "$on_rps")  | $on_mean  | $on_p95  |
| Keep-alive OFF | $(thousands "$off_rps") | $off_mean | $off_p95 |

Keep-alive served **${gain}x** as many requests per second.

EOF
}

# -----------------------------------------------------------------------------
test_transfer() {
    section_title "Transfer rate  (1 MB file, $CONCURRENCY clients, keep-alive)"
    run_ab "transfer" "$LARGE_FILE_URL" "$CONCURRENCY" yes

    local mb_per_second
    mb_per_second="$(calc "$AB_TRANSFER_KBPS / 1024")"

    record_result "Transfer rate" "$(thousands "$mb_per_second") MB/s" \
        ">= $TARGET_TRANSFER_MIN_MBPS MB/s" \
        "$(verdict_at_least "$mb_per_second" "$TARGET_TRANSFER_MIN_MBPS")"

    add_details <<EOF
## Transfer rate (bandwidth)

$CONCURRENCY clients downloaded a 1 MB file repeatedly for $DURATION seconds.
Tomcat sends files larger than 48 KB with "sendfile": the kernel copies the
file straight to the network, without passing it through Java.

| Item | Value |
|---|---:|
| Data sent per second | **$(thousands "$mb_per_second") MB/s** ($(thousands "$(calc "$mb_per_second * 8")") Mbit/s) |
| Files downloaded per second | $(thousands "$AB_RPS") |
| Files downloaded in total | $(thousands "$AB_COMPLETE") |
| Average time per download | $AB_MEAN_MS ms |

Over localhost the network is not a limit, so this shows how fast Tomcat
itself can push data. Over a real network, the network card is usually the limit.

EOF
}

# -----------------------------------------------------------------------------
test_compression() {
    section_title "Compression (gzip)  (40 KB HTML page, $CONCURRENCY clients, keep-alive)"

    run_ab "compression-off" "$TEXT_PAGE_URL" "$CONCURRENCY" yes
    local plain_rps="$AB_RPS" plain_bytes="$AB_DOC_BYTES" plain_p95="$AB_P95_MS"
    local plain_mbps
    plain_mbps="$(calc "$AB_TRANSFER_KBPS / 1024")"

    run_ab "compression-gzip" "$TEXT_PAGE_URL" "$CONCURRENCY" yes "$DURATION" "Accept-Encoding: gzip"
    local gzip_rps="$AB_RPS" gzip_bytes="$AB_DOC_BYTES" gzip_p95="$AB_P95_MS"
    local gzip_mbps
    gzip_mbps="$(calc "$AB_TRANSFER_KBPS / 1024")"

    if (( gzip_bytes >= plain_bytes )); then
        record_result "Compression (gzip)" "answers were NOT compressed" \
            "gzip active" FAIL
        add_details <<EOF
## Compression (gzip)

The page was sent uncompressed ($gzip_bytes bytes) even though the client asked
for gzip. Check COMPRESSION in settings.conf (currently "$COMPRESSION") and the
compression attributes of the Connector in $CATALINA_BASE/conf/server.xml.

EOF
        return
    fi

    local ratio speed_pct
    ratio="$(calc "$plain_bytes / $gzip_bytes")"
    speed_pct="$(calc "$gzip_rps / $plain_rps * 100")"

    record_result "Compression (gzip)" "${ratio}x smaller, $speed_pct% of uncompressed req/s" \
        "information only" INFO

    add_details <<EOF
## Compression (gzip)

$CONCURRENCY clients requested the same 40 KB HTML page, first without and then
with "Accept-Encoding: gzip". Compression makes answers smaller (less network
traffic, faster for users on slow links) but costs Tomcat CPU time.

| Item | Uncompressed | gzip |
|---|---:|---:|
| Size of one answer | $plain_bytes bytes | **$gzip_bytes bytes** |
| Requests per second | $(thousands "$plain_rps") | **$(thousands "$gzip_rps")** |
| p95 response time | $plain_p95 ms | $gzip_p95 ms |
| Data actually sent | $plain_mbps MB/s | $gzip_mbps MB/s |

Answers were **${ratio}x smaller**. With compression Tomcat served **$speed_pct%**
of the uncompressed request rate: the difference is the CPU cost of gzip.
Note: Tomcat never compresses files larger than 48 KB that it sends with
"sendfile" (see PERFORMANCE-METRICS.md).

EOF
}

# -----------------------------------------------------------------------------
test_threads() {
    local half=$(( MAX_THREADS / 2 )) double=$(( MAX_THREADS * 2 ))
    (( half < 1 )) && half=1
    section_title "Worker thread pool  (page waits $SLOW_REQUEST_MS ms, $half then $double clients, maxThreads $MAX_THREADS)"

    # The most requests/s the pool can finish when every request takes
    # SLOW_REQUEST_MS: every thread finishes 1000 / SLOW_REQUEST_MS per second.
    local pool_limit_rps
    pool_limit_rps="$(calc "$MAX_THREADS * 1000 / $SLOW_REQUEST_MS")"

    # Step 1: fewer clients than threads - nobody has to wait for a thread.
    run_sampled_ab "threads-${half}-clients" "$SLOW_PAGE_URL" "$half" yes
    local low_rps="$AB_RPS" low_p50="$AB_P50_MS" low_p99="$AB_P99_MS"
    local low_busy="$LOAD_BUSY_PEAK" low_failed=$(( AB_FAILED + AB_NON_2XX ))

    # Step 2: twice as many clients as threads - the pool is full, the other
    # requests wait in a queue until a thread is free.
    run_sampled_ab "threads-${double}-clients" "$SLOW_PAGE_URL" "$double" yes
    local high_rps="$AB_RPS" high_p50="$AB_P50_MS" high_p99="$AB_P99_MS"
    local high_busy="$LOAD_BUSY_PEAK" high_total="$LOAD_THREADS_PEAK"
    local high_failed=$(( AB_FAILED + AB_NON_2XX ))

    local efficiency_pct wait_ms
    efficiency_pct="$(calc "$high_rps / $pool_limit_rps * 100")"
    wait_ms="$(calc2 "$high_p50 - $SLOW_REQUEST_MS")"

    local verdict
    verdict="$(verdict_at_least "$efficiency_pct" "$TARGET_THREAD_POOL_MIN_PCT")"
    (( low_failed + high_failed > 0 )) && verdict=FAIL

    record_result "Thread pool" "$(thousands "$high_rps") req/s = $efficiency_pct% of pool limit, $high_busy/$MAX_THREADS busy" \
        ">= $TARGET_THREAD_POOL_MIN_PCT% of limit" "$verdict"

    add_details <<EOF
## Worker thread pool

Every request of this test waits $SLOW_REQUEST_MS ms inside Tomcat (like a page
that waits for a database), and keeps one worker thread busy meanwhile.
The pool has at most **$MAX_THREADS threads** (maxThreads), so it can finish at most
$MAX_THREADS x 1000 / $SLOW_REQUEST_MS = **$(thousands "$pool_limit_rps") requests/s**.

| Item | $half clients (fewer than threads) | $double clients (more than threads) |
|---|---:|---:|
| Requests per second | $(thousands "$low_rps") | **$(thousands "$high_rps")** |
| Busy worker threads (peak) | $low_busy | **$high_busy** of $MAX_THREADS |
| p50 response time | $low_p50 ms | $high_p50 ms |
| p99 response time | $low_p99 ms | $high_p99 ms |
| Failed or non-2xx answers | $low_failed | $high_failed |

With $double clients the pool was full: Tomcat reached **$efficiency_pct%** of the
pool limit, and a typical request waited about **$wait_ms ms** in the queue for
a free thread before its own $SLOW_REQUEST_MS ms of work started. Threads in the
pool at the peak: $high_total.

A full pool is normal under heavy load; it becomes a problem when requests
wait too long. More threads help only while the CPU and the backend
(database) still have spare capacity. Per-second samples:
\`raw/threads-${double}-clients-samples.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_sessions() {
    section_title "Sessions  ($SESSIONS_TEST_COUNT new sessions, $SESSION_DATA_BYTES bytes each, $CONCURRENCY clients)"

    # Start clean: end old sessions of /perf-test, free unused memory.
    local before after cleaned
    before="$(tomcat_status expire-sessions)"
    printf '%s\n' "$before" > "$RAW_DIR/sessions-status-before.txt"

    # ab never returns the session cookie, so every request makes a new session.
    AB_REQUEST_LIMIT="$SESSIONS_TEST_COUNT"
    run_ab "sessions" "$SESSION_PAGE_URL" "$CONCURRENCY" yes 300
    AB_REQUEST_LIMIT=""

    # Count the sessions, with only live objects left in the heap.
    after="$(tomcat_status gc)"
    printf '%s\n' "$after" > "$RAW_DIR/sessions-status-after.txt"

    local created active rejected heap_before heap_after heap_growth kb_per_session
    created=$(( $(status_value "$after" sessions_created) - $(status_value "$before" sessions_created) ))
    rejected=$(( $(status_value "$after" sessions_rejected) - $(status_value "$before" sessions_rejected) ))
    active="$(status_value "$after" sessions_active)"
    heap_before="$(status_value "$before" heap_used_mb)"
    heap_after="$(status_value "$after" heap_used_mb)"
    heap_growth="$(calc "$heap_after - $heap_before")"
    kb_per_session="$(awk -v growth="$heap_growth" -v n="$active" \
        'BEGIN { printf "%.2f", (n > 0 ? growth * 1024 / n : 0) }')"

    # Clean up, so the sessions do not use memory in the next tests.
    cleaned="$(tomcat_status expire-sessions)"
    printf '%s\n' "$cleaned" > "$RAW_DIR/sessions-status-cleaned.txt"

    local verdict
    verdict="$(verdict_at_least "$AB_RPS" "$TARGET_SESSIONS_MIN_PER_S")"
    (( AB_FAILED + AB_NON_2XX + rejected > 0 )) && verdict=FAIL

    record_result "Sessions" "$(thousands "$AB_RPS") new sessions/s, $kb_per_session KB heap each" \
        ">= $TARGET_SESSIONS_MIN_PER_S sessions/s" "$verdict"

    add_details <<EOF
## Sessions

A session is the memory Tomcat keeps for one user between requests (login,
shopping cart ...). Each request of this test created a new session and
stored $SESSION_DATA_BYTES bytes of data in it. The Java heap was measured after
a full garbage collection, so only memory really in use was counted.

| Item | Value |
|---|---:|
| Sessions created per second | **$(thousands "$AB_RPS")** |
| Sessions created (Tomcat's counter) | $(thousands "$created") of $(thousands "$SESSIONS_TEST_COUNT") requested |
| Sessions alive after the test | $(thousands "$active") |
| Sessions rejected | $rejected |
| Failed or non-2xx answers | $(( AB_FAILED + AB_NON_2XX )) |
| Heap in use before / after | $heap_before MB / $heap_after MB |
| **Heap per session** | **$kb_per_session KB** (of which $SESSION_DATA_BYTES bytes are the stored data) |
| Heap in use after ending the sessions | $(status_value "$cleaned" heap_used_mb) MB |

Sessions stay in memory until they time out ($SESSION_TIMEOUT_MINUTES minutes without
use). Rough memory need: users active within $SESSION_TIMEOUT_MINUTES minutes x heap per
session. A client that ignores the session cookie (like "ab", or a monitoring
script) creates a new session with every request.

EOF
}

# -----------------------------------------------------------------------------
# Number of "SEVERE" lines in Tomcat's own log files so far.
count_severe_log_lines() {
    cat "$LOG_DIR"/catalina.*.log "$LOG_DIR"/localhost.*.log 2>/dev/null | grep -c 'SEVERE' || true
}

test_errors() {
    section_title "Error rate under overload  ($ERROR_TEST_CONCURRENCY clients, no keep-alive)"

    local severe_before
    severe_before="$(count_severe_log_lines)"

    run_sampled_ab "errors-overload" "$DYNAMIC_PAGE_URL" "$ERROR_TEST_CONCURRENCY" no

    local bad_requests error_rate severe_after new_severe new_log_lines=""
    bad_requests=$(( AB_FAILED + AB_NON_2XX ))
    error_rate="$(awk -v bad="$bad_requests" -v all="$AB_COMPLETE" \
        'BEGIN { printf "%.3f", (all > 0 ? bad / all * 100 : 100) }')"

    severe_after="$(count_severe_log_lines)"
    new_severe=$(( severe_after - severe_before ))
    if (( new_severe > 0 )); then
        new_log_lines="$(cat "$LOG_DIR"/catalina.*.log "$LOG_DIR"/localhost.*.log 2>/dev/null |
                         grep 'SEVERE' | tail -n "$new_severe")"
    fi
    printf '%s\n' "$new_log_lines" > "$RAW_DIR/errors-new-SEVERE-log-lines.txt"

    record_result "Error rate" "$error_rate% ($bad_requests of $(thousands "$AB_COMPLETE"))" \
        "<= $TARGET_ERROR_RATE_MAX_PCT%" \
        "$(verdict_at_most "$error_rate" "$TARGET_ERROR_RATE_MAX_PCT")"

    add_details <<EOF
## Error rate under overload

$ERROR_TEST_CONCURRENCY clients, each opening a new connection for every request,
requested the dynamic page for $DURATION seconds. This is five times the
worker thread pool ($MAX_THREADS threads) and pushes Tomcat harder than normal traffic.

| Item | Value |
|---|---:|
| Requests completed | $(thousands "$AB_COMPLETE") |
| Requests per second | $(thousands "$AB_RPS") |
| Failed requests (network / wrong length) | $AB_FAILED |
| &nbsp;&nbsp;broken down as | $AB_FAILED_DETAIL |
| Non-2xx answers (HTTP errors such as 503) | $AB_NON_2XX |
| **Error rate** | **$error_rate%** |
| Error answers counted by Tomcat (status 400 and higher) | $LOAD_TOMCAT_ERRORS |
| Busy worker threads (peak) | $LOAD_BUSY_PEAK of $MAX_THREADS |
| p99 / slowest response | $AB_P99_MS ms / $AB_MAX_MS ms |
| New SEVERE lines in Tomcat's logs | $new_severe |

$(if [[ -n $new_log_lines ]]; then
    echo "Last SEVERE lines from the logs (all in \`raw/errors-new-SEVERE-log-lines.txt\`):"
    echo
    echo '```'
    printf '%s\n' "$new_log_lines" | head -10
    echo '```'
  fi)

EOF
}

# -----------------------------------------------------------------------------
test_connections() {
    section_title "Connections  ($CONNECTIONS_TEST_CLIENTS clients, keep-alive)"
    run_sampled_ab "connections-load" "$DYNAMIC_PAGE_URL" "$CONNECTIONS_TEST_CLIENTS" yes

    local used_pct
    used_pct="$(calc "$LOAD_CONN_PEAK / $MAX_CONNECTIONS * 100")"

    local verdict
    verdict="$(verdict_at_most "$used_pct" "$TARGET_CONNECTIONS_MAX_PCT")"
    (( AB_FAILED > 0 )) && verdict=FAIL

    record_result "Connections" "$LOAD_CONN_PEAK of $(thousands "$MAX_CONNECTIONS") open ($used_pct%), $AB_FAILED failed" \
        "<= $TARGET_CONNECTIONS_MAX_PCT%, 0 failed" "$verdict"

    add_details <<EOF
## Connections

$CONNECTIONS_TEST_CLIENTS clients kept their connections open (keep-alive) for
$DURATION seconds. Tomcat's counters were read once per second. Tomcat accepts
at most **$MAX_CONNECTIONS** connections (maxConnections); more wait in the
operating system's queue (acceptCount = $ACCEPT_COUNT).

| Item | Value |
|---|---:|
| Most open connections at one time | **$LOAD_CONN_PEAK** |
| Maximum connections (maxConnections) | $MAX_CONNECTIONS |
| Peak usage | **$used_pct%** |
| Busy worker threads at the same time (peak) | $LOAD_BUSY_PEAK of $MAX_THREADS |
| Failed requests | $AB_FAILED |
| Requests per second during the test | $(thousands "$AB_RPS") |

With the NIO connector an open but idle connection needs no worker thread,
so Tomcat can hold many more connections than it has threads.
Per-second samples: \`raw/connections-load-samples.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_gc() {
    section_title "Garbage collection  (JSP page, $CONCURRENCY clients)"
    run_resource_load

    local overhead_pct pauses_per_s average_pause
    overhead_pct="$(awk -v pause="$LOAD_GC_PAUSE_TOTAL_MS" -v window="$LOAD_WINDOW_MS" \
        'BEGIN { printf "%.2f", (window > 0 ? pause / window * 100 : 0) }')"
    pauses_per_s="$(awk -v n="$LOAD_GC_PAUSES" -v window="$LOAD_WINDOW_MS" \
        'BEGIN { printf "%.1f", (window > 0 ? n / window * 1000 : 0) }')"
    average_pause="$(awk -v total="$LOAD_GC_PAUSE_TOTAL_MS" -v n="$LOAD_GC_PAUSES" \
        'BEGIN { printf "%.2f", (n > 0 ? total / n : 0) }')"

    local verdict=PASS
    is_at_most "$overhead_pct" "$TARGET_GC_OVERHEAD_MAX_PCT" || verdict=FAIL
    is_at_most "$LOAD_GC_PAUSE_MAX_MS" "$TARGET_GC_PAUSE_MAX_MS" || verdict=FAIL

    record_result "Garbage collection" "$overhead_pct% of time paused, longest pause $LOAD_GC_PAUSE_MAX_MS ms" \
        "<= $TARGET_GC_OVERHEAD_MAX_PCT%, <= $TARGET_GC_PAUSE_MAX_MS ms" "$verdict"

    # Copy the pause lines of this run into the result folder.
    grep ' Pause ' "$GC_LOG" 2>/dev/null | tail -n "$LOAD_GC_PAUSES" > "$RAW_DIR/gc-pauses-during-load.txt" || true

    add_details <<EOF
## Garbage collection (GC)

Java frees unused memory automatically with a "garbage collector". During a
GC pause the application threads stop for a moment, so every request that is
running waits. Measured while $CONCURRENCY clients requested the dynamic page for
$DURATION seconds ($(thousands "$LOAD_RPS") requests/s). Pause times come from
\`$GC_LOG\`.

| Item | Value |
|---|---:|
| GC pauses | $LOAD_GC_PAUSES ($pauses_per_s per second) |
| **Time spent in GC pauses** | **$overhead_pct%** of $(calc "$LOAD_WINDOW_MS / 1000") s |
| Average pause | $average_pause ms |
| **Longest pause** | **$LOAD_GC_PAUSE_MAX_MS ms** |
| Garbage collections (Java's counter) | $LOAD_GC_COUNT |
| Heap in use: before the load / highest sample | $LOAD_HEAP_IDLE_MB MB / $LOAD_HEAP_PEAK_MB MB (maximum $HEAP_MAX_MB MB) |
| Garbage collector | $GC_OPTIONS |

Short, frequent pauses of a few ms are normal. Long pauses (hundreds of ms)
or a high share of time in GC mean the heap is too small or the application
keeps too many objects. Pause lines of this run: \`raw/gc-pauses-during-load.txt\`

EOF
}

# -----------------------------------------------------------------------------
test_cpu() {
    section_title "CPU usage under load  ($CONCURRENCY clients, $CPU_COUNT CPUs)"
    run_resource_load

    local ms_per_1000
    ms_per_1000="$(calc "$LOAD_CPU_SECONDS * 1000 / $LOAD_REQUESTS * 1000")"

    record_result "CPU usage" "$LOAD_CPU_AVG_PCT% average of all CPUs" \
        "<= $TARGET_CPU_MAX_PCT%" \
        "$(verdict_at_most "$LOAD_CPU_AVG_PCT" "$TARGET_CPU_MAX_PCT")"

    add_details <<EOF
## CPU usage

CPU used by the Tomcat process while serving $(thousands "$LOAD_RPS") dynamic pages
per second. 100% means every one of the $CPU_COUNT CPUs was fully busy with Tomcat.
(The load generator "ab" runs on the same machine and uses CPU too.)

| Item | Value |
|---|---:|
| Average CPU, % of all CPUs | **$LOAD_CPU_AVG_PCT%** |
| Busiest second, % of all CPUs | $LOAD_CPU_PEAK_PCT% |
| Same average, in CPU cores | $(calc "$LOAD_CPU_AVG_PCT * $CPU_COUNT / 100") cores |
| CPU time per 1000 requests | $ms_per_1000 ms |

The CPU time includes Java's own work: the JIT compiler and the garbage
collector. Per-second samples: \`raw/resource-load-samples.csv\`

EOF
}

# -----------------------------------------------------------------------------
test_memory() {
    section_title "Memory usage  ($CONCURRENCY clients)"
    run_resource_load

    local status
    status="$(tomcat_status)"

    record_result "Memory usage" "$LOAD_MEM_PEAK_MB MB peak (before load $LOAD_MEM_IDLE_MB MB)" \
        "<= $TARGET_MEMORY_MAX_MB MB" \
        "$(verdict_at_most "$LOAD_MEM_PEAK_MB" "$TARGET_MEMORY_MAX_MB")"

    add_details <<EOF
## Memory usage

Memory of the Tomcat Java process (PSS), just before and while $CONCURRENCY clients
requested the dynamic page (sampled once per second). The Java heap is the part where the application's
objects live; the rest is Java itself (code, compiled code, thread stacks).

| Item | Before the load | Under load (highest sample) |
|---|---:|---:|
| **Process memory (MB)** | $LOAD_MEM_IDLE_MB | **$LOAD_MEM_PEAK_MB** |
| Java heap in use (MB) | $LOAD_HEAP_IDLE_MB | $LOAD_HEAP_PEAK_MB |

| Java memory setting / counter | Value |
|---|---:|
| Heap: start size (-Xms) / maximum (-Xmx) | $HEAP_INITIAL_MB MB / $HEAP_MAX_MB MB |
| Heap reserved from the system now | $(status_value "$status" heap_committed_mb) MB |
| Non-heap memory in use (classes, compiled code) | $(status_value "$status" nonheap_used_mb) MB |
| Java threads | $(status_value "$status" jvm_threads) |

The process never gives back all memory it once used for the heap, so its size
mostly follows the heap settings, not the current load. "Heap in use" goes up
and down with every garbage collection. Per-second samples:
\`raw/resource-load-samples.csv\`

EOF
}


# =============================================================================
#  PART 5 - preparing, and writing the final report
# =============================================================================

check_ready_to_test() {
    require_root "$@"
    command -v ab   >/dev/null || die "'ab' not found. Install httpd-tools (./start-tomcat.sh does this)."
    command -v curl >/dev/null || die "'curl' not found."
    tomcat_is_serving || die "Tomcat is not serving $HEALTH_URL. Run ./start-tomcat.sh first."
    tomcat_status | grep -q '^threads_max=' ||
        die "Status page $STATUS_URL not available. Run ./start-tomcat.sh first."

    # ab needs one open file per client; the default limit (1024) is too low
    # for the tests with 1000 clients.
    ulimit -n 65536 2>/dev/null ||
        warn "Could not raise the open-files limit; tests with many clients may fail."
}

prepare_result_folder() {
    RESULT_DIR="$KIT_DIR/results/$(date +%Y%m%d-%H%M%S)"
    RAW_DIR="$RESULT_DIR/raw"
    AB_DIR="$RAW_DIR/ab-processes"
    DETAILS_FILE="$RAW_DIR/.details.md"
    mkdir -p "$RAW_DIR" "$AB_DIR"
    : > "$DETAILS_FILE"
    ln -sfn "$(basename "$RESULT_DIR")" "$KIT_DIR/results/latest"
}

# Unmeasured load before the tests. Java runs code slowly at first and
# compiles the busy parts to fast machine code after a while ("JIT warm-up").
warm_up() {
    local half=$(( (WARMUP_SECONDS + 1) / 2 ))
    log "Warm-up: $WARMUP_SECONDS seconds of load (not measured), so Java has compiled the hot code"
    ab -k -t "$half" -n 10000000 -c 10 "$DYNAMIC_PAGE_URL" > "$RAW_DIR/warm-up-dynamic.txt" 2>&1 || true
    ab -k -t "$half" -n 10000000 -c 10 "$SMALL_PAGE_URL"   > "$RAW_DIR/warm-up-static.txt"  2>&1 || true
}

write_report() {
    local tests_run="$1" started="$2" finished="$3"
    local report="$RESULT_DIR/report.md"
    local status os_name selinux cpu_model memory_total

    status="$(tomcat_status)"
    printf '%s\n' "$status" > "$RAW_DIR/status-at-end.txt"
    os_name="$(. /etc/os-release && echo "$PRETTY_NAME")"
    selinux="$(getenforce 2>/dev/null || echo 'not available')"
    cpu_model="$(awk -F': ' '/^model name/ {print $2; exit}' /proc/cpuinfo)"
    memory_total="$(awk '/^MemTotal/ {printf "%.1f GB", $2 / 1048576}' /proc/meminfo)"

    local overall="ALL PASSED"
    (( FAIL_COUNT > 0 )) && overall="$FAIL_COUNT TEST(S) FAILED"

    {
        echo "# Tomcat Performance Test Report"
        echo
        echo "**Result: $overall**"
        echo
        echo "| | |"
        echo "|---|---|"
        echo "| Test started  | $started |"
        echo "| Test finished | $finished |"
        echo "| Host | $(hostname) |"
        echo "| Operating system | $os_name (kernel $(uname -r)) |"
        echo "| CPU | $CPU_COUNT x $cpu_model |"
        echo "| Memory | $memory_total |"
        echo "| Tomcat | $(status_value "$status" tomcat_version), instance $SERVICE_NAME |"
        echo "| Java | $(status_value "$status" jvm_version), heap $HEAP_INITIAL_MB-$HEAP_MAX_MB MB, $GC_OPTIONS |"
        echo "| Connector | $(status_value "$status" connector): maxThreads $MAX_THREADS, maxConnections $MAX_CONNECTIONS, compression $COMPRESSION |"
        echo "| SELinux | $selinux |"
        echo "| Target URL | $BASE_URL/perf-test/ |"
        echo "| Tests run | $tests_run |"
        echo "| Load per run | $DURATION s, $CONCURRENCY clients (unless the test says otherwise) |"
        echo "| Load generator | ApacheBench (ab), up to $(ab_process_count 1000) processes side by side, on the same machine |"
        echo
        echo "## Summary"
        echo
        echo "PASS = target met, FAIL = target missed, INFO = measured only (no target)."
        echo "Targets are set in \`settings.conf\`. What each metric means: \`PERFORMANCE-METRICS.md\`."
        echo
        printf '| %-22s | %-58s | %-30s | %-7s |\n' "Test" "Measured" "Target" "Verdict"
        printf '|%s|%s|%s|%s|\n' "$(printf -- '-%.0s' {1..24})" "$(printf -- '-%.0s' {1..60})" \
                                 "$(printf -- '-%.0s' {1..32})" "$(printf -- '-%.0s' {1..9})"
        local row name measured target verdict
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            [[ $verdict == FAIL ]] && verdict="**FAIL**"
            printf '| %-22s | %-58s | %-30s | %-7s |\n' "$name" "$measured" "$target" "$verdict"
        done
        echo
        echo "# Details"
        echo
        cat "$DETAILS_FILE"
        echo "## Files in this folder"
        echo
        echo "- \`report.md\` - this report"
        echo "- \`summary.csv\` - the summary table, for spreadsheets"
        echo "- \`raw/\` - merged percentile tables, per-second samples, Tomcat counters before/after"
        echo "- \`raw/ab-processes/\` - unmodified output of every ApacheBench process"
        echo
        echo "Tomcat's own logs are in \`$LOG_DIR/\` (catalina.*.log, localhost.*.log, gc.log)."
    } > "$report"

    # The same summary as CSV.
    {
        echo "test,measured,target,verdict"
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            printf '"%s","%s","%s","%s"\n' "$name" "$measured" "$target" "$verdict"
        done
    } > "$RESULT_DIR/summary.csv"

    rm -f "$DETAILS_FILE"
}

print_usage() {
    # Print the comment block at the top of this file.
    sed -n '3,/^set -euo pipefail/{/^#/p}' "$0" | sed 's/^# \{0,1\}//'
}


# =============================================================================
#  MAIN
# =============================================================================
main() {
    case "${1:-}" in
        -h|--help) print_usage; exit 0 ;;
        --list)    printf '%s\n' "${ALL_TESTS[@]}"; exit 0 ;;
    esac

    # Which tests to run: the names given, or all of them.
    local tests=("$@")
    if (( ${#tests[@]} == 0 )) || [[ ${tests[0]} == all ]]; then
        tests=("${ALL_TESTS[@]}")
    fi
    local test_name
    for test_name in "${tests[@]}"; do
        if [[ " ${ALL_TESTS[*]} " != *" $test_name "* ]]; then
            echo "Unknown test: '$test_name'. Valid tests: ${ALL_TESTS[*]}" >&2
            exit 2
        fi
    done

    check_ready_to_test
    prepare_result_folder

    # Always stop background jobs (sampler, ab) if the script is interrupted.
    trap 'kill $(jobs -p) 2>/dev/null || true' EXIT

    local started finished
    started="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    log "Testing $BASE_URL/perf-test/ - tests: ${tests[*]}"
    log "Results folder: $RESULT_DIR"

    # The startup test restarts Java, which forgets its warm-up; so the
    # warm-up runs just before the first test that is not "startup".
    local warmed_up=no
    for test_name in "${tests[@]}"; do
        if [[ $test_name != startup && $warmed_up == no ]]; then
            warm_up
            warmed_up=yes
        fi
        "test_$test_name"
    done

    finished="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    write_report "${tests[*]}" "$started" "$finished"

    echo
    sed -n '/^## Summary/,/^# Details/p' "$RESULT_DIR/report.md" | sed '$d'
    echo
    ok "Report saved: $RESULT_DIR/report.md"

    # Exit code: 0 when nothing failed, otherwise 1.
    if (( FAIL_COUNT > 0 )); then
        exit 1
    fi
}

main "$@"