#!/usr/bin/env bash
# =============================================================================
#  test-samba.sh - measure Samba server performance, one metric at a time,
#                  and save an easy-to-read report
# =============================================================================
#
#  USAGE (as root, after ./start-samba.sh)
#      ./test-samba.sh                   run every test (about 2-3 minutes)
#      ./test-samba.sh seqread           run one test
#      ./test-samba.sh latency errors    run several tests
#      ./test-samba.sh --list            show the test names
#
#      DURATION=30 LOAD_RATE=1000 ./test-samba.sh latency
#                                        override settings.conf for one run
#
#  THE TESTS  (explained in detail in PERFORMANCE-METRICS.md)
#      startup      time from "start smbd" until logins and the mounted client work
#      login        new client connections (with login) per second
#      seqwrite     writing a big file, in MB/s
#      seqread      reading a big file, in MB/s
#      randread     4 KiB reads at random places, per second (IOPS)
#      randwrite    4 KiB writes at random places, per second (IOPS)
#      latency      time one request takes, at a steady realistic load
#      errors       requests that failed at that steady load
#      metadata     small files created (and deleted) per second
#      concurrency  how many clients can work at the same time, still fast
#      cpu          CPU used by smbd per request, per GB and per login
#      memory       server memory used for each client connection
#
#  OUTPUT
#      results/<date>-<time>/report.md    the human-readable report
#      results/<date>-<time>/summary.csv  one line per test (for spreadsheets)
#      results/<date>-<time>/raw/         unmodified measurement output
#      results/latest                     link to the newest result folder
#
#  EXIT CODE
#      0 = every test passed,  1 = at least one FAIL,  2 = could not run
#
#  The load comes from "fio" (the standard Linux storage benchmark), which
#  works through the test mount (the kernel's SMB client), and from
#  lib/smb-load.py, which opens its own SMB connections with Samba's client
#  library. Everything runs on this machine over 127.0.0.1; no internet
#  access is needed.
# =============================================================================

set -euo pipefail
source "$(dirname -- "${BASH_SOURCE[0]}")/lib/common.sh"

ALL_TESTS=(startup login seqwrite seqread randread randwrite latency errors metadata concurrency cpu memory)

DATA_DIR="$MOUNT_POINT/fio"          # data.0, data.1, ... for the fio tests
DATA_FILE_ON_SHARE="fio/data.0"      # the same file, as smb-load.py names it
GENERATOR_BUSY_PCT=90                # fio above this = it may be the limit

# Filled in by the functions below.
RESULT_DIR=""
RAW_DIR=""
DETAILS_FILE=""
SUMMARY_ROWS=()     # "Test|Measured|Target|Verdict"
FAIL_COUNT=0
GENERATOR_LIMITED_TESTS=()
SMBD_CPU_PER_LOGIN_MS=""   # measured by "login", shown again by "cpu"


# =============================================================================
#  PART 1 - small helpers
# =============================================================================

# Floating-point math and comparisons (bash itself only knows whole numbers).
calc()          { awk "BEGIN { printf \"%.1f\", $* }"; }
calc2()         { awk "BEGIN { printf \"%.2f\", $* }"; }
calc3()         { awk "BEGIN { printf \"%.3f\", $* }"; }
is_at_most()    { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a <= b) }'; }
is_at_least()   { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a >= b) }'; }
now_seconds()   { date +%s.%N; }

# Turn a comparison into the words PASS / FAIL.
verdict_at_most()  { if is_at_most  "$1" "$2"; then echo PASS; else echo FAIL; fi; }
verdict_at_least() { if is_at_least "$1" "$2"; then echo PASS; else echo FAIL; fi; }

# Remember one line for the summary table at the top of the report.
#   record_result "Test name" "measured value" "target" PASS|FAIL|INFO
record_result() {
    local name="$1" measured="$2" target="$3" verdict="$4"
    SUMMARY_ROWS+=("$name|$measured|$target|$verdict")
    [[ $verdict == FAIL ]] && FAIL_COUNT=$(( FAIL_COUNT + 1 ))

    local colour='1;32'                       # green  = PASS
    [[ $verdict == FAIL ]] && colour='1;31'   # red    = FAIL
    [[ $verdict == INFO ]] && colour='1;36'   # cyan   = INFO
    printf "    => \033[${colour}m%-4s\033[0m  %s   (target: %s)\n" "$verdict" "$measured" "$target"
}

# Append Markdown text (read from stdin) to the "details" part of the report.
add_details() {
    cat >> "$DETAILS_FILE"
}

section_title() {
    echo
    printf '\033[1m==> %s\033[0m\n' "$*"
}


# =============================================================================
#  PART 2 - running the load and reading its results
# =============================================================================
#
#  Every load run writes its values as "key = value" lines to
#  raw/<name>.txt. Read single values afterwards with:  value <key>
#
#  run_fio <name> [fio options ...]
#      fio through the test mount. Also saves raw/<name>.json (fio's
#      complete result).
#
#  run_load_tool <name> <mode> [smb-load.py options ...]
#      lib/smb-load.py with its own SMB connections.
#
#  Both add what the SERVER did during the run: the CPU time of the test
#  smbd, and (fio only) the kernel SMB client's request counters.
#
#  A run that already happened in this test session is not repeated (the
#  "cpu" test re-uses the "randread" and "seqread" runs, for example).
# -----------------------------------------------------------------------------
LOAD_OUTPUT=""

# Options every fio run uses:
#   --direct=1             bypass the client's page cache, so every read and
#                          write really travels to the Samba server
#   --ioengine=libaio      keep several requests in flight (--iodepth)
#   --time_based           run for exactly DURATION seconds
#   --lat_percentiles=1    record the complete time of every request
#   --continue_on_error    count failed requests instead of stopping
fio_common_options() {
    printf '%s\n' \
        --directory="$DATA_DIR" \
        --ioengine=libaio \
        --direct=1 \
        --time_based \
        --runtime="$DURATION" \
        --group_reporting \
        --lat_percentiles=1 \
        --percentile_list=50:90:95:99:99.9 \
        --continue_on_error=all \
        --output-format=json
}

run_fio() {
    local name="$1"
    shift
    LOAD_OUTPUT="$RAW_DIR/$name.txt"
    [[ -s $LOAD_OUTPUT ]] && return 0             # already measured

    local common_options=()
    mapfile -t common_options < <(fio_common_options)

    printf '    %-28s ' "$name"
    local cpu_before counters_before
    cpu_before="$(smbd_cpu_seconds)"
    counters_before="$(cifs_counters)"

    if ! fio --name="$name" "${common_options[@]}" "$@" \
            --output="$RAW_DIR/$name.json" 2> "$RAW_DIR/$name-errors.txt"; then
        echo "failed"
        die "fio failed. See $RAW_DIR/$name-errors.txt"
    fi
    [[ -s $RAW_DIR/$name-errors.txt ]] || rm -f "$RAW_DIR/$name-errors.txt"

    local cpu_after counters_after
    cpu_after="$(smbd_cpu_seconds)"
    counters_after="$(cifs_counters)"

    {
        python3 "$FIO_TO_VALUES" "$RAW_DIR/$name.json"
        echo "smbd_cpu_seconds         = $(calc3 "$cpu_after - $cpu_before")"
        counter_differences "$counters_before" "$counters_after"
    } > "$LOAD_OUTPUT"

    printf '%10.0f IOPS %8.1f MB/s   %s failed\n' \
        "$(calc "$(value read_iops) + $(value write_iops)")" \
        "$(calc "$(value read_mb_per_s) + $(value write_mb_per_s)")" "$(value errors)"

    # Note it when fio itself was almost fully busy: then the result may
    # show the limit of the load generator, not of the Samba server.
    if is_at_least "$(calc "$(value fio_cpu_user_pct) + $(value fio_cpu_system_pct)")" "$GENERATOR_BUSY_PCT"; then
        GENERATOR_LIMITED_TESTS+=("$name")
        warn "fio was ${GENERATOR_BUSY_PCT}%+ busy in '$name'; the server may be faster than measured."
    fi
}

# The options that tell smb-load.py where the test server is.
load_tool_connection_options() {
    printf '%s\n' --server "$CONNECT_ADDRESS" --share "$SHARE_NAME" \
        --credentials "$CREDENTIALS_FILE" --config "$TEST_CONF" --encrypt "$ENCRYPTION"
}

run_load_tool() {
    local name="$1" mode="$2"
    shift 2
    LOAD_OUTPUT="$RAW_DIR/$name.txt"
    [[ -s $LOAD_OUTPUT ]] && return 0             # already measured

    local connection_options=()
    mapfile -t connection_options < <(load_tool_connection_options)

    printf '    %-28s ' "$name"
    local cpu_before cpu_after
    cpu_before="$(smbd_cpu_seconds)"
    if ! python3 "$LOAD_TOOL" "$mode" "${connection_options[@]}" "$@" \
            --output "$LOAD_OUTPUT.tmp" 2> "$RAW_DIR/$name-errors.txt"; then
        echo "failed"
        die "lib/smb-load.py failed. See $RAW_DIR/$name-errors.txt"
    fi
    [[ -s $RAW_DIR/$name-errors.txt ]] || rm -f "$RAW_DIR/$name-errors.txt"
    cpu_after="$(smbd_cpu_seconds)"

    {
        cat "$LOAD_OUTPUT.tmp"
        echo "smbd_cpu_seconds         = $(calc3 "$cpu_after - $cpu_before")"
    } > "$LOAD_OUTPUT"
    rm -f "$LOAD_OUTPUT.tmp"
    echo "done, $(value errors) failed"
}

# One value from the last run output.  value <key>  (missing -> "0")
value() {
    awk -v key="$1" '$1 == key { print $3; found = 1 } END { if (!found) print 0 }' "$LOAD_OUTPUT"
}

# A latency value, rounded to 3 decimals (SMB requests often take < 1 ms).
ms() {
    calc3 "$(value "$1")"
}

# The first few error messages of the last run, as a Markdown list.
error_examples() {
    grep '^# error example:' "$LOAD_OUTPUT" | sed 's/^# error example: /- /' || true
}

# Counters of the kernel SMB client (/proc/fs/cifs/Stats) for the test share:
#   "SMBs: <requests sent>", "Reads: <n> total <failed> failed",
#   "Writes: <n> total <failed> failed", and "<n> session <n> share reconnects".
# Printed as one line: requests read_failed write_failed session_reconnects
cifs_counters() {
    SHARE_PATTERN="$CONNECT_ADDRESS\\$SHARE_NAME" awk '
        /^[0-9]+\) /          { in_share = (index($0, ENVIRON["SHARE_PATTERN"]) > 0) }
        / session .* share reconnects/ { reconnects = $1 }
        in_share && $1 == "SMBs:"   { requests = $2 }
        in_share && $1 == "Reads:"  { read_failed = $4 }
        in_share && $1 == "Writes:" { write_failed = $4 }
        END { print requests + 0, read_failed + 0, write_failed + 0, reconnects + 0 }
    ' /proc/fs/cifs/Stats 2>/dev/null || echo "0 0 0 0"
}

counter_differences() {
    local before=($1) after=($2)
    echo "client_smb_requests      = $(( after[0] - before[0] ))"
    echo "client_reads_failed      = $(( after[1] - before[1] ))"
    echo "client_writes_failed     = $(( after[2] - before[2] ))"
    echo "client_reconnects        = $(( after[3] - before[3] ))"
}


# =============================================================================
#  PART 3 - test data and caches
# =============================================================================

# Create data.0 ... data.<SEQ_STREAMS-1> (DATA_FILE_MB each) if they are
# missing. fio only writes what is missing, so this is quick the next time.
ensure_data_files() {
    mkdir -p "$DATA_DIR"
    local wanted_bytes=$(( DATA_FILE_MB * 1024 * 1024 ))
    local stream missing=no
    for (( stream = 0; stream < SEQ_STREAMS; stream++ )); do
        [[ $(stat -c %s "$DATA_DIR/data.$stream" 2>/dev/null || echo 0) -ge $wanted_bytes ]] || missing=yes
    done
    [[ $missing == no ]] && return 0

    log "Creating the test data files ($SEQ_STREAMS x $DATA_FILE_MB MB) on the Samba share"
    fio --name=create-data-files --directory="$DATA_DIR" \
        --filename_format='data.$jobnum' --numjobs="$SEQ_STREAMS" \
        --size="${DATA_FILE_MB}M" --rw=write --bs=1M --create_only=1 \
        > "$RAW_DIR/create-data-files.txt" 2>&1 ||
        die "Could not create the test data files. See $RAW_DIR/create-data-files.txt"
}

# With DROP_CACHES=yes: write all cached data to disk and empty the page
# cache, so that the next read test has to read from disk.
maybe_drop_caches() {
    [[ $DROP_CACHES == yes ]] || return 0
    sync
    echo 3 > /proc/sys/vm/drop_caches
    log "Page cache emptied (DROP_CACHES=yes): reads come from disk"
}

# Sum of the PSS (kB) of the given PIDs.
total_pss_kb() {
    local pid total=0
    for pid in "$@"; do
        total=$(( total + $(process_pss_kb "$pid") ))
    done
    echo "$total"
}

# Memory the kernel could still give to programs, in kB.
memory_available_kb() {
    awk '$1 == "MemAvailable:" { print $2 }' /proc/meminfo
}


# =============================================================================
#  PART 4 - the tests, one function per metric
# =============================================================================

# -----------------------------------------------------------------------------
test_startup() {
    section_title "Startup time  ($STARTUP_ROUNDS restart(s) of the test smbd)"

    local round start_time login_time recovered_time
    local login_ms=() recovery_s=() attempts=()
    local ready_file="$RAW_DIR/.login-waiter-ready"

    for (( round = 1; round <= STARTUP_ROUNDS; round++ )); do
        samba_stop
        local log_lines_before
        log_lines_before="$(dmesg 2>/dev/null | wc -l)"

        # 1) A client that tries to log in every 10 ms is started FIRST
        #    (Python needs a moment to start), then the server.
        local waiter_output="$RAW_DIR/startup-round-$round-login.txt"
        local connection_options=()
        mapfile -t connection_options < <(load_tool_connection_options)
        rm -f "$ready_file"
        python3 "$LOAD_TOOL" wait-login "${connection_options[@]}" --timeout 60 \
            --ready-file "$ready_file" --output "$waiter_output" 2>/dev/null &
        local waiter_pid=$!
        while [[ ! -e $ready_file ]] && kill -0 "$waiter_pid" 2>/dev/null; do
            sleep 0.05
        done

        start_time="$(now_seconds)"
        samba_start
        if ! wait "$waiter_pid"; then
            die "No login to the test smbd within 60 s after the start.
       $(grep '^# error example:' "$waiter_output" | sed 's/^# error example: //')
       Look at: $LOG_DIR/log.smbd"
        fi
        LOAD_OUTPUT="$waiter_output"
        login_time="$(value login_ok_at)"
        attempts+=("$(value attempts)")

        # 2) The mounted client (the kernel's SMB client) notices that its
        #    connection broke, connects again, logs in again and reopens
        #    the share. Until then every program using the mount waits or
        #    gets an error. Try to create a file until it works.
        local probe="$MOUNT_POINT/.startup-probe-$round"
        local deadline
        deadline="$(calc "$(now_seconds) + $RECOVERY_TIMEOUT")"
        until timeout 10 touch "$probe" 2>/dev/null; do
            is_at_most "$(now_seconds)" "$deadline" ||
                die "The mounted client could not create a file within $RECOVERY_TIMEOUT s after the restart.
       Look at: dmesg | grep -i cifs"
            sleep 0.1
        done
        recovered_time="$(now_seconds)"
        rm -f "$probe"

        # What the kernel SMB client said during this round.
        dmesg 2>/dev/null | tail -n +"$(( log_lines_before + 1 ))" | grep -iE 'cifs|smb' \
            > "$RAW_DIR/startup-round-$round-kernel-log.txt" || true

        login_ms+=("$(calc "($login_time - $start_time) * 1000")")
        recovery_s+=("$(calc2 "$recovered_time - $start_time")")
        printf '    restart %d: login works after %s ms, the mounted client works again after %s s\n' \
            "$round" "${login_ms[-1]}" "${recovery_s[-1]}"
    done
    rm -f "$ready_file"

    local average_ms slowest_recovery
    average_ms="$(printf '%s\n' "${login_ms[@]}" | awk '{ s += $1 } END { printf "%.1f", s / NR }')"
    slowest_recovery="$(printf '%s\n' "${recovery_s[@]}" | sort -n | tail -1)"

    record_result "Startup time" "$average_ms ms average" "<= $TARGET_STARTUP_MAX_MS ms" \
        "$(verdict_at_most "$average_ms" "$TARGET_STARTUP_MAX_MS")"
    record_result "Recovery after restart" "$slowest_recovery s (slowest)" \
        "<= $TARGET_RECOVERY_MAX_S s" \
        "$(verdict_at_most "$slowest_recovery" "$TARGET_RECOVERY_MAX_S")"

    add_details <<EOF
## Startup time and recovery after a restart

The test smbd was stopped and started $STARTUP_ROUNDS time(s) while the test share
stayed mounted. Two times were measured from the start command:

- **Login works**: a new client can connect, log in and open the share
  (a client tried every 10 ms while the server started).
- **Mounted client works again**: the kernel SMB client, which was connected
  before the restart, could create a file again. It must first notice that
  its connection broke, then connect and log in again. The Linux SMB client
  tries this every 3 seconds, so up to about 3 s are normal
  (see PERFORMANCE-METRICS.md 4.1).

| Round | Login works (ms) | Login attempts | Mounted client works again (s) |
|------:|-----------------:|---------------:|-------------------------------:|
$(for i in "${!login_ms[@]}"; do printf '| %5d | %16s | %14s | %30s |\n' $(( i + 1 )) "${login_ms[$i]}" "${attempts[$i]}" "${recovery_s[$i]}"; done)

Average time until a login works: **$average_ms ms**.
Slowest time until the mounted client worked again: **$slowest_recovery s**.

Kernel SMB client messages of each round: \`raw/startup-round-*-kernel-log.txt\`

EOF
}

# -----------------------------------------------------------------------------
test_login() {
    section_title "Login  ($WORKERS clients connect, log in and disconnect, again and again)"
    run_load_tool login login --workers "$WORKERS" --duration "$DURATION"

    local rate p95 verdict=PASS
    rate="$(calc "$(value logins_per_s)")"
    p95="$(ms login_p95_ms)"
    is_at_least "$rate" "$TARGET_LOGIN_MIN_PER_S" || verdict=FAIL
    is_at_most  "$p95"  "$TARGET_LOGIN_P95_MAX_MS" || verdict=FAIL
    (( $(value errors) > 0 )) && verdict=FAIL

    # CPU the server spent for one login (new smbd process + authentication).
    local logins
    logins="$(value logins)"
    if (( logins > 0 )); then
        SMBD_CPU_PER_LOGIN_MS="$(calc2 "$(value smbd_cpu_seconds) * 1000 / $logins")"
    fi

    record_result "Login (new connections)" "$rate logins/s, p95 $p95 ms" \
        ">= $TARGET_LOGIN_MIN_PER_S/s, p95 <= $TARGET_LOGIN_P95_MAX_MS ms" "$verdict"

    add_details <<EOF
## Login (new connections per second)

$WORKERS clients opened a new SMB connection, logged in and disconnected again,
as fast as they could, for $DURATION seconds. Every login is the full start of a
client session - what happens when a user opens a mapped drive or a program
connects to the share:

1. TCP connect to $CONNECT_ADDRESS:445 (forwarded to port $SAMBA_PORT); smbd starts a **new process** for the client
2. NEGOTIATE: client and server agree on the SMB dialect (SMB 3.1.1)
3. SESSION SETUP: user name and password are checked (NTLMv2)
4. TREE CONNECT: the share is opened

| Item | Value |
|---|---:|
| **Logins per second** | **$rate** |
| Logins done | $logins |
| Failed logins | $(value logins_failed) |
| Time per login: average / p50 | $(ms login_avg_ms) / $(ms login_p50_ms) ms |
| Time per login: **p95** / p99 | **$p95** / $(ms login_p99_ms) ms |
| Slowest login | $(ms login_max_ms) ms |
| smbd CPU per login | ${SMBD_CPU_PER_LOGIN_MS:-n/a} ms |

$(error_examples)

EOF
}

# -----------------------------------------------------------------------------
test_seqwrite() {
    section_title "Sequential write  ($SEQ_STREAMS stream(s), 1 MiB blocks, $SEQ_QUEUE_DEPTH in flight)"
    mkdir -p "$DATA_DIR"
    run_fio seqwrite --rw=write --bs=1M --iodepth="$SEQ_QUEUE_DEPTH" \
        --numjobs="$SEQ_STREAMS" --filename_format='data.$jobnum' \
        --size="${DATA_FILE_MB}M" --end_fsync=1

    local speed verdict
    speed="$(calc "$(value write_mb_per_s)")"
    verdict="$(verdict_at_least "$speed" "$TARGET_SEQWRITE_MIN_MBPS")"
    (( $(value errors) > 0 )) && verdict=FAIL

    record_result "Sequential write" "$speed MB/s" ">= $TARGET_SEQWRITE_MIN_MBPS MB/s" "$verdict"

    add_details <<EOF
## Sequential write

$SEQ_STREAMS stream(s) wrote ${DATA_FILE_MB} MB files from start to end, again and again,
for $DURATION seconds, in 1 MiB requests with $SEQ_QUEUE_DEPTH requests in flight.
This is like copying a big file (a backup, an ISO image) to the share.

| Item | Value |
|---|---:|
| **Speed** | **$speed MB/s** |
| Same, in network units | $(calc2 "$(value write_mb_per_s) * 8 * 1.048576 / 1000") Gbit/s |
| Data written | $(calc "$(value write_mb_total)") MB |
| Time per 1 MiB request: average / p99 | $(ms write_lat_avg_ms) / $(ms write_lat_p99_ms) ms |
| Failed requests | $(value errors) |
| smbd CPU time during the run | $(value smbd_cpu_seconds) s |

Samba writes the data into the server's page cache and answers; the data
reaches the disk shortly after, or at once when the client asks for a flush
(fio does that at the end, \`--end_fsync\`; \`strict sync = $STRICT_SYNC\`).
Encryption: $ENCRYPTION, signing: $SIGNING.

EOF
}

# -----------------------------------------------------------------------------
test_seqread() {
    section_title "Sequential read  ($SEQ_STREAMS stream(s), 1 MiB blocks, $SEQ_QUEUE_DEPTH in flight)"
    ensure_data_files
    maybe_drop_caches
    run_fio seqread --rw=read --bs=1M --iodepth="$SEQ_QUEUE_DEPTH" \
        --numjobs="$SEQ_STREAMS" --filename_format='data.$jobnum' \
        --size="${DATA_FILE_MB}M"

    local speed verdict
    speed="$(calc "$(value read_mb_per_s)")"
    verdict="$(verdict_at_least "$speed" "$TARGET_SEQREAD_MIN_MBPS")"
    (( $(value errors) > 0 )) && verdict=FAIL

    record_result "Sequential read" "$speed MB/s" ">= $TARGET_SEQREAD_MIN_MBPS MB/s" "$verdict"

    add_details <<EOF
## Sequential read

$SEQ_STREAMS stream(s) read ${DATA_FILE_MB} MB files from start to end, again and again,
for $DURATION seconds, in 1 MiB requests with $SEQ_QUEUE_DEPTH requests in flight.

| Item | Value |
|---|---:|
| **Speed** | **$speed MB/s** |
| Same, in network units | $(calc2 "$(value read_mb_per_s) * 8 * 1.048576 / 1000") Gbit/s |
| Data read | $(calc "$(value read_mb_total)") MB |
| Time per 1 MiB request: average / p99 | $(ms read_lat_avg_ms) / $(ms read_lat_p99_ms) ms |
| Failed requests | $(value errors) |
| Page cache emptied before the test | $DROP_CACHES |

$(if [[ $DROP_CACHES == yes ]]; then
    echo "The page cache was emptied first, so the data came from disk (at least on"
    echo "the first pass through the file)."
  else
    echo "The data was served from the server's memory (page cache), so this is the"
    echo "speed of Samba itself, not of the disk. Run with \`DROP_CACHES=yes\` to include the disk."
  fi)
The test runs over the loopback interface, so no network card limits the
speed; over a real network the link usually decides (10 Gbit/s = about 1180 MB/s).

EOF
}

# -----------------------------------------------------------------------------
test_randread() {
    section_title "Random read  (4 KiB, $RANDOM_JOBS jobs x $RANDOM_QUEUE_DEPTH in flight)"
    ensure_data_files
    maybe_drop_caches
    run_fio randread --rw=randread --bs=4k --iodepth="$RANDOM_QUEUE_DEPTH" \
        --numjobs="$RANDOM_JOBS" --filename=data.0 --size="${DATA_FILE_MB}M"

    local iops verdict
    iops="$(calc "$(value read_iops)")"
    verdict="$(verdict_at_least "$iops" "$TARGET_RANDREAD_MIN_IOPS")"
    (( $(value errors) > 0 )) && verdict=FAIL

    record_result "Random read" "$iops IOPS" ">= $TARGET_RANDREAD_MIN_IOPS IOPS" "$verdict"

    add_details <<EOF
## Random read (IOPS)

$RANDOM_JOBS jobs read 4 KiB blocks at random places in a ${DATA_FILE_MB} MB file, each
job with $RANDOM_QUEUE_DEPTH requests in flight, for $DURATION seconds. Every block is
one SMB READ request, so this shows how many requests per second the server
can handle - like a database or many users opening small parts of files.

| Item | Value |
|---|---:|
| **Reads per second (IOPS)** | **$iops** |
| Data rate | $(calc "$(value read_mb_per_s)") MB/s |
| Time per request: average / p95 / p99 | $(ms read_lat_avg_ms) / $(ms read_lat_p95_ms) / $(ms read_lat_p99_ms) ms |
| Failed requests | $(value errors) |
| SMB requests the client sent | $(value client_smb_requests) |
| Page cache emptied before the test | $DROP_CACHES |

All requests travel over the ONE connection of the test mount, so they are
all handled by ONE smbd process (Samba uses one process per client
connection). Samba spreads the actual file reads over helper threads
(asynchronous I/O), so one process can still use several CPU cores.

EOF
}

# -----------------------------------------------------------------------------
test_randwrite() {
    section_title "Random write  (4 KiB, $RANDOM_JOBS jobs x $RANDOM_QUEUE_DEPTH in flight)"
    ensure_data_files
    run_fio randwrite --rw=randwrite --bs=4k --iodepth="$RANDOM_QUEUE_DEPTH" \
        --numjobs="$RANDOM_JOBS" --filename=data.0 --size="${DATA_FILE_MB}M"

    local iops verdict
    iops="$(calc "$(value write_iops)")"
    verdict="$(verdict_at_least "$iops" "$TARGET_RANDWRITE_MIN_IOPS")"
    (( $(value errors) > 0 )) && verdict=FAIL

    record_result "Random write" "$iops IOPS" ">= $TARGET_RANDWRITE_MIN_IOPS IOPS" "$verdict"

    add_details <<EOF
## Random write (IOPS)

$RANDOM_JOBS jobs wrote 4 KiB blocks at random places in a ${DATA_FILE_MB} MB file, each
job with $RANDOM_QUEUE_DEPTH requests in flight, for $DURATION seconds.

| Item | Value |
|---|---:|
| **Writes per second (IOPS)** | **$iops** |
| Data rate | $(calc "$(value write_mb_per_s)") MB/s |
| Time per request: average / p95 / p99 | $(ms write_lat_avg_ms) / $(ms write_lat_p95_ms) / $(ms write_lat_p99_ms) ms |
| Failed requests | $(value errors) |
| SMB requests the client sent | $(value client_smb_requests) |

The Linux SMB client does not ask for "write-through" here, so Samba answers
when the data is in the server's memory (page cache). A client that asks for
write-through, or a flush after every write (some databases do), makes every
write wait for the disk (PERFORMANCE-METRICS.md 4.6).

EOF
}

# -----------------------------------------------------------------------------
#  run_steady_load: LOAD_RATE requests per second (70 % reads, 30 % writes,
#  4 KiB, one at a time, arriving at random moments like real users).
#  Shared by the "latency" and "errors" tests, so it runs only once.
# -----------------------------------------------------------------------------
run_steady_load() {
    ensure_data_files
    local read_rate=$(( LOAD_RATE * 70 / 100 ))
    local write_rate=$(( LOAD_RATE - read_rate ))
    run_fio "steady-load-${LOAD_RATE}-per-s" --rw=randrw --rwmixread=70 --bs=4k \
        --iodepth=1 --numjobs=1 --filename=data.0 --size="${DATA_FILE_MB}M" \
        --rate_iops="$read_rate,$write_rate" --rate_process=poisson
}

test_latency() {
    section_title "Latency  ($LOAD_RATE requests per second, 70% read / 30% write)"
    run_steady_load

    local verdict=PASS
    is_at_most "$(value read_lat_p95_ms)"  "$TARGET_LATENCY_P95_MAX_MS" || verdict=FAIL
    is_at_most "$(value read_lat_p99_ms)"  "$TARGET_LATENCY_P99_MAX_MS" || verdict=FAIL
    is_at_most "$(value write_lat_p95_ms)" "$TARGET_LATENCY_P95_MAX_MS" || verdict=FAIL
    is_at_most "$(value write_lat_p99_ms)" "$TARGET_LATENCY_P99_MAX_MS" || verdict=FAIL

    record_result "Latency (p95 / p99)" \
        "read $(ms read_lat_p95_ms) / $(ms read_lat_p99_ms) ms, write $(ms write_lat_p95_ms) / $(ms write_lat_p99_ms) ms" \
        "<= $TARGET_LATENCY_P95_MAX_MS / $TARGET_LATENCY_P99_MAX_MS ms" "$verdict"

    add_details <<EOF
## Latency (how long one request takes)

A steady load of $LOAD_RATE requests per second for $DURATION seconds: 70% reads and
30% writes of 4 KiB at random places, one request at a time, arriving at
random moments (like many independent users). The server is far from full,
so this is the waiting time users feel in normal work.
"p95 = 0.5 ms" means 95 of every 100 requests were done within 0.5 ms.

| Statistic | Read (ms) | Write (ms) |
|---|---:|---:|
| Average      | $(ms read_lat_avg_ms) | $(ms write_lat_avg_ms) |
| p50 (median) | $(ms read_lat_p50_ms) | $(ms write_lat_p50_ms) |
| p90          | $(ms read_lat_p90_ms) | $(ms write_lat_p90_ms) |
| p95          | **$(ms read_lat_p95_ms)** | **$(ms write_lat_p95_ms)** |
| p99          | **$(ms read_lat_p99_ms)** | **$(ms write_lat_p99_ms)** |
| p99.9        | $(ms read_lat_p999_ms) | $(ms write_lat_p999_ms) |
| Slowest      | $(ms read_lat_max_ms) | $(ms write_lat_max_ms) |
| Requests     | $(value read_requests) | $(value write_requests) |

Full fio result: \`raw/steady-load-${LOAD_RATE}-per-s.json\`

EOF
}

# -----------------------------------------------------------------------------
test_errors() {
    section_title "Errors  ($LOAD_RATE requests per second)"
    run_steady_load

    local failed_pct
    failed_pct="$(calc3 "$(value error_pct)")"
    local requests=$(( $(value read_requests) + $(value write_requests) ))

    record_result "Errors (failed requests)" "$failed_pct% ($(value errors) of $requests)" \
        "<= $TARGET_ERRORS_MAX_PCT%" \
        "$(verdict_at_most "$failed_pct" "$TARGET_ERRORS_MAX_PCT")"

    add_details <<EOF
## Errors

The same steady load as the latency test ($LOAD_RATE requests per second for
$DURATION seconds). A request that returned an error to the program counts as
failed. The Linux SMB client also counts, per share, the requests the
**server** answered with an error, and how often it had to **reconnect**
(a reconnect means the connection broke, for example because smbd stopped).

| Item | Value |
|---|---:|
| Requests sent by the program (fio) | $requests |
| **Failed requests (seen by the program)** | **$(value errors) ($failed_pct%)** |
| First error code (0 = none) | $(value first_error_code) |
| SMB requests the kernel client sent | $(value client_smb_requests) |
| Reads the server answered with an error | $(value client_reads_failed) |
| Writes the server answered with an error | $(value client_writes_failed) |
| Reconnects of the client (should be 0) | $(value client_reconnects) |

Linux SMB mounts are "soft" by default: when the server does not answer,
the program gets an error after a while (instead of waiting for ever), so
server problems show up here as failed requests.

EOF
}

# -----------------------------------------------------------------------------
test_metadata() {
    section_title "Metadata  ($WORKERS clients create ${SMALL_FILE_KB} KB files, then delete them)"
    run_load_tool metadata files --dir metadata --workers "$WORKERS" \
        --duration "$DURATION" --file-kb "$SMALL_FILE_KB"
    rm -rf "$MOUNT_POINT/metadata"

    local rate verdict
    rate="$(calc "$(value creates_per_s)")"
    verdict="$(verdict_at_least "$rate" "$TARGET_METADATA_MIN_PER_S")"
    (( $(value errors) > 0 )) && verdict=FAIL

    record_result "Small files (metadata)" "$rate files created/s" \
        ">= $TARGET_METADATA_MIN_PER_S files/s" "$verdict"

    add_details <<EOF
## Small files (metadata operations)

$WORKERS clients, each with its own connection, created ${SMALL_FILE_KB} KB files as fast as
they could for $DURATION seconds, then deleted all of them. Every new file costs
three SMB round trips - CREATE, WRITE and CLOSE - and a delete two more. For
each of them smbd must also check the folder for a file with the same name
in other upper/lower case (Windows names ignore case, Linux names do not),
update its lock databases, and let the file system save the new entry.
Workloads with many small files (source code, profiles, home folders) depend
on this much more than on MB/s.

| Item | Create | Delete |
|---|---:|---:|
| **Files per second** | **$rate** | $(calc "$(value deletes_per_s)") |
| Files done | $(value creates) | $(value deletes) |
| Failed | $(value creates_failed) | $(value deletes_failed) |
| Time per file: average | $(ms create_avg_ms) ms | $(ms delete_avg_ms) ms |
| Time per file: p95 | $(ms create_p95_ms) ms | $(ms delete_p95_ms) ms |
| Time per file: p99 | $(ms create_p99_ms) ms | $(ms delete_p99_ms) ms |
| Slowest | $(ms create_max_ms) ms | $(ms delete_max_ms) ms |

smbd CPU time during the whole test: $(value smbd_cpu_seconds) s.

$(error_examples)

EOF
}

# -----------------------------------------------------------------------------
test_concurrency() {
    section_title "Concurrency  (clients reading at the same time: $CONCURRENCY_LEVELS)"
    ensure_data_files
    local clients table="" capacity=0 all_kept_up=yes

    for clients in $CONCURRENCY_LEVELS; do
        run_load_tool "concurrency-${clients}-clients" readers --clients "$clients" \
            --file "$DATA_FILE_ON_SHARE" --file-mb "$DATA_FILE_MB" --duration "$DURATION"

        local iops p99 average failed connected kept_up=yes
        iops="$(calc "$(value iops)")"
        average="$(ms read_avg_ms)"
        p99="$(ms read_p99_ms)"
        failed="$(value errors)"
        connected="$(value clients_connected)"

        # A step "keeps up" when every client got connected, no request
        # failed and 99% of the requests were answered within the target.
        (( failed == 0 && connected == clients )) || kept_up=no
        is_at_most "$p99" "$TARGET_LATENCY_P99_MAX_MS" || kept_up=no

        if [[ $kept_up == yes && $all_kept_up == yes ]]; then
            capacity="$clients"
        else
            all_kept_up=no
        fi

        table+="$(printf '| %7s | %9s | %10s | %9s | %9s | %6s | %-7s |' \
            "$clients" "$connected" "$iops" "$average" "$p99" "$failed" "$kept_up")"$'\n'
    done

    local measured="$capacity clients with p99 <= $TARGET_LATENCY_P99_MAX_MS ms"
    [[ $all_kept_up == yes ]] && measured="every step OK (>= $capacity clients)"
    [[ $capacity == 0 ]] && measured="too slow already at the first step"

    record_result "Concurrency" "$measured" ">= $TARGET_CONCURRENCY_MIN clients" \
        "$(verdict_at_least "$capacity" "$TARGET_CONCURRENCY_MIN")"

    add_details <<EOF
## Concurrency

More and more clients read 4 KiB blocks at random places in the same file at
the same time, $DURATION s per step. **Every client has its own SMB connection,
so the server runs one smbd process per client** - exactly as with real
users on separate PCs. Each client sends one request and waits for the answer
before it sends the next. A step "keeps up" when every client could connect,
no request failed and the p99 time stayed within $TARGET_LATENCY_P99_MAX_MS ms.

| Clients | Connected | Total IOPS | Avg (ms) | p99 (ms) | Failed | Kept up |
|--------:|----------:|-----------:|---------:|---------:|-------:|:--------|
${table}
Result: **$measured**.

- **Total IOPS** grows with the clients until the CPU cores are all busy;
  after that it stays flat and the time per request grows.
- The clients (lib/smb-load.py, Python) run on this machine too and need
  CPU as well; on a machine with few cores they share the CPU with smbd.
$(if [[ $all_kept_up == yes ]]; then
    echo
    echo "The server kept up at every step. To find its real limit, add more clients,"
    echo "for example: CONCURRENCY_LEVELS=\"$CONCURRENCY_LEVELS 256\" ./test-samba.sh concurrency"
  fi)

EOF
}

# -----------------------------------------------------------------------------
test_cpu() {
    section_title "CPU usage of the test smbd"
    ensure_data_files

    # Re-use the randread and seqread runs (they are run now if needed).
    maybe_drop_caches
    run_fio randread --rw=randread --bs=4k --iodepth="$RANDOM_QUEUE_DEPTH" \
        --numjobs="$RANDOM_JOBS" --filename=data.0 --size="${DATA_FILE_MB}M"
    local random_cpu random_requests random_iops
    random_cpu="$(value smbd_cpu_seconds)"
    random_requests="$(value read_requests)"
    random_iops="$(calc "$(value read_iops)")"

    maybe_drop_caches
    run_fio seqread --rw=read --bs=1M --iodepth="$SEQ_QUEUE_DEPTH" \
        --numjobs="$SEQ_STREAMS" --filename_format='data.$jobnum' --size="${DATA_FILE_MB}M"
    local sequential_cpu sequential_gb
    sequential_cpu="$(value smbd_cpu_seconds)"
    sequential_gb="$(calc3 "$(value read_mb_total) / 1024")"

    local us_per_op seconds_per_gb cores_busy
    us_per_op="$(awk -v c="$random_cpu" -v n="$random_requests" 'BEGIN { printf "%.1f", (n > 0 ? c * 1000000 / n : 0) }')"
    seconds_per_gb="$(awk -v c="$sequential_cpu" -v g="$sequential_gb" 'BEGIN { printf "%.3f", (g > 0 ? c / g : 0) }')"
    cores_busy="$(calc2 "$random_cpu / $DURATION")"

    record_result "CPU per request" "$us_per_op us per 4 KiB read ($cores_busy cores busy at $random_iops IOPS)" \
        "<= $TARGET_CPU_MAX_US_PER_OP us" \
        "$(verdict_at_most "$us_per_op" "$TARGET_CPU_MAX_US_PER_OP")"

    local how_measured="CPU time of all processes of the test smbd (main process, helpers and client processes)"
    if has_systemd; then
        how_measured+=", from the control group of $SERVICE_NAME.service (cpu.stat)"
    else
        how_measured+=", from /proc/<pid>/stat"
    fi

    add_details <<EOF
## CPU usage

$how_measured, measured during the random-read run (many small requests)
and the sequential-read run (big requests).

| Item | Value |
|---|---:|
| **smbd CPU per 4 KiB read** | **$us_per_op microseconds** |
| smbd CPU during the random-read run | $random_cpu s in $DURATION s = $cores_busy cores busy |
| Requests in that run | $random_requests ($random_iops per second) |
| smbd CPU per GB read sequentially | $seconds_per_gb s |
| smbd CPU per login (from the "login" test) | ${SMBD_CPU_PER_LOGIN_MS:-not measured (run the login test)} ms |
| CPU cores in this machine | $(nproc) |
| SMB3 encryption / signing | $ENCRYPTION / $SIGNING |

Estimate the CPU a server needs: **cores = requests per second x
microseconds per request / 1,000,000**. Big requests (1 MiB) cost much less
CPU per byte than small ones. Encryption and signing add CPU for every byte.

Not included: the kernel's network stack and the SMB client, which both run
on this machine too in this test.

EOF
}

# -----------------------------------------------------------------------------
test_memory() {
    section_title "Memory usage  ($MEMORY_CONNECTIONS client connections open at the same time)"

    local all_before available_before
    all_before="$(total_pss_kb $(test_server_pids))"
    available_before="$(memory_available_kb)"

    # Open the connections in the background and wait until all are open.
    local hold_output="$RAW_DIR/memory-connections.txt"
    local ready_file="$RAW_DIR/.hold-ready"
    local connection_options=()
    mapfile -t connection_options < <(load_tool_connection_options)
    rm -f "$ready_file"
    python3 "$LOAD_TOOL" hold "${connection_options[@]}" --dir memory-test \
        --connections "$MEMORY_CONNECTIONS" --ready-file "$ready_file" \
        --output "$hold_output" 2> "$RAW_DIR/memory-errors.txt" &
    local hold_pid=$!
    while [[ ! -e $ready_file ]] && kill -0 "$hold_pid" 2>/dev/null; do
        sleep 0.2
    done
    rm -f "$ready_file"
    [[ -s $RAW_DIR/memory-errors.txt ]] || rm -f "$RAW_DIR/memory-errors.txt"
    LOAD_OUTPUT="$hold_output"
    local opened
    opened="$(value connections_open)"
    printf '    %-28s %10s connections open\n' "memory-connections" "$opened"

    sleep 1                                     # let the processes settle
    local all_busy available_busy client_pids client_count client_pss_total client_rss_total pid
    all_busy="$(total_pss_kb $(test_server_pids))"
    available_busy="$(memory_available_kb)"
    mapfile -t client_pids < <(client_process_pids)
    client_count="${#client_pids[@]}"
    client_pss_total="$(total_pss_kb "${client_pids[@]}")"
    client_rss_total=0
    for pid in "${client_pids[@]}"; do
        client_rss_total=$(( client_rss_total + $(process_rss_kb "$pid") ))
    done

    # Close the connections again (the tool deletes its files).
    kill "$hold_pid" 2>/dev/null || true
    wait "$hold_pid" 2>/dev/null || true

    local per_connection_mb rss_per_process_mb available_per_connection_mb
    if (( opened > 0 )); then
        per_connection_mb="$(calc2 "($all_busy - $all_before) / 1024 / $opened")"
        available_per_connection_mb="$(calc2 "($available_before - $available_busy) / 1024 / $opened")"
    else
        per_connection_mb=0; available_per_connection_mb=0
    fi
    if (( client_count > 0 )); then
        rss_per_process_mb="$(calc2 "$client_rss_total / 1024 / $client_count")"
    else
        rss_per_process_mb=0
    fi

    local verdict
    verdict="$(verdict_at_most "$per_connection_mb" "$TARGET_MEMORY_MAX_MB_PER_CONN")"
    (( opened < MEMORY_CONNECTIONS )) && verdict=FAIL

    record_result "Memory per connection" "$per_connection_mb MB ($opened connections: $(calc "$all_busy / 1024") MB smbd in total)" \
        "<= $TARGET_MEMORY_MAX_MB_PER_CONN MB per connection" "$verdict"

    add_details <<EOF
## Memory usage

$opened clients connected at the same time, each with its own connection and
one open file, like users who have a document open on the share. Samba starts
**one smbd process for every client connection**, so memory grows with the
number of connected clients, not with the amount of data.

The memory is counted as **PSS** (proportional set size): memory that several
smbd processes share (program code, libraries, shared databases) is divided
fairly between them, so the values can be added up without counting shared
memory many times.

| Item | Before | $opened connections open |
|---|---:|---:|
| smbd processes serving a client connection | - | $client_count |
| Memory of all test smbd processes (PSS) | $(calc "$all_before / 1024") MB | **$(calc "$all_busy / 1024") MB** |
| Memory the system could still give to programs (MemAvailable) | $(calc "$available_before / 1024") MB | $(calc "$available_busy / 1024") MB |

| Per client connection | Value |
|---|---:|
| **smbd memory (PSS)** | **$per_connection_mb MB** |
| Memory of one client process as \`top\` shows it (RSS, includes shared memory) | $rss_per_process_mb MB |
| Drop of MemAvailable (includes the test client itself; noisy) | $available_per_connection_mb MB |

The smbd process count also includes the test mount and one control
connection of the test tool (2 more than the test connections).

Estimate the memory for many users: **connected clients x MB per connection**.
One PC usually opens one connection to a server, whatever the number of
mapped drives on that server. Busy clients with many open files and locks
use more than this idle case.

$(error_examples)

EOF
}


# =============================================================================
#  PART 5 - preparing, and writing the final report
# =============================================================================

# Stop with exit code 2 ("could not run") and a clear message.
cannot_run() {
    printf '\033[1;31m[ERROR ]\033[0m %s\n' "$*" >&2
    exit 2
}

check_ready_to_test() {
    [[ $EUID -eq 0 ]]              || cannot_run "Please run as root, for example:  sudo $0"
    command -v fio >/dev/null      || cannot_run "'fio' not found. Run ./start-samba.sh first."
    python3 -c 'import samba.samba3.libsmb_samba_internal' 2>/dev/null ||
        cannot_run "Python cannot load Samba's client library (python3-samba). Run ./start-samba.sh first."
    [[ -r $CREDENTIALS_FILE ]]     || cannot_run "No credentials file $CREDENTIALS_FILE. Run ./start-samba.sh first."
    smbd_is_running                || cannot_run "The test smbd is not running. Run ./start-samba.sh first."
    port_forwarding_is_active      || cannot_run "The port forwarding $CONNECT_ADDRESS:445 is missing. Run ./start-samba.sh again."
    login_works                    || cannot_run "A login to the test smbd does not work. Run ./start-samba.sh first."
    test_mount_is_mounted          || cannot_run "The test share is not mounted on $MOUNT_POINT. Run ./start-samba.sh first."
    touch "$MOUNT_POINT/.test-samba-probe" 2>/dev/null ||
        cannot_run "Cannot write to $MOUNT_POINT. Run ./start-samba.sh again."
    rm -f "$MOUNT_POINT/.test-samba-probe"
}

prepare_result_folder() {
    RESULT_DIR="$KIT_DIR/results/$(date +%Y%m%d-%H%M%S)"
    RAW_DIR="$RESULT_DIR/raw"
    DETAILS_FILE="$RAW_DIR/.details.md"
    mkdir -p "$RAW_DIR"
    : > "$DETAILS_FILE"
    ln -sfn "$(basename "$RESULT_DIR")" "$KIT_DIR/results/latest"
}

# Remove what an interrupted earlier run may have left behind.
remove_leftovers() {
    rm -rf "$MOUNT_POINT/metadata" "$MOUNT_POINT/memory-test"
}

write_report() {
    local tests_run="$1" started="$2" finished="$3"
    local report="$RESULT_DIR/report.md"
    local os_name selinux cpu_model memory_total share_fs

    os_name="$(. /etc/os-release && echo "$PRETTY_NAME")"
    selinux="$(getenforce 2>/dev/null || echo 'not available')"
    cpu_model="$(awk -F': ' '/^model name/ { print $2; exit }' /proc/cpuinfo)"
    memory_total="$(awk '/^MemTotal/ { printf "%.1f GB", $2 / 1048576 }' /proc/meminfo)"
    share_fs="$(df --output=source,fstype "$SHARE_DIR" | tail -1 | awk '{ print $1 " (" $2 ")" }')"

    local overall="ALL PASSED"
    (( FAIL_COUNT > 0 )) && overall="$FAIL_COUNT TEST(S) FAILED"

    {
        echo "# Samba Server Performance Test Report"
        echo
        echo "**Result: $overall**"
        echo
        echo "| | |"
        echo "|---|---|"
        echo "| Test started  | $started |"
        echo "| Test finished | $finished |"
        echo "| Host | $(hostname) |"
        echo "| Operating system | $os_name (kernel $(uname -r)) |"
        echo "| CPU | $(nproc) x $cpu_model |"
        echo "| Memory | $memory_total |"
        echo "| Samba server | $(samba_version), own smbd instance on $SAMBA_ADDRESS:$SAMBA_PORT; clients connect to $CONNECT_ADDRESS:445 (forwarded) |"
        echo "| Samba settings | encryption: $ENCRYPTION, signing: $SIGNING, strict sync: $STRICT_SYNC, sendfile: $USE_SENDFILE |"
        echo "| SELinux | $selinux |"
        echo "| Share | \`$SHARE_UNC\` = \`$SHARE_DIR\` on $share_fs |"
        echo "| Test mount | \`$MOUNT_POINT\` (kernel SMB client on the same machine, over 127.0.0.1) |"
        echo "| Mount options | \`$(active_mount_options)\` |"
        echo "| Tests run | $tests_run |"
        echo "| Load per run | $DURATION s; test file $DATA_FILE_MB MB; steady load $LOAD_RATE requests/s; page cache emptied before reads: $DROP_CACHES |"
        echo "| Load generators | $(fio --version); lib/smb-load.py with $(rpm -q python3-samba 2>/dev/null || echo python3-samba) |"
        echo
        echo "## Summary"
        echo
        echo "PASS = target met, FAIL = target missed, INFO = measured only (no target)."
        echo "Targets are set in \`settings.conf\`. What each metric means: \`PERFORMANCE-METRICS.md\`."
        echo
        printf '| %-26s | %-58s | %-32s | %-7s |\n' "Test" "Measured" "Target" "Verdict"
        printf '|%s|%s|%s|%s|\n' "$(printf -- '-%.0s' {1..28})" "$(printf -- '-%.0s' {1..60})" \
                                 "$(printf -- '-%.0s' {1..34})" "$(printf -- '-%.0s' {1..9})"
        local row name measured target verdict
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            [[ $verdict == FAIL ]] && verdict="**FAIL**"
            printf '| %-26s | %-58s | %-32s | %-7s |\n' "$name" "$measured" "$target" "$verdict"
        done
        echo
        if (( ${#GENERATOR_LIMITED_TESTS[@]} > 0 )); then
            echo "> **Note:** fio (the load generator) was at least ${GENERATOR_BUSY_PCT}% busy in:"
            echo "> ${GENERATOR_LIMITED_TESTS[*]}."
            echo "> In those runs the Samba server may be faster than the numbers show."
            echo
        fi
        echo "> **Remember:** client and server run on the same machine and talk over"
        echo "> 127.0.0.1. The results show what the server can do without a network;"
        echo "> real clients also pay the network's delay and speed limit."
        echo
        echo "# Details"
        echo
        cat "$DETAILS_FILE"
        echo "## Files in this folder"
        echo
        echo "- \`report.md\` - this report"
        echo "- \`summary.csv\` - the summary table, for spreadsheets"
        echo "- \`raw/\` - fio's full JSON result of every run, the values taken from each run"
        echo "  (\"key = value\" lines) and the kernel SMB client's messages of the startup test"
    } > "$report"

    # The same summary as CSV.
    {
        echo "test,measured,target,verdict"
        for row in "${SUMMARY_ROWS[@]}"; do
            IFS='|' read -r name measured target verdict <<< "$row"
            printf '"%s","%s","%s","%s"\n' "$name" "$measured" "$target" "$verdict"
        done
    } > "$RESULT_DIR/summary.csv"

    rm -f "$DETAILS_FILE"
}

print_usage() {
    # Print the comment block at the top of this file.
    sed -n '3,/^set -euo pipefail/{/^#/p}' "$0" | sed 's/^# \{0,1\}//'
}


# =============================================================================
#  MAIN
# =============================================================================
main() {
    case "${1:-}" in
        -h|--help) print_usage; exit 0 ;;
        --list)    printf '%s\n' "${ALL_TESTS[@]}"; exit 0 ;;
    esac

    # Which tests to run: the names given, or all of them.
    local tests=("$@")
    if (( ${#tests[@]} == 0 )) || [[ ${tests[0]} == all ]]; then
        tests=("${ALL_TESTS[@]}")
    fi
    local test_name
    for test_name in "${tests[@]}"; do
        if [[ " ${ALL_TESTS[*]} " != *" $test_name "* ]]; then
            echo "Unknown test: '$test_name'. Valid tests: ${ALL_TESTS[*]}" >&2
            exit 2
        fi
    done

    # Test the server as it was started (port, share, Samba options).
    read_settings_of_running_server
    check_ready_to_test
    prepare_result_folder

    # Always stop background helpers if the script is interrupted.
    trap 'kill $(jobs -p) 2>/dev/null || true' EXIT

    local started finished
    started="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    log "Testing the Samba server $SHARE_UNC (smbd on port $SAMBA_PORT) - tests: ${tests[*]}"
    log "Results folder: $RESULT_DIR"
    remove_leftovers

    for test_name in "${tests[@]}"; do
        "test_$test_name"
    done

    finished="$(date '+%Y-%m-%d %H:%M:%S %Z')"
    write_report "${tests[*]}" "$started" "$finished"

    echo
    sed -n '/^## Summary/,/^# Details/p' "$RESULT_DIR/report.md" | sed '$d'
    echo
    ok "Report saved: $RESULT_DIR/report.md"

    # Exit code: 0 when nothing failed, otherwise 1.
    if (( FAIL_COUNT > 0 )); then
        exit 1
    fi
}

main "$@"