#!/usr/bin/env bash
# httpd-findings.sh - turn the traces from httpd-example.sh into conclusions.
# Called by httpd-example.sh; takes OUTDIR HTTPD SRV PORT.
set -uo pipefail
OUTDIR="${1:?}"; HTTPD="${2:?}"; SRV="${3:?}"; PORT="${4:?}"
SP="$OUTDIR/2-singleproc"; FK="$OUTDIR/3-forking"

csv() { [[ -s "$1" ]] && cat "$1" || true; }

# print the fenced code block that follows a given heading in a Markdown file
md_block() {
  awk -v want="$2" '
    index($0, want) { found = 1; next }
    found && /^```/ { infence = !infence; if (!infence) exit; next }
    found && infence { print }
  ' "$1"
}
# column 1 of files.csv, filtered by a regex on the access column
files_where() { python3 - "$1" "$2" <<'PY'
import csv, re, sys
path, pattern = sys.argv[1], sys.argv[2]
try:
    rows = list(csv.DictReader(open(path, newline="")))
except OSError:
    sys.exit(0)
for r in rows:
    if re.search(pattern, r["access"]):
        print(r["path"])
PY
}

echo "# Worked example: what does \`httpd\` touch?"
echo
echo "Generated $(date -Is) on $(hostname) - $( . /etc/os-release && echo "$PRETTY_NAME"), kernel $(uname -r)"
echo "SELinux: $(getenforce 2>/dev/null || echo n/a). Analysed binary: \`$HTTPD\`"
echo "($(rpm -qf "$HTTPD" 2>/dev/null || echo 'not from a package'))."
echo
echo "A private instance was started on 127.0.0.1:$PORT with its own ServerRoot,"
echo "so any system httpd kept running untouched."
echo
echo "## The headline: ldd tells you almost nothing about httpd"
echo
DECL=$(ldd "$HTTPD" 2>/dev/null | grep -c '=>' || echo 0)
MODS=$(files_where "$SP/files.csv" '.' | grep -cE '/httpd/modules/.*\.so$' || echo 0)
echo "\`ldd $HTTPD\` lists **$DECL** shared libraries. During one startup the process"
echo "actually loaded **$MODS** files from \`/usr/lib64/httpd/modules/\` - every one of"
echo "them opened by \`dlopen()\` after the loader had finished, so no amount of static"
echo "inspection would have listed them."
echo
echo "Modules loaded at runtime:"
echo '```'
files_where "$SP/files.csv" '.' | grep -E '/httpd/modules/.*\.so$' | sort || echo "(none captured)"
echo '```'
echo
echo "> This is the general lesson, not an httpd quirk. Any plugin host - httpd,"
echo "> nginx with dynamic modules, PAM, NSS, a JVM agent - hides its real code"
echo "> surface behind dlopen. Trace it or you are guessing."
echo
echo "## Configuration it read"
echo
echo '```'
files_where "$SP/files.csv" 'read|open|stat' | grep -E '\.conf$|/etc/httpd|mime\.types|magic$' | sort -u || true
echo '```'
echo
echo "## Files it created or wrote"
echo
echo '```'
files_where "$SP/files.csv" 'write|create|delete' | sort -u || true
echo '```'
echo
echo "The log files are the ones that matter operationally: httpd holds them open"
echo "for the lifetime of the process, which is why log rotation needs a reload."
echo
echo "## Content it served"
echo
echo "The request loop read these from DocumentRoot - this is how you prove which"
echo "content a worker actually touched:"
echo
echo '```'
files_where "$SP/files.csv" '.' | grep -E "$SRV/(htdocs|cgi-bin)" | sort -u || echo "(none)"
echo '```'
echo
echo "## Paths probed but absent"
echo
echo "Where httpd *would* have read configuration or content had it existed:"
echo
echo '```'
python3 - "$SP/files.csv" <<'PY' || true
import csv, sys
try:
    rows = list(csv.DictReader(open(sys.argv[1], newline="")))
except OSError:
    sys.exit(0)
for r in rows:
    if "ENOENT" in (r["errors"] or ""):
        print(r["path"])
PY
echo '```'
echo
echo "## Processes: the forking MPM"
echo
echo "Single-process mode (\`httpd -X\`) is convenient for reading a trace, but it is"
echo "not how httpd runs. Under the event MPM the parent forks a pool of workers:"
echo
echo '```'
sed -n '/## 2. Process tree/,/^###/p' "$FK/REPORT.md" 2>/dev/null | sed -n '/```/,/```/p' | sed '1d;$d' || echo "(no forking trace)"
echo '```'
echo
WORKERS=$(python3 - "$FK/processes.csv" <<'PY' || echo 0
import csv, sys
try:
    rows = list(csv.DictReader(open(sys.argv[1], newline="")))
except OSError:
    print(0); sys.exit(0)
print(sum(1 for r in rows if r.get("parent") and r.get("kind", "process") != "thread"))
PY
)
THREADS=$(python3 - "$FK/processes.csv" <<'PY' || echo 0
import csv, sys
try:
    rows = list(csv.DictReader(open(sys.argv[1], newline="")))
except OSError:
    print(0); sys.exit(0)
print(sum(1 for r in rows if r.get("kind") == "thread"))
PY
)
echo "**$WORKERS** child process(es) and **$THREADS** thread(s) were created in that"
echo "window. The distinction matters: the event MPM runs a small number of worker"
echo "*processes*, each with a pool of *threads*, and strace reports a thread as just"
echo "another pid. A tool that does not read the CLONE_THREAD flag will tell you httpd"
echo "forked dozens of processes, which is wrong - \`parse_strace.py\` separates them."
echo "Trace httpd without \`strace -f\` and you see the parent doing almost nothing"
echo "while every request is served by a child you never recorded. \`elf-trace.sh\`"
echo "always passes \`-f\`, which is the whole reason it finds them."
echo
echo "### The CGI child"
echo
echo "A CGI request makes httpd exec a program - the clearest case of a daemon"
echo "running code that is not its own:"
echo
echo '```'
python3 - "$HTTPD" "$FK/processes.csv" "$SP/processes.csv" <<'PY' || true
import csv, os, sys
httpd = os.path.realpath(sys.argv[1])
seen = {}
for path in sys.argv[2:]:
    try:
        rows = list(csv.DictReader(open(path, newline="")))
    except OSError:
        continue
    for r in rows:
        for img in (r.get("all_images") or r.get("image") or "").split(";"):
            # compare against the real binary - a substring test on "httpd"
            # would also swallow anything living under an httpd-named directory
            if not img or os.path.realpath(img) == httpd or os.path.basename(img) == "httpd":
                continue
            seen.setdefault(img, []).append(r["pid"])
for img, pids in sorted(seen.items()):
    print("%-58s exec'd %d time(s) (pids %s)"
          % (img, len(pids), ", ".join(pids[:4]) + ("..." if len(pids) > 4 else "")))
if not seen:
    print("(no non-httpd image was exec'd in this window)")
PY
echo '```'
echo
echo "Note what the trace records as the image: the **CGI script itself**, not"
echo "\`/bin/sh\`. httpd calls \`execve(\"whoami.cgi\")\` and the kernel resolves the"
echo "\`#!\` line, so the shell never appears as a separate exec. The script's own"
echo "children (\`id\`, \`cat\`) do. If you are hunting for "what did this daemon run","
echo "that indirection is exactly the kind of thing a trace catches and a config"
echo "review does not."
echo
echo "## Sockets"
echo
echo '```'
python3 - "$SP/network.csv" "$FK/network.csv" <<'PY' || true
import csv, collections, re, sys
listen, accepts, out = [], 0, collections.Counter()
for path in sys.argv[1:]:
    try:
        rows = list(csv.DictReader(open(path, newline="")))
    except OSError:
        continue
    for r in rows:
        call, detail = r["call"], r["detail"]
        if call == "bind" and "AF_INET" in detail:
            m = re.search(r'htons\((\d+)\).*?inet_addr\("([^"]+)"\)', detail)
            if m and "%s:%s" % (m.group(2), m.group(1)) not in listen:
                listen.append("%s:%s" % (m.group(2), m.group(1)))
        elif call in ("accept", "accept4"):
            accepts += 1
        elif call == "connect":
            m = re.search(r'sun_path="([^"]+)"', detail)
            if m:
                out["unix:%s  (%s)" % (m.group(1), r["result"])] += 1
            elif "AF_INET" in detail:
                m2 = re.search(r'inet_addr\("([^"]+)"\)', detail)
                out["inet:%s  (%s)" % (m2.group(1) if m2 else "?", r["result"])] += 1
print("listening on  : %s" % (", ".join(listen) or "(none captured)"))
print("connections accepted: %d" % accepts)
print("")
print("outbound connects, grouped:")
for k, n in out.most_common():
    print("  %-62s x%d" % (k, n))
PY
echo '```'
echo
echo "The listen socket and the accept count are the obvious part. The interesting"
echo "part is the outbound list: httpd itself never dials out, but every request"
echo "that needs a user or group lookup makes the C library try \`nscd\` and then"
echo "SSSD. Those are failed connects to sockets that do not exist here - harmless,"
echo "but on a host where they *do* exist, your web server's request path depends on"
echo "a name-service daemon you might not have thought of as part of it."
echo
echo "## Watching it as a daemon, without ptrace"
echo
if [[ -f "$OUTDIR/4-snapshot/SNAPSHOT.md" ]]; then
  echo "\`elf-snapshot.sh\` only reads /proc, so it is safe against a production"
  echo "daemon. What the parent held open:"
  echo
  echo '```'
  md_block "$OUTDIR/4-snapshot/SNAPSHOT.md" "Files open with write access" | head -20
  echo '```'
  echo
  echo "Mapped modules seen from /proc/PID/maps (the dlopen list again, from a"
  echo "completely different angle - useful when you may not trace at all):"
  echo
  echo '```'
  md_block "$OUTDIR/4-snapshot/SNAPSHOT.md" "Memory-mapped files" \
    | grep -E 'httpd/modules' | sort -u | head -15
  echo '```'
  echo
  echo "Those are the same modules the trace found, recovered from"
  echo "\`/proc/PID/maps\` without touching the process at all. On a host where"
  echo "you may not ptrace a production daemon, this is how you still answer"
  echo "\"what code is actually loaded in there\"."
else
  echo "_(the daemon snapshot step did not run)_"
fi
echo
echo "## Applying this to a system httpd"
echo
cat <<'TXT'
```bash
# 1. static first, it never runs anything
tools/elf-static.sh /usr/sbin/httpd > httpd-static.md

# 2. the system unit, traced from the start (stop it first)
systemctl stop httpd
tools/elf-trace.sh -T 20 -- /usr/sbin/httpd -DFOREGROUND

# 3. or attach to the running service without stopping it
tools/elf-trace.sh -T 30 -p "$(systemctl show -p MainPID --value httpd)"

# 4. or watch it with zero impact for as long as you like
tools/elf-audit.sh start /usr/sbin/httpd httpdwatch
tools/elf-audit.sh report httpdwatch
tools/elf-audit.sh stop httpdwatch
```

Three things about the *systemd* httpd that the trace will show and that
surprise people:

* **PrivateTmp=yes.** httpd's `/tmp` is a per-service namespace. A path logged
  as `/tmp/x` really lives in `/tmp/systemd-private-*-httpd.service-*/tmp/x`.
  Check with `systemctl show httpd -p PrivateTmp` and look inside with
  `nsenter -t "$MAINPID" -m ls /tmp`.
* **SELinux.** On an Enforcing host, content outside the `httpd_sys_content_t`
  label gives `EACCES` in the trace with nothing else to explain it. Confirm
  with `ausearch -m AVC -ts recent | audit2why` before blaming the config.
* **Workers change identity.** The parent starts as root and binds the port;
  the children `setuid` to `apache`. A file the parent can read may be denied
  to the worker, and only the worker's pid shows the `EACCES`.
TXT
echo
echo "## Reproducing this report"
echo
echo '```bash'
echo "examples/httpd-example.sh -o $OUTDIR -p $PORT"
echo '```'
echo
echo "Raw evidence: \`2-singleproc/strace.log\`, \`3-forking/strace.log\`."