#!/usr/bin/env python3
"""
fetch.py -- build an offline mirror of https://www.redhat.com/en/topics/

Run this on a host that HAS internet access.  It writes everything under
./site/, which ./serve.py then serves on the air-gapped machine.

Python standard library only -- no pip installs, nothing to vendor.

Why not wget:
  * Red Hat's Drupal aggregates CSS/JS behind ~530-character query strings.
    wget turns those into filenames and blows past the 255-byte limit.
    The basename already carries a content hash, so we simply drop the query.
  * Pages load Red Hat Design System components as ES modules through a
    <script type="importmap"> that maps "@rhds/elements/" to a local dist
    path.  Bare specifiers only resolve through that map, so the crawl has
    to read the map out of the HTML and feed it back into the JS walk.
  * <rh-icon icon="x" set="y"> loads its glyph with a runtime
    import(`../icons/${set}/${icon}.js`) that no static crawl can see.
"""

import argparse
import gzip
import io
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import zlib
from collections import deque

ROOT = os.path.dirname(os.path.abspath(__file__))
SITE = os.path.join(ROOT, "site")
LOGS = os.path.join(ROOT, "logs")

SITE_HOST = "www.redhat.com"
EXT_HOSTS = {"static.redhat.com"}        # third-party asset hosts we do mirror
EXT_DIR = "_ext"                         # ...stored under site/_ext/<host>/
PAGE_ROOT = "/en/topics"                 # everything at or below this is a page
START_URL = "https://www.redhat.com/en/topics"
SITEMAP_URL = "https://www.redhat.com/en/sitemap.xml"

# The RHDS element dist root, needed to synthesise <rh-icon> module URLs.
RHDS_DIST = "/modules/contrib/red_hat_shared_libs/dist/rhds-elements"

UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/128.0 Safari/537.36")

# Analytics / consent / marketing tags.  Offline these never resolve and the
# page sits there waiting on DNS, so the tags are cut out of the HTML.
TRACKER_RE = re.compile(r"""(
      /dtm\.js
    | trustarc | trustecm | consent\.truste
    | rh\.mktg\.js
    | googletagmanager | google-analytics | /gtag/ | doubleclick | adservice
    | hotjar | munchkin | marketo | 6sense | demandbase | bizible | omtrdc
    | connect\.facebook | linkedin\.com/px | snap\.licdn | analytics\.twitter
    | youtube\.com/iframe_api | player\.vimeo
)""", re.I | re.X)

ASSET_LINK_RELS = {
    "stylesheet", "preload", "modulepreload", "prefetch", "icon",
    "shortcut icon", "apple-touch-icon", "apple-touch-icon-precomposed",
    "manifest", "mask-icon",
}
ASSET_TAGS = {"script", "img", "source", "video", "audio", "embed", "track",
              "object", "use", "image", "input"}
ASSET_ATTRS = {"src", "srcset", "poster", "data-src", "data-srcset",
               "xlink:href", "data"}
REWRITE_TAGS = ASSET_TAGS | {"a", "area", "link"}

SKIP_SCHEMES = ("data:", "mailto:", "tel:", "javascript:", "blob:", "#", "about:")

# --------------------------------------------------------------------------
# small helpers
# --------------------------------------------------------------------------

def unescape_attr(value):
    """HTML-unescape an attribute value (&amp; shows up in aggregated URLs)."""
    return (value.replace("&amp;", "&").replace("&#38;", "&")
                 .replace("&quot;", '"').replace("&#39;", "'")
                 .replace("&lt;", "<").replace("&gt;", ">"))


def canonical(url):
    """One spelling per resource.

    Pages link to themselves as both http:// and https://, with and without a
    trailing slash.  Without this, every page gets crawled twice and the two
    copies fight over the same file on disk.
    """
    p = urllib.parse.urlsplit(url)
    scheme = "https" if p.scheme in ("http", "https") else p.scheme
    host = p.netloc.lower()
    path = p.path
    if (host == SITE_HOST or host in EXT_HOSTS) and path != "/":
        path = path.rstrip("/")
    return urllib.parse.urlunsplit((scheme, host, path, p.query, ""))


def norm_url(raw, base):
    """Absolutise, drop the fragment, return None for things we never fetch."""
    if not raw:
        return None
    raw = unescape_attr(raw.strip())
    if not raw or raw.startswith(SKIP_SCHEMES):
        return None
    # Red Hat's CMS occasionally emits an unresolved link token, e.g.
    # /en/topics/ai/[[url-nid:741691;title:'...']] -- not a URL.
    if "[[" in raw or any(c.isspace() for c in raw):
        return None
    # url(%23clip0_232_19576) -- a percent-encoded "#", i.e. an in-document
    # SVG fragment reference, not something to fetch
    if raw.lower().startswith("%23"):
        return None
    try:
        absolute = urllib.parse.urljoin(base, raw)
    except ValueError:
        return None
    absolute, _ = urllib.parse.urldefrag(absolute)
    parts = urllib.parse.urlsplit(absolute)
    if parts.scheme not in ("http", "https"):
        return None
    return canonical(absolute)


def is_page(url):
    """True for the topic hub itself and anything below it."""
    p = urllib.parse.urlsplit(url)
    if p.netloc != SITE_HOST:
        return False
    path = p.path.rstrip("/") or "/"
    return path == PAGE_ROOT or path.startswith(PAGE_ROOT + "/")


def mirrored_host(url):
    p = urllib.parse.urlsplit(url)
    return p.netloc == SITE_HOST or p.netloc in EXT_HOSTS


def local_href(url):
    """The URL as the offline server should see it (root-relative, no query)."""
    p = urllib.parse.urlsplit(url)
    if is_page(url):
        return (p.path.rstrip("/") or PAGE_ROOT) + "/"
    if p.netloc == SITE_HOST:
        return p.path
    return "/%s/%s%s" % (EXT_DIR, p.netloc, p.path)


def local_path(url):
    """Where the bytes land on disk, relative to site/.

    Percent-escapes are decoded here.  The server decodes a request path
    before it looks on disk, so storing "IBM%20RedHat.svg" verbatim would
    make the browser ask for "IBM RedHat.svg" and miss.  Decoding is done
    per segment, so an encoded %2F cannot invent a directory.
    """
    href = local_href(url)
    rel = href.lstrip("/")
    if href.endswith("/"):
        rel += "index.html"
    segs = []
    for seg in rel.split("/"):
        seg = urllib.parse.unquote(seg)
        segs.append(seg.replace("/", "_").replace("\\", "_"))
    return os.sep.join(segs)


def guess_kind(url, content_type):
    ct = (content_type or "").split(";")[0].strip().lower()
    if ct in ("text/html", "application/xhtml+xml"):
        return "html"
    if ct == "text/css":
        return "css"
    if ct in ("text/javascript", "application/javascript",
              "application/x-javascript", "module"):
        return "js"
    path = urllib.parse.urlsplit(url).path.lower()
    if path.endswith(".css"):
        return "css"
    if path.endswith((".js", ".mjs")):
        return "js"
    if path.endswith((".html", ".htm")):
        return "html"
    return "other"


# --------------------------------------------------------------------------
# HTTP
# --------------------------------------------------------------------------

class Downloader:
    def __init__(self, delay, timeout, retries):
        self.delay = delay
        self.timeout = timeout
        self.retries = retries
        self.last = 0.0
        self.count = 0
        self.bytes = 0

    def get(self, url):
        """Return (body_bytes, content_type) or raise the final error."""
        last_err = None
        for attempt in range(self.attempts()):
            wait = self.delay - (time.time() - self.last)
            if wait > 0:
                time.sleep(wait)
            self.last = time.time()
            req = urllib.request.Request(url, headers={
                "User-Agent": UA,
                "Accept": "*/*",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "en-US,en;q=0.9",
            })
            try:
                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
                    raw = resp.read()
                    enc = (resp.headers.get("Content-Encoding") or "").lower()
                    if enc == "gzip":
                        raw = gzip.GzipFile(fileobj=io.BytesIO(raw)).read()
                    elif enc == "deflate":
                        raw = zlib.decompress(raw, -zlib.MAX_WBITS)
                    self.count += 1
                    self.bytes += len(raw)
                    return raw, resp.headers.get("Content-Type", "")
            except urllib.error.HTTPError as exc:
                # 404/410 will not get better by trying again.
                if exc.code in (404, 403, 410):
                    raise
                last_err = exc
            except Exception as exc:                      # noqa: BLE001
                last_err = exc
            time.sleep(min(2 ** attempt, 15))
        raise last_err

    def attempts(self):
        return max(1, self.retries)


# --------------------------------------------------------------------------
# the crawler
# --------------------------------------------------------------------------

class Mirror:
    def __init__(self, args):
        self.args = args
        self.dl = Downloader(args.delay, args.timeout, args.retries)
        self.pages = deque()
        self.assets = deque()
        self.seen = set()
        self.written = {}          # local path -> url that wrote it
        self.importmap = {}        # bare prefix -> path prefix
        self.failures = []
        self.probes = set()      # speculative <rh-*> package guesses
        self.probe_misses = 0
        self.page_count = 0
        self.asset_count = 0
        self.skipped = 0
        self.resumed = 0
        self.dead_links = []

    # -- queueing ---------------------------------------------------------

    def enqueue(self, url, as_page=False):
        if not url or not mirrored_host(url):
            return
        if TRACKER_RE.search(url):
            return
        url = canonical(url)
        key = urllib.parse.urlsplit(url)._replace(query="", fragment="").geturl()
        if key in self.seen:
            return
        self.seen.add(key)
        if as_page or is_page(url):
            self.pages.append(url)
        else:
            self.assets.append(url)

    # -- disk -------------------------------------------------------------

    def write(self, url, data, kind):
        rel = local_path(url)
        dest = os.path.join(SITE, rel)
        prev = self.written.get(rel)
        if prev and prev != url:
            self.log("collision: %s and %s both map to %s" % (prev, url, rel))
        os.makedirs(os.path.dirname(dest), exist_ok=True)
        with open(dest, "wb") as fh:
            fh.write(data)
        self.written[rel] = url
        if self.args.verbose:
            print("    -> site/%s (%s, %d bytes)" % (rel, kind, len(data)))

    def log(self, message):
        print(message)
        self.failures.append(message)

    # -- extraction -------------------------------------------------------

    def read_importmap(self, text):
        for m in re.finditer(r'<script[^>]*type=["\']importmap["\'][^>]*>(.*?)</script>',
                             text, re.S | re.I):
            try:
                data = json.loads(m.group(1))
            except ValueError:
                continue
            for bare, target in (data.get("imports") or {}).items():
                if bare not in self.importmap:
                    self.importmap[bare] = target

    def resolve_bare(self, spec, base):
        """Resolve a bare module specifier through the page's import map."""
        best = None
        for prefix in self.importmap:
            if spec.startswith(prefix) and (best is None or len(prefix) > len(best)):
                best = prefix
        if best is None:
            return None
        target = self.importmap[best] + spec[len(best):]
        return norm_url(target, base)

    def js_imports(self, text, base):
        """Static import / re-export specifiers plus literal dynamic imports."""
        out = []
        patterns = (
            r'\bimport\s+[^\'"()]*?from\s*[\'"]([^\'"]+)[\'"]',
            r'\bimport\s*[\'"]([^\'"]+)[\'"]',
            r'\bexport\s+[^\'"()]*?from\s*[\'"]([^\'"]+)[\'"]',
            r'\bimport\s*\(\s*[\'"]([^\'"${}]+)[\'"]\s*\)',
        )
        for pat in patterns:
            for m in re.finditer(pat, text):
                spec = m.group(1)
                if spec.startswith((".", "/")) or "://" in spec:
                    url = norm_url(spec, base)
                else:
                    url = self.resolve_bare(spec, base)
                if url:
                    out.append(url)
        return out

    def css_refs(self, text, base):
        out = []
        for m in re.finditer(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)', text):
            url = norm_url(m.group(1), base)
            if url:
                out.append(url)
        for m in re.finditer(r'@import\s+[\'"]([^\'"]+)[\'"]', text):
            url = norm_url(m.group(1), base)
            if url:
                out.append(url)
        return out

    def rhds_component_urls(self, text, base):
        """<rh-foo> -> the element module; <rh-icon icon= set=> -> the glyph."""
        out = []
        for tag in set(re.findall(r'<(rh-[a-z0-9-]+)[\s>]', text, re.I)):
            tag = tag.lower()
            # rh-footer-block and friends ship inside the rh-footer package,
            # so fall back to progressively shorter package names.
            parts = tag.split("-")
            for cut in range(len(parts), 1, -1):
                pkg = "-".join(parts[:cut])
                url = self.resolve_bare("@rhds/elements/%s/%s.js" % (pkg, pkg), base)
                if url:
                    out.append(("component", url))
        for m in re.finditer(r'<rh-icon\b([^>]*)>', text, re.I):
            attrs = m.group(1)
            icon = re.search(r'\bicon=["\']([^"\']+)', attrs)
            iset = re.search(r'\bset=["\']([^"\']+)', attrs)
            if icon and iset:
                url = "https://%s%s/icons/%s/%s.js" % (
                    SITE_HOST, RHDS_DIST, iset.group(1), icon.group(1))
                out.append(("icon", url))
        return out

    # -- rewriting --------------------------------------------------------

    def map_page(self, raw, base):
        """<a href>: rewrite only links that stay inside the topic tree.

        Everything else (products, blog, other languages) keeps the URL it
        had.  Root-relative ones land on serve.py's "not mirrored" page;
        absolute ones still work if the browser happens to be online.
        """
        url = norm_url(raw, base)
        if url is None or not is_page(url):
            return raw
        self.enqueue(url)
        return local_href(url)

    def map_asset(self, raw, base):
        url = norm_url(raw, base)
        if url is None or TRACKER_RE.search(url) or not mirrored_host(url):
            return raw
        self.enqueue(url)
        return local_href(url)

    def map_srcset(self, value, base):
        parts = []
        for item in value.split(","):
            item = item.strip()
            if not item:
                continue
            bits = item.split(None, 1)
            new = self.map_asset(bits[0], base)
            parts.append(new if len(bits) == 1 else "%s %s" % (new, bits[1]))
        return ", ".join(parts)

    def rewrite_attrs(self, tag, attrs, base):
        """Rewrite the URL-valued attributes of one start tag."""
        rel = ""
        m = re.search(r'\brel\s*=\s*["\']([^"\']+)', attrs, re.I)
        if m:
            rel = m.group(1).strip().lower()

        def one(m):
            name, quote, value = m.group(1).lower(), m.group(2), m.group(3)
            if tag in ("a", "area") and name == "href":
                return '%s=%s%s%s' % (m.group(1), quote,
                                      self.map_page(value, base), quote)
            if tag == "link" and name == "href":
                if rel in ASSET_LINK_RELS:
                    return '%s=%s%s%s' % (m.group(1), quote,
                                          self.map_asset(value, base), quote)
                return m.group(0)          # canonical, alternate, ... left alone
            if tag in ASSET_TAGS and name in ASSET_ATTRS:
                if name in ("srcset", "data-srcset"):
                    return '%s=%s%s%s' % (m.group(1), quote,
                                          self.map_srcset(value, base), quote)
                return '%s=%s%s%s' % (m.group(1), quote,
                                      self.map_asset(value, base), quote)
            return m.group(0)

        return re.sub(r'([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*(["\'])(.*?)\2',
                      one, attrs, flags=re.S)

    def strip_trackers(self, text):
        """Cut out everything that can only reach for the network.

        Offline these hosts never resolve, so the page stalls on DNS until the
        browser gives up -- the primary nav and the consent banner are the
        visible casualties.  Three kinds go:

          * analytics / consent / marketing tags (Adobe DTM, TrustArc, ...);
          * any script or stylesheet on a host this mirror does not hold,
            which offline can only hang (one topic page pulls jQuery and
            Font Awesome from third-party CDNs);
          * every preconnect / dns-prefetch hint, including ones for hosts we
            do mirror -- their assets are served from here now.

        Inline snippets are left alone: they only push onto arrays and guard
        their own globals, so they are harmless without the network.
        """
        def kill_script(m):
            tag = m.group(0)
            if TRACKER_RE.search(tag):
                return ""
            src = re.search(r'\bsrc=["\']([^"\']+)', tag, re.I)
            if src:
                url = norm_url(src.group(1), START_URL)
                if url and not mirrored_host(url):
                    return ""
            return tag
        text = re.sub(r'<script\b[^>]*\bsrc=["\'][^"\']+["\'][^>]*>\s*</script>',
                      kill_script, text, flags=re.I)

        def kill_link(m):
            tag = m.group(0)
            if TRACKER_RE.search(tag):
                return ""
            rel = re.search(r'rel=["\']([^"\']+)', tag, re.I)
            rel = rel.group(1).strip().lower() if rel else ""
            if rel in ("preconnect", "dns-prefetch"):
                return ""
            if rel in ASSET_LINK_RELS:
                href = re.search(r'href=["\']([^"\']+)', tag, re.I)
                if href:
                    url = norm_url(href.group(1), START_URL)
                    if url and not mirrored_host(url):
                        return ""
            return tag
        text = re.sub(r'<link\b[^>]*>', kill_link, text, flags=re.I)

        # speculative prerender of /en, which is not part of the mirror
        text = re.sub(r'<script[^>]*type=["\']speculationrules["\'][^>]*>.*?</script>',
                      "", text, flags=re.S | re.I)
        return text

    def rewrite_html(self, text, base):
        text = self.strip_trackers(text)

        # Park <script>/<style> bodies so the tag regex below cannot trip over
        # comparisons like `a<b` inside JavaScript.
        blocks = []

        def stash(m):
            blocks.append(m.group(0))
            return "\x00BLK%d\x00" % (len(blocks) - 1)

        text = re.sub(r'<(script|style)\b[^>]*>.*?</\1\s*>', stash, text,
                      flags=re.S | re.I)

        def tag_sub(m):
            closing, tag, attrs = m.group(1), m.group(2).lower(), m.group(3)
            if closing or not attrs:
                return m.group(0)
            if tag not in REWRITE_TAGS:
                return m.group(0)
            return "<%s%s>" % (m.group(2), self.rewrite_attrs(tag, attrs, base))

        text = re.sub(
            r'<(/?)([a-zA-Z][a-zA-Z0-9:-]*)((?:[^>"\']|"[^"]*"|\'[^\']*\')*)>',
            tag_sub, text)

        # Put the script/style blocks back, rewritten in their own right.
        def restore(m):
            return self.rewrite_block(blocks[int(m.group(1))], base)

        return re.sub(r'\x00BLK(\d+)\x00', restore, text)

    def rewrite_block(self, block, base):
        """Handle one stashed <script> or <style> element."""
        m = re.match(r'<(script|style)\b([^>]*)>(.*)(</\1\s*>)$', block,
                     re.S | re.I)
        if not m:
            return block
        tag, attrs, body, close = m.group(1).lower(), m.group(2), m.group(3), m.group(4)

        if tag == "style":
            for ref in self.css_refs(body, base):
                self.enqueue(ref)
            return "<style%s>%s%s" % (attrs, self.rewrite_css(body, base), close)

        new_attrs = attrs
        src = re.search(r'\bsrc\s*=\s*(["\'])(.*?)\1', attrs, re.S)
        if src:
            new_attrs = (attrs[:src.start(2)] + self.map_asset(src.group(2), base)
                         + attrs[src.end(2):])
        elif re.search(r'type\s*=\s*["\']module["\']', attrs, re.I):
            # inline module, e.g. import '@rhds/elements/rh-button/rh-button.js'
            for ref in self.js_imports(body, base):
                self.enqueue(ref)
        return "<script%s>%s%s" % (new_attrs, body, close)

    def rewrite_css(self, text, base):
        def sub(m):
            return "url(%s)" % self.map_asset(m.group(1), base)
        text = re.sub(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)', sub, text)

        def imp(m):
            return '@import %s%s%s' % (m.group(1),
                                       self.map_asset(m.group(2), base),
                                       m.group(1))
        return re.sub(r'@import\s+(["\'])([^"\']+)\1', imp, text)

    def rewrite_js(self, text, base):
        """Only cross-host absolute URLs need touching; relative ones already
        line up, because the mirror keeps the site's directory layout."""
        def sub(m):
            quote, value = m.group(1), m.group(2)
            url = norm_url(value, base)
            if url and mirrored_host(url) and not is_page(url):
                self.enqueue(url)
                return "%s%s%s" % (quote, local_href(url), quote)
            return m.group(0)
        return re.sub(r'(["\'])(https?://[^"\'\s]+)\1', sub, text)

    # -- processing -------------------------------------------------------

    def have(self, url):
        """--resume: is this asset already on disk?

        Only ever asked about assets.  Pages are always re-fetched, because
        the stored copy has been rewritten -- its asset URLs have lost the
        query strings, and Drupal's private files 404 without their ?itok=
        token.  Re-reading a rewritten page would therefore mirror a set of
        broken image URLs.
        """
        return (self.args.resume
                and os.path.isfile(os.path.join(SITE, local_path(url))))

    def handle_page(self, url):
        raw, ctype = self.dl.get(url)
        text = raw.decode("utf-8", "replace")
        self.read_importmap(text)

        for link in re.findall(r'<a\b[^>]*\bhref=["\']([^"\']+)["\']', text, re.I):
            target = norm_url(link, url)
            if target and is_page(target):
                self.enqueue(target)

        for kind, comp in self.rhds_component_urls(text, url):
            if kind == "component":
                self.probes.add(comp)
            self.enqueue(comp)

        out = self.rewrite_html(text, url)
        self.write(url, out.encode("utf-8"), "html")
        self.page_count += 1

    def handle_asset(self, url):
        raw, ctype = self.dl.get(url)
        kind = guess_kind(url, ctype)
        if kind == "html" and not is_page(url):
            # Safety net: a link that slipped through as an asset but is really
            # an out-of-scope page (/en/products/..., /ja/topics, ...).
            self.skipped += 1
            return
        if kind == "css":
            text = raw.decode("utf-8", "replace")
            for ref in self.css_refs(text, url):
                self.enqueue(ref)
            raw = self.rewrite_css(text, url).encode("utf-8")
        elif kind == "js":
            text = raw.decode("utf-8", "replace")
            for ref in self.js_imports(text, url):
                self.enqueue(ref)
            raw = self.rewrite_js(text, url).encode("utf-8")
        elif kind == "html":
            text = raw.decode("utf-8", "replace")
            raw = self.rewrite_html(text, url).encode("utf-8")
        self.write(url, raw, kind)
        self.asset_count += 1

    # -- seeds ------------------------------------------------------------

    def seed_from_sitemap(self):
        print("Reading %s ..." % SITEMAP_URL)
        try:
            raw, _ = self.dl.get(SITEMAP_URL)
        except Exception as exc:                          # noqa: BLE001
            self.log("sitemap unavailable (%s); relying on link discovery" % exc)
            return
        if raw[:2] == b"\x1f\x8b":
            raw = gzip.GzipFile(fileobj=io.BytesIO(raw)).read()
        text = raw.decode("utf-8", "replace")
        found = 0
        for loc in re.findall(r'<loc>\s*([^<]+?)\s*</loc>', text):
            if is_page(loc):
                before = len(self.seen)
                self.enqueue(loc)
                found += len(self.seen) - before
        print("  %d topic URLs seeded from the sitemap" % found)

    # -- driver -----------------------------------------------------------

    def run(self):
        os.makedirs(SITE, exist_ok=True)
        os.makedirs(LOGS, exist_ok=True)
        started = time.time()

        self.enqueue(START_URL)
        if not self.args.no_sitemap:
            self.seed_from_sitemap()

        print("\n== pages ==")
        while self.pages:
            url = self.pages.popleft()
            if self.args.limit and self.page_count >= self.args.limit:
                print("  page limit reached, stopping page crawl")
                break
            n = self.page_count + 1
            print("[page %4d/%-4d] %s" % (n, n + len(self.pages), url))
            try:
                self.handle_page(url)
            except urllib.error.HTTPError as exc:
                if exc.code in (404, 410):
                    # redhat.com links to a page that no longer exists; it is
                    # 404 on the live site too, so there is nothing to mirror.
                    self.dead_links.append(url)
                    print("  dead link upstream (%d)" % exc.code)
                else:
                    self.log("  FAILED page %s: %s" % (url, exc))
            except Exception as exc:                      # noqa: BLE001
                self.log("  FAILED page %s: %s" % (url, exc))

        print("\n== assets ==")
        while self.assets:
            url = self.assets.popleft()
            n = self.asset_count + 1
            if self.have(url):
                self.asset_count += 1
                self.resumed += 1
                continue
            print("[asset %4d/%-4d] %s" % (n, n + len(self.assets),
                                           url if len(url) < 120 else url[:117] + "..."))
            try:
                self.handle_asset(url)
            except urllib.error.HTTPError as exc:
                # rh-footer-block and friends live inside the rh-footer
                # package; the shorter-prefix guesses that miss are expected.
                if exc.code == 404 and url in self.probes:
                    self.probe_misses += 1
                elif exc.code in (404, 410):
                    # referenced by the page but 404 on the live site too
                    self.dead_links.append(url)
                    print("  dead asset upstream (%d)" % exc.code)
                else:
                    self.log("  FAILED asset %s: %s" % (url, exc))
            except Exception as exc:                      # noqa: BLE001
                self.log("  FAILED asset %s: %s" % (url, exc))

        self.write_landing_page()
        manifest = self.write_manifest(started)

        elapsed = time.time() - started
        print("\nDone in %d min %02d s" % (elapsed // 60, elapsed % 60))
        print("  pages   : %d" % manifest["pages"])
        print("  assets  : %d" % self.asset_count)
        print("  skipped : %d off-tree HTML, %d component probes"
              % (self.skipped, self.probe_misses))
        print("  files   : %d on disk (%.1f MB)"
              % (manifest["files"], manifest["bytes_on_disk"] / 1e6))
        if self.resumed:
            print("  reused  : %d already on disk (--resume)" % self.resumed)
        print("  fetched : %.1f MB" % (self.dl.bytes / 1e6))
        if self.dead_links:
            print("  dead    : %d upstream 404s (see logs/dead-links.txt)"
                  % len(self.dead_links))
            with open(os.path.join(LOGS, "dead-links.txt"), "w") as fh:
                fh.write("Pages and assets that redhat.com references but "
                         "that 404 on the live site; nothing to mirror.\n\n")
                fh.write("\n".join(self.dead_links) + "\n")
        if self.failures:
            print("  PROBLEMS: %d (see logs/failures.txt)" % len(self.failures))
            with open(os.path.join(LOGS, "failures.txt"), "w") as fh:
                fh.write("\n".join(self.failures) + "\n")
        else:
            print("  problems: none")
            # do not leave a stale report from an earlier, worse run
            stale = os.path.join(LOGS, "failures.txt")
            if os.path.exists(stale):
                os.remove(stale)
        return 0 if not self.failures else 1

    def write_landing_page(self):
        """site/index.html -> the topic hub, so http://host:port/ just works."""
        dest = os.path.join(SITE, "index.html")
        with open(dest, "w", encoding="utf-8") as fh:
            fh.write('<!doctype html><meta charset="utf-8">'
                     '<title>Red Hat Topics (offline)</title>'
                     '<meta http-equiv="refresh" content="0; url=/en/topics/">'
                     '<p>Redirecting to <a href="/en/topics/">'
                     'Red Hat IT topics</a>.</p>\n')
        self.written["index.html"] = "(generated)"

    def write_manifest(self, started):
        """Describe what is actually on disk, not just what this run wrote."""
        page_paths = []
        files = 0
        size = 0
        for dirpath, _dirs, names in os.walk(SITE):
            for name in names:
                files += 1
                size += os.path.getsize(os.path.join(dirpath, name))
                if name == "index.html":
                    rel = os.path.relpath(os.path.join(dirpath, name), SITE)
                    rel = rel.replace(os.sep, "/")[:-len("index.html")]
                    if rel.startswith("en/topics"):
                        page_paths.append("/" + rel)
        manifest = {
            "source": START_URL,
            "built": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.localtime(started)),
            "pages": len(page_paths),
            "files": files,
            "bytes_on_disk": size,
            "bytes_fetched_this_run": self.dl.bytes,
            "page_paths": sorted(page_paths),
        }
        with open(os.path.join(ROOT, "manifest.json"), "w") as fh:
            json.dump(manifest, fh, indent=2)
        return manifest


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--delay", type=float, default=1.0,
                    help="seconds between requests (default 1.0; robots.txt "
                         "advertises Crawl-delay: 10)")
    ap.add_argument("--timeout", type=float, default=60.0, help="socket timeout")
    ap.add_argument("--retries", type=int, default=3, help="attempts per URL")
    ap.add_argument("--limit", type=int, default=0,
                    help="stop after N pages (smoke tests)")
    ap.add_argument("--resume", action="store_true",
                    help="skip assets already present in site/ (finish an "
                         "interrupted crawl; pages are always re-fetched)")
    ap.add_argument("--no-sitemap", action="store_true",
                    help="discover pages by following links only")
    ap.add_argument("-v", "--verbose", action="store_true")
    args = ap.parse_args()
    try:
        return Mirror(args).run()
    except KeyboardInterrupt:
        print("\ninterrupted")
        return 130


if __name__ == "__main__":
    sys.exit(main())