#!/usr/bin/env python3
"""
check.py -- verify the offline mirror is self-contained.

Run it on the air-gapped machine, with no network and no server needed:

    ./check.py

It walks every mirrored page, collects every local URL the page depends on
(stylesheets, scripts, images, fonts, ES-module imports, CSS url(), RHDS
icon modules) and reports the ones that are not on disk.

"The pages load" is not the same as "the pages are complete" -- a missing
web-component module just means that element silently never upgrades.
"""

import argparse
import json
import os
import re
import sys
from urllib.parse import urlsplit, urljoin, unquote

ROOT = os.path.dirname(os.path.abspath(__file__))
SITE = os.path.join(ROOT, "site")

TRACKER_RE = re.compile(
    r"(/dtm\.js|trustarc|trustecm|googletagmanager|google-analytics|hotjar"
    r"|munchkin|marketo|6sense|rh\.mktg\.js)", re.I)

ASSET_RELS = {"stylesheet", "preload", "modulepreload", "prefetch", "icon",
              "shortcut icon", "apple-touch-icon", "manifest", "mask-icon"}

RHDS_ICONS = "/modules/contrib/red_hat_shared_libs/dist/rhds-elements/icons"


def unescape(v):
    return v.replace("&", "&").replace("&", "&")


def to_file(url):
    """Local URL -> path under site/, or None if it is not ours to check."""
    p = urlsplit(url)
    if p.scheme or p.netloc:
        return None                      # absolute, deliberately left external
    path = p.path
    if not path.startswith("/"):
        return None
    rel = path.lstrip("/")
    if path.endswith("/"):
        rel += "index.html"
    segs = [unquote(seg).replace("/", "_").replace("\\", "_")
            for seg in rel.split("/")]
    return os.path.join(SITE, *segs)


class Checker:
    def __init__(self, verbose):
        self.verbose = verbose
        self.missing = {}                # url -> set of referrers
        self.checked = set()
        self.pages = 0
        self.refs = 0
        self.trackers = []

    def note(self, url, referrer):
        if url is None:
            return
        self.refs += 1
        target = to_file(url)
        if target is None:
            return
        if not os.path.isfile(target):
            self.missing.setdefault(url, set()).add(referrer)

    # -- extraction -------------------------------------------------------

    def from_html(self, text, referrer):
        if TRACKER_RE.search(text):
            for m in TRACKER_RE.finditer(text):
                self.trackers.append((referrer, m.group(1)))

        # <link rel="stylesheet|preload|icon" href=...>
        for m in re.finditer(r'<link\b([^>]*)>', text, re.I):
            attrs = m.group(1)
            rel = re.search(r'rel=["\']([^"\']+)', attrs, re.I)
            href = re.search(r'href=["\']([^"\']+)', attrs, re.I)
            if rel and href and rel.group(1).strip().lower() in ASSET_RELS:
                self.note(unescape(href.group(1)), referrer)

        # <script src=...>
        for m in re.finditer(r'<script\b[^>]*\bsrc=["\']([^"\']+)', text, re.I):
            self.note(unescape(m.group(1)), referrer)

        # img / source / video
        for m in re.finditer(r'<(?:img|source|video|embed)\b([^>]*)>', text, re.I):
            attrs = m.group(1)
            for name in ("src", "poster"):
                a = re.search(r'\b%s=["\']([^"\']+)' % name, attrs, re.I)
                if a:
                    self.note(unescape(a.group(1)), referrer)
            a = re.search(r'\bsrcset=["\']([^"\']+)', attrs, re.I)
            if a:
                for item in unescape(a.group(1)).split(","):
                    item = item.strip().split(None, 1)
                    if item and item[0]:
                        self.note(item[0], referrer)

        # <rh-icon icon="x" set="y"> -> runtime import(`../icons/${set}/${icon}.js`)
        for m in re.finditer(r'<rh-icon\b([^>]*)>', text, re.I):
            icon = re.search(r'\bicon=["\']([^"\']+)', m.group(1))
            iset = re.search(r'\bset=["\']([^"\']+)', m.group(1))
            if icon and iset:
                self.note("%s/%s/%s.js" % (RHDS_ICONS, iset.group(1),
                                           icon.group(1)), referrer)

        # inline <style>
        for m in re.finditer(r'<style\b[^>]*>(.*?)</style>', text, re.S | re.I):
            self.from_css(m.group(1), referrer, referrer)

    def from_css(self, text, base_url, referrer):
        for m in re.finditer(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)', text):
            raw = m.group(1).strip()
            # "#clip0_232_19576", raw or percent-encoded, is an in-document
            # SVG fragment reference, not a file
            if raw.startswith(("data:", "#")) or raw.lower().startswith("%23"):
                continue
            self.note(urljoin(base_url, raw), referrer)
        for m in re.finditer(r'@import\s+[\'"]([^\'"]+)[\'"]', text):
            self.note(urljoin(base_url, m.group(1)), referrer)

    def from_js(self, text, base_url, referrer, importmap):
        pats = (r'\bimport\s+[^\'"()]*?from\s*[\'"]([^\'"]+)[\'"]',
                r'\bimport\s*[\'"]([^\'"]+)[\'"]',
                r'\bexport\s+[^\'"()]*?from\s*[\'"]([^\'"]+)[\'"]',
                r'\bimport\s*\(\s*[\'"]([^\'"${}]+)[\'"]\s*\)')
        for pat in pats:
            for m in re.finditer(pat, text):
                spec = m.group(1)
                if spec.startswith((".", "/")):
                    self.note(urljoin(base_url, spec), referrer)
                elif "://" not in spec:
                    for prefix, target in importmap.items():
                        if spec.startswith(prefix):
                            self.note(target + spec[len(prefix):], referrer)
                            break

    # -- walk -------------------------------------------------------------

    def run(self):
        if not os.path.isdir(SITE):
            sys.exit("No mirror at %s -- run ./fetch.py first." % SITE)

        importmap = {}
        pages = []
        for dirpath, _dirs, files in os.walk(SITE):
            for name in files:
                full = os.path.join(dirpath, name)
                rel = "/" + os.path.relpath(full, SITE).replace(os.sep, "/")
                if name.endswith(".html"):
                    pages.append((rel, full))

        # The import map is the same on every topic page; read it from the
        # hub rather than from whichever file os.walk happened to yield first.
        hub = os.path.join(SITE, "en", "topics", "index.html")
        for candidate in [hub] + [full for _rel, full in pages]:
            if not os.path.isfile(candidate):
                continue
            text = open(candidate, encoding="utf-8", errors="replace").read()
            m = re.search(r'<script[^>]*type=["\']importmap["\'][^>]*>(.*?)</script>',
                          text, re.S | re.I)
            if m:
                try:
                    importmap = json.loads(m.group(1)).get("imports", {})
                    break
                except ValueError:
                    pass
        if not importmap:
            print("WARNING: no <script type=\"importmap\"> found; bare module "
                  "specifiers cannot be verified")

        for rel, full in pages:
            self.pages += 1
            text = open(full, encoding="utf-8", errors="replace").read()
            self.from_html(text, rel)

        # follow CSS and JS that the pages pulled in
        for dirpath, _dirs, files in os.walk(SITE):
            for name in files:
                full = os.path.join(dirpath, name)
                rel = "/" + os.path.relpath(full, SITE).replace(os.sep, "/")
                if name.endswith(".css"):
                    self.from_css(open(full, encoding="utf-8",
                                       errors="replace").read(), rel, rel)
                elif name.endswith((".js", ".mjs")):
                    self.from_js(open(full, encoding="utf-8",
                                      errors="replace").read(), rel, rel,
                                 importmap)

        return self.report()

    def report(self):
        files = sum(len(f) for _d, _s, f in os.walk(SITE))
        size = sum(os.path.getsize(os.path.join(d, f))
                   for d, _s, fs in os.walk(SITE) for f in fs)
        print("pages       : %d" % self.pages)
        print("files       : %d" % files)
        print("size        : %.1f MB" % (size / 1e6))
        print("references  : %d checked" % self.refs)

        if self.trackers:
            print("trackers    : %d leftover references!" % len(self.trackers))
            for referrer, hit in self.trackers[:10]:
                print("    %s -> %s" % (referrer, hit))
        else:
            print("trackers    : none left in the HTML")

        if not self.missing:
            print("missing     : none -- the mirror is self-contained")
            return 0

        print("missing     : %d local URLs" % len(self.missing))
        for url in sorted(self.missing)[:40 if not self.verbose else 10000]:
            refs = sorted(self.missing[url])
            print("    %s" % url)
            print("        referenced by %d file(s), e.g. %s"
                  % (len(refs), refs[0]))
        if not self.verbose and len(self.missing) > 40:
            print("    ... %d more (use -v)" % (len(self.missing) - 40))
        return 1


def main():
    ap = argparse.ArgumentParser(description="Verify the offline mirror.")
    ap.add_argument("-v", "--verbose", action="store_true",
                    help="list every missing URL")
    return Checker(ap.parse_args().verbose).run()


if __name__ == "__main__":
    sys.exit(main())