#!/usr/bin/env python3
"""Strip comments from an exported OSS-mirror tree (see export-oss-mirror.sh).

Policy (D5.10, revised 2026-10-16): strip as much as possible. The only
comments that survive are ones a machine or a licence needs:
  - the ASF-licensed Maven wrapper files, untouched (Apache-2.0 section 4)
  - a shebang on line 1
  - Dockerfile parser directives (# syntax= / escape= / check=) at the top
  - Go compiler directives (//go:..., // +build)

Every stripper is line-preserving: it blanks comment text in place, so line i
of the output corresponds to line i of the input. A shared post-pass then
drops lines that held only a comment and right-strips lines that lost a
trailing one. Two safety nets follow:
  1. any line the stripper did not change must survive verbatim;
  2. a per-language oracle proves the file still means the same thing:
       sh    `bash --pretty-print` of both versions is identical
       py    the AST is identical once docstrings are removed from the original
       yaml  every document loads to the same data
       xml   the comment-free canonical form is identical
       go    the token stream (go/scanner, comments skipped) is identical
       js    `node --check` still parses it
Net 1 alone once missed a real break: an earlier version fed the Dockerfile
named `Dockerfile.java` to the Java scanner, and the `/*` in
`rm -rf /var/lib/apt/lists/*` opened a block comment that ate ten lines.

Contexts where a comment marker is not a comment, all handled:
  java  "https://..." string and text-block literals, '"' char literals
  sh    ${x#y}, $#, quoted strings spanning lines, heredoc bodies
  yml   block scalars (| and >), quoted scalars containing " #"
  xml   <![CDATA[ ]]> sections
  js/go backtick template and raw strings
"""
import ast
import io
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
import tokenize
import xml.etree.ElementTree as ET

UNVERIFIED = []
SKIPPED_AWK = []
LICENCE_EXEMPT = {"mvnw", "mvnw.bat", "mvnw.cmd", "maven-wrapper.properties"}
_HEREDOC = re.compile(r"<<-?\s*(['\"]?)([A-Za-z_][A-Za-z0-9_]*)\1")
_DOCKER_DIRECTIVE = re.compile(r"^#\s*(syntax|escape|check)\s*=", re.I)
_GO_DIRECTIVE = ("//go:", "// +build")


def _blank(src, spans):
    """Remove the given char spans but keep their newlines (line-preserving)."""
    out, pos = [], 0
    for a, b in sorted(spans):
        out.append(src[pos:a])
        out.append("\n" * src.count("\n", a, b))
        pos = b
    out.append(src[pos:])
    return "".join(out)


def _scan(src, kind, quotes):
    """C-like scanner. Returns the comment spans."""
    spans, i, n = [], 0, len(src)
    while i < n:
        c = src[i]
        if kind == "java" and src.startswith('"""', i):
            i = src.index('"""', i + 3) + 3
        elif c in quotes:
            j = i + 1
            while j < n and src[j] != c:
                j += 2 if src[j] == "\\" else 1
            i = j + 1
        elif src.startswith("//", i):
            j = src.find("\n", i)
            j = n if j < 0 else j
            if not (kind == "go" and src.startswith(_GO_DIRECTIVE, i)):
                spans.append((i, j))
            i = j
        elif src.startswith("/*", i):
            j = src.find("*/", i + 2)
            j = n if j < 0 else j + 2
            spans.append((i, j))
            i = j
        else:
            i += 1
    return spans


def strip_java(src):
    return _blank(src, _scan(src, "java", "\"'"))


def strip_js(src):
    return _blank(src, _scan(src, "js", "\"'`"))


def strip_go(src):
    return _blank(src, _scan(src, "go", "\"'`"))


def strip_xml(src):
    spans, i, n = [], 0, len(src)
    while i < n:
        if src.startswith("<![CDATA[", i):
            j = src.find("]]>", i)
            i = n if j < 0 else j + 3
        elif src.startswith("<!--", i):
            j = src.find("-->", i)
            j = n if j < 0 else j + 3
            spans.append((i, j))
            i = j
        else:
            i += 1
    return _blank(src, spans)


def strip_properties(src):
    """Whole-line # and ! comments. A trailing # is part of the value.
    A line starting with // is not a comment in this format but a junk key
    ("//" = "rest of line") that nothing reads -- it goes too."""
    out = []
    for line in src.split("\n"):
        s = line.lstrip()
        out.append("" if s.startswith(("#", "!", "//")) else line)
    return "\n".join(out)


def strip_dockerfile(src):
    """Whole-line comments only: mid-line # is an instruction argument."""
    out, header = [], True
    for line in src.split("\n"):
        s = line.strip()
        if header and _DOCKER_DIRECTIVE.match(s):
            out.append(line)
            continue
        if s and not s.startswith("#"):
            header = False
        out.append("" if s.startswith("#") else line)
    return "\n".join(out)


def strip_ignorefile(src):
    return "\n".join("" if l.startswith("#") else l for l in src.split("\n"))


_SH_WORD_BREAK = set(" \t;&|()<>")


_AWK_CMD = re.compile(r"(^|[\s|;&(])g?awk\b[^|;&]*$")


def _sh_scan(src):
    """Quote-aware: # starts a comment only at the start of a word, outside
    quotes, ${...} and heredoc bodies. State carries across lines, so text
    inside a multi-line '...' string is never treated as shell. Returns the
    comment-free text and the (line, col) spans of every single-quoted string,
    flagged when it opens an awk program."""
    lines, out, awk = src.split("\n"), [], []
    quote, brace, heredocs, awk_open, sq_open = None, 0, [], None, None
    for idx, line in enumerate(lines):
        if heredocs:                             # heredoc body is data
            out.append(line)
            if line.strip() == heredocs[0]:
                heredocs.pop(0)
            continue
        if idx == 0 and line.startswith("#!"):
            out.append(line)
            continue
        i, n, cut = 0, len(line), None
        while i < n:
            c = line[i]
            if quote == "'":
                if c == "'":
                    quote = None
                    awk.append((sq_open, (idx, i), sq_open == awk_open))
                    awk_open = None
            elif quote == "$'":
                if c == "\\":
                    i += 1
                elif c == "'":
                    quote = None
            elif quote in ('"', "`"):
                if c == "\\":
                    i += 1
                elif c == quote:
                    quote = None
            elif brace:
                if c == "{":
                    brace += 1
                elif c == "}":
                    brace -= 1
            elif c == "\\":
                i += 1
            elif c == "$" and line.startswith("${", i):
                brace, i = 1, i + 1
            elif c == "$" and line.startswith("$'", i):
                quote, i = "$'", i + 1
            elif c in "'\"`":
                quote = c
                if c == "'":
                    sq_open = (idx, i + 1)
                    if i and line[i - 1] in " \t" and _AWK_CMD.search(line[:i]):
                        awk_open = sq_open
            elif c == "#" and (i == 0 or line[i - 1] in _SH_WORD_BREAK):
                cut = i
                break
            i += 1
        code = line if cut is None else line[:cut]
        out.append(code)
        if quote is None and not brace:
            for m in _HEREDOC.finditer(code):
                heredocs.append(m.group(2))
    return "\n".join(out), awk


def strip_awk(prog):
    """Line-preserving awk comment strip: # outside "strings" and /regex/."""
    out, i, n, prev = [], 0, len(prog), "("
    while i < n:
        c = prog[i]
        if c == '"':
            j = i + 1
            while j < n and prog[j] != '"':
                j += 2 if prog[j] == "\\" else 1
            out.append(prog[i:j + 1]); i = j + 1; prev = '"'
        elif c == "/" and prev in "(,~!&|{};:?=\n+-*%<>[":
            j = i + 1
            while j < n and prog[j] not in "/\n":
                j += 2 if prog[j] == "\\" else 1
            out.append(prog[i:j + 1]); i = j + 1; prev = "/"
        elif c == "#":
            j = prog.find("\n", i)
            j = n if j < 0 else j
            i = j
        else:
            out.append(c); i += 1
            if not c.isspace() or c == "\n":
                prev = c
    return "".join(out)


def _gawk_norm(prog):
    r = subprocess.run(["gawk", "-o-", "--", prog], capture_output=True, text=True)
    if r.returncode != 0:
        return None
    lines = []
    for l in r.stdout.split("\n"):
        if l.lstrip("\t").startswith("#"):
            continue
        l = re.sub(r"\t#.*$", "", l).rstrip()
        if l:
            lines.append(l)
    return lines


_SPLICE = re.compile(r"'(\"\$(?:\{[^}]*\}|\w+)\")'")


def _strip_awk_regions(src, regions):
    """Strip comments inside each single-quoted awk program; each one is proven
    equivalent with gawk's pretty-printer. A program spliced with a shell
    variable ('...'"$x"'...') is stripped as one program, with each splice
    stood in by a placeholder identifier while it is stripped and checked."""
    lines = src.split("\n")
    programs, k = [], 0
    while k < len(regions):
        if not regions[k][2]:
            k += 1
            continue
        chain = [regions[k]]
        while k + 1 < len(regions):
            (_, _), (l1, c1), _ = chain[-1]
            (n0, nc0), _, _ = regions[k + 1]
            m = _SPLICE.match(lines[l1], c1)
            if m and n0 == l1 and nc0 == m.end():
                chain.append(regions[k + 1]); k += 1
            else:
                break
        programs.append(chain)
        k += 1

    def text(a, b):
        (l0, c0), (l1, c1) = a, b
        if l0 == l1:
            return lines[l0][c0:c1]
        return "\n".join([lines[l0][c0:]] + lines[l0 + 1:l1] + [lines[l1][:c1]])

    edits = []
    for chain in programs:
        frags = [text(a, b) for a, b, _ in chain]
        marks = [f"__splice{n}__" for n in range(len(frags) - 1)]
        prog = "".join(f + (marks[n] if n < len(marks) else "") for n, f in enumerate(frags))
        new = strip_awk(prog)
        if new == prog:
            continue
        where = f"awk program at line {chain[0][0][0] + 1}"
        if not shutil.which("gawk"):
            SKIPPED_AWK.append(f"no gawk to verify {where}")
            continue
        a, b = _gawk_norm(prog), _gawk_norm(new)
        if a is None:
            SKIPPED_AWK.append(f"{where} does not parse standalone")
            continue
        if a != b:
            raise SystemExit(f"FATAL: {where} changed meaning")
        parts = re.split(r"__splice\d+__", new)
        if len(parts) != len(frags):
            raise SystemExit(f"FATAL: {where} lost a splice")
        edits += [(a_, b_, part) for (a_, b_, _), part in zip(chain, parts)]
    for (l0, c0), (l1, c1), part in sorted(edits, reverse=True):
        nl = part.split("\n")
        if l0 == l1:
            lines[l0] = lines[l0][:c0] + nl[0] + lines[l0][c1:]
        else:
            lines[l0:l1 + 1] = ([lines[l0][:c0] + nl[0]] + nl[1:-1]
                                + [nl[-1] + lines[l1][c1:]])
    return "\n".join(lines)


def strip_sh(src):
    return _sh_scan(src)[0]


def strip_yaml(src):
    out, block_indent = [], None
    for line in src.split("\n"):
        stripped, indent = line.strip(), len(line) - len(line.lstrip())
        if block_indent is not None:             # block scalar body is data
            if stripped and indent <= block_indent:
                block_indent = None
            else:
                out.append(line)
                continue
        quote, cut = None, None
        for i, c in enumerate(line):
            if quote:
                if c == quote:
                    quote = None
            elif c in "'\"" and (i == 0 or line[i - 1] in " \t:[{,-"):
                quote = c
            elif c == "#" and (i == 0 or line[i - 1] in " \t"):
                cut = i
                break
        code = line if cut is None else line[:cut]
        out.append(code)
        if re.search(r":\s*[|>][+-]?\d*\s*$", code):
            block_indent = indent
    return "\n".join(out)


def _docstring_nodes(tree):
    for node in ast.walk(tree):
        if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef,
                             ast.AsyncFunctionDef)) and node.body:
            first = node.body[0]
            if (isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant)
                    and isinstance(first.value.value, str)):
                yield node, first


def strip_py(src):
    lines = src.split("\n")
    starts, pos = [], 0
    for l in lines:
        starts.append(pos)
        pos += len(l) + 1
    off = lambda ln, col: starts[ln - 1] + col
    spans, keep_first = [], lines[0].startswith("#!")
    for tok in tokenize.generate_tokens(io.StringIO(src).readline):
        if tok.type == tokenize.COMMENT and not (keep_first and tok.start[0] == 1):
            spans.append((off(*tok.start), off(*tok.end)))
    edits = [(a, b, "") for a, b in spans]
    for parent, doc in _docstring_nodes(ast.parse(src)):
        a = off(doc.lineno, doc.col_offset)
        b = off(doc.end_lineno, doc.end_col_offset)
        edits.append((a, b, "pass" if len(parent.body) == 1 else ""))  # sole statement keeps a body
    out, p = [], 0
    for a, b, r in sorted(edits):
        out.append(src[p:a]); out.append(r + "\n" * src.count("\n", a, b)); p = b
    out.append(src[p:])
    return "".join(out)


def _postpass(src, new):
    """Drop lines that held only a comment, right-strip lines that lost one."""
    old_l, new_l = src.split("\n"), new.split("\n")
    assert len(old_l) == len(new_l), "stripper is not line-preserving"
    out, dropped = [], False
    for o, m in zip(old_l, new_l):
        if o != m and not m.strip() and o.strip():
            dropped = True                       # the whole line was comment
            continue
        line = o if o == m else m.rstrip()
        # a dropped block must not leave two blank lines behind; blank runs
        # anywhere else (heredocs, block scalars, strings) are left alone
        if dropped and not line.strip() and out and not out[-1].strip():
            continue
        dropped = False
        out.append(line)
    return "\n".join(out)


# ---- oracles: prove the stripped file means the same thing --------------

def _oracle_sh(src, new):
    with tempfile.TemporaryDirectory() as d:
        a, b = pathlib.Path(d, "a.sh"), pathlib.Path(d, "b.sh")
        a.write_text(src); b.write_text(new)
        pa = subprocess.run(["bash", "--pretty-print", str(a)], capture_output=True, text=True)
        pb = subprocess.run(["bash", "--pretty-print", str(b)], capture_output=True, text=True)
    if pa.returncode != 0:
        return None                              # original itself unparsable here
    # pretty-print renders a removed comment block as a blank line, and blank
    # lines carry no meaning outside strings, which the stripper never touches
    norm = lambda t: [l.rstrip() for l in t.split("\n") if l.strip()]
    return pb.returncode == 0 and norm(pa.stdout) == norm(pb.stdout)


class _NoDoc(ast.NodeTransformer):
    def visit(self, node):
        node = super().visit(node)
        body = getattr(node, "body", None)
        if (isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef,
                              ast.AsyncFunctionDef)) and body
                and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant)
                and isinstance(body[0].value.value, str)):
            node.body = body[1:] or [ast.Pass()]
        return node


def _oracle_py(src, new):
    return ast.dump(_NoDoc().visit(ast.parse(src))) == ast.dump(ast.parse(new))


def _oracle_yaml(src, new):
    import yaml
    return list(yaml.safe_load_all(src)) == list(yaml.safe_load_all(new))


def _oracle_xml(src, new):
    c = lambda t: ET.canonicalize(t, with_comments=False, strip_text=True)
    return c(src) == c(new)


_GO_ORACLE = r'''package main
import ("fmt";"go/scanner";"go/token";"os")
func main(){src,_:=os.ReadFile(os.Args[1]);fs:=token.NewFileSet()
f:=fs.AddFile("x",-1,len(src));var s scanner.Scanner;bad:=false
s.Init(f,src,func(token.Position,string){bad=true},0)
for{_,t,l:=s.Scan();if t==token.EOF{break};fmt.Printf("%s %q\n",t,l)}
if bad{os.Exit(2)}}'''


def _oracle_go(src, new):
    if not shutil.which("go"):
        return None
    with tempfile.TemporaryDirectory() as d:
        prog = pathlib.Path(d, "p.go"); prog.write_text(_GO_ORACLE)
        exe = pathlib.Path(d, "p")
        subprocess.run(["go", "build", "-o", str(exe), str(prog)], check=True,
                       capture_output=True, env={**os.environ, "GO111MODULE": "off"})
        outs = []
        for name, txt in (("a.go", src), ("b.go", new)):
            f = pathlib.Path(d, name); f.write_text(txt)
            r = subprocess.run([str(exe), str(f)], capture_output=True, text=True)
            if r.returncode != 0:
                return False
            outs.append(r.stdout)
    return outs[0] == outs[1]


def _oracle_js(src, new):
    if not shutil.which("node"):
        return None
    with tempfile.TemporaryDirectory() as d:
        f = pathlib.Path(d, "b.mjs"); f.write_text(new)
        return subprocess.run(["node", "--check", str(f)], capture_output=True).returncode == 0


STRIPPERS = {
    ".java": (strip_java, None), ".js": (strip_js, _oracle_js),
    ".go": (strip_go, _oracle_go), ".mod": (strip_go, None),
    ".sh": (strip_sh, _oracle_sh), ".bash": (strip_sh, _oracle_sh),
    ".py": (strip_py, _oracle_py),
    ".yml": (strip_yaml, _oracle_yaml), ".yaml": (strip_yaml, _oracle_yaml),
    ".xml": (strip_xml, _oracle_xml),
    ".properties": (strip_properties, None), ".conf": (strip_properties, None),
}


def stripper_for(path):
    """Classify by what the file IS, not by the characters after the last dot."""
    if path.name in LICENCE_EXEMPT:
        return None
    if path.name.startswith("Dockerfile"):
        return strip_dockerfile, None
    if path.name in (".gitignore", ".dockerignore"):
        return strip_ignorefile, None
    return STRIPPERS.get(path.suffix)


def strip_text(path, src):
    fn, oracle = stripper_for(path)
    orig = src
    if fn is strip_sh:                           # awk programs first, own oracle
        SKIPPED_AWK.clear()
        src = _strip_awk_regions(src, _sh_scan(src)[1])
        UNVERIFIED.extend(f"{path}: {m}" for m in SKIPPED_AWK)
    blanked = fn(src)
    new = _postpass(orig, blanked)
    survivors = set(new.split("\n"))
    for o, m in zip(orig.split("\n"), blanked.split("\n")):
        if o == m and o.strip() and o not in survivors:
            raise SystemExit(f"FATAL: {path} lost an untouched line: {o!r}")
    if oracle and new != src:
        ok = oracle(src, new)
        if ok is False:
            raise SystemExit(f"FATAL: {path} changed meaning after stripping")
        if ok is None:
            UNVERIFIED.append(str(path))
    return new


def main(tree):
    touched = 0
    for path in sorted(tree.rglob("*")):
        if not path.is_file() or ".git" in path.parts or stripper_for(path) is None:
            continue
        src = path.read_text(encoding="utf-8")
        new = strip_text(path, src)
        if new != src:
            path.write_text(new, encoding="utf-8")
            touched += 1
    print(f"  comments stripped from {touched} file(s)")
    for u in UNVERIFIED:
        print(f"  WARN: no oracle could check {u}")


if __name__ == "__main__":
    if len(sys.argv) != 2:
        sys.exit("usage: oss-strip-comments.py <exported-tree>")
    main(pathlib.Path(sys.argv[1]))