Skip to content

Deslop Scan

#!/usr/bin/env python3
"""Scan prose for LLM slop patterns.

Usage:
    deslop_scan.py FILE [FILE ...]     scan files (directories are walked)
    deslop_scan.py -                   scan stdin
    deslop_scan.py --comments src/     also scan comments and docstrings in .py files
    deslop_scan.py --explain FILE      add the catalogue entries for the rules that fired
    deslop_scan.py --check-catalogue   check that rules and catalogue entries match

Options:
    --colon-triple   enable the colon-into-a-triple check (noisy in docs)
    --em-dash        enable em-dash density reporting
    --all            enable every optional check
    --only IDS       scan with only these rules (comma separated)
    --skip IDS       scan with every rule except these
    --all-hits       report hits below a rule's noise threshold too
    --list-rules     print the rule table and exit
    --summary        add a corpus-wide total when scanning several files
    --quiet          omit files with no hits
    --json           emit JSON instead of a text report
    --context N      characters of match text to show (default 90)
    --comments       include .py files when walking directories
    --explain        print the patterns.md entries for the rules that
                     fired, then the patterns no regex catches
    --check-catalogue
                     exit 1 if a rule has no catalogue entry or an entry names
                     no rule

A .py file is scanned for its comments and docstrings only, whether it is named
on the command line or found with --comments; the code itself is skipped, and
so are the rules in COMMENT_SKIP.

Fenced code blocks, indented code blocks, inline code, blockquotes, RST
directives and section underlines are skipped.

Some rules only fire above a count, because one instance is a style choice and
several are a tic. `--all-hits` shows those, and the report footer counts them.

Exit status: 0 clean, 1 hits found, 2 no file could be read.
"""

from __future__ import annotations

import argparse
import ast
import io
import json
import os
import re
import sys
import tokenize
from dataclasses import dataclass, field

I = re.IGNORECASE  # noqa: E741

TEXT_SUFFIXES = {".md", ".markdown", ".rst", ".txt"}

CATALOGUE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "patterns.md")

# These fire on ordinary comments and docstrings: a why-comment ends in its
# reason (", since ...", "instead of ..."), docstrings open with the same verb
# ("Return ..."), numpydoc parameter lists read as fragments, and a "# ..." line
# in a docstring's usage block isn't a heading.
COMMENT_SKIP = {
    "antithesis",
    "choppy-run",
    "fragment-run",
    "justification-tail",
    "repeated-openers",
    "short-section",
}


@dataclass(frozen=True)
class Rule:
    id: str
    description: str
    pattern: str
    exclude: str | None = None
    optional: bool = False
    min_count: int = 1
    min_per_1k: float = 0.0
    density_only: bool = False
    count_pattern: str | None = None

    @property
    def note(self) -> str:
        bits = []
        if self.optional:
            bits.append("off by default")
        if self.min_count > 1:
            bits.append(f"needs {self.min_count}+")
        if self.min_per_1k:
            bits.append(f"and {self.min_per_1k}+ per 1000 words")
        if self.density_only:
            bits.append("density only")
        return ", ".join(bits)


# ---------------------------------------------------------------- regex rules

RULES: list[Rule] = [
    # -- rhetorical tics -----------------------------------------------------
    Rule(
        "no-x-no-y",
        '"No X, no Y" chain',
        r"\bno\s+[\w'’-]+(?:\s+[\w'’-]+){0,2}\s*[,;]\s*(?:and\s+|or\s+)?"
        r"no\s+[\w'’-]+(?:\s+[\w'’-]+){0,2}"
        r"(?:\s*[,;]\s*(?:and\s+|or\s+)?no\s+[\w'’-]+(?:\s+[\w'’-]+){0,2})*",
        count_pattern=r"\bno\s+",
    ),
    Rule(
        "thats-the-whole",
        '"That\'s the whole ..."',
        r"\b(?:that|this)(?:'s|’s| is)\s+the\s+whole\s+[\w-]+",
        # "the whole file" names a thing; "the whole point" inflates one
        exclude=r"whole\s+(?:file|directory|folder|list|output|archive|dataset|"
        r"repo(?:sitory)?|document|module|package|page|suite|log|table|history)",
    ),
    Rule(
        "did-not-chain",
        '"Did not X, did not Y" chain',
        r"\b(?:did\s+not|didn['’]t)\s+[\w'’-]+(?:\s+[\w'’-]+){0,2}\s*[,;]\s*(?:and\s+|or\s+)?"
        r"(?:did\s+not|didn['’]t)\s+[\w'’-]+(?:\s+[\w'’-]+){0,2}"
        r"(?:\s*[,;]\s*(?:and\s+|or\s+)?(?:did\s+not|didn['’]t)\s+[\w'’-]+(?:\s+[\w'’-]+){0,2})*",
        count_pattern=r"\b(?:did\s+not|didn['’]t)\b",
    ),
    Rule(
        "dont-verb-it",
        '"Don\'t VERB it ... VERB it"',
        r"\b(?:do\s+not|don['’]t)\s+(\w+)(?:\s+(?:of|about|at|as|to))?\s+(?:it|this|that)\b"
        r"[^.!?]{0,60}[.!?;:—-]\s*"
        r"(?:\w+[\s,]+){0,3}\1(?:\s+(?:of|about|at|as|to))?\s+(?:it|this|that)\b",
    ),
    Rule(
        "sit-with-that",
        '"Sit with that"',
        r"\bsit\s+with\s+(?:that|this|it|the\s+\w+)",
    ),
    Rule(
        "you-already-know",
        '"You already know"',
        r"\byou\s+already\s+know\b",
    ),
    Rule(
        "is-the-entire",
        '"is the entire ..."',
        r"\b(?:is|are|was|were)\s+the\s+entire\s+(?:point|game|thing|idea|pitch|trick|"
        r"premise|story|argument|appeal|value\s+proposition|business\s+model)\b",
    ),
    Rule(
        "the-entire-x-is",
        '"The entire ... is"',
        r"\bthe\s+entire\s+(?:point|game|thing|idea|pitch|trick|premise|business\s+model)\s+(?:is|was)\b",
    ),
    Rule(
        "is-the-whole",
        '"is the whole ..."',
        r"\b(?:is|are|was|were)\s+the\s+whole\s+(?:point|trick|pitch|idea|thing|game|premise|story)\b"
        r"|\bhere(?:'s|’s| is)\s+the\s+whole\s+[\w-]+",
    ),
    Rule(
        "is-real-and",
        '"is real, and / not"',
        r"\b(?:is|are|was|were)\s+real\s*[,;]?\s+(?:and|not|but)\b"
        r"|\bis\s+the\s+real\s+[\w-]+\s+and\s+it\b",
        exclude=r"real\s+(?:estate|time|numbers?|world|name|money|terms|analysis|user)",
    ),
    Rule(
        "punchline",
        '"The punchline is"',
        r"\bthe\s+punchline\s*(?:is\b|was\b|[:?])",
    ),
    Rule(
        "worth-naming",
        '"Worth naming"',
        r"\b(?:is|it['’]s|its)\s+worth\s+(?:naming|stating|pausing\s+on)\b"
        r"|^\s*worth\s+naming\s*:",
        exclude=r"naming\s+names",
    ),
    Rule(
        "thats-not-nothing",
        '"That\'s not nothing"',
        r"\b(?:that|this|it|which)(?:'s|’s| is)\s+not\s+nothing\b",
    ),
    Rule(
        "performative-honesty",
        "Performative honesty",
        r"\bI\s+(?:won['’]t|will\s+not)\s+pretend\b"
        r"|\bI['’]?ll\s+be\s+honest\b|\bI\s+will\s+be\s+honest\b"
        r"|\blet['’]?s\s+be\s+honest\b|\bif\s+I['’]?m\s+being\s+honest\b"
        r"|\bto\s+be\s+(?:clear|honest|fair)\b|\bin\s+all\s+honesty\b"
        r"|(?:^|(?<=[.!?]\s)|(?<=\n))(?:Honestly|Look|Truthfully|Frankly)\s*[,:]",
    ),
    Rule(
        "thats-the-part",
        '"That\'s the part ..."',
        r"\b(?:that|this)(?:'s|’s| is)\s+the\s+part\b"
        r"|\bmy\s+favou?rite\s+part\s+(?:of|is|about)\b"
        r"|\bthe\s+part\s+that\s+(?:makes\s+me|I\s+(?:like|trust|keep))\b",
    ),
    Rule(
        "only-x-i-trust",
        '"The only X I trust"',
        r"\bthe\s+only\s+[\w\s'’-]{0,25}?(?:I\s+trust|that\s+(?:matters|counts)|it\s+needs|"
        r"worth\s+\w+|you\s+need)\b",
    ),
    Rule(
        "take-my-word",
        '"Don\'t take my word for it"',
        r"\btake\s+my\s+word\s+for\b",
    ),
    Rule(
        "turns-out",
        '"Turns out ..."',
        r"(?:^|(?<=[.!?]\s)|(?<=\n)|(?<=—)|(?<=–))\s*Turns\s+out\b"
        r"|\bit\s+turn(?:s|ed)\s+out\s+that\b",
    ),
    Rule(
        "fits-in-head",
        '"Fits in your head" / dev-blog boilerplate',
        r"\b(?:hold|fit|fits|holds)\s+(?:it\s+)?in\s+your\s+head\b"
        r"|\bsmall\s+enough\s+to\s+(?:hold|fit)\b"
        r"|\bbatteries[\s-]included\b|\bit\s+just\s+works\b|\bzero[\s-]config\b"
        r"|\bsane\s+defaults\b|\bjust\s+works,?\s+out\s+of\s+the\s+box\b",
    ),
    Rule(
        "heres-the-twist",
        '"Here\'s the twist"',
        r"\bhere(?:'s|’s| is)\s+the\s+(?:twist|thing|catch|kicker|rub|trick|punchline|"
        r"first|best\s+part|problem)\b",
    ),
    Rule(
        "x-is-dead",
        '"X is dead"',
        r"\b[\w-]+\s+(?:is|are)\s+dead\b(?!\s*(?:code|letter|end|link))" r"|\blong\s+live\s+[\w-]+",
        # a dead process, socket or kernel is a fact about software, not a headline
        exclude=r"\b(?:process(?:es)?|thread|socket|connection|session|kernel|server|worker|"
        r"client|channel|node|pixel|battery|link|branch|task|job)\s+(?:is|are)\s+dead",
    ),
    Rule(
        "thats-why-mattered",
        '"That\'s why X mattered"',
        r"\b(?:that|this)(?:'s|’s| is)\s+why\s+[^.!?]{3,70}\bmatter(?:ed|s)\b",
    ),
    Rule(
        "stranded-auxiliary",
        "Stranded auxiliary contrast",
        r"\b(?:didn['’]t|doesn['’]t|don['’]t|wasn['’]t|weren['’]t|isn['’]t|aren['’]t|hasn['’]t|"
        r"haven['’]t|hadn['’]t|wouldn['’]t|won['’]t|couldn['’]t|can['’]t|shouldn['’]t)\s*[.;]"
        r"|\b(?:did|does|was|were|is|are|has|have|had|would|will|could|can|should)\s+not\s*[.;]",
        min_count=3,
    ),
    Rule(
        "end-of-day",
        '"At the end of the day"',
        # the figurative use opens a clause; "runs at the end of the day" is a time
        r"(?:^|(?<=[.!?]\s)|(?<=\n))\s*At\s+the\s+end\s+of\s+the\s+day\b"
        r"|\bat\s+the\s+end\s+of\s+the\s+day\s*,"
        r"|\bwhen\s+all\s+is\s+said\s+and\s+done\b",
    ),
    Rule(
        "in-conclusion",
        '"In conclusion"',
        r"(?:^|(?<=[.!?]\s)|(?<=\n))\s*(?:In\s+conclusion|In\s+summary|To\s+summari[sz]e|"
        r"To\s+sum\s+up|All\s+in\s+all|In\s+closing)\b",
    ),
    Rule(
        "dive-in",
        '"Let\'s dive in"',
        r"\b(?:dive[sd]?|diving)\s+(?:deep\s+)?into\b|\bdeep[\s-]dive\b"
        r"|\blet['’]s\s+(?:dive|get\s+started|take\s+a\s+look|explore|jump\s+in|unpack)\b"
        r"|\bbuckle\s+up\b|\bread\s+on\b|\bstay\s+tuned\b",
    ),
    Rule(
        "when-it-comes-to",
        '"When it comes to"',
        r"\bwhen\s+it\s+comes\s+to\b",
    ),
    Rule(
        "whether-youre",
        '"Whether you\'re X or Y"',
        r"\bwhether\s+you(?:['’]re|\s+are)\b"
        r"|\bno\s+matter\s+(?:your|what\s+your|where\s+you|how\s+you)\b",
    ),
    Rule(
        "world-of",
        '"In the world of"',
        r"\bin\s+the\s+(?:world|realm)\s+of\b|\bwelcome\s+to\s+the\s+world\s+of\b",
    ),
    Rule(
        "takeaway",
        '"The key takeaway"',
        r"\bkey\s+takeaways?\b|\bthe\s+takeaway\s+(?:here\s+)?is\b"
        r"|\bthe\s+bottom\s+line\s+is\b|\bwhat\s+this\s+means\s+for\s+you\b",
    ),
    Rule(
        "think-of-it-as",
        '"Think of it as"',
        r"\bthink\s+of\s+(?:it|this|them|these)\s+as\b"
        r"|\bimagine\s+(?:a|an|you|for\s+a\s+moment)\b|\bpicture\s+this\b",
    ),
    Rule(
        "beauty-of",
        '"The beauty of X is"',
        r"\bthe\s+(?:beauty|magic|elegance|real\s+power|true\s+power)\s+(?:of\s+[^.!?]{1,40}\s+is|here\s+is)\b",
    ),
    Rule(
        "filler-opener",
        "Filler sentence opener",
        r"(?:^|(?<=[.!?]\s)|(?<=\n))\s*(?:Simply\s+put|In\s+essence|At\s+its\s+core|"
        r"Fundamentally|Needless\s+to\s+say|It\s+goes\s+without\s+saying|Make\s+no\s+mistake)\b",
    ),
    Rule(
        "furthermore",
        "Essay connective",
        r"(?:^|(?<=[.!?]\s)|(?<=\n))\s*(?:Moreover|Furthermore|Additionally|Notably|"
        r"Consequently|Nevertheless|Firstly|Secondly)\s*,",
        min_count=2,
    ),
    Rule(
        "antithesis",
        '"X rather than Y" frame',
        r"\brather\s+than\b|\binstead\s+of\b|\bas\s+opposed\s+to\b"
        r"|,\s*not\s+(?:a|an|the|by|as|to|because)\b",
        min_count=4,
        min_per_1k=3.5,
    ),
    Rule(
        "justification-tail",
        "Trailing justification clause",
        r",\s*(?:because|since|so\s+that|so\s+|which\s+means|which\s+is\s+why|"
        r"at\s+which\s+point)\b",
        min_count=5,
        min_per_1k=6.0,
    ),
    Rule(
        "count-preview",
        "Counted-list announcement",
        r"(?:^|(?<=\n)|(?<=[.!?]\s))\s*\*{0,2}(?:One|Two|Three|Four|Five|Six)\s+"
        r"(?:\w+\s+){0,2}(?:things?|ways?|cases?|limits?|mechanisms?|audiences?|"
        r"boundaries|reasons?|problems?|options?|caveats?|concerns?|questions?|steps?|"
        r"levels?|kinds?|shapes?|postures?|failures?|decisions?|choices?|areas?|places?|"
        r"parts?|pieces?|points?|lessons?|takeaways?|principles?|rules?|patterns?|"
        r"tradeoffs?|goals?|phases?|groups?|categories?|properties?)\b",
        min_count=3,
    ),
    Rule(
        "cleft-emphasis",
        '"X is what makes Y"',
        r"\bis\s+what\s+(?:makes|lets|allows|gives|keeps|does|matters|counts|separates)\b"
        r"|\bis\s+how\s+(?:someone|you|we|it|that|this|anyone)\b"
        r"|\bis\s+where\s+[^.!?]{0,40}?(?:comes?\s+from|lives?|happens?|begins?)\b"
        r"|\bis\s+the\s+(?:reason|thing)\s+that\b",
        min_count=2,
    ),
    Rule(
        "business-story",
        '"The X story"',
        r"\bthe\s+(?:\w+[\s-]){1,4}story\b",
        exclude=r"story\s+of|user\s+story|short\s+story|whole\s+story|story\s+points?",
    ),
    Rule(
        "figurative-inflation",
        "Inflated figure of speech",
        r"\bstops?\s+being\s+(?:hypothetical|theoretical|optional|a\s+problem)\b"
        r"|\bsurviv(?:e|ed|es|ing)\s+contact\s+with\b|\bdoes\s+the\s+heavy\s+lifting\b"
        r"|\bwhere\s+the\s+rubber\s+meets\b|\bunder\s+the\s+pressure\s+of\b"
        r"|\bthe\s+sharp\s+end\b|\bmoves?\s+the\s+needle\b",
    ),
    Rule(
        "vague-quantifier",
        "Unsourced quantifier",
        r"\balmost\s+(?:everyone|all|every|any)\b|\bthe\s+vast\s+majority\b"
        r"|\bmost\s+(?:people|users|teams|customers)\s+who\b|\ba\s+fraction\s+of\s+the\b"
        r"|\bfar\s+(?:cheaper|faster|better|more|less|easier|harder)\b"
        r"|\borders?\s+of\s+magnitude\b|\bnine\s+times\s+out\s+of\s+ten\b",
        min_count=2,
    ),
    Rule(
        "assistant-register",
        "Chat-assistant register",
        r"\bI\s+hope\s+this\s+helps\b|\bgreat\s+question\b|\byou(?:['’]re|\s+are)\s+absolutely\s+right\b"
        r"|\bhappy\s+to\s+help\b|\bfeel\s+free\s+to\s+(?:reach\s+out|ask|let\s+me\s+know)\b"
        r"|\bdon['’]t\s+hesitate\s+to\b|\blet\s+me\s+know\s+if\s+you\s+(?:have\s+any|need|would)\b"
        r"|(?:^|(?<=\n))\s*(?:Certainly|Absolutely|Of\s+course)\s*[!,]",
    ),
    # -- signs of AI writing -------------------------------------------------
    Rule(
        "ai-vocab",
        "AI vocabulary word",
        r"\b(?:delve[sd]?|delving|tapestry|interplay|garner(?:ed|ing|s)?|bolster(?:ed|ing|s)?|"
        r"vibrant|bustling|multifaceted|ever-evolving|testament|myriad|plethora|paramount|"
        r"intricate(?:ly)?|transformative|unwavering|indelible|beacon|cornerstone)\b",
    ),
    Rule(
        "ai-vocab-soft",
        "AI vocabulary word (ambiguous)",
        r"\b(?:meticulous(?:ly)?|pivotal|underscor(?:e|es|ed|ing)|seamless(?:ly)?|realm|"
        r"foster(?:ing|s|ed)?|harness(?:ing|es|ed)?|nuanced|holistic|profound(?:ly)?|"
        r"elevat(?:e|es|ed|ing)|unlock(?:s|ed|ing)?|showcas(?:e|es|ed|ing)|embark(?:ed|ing|s)?|"
        r"leverag(?:e|es|ed|ing)|crucial(?:ly)?|robust(?:ness)?|comprehensive(?:ly)?)\b",
        min_count=2,
    ),
    Rule(
        "not-just-but",
        '"Not just X, but Y"',
        r"\bnot\s+(?:just|only|merely|simply)\s+[^,;.!?]{1,50}[,;]?\s*but\s+(?:also\s+)?"
        r"|\bit(?:'s|’s| is)\s+not\s+[^—–.!?]{1,45}[—–]\s*it(?:'s|’s| is)\b"
        r"|\bit(?:'s|’s| is)\s+not\s+(?:about\s+)?[^.!?]{1,45}\.\s*It(?:'s|’s| is)\s+(?:about\s+)?",
    ),
    Rule(
        "important-to-note",
        '"It\'s important to note"',
        r"\bit\s+(?:is|was)\s+important\s+to\s+note\b|\bit(?:'s|’s)\s+important\s+to\s+note\b"
        r"|\b(?:it(?:'s|’s)\s+|it\s+is\s+)?worth\s+(?:noting|mentioning|pausing|considering|"
        r"asking|remembering)\b|\bshould\s+be\s+noted\b|\bimportantly,",
    ),
    Rule(
        "testament",
        '"Stands as a testament"',
        r"\b(?:stands|serves|stand|serve)\s+as\s+a\s+(?:testament|reminder|symbol|"
        r"powerful\s+\w+)\b|\bis\s+a\s+testament\s+to\b",
    ),
    Rule(
        "crucial-role",
        '"Plays a crucial role"',
        r"\bplay(?:s|ed|ing)?\s+an?\s+(?:crucial|pivotal|vital|key|significant|important|"
        r"central|essential|major)\s+role\b",
    ),
    Rule(
        "evolving-landscape",
        '"Ever-evolving landscape"',
        r"\bever[\s-](?:evolving|changing|shifting|expanding|growing)\b"
        r"|\b(?:evolving|changing|shifting|digital|modern|current|competitive)\s+landscape\b"
        r"|\bin\s+today(?:'s|’s)\s+(?:fast[\s-]paced|digital|modern|competitive)\b"
        r"|\bin\s+an\s+era\s+(?:of|where)\b",
    ),
    Rule(
        "experts-argue",
        '"Experts argue"',
        r"\b(?:experts|critics|observers|analysts|commentators|researchers|scholars)\s+"
        r"(?:argue|say|agree|note|noted|believe|suggest|point\s+out|contend|warn)\b"
        r"|\bsome\s+(?:critics|experts|observers)\s+have\s+\w+\b"
        r"|\bindustry\s+reports\s+indicate\b|\b(?:studies|reports|surveys)\s+(?:show|suggest|indicate)\b"
        r"|\bit\s+is\s+(?:widely|generally)\s+(?:believed|accepted|agreed)\b",
    ),
    Rule(
        "despite-challenges",
        '"Despite these challenges"',
        r"\bdespite\s+(?:these|its|the)\s+challenges\b|\bfac(?:es|ed|ing)\s+(?:several|a\s+number\s+of|"
        r"numerous|significant)\s+challenges\b|\bchallenges\s+remain\b|\bremains\s+to\s+be\s+seen\b"
        r"|\b(?:only\s+)?time\s+will\s+tell\b|\bnot\s+without\s+its\s+challenges\b",
    ),
    Rule(
        "participle-tail",
        "Participle sentence tail",
        r",\s*(?:highlighting|underscoring|showcasing|reflecting|emphasi[sz]ing|demonstrating|"
        r"illustrating|signal(?:l)?ing|ensuring|cementing|solidifying|marking|paving|"
        r"contributing\s+to|allowing\s+for|making\s+it\s+(?:a|an|one))\b",
    ),
    Rule(
        "hype",
        "Marketing hype word",
        r"\b(?:revolutioni[sz](?:e|es|ed|ing)|supercharg(?:e|es|ed|ing)|unleash(?:es|ed|ing)?|"
        r"empower(?:s|ed|ing)?|streamlin(?:e|es|ed|ing)|game[\s-]chang(?:er|ing)|"
        r"cutting[\s-]edge|state[\s-]of[\s-]the[\s-]art|best[\s-]in[\s-]class|next[\s-]level|"
        r"effortless(?:ly)?|blazing(?:ly)?[\s-]fast|lightning[\s-]fast|world[\s-]class|"
        r"industry[\s-]leading|battle[\s-]tested|future[\s-]proof(?:ed|ing)?)\b",
    ),
    Rule(
        "tidy-closer",
        "Tidy closing sentence",
        r"(?:^|(?<=\n)|(?<=[.!?]\s))(?:ultimately|in the end|at the end of the day|"
        r"in conclusion|all in all)\b",
    ),
    Rule(
        "promotional",
        "Promotional boilerplate",
        r"\bnestled\s+(?:in|among|between)\b|\bin\s+the\s+heart\s+of\b"
        r"|\brich\s+(?:tapestry|heritage|history|culture)\b|\bhidden\s+gem\b"
        r"|\bboast(?:s|ed|ing)?\s+(?:a|an|its|some|impressive|over)\b|\bbreathtaking\b"
        r"|\bstunning\s+(?:views?|scenery|architecture)\b|\bmust[\s-]visit\b"
        r"|\ba\s+(?:true|real)\s+(?:testament|delight|treat)\b|\bvibrant\s+(?:culture|community|city)\b",
    ),
    Rule(
        "chatbot-leftover",
        "Chatbot leftover",
        r"\bas\s+an\s+AI\s+(?:language\s+)?model\b|\bas\s+of\s+my\s+last\s+(?:update|training)\b"
        r"|\bknowledge\s+cut[\s-]?off\b|\bI\s+(?:cannot|can't)\s+browse\s+the\s+internet\b"
        r"|oaicite|contentReference|turn\d+(?:search|view|news)\d*|utm_source=|:contentReference",
    ),
    # -- optional ------------------------------------------------------------
    Rule(
        "colon-triple",
        "Colon opening onto a triple",
        r":\s+[^,:;.!?\n]{2,40},\s+[^,:;.!?\n]{2,40},\s+(?:and\s+|or\s+)?[^,:;.!?\n]{2,40}",
        optional=True,
    ),
    Rule(
        "em-dash",
        "Em-dash",
        r"[—–]|(?<=\w)\s--\s(?=\w)",
        optional=True,
        density_only=True,
    ),
]

# rules found by looking at markdown structure rather than by regex over prose
STRUCTURAL_RULES: list[Rule] = [
    Rule("bold-lead-bullets", "Bold-label bullet run", "", min_count=1),
    Rule("bold-lead-paragraph", "Bold-label paragraph slot", "", min_count=3),
    Rule("emoji-decoration", "Emoji in heading or bullet", ""),
    Rule("short-section", "Heading over a very short section", "", min_count=2),
]

SENTENCE_RULES: list[Rule] = [
    Rule("stacked-questions", "Stacked rhetorical questions", ""),
    Rule("repeated-openers", "Repeated sentence openers", ""),
    Rule("echoing-run", "Echoing sentence run", ""),
    Rule("repeated-frame", "Repeated sentence frame", ""),
    Rule("fragment-run", "Run of sentence fragments", ""),
    Rule("choppy-run", "Run of short declarative sentences", ""),
]

ALL_RULES: list[Rule] = RULES + STRUCTURAL_RULES + SENTENCE_RULES
BY_ID: dict[str, Rule] = {r.id: r for r in ALL_RULES}
DESCRIPTIONS = {r.id: r.description for r in ALL_RULES}

COMPILED = {r.id: re.compile(r.pattern, I) for r in RULES}
COMPILED_EXCL = {r.id: re.compile(r.exclude, I) for r in RULES if r.exclude}
COMPILED_COUNT = {r.id: re.compile(r.count_pattern, I) for r in RULES if r.count_pattern}

# ------------------------------------------------------------- masking prose

FENCE_RE = re.compile(r"^(\s*)(`{3,}|~{3,})")
INLINE_CODE_RE = re.compile(r"`[^`\n]+`")
LINK_TARGET_RE = re.compile(r"\]\([^)\s]+\)")
URL_RE = re.compile(r"\bhttps?://\S+")
HTML_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
HTML_TAG_RE = re.compile(r"</?[a-zA-Z][^>\n]*>")
RST_DIRECTIVE_RE = re.compile(r"^(\s*)\.\.(?:\s|$)")
RST_UNDERLINE_RE = re.compile(r"^\s*([=~^\"'`*+#_:.-])\1{2,}\s*$")
LIST_ITEM_RE = re.compile(r"^(?:[-*+]|\d+[.)])\s")


def _blank(line: str) -> str:
    return " " * len(line)


def _docstring_lines(tree: ast.AST) -> set[int]:
    lines = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef):
            first = node.body[0] if node.body else None
            if (
                isinstance(first, ast.Expr)
                and isinstance(first.value, ast.Constant)
                and isinstance(first.value.value, str)
            ):
                lines.add(first.lineno)
    return lines


def python_prose(source: str) -> str:
    """Keep only the comments and docstrings of Python source.

    Line numbers and the columns of kept text match the source. Lines with no
    prose come back empty. The sentence splitter then treats two comments with
    code between them as two paragraphs. Raises SyntaxError or
    tokenize.TokenError if the source doesn't parse.
    """
    docstring_lines = _docstring_lines(ast.parse(source))
    lines = source.split("\n")
    keep = [[False] * len(line) for line in lines]
    in_docstring = [False] * len(lines)

    def mark(row: int, start: int, end_row: int, end: int) -> None:
        for r in range(row, end_row + 1):
            lo = start if r == row else 0
            hi = end if r == end_row else len(lines[r])
            for c in range(lo, hi):
                keep[r][c] = True

    seen_docstring_rows = set()
    for tok in tokenize.generate_tokens(io.StringIO(source).readline):
        (row, col), (end_row, end_col) = tok.start, tok.end
        if tok.type == tokenize.COMMENT and not (row == 1 and tok.string.startswith("#!")):
            mark(row - 1, col + 1, end_row - 1, end_col)
        elif (
            tok.type == tokenize.STRING
            and row in docstring_lines
            and row not in seen_docstring_rows
        ):
            seen_docstring_rows.add(row)
            body = tok.string.lstrip("rRuUbBfF")
            quote = body[:3] if body[:3] in ('"""', "'''") else body[0]
            prefix = len(tok.string) - len(body) + len(quote)
            mark(row - 1, col + prefix, end_row - 1, end_col - len(quote))
            for r in range(row - 1, end_row):
                in_docstring[r] = True

    out = []
    in_doctest = False  # a doctest's expected output runs until the next blank line
    for r, line in enumerate(lines):
        text = "".join(ch if keep[r][c] else " " for c, ch in enumerate(line))
        stripped = text.strip()
        in_doctest = (
            in_docstring[r] and bool(stripped) and (in_doctest or stripped.startswith(">>>"))
        )
        if not stripped or in_doctest or RST_UNDERLINE_RE.match(text):
            text = ""
        out.append(text)
    return "\n".join(out)


def mask_inline(text: str, keep_urls: bool = False) -> str:
    """Blank out HTML tags, inline code, link targets and URLs, preserving offsets.

    ``mask_non_prose`` ends with this, and it's all the masking the output of
    ``python_prose`` needs.
    """
    text = HTML_TAG_RE.sub(lambda m: _blank(m.group(0)), text)
    text = INLINE_CODE_RE.sub(lambda m: _blank(m.group(0)), text)
    text = LINK_TARGET_RE.sub(lambda m: _blank(m.group(0)), text)
    if not keep_urls:
        text = URL_RE.sub(lambda m: _blank(m.group(0)), text)
    return text


def mask_non_prose(text: str, keep_urls: bool = False) -> str:
    """Blank out code and quotes, preserving offsets so positions stay valid.

    Indented blocks are only code when they do not hang off a list item, so
    that wrapped list continuations stay in the prose the rules see.
    """
    text = HTML_COMMENT_RE.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), text)
    out: list[str] = []
    fence: str | None = None
    directive_indent: int | None = None
    prev_blank = True
    in_code_block = False
    in_list = False
    for line in text.split("\n"):
        stripped = line.strip()
        blank = not stripped
        indent = len(line) - len(line.lstrip())

        if fence is not None:
            out.append(_blank(line))
            if stripped.startswith(fence):
                fence = None
            prev_blank = False
            continue

        if blank:
            out.append(line)
            prev_blank = True
            continue

        m = FENCE_RE.match(line)
        if m:
            fence = m.group(2)[0] * 3
            out.append(_blank(line))
            prev_blank = False
            in_code_block = False
            continue

        if directive_indent is not None:
            if indent > directive_indent:
                out.append(_blank(line))
                prev_blank = False
                continue
            directive_indent = None

        if RST_DIRECTIVE_RE.match(line):
            directive_indent = indent
            out.append(_blank(line))
            prev_blank = False
            in_code_block = False
            continue

        if RST_UNDERLINE_RE.match(line) or stripped.startswith(">"):
            out.append(_blank(line))
            prev_blank = False
            in_code_block = False
            continue

        body = line.lstrip()
        is_list = bool(LIST_ITEM_RE.match(body))
        code_indent = 8 if in_list else 4
        if indent >= code_indent and (in_code_block or prev_blank) and not is_list:
            out.append(_blank(line))
            in_code_block = True
            prev_blank = False
            continue

        if indent < code_indent:
            in_code_block = False
        if is_list:
            in_list = True
        elif indent == 0:
            in_list = False
        out.append(line)
        prev_blank = False

    return mask_inline("\n".join(out), keep_urls)


# ------------------------------------------------------- sentence-level rules

SENTENCE_END_RE = re.compile(r"(?<=[.!?])[\"')\]]*\s+|\n{2,}")
ABBREV = {
    "e.g",
    "i.e",
    "etc",
    "vs",
    "cf",
    "al",
    "mr",
    "mrs",
    "ms",
    "dr",
    "st",
    "fig",
    "no",
    "approx",
    "ca",
    "ibid",
}
# digits belong to the token: L1 and L4 are different openers, not two "L"s
WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9'’-]*")

FUNCTION_WORDS = {
    "a",
    "an",
    "the",
    "is",
    "are",
    "was",
    "were",
    "be",
    "been",
    "being",
    "am",
    "in",
    "on",
    "at",
    "of",
    "to",
    "for",
    "with",
    "from",
    "by",
    "into",
    "onto",
    "and",
    "or",
    "but",
    "not",
    "no",
    "as",
    "that",
    "this",
    "these",
    "those",
    "it",
    "its",
    "there",
    "here",
    "you",
    "your",
    "we",
    "our",
    "they",
    "their",
    "i",
    "he",
    "she",
    "them",
    "has",
    "have",
    "had",
    "do",
    "does",
    "did",
    "can",
    "will",
    "would",
    "should",
    "could",
    "may",
    "might",
    "must",
    "if",
    "when",
    "than",
    "then",
    "so",
    "just",
    "only",
    "all",
    "every",
    "each",
    "some",
    "more",
    "most",
    "one",
    "two",
    "three",
    "up",
    "out",
    "over",
    "about",
}

OPENER_STOPLIST = {
    "the",
    "a",
    "an",
    "it",
    "this",
    "that",
    "these",
    "those",
    "i",
    "we",
    "you",
    "they",
    "he",
    "she",
    "there",
    "and",
    "but",
    "so",
    "if",
    "in",
    "to",
    "for",
    "of",
    "on",
    "at",
    "as",
    "its",
    "his",
    "her",
    "their",
    "our",
    "my",
    "your",
}


@dataclass
class Sentence:
    text: str
    start: int


STRUCTURAL_RE = re.compile(r"^(?:#{1,6}\s|\||[-*+]\s|\d+[.)]\s|!\[|\[!|:\w+:|\.\.\s)")


def is_structural(text: str) -> bool:
    """Headings, table rows and list items. Parallelism there is deliberate
    formatting, not an echoing run, so they break runs instead of joining them."""
    return bool(STRUCTURAL_RE.match(text.strip()))


def _tail_word(text: str) -> str:
    words = WORD_RE.findall(text)
    return words[-1].lower() if words else ""


def split_sentences(masked: str) -> list[Sentence]:
    sentences: list[Sentence] = []
    pos = 0
    for part in SENTENCE_END_RE.split(masked):
        if part is None:
            continue
        idx = masked.find(part, pos) if part else pos
        if idx < 0:
            idx = pos
        stripped = part.strip()
        if stripped:
            offset = idx + (len(part) - len(part.lstrip()))
            sentences.append(Sentence(stripped, offset))
        pos = idx + len(part)
    # rejoin fragments that were split on an abbreviation
    merged: list[Sentence] = []
    for s in sentences:
        if merged and WORD_RE.search(s.text) and _tail_word(merged[-1].text) in ABBREV:
            merged[-1] = Sentence(merged[-1].text + " " + s.text, merged[-1].start)
            continue
        merged.append(s)
    return merged


def skeleton(sentence: str) -> tuple[str, ...] | None:
    words = [w.lower() for w in WORD_RE.findall(sentence)]
    if len(words) < 5:
        return None
    skel = tuple(w if w in FUNCTION_WORDS else "*" for w in words)
    if sum(1 for t in skel if t != "*") < 3:
        return None
    return skel


def first_significant_word(sentence: str) -> str | None:
    words = WORD_RE.findall(sentence)
    return words[0].lower() if words else None


def scan_sentences(masked: str) -> list[dict]:
    hits: list[dict] = []
    sentences = split_sentences(masked)

    # stacked rhetorical questions
    run: list[Sentence] = []
    for s in sentences + [Sentence("", len(masked))]:
        if s.text.endswith("?"):
            run.append(s)
            continue
        if len(run) >= 2:
            hits.append(_run_hit("stacked-questions", run))
        run = []

    # repeated sentence openers
    run = []
    current: str | None = None
    for s in sentences + [Sentence("", len(masked))]:
        word = None if is_structural(s.text) else first_significant_word(s.text)
        if word is not None and word == current:
            run.append(s)
            continue
        if len(run) >= 3 and current not in OPENER_STOPLIST:
            hits.append(_run_hit("repeated-openers", run))
        current = word
        run = [s] if word else []

    # echoing sentence runs
    run = []
    current_skel: tuple[str, ...] | None = None
    for s in sentences + [Sentence("", len(masked))]:
        skel = None if is_structural(s.text) else skeleton(s.text)
        if skel is not None and skel == current_skel:
            run.append(s)
            continue
        if len(run) >= 2:
            hits.append(_run_hit("echoing-run", run))
        current_skel = skel
        run = [s] if skel else []

    # repeated sentence frames: consecutive sentences sharing both a content word
    # near the front and the same connective. Catches the parallelism an exact
    # skeleton match cannot, where the clauses hanging off the frame differ in
    # length. Both signals are needed: sharing only the noun is just staying on
    # topic, and sharing only the connective is just a habit.
    run = []
    shared: set[str] = set()
    shared_conn: set[str] = set()
    for s in sentences + [Sentence("", len(masked))]:
        head = None if is_structural(s.text) else frame_words(s.text)
        conn = connectives(s.text) if head else set()
        if head and conn and (not run or (shared & head and shared_conn & conn)):
            shared = (shared & head) if run else head
            shared_conn = (shared_conn & conn) if run else conn
            run.append(s)
            continue
        if len(run) >= 3:
            hits.append(_run_hit("repeated-frame", run, sorted(shared)[:1]))
        run = [s] if (head and conn) else []
        shared = head or set()
        shared_conn = conn

    # runs of verbless fragments, which is how a counted list gets padded out.
    # Short ones are bylines, dates and captions, not prose.
    run = []
    for s in sentences + [Sentence("", len(masked))]:
        if (
            s.text
            and not is_structural(s.text)
            and len(WORD_RE.findall(s.text)) >= 5
            and not has_finite_verb(s.text)
        ):
            run.append(s)
            continue
        if len(run) >= 2:
            hits.append(_run_hit("fragment-run", run))
        run = []

    # runs of short declarative sentences in one paragraph, one idea chopped
    # into beats. A single short sentence is emphasis; two or more in a row
    # is the staccato rhythm LLM prose uses to sound decisive.
    def is_short(s: Sentence) -> bool:
        return (
            bool(s.text)
            and not is_structural(s.text)
            and s.text.endswith(".")
            and 2 <= len(WORD_RE.findall(s.text)) <= 6
        )

    run = []
    for s in sentences + [Sentence("", len(masked))]:
        same_para = bool(run) and "\n\n" not in masked[run[-1].start : s.start]
        if is_short(s) and (not run or same_para):
            run.append(s)
            continue
        if len(run) >= 2:
            hits.append(_run_hit("choppy-run", run))
        run = [s] if is_short(s) else []

    return hits


def frame_words(sentence: str, depth: int = 4) -> set[str]:
    """The content words near the front of a sentence, which is the part a
    parallel run holds fixed while the rest of the sentence varies."""
    words = [w.lower() for w in WORD_RE.findall(sentence)]
    return {w for w in words[:depth] if w not in FUNCTION_WORDS and len(w) > 2}


FRAME_CONNECTIVES = (
    ", which means",
    ", which is",
    ", so that",
    ", at which point",
    ", rather than",
    ", instead of",
    ", because",
    ", since",
    ", so ",
    " means ",
)


def connectives(sentence: str) -> set[str]:
    low = sentence.lower()
    return {c for c in FRAME_CONNECTIVES if c in low}


AUXILIARIES = {
    "is",
    "are",
    "was",
    "were",
    "be",
    "been",
    "being",
    "am",
    "has",
    "have",
    "had",
    "do",
    "does",
    "did",
    "can",
    "will",
    "would",
    "should",
    "could",
    "may",
    "might",
    "must",
    "shall",
    "ought",
}


def has_finite_verb(sentence: str) -> bool:
    """Cheap and deliberately generous: an auxiliary, or any inflected word.
    A sentence this says has no verb is almost always a real fragment."""
    for word in WORD_RE.findall(sentence.lower()):
        if word in AUXILIARIES:
            return True
        if len(word) > 3 and word.endswith(("s", "ed", "ing")):
            return True
    return False


def _run_hit(rule_id: str, run: list[Sentence], extra: list[str] | None = None) -> dict:
    text = " ".join(s.text for s in run)
    if extra:
        text = f'on "{extra[0]}": {text}'
    return {
        "rule": rule_id,
        "start": run[0].start,
        "match": text,
        "count": len(run),
    }


# ----------------------------------------------------------- structural rules

# pictographs and the tick/cross/warning dingbats, not arrows or symbols: "↔"
# in a title is notation, "🚀" is decoration
EMOJI_RE = re.compile("[\U0001f300-\U0001faff✅❌⚠✨⭐❗✔ℹ🌟]")
HEADING_RE = re.compile(r"^\s*#{1,6}\s")
# unordered only: a numbered list of bolded step names is how plans are written
BOLD_LEAD_RE = re.compile(r"^\s*[-*+]\s+\*\*[^*\n]{2,60}\*\*\s*[:—–-]?\s+\S")
# a bolded phrase is a definition-list term; a bolded *sentence* is a slot label
# the model wrote to look organised ("**What it delivers.**")
BOLD_PARA_RE = re.compile(r"^\*\*[^*\n]{2,80}[^*\s.]\.\*\*")


SHORT_SECTION_WORDS = 40


def scan_structure(masked: str) -> list[dict]:
    """Markdown shapes LLMs reach for: every bullet a bold label, emoji in
    headings. Runs of bold-label bullets are one hit, not one per bullet."""
    hits: list[dict] = []
    offset = 0
    run: list[tuple[int, str]] = []

    def flush(run: list[tuple[int, str]]) -> None:
        if len(run) >= 3:
            hits.append(
                {
                    "rule": "bold-lead-bullets",
                    "start": run[0][0],
                    "match": run[0][1].strip(),
                    "count": len(run),
                }
            )

    # headings over sections too short to need one. A heading followed directly
    # by a deeper subheading is a parent, not a short section.
    section: list | None = None  # [start, level, text, words]

    def close_section(next_level: int | None) -> None:
        if section is None:
            return
        start, level, text, words = section
        if (next_level is None or next_level <= level) and words < SHORT_SECTION_WORDS:
            hits.append({"rule": "short-section", "start": start, "match": text, "count": 0})

    for line in masked.split("\n"):
        stripped = line.strip()
        heading = HEADING_RE.match(line)
        if heading:
            level = len(stripped) - len(stripped.lstrip("#"))
            close_section(level)
            section = [offset, level, stripped, 0]
        elif section is not None and stripped:
            section[3] += len(WORD_RE.findall(stripped))
        if BOLD_LEAD_RE.match(line):
            run.append((offset, line))
        elif stripped:
            flush(run)
            run = []
        if BOLD_PARA_RE.match(line):
            hits.append(
                {
                    "rule": "bold-lead-paragraph",
                    "start": offset,
                    "match": BOLD_PARA_RE.match(line).group(0),
                    "count": 0,
                }
            )
        if (HEADING_RE.match(line) or LIST_ITEM_RE.match(stripped)) and EMOJI_RE.search(line):
            hits.append(
                {
                    "rule": "emoji-decoration",
                    "start": offset,
                    "match": stripped,
                    "count": 0,
                }
            )
        offset += len(line) + 1
    flush(run)
    close_section(None)
    return hits


# ------------------------------------------------------------------ reporting


@dataclass
class Hit:
    rule: str
    line: int
    col: int
    match: str
    count: int = 0
    suppressed: bool = False


@dataclass
class FileReport:
    path: str
    words: int
    hits: list[Hit] = field(default_factory=list)
    error: str | None = None

    @property
    def listed(self) -> list[Hit]:
        return [h for h in self.hits if not h.suppressed and not BY_ID[h.rule].density_only]

    @property
    def suppressed(self) -> list[Hit]:
        return [h for h in self.hits if h.suppressed]

    def density_counts(self) -> dict[str, int]:
        counts: dict[str, int] = {}
        for h in self.hits:
            if BY_ID[h.rule].density_only:
                counts[h.rule] = counts.get(h.rule, 0) + 1
        return counts


def line_col(text: str, offset: int) -> tuple[int, int]:
    line = text.count("\n", 0, offset) + 1
    last_nl = text.rfind("\n", 0, offset)
    return line, offset - last_nl


def select_rules(only: set[str], skip: set[str], optional: set[str]) -> list[Rule]:
    chosen = []
    for rule in ALL_RULES:
        if only:
            if rule.id in only:
                chosen.append(rule)
            continue
        if rule.id in skip:
            continue
        if rule.optional and rule.id not in optional:
            continue
        chosen.append(rule)
    return chosen


def scan_text(
    text: str,
    path: str,
    rules: list[Rule],
    context: int,
    all_hits: bool = False,
    python: bool = False,
) -> FileReport:
    if python:
        # Hits report positions in `text`, so swap the source for its prose,
        # which keeps the same line numbers and columns.
        text = python_prose(text)
        mask = mask_inline
        rules = [r for r in rules if r.id not in COMMENT_SKIP]
    else:
        mask = mask_non_prose
    masked = mask(text)
    masked_with_urls = mask(text, keep_urls=True)
    report = FileReport(path=path, words=len(WORD_RE.findall(masked)))
    active = {r.id for r in rules}

    for rule in rules:
        if rule.id not in COMPILED:
            continue
        haystack = masked_with_urls if rule.id == "chatbot-leftover" else masked
        excl = COMPILED_EXCL.get(rule.id)
        counter = COMPILED_COUNT.get(rule.id)
        for m in COMPILED[rule.id].finditer(haystack):
            snippet = re.sub(r"\s+", " ", m.group(0)).strip()
            window = haystack[max(0, m.start() - 20) : m.end() + 20]
            if excl and excl.search(window):
                continue
            line, col = line_col(text, m.start())
            count = len(counter.findall(snippet)) if counter else 0
            report.hits.append(Hit(rule.id, line, col, snippet[:context], count))

    for raw in scan_sentences(masked) + scan_structure(masked):
        if raw["rule"] not in active:
            continue
        line, col = line_col(text, raw["start"])
        match = re.sub(r"\s+", " ", raw["match"])[:context]
        report.hits.append(Hit(raw["rule"], line, col, match, raw["count"]))

    if not all_hits:
        per_rule: dict[str, int] = {}
        for h in report.hits:
            per_rule[h.rule] = per_rule.get(h.rule, 0) + 1
        for h in report.hits:
            rule = BY_ID[h.rule]
            rate = per_rule[h.rule] / report.words * 1000 if report.words else 0
            if per_rule[h.rule] < rule.min_count or rate < rule.min_per_1k:
                h.suppressed = True

    report.hits.sort(key=lambda h: (h.line, h.col))
    return report


def plural(n: int, word: str) -> str:
    return f"{n} {word}" if n == 1 else f"{n} {word}s"


def print_report(report: FileReport) -> None:
    header = f"{report.path}  ({plural(report.words, 'word')})"
    print(header)
    print("-" * len(header))
    if report.error:
        print(f"error: {report.error}\n")
        return

    listed = report.listed
    by_rule: dict[str, list[Hit]] = {}
    for h in listed:
        by_rule.setdefault(h.rule, []).append(h)

    for rule in sorted(by_rule, key=lambda r: (-len(by_rule[r]), r)):
        hits = by_rule[rule]
        print(f"\n[{rule}] {DESCRIPTIONS[rule]}  ({len(hits)})")
        for h in hits:
            badge = f" x{h.count}" if h.count else ""
            print(f"  {h.line}:{h.col}{badge}  {h.match}")

    total = len(listed)
    if not total:
        print("clean")
    else:
        per_1k = total / report.words * 1000 if report.words else 0
        print(
            f"\n{plural(total, 'hit')} across {plural(len(by_rule), 'pattern')} "
            f"({per_1k:.1f} per 1000 words)"
        )

    for rule, count in sorted(report.density_counts().items()):
        rate = count / report.words * 1000 if report.words else 0
        verdict = "high" if rate > 6.7 else "ok"
        print(
            f"{DESCRIPTIONS[rule].lower()} density: {count} in "
            f"{plural(report.words, 'word')} ({rate:.1f} per 1000, {verdict})"
        )

    below: dict[str, int] = {}
    for h in report.suppressed:
        below[h.rule] = below.get(h.rule, 0) + 1
    if below:
        parts = [f"{r} x{n} (needs {BY_ID[r].min_count})" for r, n in sorted(below.items())]
        print(f"below threshold, not listed: {', '.join(parts)}")
    print()


def print_summary(reports: list[FileReport]) -> None:
    by_rule: dict[str, int] = {}
    words = 0
    for r in reports:
        words += r.words
        for h in r.listed:
            by_rule[h.rule] = by_rule.get(h.rule, 0) + 1
    total = sum(by_rule.values())
    header = f"summary: {plural(len(reports), 'file')}, {plural(words, 'word')}"
    print(header)
    print("=" * len(header))
    for rule, n in sorted(by_rule.items(), key=lambda kv: (-kv[1], kv[0])):
        print(f"  {n:5d}  {rule}")
    per_1k = total / words * 1000 if words else 0
    print(f"  {total:5d}  total ({per_1k:.1f} per 1000 words)\n")


def print_rules(optional: set[str]) -> None:
    for group, rules in (
        ("regex", [r for r in RULES if not r.optional]),
        ("structural", STRUCTURAL_RULES),
        ("sentence", SENTENCE_RULES),
        ("optional", [r for r in RULES if r.optional]),
    ):
        print(f"\n{group}")
        for rule in rules:
            note = f"  [{rule.note}]" if rule.note else ""
            print(f"  {rule.id:22s} {rule.description}{note}")
    print()


def catalogue_entries(text: str) -> tuple[dict[str, str], str]:
    """Split patterns.md into ``{rule id: entry}`` and the no-regex section.

    An entry runs from its ``###`` heading to the next heading or ``---`` line.
    A heading that names two ids maps both to the same entry.
    """
    entries: dict[str, str] = {}
    judgment: list[str] = []
    ids: list[str] = []
    block: list[str] = []
    in_judgment = False

    def flush() -> None:
        for rule_id in ids:
            entries[rule_id] = "\n".join(block).strip()

    for line in text.splitlines():
        if line.startswith("## "):
            flush()
            ids, block = [], []
            in_judgment = line.strip() == "## Patterns no regex catches"
        if in_judgment:
            judgment.append(line)
        elif line.startswith("### ") or line.strip() == "---":
            flush()
            ids = re.findall(r"`\[([a-z0-9-]+)\]`", line) if line.startswith("### ") else []
            block = [line]
        elif ids:
            block.append(line)
    flush()
    return entries, "\n".join(judgment).strip()


def check_catalogue(path: str = CATALOGUE) -> list[str]:
    with open(path, encoding="utf-8") as fh:
        entries, _ = catalogue_entries(fh.read())
    missing = sorted(set(BY_ID) - set(entries))
    unknown = sorted(set(entries) - set(BY_ID))
    return [f"rule [{r}] has no entry in patterns.md" for r in missing] + [
        f"patterns.md has an entry for [{r}], which no rule reports" for r in unknown
    ]


def print_explanation(rule_ids: set[str], path: str = CATALOGUE) -> None:
    with open(path, encoding="utf-8") as fh:
        entries, judgment = catalogue_entries(fh.read())
    shown = []
    for rule_id in sorted(rule_ids):
        entry = entries.get(rule_id)
        if entry and entry not in shown:
            shown.append(entry)
    print("\n\n".join([*shown, judgment]), end="\n\n")


def collect_paths(paths: list[str], python: bool = False) -> list[str]:
    suffixes = TEXT_SUFFIXES | {".py"} if python else TEXT_SUFFIXES
    found: list[str] = []
    for path in paths:
        if path == "-" or not os.path.isdir(path):
            found.append(path)
            continue
        for root, dirs, files in os.walk(path):
            dirs[:] = sorted(d for d in dirs if not d.startswith(".") and d != "node_modules")
            for name in sorted(files):
                if os.path.splitext(name)[1].lower() in suffixes:
                    found.append(os.path.join(root, name))
    return found


def main(argv: list[str]) -> int:
    ap = argparse.ArgumentParser(
        add_help=True, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    ap.add_argument("paths", nargs="*", metavar="FILE")
    ap.add_argument("--colon-triple", action="store_true")
    ap.add_argument("--em-dash", action="store_true")
    ap.add_argument("--all", action="store_true")
    ap.add_argument("--only", default="")
    ap.add_argument("--skip", default="")
    ap.add_argument("--all-hits", action="store_true")
    ap.add_argument("--list-rules", action="store_true")
    ap.add_argument("--summary", action="store_true")
    ap.add_argument("--quiet", action="store_true")
    ap.add_argument("--json", action="store_true")
    ap.add_argument("--context", type=int, default=90)
    ap.add_argument("--comments", action="store_true")
    ap.add_argument("--explain", action="store_true")
    ap.add_argument("--check-catalogue", action="store_true")
    args = ap.parse_args(argv)

    if args.check_catalogue:
        problems = check_catalogue()
        for problem in problems:
            print(problem)
        return 1 if problems else 0

    optional = {r.id for r in RULES if r.optional} if args.all else set()
    if args.colon_triple:
        optional.add("colon-triple")
    if args.em_dash:
        optional.add("em-dash")

    if args.list_rules:
        print_rules(optional)
        return 0
    if not args.paths:
        ap.error("no files given")

    only = {s.strip() for s in args.only.split(",") if s.strip()}
    skip = {s.strip() for s in args.skip.split(",") if s.strip()}
    unknown = (only | skip) - set(BY_ID)
    if unknown:
        ap.error(f"unknown rule(s): {', '.join(sorted(unknown))}")
    rules = select_rules(only, skip, optional)

    reports: list[FileReport] = []
    for path in collect_paths(args.paths, python=args.comments):
        if path == "-":
            text, label = sys.stdin.read(), "<stdin>"
        else:
            label = path
            try:
                with open(path, encoding="utf-8", errors="replace") as fh:
                    text = fh.read()
            except OSError as exc:
                reports.append(FileReport(path=label, words=0, error=str(exc)))
                continue
        try:
            is_python = path.endswith(".py")
            reports.append(
                scan_text(text, label, rules, args.context, args.all_hits, python=is_python)
            )
        except Exception as exc:  # noqa: BLE001 - a broken rule must not lose the other files
            reports.append(FileReport(path=label, words=0, error=f"{type(exc).__name__}: {exc}"))

    if args.json:
        print(
            json.dumps(
                [
                    {
                        "path": r.path,
                        "words": r.words,
                        "error": r.error,
                        "hits": [
                            {
                                "rule": h.rule,
                                "description": DESCRIPTIONS[h.rule],
                                "line": h.line,
                                "col": h.col,
                                "match": h.match,
                                "count": h.count,
                                "suppressed": h.suppressed,
                            }
                            for h in r.hits
                        ],
                        "totals": {
                            "listed": len(r.listed),
                            "suppressed": len(r.suppressed),
                            "density": r.density_counts(),
                        },
                    }
                    for r in reports
                ],
                indent=2,
            )
        )
    else:
        shown = [r for r in reports if r.listed or r.error or not args.quiet]
        for r in shown:
            print_report(r)
        if args.summary and len(reports) > 1:
            print_summary(reports)
        fired = {h.rule for r in reports for h in r.listed}
        if args.explain and fired:
            print_explanation(fired)

    if all(r.error for r in reports):
        for r in reports:
            print(f"{r.path}: {r.error}", file=sys.stderr)
        return 2
    return 1 if any(r.listed for r in reports) else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))