Bug: Inconsistent help-flag handling across graphify subcommands #821

#!/usr/bin/env python3
"""
Probe every `graphify` subcommand with `-h`, `--help`, `-?`, and a deliberately
bogus flag, then summarize which commands emit a usage/help message vs.
crash with a Traceback vs. silently do nothing.

Output:
  /tmp/graphify-cmd-help/raw/<cmd>__<flag>.txt   per-invocation raw output
  /tmp/graphify-cmd-help/results.md              markdown summary table
  /tmp/graphify-cmd-help/results.csv             same data, csv form

Run:
  python3 /tmp/graphify-cmd-help/test_help.py
"""

import csv
import datetime
import re
import subprocess
import sys
from pathlib import Path

OUTDIR = Path("/tmp/graphify-cmd-help")
RAWDIR = OUTDIR / "raw"
RAWDIR.mkdir(parents=True, exist_ok=True)

# Subcommands extracted from `graphify --help` output (2026-05-11).
# Multi-word commands (e.g. "global add", "hook install") are listed in full.
COMMANDS = [
    "install",
    "uninstall",
    "path",
    "explain",
    "clone",
    "merge-driver",
    "merge-graphs",
    "add",
    "watch",
    "update",
    "cluster-only",
    "query",
    "save-result",
    "check-update",
    "tree",
    "extract",
    "global add",
    "global remove",
    "global list",
    "global path",
    "benchmark",
    "export callflow-html",
    "hook install",
    "hook uninstall",
    "hook status",
    "gemini install",
    "gemini uninstall",
    "cursor install",
    "cursor uninstall",
    "claude install",
    "claude uninstall",
    "codex install",
    "codex uninstall",
    "opencode install",
    "opencode uninstall",
    "aider install",
    "aider uninstall",
    "copilot install",
    "copilot uninstall",
    "vscode install",
    "vscode uninstall",
    "claw install",
    "claw uninstall",
    "droid install",
    "droid uninstall",
    "trae install",
    "trae uninstall",
    "trae-cn install",
    "trae-cn uninstall",
    "antigravity install",
    "antigravity uninstall",
    "hermes install",
    "hermes uninstall",
    "kiro install",
    "kiro uninstall",
    "pi install",
    "pi uninstall",
]

# Flags to probe. The last is a deliberately invalid flag — a well-behaved CLI
# should reject it with a usage message.
FLAGS = ["-h", "--help", "-?", "--xyz-bogus-test-flag-987"]

# Classification labels.
HELP = "HELP"                # Usage/help message shown
TRACEBACK = "TRACEBACK"      # Python Traceback in output
RAN = "RAN_OK"               # Exit 0 with no help (probably executed the command!)
NO_HELP = "NO_HELP"          # Non-zero exit and no help message
TIMEOUT = "TIMEOUT"          # Command hung past the deadline

# Glyphs for the markdown table.
GLYPH = {
    HELP: "OK",
    TRACEBACK: "TRACEBACK",
    RAN: "RAN",
    NO_HELP: "no",
    TIMEOUT: "timeout",
}


def safe_name(cmd: str, flag: str) -> str:
    s = f"{cmd}__{flag}"
    return re.sub(r"[^A-Za-z0-9._-]+", "_", s).strip("_")


def classify(returncode: int | None, output: str) -> str:
    if returncode is None:
        return TIMEOUT
    if "Traceback (most recent call last)" in output:
        return TRACEBACK
    head = "\n".join(output.splitlines()[:25])
    if re.search(r"(?i)^\s*usage:", head, flags=re.MULTILINE):
        return HELP
    if re.search(r"(?i)usage:\s*graphify", output):
        return HELP
    if returncode == 0:
        return RAN
    return NO_HELP


def run_one(cmd: str, flag: str) -> tuple[str, str]:
    argv = ["graphify", *cmd.split(), flag]
    try:
        proc = subprocess.run(
            argv,
            stdin=subprocess.DEVNULL,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            text=True,
            timeout=15,
        )
        output = proc.stdout
        rc = proc.returncode
    except subprocess.TimeoutExpired as e:
        output = (e.stdout or "") + "\n[timed out after 15s]"
        rc = None

    raw_path = RAWDIR / f"{safe_name(cmd, flag)}.txt"
    raw_path.write_text(
        f"$ {' '.join(argv)}\n"
        f"--- exit: {rc if rc is not None else 'TIMEOUT'} ---\n"
        f"{output}"
    )
    return classify(rc, output), output


def main() -> int:
    results: list[dict[str, str]] = []
    for cmd in COMMANDS:
        row = {"command": cmd}
        for flag in FLAGS:
            label, _ = run_one(cmd, flag)
            row[flag] = label
            print(f"{cmd:35s} {flag:35s} -> {label}", flush=True)
        results.append(row)

    # CSV
    csv_path = OUTDIR / "results.csv"
    with csv_path.open("w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=["command", *FLAGS])
        w.writeheader()
        for r in results:
            w.writerow(r)

    # Markdown
    md_lines: list[str] = []
    md_lines.append("# graphify help-flag consistency probe")
    md_lines.append("")
    md_lines.append(
        f"Generated by `{Path(__file__).name}` on "
        f"{datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')}."
    )
    md_lines.append("")
    md_lines.append("Legend:")
    md_lines.append("")
    md_lines.append("- **OK** — command printed a usage/help message")
    md_lines.append("- **TRACEBACK** — command crashed with a Python Traceback")
    md_lines.append("- **RAN** — exit 0 with no help shown (probably executed the underlying command!)")
    md_lines.append("- **no** — non-zero exit, no help message")
    md_lines.append("- **timeout** — command did not return within 15s")
    md_lines.append("")
    md_lines.append("The `--xyz-bogus-test-flag-987` column probes how the command handles an unknown flag (i.e. incorrect usage).")
    md_lines.append("")
    header = "| command | `-h` | `--help` | `-?` | bogus flag |"
    sep    = "|---|---|---|---|---|"
    md_lines.append(header)
    md_lines.append(sep)
    for r in results:
        cells = [f"`{r['command']}`"] + [GLYPH[r[f]] for f in FLAGS]
        md_lines.append("| " + " | ".join(cells) + " |")
    md_lines.append("")

    # Summary
    totals: dict[str, int] = {}
    for r in results:
        for f in FLAGS:
            totals[r[f]] = totals.get(r[f], 0) + 1
    md_lines.append("## Totals across all (command, flag) probes")
    md_lines.append("")
    md_lines.append("| outcome | count |")
    md_lines.append("|---|---|")
    for label in (HELP, TRACEBACK, RAN, NO_HELP, TIMEOUT):
        md_lines.append(f"| {GLYPH[label]} | {totals.get(label, 0)} |")
    md_lines.append("")

    # Per-flag tallies
    md_lines.append("## Per-flag tallies")
    md_lines.append("")
    md_lines.append("| flag | HELP | TRACEBACK | RAN | NO_HELP | TIMEOUT |")
    md_lines.append("|---|---|---|---|---|---|")
    for f in FLAGS:
        c = {HELP: 0, TRACEBACK: 0, RAN: 0, NO_HELP: 0, TIMEOUT: 0}
        for r in results:
            c[r[f]] += 1
        md_lines.append(
            f"| `{f}` | {c[HELP]} | {c[TRACEBACK]} | {c[RAN]} | {c[NO_HELP]} | {c[TIMEOUT]} |"
        )
    md_lines.append("")

    # Notable failures
    md_lines.append("## Notable failures")
    md_lines.append("")
    md_lines.append("Commands that emitted a Traceback for at least one help flag:")
    md_lines.append("")
    for r in results:
        bad = [f for f in FLAGS if r[f] == TRACEBACK]
        if bad:
            md_lines.append(f"- `{r['command']}` → {', '.join(bad)}")
    md_lines.append("")

    # Sample Traceback for the benchmark command (the bug the user reported).
    bench_raw = RAWDIR / "benchmark__--help.txt"
    if bench_raw.exists():
        md_lines.append("### Sample Traceback: `graphify benchmark --help`")
        md_lines.append("")
        md_lines.append("```")
        md_lines.append(bench_raw.read_text().rstrip())
        md_lines.append("```")
        md_lines.append("")
    md_lines.append("Commands where every help-flag variant failed to show usage:")
    md_lines.append("")
    for r in results:
        helpish = [f for f in FLAGS[:3] if r[f] == HELP]
        if not helpish:
            md_lines.append(f"- `{r['command']}`")
    md_lines.append("")

    (OUTDIR / "results.md").write_text("\n".join(md_lines))
    print(f"\nWrote {OUTDIR / 'results.md'}")
    print(f"Wrote {csv_path}")
    print(f"Raw outputs in {RAWDIR}/")
    return 0


if __name__ == "__main__":
    sys.exit(main())