"""Python 3.11+ toy wiki pipeline; no network, LLM or production promotion. Read a trusted immutable source snapshot. Symlink checks are not a sandbox against another process replacing paths concurrently. Caps bound retained text, file count and per-file read time/size; hashes cover complete accepted files. """ from __future__ import annotations import hashlib import json import os from pathlib import Path MAX_TEXT_BYTES = 64 * 1024 MAX_SOURCE_BYTES = 16 * 1024 * 1024 MAX_FILES = 1000 EXCLUDED_DIRS = {".git", "node_modules", ".venv", "__pycache__", "wiki", "staging", "state"} TEXT_SUFFIXES = {".md", ".markdown", ".txt", ".py", ".ts", ".tsx"} REQUIRED_SECTIONS = ("Summary", "Claims", "Evidence", "Related") ALLOWED_KINDS = {name: {"required_sections": REQUIRED_SECTIONS} for name in ("concept", "decision", "runbook")} def scan_source_records(root: Path) -> list[dict]: root = root.resolve(strict=True) records: list[dict] = [] for directory, dirs, files in os.walk(root, followlinks=False): dirs[:] = sorted(name for name in dirs if name not in EXCLUDED_DIRS and not (Path(directory) / name).is_symlink()) for name in sorted(files): path = Path(directory) / name if path.is_symlink() or path.suffix.lower() not in TEXT_SUFFIXES: continue resolved = path.resolve(strict=True) if not resolved.is_relative_to(root) or not resolved.is_file(): raise ValueError("source escaped snapshot root") if len(records) >= MAX_FILES: raise ValueError("source file count exceeds budget") digest, prefix, total = hashlib.sha256(), bytearray(), 0 with resolved.open("rb") as stream: while block := stream.read(64 * 1024): total += len(block) if total > MAX_SOURCE_BYTES: raise ValueError(f"source exceeds per-file budget: {name}") digest.update(block) prefix.extend(block[:max(0, MAX_TEXT_BYTES - len(prefix))]) relative = resolved.relative_to(root).as_posix() records.append({ "source_id": f"source:{relative}", "source_path": relative, "sha256": digest.hexdigest(), "text": prefix.decode("utf-8", errors="replace"), "truncated": total > MAX_TEXT_BYTES, }) return sorted(records, key=lambda item: item["source_id"]) def _single_line(value: str) -> str: if not isinstance(value, str) or not value.strip() or "\n" in value or "\r" in value: raise ValueError("expected nonempty single-line text") return value def validate_proposal(proposal: dict) -> None: if proposal["kind"] not in ALLOWED_KINDS: raise ValueError("unknown kind") for field in ("page_id", "kind", "title", "summary"): _single_line(proposal[field]) evidence = proposal["evidence"] claims = proposal["claims"] if not evidence or not claims: raise ValueError("claims and evidence are required") ids = [item["evidence_id"] for item in evidence] if len(ids) != len(set(ids)): raise ValueError("duplicate evidence ID") for item in evidence: for field in ("evidence_id", "source_id", "quote"): _single_line(item[field]) for claim in claims: _single_line(claim["text"]) if not claim["evidence_ids"] or not set(claim["evidence_ids"]) <= set(ids): raise ValueError("claim has missing evidence") for related in proposal.get("related_page_ids", []): _single_line(related) def render_page(proposal: dict) -> str: """Common MVP sections for all kinds; JSON scalar strings are valid YAML. This renderer does not establish semantic support or sanitize arbitrary HTML for public publication. Treat output as a review candidate, not safe HTML. """ validate_proposal(proposal) header = {"id": proposal["page_id"], "kind": proposal["kind"], "title": proposal["title"], "status": "candidate", "sources": sorted({e["source_id"] for e in proposal["evidence"]})} lines = ["---", *(f"{key}: {json.dumps(value, ensure_ascii=False)}" for key, value in header.items()), "---", "", f"# {proposal['title']}", "", "## Summary", "", proposal["summary"], "", "## Claims", ""] for claim in proposal["claims"]: lines.append(f"- {claim['text']} — {', '.join(claim['evidence_ids'])}") lines.extend(["", "## Evidence", ""]) for e in proposal["evidence"]: lines.append(f"- {e['evidence_id']} · {e['source_id']} · {e['quote']}") lines.extend(["", "## Related", ""]) lines.extend(f"- [[{item}]]" for item in sorted(set(proposal.get("related_page_ids", [])))) return "\n".join(lines).rstrip() + "\n" def validate_rendered(body: str, kind: str) -> list[str]: if kind not in ALLOWED_KINDS: return ["unknown kind"] headings = [line[3:] for line in body.splitlines() if line.startswith("## ")] return [f"missing section: {name}" for name in ALLOWED_KINDS[kind]["required_sections"] if name not in headings] if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("source_root", type=Path) args = parser.parse_args() print(json.dumps(scan_source_records(args.source_root), ensure_ascii=False, indent=2))