"""Inspect a pinned E5 tokenizer, without downloading model weights.

Python 3.12, tokenizers==0.21.4. Network is needed for the first download.
Run: python tokenizer_probe.py
The cache is outside the project; tokenizer files are not vendored.
"""
from pathlib import Path
import hashlib
import json
import urllib.request
import tokenizers
from tokenizers import Tokenizer

MODEL = "intfloat/multilingual-e5-small"
REVISION = "614241f622f53c4eeff9890bdc4f31cfecc418b3"
TEXT = "E17 알람의 원인은 냉각수 유량 저하입니다."


def main():
    cache = Path.home() / ".cache" / "blog-foundations" / REVISION
    cache.mkdir(parents=True, exist_ok=True)
    target = cache / "tokenizer.json"
    if not target.exists():
        with urllib.request.urlopen(
            f"https://huggingface.co/{MODEL}/resolve/{REVISION}/tokenizer.json",
            timeout=60,
        ) as response:
            data = response.read()
        # Validate before persisting, so an error response is not a valid cache entry.
        Tokenizer.from_str(data.decode())
        target.write_bytes(data)
    tokenizer = Tokenizer.from_file(str(target))
    tokenizer.no_truncation()
    tokenizer.no_padding()
    rows = []
    for prefix in ["", "query: ", "passage: "]:
        raw = tokenizer.encode(prefix + TEXT, add_special_tokens=False)
        framed = tokenizer.encode(prefix + TEXT, add_special_tokens=True)
        rows.append({"prefix": prefix, "body_tokens": len(raw.ids),
                     "with_special_tokens": len(framed.ids), "pieces": raw.tokens})
    result = {"model": MODEL, "revision": REVISION, "tokenizers": tokenizers.__version__,
              "text": TEXT, "tokenizer_sha256": hashlib.sha256(target.read_bytes()).hexdigest(),
              "rows": rows,
              "scope": "tokenization only; not embedding/generation quality or provider billing"}
    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()