#!/usr/bin/env python3 """Measure A/B generation runs with a real tokenizer (tiktoken cl100k_base + o200k_base). Parses protocol cells ([Sk|Ij] one-line) and control items (### k.j headings). A cell/item is COMPLETE only if all its required parts are present and non-empty. Writes results/measurements.json and prints a markdown summary. """ import json, re, statistics, sys from pathlib import Path import tiktoken ROOT = Path(__file__).parent RAW = ROOT / "raw" OUT = ROOT / "results" / "measurements.json" ENCS = {name: tiktoken.get_encoding(name) for name in ("cl100k_base", "o200k_base")} def toks(text, enc="cl100k_base"): return len(ENCS[enc].encode(text)) def parse_protocol(path, stages, per_stage): text = path.read_text(encoding="utf-8") lines = text.splitlines() cells, scaffold_lines = [], [] cell_re = re.compile(r"^\[S(\d+)\|I(\d+)\]\s*(.*)$") for ln in lines: m = cell_re.match(ln.strip()) if m: k, j, rest = int(m.group(1)), int(m.group(2)), m.group(3) parts = [p.strip() for p in rest.split("|")] complete = len(parts) >= 3 and all(parts[:3]) def_words = edge_words = None if len(parts) >= 3: d = re.sub(r"^(Def|Definition)\s*:?\s*", "", parts[1]) e = parts[-1] def_words = len(d.split()) edge_words = len(e.split()) cells.append({"coord": f"S{k}|I{j}", "stage": k, "item": j, "text": ln.strip(), "complete": bool(complete), "def_words": def_words, "edge_words": edge_words}) elif ln.strip(): scaffold_lines.append(ln.strip()) return text, cells, scaffold_lines def parse_control(path, stages, per_stage): text = path.read_text(encoding="utf-8") head_re = re.compile(r"^###\s+(\d+)\.(\d+)\s+.*$", re.M) matches = list(head_re.finditer(text)) cells = [] for i, m in enumerate(matches): start = m.start() end = matches[i + 1].start() if i + 1 < len(matches) else len(text) chunk = text[start:end] # cut trailing next-stage heading if present in the last chunk of a stage chunk = re.split(r"^##\s+(Stage|Category)\s+\d+", chunk, flags=re.M)[0].strip() has_def = bool(re.search(r"\*\*Definition:\*\*\s*\S", chunk)) has_flow = bool(re.search(r"\*\*Execution flow:\*\*\s*\S", chunk)) has_edge = bool(re.search(r"\*\*Edge cases:\*\*\s*\S", chunk)) cells.append({"coord": f"S{m.group(1)}|I{m.group(2)}", "stage": int(m.group(1)), "item": int(m.group(2)), "text": chunk, "complete": has_def and has_flow and has_edge, "def_words": None, "edge_words": None}) scaffold = [ln.strip() for ln in text.splitlines() if re.match(r"^##\s+(Stage|Category)\s+\d+", ln.strip())] return text, cells, scaffold def summarise(name, path, stages, per_stage, parser): if not path.exists(): return {"run": name, "missing_file": True} text, cells, scaffold = parser(path, stages, per_stage) expected = {(k, j) for k in range(1, stages + 1) for j in range(1, per_stage + 1)} found = {(c["stage"], c["item"]) for c in cells} for enc in ("cl100k_base", "o200k_base"): for c in cells: c[f"tokens_{enc}"] = toks(c["text"], enc) per_cell = [c["tokens_cl100k_base"] for c in cells] stage_means = {} for k in range(1, stages + 1): vals = [c["tokens_cl100k_base"] for c in cells if c["stage"] == k] if vals: stage_means[k] = round(statistics.mean(vals), 1) drift = None if 1 in stage_means and stages in stage_means and stage_means[1]: drift = round((stage_means[stages] - stage_means[1]) / stage_means[1] * 100, 2) complete_cells = [c for c in cells if c["complete"]] last_complete = complete_cells[-1]["coord"] if complete_cells else None res = { "run": name, "file": str(path), "chars": len(text), "words": len(text.split()), "expected_cells": stages * per_stage, "cells_found": len(cells), "cells_complete": len(complete_cells), "missing_coords": sorted(f"S{k}|I{j}" for k, j in expected - found), "incomplete_coords": [c["coord"] for c in cells if not c["complete"]], "last_complete_coord": last_complete, "total_tokens_cl100k": toks(text), "total_tokens_o200k": toks(text, "o200k_base"), "scaffold_tokens_cl100k": sum(toks(s) for s in scaffold), "per_cell_tokens_cl100k": { "mean": round(statistics.mean(per_cell), 1) if per_cell else None, "median": round(statistics.median(per_cell), 1) if per_cell else None, "min": min(per_cell) if per_cell else None, "max": max(per_cell) if per_cell else None, "stdev": round(statistics.pstdev(per_cell), 1) if len(per_cell) > 1 else None, }, "stage_means_cl100k": stage_means, "drift_last_vs_first_stage_pct": drift, "cells": [{k: v for k, v in c.items() if k != "text"} for c in cells], } defs = [c["def_words"] for c in cells if c["def_words"] is not None] edges = [c["edge_words"] for c in cells if c["edge_words"] is not None] if defs: res["protocol_word_bounds"] = { "def_le15_compliant": sum(1 for w in defs if w <= 15), "def_n": len(defs), "def_median": statistics.median(defs), "def_max": max(defs), "edge_le12_compliant": sum(1 for w in edges if w <= 12), "edge_n": len(edges), "edge_median": statistics.median(edges), "edge_max": max(edges), } return res def main(): runs = [ ("control36", RAW / "control36.md", 6, 6, parse_control), ("protocol36", RAW / "protocol36.md", 6, 6, parse_protocol), ("control80", RAW / "control80.md", 8, 10, parse_control), ("protocol80", RAW / "protocol80.md", 8, 10, parse_protocol), ] results = [summarise(*r) for r in runs] extra = [Path(p) for p in sys.argv[1:]] for p in extra: # e.g. resume36.md measured as protocol continuation results.append(summarise(p.stem, p, 6, 6, parse_protocol)) OUT.parent.mkdir(parents=True, exist_ok=True) OUT.write_text(json.dumps(results, indent=2), encoding="utf-8") for r in results: if r.get("missing_file"): print(f"{r['run']}: FILE MISSING"); continue pc = r["per_cell_tokens_cl100k"] print(f"{r['run']}: complete {r['cells_complete']}/{r['expected_cells']} " f"last={r['last_complete_coord']} total_tok_cl100k={r['total_tokens_cl100k']} " f"o200k={r['total_tokens_o200k']} mean/cell={pc['mean']} median={pc['median']} " f"min={pc['min']} max={pc['max']} drift={r['drift_last_vs_first_stage_pct']}% " f"scaffold_tok={r['scaffold_tokens_cl100k']}") if "protocol_word_bounds" in r: b = r["protocol_word_bounds"] print(f" bounds: Def<=15w {b['def_le15_compliant']}/{b['def_n']} (median {b['def_median']}, max {b['def_max']}); " f"Edge<=12w {b['edge_le12_compliant']}/{b['edge_n']} (median {b['edge_median']}, max {b['edge_max']})") if r["missing_coords"]: print(f" missing: {r['missing_coords']}") if r["incomplete_coords"]: print(f" incomplete: {r['incomplete_coords']}") if __name__ == "__main__": main()