#!/usr/bin/env python3 """Build data.js from the swarm's audit JSON + curated extras. Spans are real: first commit of the earliest PR -> last commit / merge of the latest. Any PR number not already cached in pr_spans.json is fetched from the GitHub GraphQL API. """ import json, os, re, subprocess, sys from collections import Counter HERE = os.path.dirname(os.path.abspath(__file__)) SWARM = os.path.join(HERE, "swarm") OUT = "/Users/arthurzucker/Work/transformers-roadmap-space/data.js" SPANS = os.path.join(HERE, "pr_spans.json") # ---------------------------------------------------------------- tracks TRACKS = [ ("Deprecation → v5", 258, "Cut the library to one framework, then ship the major version."), ("Speed & serving", 8, "Make transformers fast enough to serve, not just to prototype."), ("Continuous batching", 14, "Serving-grade batching inside the library — the big 2026 push."), ("vLLM & SGLang", 265, "Be the model definition that inference engines read from."), ("Standardization & modular", 34, "Cement transformers as the source of truth, then let others read from it."), ("API & weight loading", 122, "One way to do things, and other libraries can build on it."), ("Training & the 5 parallelisms",48, "PP · TP · EP · DP · CP · FSDP behind one native API."), ("Quantization & kernels", 350, "Simplify, expand to other libraries, quantization by default."), ("Export & on-device", 190, "Get a transformers model out of Python and onto a device."), ("Tests, CI & infra", 150, "Weekly releases mean the tests have to stay green."), ("Multimodality", 220, "Backend-agnostic processors; serve every modality."), ] FILE_TRACK = { # swarm file -> default track when the item has no `track` "cb.json": "Continuous batching", "vllm.json": "vLLM & SGLang", "training.json": "Training & the 5 parallelisms", "multimodal.json": "Multimodality", "quant.json": "Quantization & kernels", "standard.json": "Standardization & modular", "infra.json": "Tests, CI & infra", "speed.json": "Speed & serving", } # curated items the swarm cannot find (they shipped outside this repo) -------- EXTRA = [ ("Standardization & modular", dict( t="Visual modular — the Atlas", tag="SVG diagram for every model", s="wip", prs=[], at=["2026-06-02","2026-07-01"], d="Generated architecture gallery: a full view per model plus a diff view of what each modular file overrides vs its parent.", links=[["Atlas: model-architectures","https://huggingface.co/spaces/transformers-community/model-architectures"]])), ("Standardization & modular", dict( t="Transpilation: transformers → MLX", tag="conversion skill, 51★", s="done", prs=[], at=["2026-04-15","2026-06-16"], d="Agent Skill converting transformers LLMs to mlx-lm — MoEs, MLA, hybrid SSM+attention, GatedDeltaNet, with per-layer logit comparison.", links=[["huggingface/transformers-to-mlx","https://github.com/huggingface/transformers-to-mlx"], ["MLX docs","https://huggingface.co/docs/transformers/community_integrations/mlx"]])), ("Speed & serving", dict( t="TGI backend", tag="shipped outside this repo", s="done", prs=[], note="Landed in huggingface/text-generation-inference — no transformers PR to link.", links=[["text-generation-inference","https://github.com/huggingface/text-generation-inference"]])), ("Tests, CI & infra", dict( t="Public CI dashboard", tag="daily CI, AMD + Nvidia", s="done", prs=[], at=["2025-08-05","2026-07-12"], d="Daily CI status for important models across AMD MI300 and Nvidia A10, published as a Space.", links=[["CI dashboard Space","https://huggingface.co/spaces/transformers-community/transformers-ci-dashboard"]])), ] UNTIL = {"v5.0.0 released": "2026-01-26"} # real-world end dates that beat the last PR # ---------------------------------------------------------------- spans SP = json.load(open(SPANS)) if os.path.exists(SPANS) else {} def fetch(nums): nums = [n for n in nums if str(n) not in SP] if not nums: return print(f" fetching {len(nums)} new PRs…", file=sys.stderr) for i in range(0, len(nums), 30): ch = nums[i:i+30] q = "query{repository(owner:\"huggingface\",name:\"transformers\"){" for n in ch: q += (f'p{n}:pullRequest(number:{n}){{number title state createdAt mergedAt bodyText ' f'author{{login}} additions deletions changedFiles ' f'first:commits(first:1){{nodes{{commit{{committedDate}}}}}} ' f'last:commits(last:1){{nodes{{commit{{committedDate}}}}}} commits{{totalCount}} }} ') q += "}}" r = subprocess.run(["gh","api","graphql","-f",f"query={q}"], capture_output=True, text=True) if r.returncode: print(" GraphQL error:", r.stderr[:200], file=sys.stderr); continue data = json.loads(r.stdout).get("data",{}).get("repository") or {} for _, v in data.items(): if not v: continue fc, lc = v["first"]["nodes"], v["last"]["nodes"] body = re.sub(r"\s+"," ", v.get("bodyText") or "").strip() body = re.sub(r"(?i)^(#*\s*what does this pr do\??)\s*", "", body) SP[str(v["number"])] = { "t": v["title"], "state": v["state"], "merged": (v["mergedAt"] or "")[:10], "first": fc[0]["commit"]["committedDate"][:10] if fc else v["createdAt"][:10], "last": lc[0]["commit"]["committedDate"][:10] if lc else (v["mergedAt"] or v["createdAt"])[:10], "by": (v.get("author") or {}).get("login",""), "add": v["additions"], "del": v["deletions"], "f": v["changedFiles"], "c": v["commits"]["totalCount"], "body": body[:300]} json.dump(SP, open(SPANS,"w"), indent=0) def span(prs): f, l = [], [] for n in prs: v = SP.get(str(n)) if not v: continue f.append(v["first"]); l.append(max(v["last"], v["merged"] or v["last"])) return [min(f), max(l)] if f else None def prmeta(prs): out = [] for n in prs: v = SP.get(str(n)) if not v: continue out.append({"n": n, "d": v["merged"] or v["last"], "t": v["t"], "st": v["state"], "by": v.get("by",""), "a": v.get("add",0), "r": v.get("del",0), "f": v.get("f",v.get("files",0)), "c": v.get("c",v.get("commits",0)), "b": v.get("body","")}) return sorted(out, key=lambda r: r["d"]) def rollup(prs): who = Counter(); a=r=f=c=0 for n in prs: v = SP.get(str(n)) if not v: continue if v.get("by"): who[v["by"]] += 1 a+=v.get("add",0); r+=v.get("del",0); f+=v.get("f",v.get("files",0)); c+=v.get("c",v.get("commits",0)) return {"owners":[[k,n] for k,n in who.most_common(4)], "n":len(prs), "add":a, "del":r, "files":f, "commits":c} # ---------------------------------------------------------------- assemble buckets = {name: [] for name,_,_ in TRACKS} allprs, seen_files = set(), [] for fn, default_track in FILE_TRACK.items(): p = os.path.join(SWARM, fn) if not os.path.exists(p): print(f" !! missing {fn}", file=sys.stderr); continue seen_files.append(fn) for it in json.load(open(p))["items"]: tr = it.get("track") or default_track if tr not in buckets: print(f" !! unknown track {tr!r} in {fn}, using {default_track}", file=sys.stderr); tr = default_track prs = [int(x) for x in it.get("prs", [])] allprs.update(prs) buckets[tr].append(dict(t=it["t"], tag=it.get("tag",""), s=it.get("s","done"), prs=prs, d=it.get("d"), note=it.get("note"), links=[], src=fn)) for tr, item in EXTRA: item.setdefault("links", []); item.setdefault("d", None); item.setdefault("note", None) item.setdefault("src", "curated") buckets[tr].append(item) fetch(sorted(allprs)) T = [] for name, hue, blurb in TRACKS: items = buckets[name] if not items: continue for i in items: sp = i.pop("at", None) or span(i["prs"]) u = UNTIL.get(i["t"]) if u: sp = [sp[0], max(sp[1], u)] if sp else [u, u] i["span"] = sp i["roll"] = rollup(i["prs"]) i["prs"] = prmeta(i["prs"]) # oldest work first within a track items.sort(key=lambda i: (i["span"] or ["9999-99-99"])[0]) T.append(dict(name=name, hue=hue, blurb=blurb, items=items)) MODELS = json.load(open(os.path.join(HERE,"models.json"))) RELEASES = json.load(open(os.path.join(HERE,"releases.json"))) spans = [i["span"] for t in T for i in t["items"] if i["span"]] lo = min(min(s[0] for s in spans), MODELS[0]["d"]) hi = max(max(s[1] for s in spans), MODELS[-1]["d"]) mw = Counter(m["by"] for m in MODELS) MODEL_ROW = {"n":len(MODELS), "span":[MODELS[0]["d"], MODELS[-1]["d"]], "owners":[[k,n] for k,n in mw.most_common(4)], "authors":len(mw)} hdr = ("// Generated by build2.py — do not hand-edit.\n" "// Spans are REAL: first commit of the earliest PR -> last commit / merge of the latest.\n" "// Source board: https://miro.com/app/board/uXjVL_vDn1k=/\n\n") open(OUT,"w").write(hdr + "const AXIS = " + json.dumps([lo,hi]) + ";\n" + "const RELEASES = " + json.dumps(RELEASES) + ";\n" + "const MODELS = " + json.dumps(MODELS, ensure_ascii=False) + ";\n" + "const MODEL_ROW = " + json.dumps(MODEL_ROW, ensure_ascii=False) + ";\n\n" + "const TRACKS = " + json.dumps(T, indent=1, ensure_ascii=False) + ";\n") n_items = sum(len(t["items"]) for t in T) n_prs = len({p["n"] for t in T for i in t["items"] for p in i["prs"]}) missing = [p for p in allprs if str(p) not in SP] print(f"files {len(seen_files)}/{len(FILE_TRACK)} | tracks {len(T)} | items {n_items} | prs {n_prs} | span {lo}→{hi}") if missing: print(" unresolved PR numbers:", missing[:20]) for t in T: print(f" {t['name']:32s} {len(t['items']):2d} items")