Download build2.py from the-arthurs/transformers-roadmap-timeline: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/spaces/the-arthurs/transformers-roadmap-timeline/resolve/main/build2.py
- Command line
-
hf download hf://spaces/the-arthurs/transformers-roadmap-timeline/build2.py
-
curl -L -o build2.py https://huggingface.co/spaces/the-arthurs/transformers-roadmap-timeline/resolve/main/build2.py
10.1 kB
| #!/usr/bin/env python3 | |
| """Build data.js from the swarm's audit JSON + curated extras. | |
| Spans are real: first commit of the earliest PR -> last commit / merge of the latest. | |
| Any PR number not already cached in pr_spans.json is fetched from the GitHub GraphQL API. | |
| """ | |
| import json, os, re, subprocess, sys | |
| from collections import Counter | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| SWARM = os.path.join(HERE, "swarm") | |
| OUT = "/Users/arthurzucker/Work/transformers-roadmap-space/data.js" | |
| SPANS = os.path.join(HERE, "pr_spans.json") | |
| # ---------------------------------------------------------------- tracks | |
| TRACKS = [ | |
| ("Deprecation → v5", 258, "Cut the library to one framework, then ship the major version."), | |
| ("Speed & serving", 8, "Make transformers fast enough to serve, not just to prototype."), | |
| ("Continuous batching", 14, "Serving-grade batching inside the library — the big 2026 push."), | |
| ("vLLM & SGLang", 265, "Be the model definition that inference engines read from."), | |
| ("Standardization & modular", 34, "Cement transformers as the source of truth, then let others read from it."), | |
| ("API & weight loading", 122, "One way to do things, and other libraries can build on it."), | |
| ("Training & the 5 parallelisms",48, "PP · TP · EP · DP · CP · FSDP behind one native API."), | |
| ("Quantization & kernels", 350, "Simplify, expand to other libraries, quantization by default."), | |
| ("Export & on-device", 190, "Get a transformers model out of Python and onto a device."), | |
| ("Tests, CI & infra", 150, "Weekly releases mean the tests have to stay green."), | |
| ("Multimodality", 220, "Backend-agnostic processors; serve every modality."), | |
| ] | |
| FILE_TRACK = { # swarm file -> default track when the item has no `track` | |
| "cb.json": "Continuous batching", | |
| "vllm.json": "vLLM & SGLang", | |
| "training.json": "Training & the 5 parallelisms", | |
| "multimodal.json": "Multimodality", | |
| "quant.json": "Quantization & kernels", | |
| "standard.json": "Standardization & modular", | |
| "infra.json": "Tests, CI & infra", | |
| "speed.json": "Speed & serving", | |
| } | |
| # curated items the swarm cannot find (they shipped outside this repo) -------- | |
| EXTRA = [ | |
| ("Standardization & modular", dict( | |
| t="Visual modular — the Atlas", tag="SVG diagram for every model", s="wip", prs=[], | |
| at=["2026-06-02","2026-07-01"], | |
| d="Generated architecture gallery: a full view per model plus a diff view of what each modular file overrides vs its parent.", | |
| links=[["Atlas: model-architectures","https://huggingface.co/spaces/transformers-community/model-architectures"]])), | |
| ("Standardization & modular", dict( | |
| t="Transpilation: transformers → MLX", tag="conversion skill, 51★", s="done", prs=[], | |
| at=["2026-04-15","2026-06-16"], | |
| d="Agent Skill converting transformers LLMs to mlx-lm — MoEs, MLA, hybrid SSM+attention, GatedDeltaNet, with per-layer logit comparison.", | |
| links=[["huggingface/transformers-to-mlx","https://github.com/huggingface/transformers-to-mlx"], | |
| ["MLX docs","https://huggingface.co/docs/transformers/community_integrations/mlx"]])), | |
| ("Speed & serving", dict( | |
| t="TGI backend", tag="shipped outside this repo", s="done", prs=[], | |
| note="Landed in huggingface/text-generation-inference — no transformers PR to link.", | |
| links=[["text-generation-inference","https://github.com/huggingface/text-generation-inference"]])), | |
| ("Tests, CI & infra", dict( | |
| t="Public CI dashboard", tag="daily CI, AMD + Nvidia", s="done", prs=[], | |
| at=["2025-08-05","2026-07-12"], | |
| d="Daily CI status for important models across AMD MI300 and Nvidia A10, published as a Space.", | |
| links=[["CI dashboard Space","https://huggingface.co/spaces/transformers-community/transformers-ci-dashboard"]])), | |
| ] | |
| UNTIL = {"v5.0.0 released": "2026-01-26"} # real-world end dates that beat the last PR | |
| # ---------------------------------------------------------------- spans | |
| SP = json.load(open(SPANS)) if os.path.exists(SPANS) else {} | |
| def fetch(nums): | |
| nums = [n for n in nums if str(n) not in SP] | |
| if not nums: return | |
| print(f" fetching {len(nums)} new PRs…", file=sys.stderr) | |
| for i in range(0, len(nums), 30): | |
| ch = nums[i:i+30] | |
| q = "query{repository(owner:\"huggingface\",name:\"transformers\"){" | |
| for n in ch: | |
| q += (f'p{n}:pullRequest(number:{n}){{number title state createdAt mergedAt bodyText ' | |
| f'author{{login}} additions deletions changedFiles ' | |
| f'first:commits(first:1){{nodes{{commit{{committedDate}}}}}} ' | |
| f'last:commits(last:1){{nodes{{commit{{committedDate}}}}}} commits{{totalCount}} }} ') | |
| q += "}}" | |
| r = subprocess.run(["gh","api","graphql","-f",f"query={q}"], capture_output=True, text=True) | |
| if r.returncode: | |
| print(" GraphQL error:", r.stderr[:200], file=sys.stderr); continue | |
| data = json.loads(r.stdout).get("data",{}).get("repository") or {} | |
| for _, v in data.items(): | |
| if not v: continue | |
| fc, lc = v["first"]["nodes"], v["last"]["nodes"] | |
| body = re.sub(r"\s+"," ", v.get("bodyText") or "").strip() | |
| body = re.sub(r"(?i)^(#*\s*what does this pr do\??)\s*", "", body) | |
| SP[str(v["number"])] = { | |
| "t": v["title"], "state": v["state"], "merged": (v["mergedAt"] or "")[:10], | |
| "first": fc[0]["commit"]["committedDate"][:10] if fc else v["createdAt"][:10], | |
| "last": lc[0]["commit"]["committedDate"][:10] if lc else (v["mergedAt"] or v["createdAt"])[:10], | |
| "by": (v.get("author") or {}).get("login",""), "add": v["additions"], "del": v["deletions"], | |
| "f": v["changedFiles"], "c": v["commits"]["totalCount"], "body": body[:300]} | |
| json.dump(SP, open(SPANS,"w"), indent=0) | |
| def span(prs): | |
| f, l = [], [] | |
| for n in prs: | |
| v = SP.get(str(n)) | |
| if not v: continue | |
| f.append(v["first"]); l.append(max(v["last"], v["merged"] or v["last"])) | |
| return [min(f), max(l)] if f else None | |
| def prmeta(prs): | |
| out = [] | |
| for n in prs: | |
| v = SP.get(str(n)) | |
| if not v: continue | |
| out.append({"n": n, "d": v["merged"] or v["last"], "t": v["t"], "st": v["state"], "by": v.get("by",""), | |
| "a": v.get("add",0), "r": v.get("del",0), "f": v.get("f",v.get("files",0)), | |
| "c": v.get("c",v.get("commits",0)), "b": v.get("body","")}) | |
| return sorted(out, key=lambda r: r["d"]) | |
| def rollup(prs): | |
| who = Counter(); a=r=f=c=0 | |
| for n in prs: | |
| v = SP.get(str(n)) | |
| if not v: continue | |
| if v.get("by"): who[v["by"]] += 1 | |
| a+=v.get("add",0); r+=v.get("del",0); f+=v.get("f",v.get("files",0)); c+=v.get("c",v.get("commits",0)) | |
| return {"owners":[[k,n] for k,n in who.most_common(4)], "n":len(prs), "add":a, "del":r, "files":f, "commits":c} | |
| # ---------------------------------------------------------------- assemble | |
| buckets = {name: [] for name,_,_ in TRACKS} | |
| allprs, seen_files = set(), [] | |
| for fn, default_track in FILE_TRACK.items(): | |
| p = os.path.join(SWARM, fn) | |
| if not os.path.exists(p): | |
| print(f" !! missing {fn}", file=sys.stderr); continue | |
| seen_files.append(fn) | |
| for it in json.load(open(p))["items"]: | |
| tr = it.get("track") or default_track | |
| if tr not in buckets: | |
| print(f" !! unknown track {tr!r} in {fn}, using {default_track}", file=sys.stderr); tr = default_track | |
| prs = [int(x) for x in it.get("prs", [])] | |
| allprs.update(prs) | |
| buckets[tr].append(dict(t=it["t"], tag=it.get("tag",""), s=it.get("s","done"), prs=prs, | |
| d=it.get("d"), note=it.get("note"), links=[], src=fn)) | |
| for tr, item in EXTRA: | |
| item.setdefault("links", []); item.setdefault("d", None); item.setdefault("note", None) | |
| item.setdefault("src", "curated") | |
| buckets[tr].append(item) | |
| fetch(sorted(allprs)) | |
| T = [] | |
| for name, hue, blurb in TRACKS: | |
| items = buckets[name] | |
| if not items: continue | |
| for i in items: | |
| sp = i.pop("at", None) or span(i["prs"]) | |
| u = UNTIL.get(i["t"]) | |
| if u: sp = [sp[0], max(sp[1], u)] if sp else [u, u] | |
| i["span"] = sp | |
| i["roll"] = rollup(i["prs"]) | |
| i["prs"] = prmeta(i["prs"]) | |
| # oldest work first within a track | |
| items.sort(key=lambda i: (i["span"] or ["9999-99-99"])[0]) | |
| T.append(dict(name=name, hue=hue, blurb=blurb, items=items)) | |
| MODELS = json.load(open(os.path.join(HERE,"models.json"))) | |
| RELEASES = json.load(open(os.path.join(HERE,"releases.json"))) | |
| spans = [i["span"] for t in T for i in t["items"] if i["span"]] | |
| lo = min(min(s[0] for s in spans), MODELS[0]["d"]) | |
| hi = max(max(s[1] for s in spans), MODELS[-1]["d"]) | |
| mw = Counter(m["by"] for m in MODELS) | |
| MODEL_ROW = {"n":len(MODELS), "span":[MODELS[0]["d"], MODELS[-1]["d"]], | |
| "owners":[[k,n] for k,n in mw.most_common(4)], "authors":len(mw)} | |
| hdr = ("// Generated by build2.py — do not hand-edit.\n" | |
| "// Spans are REAL: first commit of the earliest PR -> last commit / merge of the latest.\n" | |
| "// Source board: https://miro.com/app/board/uXjVL_vDn1k=/\n\n") | |
| open(OUT,"w").write(hdr | |
| + "const AXIS = " + json.dumps([lo,hi]) + ";\n" | |
| + "const RELEASES = " + json.dumps(RELEASES) + ";\n" | |
| + "const MODELS = " + json.dumps(MODELS, ensure_ascii=False) + ";\n" | |
| + "const MODEL_ROW = " + json.dumps(MODEL_ROW, ensure_ascii=False) + ";\n\n" | |
| + "const TRACKS = " + json.dumps(T, indent=1, ensure_ascii=False) + ";\n") | |
| n_items = sum(len(t["items"]) for t in T) | |
| n_prs = len({p["n"] for t in T for i in t["items"] for p in i["prs"]}) | |
| missing = [p for p in allprs if str(p) not in SP] | |
| print(f"files {len(seen_files)}/{len(FILE_TRACK)} | tracks {len(T)} | items {n_items} | prs {n_prs} | span {lo}→{hi}") | |
| if missing: print(" unresolved PR numbers:", missing[:20]) | |
| for t in T: print(f" {t['name']:32s} {len(t['items']):2d} items") | |