#!/usr/bin/env python3 """ MLX Bench: how fast does this Mac run MLX models? Measures time to first token, prompt and generation speed and peak memory with Apple's open-source MLX framework (mlx-lm), then shares the numbers with the open leaderboard at https://mlx.app/bench. pip install -U mlx-lm python3 bench.py # a model sized for this Mac python3 bench.py --model mlx-community/Qwen2.5-7B-Instruct-4bit python3 bench.py --wallet alex.mlx # credit the run to your MLX wallet python3 bench.py --context 8192 # long-context run (8192 or 65536, like Apple's method) python3 bench.py --dry-run # measure without sharing A pod (2 to 4 Mac Studios joined by Thunderbolt 5, macOS 26.2+, MLX 0.30.1+) runs it with mlx.launch: mlx.launch --hostfile hosts.json --backend jaccl --env MLX_METAL_FAST_SYNCH=1 -- \ python3 bench.py --model mlx-community/Qwen3.8-27B-4bit --context 8192 # tensor parallel … python3 bench.py --model … --context 65536 --pipeline # pipeline parallel What is sent: chip name, GPU core count, memory size, macOS / MLX / mlx-lm versions, the model name and the measured speeds. No files, no hostname, no personal data. MLX Network is independent and not affiliated with Apple Inc. """ import argparse, json, os, platform, subprocess, sys, time, urllib.request API = "https://mlx.app/api/bench" PROMPT_TOKENS = 512 GEN_TOKENS = 128 CONTEXTS = (512, 8192, 65536) def sh(cmd): try: return subprocess.run(cmd, capture_output=True, text=True, timeout=30).stdout.strip() except Exception: return "" def machine(): chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]) mem = sh(["sysctl", "-n", "hw.memsize"]) cores = None try: data = json.loads(sh(["system_profiler", "SPDisplaysDataType", "-json"]) or "{}") for gpu in data.get("SPDisplaysDataType", []): v = gpu.get("sppci_cores") if v: cores = int(str(v).split()[0]) break except Exception: pass return { "chip": chip, "memGB": round(int(mem) / 2**30) if mem.isdigit() else 0, "gpuCores": cores, "macos": platform.mac_ver()[0], } def default_model(mem_gb): if mem_gb >= 48: return "mlx-community/Qwen2.5-14B-Instruct-4bit" if mem_gb >= 24: return "mlx-community/Qwen2.5-7B-Instruct-4bit" if mem_gb >= 16: return "mlx-community/Llama-3.2-3B-Instruct-4bit" return "mlx-community/Llama-3.2-1B-Instruct-4bit" def run(model_id, context=PROMPT_TOKENS, pipeline=False, prefill_step=None): import mlx.core as mx import mlx_lm from mlx_lm import load, stream_generate # a pod: every machine runs this script under mlx.launch and shares the model group = mx.distributed.init() size = group.size() if size > 1: from mlx_lm.utils import sharded_load model, tokenizer = sharded_load(model_id, group if pipeline else None, None if pipeline else group)[:2] else: model, tokenizer = load(model_id) # a fixed text, repeated to exactly `context` tokens (the same prompt on every Mac) unit = tokenizer.encode("The Apple Silicon supercomputer measures how fast every Mac runs open models. ") ids = (unit * (context // max(1, len(unit)) + 1))[:context] step = prefill_step or (8192 if context >= 8192 else 2048) # warm up once so compilation and loading don't count for _ in stream_generate(model, tokenizer, ids[:64], max_tokens=8): pass try: mx.reset_peak_memory() except Exception: pass last = None first = None t0 = time.perf_counter() for last in stream_generate(model, tokenizer, ids, max_tokens=GEN_TOKENS, prefill_step_size=step): if first is None: first = time.perf_counter() - t0 if last is None: raise SystemExit("The model produced no output.") return group.rank(), { "model": model_id, "promptTokens": int(getattr(last, "prompt_tokens", len(ids))), "promptTps": float(last.prompt_tps), "genTokens": int(getattr(last, "generation_tokens", GEN_TOKENS)), "genTps": float(last.generation_tps), "peakMemGB": float(getattr(last, "peak_memory", 0.0)), "ttftMs": round((first or 0) * 1000, 1), "mlx": getattr(mx, "__version__", ""), "mlxLm": getattr(mlx_lm, "__version__", ""), "machines": size, "mode": "single" if size == 1 else ("pipeline" if pipeline else "tensor"), "backend": os.environ.get("MLX_BENCH_BACKEND", "jaccl" if size > 1 else ""), } def main(): ap = argparse.ArgumentParser(description="Benchmark MLX on this Mac and share it with mlx.app/bench") ap.add_argument("--model", help="Hugging Face repo of an MLX model") ap.add_argument("--wallet", help="your .mlx wallet name, to credit the run") ap.add_argument("--context", type=int, default=PROMPT_TOKENS, choices=CONTEXTS, help="prompt length in tokens (512, 8192 or 65536)") ap.add_argument("--pipeline", action="store_true", help="in a pod, split the model by layers (pipeline) instead of by width (tensor)") ap.add_argument("--prefill-step-size", type=int, help="prefill chunk size (default 2048, or 8192 for long contexts)") ap.add_argument("--dry-run", action="store_true", help="measure only, don't share") a = ap.parse_args() if platform.system() != "Darwin" or platform.machine() != "arm64": raise SystemExit("MLX Bench runs on Apple Silicon Macs.") try: import mlx_lm # noqa: F401 except ImportError: raise SystemExit("Install mlx-lm first: pip install -U mlx-lm") m = machine() model_id = a.model or default_model(m["memGB"]) t = time.time() rank, r = run(model_id, a.context, a.pipeline, a.prefill_step_size) if rank != 0: return # in a pod, the first machine reports for all of them pod = f" · pod of {r['machines']} ({r['mode']})" if r["machines"] > 1 else "" print(f"{m['chip']} · {m['gpuCores'] or '?'}-core GPU · {m['memGB']} GB · macOS {m['macos']}{pod}") print(f"{model_id}: {a.context:,}-token prompt, {GEN_TOKENS} new tokens") print(f" first token {r['ttftMs']:.0f} ms") print(f" prompt {r['promptTps']:.1f} tokens/s") print(f" generation {r['genTps']:.1f} tokens/s") print(f" peak memory {r['peakMemGB']:.2f} GB ({time.time() - t:.0f} s)") body = {**m, **r} if a.wallet: body["wallet"] = a.wallet if a.dry_run: print(json.dumps(body, indent=2)) return req = urllib.request.Request(API, data=json.dumps(body).encode(), headers={"content-type": "application/json"}, method="POST") try: with urllib.request.urlopen(req, timeout=20) as res: json.loads(res.read()) print("Shared. See how your Mac compares: https://mlx.app/bench") except urllib.error.HTTPError as e: print("Not shared:", json.loads(e.read() or b"{}").get("error", e.reason)) except Exception as e: print("Not shared:", e) if __name__ == "__main__": main()