{
 "llamacpp": {
  "small": {
   "metric": "llama.cpp llama-bench, Llama-2 7B Q4_0, tg128 (text generation tok/s); best of FA on/off as posted",
   "short": "Small model: Llama-2 7B Q4_0",
   "chips": {
    "strix-halo": {
     "label": "AMD Ryzen AI Max+ 395 (Radeon 8060S)",
     "tps": 50.95,
     "backend": "ROCm",
     "author": "kresle",
     "date": "2026-02-02",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-15671492",
     "pp": 554.48,
     "pp_metric": "pp512"
    },
    "gb10": {
     "label": "NVIDIA GB10 (DGX Spark)",
     "tps": 57.21,
     "backend": "CUDA",
     "author": "ggerganov",
     "date": "2025-10-15",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-14687095",
     "pp": 3062.31,
     "pp_metric": "pp512"
    },
    "m5-ultra-80": {
     "label": "Apple M5 Ultra 36/80 (256GB)",
     "tps": 179.1,
     "backend": "Metal",
     "author": "see thread",
     "date": "2026-09-30",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/4167#discussioncomment-18683605",
     "pp": 4944.66,
     "pp_metric": "pp512"
    },
    "m5-max-40": {
     "label": "Apple M5 Max 40-core GPU (MacBook Pro 16, 128GB)",
     "tps": 119.08,
     "backend": "Metal",
     "author": "see thread",
     "date": "2026-06-22",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/4167#discussioncomment-17386908",
     "pp": 3347.44,
     "pp_metric": "pp512"
    },
    "m5-pro": {
     "label": "Apple M5 Pro (Mac mini, 64GB)",
     "tps": 67.59,
     "backend": "Metal",
     "author": "see thread",
     "date": "2026-09-24",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/4167#discussioncomment-18583406",
     "pp": 1340.44,
     "pp_metric": "pp512"
    },
    "rtx-5090": {
     "label": "NVIDIA RTX 5090",
     "tps": 314.0,
     "backend": "CUDA",
     "author": "totaldev",
     "date": "2025-12-04",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-15160999",
     "pp": 14805.32,
     "pp_metric": "pp512"
    },
    "rtx-4090": {
     "label": "NVIDIA RTX 4090",
     "tps": 191.57,
     "backend": "CUDA",
     "author": "deehw",
     "date": "2025-08-24",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-14200008",
     "pp": 12506.97,
     "pp_metric": "pp512"
    },
    "rtx-3090": {
     "label": "NVIDIA RTX 3090",
     "tps": 172.26,
     "backend": "CUDA",
     "author": "slaren",
     "date": "2025-08-01",
     "url": "https://github.com/ggml-org/llama.cpp/discussions/15013#discussioncomment-13963584",
     "pp": 6924.01,
     "pp_metric": "pp512"
    }
   },
   "pp_note": "pp512 from the same llama-bench run and flash-attention setting as the tg128 figure, same source link."
  },
  "large": {
   "metric": "gpt-oss-120b MXFP4 (59 GiB weights), text generation tok/s at empty context, single request",
   "short": "Large model: gpt-oss-120b",
   "min_mem_gb": 96,
   "fit_rule": "Needs about 59 GiB for weights plus context and OS, so devices with 64GB or less are marked 'doesn't fit'.",
   "chips": {
    "gb10": {
     "label": "NVIDIA GB10 (DGX Spark)",
     "tps": 58.72,
     "engine": "llama.cpp CUDA, llama-bench tg32, b7946",
     "date": "2026-02-05",
     "url": "https://github.com/ggml-org/llama.cpp/blob/master/benches/dgx-spark/dgx-spark.md",
     "pp": 2443.91,
     "pp_metric": "pp2048"
    },
    "strix-halo": {
     "label": "Ryzen AI Max+ 395, 128GB",
     "tps": 52.74,
     "engine": "llama.cpp Vulkan, llama-bench tg32 (comment in thread)",
     "date": "2025-10",
     "url": "https://www.reddit.com/r/LocalLLaMA/comments/1o6u5o4/gptoss20120b_amd_strix_halo_vs_nvidia_dgx_spark/"
    },
    "m5-max-40": {
     "label": "Apple M5 Max 40-core, 128GB (MacBook Pro)",
     "tps": 79.1,
     "engine": "Ollama 0.34.1 (not llama-bench)",
     "date": "2026-09-16",
     "url": "https://bestllmfor.com/guides/macbook-pro-m5-max-128gb-ai-benchmark/",
     "needs_mem_gb": 128
    },
    "m5-ultra-80": {
     "label": "Apple M5 Ultra 80-core",
     "tps": 130,
     "engine": "LM Studio, spoken in video (not llama-bench)",
     "date": "2026",
     "url": "https://www.youtube.com/watch?v=c_58D7ixOQI",
     "needs_mem_gb": 96
    }
   }
  },
  "mid": {
   "short": "Qwen3-Coder-30B-A3B",
   "model": "qwen3-coder-30b-a3b",
   "metric": "Qwen3-Coder-30B-A3B (MoE, 3B active), llama-bench text generation tok/s at empty context, single request. Quantization differs per source: Q8_0 on GB10, UD-Q4_K_XL on Strix Halo, so compare with care.",
   "why": "Chosen over gpt-oss-20b because it has public results for more of our machines (Strix Halo incl. 64GB boxes, and GB10). gpt-oss-20b has results only for GB10 and M5 Max.",
   "min_mem_gb": 32,
   "chips": {
    "gb10": {
     "label": "NVIDIA GB10 (DGX Spark)",
     "tps": 61.06,
     "pp": 2986.97,
     "pp_metric": "pp2048",
     "engine": "llama.cpp CUDA llama-bench tg32, Q8_0",
     "date": "2026-02-05",
     "url": "https://github.com/ggml-org/llama.cpp/blob/master/benches/dgx-spark/dgx-spark.md"
    },
    "strix-halo": {
     "label": "AMD Ryzen AI Max+ 395 (Beelink GTR9 Pro)",
     "tps": 94.64,
     "pp": 1264.16,
     "pp_metric": "pp512",
     "engine": "llama.cpp Vulkan llama-bench tg128, UD-Q4_K_XL, b10687",
     "date": "2026-08-30",
     "url": "https://github.com/hogeheer499-commits/strix-halo-guide/blob/HEAD/data/benchmarks.csv"
    }
   }
  }
 },
 "llm_speed": {
  "source": "llm-speed.com",
  "url": "https://llm-speed.com/data",
  "license": "CC BY 4.0",
  "license_url": "https://creativecommons.org/licenses/by/4.0/",
  "changes": "Aggregated to median top_decode_tps per accelerator; mixed models, runtimes and workloads.",
  "imported": "2026-10-16",
  "groups": [
   {
    "accelerator": "RTX 5090",
    "runs": 165,
    "median_decode_tps": 71.3,
    "models": [
     "Codestral-22B-v0.1",
     "DeepSeek-Coder-V2-Lite-Instruct",
     "Llama-3.1-8B-Instruct",
     "Qwen2.5-14B-Instruct",
     "Qwen2.5-32B-Instruct",
     "Qwen2.5-7B-Instruct",
     "Qwen2.5-Coder-14B-Instruct",
     "Qwen2.5-Coder-32B-Instruct"
    ],
    "example_permalink": "https://llm-speed.com/r/r_bfpto9so2o1"
   },
   {
    "accelerator": "M3 Ultra (60-core GPU)",
    "runs": 21,
    "median_decode_tps": 80.3,
    "models": [
     "lmstudio-community-Qwen3-Next-80B-A3B-Instruct-MLX-4bit",
     "mlx-community-Qwen2.5-14B-Instruct-4bit",
     "mlx-community-Qwen2.5-32B-Instruct-4bit",
     "mlx-community-Qwen2.5-7B-Instruct-4bit",
     "mlx-community/Codestral-22B-v0.1-4bit",
     "mlx-community/DeepSeek-Coder-V2-Lite-Instruct-4bit",
     "mlx-community/Llama-3.1-8B-Instruct-4bit",
     "mlx-community/Qwen2.5-Coder-14B-Instruct-4bit"
    ],
    "example_permalink": "https://llm-speed.com/r/r_1pl79r50ofy"
   },
   {
    "accelerator": "RTX 4090",
    "runs": 14,
    "median_decode_tps": 131.1,
    "models": [
     "deepseek-r1",
     "gemma3",
     "glm-4.7-flash",
     "gpt-oss",
     "llama3.1",
     "qwen2.5-coder",
     "qwen3-coder",
     "qwen3.6"
    ],
    "example_permalink": "https://llm-speed.com/r/r_o636l3cc-rr"
   },
   {
    "accelerator": "M3 Pro (18-core GPU)",
    "runs": 11,
    "median_decode_tps": 19.4,
    "models": [
     "mlx-community-Qwen2.5-0.5B-Instruct-4bit",
     "mlx-community-Qwen2.5-14B-Instruct-4bit",
     "mlx-community-Qwen2.5-7B-Instruct-4bit",
     "mlx-community/Llama-3.1-8B-Instruct-4bit",
     "mlx-community/Qwen2.5-32B-Instruct-4bit",
     "mlx-community/Qwen3-32B-4bit",
     "mlx-community/gemma-2-9b-it-4bit",
     "mlx-community/phi-4-4bit"
    ],
    "example_permalink": "https://llm-speed.com/r/r_pnrrpcdqfo4"
   },
   {
    "accelerator": "M4 Max (40-core GPU)",
    "runs": 5,
    "median_decode_tps": 109.1,
    "models": [
     "qwen3-coder",
     "qwen3-coder-bench-32k"
    ],
    "example_permalink": "https://llm-speed.com/r/r_roktphpc--8"
   },
   {
    "accelerator": "RTX 3090",
    "runs": 4,
    "median_decode_tps": 137.7,
    "models": [
     "deepseek-coder-v2",
     "llama3.1",
     "qwen2.5-coder"
    ],
    "example_permalink": "https://llm-speed.com/r/r_o2-1w665rtq"
   }
  ]
 }
}