{ "schema": "aero.public.inference_benchmark.v1", "suite": "cache-neutral", "publication_date": "2026-09-24", "scope": "Synthetic inference and mocked tools; not CFD/FEA", "rows": [ { "system_id": "3090", "model": "gemma3:12b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 15.250062100007199, "wall_seconds_mean": 15.310228959994856, "wall_seconds_min": 13.294200700009242, "wall_seconds_max": 17.01875409996137, "wall_seconds_stdev": 1.5587532673654698, "wall_seconds_p95": 17.01875409996137, "prompt_tokens_median": 15368.0, "prompt_tokens_mean": 15383.0, "prompt_tokens_min": 15292.0, "prompt_tokens_max": 15458.0, "prompt_tokens_stdev": 73.64102117705865, "prompt_tokens_p95": 15458.0, "output_tokens_median": 555.0, "output_tokens_mean": 544.8, "output_tokens_min": 428.0, "output_tokens_max": 633.0, "output_tokens_stdev": 82.31464025311658, "output_tokens_p95": 633.0, "prefill_tokens_per_second_median": 5455.050145427775, "prefill_tokens_per_second_mean": 5355.755767719014, "prefill_tokens_per_second_min": 4903.200021569259, "prefill_tokens_per_second_max": 5652.367774244306, "prefill_tokens_per_second_stdev": 297.2036930334741, "prefill_tokens_per_second_p95": 5652.367774244306, "decode_tokens_per_second_median": 63.002455623749334, "decode_tokens_per_second_mean": 63.105874650084786, "decode_tokens_per_second_min": 62.310606865447504, "decode_tokens_per_second_max": 64.20948972769676, "decode_tokens_per_second_stdev": 0.7004021505583445, "decode_tokens_per_second_p95": 64.20948972769676, "prefill_seconds_median": 2.817206, "prefill_seconds_mean": 2.8797585998, "prefill_seconds_min": 2.71373, "prefill_seconds_max": 3.152635, "prefill_seconds_stdev": 0.17036202815038864, "prefill_seconds_p95": 3.152635, "decode_seconds_median": 8.781147, "decode_seconds_mean": 8.6394523998, "decode_seconds_min": 6.793386, "decode_seconds_max": 10.079099, "decode_seconds_stdev": 1.3559664534960998, "decode_seconds_p95": 10.079099, "peak_gpu_memory_used_mib_median": 11467.0, "peak_system_memory_used_bytes_median": 50810404864.0, "gpu_energy_j_observed_median": 4991.832356247902 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 4096, "n": 5, "wall_seconds_median": 3.8892678000265732, "wall_seconds_mean": 3.9239291800069624, "wall_seconds_min": 3.794923799985554, "wall_seconds_max": 4.066444299998693, "wall_seconds_stdev": 0.10910304795343757, "wall_seconds_p95": 4.066444299998693, "ttft_seconds_median": 2.8154166000313126, "ttft_seconds_mean": 2.824477059999481, "ttft_seconds_min": 2.699208899983205, "ttft_seconds_max": 2.9379825000069104, "ttft_seconds_stdev": 0.09256091384044365, "ttft_seconds_p95": 2.9379825000069104, "prompt_tokens_median": 3755.0, "prompt_tokens_mean": 3755.0, "prompt_tokens_min": 3754.0, "prompt_tokens_max": 3756.0, "prompt_tokens_stdev": 0.7071067811865476, "prompt_tokens_p95": 3756.0, "output_tokens_median": 72.0, "output_tokens_mean": 72.0, "output_tokens_min": 70.0, "output_tokens_max": 73.0, "output_tokens_stdev": 1.224744871391589, "output_tokens_p95": 73.0, "prefill_tokens_per_second_median": 2473.3367057658056, "prefill_tokens_per_second_mean": 2457.885501607207, "prefill_tokens_per_second_min": 2348.1534719081656, "prefill_tokens_per_second_max": 2519.826437307658, "prefill_tokens_per_second_stdev": 69.08176667964976, "prefill_tokens_per_second_p95": 2519.826437307658, "decode_tokens_per_second_median": 65.06663003656384, "decode_tokens_per_second_mean": 65.50541346893469, "decode_tokens_per_second_min": 64.52110108843513, "decode_tokens_per_second_max": 66.64268747135733, "decode_tokens_per_second_stdev": 1.0435036433228404, "decode_tokens_per_second_p95": 66.64268747135733, "load_seconds_median": 0.5834082, "load_seconds_mean": 0.5734107, "load_seconds_min": 0.5321438, "load_seconds_max": 0.5929104, "load_seconds_stdev": 0.024352730406465743, "load_seconds_p95": 0.5929104, "peak_gpu_memory_used_mib_median": 10902.0, "peak_system_memory_used_bytes_median": 49595109376.0, "gpu_energy_j_observed_median": 1255.7060613453389 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 8192, "n": 5, "wall_seconds_median": 5.413977899996098, "wall_seconds_mean": 5.380829600000288, "wall_seconds_min": 5.228105399990454, "wall_seconds_max": 5.521683400031179, "wall_seconds_stdev": 0.122154328061734, "wall_seconds_p95": 5.521683400031179, "ttft_seconds_median": 4.28658740001265, "ttft_seconds_mean": 4.252587740006857, "ttft_seconds_min": 4.119694299995899, "ttft_seconds_max": 4.389525400009006, "ttft_seconds_stdev": 0.11549995394728844, "ttft_seconds_p95": 4.389525400009006, "prompt_tokens_median": 7825.0, "prompt_tokens_mean": 7824.6, "prompt_tokens_min": 7823.0, "prompt_tokens_max": 7826.0, "prompt_tokens_stdev": 1.140175425099138, "prompt_tokens_p95": 7826.0, "output_tokens_median": 72.0, "output_tokens_mean": 72.4, "output_tokens_min": 72.0, "output_tokens_max": 73.0, "output_tokens_stdev": 0.5477225575051661, "output_tokens_p95": 73.0, "prefill_tokens_per_second_median": 2880.263843210583, "prefill_tokens_per_second_mean": 2886.8105365675156, "prefill_tokens_per_second_min": 2874.6408994621343, "prefill_tokens_per_second_max": 2907.0199478638756, "prefill_tokens_per_second_stdev": 13.642038834127419, "prefill_tokens_per_second_p95": 2907.0199478638756, "decode_tokens_per_second_median": 64.2274319761109, "decode_tokens_per_second_mean": 64.20668629941896, "decode_tokens_per_second_min": 63.63482097403699, "decode_tokens_per_second_max": 64.99602802050985, "decode_tokens_per_second_stdev": 0.511047776929915, "decode_tokens_per_second_p95": 64.99602802050985, "load_seconds_median": 0.5716318, "load_seconds_mean": 0.57495126, "load_seconds_min": 0.5567888, "load_seconds_max": 0.5963928, "load_seconds_stdev": 0.016294201705760236, "load_seconds_p95": 0.5963928, "peak_gpu_memory_used_mib_median": 11445.0, "peak_system_memory_used_bytes_median": 50285895680.0, "gpu_energy_j_observed_median": 1760.9491747415066 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 16384, "n": 5, "wall_seconds_median": 8.56556359998649, "wall_seconds_mean": 8.666807460004929, "wall_seconds_min": 8.388355900009628, "wall_seconds_max": 8.98048149998067, "wall_seconds_stdev": 0.25218584436175207, "wall_seconds_p95": 8.98048149998067, "ttft_seconds_median": 7.3863080000155605, "ttft_seconds_mean": 7.493321900011506, "ttft_seconds_min": 7.197479900030885, "ttft_seconds_max": 7.80482119997032, "ttft_seconds_stdev": 0.26266868460334636, "ttft_seconds_p95": 7.80482119997032, "prompt_tokens_median": 15966.0, "prompt_tokens_mean": 15965.4, "prompt_tokens_min": 15964.0, "prompt_tokens_max": 15966.0, "prompt_tokens_stdev": 0.8944271909999159, "prompt_tokens_p95": 15966.0, "output_tokens_median": 71.0, "output_tokens_mean": 71.0, "output_tokens_min": 70.0, "output_tokens_max": 72.0, "output_tokens_stdev": 1.0, "output_tokens_p95": 72.0, "prefill_tokens_per_second_median": 2931.371105412957, "prefill_tokens_per_second_mean": 2912.7548122520793, "prefill_tokens_per_second_min": 2853.3362979623776, "prefill_tokens_per_second_max": 2951.4972770981744, "prefill_tokens_per_second_stdev": 42.67452287761288, "prefill_tokens_per_second_p95": 2951.4972770981744, "decode_tokens_per_second_median": 61.03685928736851, "decode_tokens_per_second_mean": 60.56653289463804, "decode_tokens_per_second_min": 59.42774477651347, "decode_tokens_per_second_max": 61.398975660422735, "decode_tokens_per_second_stdev": 0.9391535961743639, "decode_tokens_per_second_p95": 61.398975660422735, "load_seconds_median": 0.5853953, "load_seconds_mean": 0.57641438, "load_seconds_min": 0.5116205, "load_seconds_max": 0.614046, "load_seconds_stdev": 0.03837861171677525, "load_seconds_p95": 0.614046, "peak_gpu_memory_used_mib_median": 11978.0, "peak_system_memory_used_bytes_median": 51964956672.0, "gpu_energy_j_observed_median": 3085.3840927076344 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 32768, "n": 5, "wall_seconds_median": 16.494066099985503, "wall_seconds_mean": 16.408765640005004, "wall_seconds_min": 15.760115500015672, "wall_seconds_max": 16.71107389999088, "wall_seconds_stdev": 0.3750550822791153, "wall_seconds_p95": 16.71107389999088, "ttft_seconds_median": 15.227572900010273, "ttft_seconds_mean": 15.072798440000042, "ttft_seconds_min": 14.452683900017291, "ttft_seconds_max": 15.28040129999863, "ttft_seconds_stdev": 0.3487622329943361, "ttft_seconds_p95": 15.28040129999863, "prompt_tokens_median": 32244.0, "prompt_tokens_mean": 32244.6, "prompt_tokens_min": 32244.0, "prompt_tokens_max": 32246.0, "prompt_tokens_stdev": 0.8944271909999159, "prompt_tokens_p95": 32246.0, "output_tokens_median": 72.0, "output_tokens_mean": 73.2, "output_tokens_min": 69.0, "output_tokens_max": 79.0, "output_tokens_stdev": 4.207136793592526, "output_tokens_p95": 79.0, "prefill_tokens_per_second_median": 2678.9470850623024, "prefill_tokens_per_second_mean": 2682.491077486075, "prefill_tokens_per_second_min": 2661.8530307760866, "prefill_tokens_per_second_max": 2717.16965398598, "prefill_tokens_per_second_stdev": 22.215475962320394, "prefill_tokens_per_second_p95": 2717.16965398598, "decode_tokens_per_second_median": 55.16052074434768, "decode_tokens_per_second_mean": 54.87679749180499, "decode_tokens_per_second_min": 53.595163572439226, "decode_tokens_per_second_max": 55.72515384988127, "decode_tokens_per_second_stdev": 0.8227144003339798, "decode_tokens_per_second_p95": 55.72515384988127, "load_seconds_median": 0.6009852, "load_seconds_mean": 0.59675208, "load_seconds_min": 0.5605209, "load_seconds_max": 0.6357547, "load_seconds_stdev": 0.02883680813157033, "load_seconds_p95": 0.6357547, "peak_gpu_memory_used_mib_median": 12703.0, "peak_system_memory_used_bytes_median": 55253917696.0, "gpu_energy_j_observed_median": 5857.519420833588 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 15.439900700002909, "wall_seconds_mean": 15.26069694000762, "wall_seconds_min": 13.74584660003893, "wall_seconds_max": 15.961318399989977, "wall_seconds_stdev": 0.8826295546479545, "wall_seconds_p95": 15.961318399989977, "ttft_seconds_median": 7.663781000010204, "ttft_seconds_mean": 7.737743760005105, "ttft_seconds_min": 7.323985800030641, "ttft_seconds_max": 8.363466500013601, "ttft_seconds_stdev": 0.42932905910577024, "ttft_seconds_p95": 8.363466500013601, "prompt_tokens_median": 16342.0, "prompt_tokens_mean": 16342.2, "prompt_tokens_min": 16341.0, "prompt_tokens_max": 16344.0, "prompt_tokens_stdev": 1.3038404810405297, "prompt_tokens_p95": 16344.0, "output_tokens_median": 461.0, "output_tokens_mean": 449.0, "output_tokens_min": 385.0, "output_tokens_max": 504.0, "output_tokens_stdev": 47.460509900337144, "output_tokens_p95": 504.0, "prefill_tokens_per_second_median": 2882.5326494182723, "prefill_tokens_per_second_mean": 2881.8162469086583, "prefill_tokens_per_second_min": 2782.0161651071444, "prefill_tokens_per_second_max": 2963.596781085212, "prefill_tokens_per_second_stdev": 77.87477916690666, "prefill_tokens_per_second_p95": 2963.596781085212, "decode_tokens_per_second_median": 59.76962384745076, "decode_tokens_per_second_mean": 59.735967509496156, "decode_tokens_per_second_min": 59.29428990843341, "decode_tokens_per_second_max": 60.09048625722233, "decode_tokens_per_second_stdev": 0.3178824188210302, "decode_tokens_per_second_p95": 60.09048625722233, "load_seconds_median": 0.5513586, "load_seconds_mean": 0.57255808, "load_seconds_min": 0.5443296, "load_seconds_max": 0.6308371, "load_seconds_stdev": 0.037172806026408635, "load_seconds_p95": 0.6308371, "peak_gpu_memory_used_mib_median": 12229.0, "peak_system_memory_used_bytes_median": 52157394944.0, "gpu_energy_j_observed_median": 5696.69774291873 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 5.1247525999788195, "wall_seconds_mean": 4.965647040004842, "wall_seconds_min": 4.39668000000529, "wall_seconds_max": 5.342855000053532, "wall_seconds_stdev": 0.41331820316183093, "wall_seconds_p95": 5.342855000053532, "ttft_seconds_median": 3.0642570999916643, "ttft_seconds_mean": 3.093392219999805, "ttft_seconds_min": 3.0504867000272498, "ttft_seconds_max": 3.2230416999664158, "ttft_seconds_stdev": 0.072764259231271, "ttft_seconds_p95": 3.2230416999664158, "prompt_tokens_median": 4148.0, "prompt_tokens_mean": 4147.4, "prompt_tokens_min": 4146.0, "prompt_tokens_max": 4148.0, "prompt_tokens_stdev": 0.8944271909999159, "prompt_tokens_p95": 4148.0, "output_tokens_median": 129.0, "output_tokens_mean": 118.6, "output_tokens_min": 85.0, "output_tokens_max": 148.0, "output_tokens_stdev": 24.925890154616344, "output_tokens_p95": 148.0, "prefill_tokens_per_second_median": 2471.905023545074, "prefill_tokens_per_second_mean": 2480.6921243763777, "prefill_tokens_per_second_min": 2417.429738348781, "prefill_tokens_per_second_max": 2546.0614307907704, "prefill_tokens_per_second_stdev": 46.69257436410747, "prefill_tokens_per_second_p95": 2546.0614307907704, "decode_tokens_per_second_median": 62.89951986699835, "decode_tokens_per_second_mean": 63.34452354448217, "decode_tokens_per_second_min": 62.38794958008459, "decode_tokens_per_second_max": 64.89830479488847, "decode_tokens_per_second_stdev": 1.0124487887580693, "decode_tokens_per_second_p95": 64.89830479488847, "load_seconds_median": 0.619134, "load_seconds_mean": 0.6127801, "load_seconds_min": 0.5545744, "load_seconds_max": 0.6889678, "load_seconds_stdev": 0.05355983241553318, "load_seconds_p95": 0.6889678, "peak_gpu_memory_used_mib_median": 11430.0, "peak_system_memory_used_bytes_median": 50638450688.0, "gpu_energy_j_observed_median": 1682.8222725749017 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 33.88235909998184, "wall_seconds_mean": 33.772788040013985, "wall_seconds_min": 33.34486390004167, "wall_seconds_max": 34.10327680001501, "wall_seconds_stdev": 0.3383141413140818, "wall_seconds_p95": 34.10327680001501, "ttft_seconds_median": 15.634014899958856, "ttft_seconds_mean": 15.487217580003199, "ttft_seconds_min": 14.8934177000192, "ttft_seconds_max": 15.993335000006482, "ttft_seconds_stdev": 0.45066414874862515, "ttft_seconds_p95": 15.993335000006482, "prompt_tokens_median": 32600.0, "prompt_tokens_mean": 32599.6, "prompt_tokens_min": 32598.0, "prompt_tokens_max": 32601.0, "prompt_tokens_stdev": 1.140175425099138, "prompt_tokens_p95": 32601.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 2650.888620963016, "prefill_tokens_per_second_mean": 2661.6044482694692, "prefill_tokens_per_second_min": 2645.1767760003154, "prefill_tokens_per_second_max": 2683.0900308390746, "prefill_tokens_per_second_stdev": 17.814603584403496, "prefill_tokens_per_second_p95": 2683.0900308390746, "decode_tokens_per_second_median": 54.69606463455836, "decode_tokens_per_second_mean": 54.697325113086244, "decode_tokens_per_second_min": 54.205392580116516, "decode_tokens_per_second_max": 55.224298735623115, "decode_tokens_per_second_stdev": 0.3709884347424741, "decode_tokens_per_second_p95": 55.224298735623115, "load_seconds_median": 0.6318364, "load_seconds_mean": 0.6392795, "load_seconds_min": 0.547512, "load_seconds_max": 0.7533612, "load_seconds_stdev": 0.07553356462983855, "load_seconds_p95": 0.7533612, "peak_gpu_memory_used_mib_median": 12991.0, "peak_system_memory_used_bytes_median": 51909218304.0, "gpu_energy_j_observed_median": 12683.678114438057 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 33.501322600001004, "wall_seconds_mean": 33.844878959993366, "wall_seconds_min": 32.95163719996344, "wall_seconds_max": 35.074595500016585, "wall_seconds_stdev": 0.9014366430672999, "wall_seconds_p95": 35.074595500016585, "ttft_seconds_median": 15.541699900000822, "ttft_seconds_mean": 15.536391039995943, "ttft_seconds_min": 14.93693099997472, "ttft_seconds_max": 16.216645599983167, "ttft_seconds_stdev": 0.550862463469368, "ttft_seconds_p95": 16.216645599983167, "prompt_tokens_median": 32591.0, "prompt_tokens_mean": 32590.8, "prompt_tokens_min": 32589.0, "prompt_tokens_max": 32592.0, "prompt_tokens_stdev": 1.3038404810405297, "prompt_tokens_p95": 32592.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 2660.929839529406, "prefill_tokens_per_second_mean": 2654.9918988298346, "prefill_tokens_per_second_min": 2606.6611510068637, "prefill_tokens_per_second_max": 2704.0602637634847, "prefill_tokens_per_second_stdev": 37.53664141403212, "prefill_tokens_per_second_p95": 2704.0602637634847, "decode_tokens_per_second_median": 54.73949610651649, "decode_tokens_per_second_mean": 54.65476476406066, "decode_tokens_per_second_min": 53.075761615948885, "decode_tokens_per_second_max": 55.875473684328156, "decode_tokens_per_second_stdev": 1.1858216726481101, "decode_tokens_per_second_p95": 55.875473684328156, "load_seconds_median": 0.6411806, "load_seconds_mean": 0.64651982, "load_seconds_min": 0.6060859, "load_seconds_max": 0.6918602, "load_seconds_stdev": 0.03173157707374472, "load_seconds_p95": 0.6918602, "peak_gpu_memory_used_mib_median": 12968.0, "peak_system_memory_used_bytes_median": 52509642752.0, "gpu_energy_j_observed_median": 12795.212949161529 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 27.931950800004415, "wall_seconds_mean": 27.43398042001063, "wall_seconds_min": 26.0722452000482, "wall_seconds_max": 28.63045619998593, "wall_seconds_stdev": 1.2669282898666048, "wall_seconds_p95": 28.63045619998593, "prompt_tokens_median": 15323.0, "prompt_tokens_mean": 15322.8, "prompt_tokens_min": 15296.0, "prompt_tokens_max": 15358.0, "prompt_tokens_stdev": 22.664950915455343, "prompt_tokens_p95": 15358.0, "output_tokens_median": 544.0, "output_tokens_mean": 532.8, "output_tokens_min": 495.0, "output_tokens_max": 562.0, "output_tokens_stdev": 31.76003778335284, "output_tokens_p95": 562.0, "prefill_tokens_per_second_median": 7496.48946715368, "prefill_tokens_per_second_mean": 7497.435627471168, "prefill_tokens_per_second_min": 7446.674010458677, "prefill_tokens_per_second_max": 7537.639437899928, "prefill_tokens_per_second_stdev": 36.42302915491628, "prefill_tokens_per_second_p95": 7537.639437899928, "decode_tokens_per_second_median": 23.943560681844446, "decode_tokens_per_second_mean": 23.939409748052242, "decode_tokens_per_second_min": 23.90244137530423, "decode_tokens_per_second_max": 23.97208282249971, "decode_tokens_per_second_stdev": 0.030149685997465803, "decode_tokens_per_second_p95": 23.97208282249971, "prefill_seconds_median": 2.040642478, "prefill_seconds_mean": 2.0437730276, "prefill_seconds_min": 2.032997217, "prefill_seconds_max": 2.057697165, "prefill_seconds_stdev": 0.009248562110536823, "prefill_seconds_p95": 2.057697165, "decode_seconds_median": 22.759181435000002, "decode_seconds_mean": 22.256264860399998, "decode_seconds_min": 20.649019264, "decode_seconds_max": 23.471863999999997, "decode_seconds_stdev": 1.327419355497639, "decode_seconds_p95": 23.471863999999997, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 58920325120.0, "gpu_energy_j_observed_median": 1214.901846038103 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 4096, "n": 5, "wall_seconds_median": 5.303116599970963, "wall_seconds_mean": 5.251062819990329, "wall_seconds_min": 4.994485000032, "wall_seconds_max": 5.362568399985321, "wall_seconds_stdev": 0.14761389009056114, "wall_seconds_p95": 5.362568399985321, "ttft_seconds_median": 2.1304803999955766, "ttft_seconds_mean": 2.1370303599978797, "ttft_seconds_min": 2.1085351000074297, "ttft_seconds_max": 2.17162089998601, "ttft_seconds_stdev": 0.02352971792079961, "ttft_seconds_p95": 2.17162089998601, "prompt_tokens_median": 3754.0, "prompt_tokens_mean": 3754.0, "prompt_tokens_min": 3753.0, "prompt_tokens_max": 3755.0, "prompt_tokens_stdev": 0.7071067811865476, "prompt_tokens_p95": 3755.0, "output_tokens_median": 76.0, "output_tokens_mean": 74.4, "output_tokens_min": 69.0, "output_tokens_max": 76.0, "output_tokens_stdev": 3.0495901363953815, "output_tokens_p95": 76.0, "prefill_tokens_per_second_median": 1929.5290907443048, "prefill_tokens_per_second_mean": 1929.8935659795898, "prefill_tokens_per_second_min": 1927.2751342804052, "prefill_tokens_per_second_max": 1933.5451460037034, "prefill_tokens_per_second_stdev": 2.462845050287424, "prefill_tokens_per_second_p95": 1933.5451460037034, "decode_tokens_per_second_median": 24.422959782243844, "decode_tokens_per_second_mean": 24.425276714878756, "decode_tokens_per_second_min": 24.39598638567342, "decode_tokens_per_second_max": 24.461791494557268, "decode_tokens_per_second_stdev": 0.023862126004536625, "decode_tokens_per_second_p95": 24.461791494557268, "load_seconds_median": 0.1588862, "load_seconds_mean": 0.1601791176, "load_seconds_min": 0.134295715, "load_seconds_max": 0.176928944, "load_seconds_stdev": 0.017660964060054546, "load_seconds_p95": 0.176928944, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 21752684544.0, "gpu_energy_j_observed_median": 314.06043271541597 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 8192, "n": 5, "wall_seconds_median": 7.624518800002988, "wall_seconds_mean": 7.583947059989441, "wall_seconds_min": 7.442317099950742, "wall_seconds_max": 7.764811199973337, "wall_seconds_stdev": 0.1386236701843631, "wall_seconds_p95": 7.764811199973337, "ttft_seconds_median": 4.3120061000227, "ttft_seconds_mean": 4.309781539998949, "ttft_seconds_min": 4.277088199974969, "ttft_seconds_max": 4.332892600039486, "ttft_seconds_stdev": 0.021026826171276325, "ttft_seconds_p95": 4.332892600039486, "prompt_tokens_median": 7824.0, "prompt_tokens_mean": 7823.6, "prompt_tokens_min": 7822.0, "prompt_tokens_max": 7825.0, "prompt_tokens_stdev": 1.140175425099138, "prompt_tokens_p95": 7825.0, "output_tokens_median": 77.0, "output_tokens_mean": 76.0, "output_tokens_min": 72.0, "output_tokens_max": 81.0, "output_tokens_stdev": 3.605551275463989, "output_tokens_p95": 81.0, "prefill_tokens_per_second_median": 1914.3600673973658, "prefill_tokens_per_second_mean": 1913.9336362294475, "prefill_tokens_per_second_min": 1909.7892473610646, "prefill_tokens_per_second_max": 1917.3945130951588, "prefill_tokens_per_second_stdev": 3.1400432924782247, "prefill_tokens_per_second_p95": 1917.3945130951588, "decode_tokens_per_second_median": 23.587374766218904, "decode_tokens_per_second_mean": 23.587098989557955, "decode_tokens_per_second_min": 23.57466691108494, "decode_tokens_per_second_max": 23.60397592241817, "decode_tokens_per_second_stdev": 0.011589523853598422, "decode_tokens_per_second_p95": 23.60397592241817, "load_seconds_median": 0.171432714, "load_seconds_mean": 0.1675103644, "load_seconds_min": 0.151681892, "load_seconds_max": 0.182520084, "load_seconds_stdev": 0.013103130410946108, "load_seconds_p95": 0.182520084, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 22281527296.0, "gpu_energy_j_observed_median": 487.7271253848076 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 16384, "n": 5, "wall_seconds_median": 12.217662499984726, "wall_seconds_mean": 12.164968939987011, "wall_seconds_min": 11.974300400004722, "wall_seconds_max": 12.250874199962709, "wall_seconds_stdev": 0.11193952107799794, "wall_seconds_p95": 12.250874199962709, "ttft_seconds_median": 8.731814400001895, "ttft_seconds_mean": 8.72760451999493, "ttft_seconds_min": 8.700153800018597, "ttft_seconds_max": 8.743327100004535, "ttft_seconds_stdev": 0.016755571097308488, "ttft_seconds_p95": 8.743327100004535, "prompt_tokens_median": 15965.0, "prompt_tokens_mean": 15964.4, "prompt_tokens_min": 15963.0, "prompt_tokens_max": 15965.0, "prompt_tokens_stdev": 0.8944271909999159, "prompt_tokens_p95": 15965.0, "output_tokens_median": 77.0, "output_tokens_mean": 75.6, "output_tokens_min": 72.0, "output_tokens_max": 77.0, "output_tokens_stdev": 2.1908902300206643, "output_tokens_p95": 77.0, "prefill_tokens_per_second_median": 1889.5147921498765, "prefill_tokens_per_second_mean": 1890.246690106256, "prefill_tokens_per_second_min": 1887.8982079153755, "prefill_tokens_per_second_max": 1896.1107715651588, "prefill_tokens_per_second_stdev": 3.389300875916402, "prefill_tokens_per_second_p95": 1896.1107715651588, "decode_tokens_per_second_median": 22.361357243854002, "decode_tokens_per_second_mean": 22.350704365208323, "decode_tokens_per_second_min": 22.29923601999757, "decode_tokens_per_second_max": 22.397440482400054, "decode_tokens_per_second_stdev": 0.038623063508181475, "decode_tokens_per_second_p95": 22.397440482400054, "load_seconds_median": 0.170054952, "load_seconds_mean": 0.1587434626, "load_seconds_min": 0.136372752, "load_seconds_max": 0.175931581, "load_seconds_stdev": 0.01944548577548398, "load_seconds_p95": 0.175931581, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 23404138496.0, "gpu_energy_j_observed_median": 887.3743614304065 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 32768, "n": 5, "wall_seconds_median": 22.140859699982684, "wall_seconds_mean": 22.11485739998752, "wall_seconds_min": 21.89104319998296, "wall_seconds_max": 22.233898099977523, "wall_seconds_stdev": 0.13194427149392857, "wall_seconds_p95": 22.233898099977523, "ttft_seconds_median": 18.33773619995918, "ttft_seconds_mean": 18.31452079999726, "ttft_seconds_min": 18.2486315999995, "ttft_seconds_max": 18.345240800001193, "ttft_seconds_stdev": 0.040754333630193866, "ttft_seconds_p95": 18.345240800001193, "prompt_tokens_median": 32243.0, "prompt_tokens_mean": 32243.6, "prompt_tokens_min": 32243.0, "prompt_tokens_max": 32245.0, "prompt_tokens_stdev": 0.8944271909999159, "prompt_tokens_p95": 32245.0, "output_tokens_median": 75.0, "output_tokens_mean": 75.2, "output_tokens_min": 72.0, "output_tokens_max": 77.0, "output_tokens_stdev": 2.04939015319192, "output_tokens_p95": 77.0, "prefill_tokens_per_second_median": 1800.3659368088581, "prefill_tokens_per_second_mean": 1801.0232137338187, "prefill_tokens_per_second_min": 1799.7078833318237, "prefill_tokens_per_second_max": 1803.7149335257898, "prefill_tokens_per_second_stdev": 1.608183610599127, "prefill_tokens_per_second_p95": 1803.7149335257898, "decode_tokens_per_second_median": 20.059087760616496, "decode_tokens_per_second_mean": 20.060044170105115, "decode_tokens_per_second_min": 20.04057233642875, "decode_tokens_per_second_max": 20.08046523022695, "decode_tokens_per_second_stdev": 0.015162832155933538, "decode_tokens_per_second_p95": 20.08046523022695, "load_seconds_median": 0.186293524, "load_seconds_mean": 0.1789194954, "load_seconds_min": 0.134193716, "load_seconds_max": 0.204142142, "load_seconds_stdev": 0.02631814777897523, "load_seconds_p95": 0.204142142, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 25179545600.0, "gpu_energy_j_observed_median": 1763.715427210331 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 65536, "n": 5, "wall_seconds_median": 44.450516499986406, "wall_seconds_mean": 44.355595719988926, "wall_seconds_min": 43.99572829995304, "wall_seconds_max": 44.525265000003856, "wall_seconds_stdev": 0.21105886303693283, "wall_seconds_p95": 44.525265000003856, "ttft_seconds_median": 39.98545689997263, "ttft_seconds_mean": 39.94977911999449, "ttft_seconds_min": 39.75334070000099, "ttft_seconds_max": 40.04904329997953, "ttft_seconds_stdev": 0.12020365024241317, "ttft_seconds_p95": 40.04904329997953, "prompt_tokens_median": 64749.0, "prompt_tokens_mean": 64748.6, "prompt_tokens_min": 64747.0, "prompt_tokens_max": 64750.0, "prompt_tokens_stdev": 1.51657508881031, "prompt_tokens_p95": 64750.0, "output_tokens_median": 74.0, "output_tokens_mean": 73.6, "output_tokens_min": 70.0, "output_tokens_max": 76.0, "output_tokens_stdev": 2.6076809620810595, "output_tokens_p95": 76.0, "prefill_tokens_per_second_median": 1647.5613522262074, "prefill_tokens_per_second_mean": 1648.8288777938985, "prefill_tokens_per_second_min": 1646.6460150093308, "prefill_tokens_per_second_max": 1653.9433806954885, "prefill_tokens_per_second_stdev": 2.9886397708249928, "prefill_tokens_per_second_p95": 1653.9433806954885, "decode_tokens_per_second_median": 16.943152205586994, "decode_tokens_per_second_mean": 16.939806364274805, "decode_tokens_per_second_min": 16.919108858818714, "decode_tokens_per_second_max": 16.95337102427748, "decode_tokens_per_second_stdev": 0.014634610000066163, "decode_tokens_per_second_p95": 16.95337102427748, "load_seconds_median": 0.155838701, "load_seconds_mean": 0.1630634198, "load_seconds_min": 0.145294082, "load_seconds_max": 0.19072207, "load_seconds_stdev": 0.019809172096383613, "load_seconds_p95": 0.19072207, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 28284604416.0, "gpu_energy_j_observed_median": 3737.974726675749 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 131072, "n": 5, "wall_seconds_median": 98.65267039998434, "wall_seconds_mean": 98.71802075997694, "wall_seconds_min": 98.58487449999666, "wall_seconds_max": 99.00234129995806, "wall_seconds_stdev": 0.16419479587461092, "wall_seconds_p95": 99.00234129995806, "ttft_seconds_median": 92.87623599998187, "ttft_seconds_mean": 92.86890469999052, "ttft_seconds_min": 92.79717640002491, "ttft_seconds_max": 92.91238609998254, "ttft_seconds_stdev": 0.04385798439701495, "ttft_seconds_p95": 92.91238609998254, "prompt_tokens_median": 129815.0, "prompt_tokens_mean": 129814.2, "prompt_tokens_min": 129813.0, "prompt_tokens_max": 129815.0, "prompt_tokens_stdev": 1.0954451150103321, "prompt_tokens_p95": 129815.0, "output_tokens_median": 74.0, "output_tokens_mean": 75.0, "output_tokens_min": 74.0, "output_tokens_max": 79.0, "output_tokens_stdev": 2.23606797749979, "output_tokens_p95": 79.0, "prefill_tokens_per_second_median": 1418.909719657166, "prefill_tokens_per_second_mean": 1418.96871175671, "prefill_tokens_per_second_min": 1418.4451545897905, "prefill_tokens_per_second_max": 1419.5474451564335, "prefill_tokens_per_second_stdev": 0.5036844006855922, "prefill_tokens_per_second_p95": 1419.5474451564335, "decode_tokens_per_second_median": 12.986455620098058, "decode_tokens_per_second_mean": 12.98647963637432, "decode_tokens_per_second_min": 12.975118970075679, "decode_tokens_per_second_max": 12.999150930838635, "decode_tokens_per_second_stdev": 0.008722877058857603, "decode_tokens_per_second_p95": 12.999150930838635, "load_seconds_median": 0.188572795, "load_seconds_mean": 0.1864948224, "load_seconds_min": 0.172842136, "load_seconds_max": 0.198248102, "load_seconds_stdev": 0.011904352962838437, "load_seconds_p95": 0.198248102, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 35325878272.0, "gpu_energy_j_observed_median": 8288.747153700586 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 30.595761200005654, "wall_seconds_mean": 30.439709800004493, "wall_seconds_min": 28.157903499959502, "wall_seconds_max": 32.322964899998624, "wall_seconds_stdev": 1.4893297846329752, "wall_seconds_p95": 32.322964899998624, "ttft_seconds_median": 9.045793800032698, "ttft_seconds_mean": 9.057788720005192, "ttft_seconds_min": 9.036147099977825, "ttft_seconds_max": 9.10183490003692, "ttft_seconds_stdev": 0.026807682780800396, "ttft_seconds_p95": 9.10183490003692, "prompt_tokens_median": 16341.0, "prompt_tokens_mean": 16341.2, "prompt_tokens_min": 16340.0, "prompt_tokens_max": 16343.0, "prompt_tokens_stdev": 1.3038404810405297, "prompt_tokens_p95": 16343.0, "output_tokens_median": 467.0, "output_tokens_mean": 463.6, "output_tokens_min": 414.0, "output_tokens_max": 505.0, "output_tokens_stdev": 32.585272747055534, "output_tokens_p95": 505.0, "prefill_tokens_per_second_median": 1865.9593806939831, "prefill_tokens_per_second_mean": 1864.9931913747523, "prefill_tokens_per_second_min": 1859.009009152938, "prefill_tokens_per_second_max": 1869.2766510346985, "prefill_tokens_per_second_stdev": 3.8794696096942474, "prefill_tokens_per_second_p95": 1869.2766510346985, "decode_tokens_per_second_median": 21.908803590481785, "decode_tokens_per_second_mean": 21.90269750121584, "decode_tokens_per_second_min": 21.884336629565517, "decode_tokens_per_second_max": 21.920556651674577, "decode_tokens_per_second_stdev": 0.01676520097345648, "decode_tokens_per_second_p95": 21.920556651674577, "load_seconds_median": 0.161482251, "load_seconds_mean": 0.173763677, "load_seconds_min": 0.155857655, "load_seconds_max": 0.206488412, "load_seconds_stdev": 0.021267113957342768, "load_seconds_p95": 0.206488412, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 60561948672.0, "gpu_energy_j_observed_median": 1767.0235483384129 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 7.435787199996412, "wall_seconds_mean": 9.929244880005717, "wall_seconds_min": 6.577971300052013, "wall_seconds_max": 18.6215741999913, "wall_seconds_stdev": 5.101124902583494, "wall_seconds_p95": 18.6215741999913, "ttft_seconds_median": 2.3737117999698967, "ttft_seconds_mean": 3.399745719996281, "ttft_seconds_min": 2.334583499992732, "ttft_seconds_max": 5.2361119000124745, "ttft_seconds_stdev": 1.4364654902286156, "ttft_seconds_p95": 5.2361119000124745, "prompt_tokens_median": 4147.0, "prompt_tokens_mean": 4146.4, "prompt_tokens_min": 4145.0, "prompt_tokens_max": 4147.0, "prompt_tokens_stdev": 0.8944271909999159, "prompt_tokens_p95": 4147.0, "output_tokens_median": 110.0, "output_tokens_mean": 110.4, "output_tokens_min": 100.0, "output_tokens_max": 121.0, "output_tokens_stdev": 10.502380682492898, "output_tokens_p95": 121.0, "prefill_tokens_per_second_median": 1895.969384242992, "prefill_tokens_per_second_mean": 1500.5273687157294, "prefill_tokens_per_second_min": 831.4052678620861, "prefill_tokens_per_second_max": 1909.1241448091168, "prefill_tokens_per_second_stdev": 554.9560413690413, "prefill_tokens_per_second_p95": 1909.1241448091168, "decode_tokens_per_second_median": 23.991092699721793, "decode_tokens_per_second_mean": 20.145323361826872, "decode_tokens_per_second_min": 9.122373996319366, "decode_tokens_per_second_max": 24.04884297884067, "decode_tokens_per_second_stdev": 6.456159159249896, "decode_tokens_per_second_p95": 24.04884297884067, "load_seconds_median": 0.177169571, "load_seconds_mean": 0.1864072498, "load_seconds_min": 0.125800947, "load_seconds_max": 0.264506391, "load_seconds_stdev": 0.05264107515315704, "load_seconds_p95": 0.264506391, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 58926747648.0, "gpu_energy_j_observed_median": 422.2897865319252 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 70.29821310000261, "wall_seconds_mean": 70.29600400000345, "wall_seconds_min": 70.27188860002207, "wall_seconds_max": 70.32819660002133, "wall_seconds_stdev": 0.024352270809909732, "wall_seconds_p95": 70.32819660002133, "ttft_seconds_median": 18.746604099986143, "ttft_seconds_mean": 18.74271836001426, "ttft_seconds_min": 18.71749250002904, "ttft_seconds_max": 18.76964680000674, "ttft_seconds_stdev": 0.021014125576700522, "ttft_seconds_p95": 18.76964680000674, "prompt_tokens_median": 32599.0, "prompt_tokens_mean": 32598.6, "prompt_tokens_min": 32597.0, "prompt_tokens_max": 32600.0, "prompt_tokens_stdev": 1.140175425099138, "prompt_tokens_p95": 32600.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1776.961495496446, "prefill_tokens_per_second_mean": 1776.5871438486968, "prefill_tokens_per_second_min": 1774.7959665547698, "prefill_tokens_per_second_max": 1777.373564128234, "prefill_tokens_per_second_stdev": 1.0242298158285021, "prefill_tokens_per_second_p95": 1777.373564128234, "decode_tokens_per_second_median": 19.563171658195802, "decode_tokens_per_second_mean": 19.561175904227817, "decode_tokens_per_second_min": 19.551370066724914, "decode_tokens_per_second_max": 19.568187353024932, "decode_tokens_per_second_stdev": 0.006263754087129324, "decode_tokens_per_second_p95": 19.568187353024932, "load_seconds_median": 0.159060134, "load_seconds_mean": 0.1620075106, "load_seconds_min": 0.137114803, "load_seconds_max": 0.184287242, "load_seconds_stdev": 0.01771652148970104, "load_seconds_p95": 0.184287242, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 62327312384.0, "gpu_energy_j_observed_median": 4044.7319762408747 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 70.37869680003496, "wall_seconds_mean": 70.34962903999258, "wall_seconds_min": 70.0851224999642, "wall_seconds_max": 70.56000569998287, "wall_seconds_stdev": 0.17962229429041415, "wall_seconds_p95": 70.56000569998287, "ttft_seconds_median": 18.74087579996558, "ttft_seconds_mean": 18.756515879987273, "ttft_seconds_min": 18.691443699994124, "ttft_seconds_max": 18.817869599966798, "ttft_seconds_stdev": 0.05707155342450909, "ttft_seconds_p95": 18.817869599966798, "prompt_tokens_median": 32590.0, "prompt_tokens_mean": 32589.8, "prompt_tokens_min": 32588.0, "prompt_tokens_max": 32591.0, "prompt_tokens_stdev": 1.3038404810405297, "prompt_tokens_p95": 32591.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1776.7527025230386, "prefill_tokens_per_second_mean": 1777.319263633988, "prefill_tokens_per_second_min": 1773.050924230411, "prefill_tokens_per_second_max": 1783.9279349162146, "prefill_tokens_per_second_stdev": 4.520464013839378, "prefill_tokens_per_second_p95": 1783.9279349162146, "decode_tokens_per_second_median": 19.55115304019137, "decode_tokens_per_second_mean": 19.546648337017043, "decode_tokens_per_second_min": 19.461277243363533, "decode_tokens_per_second_max": 19.614623419128506, "decode_tokens_per_second_stdev": 0.05482410881081344, "decode_tokens_per_second_p95": 19.614623419128506, "load_seconds_median": 0.155217221, "load_seconds_mean": 0.163550825, "load_seconds_min": 0.130536607, "load_seconds_max": 0.189441795, "load_seconds_stdev": 0.02487498037220453, "load_seconds_p95": 0.189441795, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 62186553344.0, "gpu_energy_j_observed_median": 3995.6306132781506 } ] }