{ "schema": "aero.public.inference_benchmark.v1", "suite": "warm-cache", "publication_date": "2026-09-24", "scope": "Synthetic inference and mocked tools; not CFD/FEA", "rows": [ { "system_id": "3090", "model": "gemma3:12b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 16.731716500013135, "wall_seconds_mean": 16.541558339993934, "wall_seconds_min": 14.931366299977526, "wall_seconds_max": 18.025823000003584, "wall_seconds_stdev": 1.133760985330175, "wall_seconds_p95": 18.025823000003584, "prompt_tokens_median": 15365.0, "prompt_tokens_mean": 15340.8, "prompt_tokens_min": 15244.0, "prompt_tokens_max": 15365.0, "prompt_tokens_stdev": 54.11284505549491, "prompt_tokens_p95": 15365.0, "output_tokens_median": 710.0, "output_tokens_mean": 691.8, "output_tokens_min": 619.0, "output_tokens_max": 710.0, "output_tokens_stdev": 40.69643719049617, "output_tokens_p95": 710.0, "prefill_tokens_per_second_median": 6504.554458128068, "prefill_tokens_per_second_mean": 6712.76221291863, "prefill_tokens_per_second_min": 5559.91994294252, "prefill_tokens_per_second_max": 8445.944074154104, "prefill_tokens_per_second_stdev": 1080.9062780646875, "prefill_tokens_per_second_p95": 8445.944074154104, "decode_tokens_per_second_median": 62.702923475690916, "decode_tokens_per_second_mean": 62.3548261167683, "decode_tokens_per_second_min": 60.901000277485544, "decode_tokens_per_second_max": 63.45766328148335, "decode_tokens_per_second_stdev": 1.113204245380119, "decode_tokens_per_second_p95": 63.45766328148335, "prefill_seconds_median": 2.362191, "prefill_seconds_mean": 2.3303184, "prefill_seconds_min": 1.8048899999999999, "prefill_seconds_max": 2.763529, "prefill_seconds_stdev": 0.35227479135300055, "prefill_seconds_p95": 2.763529, "decode_seconds_median": 11.229915, "decode_seconds_mean": 11.0991108, "decode_seconds_min": 9.871948, "decode_seconds_max": 11.658265, "decode_seconds_stdev": 0.7148734400421521, "decode_seconds_p95": 11.658265, "peak_gpu_memory_used_mib_median": 11438.0, "peak_system_memory_used_bytes_median": 44859408384.0, "gpu_energy_j_observed_median": 5604.693192667961 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 4096, "n": 5, "wall_seconds_median": 2.08706049999455, "wall_seconds_mean": 2.079625399992801, "wall_seconds_min": 1.9897436000173911, "wall_seconds_max": 2.1694802999845706, "wall_seconds_stdev": 0.06529933772452794, "wall_seconds_p95": 2.1694802999845706, "ttft_seconds_median": 0.8729013999691233, "ttft_seconds_mean": 0.8816336799878627, "ttft_seconds_min": 0.8198262000223622, "ttft_seconds_max": 0.9831509999930859, "ttft_seconds_stdev": 0.06117173179486046, "ttft_seconds_p95": 0.9831509999930859, "prompt_tokens_median": 3721.0, "prompt_tokens_mean": 3721.0, "prompt_tokens_min": 3721.0, "prompt_tokens_max": 3721.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 3721.0, "output_tokens_median": 79.0, "output_tokens_mean": 79.0, "output_tokens_min": 79.0, "output_tokens_max": 79.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 79.0, "prefill_tokens_per_second_median": 15000.826436285794, "prefill_tokens_per_second_mean": 15049.800326644061, "prefill_tokens_per_second_min": 14365.186909573831, "prefill_tokens_per_second_max": 15741.00317696678, "prefill_tokens_per_second_stdev": 632.7972762560886, "prefill_tokens_per_second_p95": 15741.00317696678, "decode_tokens_per_second_median": 65.78706309895374, "decode_tokens_per_second_mean": 65.98812307795565, "decode_tokens_per_second_min": 64.616866965322, "decode_tokens_per_second_max": 67.56144242824377, "decode_tokens_per_second_stdev": 1.1379744366608056, "decode_tokens_per_second_p95": 67.56144242824377, "load_seconds_median": 0.5967649, "load_seconds_mean": 0.59978946, "load_seconds_min": 0.5471606, "load_seconds_max": 0.6937586, "load_seconds_stdev": 0.05762971671492929, "load_seconds_p95": 0.6937586, "peak_gpu_memory_used_mib_median": 10892.0, "peak_system_memory_used_bytes_median": 44653137920.0, "gpu_energy_j_observed_median": 640.928237888813 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 8192, "n": 5, "wall_seconds_median": 1.9324110000161454, "wall_seconds_mean": 1.9366768199950457, "wall_seconds_min": 1.8876191999879666, "wall_seconds_max": 2.0035452999873087, "wall_seconds_stdev": 0.045701851464395586, "wall_seconds_p95": 2.0035452999873087, "ttft_seconds_median": 0.816715499968268, "ttft_seconds_mean": 0.8213770799920894, "ttft_seconds_min": 0.782776299980469, "ttft_seconds_max": 0.886808299983386, "ttft_seconds_stdev": 0.03956483425117247, "ttft_seconds_p95": 0.886808299983386, "prompt_tokens_median": 7791.0, "prompt_tokens_mean": 7791.0, "prompt_tokens_min": 7791.0, "prompt_tokens_max": 7791.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 7791.0, "output_tokens_median": 72.0, "output_tokens_mean": 72.0, "output_tokens_min": 72.0, "output_tokens_max": 72.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 72.0, "prefill_tokens_per_second_median": 35842.44230981561, "prefill_tokens_per_second_mean": 35823.696601157266, "prefill_tokens_per_second_min": 29757.653304814467, "prefill_tokens_per_second_max": 38949.74178485904, "prefill_tokens_per_second_stdev": 3732.501106823881, "prefill_tokens_per_second_p95": 38949.74178485904, "decode_tokens_per_second_median": 64.80519737682963, "decode_tokens_per_second_mean": 64.59600537248113, "decode_tokens_per_second_min": 63.22472437971716, "decode_tokens_per_second_max": 65.26373801685254, "decode_tokens_per_second_stdev": 0.8278051346727496, "decode_tokens_per_second_p95": 65.26373801685254, "load_seconds_median": 0.5433629, "load_seconds_mean": 0.54644388, "load_seconds_min": 0.530647, "load_seconds_max": 0.5691809, "load_seconds_stdev": 0.014162893545705975, "load_seconds_p95": 0.5691809, "peak_gpu_memory_used_mib_median": 11444.0, "peak_system_memory_used_bytes_median": 44177502208.0, "gpu_energy_j_observed_median": 587.7048515725136 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 16384, "n": 5, "wall_seconds_median": 2.0611767999944277, "wall_seconds_mean": 2.078317899990361, "wall_seconds_min": 2.0107550999964587, "wall_seconds_max": 2.168754899990745, "wall_seconds_stdev": 0.05936494447140963, "wall_seconds_p95": 2.168754899990745, "ttft_seconds_median": 0.9047881999867968, "ttft_seconds_mean": 0.9208425399963744, "ttft_seconds_min": 0.8440043000155129, "ttft_seconds_max": 0.9925371999852359, "ttft_seconds_stdev": 0.057653051016289196, "ttft_seconds_p95": 0.9925371999852359, "prompt_tokens_median": 15931.0, "prompt_tokens_mean": 15931.0, "prompt_tokens_min": 15931.0, "prompt_tokens_max": 15931.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15931.0, "output_tokens_median": 70.0, "output_tokens_mean": 70.0, "output_tokens_min": 70.0, "output_tokens_max": 70.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 70.0, "prefill_tokens_per_second_median": 59937.16962320585, "prefill_tokens_per_second_mean": 59773.39968910843, "prefill_tokens_per_second_min": 50204.04947608918, "prefill_tokens_per_second_max": 66922.91535391724, "prefill_tokens_per_second_stdev": 6241.932052387908, "prefill_tokens_per_second_p95": 66922.91535391724, "decode_tokens_per_second_median": 60.402783015082576, "decode_tokens_per_second_mean": 60.51956644102034, "decode_tokens_per_second_min": 59.544214556518945, "decode_tokens_per_second_max": 61.622159328494526, "decode_tokens_per_second_stdev": 0.8194799740384817, "decode_tokens_per_second_p95": 61.622159328494526, "load_seconds_median": 0.5251512, "load_seconds_mean": 0.52849554, "load_seconds_min": 0.4933514, "load_seconds_max": 0.5857867, "load_seconds_stdev": 0.037301017348498694, "load_seconds_p95": 0.5857867, "peak_gpu_memory_used_mib_median": 11953.0, "peak_system_memory_used_bytes_median": 44377329664.0, "gpu_energy_j_observed_median": 627.6881807780265 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 32768, "n": 5, "wall_seconds_median": 2.573112800018862, "wall_seconds_mean": 2.6040672800154425, "wall_seconds_min": 2.427987800037954, "wall_seconds_max": 2.835893999959808, "wall_seconds_stdev": 0.15835161302716536, "wall_seconds_p95": 2.835893999959808, "ttft_seconds_median": 1.2491398000274785, "ttft_seconds_mean": 1.2972310599987396, "ttft_seconds_min": 1.1641691999975592, "ttft_seconds_max": 1.4383806999539956, "ttft_seconds_stdev": 0.11387234215105652, "ttft_seconds_p95": 1.4383806999539956, "prompt_tokens_median": 32211.0, "prompt_tokens_mean": 32211.0, "prompt_tokens_min": 32211.0, "prompt_tokens_max": 32211.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32211.0, "output_tokens_median": 70.0, "output_tokens_mean": 70.0, "output_tokens_min": 70.0, "output_tokens_max": 70.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 70.0, "prefill_tokens_per_second_median": 99504.1935035448, "prefill_tokens_per_second_mean": 97388.37521606573, "prefill_tokens_per_second_min": 84291.09750353274, "prefill_tokens_per_second_max": 105239.59970333875, "prefill_tokens_per_second_stdev": 8405.78198888589, "prefill_tokens_per_second_p95": 105239.59970333875, "decode_tokens_per_second_median": 54.596690348631064, "decode_tokens_per_second_mean": 53.729550437947594, "decode_tokens_per_second_min": 50.1607652526347, "decode_tokens_per_second_max": 55.72786966684287, "decode_tokens_per_second_stdev": 2.319702526656472, "decode_tokens_per_second_p95": 55.72786966684287, "load_seconds_median": 0.6682103, "load_seconds_mean": 0.64524588, "load_seconds_min": 0.5253621, "load_seconds_max": 0.7144731, "load_seconds_stdev": 0.07914472530283996, "load_seconds_p95": 0.7144731, "peak_gpu_memory_used_mib_median": 12715.0, "peak_system_memory_used_bytes_median": 44686053376.0, "gpu_energy_j_observed_median": 740.1698657512665 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 18.084672899974976, "wall_seconds_mean": 18.023657119995914, "wall_seconds_min": 17.54265789996134, "wall_seconds_max": 18.346895300026517, "wall_seconds_stdev": 0.29388760045992646, "wall_seconds_p95": 18.346895300026517, "ttft_seconds_median": 1.1036828000214882, "ttft_seconds_mean": 1.2063489999971353, "ttft_seconds_min": 0.973166499985382, "ttft_seconds_max": 1.4751721000066027, "ttft_seconds_stdev": 0.21930417344819314, "ttft_seconds_p95": 1.4751721000066027, "prompt_tokens_median": 16309.0, "prompt_tokens_mean": 16309.0, "prompt_tokens_min": 16309.0, "prompt_tokens_max": 16309.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16309.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 49783.88014505672, "prefill_tokens_per_second_mean": 42732.361025761296, "prefill_tokens_per_second_min": 20051.73689917944, "prefill_tokens_per_second_max": 61391.42349504623, "prefill_tokens_per_second_stdev": 19503.096371205036, "prefill_tokens_per_second_p95": 61391.42349504623, "decode_tokens_per_second_median": 59.27286655318679, "decode_tokens_per_second_mean": 59.47242927419142, "decode_tokens_per_second_min": 58.80060409388622, "decode_tokens_per_second_max": 60.354029498173986, "decode_tokens_per_second_stdev": 0.7118944300388519, "decode_tokens_per_second_p95": 60.354029498173986, "load_seconds_median": 0.581086, "load_seconds_mean": 0.59934058, "load_seconds_min": 0.5394787, "load_seconds_max": 0.68993, "load_seconds_stdev": 0.05997523419255653, "load_seconds_p95": 0.68993, "peak_gpu_memory_used_mib_median": 12250.0, "peak_system_memory_used_bytes_median": 44899364864.0, "gpu_energy_j_observed_median": 6881.941198734045 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 2.6517274000216275, "wall_seconds_mean": 2.6119459000066856, "wall_seconds_min": 2.5110518999863416, "wall_seconds_max": 2.698217900004238, "wall_seconds_stdev": 0.0854880704214838, "wall_seconds_p95": 2.698217900004238, "ttft_seconds_median": 0.8987082000239752, "ttft_seconds_mean": 0.8826888200128451, "ttft_seconds_min": 0.8198618000023998, "ttft_seconds_max": 0.9428185999859124, "ttft_seconds_stdev": 0.05443670307561511, "ttft_seconds_p95": 0.9428185999859124, "prompt_tokens_median": 4113.0, "prompt_tokens_mean": 4113.0, "prompt_tokens_min": 4113.0, "prompt_tokens_max": 4113.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 4113.0, "output_tokens_median": 111.0, "output_tokens_mean": 111.0, "output_tokens_min": 111.0, "output_tokens_max": 111.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 111.0, "prefill_tokens_per_second_median": 14953.264232504536, "prefill_tokens_per_second_mean": 15993.913788813796, "prefill_tokens_per_second_min": 14231.39071793612, "prefill_tokens_per_second_max": 19041.578511210595, "prefill_tokens_per_second_stdev": 2161.728653392486, "prefill_tokens_per_second_p95": 19041.578511210595, "decode_tokens_per_second_median": 64.32358821313709, "decode_tokens_per_second_mean": 64.22509330479484, "decode_tokens_per_second_min": 62.4377029225345, "decode_tokens_per_second_max": 65.64155453393906, "decode_tokens_per_second_stdev": 1.3580687339559216, "decode_tokens_per_second_p95": 65.64155453393906, "load_seconds_median": 0.5814766, "load_seconds_mean": 0.58499364, "load_seconds_min": 0.5526742, "load_seconds_max": 0.6201766, "load_seconds_stdev": 0.024534272619908655, "load_seconds_p95": 0.6201766, "peak_gpu_memory_used_mib_median": 20573.0, "peak_system_memory_used_bytes_median": 45347430400.0, "gpu_energy_j_observed_median": 826.763846719265 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 19.43443679995835, "wall_seconds_mean": 19.386967939976604, "wall_seconds_min": 19.077601699973457, "wall_seconds_max": 19.579395399952773, "wall_seconds_stdev": 0.19023796308109514, "wall_seconds_p95": 19.579395399952773, "ttft_seconds_median": 1.4358543999842368, "ttft_seconds_mean": 1.4563474999857136, "ttft_seconds_min": 1.3730071000172757, "ttft_seconds_max": 1.562482999986969, "ttft_seconds_stdev": 0.07243905927627176, "ttft_seconds_p95": 1.562482999986969, "prompt_tokens_median": 32566.0, "prompt_tokens_mean": 32566.0, "prompt_tokens_min": 32566.0, "prompt_tokens_max": 32566.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32566.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 50059.79629293728, "prefill_tokens_per_second_mean": 50934.67764884267, "prefill_tokens_per_second_min": 43432.2203432327, "prefill_tokens_per_second_max": 56764.46390473136, "prefill_tokens_per_second_stdev": 5320.854956264583, "prefill_tokens_per_second_p95": 56764.46390473136, "decode_tokens_per_second_median": 55.58249762732213, "decode_tokens_per_second_mean": 55.779726927518006, "decode_tokens_per_second_min": 55.12004290985101, "decode_tokens_per_second_max": 56.48708086797823, "decode_tokens_per_second_stdev": 0.5499873672590877, "decode_tokens_per_second_p95": 56.48708086797823, "load_seconds_median": 0.5613327, "load_seconds_mean": 0.55635998, "load_seconds_min": 0.5235459, "load_seconds_max": 0.5878585, "load_seconds_stdev": 0.0289061802195828, "load_seconds_p95": 0.5878585, "peak_gpu_memory_used_mib_median": 12968.0, "peak_system_memory_used_bytes_median": 44653490176.0, "gpu_energy_j_observed_median": 7413.12799840331 }, { "system_id": "3090", "model": "gemma3:12b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 19.604493199964054, "wall_seconds_mean": 19.624288439983502, "wall_seconds_min": 19.470551599981263, "wall_seconds_max": 19.89395749999676, "wall_seconds_stdev": 0.16281148036687693, "wall_seconds_p95": 19.89395749999676, "ttft_seconds_median": 1.6604224999900907, "ttft_seconds_mean": 1.620401579979807, "ttft_seconds_min": 1.4692287999787368, "ttft_seconds_max": 1.752907999965828, "ttft_seconds_stdev": 0.11637875785130933, "ttft_seconds_p95": 1.752907999965828, "prompt_tokens_median": 32557.0, "prompt_tokens_mean": 32557.0, "prompt_tokens_min": 32557.0, "prompt_tokens_max": 32557.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32557.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 49735.86842611802, "prefill_tokens_per_second_mean": 48676.28792909166, "prefill_tokens_per_second_min": 42918.06455440063, "prefill_tokens_per_second_max": 53171.21369252252, "prefill_tokens_per_second_stdev": 3979.866513997183, "prefill_tokens_per_second_p95": 53171.21369252252, "decode_tokens_per_second_median": 55.75594241258639, "decode_tokens_per_second_mean": 55.55543163763573, "decode_tokens_per_second_min": 54.84922551248099, "decode_tokens_per_second_max": 56.06096697431617, "decode_tokens_per_second_stdev": 0.5313373032432939, "decode_tokens_per_second_p95": 56.06096697431617, "load_seconds_median": 0.674093, "load_seconds_mean": 0.65776336, "load_seconds_min": 0.5730311, "load_seconds_max": 0.7261291, "load_seconds_stdev": 0.05707941188088748, "load_seconds_p95": 0.7261291, "peak_gpu_memory_used_mib_median": 13015.0, "peak_system_memory_used_bytes_median": 45008302080.0, "gpu_energy_j_observed_median": 7458.223459795712 }, { "system_id": "3090", "model": "gemma3:4b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 9.864410399983171, "wall_seconds_mean": 10.02300574000692, "wall_seconds_min": 9.641889900027309, "wall_seconds_max": 10.57474509999156, "wall_seconds_stdev": 0.43148906826527467, "wall_seconds_p95": 10.57474509999156, "prompt_tokens_median": 15556.0, "prompt_tokens_mean": 15556.0, "prompt_tokens_min": 15556.0, "prompt_tokens_max": 15556.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15556.0, "output_tokens_median": 810.0, "output_tokens_mean": 810.0, "output_tokens_min": 810.0, "output_tokens_max": 810.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 810.0, "prefill_tokens_per_second_median": 18775.738790801868, "prefill_tokens_per_second_mean": 17729.37540532664, "prefill_tokens_per_second_min": 14342.151968173624, "prefill_tokens_per_second_max": 20240.686694832224, "prefill_tokens_per_second_stdev": 2525.041888644628, "prefill_tokens_per_second_p95": 20240.686694832224, "decode_tokens_per_second_median": 127.8439356992353, "decode_tokens_per_second_mean": 125.9479387937026, "decode_tokens_per_second_min": 121.61959462837781, "decode_tokens_per_second_max": 130.15778337117015, "decode_tokens_per_second_stdev": 3.9982642514910514, "decode_tokens_per_second_p95": 130.15778337117015, "prefill_seconds_median": 0.828516, "prefill_seconds_mean": 0.8927876, "prefill_seconds_min": 0.768551, "prefill_seconds_max": 1.084635, "prefill_seconds_stdev": 0.13532111072667116, "prefill_seconds_p95": 1.084635, "decode_seconds_median": 6.33585, "decode_seconds_mean": 6.436458, "decode_seconds_min": 6.223216, "decode_seconds_max": 6.660111000000001, "decode_seconds_stdev": 0.2059229373224364, "decode_seconds_p95": 6.660111000000001, "peak_gpu_memory_used_mib_median": 6007.0, "peak_system_memory_used_bytes_median": 44028944384.0, "gpu_energy_j_observed_median": 3047.999830610752 }, { "system_id": "3090", "model": "gemma3:4b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 3.3977833000244573, "wall_seconds_mean": 3.4074757199850865, "wall_seconds_min": 3.381815699976869, "wall_seconds_max": 3.452301599958446, "wall_seconds_stdev": 0.02904155930044744, "wall_seconds_p95": 3.452301599958446, "ttft_seconds_median": 0.7387341999565251, "ttft_seconds_mean": 0.7356045999913476, "ttft_seconds_min": 0.7044905999791808, "ttft_seconds_max": 0.7603728999965824, "ttft_seconds_stdev": 0.020084271998688008, "ttft_seconds_p95": 0.7603728999965824, "prompt_tokens_median": 16309.0, "prompt_tokens_mean": 16309.0, "prompt_tokens_min": 16309.0, "prompt_tokens_max": 16309.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16309.0, "output_tokens_median": 337.0, "output_tokens_mean": 337.0, "output_tokens_min": 337.0, "output_tokens_max": 337.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 337.0, "prefill_tokens_per_second_median": 221631.83214198353, "prefill_tokens_per_second_mean": 218418.4364909581, "prefill_tokens_per_second_min": 181442.9548867998, "prefill_tokens_per_second_max": 243381.58483808386, "prefill_tokens_per_second_stdev": 23019.726761810765, "prefill_tokens_per_second_p95": 243381.58483808386, "decode_tokens_per_second_median": 125.52388529575215, "decode_tokens_per_second_mean": 126.20502378773561, "decode_tokens_per_second_min": 124.2369290400688, "decode_tokens_per_second_max": 128.45259222667423, "decode_tokens_per_second_stdev": 1.762198539719259, "decode_tokens_per_second_p95": 128.45259222667423, "load_seconds_median": 0.5275578, "load_seconds_mean": 0.54066082, "load_seconds_min": 0.5209501, "load_seconds_max": 0.5849438, "load_seconds_stdev": 0.026027895344841092, "load_seconds_p95": 0.5849438, "peak_gpu_memory_used_mib_median": 6268.0, "peak_system_memory_used_bytes_median": 43074719744.0, "gpu_energy_j_observed_median": 1095.3904283249378 }, { "system_id": "3090", "model": "gemma3:4b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 3.623957200034056, "wall_seconds_mean": 3.6844438199885188, "wall_seconds_min": 3.6022194999968633, "wall_seconds_max": 3.8032898999517784, "wall_seconds_stdev": 0.09837096807481435, "wall_seconds_p95": 3.8032898999517784, "ttft_seconds_median": 0.6496930999564938, "ttft_seconds_mean": 0.6795930999913253, "ttft_seconds_min": 0.623220500012394, "ttft_seconds_max": 0.7652141999569722, "ttft_seconds_stdev": 0.06578602342049512, "ttft_seconds_p95": 0.7652141999569722, "prompt_tokens_median": 4113.0, "prompt_tokens_mean": 4113.0, "prompt_tokens_min": 4113.0, "prompt_tokens_max": 4113.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 4113.0, "output_tokens_median": 387.0, "output_tokens_mean": 387.0, "output_tokens_min": 387.0, "output_tokens_max": 387.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 387.0, "prefill_tokens_per_second_median": 48400.76255030714, "prefill_tokens_per_second_mean": 46269.20282530687, "prefill_tokens_per_second_min": 38519.17061566989, "prefill_tokens_per_second_max": 54267.65710968321, "prefill_tokens_per_second_stdev": 6759.98597234267, "prefill_tokens_per_second_p95": 54267.65710968321, "decode_tokens_per_second_median": 128.9753657051503, "decode_tokens_per_second_mean": 128.8262773506386, "decode_tokens_per_second_min": 127.09639259714417, "decode_tokens_per_second_max": 130.60897192514588, "decode_tokens_per_second_stdev": 1.561483048154041, "decode_tokens_per_second_p95": 130.60897192514588, "load_seconds_median": 0.531614, "load_seconds_mean": 0.5531714400000001, "load_seconds_min": 0.501121, "load_seconds_max": 0.6309968, "load_seconds_stdev": 0.056463033251748, "load_seconds_p95": 0.6309968, "peak_gpu_memory_used_mib_median": 6011.0, "peak_system_memory_used_bytes_median": 43544125440.0, "gpu_energy_j_observed_median": 1206.9939522027971 }, { "system_id": "3090", "model": "gemma3:4b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 9.735210300015751, "wall_seconds_mean": 9.723726780014113, "wall_seconds_min": 9.436986299988348, "wall_seconds_max": 10.059978800010867, "wall_seconds_stdev": 0.23082308746376654, "wall_seconds_p95": 10.059978800010867, "ttft_seconds_median": 0.8585128999548033, "ttft_seconds_mean": 0.871244759997353, "ttft_seconds_min": 0.8408531000022776, "ttft_seconds_max": 0.925029500038363, "ttft_seconds_stdev": 0.03461446546372492, "ttft_seconds_p95": 0.925029500038363, "prompt_tokens_median": 32566.0, "prompt_tokens_mean": 32566.0, "prompt_tokens_min": 32566.0, "prompt_tokens_max": 32566.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32566.0, "output_tokens_median": 996.0, "output_tokens_mean": 996.0, "output_tokens_min": 996.0, "output_tokens_max": 996.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 996.0, "prefill_tokens_per_second_median": 376672.79688168684, "prefill_tokens_per_second_mean": 370374.7071991663, "prefill_tokens_per_second_min": 312800.76072653226, "prefill_tokens_per_second_max": 409243.993163768, "prefill_tokens_per_second_stdev": 41211.84454975377, "prefill_tokens_per_second_p95": 409243.993163768, "decode_tokens_per_second_median": 112.07650188982812, "decode_tokens_per_second_mean": 112.5814248192731, "decode_tokens_per_second_min": 109.04375632530423, "decode_tokens_per_second_max": 116.12749773224917, "decode_tokens_per_second_stdev": 2.593294456756201, "decode_tokens_per_second_p95": 116.12749773224917, "load_seconds_median": 0.5333629, "load_seconds_mean": 0.53646642, "load_seconds_min": 0.5116779, "load_seconds_max": 0.5761791, "load_seconds_stdev": 0.023927566598737137, "load_seconds_p95": 0.5761791, "peak_gpu_memory_used_mib_median": 6985.0, "peak_system_memory_used_bytes_median": 43859914752.0, "gpu_energy_j_observed_median": 3399.8806562244895 }, { "system_id": "3090", "model": "gemma3:4b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 9.778850500006229, "wall_seconds_mean": 9.961344019987155, "wall_seconds_min": 9.717304499994498, "wall_seconds_max": 10.518512499984354, "wall_seconds_stdev": 0.3389817625227423, "wall_seconds_p95": 10.518512499984354, "ttft_seconds_median": 0.8985363999963738, "ttft_seconds_mean": 0.9561262600007467, "ttft_seconds_min": 0.8683201000094414, "ttft_seconds_max": 1.1779649999807589, "ttft_seconds_stdev": 0.12802219473997173, "ttft_seconds_p95": 1.1779649999807589, "prompt_tokens_median": 32557.0, "prompt_tokens_mean": 32557.0, "prompt_tokens_min": 32557.0, "prompt_tokens_max": 32557.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32557.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 374373.303895865, "prefill_tokens_per_second_mean": 345042.79485971684, "prefill_tokens_per_second_min": 191534.29815272384, "prefill_tokens_per_second_max": 427790.5525261152, "prefill_tokens_per_second_stdev": 95750.10768820797, "prefill_tokens_per_second_p95": 427790.5525261152, "decode_tokens_per_second_median": 112.48523068921051, "decode_tokens_per_second_mean": 111.16575586754406, "decode_tokens_per_second_min": 107.07348881831555, "decode_tokens_per_second_max": 113.21994582199152, "decode_tokens_per_second_stdev": 2.645961047618812, "decode_tokens_per_second_p95": 113.21994582199152, "load_seconds_median": 0.5343985, "load_seconds_mean": 0.54927388, "load_seconds_min": 0.5301991, "load_seconds_max": 0.5784814, "load_seconds_stdev": 0.02282202296208644, "load_seconds_p95": 0.5784814, "peak_gpu_memory_used_mib_median": 6991.0, "peak_system_memory_used_bytes_median": 43840729088.0, "gpu_energy_j_observed_median": 3565.318722809553 }, { "system_id": "3090", "model": "qwen2.5-coder:14b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 12.782995299960021, "wall_seconds_mean": 12.688661879987922, "wall_seconds_min": 12.481156199995894, "wall_seconds_max": 12.799847699992824, "wall_seconds_stdev": 0.14765473109079266, "wall_seconds_p95": 12.799847699992824, "prompt_tokens_median": 15128.0, "prompt_tokens_mean": 15128.0, "prompt_tokens_min": 15128.0, "prompt_tokens_max": 15128.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15128.0, "output_tokens_median": 712.0, "output_tokens_mean": 712.0, "output_tokens_min": 712.0, "output_tokens_max": 712.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 712.0, "prefill_tokens_per_second_median": 84863.85208288922, "prefill_tokens_per_second_mean": 84385.94548732758, "prefill_tokens_per_second_min": 80401.36907672357, "prefill_tokens_per_second_max": 86218.58988607154, "prefill_tokens_per_second_stdev": 2302.4095954238023, "prefill_tokens_per_second_p95": 86218.58988607154, "decode_tokens_per_second_median": 63.661230694977675, "decode_tokens_per_second_mean": 63.69070894456256, "decode_tokens_per_second_min": 63.302843649088864, "decode_tokens_per_second_max": 64.18996768503929, "decode_tokens_per_second_stdev": 0.36780303590523905, "decode_tokens_per_second_p95": 64.18996768503929, "prefill_seconds_median": 0.178262, "prefill_seconds_mean": 0.1793818, "prefill_seconds_min": 0.175461, "prefill_seconds_max": 0.188156, "prefill_seconds_stdev": 0.0050511242016010585, "prefill_seconds_p95": 0.188156, "decode_seconds_median": 11.184201, "decode_seconds_mean": 11.1793224, "decode_seconds_min": 11.092076, "decode_seconds_max": 11.24752, "decode_seconds_stdev": 0.06446832330920323, "decode_seconds_p95": 11.24752, "peak_gpu_memory_used_mib_median": 12089.0, "peak_system_memory_used_bytes_median": 42171920384.0, "gpu_energy_j_observed_median": 4932.851884120702 }, { "system_id": "3090", "model": "qwen2.5-coder:14b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 10.204446599993389, "wall_seconds_mean": 10.189163119986187, "wall_seconds_min": 10.077290999994148, "wall_seconds_max": 10.234034899971448, "wall_seconds_stdev": 0.0644263205397144, "wall_seconds_p95": 10.234034899971448, "ttft_seconds_median": 0.383273699961137, "ttft_seconds_mean": 0.3798545399797149, "ttft_seconds_min": 0.3394653999712318, "ttft_seconds_max": 0.40963329997612163, "ttft_seconds_stdev": 0.025581612651698053, "ttft_seconds_p95": 0.40963329997612163, "prompt_tokens_median": 16025.0, "prompt_tokens_mean": 16025.0, "prompt_tokens_min": 16025.0, "prompt_tokens_max": 16025.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16025.0, "output_tokens_median": 524.0, "output_tokens_mean": 524.0, "output_tokens_min": 524.0, "output_tokens_max": 524.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 524.0, "prefill_tokens_per_second_median": 643806.998513519, "prefill_tokens_per_second_mean": 647207.3896666799, "prefill_tokens_per_second_min": 630905.5118110237, "prefill_tokens_per_second_max": 675989.2010461487, "prefill_tokens_per_second_stdev": 17384.62942964946, "prefill_tokens_per_second_p95": 675989.2010461487, "decode_tokens_per_second_median": 53.361064793571096, "decode_tokens_per_second_mean": 53.42647070335523, "decode_tokens_per_second_min": 53.23387673878705, "decode_tokens_per_second_max": 53.81815134239313, "decode_tokens_per_second_stdev": 0.22567986273808385, "decode_tokens_per_second_p95": 53.81815134239313, "load_seconds_median": 0.250517, "load_seconds_mean": 0.2450655, "load_seconds_min": 0.2158411, "load_seconds_max": 0.2617597, "load_seconds_stdev": 0.01772738210467072, "load_seconds_p95": 0.2617597, "peak_gpu_memory_used_mib_median": 14417.0, "peak_system_memory_used_bytes_median": 42629914624.0, "gpu_energy_j_observed_median": 4003.624134478569 }, { "system_id": "3090", "model": "qwen2.5-coder:14b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 42.79090299998643, "wall_seconds_mean": 42.96890778000234, "wall_seconds_min": 42.39404189999914, "wall_seconds_max": 43.93687079998199, "wall_seconds_stdev": 0.5832464298389797, "wall_seconds_p95": 43.93687079998199, "ttft_seconds_median": 22.27993570000399, "ttft_seconds_mean": 22.57221766001312, "ttft_seconds_min": 22.023156100010965, "ttft_seconds_max": 23.724085199995898, "ttft_seconds_stdev": 0.6841084984488183, "ttft_seconds_p95": 23.724085199995898, "prompt_tokens_median": 31994.0, "prompt_tokens_mean": 31994.0, "prompt_tokens_min": 31994.0, "prompt_tokens_max": 31994.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31994.0, "output_tokens_median": 898.0, "output_tokens_mean": 898.0, "output_tokens_min": 898.0, "output_tokens_max": 898.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 898.0, "prefill_tokens_per_second_median": 1472.5310412525932, "prefill_tokens_per_second_mean": 1471.7053147082481, "prefill_tokens_per_second_min": 1444.8085091336475, "prefill_tokens_per_second_max": 1500.8759488077176, "prefill_tokens_per_second_stdev": 21.11394184977594, "prefill_tokens_per_second_p95": 1500.8759488077176, "decode_tokens_per_second_median": 44.08864547505172, "decode_tokens_per_second_mean": 44.03414254278862, "decode_tokens_per_second_min": 43.72229835755579, "decode_tokens_per_second_max": 44.43228356703165, "decode_tokens_per_second_stdev": 0.28680018152017434, "decode_tokens_per_second_p95": 44.43228356703165, "load_seconds_median": 0.2379115, "load_seconds_mean": 0.23647602, "load_seconds_min": 0.1941602, "load_seconds_max": 0.2885598, "load_seconds_stdev": 0.036176956877714286, "load_seconds_p95": 0.2885598, "peak_gpu_memory_used_mib_median": 16744.0, "peak_system_memory_used_bytes_median": 48660422656.0, "gpu_energy_j_observed_median": 17488.425433937304 }, { "system_id": "3090", "model": "qwen2.5vl:7b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 9.099808000028133, "wall_seconds_mean": 9.01255778002087, "wall_seconds_min": 8.746828800009098, "wall_seconds_max": 9.166103199997451, "wall_seconds_stdev": 0.1763821852818659, "wall_seconds_p95": 9.166103199997451, "prompt_tokens_median": 15294.0, "prompt_tokens_mean": 15294.0, "prompt_tokens_min": 15294.0, "prompt_tokens_max": 15294.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15294.0, "output_tokens_median": 870.0, "output_tokens_mean": 870.0, "output_tokens_min": 870.0, "output_tokens_max": 870.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 870.0, "prefill_tokens_per_second_median": 163336.35926736798, "prefill_tokens_per_second_mean": 163405.93271781577, "prefill_tokens_per_second_min": 159058.9997191975, "prefill_tokens_per_second_max": 167697.3684210526, "prefill_tokens_per_second_stdev": 3401.3161587501236, "prefill_tokens_per_second_p95": 167697.3684210526, "decode_tokens_per_second_median": 114.86020390986775, "decode_tokens_per_second_mean": 115.21406782333717, "decode_tokens_per_second_min": 113.22589020602557, "decode_tokens_per_second_max": 118.03463380365386, "decode_tokens_per_second_stdev": 1.801814165286888, "decode_tokens_per_second_p95": 118.03463380365386, "prefill_seconds_median": 0.093635, "prefill_seconds_mean": 0.0936276, "prefill_seconds_min": 0.0912, "prefill_seconds_max": 0.096153, "prefill_seconds_stdev": 0.0019499690510364526, "prefill_seconds_p95": 0.096153, "decode_seconds_median": 7.574425, "decode_seconds_mean": 7.5526256, "decode_seconds_min": 7.370718, "decode_seconds_max": 7.683755, "decode_seconds_stdev": 0.11705794991712418, "decode_seconds_p95": 7.683755, "peak_gpu_memory_used_mib_median": 14124.0, "peak_system_memory_used_bytes_median": 44384051200.0, "gpu_energy_j_observed_median": 3342.1711628532407 }, { "system_id": "3090", "model": "qwen2.5vl:7b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 7.261640699987765, "wall_seconds_mean": 7.323952119995374, "wall_seconds_min": 7.16484969999874, "wall_seconds_max": 7.497420199972112, "wall_seconds_stdev": 0.14392147851019096, "wall_seconds_p95": 7.497420199972112, "ttft_seconds_median": 0.36617250001290813, "ttft_seconds_mean": 0.3753004599944688, "ttft_seconds_min": 0.35619039996527135, "ttft_seconds_max": 0.41525799996452406, "ttft_seconds_stdev": 0.02416293745513602, "ttft_seconds_p95": 0.41525799996452406, "prompt_tokens_median": 16015.0, "prompt_tokens_mean": 16015.0, "prompt_tokens_min": 16015.0, "prompt_tokens_max": 16015.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16015.0, "output_tokens_median": 721.0, "output_tokens_mean": 721.0, "output_tokens_min": 721.0, "output_tokens_max": 721.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 721.0, "prefill_tokens_per_second_median": 1041219.6866263571, "prefill_tokens_per_second_mean": 1027788.8045231969, "prefill_tokens_per_second_min": 930509.5578409156, "prefill_tokens_per_second_max": 1076204.5561454203, "prefill_tokens_per_second_stdev": 57118.268290959815, "prefill_tokens_per_second_p95": 1076204.5561454203, "decode_tokens_per_second_median": 104.58198175747917, "decode_tokens_per_second_mean": 103.80685456181861, "decode_tokens_per_second_min": 101.81729035523361, "decode_tokens_per_second_max": 105.9139966595225, "decode_tokens_per_second_stdev": 1.829738785674161, "decode_tokens_per_second_p95": 105.9139966595225, "load_seconds_median": 0.2574908, "load_seconds_mean": 0.26452184, "load_seconds_min": 0.2499076, "load_seconds_max": 0.296476, "load_seconds_stdev": 0.019019395381241748, "load_seconds_p95": 0.296476, "peak_gpu_memory_used_mib_median": 14818.0, "peak_system_memory_used_bytes_median": 43690700800.0, "gpu_energy_j_observed_median": 2917.9206222140792 }, { "system_id": "3090", "model": "qwen2.5vl:7b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 2.161503800016362, "wall_seconds_mean": 2.1737187599996104, "wall_seconds_min": 2.1514431999530643, "wall_seconds_max": 2.2361019999952987, "wall_seconds_stdev": 0.03532893746625229, "wall_seconds_p95": 2.2361019999952987, "ttft_seconds_median": 0.2878678999841213, "ttft_seconds_mean": 0.293762920005247, "ttft_seconds_min": 0.27243620000081137, "ttft_seconds_max": 0.3177817999967374, "ttft_seconds_stdev": 0.02009019633530335, "ttft_seconds_p95": 0.3177817999967374, "prompt_tokens_median": 4053.0, "prompt_tokens_mean": 4053.0, "prompt_tokens_min": 4053.0, "prompt_tokens_max": 4053.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 4053.0, "output_tokens_median": 219.0, "output_tokens_mean": 219.0, "output_tokens_min": 219.0, "output_tokens_max": 219.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 219.0, "prefill_tokens_per_second_median": 307488.05098247476, "prefill_tokens_per_second_mean": 310131.93189813127, "prefill_tokens_per_second_min": 287548.7761617595, "prefill_tokens_per_second_max": 338285.61889658624, "prefill_tokens_per_second_stdev": 18314.33285982466, "prefill_tokens_per_second_p95": 338285.61889658624, "decode_tokens_per_second_median": 116.4188292305885, "decode_tokens_per_second_mean": 116.54502761923055, "decode_tokens_per_second_min": 114.25655762208626, "decode_tokens_per_second_max": 118.18187705199355, "decode_tokens_per_second_stdev": 1.496021434087526, "decode_tokens_per_second_p95": 118.18187705199355, "load_seconds_median": 0.2447413, "load_seconds_mean": 0.24761088, "load_seconds_min": 0.2256763, "load_seconds_max": 0.2713958, "load_seconds_stdev": 0.018477330535875584, "load_seconds_p95": 0.2713958, "peak_gpu_memory_used_mib_median": 14124.0, "peak_system_memory_used_bytes_median": 44026568704.0, "gpu_energy_j_observed_median": 812.7290371143818 }, { "system_id": "3090", "model": "qwen2.5vl:7b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 11.248735799978022, "wall_seconds_mean": 11.303928299993277, "wall_seconds_min": 11.055825699993875, "wall_seconds_max": 11.643633100029547, "wall_seconds_stdev": 0.22481269015599115, "wall_seconds_p95": 11.643633100029547, "ttft_seconds_median": 0.4853267999715172, "ttft_seconds_mean": 0.4791473199846223, "ttft_seconds_min": 0.4420090999919921, "ttft_seconds_max": 0.5067179999896325, "ttft_seconds_stdev": 0.029040261912085263, "ttft_seconds_p95": 0.5067179999896325, "prompt_tokens_median": 31984.0, "prompt_tokens_mean": 31984.0, "prompt_tokens_min": 31984.0, "prompt_tokens_max": 31984.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31984.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1609986.9123124937, "prefill_tokens_per_second_mean": 1648747.1537854697, "prefill_tokens_per_second_min": 1527849.4315467658, "prefill_tokens_per_second_max": 1814900.981671679, "prefill_tokens_per_second_stdev": 108254.1985816149, "prefill_tokens_per_second_p95": 1814900.981671679, "decode_tokens_per_second_median": 93.11090132190478, "decode_tokens_per_second_mean": 92.42517303476095, "decode_tokens_per_second_min": 89.79733459959547, "decode_tokens_per_second_max": 94.35821833189578, "decode_tokens_per_second_stdev": 1.7392557478365744, "decode_tokens_per_second_p95": 94.35821833189578, "load_seconds_median": 0.2737853, "load_seconds_mean": 0.26563148000000003, "load_seconds_min": 0.2277083, "load_seconds_max": 0.295556, "load_seconds_stdev": 0.02933514521383864, "load_seconds_p95": 0.295556, "peak_gpu_memory_used_mib_median": 16107.0, "peak_system_memory_used_bytes_median": 44565454848.0, "gpu_energy_j_observed_median": 4480.7980636978145 }, { "system_id": "3090", "model": "qwen2.5vl:7b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 11.436872599995695, "wall_seconds_mean": 11.462223920004908, "wall_seconds_min": 11.299816099985037, "wall_seconds_max": 11.67697520001093, "wall_seconds_stdev": 0.15832371912257887, "wall_seconds_p95": 11.67697520001093, "ttft_seconds_median": 0.4886201999615878, "ttft_seconds_mean": 0.49173821998992934, "ttft_seconds_min": 0.4795643999823369, "ttft_seconds_max": 0.503941900038626, "ttft_seconds_stdev": 0.01024902971603981, "ttft_seconds_p95": 0.503941900038626, "prompt_tokens_median": 31979.0, "prompt_tokens_mean": 31979.0, "prompt_tokens_min": 31979.0, "prompt_tokens_max": 31979.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31979.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1687902.4596220837, "prefill_tokens_per_second_mean": 1676996.733068923, "prefill_tokens_per_second_min": 1605210.320248971, "prefill_tokens_per_second_max": 1762608.1684396185, "prefill_tokens_per_second_stdev": 62045.771082716434, "prefill_tokens_per_second_p95": 1762608.1684396185, "decode_tokens_per_second_median": 91.35178164293255, "decode_tokens_per_second_mean": 91.18847128926902, "decode_tokens_per_second_min": 89.52901738411789, "decode_tokens_per_second_max": 92.49152846967863, "decode_tokens_per_second_stdev": 1.2952044966253584, "decode_tokens_per_second_p95": 92.49152846967863, "load_seconds_median": 0.2755871, "load_seconds_mean": 0.27583112, "load_seconds_min": 0.2650886, "load_seconds_max": 0.2888119, "load_seconds_stdev": 0.00877140570558676, "load_seconds_p95": 0.2888119, "peak_gpu_memory_used_mib_median": 16113.0, "peak_system_memory_used_bytes_median": 43836182528.0, "gpu_energy_j_observed_median": 4517.877874860764 }, { "system_id": "spark", "model": "aero-specialist-verification-v5-e1:v4", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 38.77896889997646, "wall_seconds_mean": 39.44669732002076, "wall_seconds_min": 38.53873270004988, "wall_seconds_max": 42.33554040000308, "wall_seconds_stdev": 1.6191531271147173, "wall_seconds_p95": 42.33554040000308, "prompt_tokens_median": 15314.0, "prompt_tokens_mean": 15314.0, "prompt_tokens_min": 15314.0, "prompt_tokens_max": 15314.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15314.0, "output_tokens_median": 870.0, "output_tokens_mean": 870.0, "output_tokens_min": 870.0, "output_tokens_max": 870.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 870.0, "prefill_tokens_per_second_median": 32250.855909685353, "prefill_tokens_per_second_mean": 26515.256013258353, "prefill_tokens_per_second_min": 3685.131272384357, "prefill_tokens_per_second_max": 32370.0781709909, "prefill_tokens_per_second_stdev": 12763.100228851557, "prefill_tokens_per_second_p95": 32370.0781709909, "decode_tokens_per_second_median": 24.62174149117339, "decode_tokens_per_second_mean": 24.619277560851764, "decode_tokens_per_second_min": 24.6068997142657, "decode_tokens_per_second_max": 24.63123045992774, "decode_tokens_per_second_stdev": 0.009665181733478283, "decode_tokens_per_second_p95": 24.63123045992774, "prefill_seconds_median": 0.474840111, "prefill_seconds_mean": 1.2113330094, "prefill_seconds_min": 0.473091227, "prefill_seconds_max": 4.155618584, "prefill_seconds_stdev": 1.645906813388097, "prefill_seconds_p95": 4.155618584, "decode_seconds_median": 35.334624901, "decode_seconds_mean": 35.338165595, "decode_seconds_min": 35.321012542, "decode_seconds_max": 35.355937159999996, "decode_seconds_stdev": 0.013873853413742724, "decode_seconds_p95": 35.355937159999996, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 21638909952.0, "gpu_energy_j_observed_median": 1470.3592932188512 }, { "system_id": "spark", "model": "aero-specialist-verification-v5-e1:v4", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 31.112300399981905, "wall_seconds_mean": 31.110283920005894, "wall_seconds_min": 31.085892000002787, "wall_seconds_max": 31.131041000015102, "wall_seconds_stdev": 0.018423811528793674, "wall_seconds_p95": 31.131041000015102, "ttft_seconds_median": 0.17277790000662208, "ttft_seconds_mean": 0.17517926002619788, "ttft_seconds_min": 0.15903219999745488, "ttft_seconds_max": 0.18924220005283132, "ttft_seconds_stdev": 0.011564231183364224, "ttft_seconds_p95": 0.18924220005283132, "prompt_tokens_median": 16009.0, "prompt_tokens_mean": 16009.0, "prompt_tokens_min": 16009.0, "prompt_tokens_max": 16009.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16009.0, "output_tokens_median": 624.0, "output_tokens_mean": 624.0, "output_tokens_min": 624.0, "output_tokens_max": 624.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 624.0, "prefill_tokens_per_second_median": 305132.0876571736, "prefill_tokens_per_second_mean": 305787.193525049, "prefill_tokens_per_second_min": 303335.42586586595, "prefill_tokens_per_second_max": 310905.32118273363, "prefill_tokens_per_second_stdev": 3115.5360474117556, "prefill_tokens_per_second_p95": 310905.32118273363, "decode_tokens_per_second_median": 20.281017890141896, "decode_tokens_per_second_mean": 20.280179632499962, "decode_tokens_per_second_min": 20.269299843959995, "decode_tokens_per_second_max": 20.28774625774089, "decode_tokens_per_second_stdev": 0.007878421566572359, "decode_tokens_per_second_p95": 20.28774625774089, "load_seconds_median": 0.080504161, "load_seconds_mean": 0.081710283, "load_seconds_min": 0.070270352, "load_seconds_max": 0.090440477, "load_seconds_stdev": 0.008639080947804316, "load_seconds_p95": 0.090440477, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 23360479232.0, "gpu_energy_j_observed_median": 1372.469312888384 }, { "system_id": "spark", "model": "aero-specialist-verification-v5-e1:v4", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 32.687195499951486, "wall_seconds_mean": 32.67796999999555, "wall_seconds_min": 32.59242960001575, "wall_seconds_max": 32.72138790000463, "wall_seconds_stdev": 0.05176755260003535, "wall_seconds_p95": 32.72138790000463, "ttft_seconds_median": 0.20115329994587228, "ttft_seconds_mean": 0.20833454000530766, "ttft_seconds_min": 0.17945730005158111, "ttft_seconds_max": 0.23408999998355284, "ttft_seconds_stdev": 0.02289021808040723, "ttft_seconds_p95": 0.23408999998355284, "prompt_tokens_median": 31978.0, "prompt_tokens_mean": 31978.0, "prompt_tokens_min": 31978.0, "prompt_tokens_max": 31978.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31978.0, "output_tokens_median": 543.0, "output_tokens_mean": 543.0, "output_tokens_min": 543.0, "output_tokens_max": 543.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 543.0, "prefill_tokens_per_second_median": 508523.1132644353, "prefill_tokens_per_second_mean": 507934.46728637867, "prefill_tokens_per_second_min": 500089.71839121927, "prefill_tokens_per_second_max": 513691.811225954, "prefill_tokens_per_second_stdev": 5105.130897157127, "prefill_tokens_per_second_p95": 513691.811225954, "decode_tokens_per_second_median": 16.81821765258613, "decode_tokens_per_second_mean": 16.81194745213141, "decode_tokens_per_second_min": 16.789668882691075, "decode_tokens_per_second_max": 16.841253244943136, "decode_tokens_per_second_stdev": 0.022093048506048454, "decode_tokens_per_second_p95": 16.841253244943136, "load_seconds_median": 0.076744099, "load_seconds_mean": 0.07759026699999999, "load_seconds_min": 0.054377589, "load_seconds_max": 0.093785092, "load_seconds_stdev": 0.01549692377613633, "load_seconds_p95": 0.093785092, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 25833684992.0, "gpu_energy_j_observed_median": 1470.340517948866 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 27.396292500023264, "wall_seconds_mean": 27.198941780021414, "wall_seconds_min": 26.820001400017645, "wall_seconds_max": 27.49878359999275, "wall_seconds_stdev": 0.3426617489945916, "wall_seconds_p95": 27.49878359999275, "prompt_tokens_median": 15146.0, "prompt_tokens_mean": 15146.0, "prompt_tokens_min": 15146.0, "prompt_tokens_max": 15146.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15146.0, "output_tokens_median": 525.0, "output_tokens_mean": 525.0, "output_tokens_min": 525.0, "output_tokens_max": 525.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 525.0, "prefill_tokens_per_second_median": 7435.590149051519, "prefill_tokens_per_second_mean": 7440.744439943265, "prefill_tokens_per_second_min": 7414.449646031648, "prefill_tokens_per_second_max": 7471.648798787843, "prefill_tokens_per_second_stdev": 23.938004039061564, "prefill_tokens_per_second_p95": 7471.648798787843, "decode_tokens_per_second_median": 24.19809605126013, "decode_tokens_per_second_mean": 24.189645758542106, "decode_tokens_per_second_min": 24.157347423060237, "decode_tokens_per_second_max": 24.200025502733617, "decode_tokens_per_second_stdev": 0.018176262923067586, "decode_tokens_per_second_p95": 24.200025502733617, "prefill_seconds_median": 2.0369600390000002, "prefill_seconds_mean": 2.035565856, "prefill_seconds_min": 2.027129541, "prefill_seconds_max": 2.042767936, "prefill_seconds_stdev": 0.006544269695718283, "prefill_seconds_p95": 2.042767936, "decode_seconds_median": 21.695921815, "decode_seconds_mean": 21.703510775199998, "decode_seconds_min": 21.694192014000002, "decode_seconds_max": 21.732518509, "decode_seconds_stdev": 0.01632409220964235, "decode_seconds_p95": 21.732518509, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 21992476672.0, "gpu_energy_j_observed_median": 1192.2831065511702 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 4096, "n": 5, "wall_seconds_median": 5.501201900013257, "wall_seconds_mean": 5.503906579990871, "wall_seconds_min": 5.4736230999697, "wall_seconds_max": 5.530512199969962, "wall_seconds_stdev": 0.02430873145696648, "wall_seconds_p95": 5.530512199969962, "ttft_seconds_median": 2.1274371999897994, "ttft_seconds_mean": 2.1286626799963413, "ttft_seconds_min": 2.11580969998613, "ttft_seconds_max": 2.1390023000421934, "ttft_seconds_stdev": 0.008776446403003155, "ttft_seconds_p95": 2.1390023000421934, "prompt_tokens_median": 3720.0, "prompt_tokens_mean": 3720.0, "prompt_tokens_min": 3720.0, "prompt_tokens_max": 3720.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 3720.0, "output_tokens_median": 81.0, "output_tokens_mean": 81.0, "output_tokens_min": 81.0, "output_tokens_max": 81.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 81.0, "prefill_tokens_per_second_median": 1919.3568386305706, "prefill_tokens_per_second_mean": 1918.7146897118175, "prefill_tokens_per_second_min": 1913.8964996043612, "prefill_tokens_per_second_max": 1922.621713761549, "prefill_tokens_per_second_stdev": 3.4257156084086793, "prefill_tokens_per_second_p95": 1922.621713761549, "decode_tokens_per_second_median": 24.395816388794895, "decode_tokens_per_second_mean": 24.408948198991133, "decode_tokens_per_second_min": 24.298893532873755, "decode_tokens_per_second_max": 24.481310514799876, "decode_tokens_per_second_stdev": 0.07396761275492152, "decode_tokens_per_second_p95": 24.481310514799876, "load_seconds_median": 0.16036853, "load_seconds_mean": 0.1620756664, "load_seconds_min": 0.148883932, "load_seconds_max": 0.17416505, "load_seconds_stdev": 0.010124096980409895, "load_seconds_p95": 0.17416505, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 21726654464.0, "gpu_energy_j_observed_median": 328.7156189405918 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 8192, "n": 5, "wall_seconds_median": 7.613619200012181, "wall_seconds_mean": 7.601256619987543, "wall_seconds_min": 7.553142299992032, "wall_seconds_max": 7.6377936999779195, "wall_seconds_stdev": 0.032133344702326254, "wall_seconds_p95": 7.6377936999779195, "ttft_seconds_median": 4.310302399971988, "ttft_seconds_mean": 4.316628480004146, "ttft_seconds_min": 4.2832163000130095, "ttft_seconds_max": 4.365589399996679, "ttft_seconds_stdev": 0.03052427959859738, "ttft_seconds_p95": 4.365589399996679, "prompt_tokens_median": 7790.0, "prompt_tokens_mean": 7790.0, "prompt_tokens_min": 7790.0, "prompt_tokens_max": 7790.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 7790.0, "output_tokens_median": 76.0, "output_tokens_mean": 76.0, "output_tokens_min": 76.0, "output_tokens_max": 76.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 76.0, "prefill_tokens_per_second_median": 1900.9601304516052, "prefill_tokens_per_second_mean": 1902.2648665604695, "prefill_tokens_per_second_min": 1899.194959512978, "prefill_tokens_per_second_max": 1907.8012566589896, "prefill_tokens_per_second_stdev": 3.437262897557336, "prefill_tokens_per_second_p95": 1907.8012566589896, "decode_tokens_per_second_median": 23.5726838997051, "decode_tokens_per_second_mean": 23.551287508861904, "decode_tokens_per_second_min": 23.4547708631419, "decode_tokens_per_second_max": 23.585791430395766, "decode_tokens_per_second_stdev": 0.05456170512704877, "decode_tokens_per_second_p95": 23.585791430395766, "load_seconds_median": 0.16389136, "load_seconds_mean": 0.1667443274, "load_seconds_min": 0.130910392, "load_seconds_max": 0.199659226, "load_seconds_stdev": 0.02491517996999514, "load_seconds_p95": 0.199659226, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 22175645696.0, "gpu_energy_j_observed_median": 488.0224811184406 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 16384, "n": 5, "wall_seconds_median": 12.098093700013123, "wall_seconds_mean": 12.087600460008252, "wall_seconds_min": 12.040089700021781, "wall_seconds_max": 12.132126800017431, "wall_seconds_stdev": 0.038310662783105354, "wall_seconds_p95": 12.132126800017431, "ttft_seconds_median": 8.742501400003675, "ttft_seconds_mean": 8.749498259997926, "ttft_seconds_min": 8.724438600009307, "ttft_seconds_max": 8.78070200001821, "ttft_seconds_stdev": 0.025351500288718666, "ttft_seconds_p95": 8.78070200001821, "prompt_tokens_median": 15930.0, "prompt_tokens_mean": 15930.0, "prompt_tokens_min": 15930.0, "prompt_tokens_max": 15930.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15930.0, "output_tokens_median": 73.0, "output_tokens_mean": 73.0, "output_tokens_min": 73.0, "output_tokens_max": 73.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 73.0, "prefill_tokens_per_second_median": 1878.4045326015987, "prefill_tokens_per_second_mean": 1878.7598153741599, "prefill_tokens_per_second_min": 1875.5388041518806, "prefill_tokens_per_second_max": 1881.9502471596509, "prefill_tokens_per_second_stdev": 2.62486840760317, "prefill_tokens_per_second_p95": 1881.9502471596509, "decode_tokens_per_second_median": 22.225691087655736, "decode_tokens_per_second_mean": 22.24454006993666, "decode_tokens_per_second_min": 22.197441849550113, "decode_tokens_per_second_max": 22.31964069340984, "decode_tokens_per_second_stdev": 0.05251728816652165, "decode_tokens_per_second_p95": 22.31964069340984, "load_seconds_median": 0.164777008, "load_seconds_mean": 0.1675186698, "load_seconds_min": 0.144740956, "load_seconds_max": 0.197781229, "load_seconds_stdev": 0.01907273347574161, "load_seconds_p95": 0.197781229, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 23536414720.0, "gpu_energy_j_observed_median": 887.2105590569973 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 32768, "n": 5, "wall_seconds_median": 21.952726599993184, "wall_seconds_mean": 21.94743185999105, "wall_seconds_min": 21.90614999999525, "wall_seconds_max": 22.00619809998898, "wall_seconds_stdev": 0.04153023067364845, "wall_seconds_p95": 22.00619809998898, "ttft_seconds_median": 18.403323899954557, "ttft_seconds_mean": 18.390337640000507, "ttft_seconds_min": 18.348710099991877, "ttft_seconds_max": 18.414860300021246, "ttft_seconds_stdev": 0.02796699020744371, "ttft_seconds_p95": 18.414860300021246, "prompt_tokens_median": 32210.0, "prompt_tokens_mean": 32210.0, "prompt_tokens_min": 32210.0, "prompt_tokens_max": 32210.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32210.0, "output_tokens_median": 70.0, "output_tokens_mean": 70.0, "output_tokens_min": 70.0, "output_tokens_max": 70.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 70.0, "prefill_tokens_per_second_median": 1790.5728077211606, "prefill_tokens_per_second_mean": 1790.6030137865796, "prefill_tokens_per_second_min": 1789.7333407145618, "prefill_tokens_per_second_max": 1791.8910316342922, "prefill_tokens_per_second_stdev": 0.854973434427421, "prefill_tokens_per_second_p95": 1791.8910316342922, "decode_tokens_per_second_median": 19.9936265631109, "decode_tokens_per_second_mean": 19.987282437180546, "decode_tokens_per_second_min": 19.954953562397108, "decode_tokens_per_second_max": 20.014025188403227, "decode_tokens_per_second_stdev": 0.024463617141623054, "decode_tokens_per_second_p95": 20.014025188403227, "load_seconds_median": 0.172744958, "load_seconds_mean": 0.168901149, "load_seconds_min": 0.141376223, "load_seconds_max": 0.185953533, "load_seconds_stdev": 0.016475323936135995, "load_seconds_p95": 0.185953533, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 25203544064.0, "gpu_energy_j_observed_median": 1754.6665540921688 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 65536, "n": 5, "wall_seconds_median": 44.87468130001798, "wall_seconds_mean": 44.860289800004104, "wall_seconds_min": 44.77661299996544, "wall_seconds_max": 44.938310500001535, "wall_seconds_stdev": 0.07265927836848143, "wall_seconds_p95": 44.938310500001535, "ttft_seconds_median": 40.26792770001339, "ttft_seconds_mean": 40.24517777999863, "ttft_seconds_min": 40.172325299994554, "ttft_seconds_max": 40.31653109996114, "ttft_seconds_stdev": 0.061624687730487696, "ttft_seconds_p95": 40.31653109996114, "prompt_tokens_median": 64715.0, "prompt_tokens_mean": 64715.0, "prompt_tokens_min": 64715.0, "prompt_tokens_max": 64715.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 64715.0, "output_tokens_median": 77.0, "output_tokens_mean": 77.0, "output_tokens_min": 77.0, "output_tokens_max": 77.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 77.0, "prefill_tokens_per_second_median": 1636.017214902574, "prefill_tokens_per_second_mean": 1636.8224412615998, "prefill_tokens_per_second_min": 1635.4117295540211, "prefill_tokens_per_second_max": 1640.199611550118, "prefill_tokens_per_second_stdev": 1.9513327191150807, "prefill_tokens_per_second_p95": 1640.199611550118, "decode_tokens_per_second_median": 16.90517772461752, "decode_tokens_per_second_mean": 16.906912687577815, "decode_tokens_per_second_min": 16.899079086835954, "decode_tokens_per_second_max": 16.91829891287256, "decode_tokens_per_second_stdev": 0.007156307860222009, "decode_tokens_per_second_p95": 16.91829891287256, "load_seconds_median": 0.187169328, "load_seconds_mean": 0.1818337052, "load_seconds_min": 0.167913369, "load_seconds_max": 0.197259397, "load_seconds_stdev": 0.01280733509583608, "load_seconds_p95": 0.197259397, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 28239949824.0, "gpu_energy_j_observed_median": 3755.4869084715842 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "context_scaling", "context_setting": 131072, "n": 5, "wall_seconds_median": 99.04780160001246, "wall_seconds_mean": 99.00619157999753, "wall_seconds_min": 98.89559109997936, "wall_seconds_max": 99.06824649998453, "wall_seconds_stdev": 0.07262266127817416, "wall_seconds_p95": 99.06824649998453, "ttft_seconds_median": 93.23713520000456, "ttft_seconds_mean": 93.22840816000243, "ttft_seconds_min": 93.12931360001676, "ttft_seconds_max": 93.29855780000798, "ttft_seconds_stdev": 0.06694068743731753, "ttft_seconds_p95": 93.29855780000798, "prompt_tokens_median": 129780.0, "prompt_tokens_mean": 129780.0, "prompt_tokens_min": 129780.0, "prompt_tokens_max": 129780.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 129780.0, "output_tokens_median": 74.0, "output_tokens_mean": 74.0, "output_tokens_min": 74.0, "output_tokens_max": 74.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 74.0, "prefill_tokens_per_second_median": 1413.2171628023918, "prefill_tokens_per_second_mean": 1413.0596370279693, "prefill_tokens_per_second_min": 1412.1403395906993, "prefill_tokens_per_second_max": 1413.6129217507048, "prefill_tokens_per_second_stdev": 0.6201312723185632, "prefill_tokens_per_second_p95": 1413.6129217507048, "decode_tokens_per_second_median": 12.919148516205023, "decode_tokens_per_second_mean": 12.92006739456108, "decode_tokens_per_second_min": 12.914077583532183, "decode_tokens_per_second_max": 12.926322078653415, "decode_tokens_per_second_stdev": 0.004517283258763278, "decode_tokens_per_second_p95": 12.926322078653415, "load_seconds_median": 0.181139881, "load_seconds_mean": 0.1768539292, "load_seconds_min": 0.146755365, "load_seconds_max": 0.199714033, "load_seconds_stdev": 0.019891039647664106, "load_seconds_p95": 0.199714033, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 35129073664.0, "gpu_energy_j_observed_median": 8284.923661469222 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 54.696462899970356, "wall_seconds_mean": 54.702057299984155, "wall_seconds_min": 54.648396599979606, "wall_seconds_max": 54.764334100007545, "wall_seconds_stdev": 0.041403562854132846, "wall_seconds_p95": 54.764334100007545, "ttft_seconds_median": 9.114731399982702, "ttft_seconds_mean": 9.11453153998591, "ttft_seconds_min": 9.078029899974354, "ttft_seconds_max": 9.171149100002367, "ttft_seconds_stdev": 0.03719337592335195, "ttft_seconds_p95": 9.171149100002367, "prompt_tokens_median": 16308.0, "prompt_tokens_mean": 16308.0, "prompt_tokens_min": 16308.0, "prompt_tokens_max": 16308.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16308.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1852.2480435337006, "prefill_tokens_per_second_mean": 1851.2777119508792, "prefill_tokens_per_second_min": 1847.0198583245005, "prefill_tokens_per_second_max": 1854.4921713716606, "prefill_tokens_per_second_stdev": 3.0826075182414567, "prefill_tokens_per_second_p95": 1854.4921713716606, "decode_tokens_per_second_median": 22.14313873296103, "decode_tokens_per_second_mean": 22.139935473391983, "decode_tokens_per_second_min": 22.129707950823587, "decode_tokens_per_second_max": 22.145178465254336, "decode_tokens_per_second_stdev": 0.006298889555993124, "decode_tokens_per_second_p95": 22.145178465254336, "load_seconds_median": 0.187220382, "load_seconds_mean": 0.1833007556, "load_seconds_min": 0.15201197, "load_seconds_max": 0.208945459, "load_seconds_stdev": 0.02171077704282952, "load_seconds_p95": 0.208945459, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 23515156480.0, "gpu_energy_j_observed_median": 2920.858564103842 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 6.960013400006574, "wall_seconds_mean": 6.9547031400026755, "wall_seconds_min": 6.9356514000101015, "wall_seconds_max": 6.96378689998528, "wall_seconds_stdev": 0.011491988773486601, "wall_seconds_p95": 6.96378689998528, "ttft_seconds_median": 2.348633499990683, "ttft_seconds_mean": 2.3501651800004764, "ttft_seconds_min": 2.3181875000009313, "ttft_seconds_max": 2.384216899983585, "ttft_seconds_stdev": 0.023509515249025642, "ttft_seconds_p95": 2.384216899983585, "prompt_tokens_median": 4112.0, "prompt_tokens_mean": 4112.0, "prompt_tokens_min": 4112.0, "prompt_tokens_max": 4112.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 4112.0, "output_tokens_median": 115.0, "output_tokens_mean": 115.0, "output_tokens_min": 115.0, "output_tokens_max": 115.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 115.0, "prefill_tokens_per_second_median": 1901.5614151775949, "prefill_tokens_per_second_mean": 1902.2715145559873, "prefill_tokens_per_second_min": 1898.240082760904, "prefill_tokens_per_second_max": 1906.9712194310646, "prefill_tokens_per_second_stdev": 3.5344956925612347, "prefill_tokens_per_second_p95": 1906.9712194310646, "decode_tokens_per_second_median": 25.38691963032847, "decode_tokens_per_second_mean": 25.379261183061292, "decode_tokens_per_second_min": 25.340824280779263, "decode_tokens_per_second_max": 25.42049412593003, "decode_tokens_per_second_stdev": 0.030098957428339408, "decode_tokens_per_second_p95": 25.42049412593003, "load_seconds_median": 0.164101518, "load_seconds_mean": 0.1637178736, "load_seconds_min": 0.130332092, "load_seconds_max": 0.193998743, "load_seconds_stdev": 0.02270931238126789, "load_seconds_p95": 0.193998743, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 48377782272.0, "gpu_energy_j_observed_median": 391.99785560369486 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 68.26462600001832, "wall_seconds_mean": 68.26868746000109, "wall_seconds_min": 68.1673305000295, "wall_seconds_max": 68.35391910001636, "wall_seconds_stdev": 0.06761275498887735, "wall_seconds_p95": 68.35391910001636, "ttft_seconds_median": 18.568253099976573, "ttft_seconds_mean": 18.58423553999746, "ttft_seconds_min": 18.52992510003969, "ttft_seconds_max": 18.634360099968035, "ttft_seconds_stdev": 0.04329069623076725, "ttft_seconds_p95": 18.634360099968035, "prompt_tokens_median": 32565.0, "prompt_tokens_mean": 32565.0, "prompt_tokens_min": 32565.0, "prompt_tokens_max": 32565.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32565.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1789.8269545147848, "prefill_tokens_per_second_mean": 1789.4502175698135, "prefill_tokens_per_second_min": 1788.662150984916, "prefill_tokens_per_second_max": 1790.2273529217891, "prefill_tokens_per_second_stdev": 0.7270450326924504, "prefill_tokens_per_second_p95": 1790.2273529217891, "decode_tokens_per_second_median": 20.324941577084996, "decode_tokens_per_second_mean": 20.31158943663835, "decode_tokens_per_second_min": 20.261477535272086, "decode_tokens_per_second_max": 20.33204086654584, "decode_tokens_per_second_stdev": 0.028829964886661962, "decode_tokens_per_second_p95": 20.33204086654584, "load_seconds_median": 0.158522801, "load_seconds_mean": 0.1671504086, "load_seconds_min": 0.129011841, "load_seconds_max": 0.205212952, "load_seconds_stdev": 0.02971307046592241, "load_seconds_p95": 0.205212952, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 25448681472.0, "gpu_energy_j_observed_median": 4027.035607172251 }, { "system_id": "spark", "model": "gemma3:12b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 69.77884599997196, "wall_seconds_mean": 69.778426139988, "wall_seconds_min": 69.64366330002667, "wall_seconds_max": 69.89773229998536, "wall_seconds_stdev": 0.09265241954100263, "wall_seconds_p95": 69.89773229998536, "ttft_seconds_median": 18.667896599974483, "ttft_seconds_mean": 18.660802659997717, "ttft_seconds_min": 18.575277500029188, "ttft_seconds_max": 18.72083589999238, "ttft_seconds_stdev": 0.05769613759480938, "ttft_seconds_p95": 18.72083589999238, "prompt_tokens_median": 32556.0, "prompt_tokens_mean": 32556.0, "prompt_tokens_min": 32556.0, "prompt_tokens_max": 32556.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32556.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 1783.3871949882305, "prefill_tokens_per_second_mean": 1784.5961803418177, "prefill_tokens_per_second_min": 1779.6017604753379, "prefill_tokens_per_second_max": 1792.1205564989878, "prefill_tokens_per_second_stdev": 5.108653686135364, "prefill_tokens_per_second_p95": 1792.1205564989878, "decode_tokens_per_second_median": 19.72602021668627, "decode_tokens_per_second_mean": 19.73161279531152, "decode_tokens_per_second_min": 19.721572637297353, "decode_tokens_per_second_max": 19.75763583815387, "decode_tokens_per_second_stdev": 0.014715391335030942, "decode_tokens_per_second_p95": 19.75763583815387, "load_seconds_median": 0.172553383, "load_seconds_mean": 0.1726917858, "load_seconds_min": 0.143279841, "load_seconds_max": 0.192895002, "load_seconds_stdev": 0.02106958047297582, "load_seconds_p95": 0.192895002, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 25423945728.0, "gpu_energy_j_observed_median": 3983.250004781484 }, { "system_id": "spark", "model": "gemma3:4b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 13.403588699991815, "wall_seconds_mean": 13.437139460002072, "wall_seconds_min": 13.347203399986029, "wall_seconds_max": 13.653546700021252, "wall_seconds_stdev": 0.12442524963488595, "wall_seconds_p95": 13.653546700021252, "prompt_tokens_median": 15309.0, "prompt_tokens_mean": 15309.0, "prompt_tokens_min": 15309.0, "prompt_tokens_max": 15309.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15309.0, "output_tokens_median": 643.0, "output_tokens_mean": 643.0, "output_tokens_min": 643.0, "output_tokens_max": 643.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 643.0, "prefill_tokens_per_second_median": 21916.059008773424, "prefill_tokens_per_second_mean": 21940.607399967845, "prefill_tokens_per_second_min": 21848.601199075038, "prefill_tokens_per_second_max": 22102.242246849528, "prefill_tokens_per_second_stdev": 101.27455691851907, "prefill_tokens_per_second_p95": 22102.242246849528, "decode_tokens_per_second_median": 67.27403380941357, "decode_tokens_per_second_mean": 67.31432462370181, "decode_tokens_per_second_min": 67.16245591924337, "decode_tokens_per_second_max": 67.5158436941866, "decode_tokens_per_second_stdev": 0.16071028856307834, "decode_tokens_per_second_p95": 67.5158436941866, "prefill_seconds_median": 0.6985288730000001, "prefill_seconds_mean": 0.6977591722, "prefill_seconds_min": 0.692644657, "prefill_seconds_max": 0.700685589, "prefill_seconds_stdev": 0.0032098186694219572, "prefill_seconds_p95": 0.700685589, "decode_seconds_median": 9.557922479, "decode_seconds_mean": 9.5522451418, "decode_seconds_min": 9.523690512, "decode_seconds_max": 9.573801184, "decode_seconds_stdev": 0.022792273052317426, "decode_seconds_p95": 9.573801184, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 16059179008.0, "gpu_energy_j_observed_median": 429.06734197020535 }, { "system_id": "spark", "model": "gemma3:4b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 15.039582900004461, "wall_seconds_mean": 15.048877700010781, "wall_seconds_min": 15.02641940000467, "wall_seconds_max": 15.080306800024118, "wall_seconds_stdev": 0.022621201585608937, "wall_seconds_p95": 15.080306800024118, "ttft_seconds_median": 3.1641445999848656, "ttft_seconds_mean": 3.1687761000008323, "ttft_seconds_min": 3.1461836000089534, "ttft_seconds_max": 3.1948142999899574, "ttft_seconds_stdev": 0.021082669752264905, "ttft_seconds_p95": 3.1948142999899574, "prompt_tokens_median": 16308.0, "prompt_tokens_mean": 16308.0, "prompt_tokens_min": 16308.0, "prompt_tokens_max": 16308.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16308.0, "output_tokens_median": 729.0, "output_tokens_mean": 729.0, "output_tokens_min": 729.0, "output_tokens_max": 729.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 729.0, "prefill_tokens_per_second_median": 5696.816168703371, "prefill_tokens_per_second_mean": 5694.173846782061, "prefill_tokens_per_second_min": 5670.24064041275, "prefill_tokens_per_second_max": 5705.039484288611, "prefill_tokens_per_second_stdev": 14.217146570745015, "prefill_tokens_per_second_p95": 5705.039484288611, "decode_tokens_per_second_median": 63.053190594232014, "decode_tokens_per_second_mean": 63.049471906537285, "decode_tokens_per_second_min": 62.97133273224389, "decode_tokens_per_second_max": 63.1599314989424, "decode_tokens_per_second_stdev": 0.07495092574829858, "decode_tokens_per_second_p95": 63.1599314989424, "load_seconds_median": 0.173340694, "load_seconds_mean": 0.1751594668, "load_seconds_min": 0.156661941, "load_seconds_max": 0.200234841, "load_seconds_stdev": 0.017177552672466317, "load_seconds_p95": 0.200234841, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 17069637632.0, "gpu_energy_j_observed_median": 702.1070633029939 }, { "system_id": "spark", "model": "gemma3:4b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 5.8957928000018, "wall_seconds_mean": 5.906690039997921, "wall_seconds_min": 5.884041299985256, "wall_seconds_max": 5.932959500001743, "wall_seconds_stdev": 0.022435822878166558, "wall_seconds_p95": 5.932959500001743, "ttft_seconds_median": 0.9116524999844842, "ttft_seconds_mean": 0.9209780999924988, "ttft_seconds_min": 0.9083134999964386, "ttft_seconds_max": 0.9394960000063293, "ttft_seconds_stdev": 0.015393988044943099, "ttft_seconds_p95": 0.9394960000063293, "prompt_tokens_median": 4112.0, "prompt_tokens_mean": 4112.0, "prompt_tokens_min": 4112.0, "prompt_tokens_max": 4112.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 4112.0, "output_tokens_median": 342.0, "output_tokens_mean": 342.0, "output_tokens_min": 342.0, "output_tokens_max": 342.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 342.0, "prefill_tokens_per_second_median": 5783.4522264321295, "prefill_tokens_per_second_mean": 5784.655262776332, "prefill_tokens_per_second_min": 5772.615507172452, "prefill_tokens_per_second_max": 5801.293115007273, "prefill_tokens_per_second_stdev": 11.95369001937274, "prefill_tokens_per_second_p95": 5801.293115007273, "decode_tokens_per_second_median": 71.1289251438206, "decode_tokens_per_second_mean": 71.16291952662473, "decode_tokens_per_second_min": 71.08197094304083, "decode_tokens_per_second_max": 71.26999576242567, "decode_tokens_per_second_stdev": 0.07482851376493523, "decode_tokens_per_second_p95": 71.26999576242567, "load_seconds_median": 0.176041667, "load_seconds_mean": 0.1826403554, "load_seconds_min": 0.158704253, "load_seconds_max": 0.204869977, "load_seconds_stdev": 0.01928261670140965, "load_seconds_p95": 0.204869977, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 38195003392.0, "gpu_energy_j_observed_median": 241.97279193520544 }, { "system_id": "spark", "model": "gemma3:4b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 23.388107399980072, "wall_seconds_mean": 23.390238399989904, "wall_seconds_min": 23.344776499958243, "wall_seconds_max": 23.43585880001774, "wall_seconds_stdev": 0.03760674679960412, "wall_seconds_p95": 23.43585880001774, "ttft_seconds_median": 6.299955900001805, "ttft_seconds_mean": 6.295649800007231, "ttft_seconds_min": 6.2683367999852635, "ttft_seconds_max": 6.324865300033707, "ttft_seconds_stdev": 0.022641534123767845, "ttft_seconds_p95": 6.324865300033707, "prompt_tokens_median": 32565.0, "prompt_tokens_mean": 32565.0, "prompt_tokens_min": 32565.0, "prompt_tokens_max": 32565.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32565.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 5527.812924481268, "prefill_tokens_per_second_mean": 5529.668952016216, "prefill_tokens_per_second_min": 5523.513969236686, "prefill_tokens_per_second_max": 5540.174306973242, "prefill_tokens_per_second_stdev": 6.247843891952216, "prefill_tokens_per_second_p95": 5540.174306973242, "decode_tokens_per_second_median": 60.13543214208414, "decode_tokens_per_second_mean": 60.14189177504434, "decode_tokens_per_second_min": 60.082151895632705, "decode_tokens_per_second_max": 60.25100015488375, "decode_tokens_per_second_stdev": 0.0675965792807018, "decode_tokens_per_second_p95": 60.25100015488375, "load_seconds_median": 0.175931397, "load_seconds_mean": 0.172825568, "load_seconds_min": 0.157692977, "load_seconds_max": 0.183919676, "load_seconds_stdev": 0.011670634851335869, "load_seconds_p95": 0.183919676, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 40165699584.0, "gpu_energy_j_observed_median": 1199.2185351574424 }, { "system_id": "spark", "model": "gemma3:4b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 23.56643940001959, "wall_seconds_mean": 23.568563219998033, "wall_seconds_min": 23.476335599960294, "wall_seconds_max": 23.644589799980167, "wall_seconds_stdev": 0.06541828562359413, "wall_seconds_p95": 23.644589799980167, "ttft_seconds_median": 6.323581500037108, "ttft_seconds_mean": 6.3270984999951905, "ttft_seconds_min": 6.284043399966322, "ttft_seconds_max": 6.3994794999598525, "ttft_seconds_stdev": 0.04575270232319688, "ttft_seconds_p95": 6.3994794999598525, "prompt_tokens_median": 32556.0, "prompt_tokens_mean": 32556.0, "prompt_tokens_min": 32556.0, "prompt_tokens_max": 32556.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 32556.0, "output_tokens_median": 959.0, "output_tokens_mean": 959.0, "output_tokens_min": 959.0, "output_tokens_max": 959.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 959.0, "prefill_tokens_per_second_median": 5467.367446158801, "prefill_tokens_per_second_mean": 5472.051952792185, "prefill_tokens_per_second_min": 5460.863685540756, "prefill_tokens_per_second_max": 5484.58503105641, "prefill_tokens_per_second_stdev": 9.912901160697784, "prefill_tokens_per_second_p95": 5484.58503105641, "decode_tokens_per_second_median": 57.086654193132325, "decode_tokens_per_second_mean": 57.09358289248874, "decode_tokens_per_second_min": 57.00723810719411, "decode_tokens_per_second_max": 57.18132378343071, "decode_tokens_per_second_stdev": 0.07414952086064626, "decode_tokens_per_second_p95": 57.18132378343071, "load_seconds_median": 0.158136956, "load_seconds_mean": 0.1646635372, "load_seconds_min": 0.142523585, "load_seconds_max": 0.197139812, "load_seconds_stdev": 0.020902574716028972, "load_seconds_p95": 0.197139812, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 18167279616.0, "gpu_energy_j_observed_median": 1161.0176533305646 }, { "system_id": "spark", "model": "qwen2.5vl:7b", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 24.57027799997013, "wall_seconds_mean": 24.57302913998719, "wall_seconds_min": 24.479171499959193, "wall_seconds_max": 24.706208299961872, "wall_seconds_stdev": 0.08408826505326963, "wall_seconds_p95": 24.706208299961872, "prompt_tokens_median": 15294.0, "prompt_tokens_mean": 15294.0, "prompt_tokens_min": 15294.0, "prompt_tokens_max": 15294.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15294.0, "output_tokens_median": 870.0, "output_tokens_mean": 870.0, "output_tokens_min": 870.0, "output_tokens_max": 870.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 870.0, "prefill_tokens_per_second_median": 76757.67047847748, "prefill_tokens_per_second_mean": 76706.18521231637, "prefill_tokens_per_second_min": 76342.4881956663, "prefill_tokens_per_second_max": 76877.40295606432, "prefill_tokens_per_second_stdev": 215.12218334206935, "prefill_tokens_per_second_p95": 76877.40295606432, "decode_tokens_per_second_median": 40.15977939178132, "decode_tokens_per_second_mean": 40.16457221010843, "decode_tokens_per_second_min": 40.087932995298836, "decode_tokens_per_second_max": 40.29138304183784, "decode_tokens_per_second_stdev": 0.08195423711468446, "decode_tokens_per_second_p95": 40.29138304183784, "prefill_seconds_median": 0.199250445, "prefill_seconds_mean": 0.1993854402, "prefill_seconds_min": 0.198940123, "prefill_seconds_max": 0.200334052, "prefill_seconds_stdev": 0.0005607555566026122, "prefill_seconds_p95": 0.200334052, "decode_seconds_median": 21.663465616, "decode_seconds_mean": 21.6609525866, "decode_seconds_min": 21.592706289000002, "decode_seconds_max": 21.702291313, "decode_seconds_stdev": 0.044145181671854, "decode_seconds_p95": 21.702291313, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 27191349248.0, "gpu_energy_j_observed_median": 1066.741272279024 }, { "system_id": "spark", "model": "qwen2.5vl:7b", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 24.151167000003625, "wall_seconds_mean": 24.154775900009554, "wall_seconds_min": 24.12690550001571, "wall_seconds_max": 24.18344450002769, "wall_seconds_stdev": 0.02062894497365567, "wall_seconds_p95": 24.18344450002769, "ttft_seconds_median": 0.17950859997654334, "ttft_seconds_mean": 0.17152141999686138, "ttft_seconds_min": 0.14929020003182814, "ttft_seconds_max": 0.1899834999931045, "ttft_seconds_stdev": 0.016830157656121822, "ttft_seconds_p95": 0.1899834999931045, "prompt_tokens_median": 16015.0, "prompt_tokens_mean": 16015.0, "prompt_tokens_min": 16015.0, "prompt_tokens_max": 16015.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16015.0, "output_tokens_median": 769.0, "output_tokens_mean": 769.0, "output_tokens_min": 769.0, "output_tokens_max": 769.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 769.0, "prefill_tokens_per_second_median": 489823.2528686622, "prefill_tokens_per_second_mean": 486032.46008690854, "prefill_tokens_per_second_min": 469599.77051041875, "prefill_tokens_per_second_max": 497741.9330375952, "prefill_tokens_per_second_stdev": 10764.70296917355, "prefill_tokens_per_second_p95": 497741.9330375952, "decode_tokens_per_second_median": 32.374204629230334, "decode_tokens_per_second_mean": 32.3727266299309, "decode_tokens_per_second_min": 32.36550379700679, "decode_tokens_per_second_max": 32.37886782956181, "decode_tokens_per_second_stdev": 0.006116479980025489, "decode_tokens_per_second_p95": 32.37886782956181, "load_seconds_median": 0.102945779, "load_seconds_mean": 0.0968954076, "load_seconds_min": 0.084375955, "load_seconds_max": 0.106030265, "load_seconds_stdev": 0.010682427482287288, "load_seconds_p95": 0.106030265, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 29825994752.0, "gpu_energy_j_observed_median": 1332.9609519422054 }, { "system_id": "spark", "model": "qwen2.5vl:7b", "workload": "interactive", "context_setting": 8192, "n": 5, "wall_seconds_median": 5.3058400000445545, "wall_seconds_mean": 5.312536740000359, "wall_seconds_min": 5.295013100025244, "wall_seconds_max": 5.332832099986263, "wall_seconds_stdev": 0.016658054735564566, "wall_seconds_p95": 5.332832099986263, "ttft_seconds_median": 0.12267630000133067, "ttft_seconds_mean": 0.11449284000555053, "ttft_seconds_min": 0.0945260000298731, "ttft_seconds_max": 0.1279958999948576, "ttft_seconds_stdev": 0.01614021686773989, "ttft_seconds_p95": 0.1279958999948576, "prompt_tokens_median": 4053.0, "prompt_tokens_mean": 4053.0, "prompt_tokens_min": 4053.0, "prompt_tokens_max": 4053.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 4053.0, "output_tokens_median": 210.0, "output_tokens_mean": 210.0, "output_tokens_min": 210.0, "output_tokens_max": 210.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 210.0, "prefill_tokens_per_second_median": 156671.16489050607, "prefill_tokens_per_second_mean": 156028.79724051311, "prefill_tokens_per_second_min": 153554.0720793347, "prefill_tokens_per_second_max": 158286.20058716252, "prefill_tokens_per_second_stdev": 1962.449723225858, "prefill_tokens_per_second_p95": 158286.20058716252, "decode_tokens_per_second_median": 41.1292041004281, "decode_tokens_per_second_mean": 41.13486457909118, "decode_tokens_per_second_min": 41.10580684498221, "decode_tokens_per_second_max": 41.1768886709116, "decode_tokens_per_second_stdev": 0.027402532933568768, "decode_tokens_per_second_p95": 41.1768886709116, "load_seconds_median": 0.084983329, "load_seconds_mean": 0.0755209462, "load_seconds_min": 0.053063014, "load_seconds_max": 0.090540795, "load_seconds_stdev": 0.017409089010076997, "load_seconds_p95": 0.090540795, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 34364604416.0, "gpu_energy_j_observed_median": 238.23749910712243 }, { "system_id": "spark", "model": "qwen2.5vl:7b", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 39.15960640000412, "wall_seconds_mean": 39.17011492000893, "wall_seconds_min": 39.09613720001653, "wall_seconds_max": 39.28840730001684, "wall_seconds_stdev": 0.07140969640689108, "wall_seconds_p95": 39.28840730001684, "ttft_seconds_median": 0.1811472999979742, "ttft_seconds_mean": 0.18232986001530663, "ttft_seconds_min": 0.1597015000297688, "ttft_seconds_max": 0.19782300002407283, "ttft_seconds_stdev": 0.01500677240623016, "ttft_seconds_p95": 0.19782300002407283, "prompt_tokens_median": 31984.0, "prompt_tokens_mean": 31984.0, "prompt_tokens_min": 31984.0, "prompt_tokens_max": 31984.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31984.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 747434.0547304287, "prefill_tokens_per_second_mean": 760665.927886558, "prefill_tokens_per_second_min": 740808.0644684604, "prefill_tokens_per_second_max": 797678.4723280091, "prefill_tokens_per_second_stdev": 23818.053468872455, "prefill_tokens_per_second_p95": 797678.4723280091, "decode_tokens_per_second_median": 25.85502401197771, "decode_tokens_per_second_mean": 25.84569989062763, "decode_tokens_per_second_min": 25.78900120272505, "decode_tokens_per_second_max": 25.877249498346345, "decode_tokens_per_second_stdev": 0.033873346119974, "decode_tokens_per_second_p95": 25.877249498346345, "load_seconds_median": 0.060180439, "load_seconds_mean": 0.0680119292, "load_seconds_min": 0.057844244, "load_seconds_max": 0.101198976, "load_seconds_stdev": 0.01858466998856527, "load_seconds_p95": 0.101198976, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 43781812224.0, "gpu_energy_j_observed_median": 2387.9249830722815 }, { "system_id": "spark", "model": "qwen2.5vl:7b", "workload": "overnight", "context_setting": 36864, "n": 5, "wall_seconds_median": 39.98556170001393, "wall_seconds_mean": 39.97792683999287, "wall_seconds_min": 39.95924629998626, "wall_seconds_max": 39.98879339999985, "wall_seconds_stdev": 0.013209009476358778, "wall_seconds_p95": 39.98879339999985, "ttft_seconds_median": 0.20248219999484718, "ttft_seconds_mean": 0.20757741999113932, "ttft_seconds_min": 0.19708319997880608, "ttft_seconds_max": 0.2195740999886766, "ttft_seconds_stdev": 0.01083971696636673, "ttft_seconds_p95": 0.2195740999886766, "prompt_tokens_median": 31979.0, "prompt_tokens_mean": 31979.0, "prompt_tokens_min": 31979.0, "prompt_tokens_max": 31979.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31979.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 748465.3990103003, "prefill_tokens_per_second_mean": 748704.121131426, "prefill_tokens_per_second_min": 748070.4593995688, "prefill_tokens_per_second_max": 749411.5365366655, "prefill_tokens_per_second_stdev": 604.7471002947648, "prefill_tokens_per_second_p95": 749411.5365366655, "decode_tokens_per_second_median": 25.33095324331622, "decode_tokens_per_second_mean": 25.334622744130286, "decode_tokens_per_second_min": 25.328208255428088, "decode_tokens_per_second_max": 25.3449498992134, "decode_tokens_per_second_stdev": 0.007619750818213886, "decode_tokens_per_second_p95": 25.3449498992134, "load_seconds_median": 0.081204725, "load_seconds_mean": 0.0845625884, "load_seconds_min": 0.063515136, "load_seconds_max": 0.108827023, "load_seconds_stdev": 0.017857618634440447, "load_seconds_p95": 0.108827023, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 36968321024.0, "gpu_energy_j_observed_median": 2378.959251116514 }, { "system_id": "spark", "model": "qwen3-coder:30b-a3b-q4_K_M", "workload": "agent_loop", "context_setting": 8192, "n": 5, "wall_seconds_median": 15.026726400014013, "wall_seconds_mean": 15.032175960007589, "wall_seconds_min": 14.995993999997154, "wall_seconds_max": 15.101637799991295, "wall_seconds_stdev": 0.04233964933458347, "wall_seconds_p95": 15.101637799991295, "prompt_tokens_median": 15294.0, "prompt_tokens_mean": 15294.0, "prompt_tokens_min": 15294.0, "prompt_tokens_max": 15294.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 15294.0, "output_tokens_median": 870.0, "output_tokens_mean": 870.0, "output_tokens_min": 870.0, "output_tokens_max": 870.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 870.0, "prefill_tokens_per_second_median": 55216.99254530526, "prefill_tokens_per_second_mean": 55137.18938729583, "prefill_tokens_per_second_min": 54681.620341476955, "prefill_tokens_per_second_max": 55318.69443844531, "prefill_tokens_per_second_stdev": 258.12021916419735, "prefill_tokens_per_second_p95": 55318.69443844531, "decode_tokens_per_second_median": 71.61741137806472, "decode_tokens_per_second_mean": 71.58615685862844, "decode_tokens_per_second_min": 71.43312686217227, "decode_tokens_per_second_max": 71.71441028939961, "decode_tokens_per_second_stdev": 0.1355881393820036, "decode_tokens_per_second_p95": 71.71441028939961, "prefill_seconds_median": 0.276979953, "prefill_seconds_mean": 0.2773857336, "prefill_seconds_min": 0.276470733, "prefill_seconds_max": 0.279691785, "prefill_seconds_stdev": 0.0013062094007760862, "prefill_seconds_p95": 0.279691785, "decode_seconds_median": 12.147883919, "decode_seconds_mean": 12.1532225766, "decode_seconds_min": 12.13145303, "decode_seconds_max": 12.179223257, "decode_seconds_stdev": 0.023026600001215404, "decode_seconds_p95": 12.179223257, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 31412097024.0, "gpu_energy_j_observed_median": 670.9777383220196 }, { "system_id": "spark", "model": "qwen3-coder:30b-a3b-q4_K_M", "workload": "engineering_validity", "context_setting": 20480, "n": 5, "wall_seconds_median": 21.53753059997689, "wall_seconds_mean": 21.535121739993336, "wall_seconds_min": 21.49870729999384, "wall_seconds_max": 21.58058930002153, "wall_seconds_stdev": 0.03148606336189136, "wall_seconds_p95": 21.58058930002153, "ttft_seconds_median": 0.15814569999929518, "ttft_seconds_mean": 0.14821505999425427, "ttft_seconds_min": 0.12010189995635301, "ttft_seconds_max": 0.1652468999964185, "ttft_seconds_stdev": 0.018626382536964418, "ttft_seconds_p95": 0.1652468999964185, "prompt_tokens_median": 16004.0, "prompt_tokens_mean": 16004.0, "prompt_tokens_min": 16004.0, "prompt_tokens_max": 16004.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 16004.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 610605.8466005805, "prefill_tokens_per_second_mean": 606642.9034257189, "prefill_tokens_per_second_min": 571427.3268444696, "prefill_tokens_per_second_max": 625660.0774412677, "prefill_tokens_per_second_stdev": 21389.719030417953, "prefill_tokens_per_second_p95": 625660.0774412677, "decode_tokens_per_second_median": 47.40268316864143, "decode_tokens_per_second_mean": 47.40158048267745, "decode_tokens_per_second_min": 47.361635109210646, "decode_tokens_per_second_max": 47.44624495169178, "decode_tokens_per_second_stdev": 0.03142808785285281, "decode_tokens_per_second_p95": 47.44624495169178, "load_seconds_median": 0.074388568, "load_seconds_mean": 0.07337405079999999, "load_seconds_min": 0.053787283, "load_seconds_max": 0.098231261, "load_seconds_stdev": 0.018860030938986916, "load_seconds_p95": 0.098231261, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 32696320000.0, "gpu_energy_j_observed_median": 780.2369675934316 }, { "system_id": "spark", "model": "qwen3-coder:30b-a3b-q4_K_M", "workload": "large_context", "context_setting": 36864, "n": 5, "wall_seconds_median": 29.45893680001609, "wall_seconds_mean": 29.458411900012287, "wall_seconds_min": 29.438212500012014, "wall_seconds_max": 29.48841570003424, "wall_seconds_stdev": 0.019198378982888176, "wall_seconds_p95": 29.48841570003424, "ttft_seconds_median": 0.19226260000141338, "ttft_seconds_mean": 0.19470222000963985, "ttft_seconds_min": 0.18062820000341162, "ttft_seconds_max": 0.21975860005477443, "ttft_seconds_stdev": 0.014967432515909623, "ttft_seconds_p95": 0.21975860005477443, "prompt_tokens_median": 31973.0, "prompt_tokens_mean": 31973.0, "prompt_tokens_min": 31973.0, "prompt_tokens_max": 31973.0, "prompt_tokens_stdev": 0.0, "prompt_tokens_p95": 31973.0, "output_tokens_median": 1000.0, "output_tokens_mean": 1000.0, "output_tokens_min": 1000.0, "output_tokens_max": 1000.0, "output_tokens_stdev": 0.0, "output_tokens_p95": 1000.0, "prefill_tokens_per_second_median": 921500.7749592966, "prefill_tokens_per_second_mean": 926161.6543559779, "prefill_tokens_per_second_min": 897640.1102424349, "prefill_tokens_per_second_max": 970221.2506826491, "prefill_tokens_per_second_stdev": 29008.30692901473, "prefill_tokens_per_second_p95": 970221.2506826491, "decode_tokens_per_second_median": 34.47154932750504, "decode_tokens_per_second_mean": 34.4778525681765, "decode_tokens_per_second_min": 34.452630002468275, "decode_tokens_per_second_max": 34.518959374713134, "decode_tokens_per_second_stdev": 0.025748654986126194, "decode_tokens_per_second_p95": 34.518959374713134, "load_seconds_median": 0.076964346, "load_seconds_mean": 0.0803426282, "load_seconds_min": 0.068247405, "load_seconds_max": 0.098883641, "load_seconds_stdev": 0.012925756916131357, "load_seconds_p95": 0.098883641, "peak_gpu_memory_used_mib_median": null, "peak_system_memory_used_bytes_median": 34379694080.0, "gpu_energy_j_observed_median": 995.3633492875099 } ] }