{ "status": "PARTIAL_RESULTS_FINAL_NO_SUPPLEMENT", "missing": [ "vllm slo-0.8: 3 rounds not executed", "vllm slo-1.1: 3 rounds not executed", "OpenWebUI screenshots not captured" ], "aggregation": "Median/min/max of per-round statistics; not pooled percentiles", "quality_verified": "Limited arithmetic, memory, tool, fixture vision and SSE smoke checks passed; not a quality benchmark.", "client_arrival_timing_verified": false, "phases": [ { "engine": "sglang", "stage": "core", "case": "decode-c1", "file": "sglang/core/sglang-decode-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 84.42484433809295, "output_tok_s": 48.51652415960526, "ttft_p50_s": 0.2373232648242265, "ttft_p95_s": 0.24694492197595536, "ttft_p99_s": 0.24723462033085525, "mean_request_tpot_ms": 20.18909419976984, "request_tpot_p95_ms": 20.211700206976058, "e2e_p95_s": 10.56673616755288, "full_stream_e2e_available": true, "token_e2e_p95_s": 10.566720018885098, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.09475883624922903, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.09475883624922903, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.09475883624922903, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.09475883624922903, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "decode-c1", "file": "sglang/core/sglang-decode-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 84.39079887699336, "output_tok_s": 48.536096997615374, "ttft_p50_s": 0.22770833387039602, "ttft_p95_s": 0.22870647183153778, "ttft_p99_s": 0.22880270559806376, "mean_request_tpot_ms": 20.19936801788694, "request_tpot_p95_ms": 20.219082154658004, "e2e_p95_s": 10.559498811978846, "full_stream_e2e_available": true, "token_e2e_p95_s": 10.559484610427171, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.09479706444846753, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.09479706444846753, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.09479706444846753, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.09479706444846753, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "decode-c1", "file": "sglang/core/sglang-decode-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 84.45719575602561, "output_tok_s": 48.497939853843306, "ttft_p50_s": 0.2377315240446478, "ttft_p95_s": 0.24784173434600235, "ttft_p99_s": 0.24792676528915764, "mean_request_tpot_ms": 20.19596505416247, "request_tpot_p95_ms": 20.222119238171107, "e2e_p95_s": 10.564405523194, "full_stream_e2e_available": true, "token_e2e_p95_s": 10.564387026871554, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.09472253877703771, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.09472253877703771, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.09472253877703771, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.09472253877703771, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "decode-c32", "file": "sglang/core/sglang-decode-c32-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 39.987835094798356, "output_tok_s": 819.4492130498578, "ttft_p50_s": 5.557435345137492, "ttft_p95_s": 5.683949015289545, "ttft_p99_s": 5.689705312228762, "mean_request_tpot_ms": 29.412606192380007, "request_tpot_p95_ms": 33.62344485172663, "e2e_p95_s": 20.114979301765562, "full_stream_e2e_available": true, "token_e2e_p95_s": 20.114969361503608, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.05001521075743761, "pass_fraction_ttft2s_tpot50ms": 0.03125, "goodput_ttft3s_tpot50ms_rps": 0.22506844840846926, "pass_fraction_ttft3s_tpot50ms": 0.140625, "goodput_ttft5s_tpot50ms_rps": 0.5251597129530949, "pass_fraction_ttft5s_tpot50ms": 0.328125 }, { "engine": "sglang", "stage": "core", "case": "decode-c32", "file": "sglang/core/sglang-decode-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 39.73307926207781, "output_tok_s": 824.7032600686088, "ttft_p50_s": 5.5573560821358114, "ttft_p95_s": 5.679089043894782, "ttft_p99_s": 5.681537591237575, "mean_request_tpot_ms": 29.78502103882929, "request_tpot_p95_ms": 33.160563055225396, "e2e_p95_s": 19.87822789240163, "full_stream_e2e_available": true, "token_e2e_p95_s": 19.878219857113436, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.05033589233817192, "pass_fraction_ttft2s_tpot50ms": 0.03125, "goodput_ttft3s_tpot50ms_rps": 0.40268713870537537, "pass_fraction_ttft3s_tpot50ms": 0.25, "goodput_ttft5s_tpot50ms_rps": 0.7298704389034929, "pass_fraction_ttft5s_tpot50ms": 0.453125 }, { "engine": "sglang", "stage": "core", "case": "decode-c32", "file": "sglang/core/sglang-decode-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 39.607141823042184, "output_tok_s": 827.3255400857179, "ttft_p50_s": 5.552005053963512, "ttft_p95_s": 5.673246767208911, "ttft_p99_s": 5.683081572433002, "mean_request_tpot_ms": 29.250298274370913, "request_tpot_p95_ms": 33.473784309163065, "e2e_p95_s": 20.029147566552275, "full_stream_e2e_available": true, "token_e2e_p95_s": 20.029140169639142, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.05049594360874743, "pass_fraction_ttft2s_tpot50ms": 0.03125, "goodput_ttft3s_tpot50ms_rps": 0.3029756616524846, "pass_fraction_ttft3s_tpot50ms": 0.1875, "goodput_ttft5s_tpot50ms_rps": 0.5807033515005955, "pass_fraction_ttft5s_tpot50ms": 0.359375 }, { "engine": "sglang", "stage": "core", "case": "decode-c8", "file": "sglang/core/sglang-decode-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 23.394785780925304, "output_tok_s": 350.1634969737258, "ttft_p50_s": 1.448188545415178, "ttft_p95_s": 1.465820539277047, "ttft_p99_s": 1.466263275872916, "mean_request_tpot_ms": 20.069967273375795, "request_tpot_p95_ms": 20.32843019964571, "e2e_p95_s": 11.809660215047188, "full_stream_e2e_available": true, "token_e2e_p95_s": 11.809639187762514, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.6839130800268082, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.6839130800268082, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.6839130800268082, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "decode-c8", "file": "sglang/core/sglang-decode-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 23.660802009049803, "output_tok_s": 346.2266408749254, "ttft_p50_s": 1.4503454531077296, "ttft_p95_s": 1.4581511460710317, "ttft_p99_s": 1.4666330514941364, "mean_request_tpot_ms": 20.328408288163427, "request_tpot_p95_ms": 20.548386023828208, "e2e_p95_s": 11.931524883024395, "full_stream_e2e_available": true, "token_e2e_p95_s": 11.931516422890127, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.6762239079588387, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.6762239079588387, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.6762239079588387, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "decode-c8", "file": "sglang/core/sglang-decode-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 23.23380251415074, "output_tok_s": 352.5897233141495, "ttft_p50_s": 1.4539049200247973, "ttft_p95_s": 1.4701936388155445, "ttft_p99_s": 1.4705270695267245, "mean_request_tpot_ms": 19.90083777578092, "request_tpot_p95_ms": 20.043503221510793, "e2e_p95_s": 11.6389586313162, "full_stream_e2e_available": true, "token_e2e_p95_s": 11.638950228923932, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.6886518033479483, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.6886518033479483, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.6886518033479483, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "long-c1", "file": "sglang/core/sglang-long-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 60.75507844891399, "output_tok_s": 16.8545581067932, "ttft_p50_s": 5.038191937142983, "ttft_p95_s": 5.064343739300966, "ttft_p99_s": 5.067687226831913, "mean_request_tpot_ms": 20.123470527409598, "request_tpot_p95_ms": 20.24613352782115, "e2e_p95_s": 7.61687787885312, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.61686287720222, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c1", "file": "sglang/core/sglang-long-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 60.763075372204185, "output_tok_s": 16.852339907542344, "ttft_p50_s": 5.0348457403015345, "ttft_p95_s": 5.060727500542998, "ttft_p99_s": 5.0612430898845195, "mean_request_tpot_ms": 20.131148656419526, "request_tpot_p95_ms": 20.240636005278997, "e2e_p95_s": 7.61073174267076, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.610717630758882, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c1", "file": "sglang/core/sglang-long-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 60.85527942283079, "output_tok_s": 16.826806313468847, "ttft_p50_s": 5.043643031967804, "ttft_p95_s": 5.0802262338344, "ttft_p99_s": 5.082711985455826, "mean_request_tpot_ms": 20.158018832054633, "request_tpot_p95_ms": 20.234935085553175, "e2e_p95_s": 7.631990188267082, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.631974293501116, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c32", "file": "sglang/core/sglang-long-c32-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 359.30970406206325, "output_tok_s": 22.799272904093353, "ttft_p50_s": 59.85346938553266, "ttft_p95_s": 163.02333678153803, "ttft_p99_s": 172.89447315484284, "mean_request_tpot_ms": 831.0106538892009, "request_tpot_p95_ms": 1292.964965114223, "e2e_p95_s": 285.8280833825702, "full_stream_e2e_available": true, "token_e2e_p95_s": 285.8280466715805, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c32", "file": "sglang/core/sglang-long-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 359.40180941997096, "output_tok_s": 22.79343004204918, "ttft_p50_s": 55.839272066019475, "ttft_p95_s": 154.68304573709608, "ttft_p99_s": 167.2163947788905, "mean_request_tpot_ms": 861.0626236304382, "request_tpot_p95_ms": 1331.2565517239636, "e2e_p95_s": 285.99859518483277, "full_stream_e2e_available": true, "token_e2e_p95_s": 285.9985818383051, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c32", "file": "sglang/core/sglang-long-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 359.57561472896487, "output_tok_s": 22.782412556465584, "ttft_p50_s": 56.55914409807883, "ttft_p95_s": 149.4713405372342, "ttft_p99_s": 166.18098039661996, "mean_request_tpot_ms": 874.415656850835, "request_tpot_p95_ms": 1330.0665863047127, "e2e_p95_s": 286.04614139513575, "full_stream_e2e_available": true, "token_e2e_p95_s": 286.0461279868614, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c8", "file": "sglang/core/sglang-long-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 104.4472956219688, "output_tok_s": 19.60797536982122, "ttft_p50_s": 34.743653156096116, "ttft_p95_s": 49.71797822928056, "ttft_p99_s": 49.77683453960344, "mean_request_tpot_ms": 150.17712575784066, "request_tpot_p95_ms": 295.2013193734181, "e2e_p95_s": 52.26222040166613, "full_stream_e2e_available": true, "token_e2e_p95_s": 52.26221369160339, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c8", "file": "sglang/core/sglang-long-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 104.52083779592067, "output_tok_s": 19.5941789521317, "ttft_p50_s": 33.026117810979486, "ttft_p95_s": 47.06338689045515, "ttft_p99_s": 49.23594299990218, "mean_request_tpot_ms": 166.8781289793221, "request_tpot_p95_ms": 312.2237857603505, "e2e_p95_s": 52.29953672760166, "full_stream_e2e_available": true, "token_e2e_p95_s": 52.29952943511307, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "long-c8", "file": "sglang/core/sglang-long-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 104.67420241702348, "output_tok_s": 19.565470313695247, "ttft_p50_s": 33.08065340737812, "ttft_p95_s": 49.79768008226529, "ttft_p99_s": 49.9298652020283, "mean_request_tpot_ms": 160.45640439127837, "request_tpot_p95_ms": 312.1596708633445, "e2e_p95_s": 52.48119758744724, "full_stream_e2e_available": true, "token_e2e_p95_s": 52.48118782485835, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c1", "file": "sglang/core/sglang-prefill-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 32.86987294675782, "output_tok_s": 31.15314749341019, "ttft_p50_s": 1.5508908156771213, "ttft_p95_s": 1.5602353524882346, "ttft_p99_s": 1.5617607897613197, "mean_request_tpot_ms": 20.15396488388619, "request_tpot_p95_ms": 20.211176678545655, "e2e_p95_s": 4.117015044391155, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.116997827100567, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.24338396479226712, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.24338396479226712, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.24338396479226712, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c1", "file": "sglang/core/sglang-prefill-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 32.88819012604654, "output_tok_s": 31.13579665148616, "ttft_p50_s": 1.5451768508646637, "ttft_p95_s": 1.5674305916298181, "ttft_p99_s": 1.5705121257249266, "mean_request_tpot_ms": 20.1766064342874, "request_tpot_p95_ms": 20.229295031790894, "e2e_p95_s": 4.120126041560434, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.120110503840261, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.24324841133973563, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.24324841133973563, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.24324841133973563, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c1", "file": "sglang/core/sglang-prefill-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 32.94149047927931, "output_tok_s": 31.085417966867993, "ttft_p50_s": 1.5663332724943757, "ttft_p95_s": 1.5766864896053447, "ttft_p99_s": 1.5780721961008384, "mean_request_tpot_ms": 20.098928373130992, "request_tpot_p95_ms": 20.191151162015466, "e2e_p95_s": 4.1264671427896245, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.126452111685649, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.2428548278661562, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.2428548278661562, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.2428548278661562, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c32", "file": "sglang/core/sglang-prefill-c32-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 106.73568693222478, "output_tok_s": 76.75033754363497, "ttft_p50_s": 13.773834059014916, "ttft_p95_s": 47.97562206678556, "ttft_p99_s": 50.63721838030964, "mean_request_tpot_ms": 255.7715372478967, "request_tpot_p95_ms": 370.4678177221231, "e2e_p95_s": 86.00287257365414, "full_stream_e2e_available": true, "token_e2e_p95_s": 86.00285777836105, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c32", "file": "sglang/core/sglang-prefill-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 106.67981973476708, "output_tok_s": 76.79053095859533, "ttft_p50_s": 13.699585764668882, "ttft_p95_s": 49.28513757132457, "ttft_p99_s": 50.67829830430448, "mean_request_tpot_ms": 254.95234315522387, "request_tpot_p95_ms": 381.2629947160172, "e2e_p95_s": 85.95466712096238, "full_stream_e2e_available": true, "token_e2e_p95_s": 82.07828260697424, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c32", "file": "sglang/core/sglang-prefill-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 106.58817696105689, "output_tok_s": 76.85655420293972, "ttft_p50_s": 13.702325479127467, "ttft_p95_s": 48.04473857576957, "ttft_p99_s": 50.73546406908426, "mean_request_tpot_ms": 255.60579977728614, "request_tpot_p95_ms": 369.58377112792147, "e2e_p95_s": 85.83662466031963, "full_stream_e2e_available": true, "token_e2e_p95_s": 85.83661192541474, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c8", "file": "sglang/core/sglang-prefill-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 32.216130409389734, "output_tok_s": 63.570639116952684, "ttft_p50_s": 9.04577746707946, "ttft_p95_s": 13.624190002330579, "ttft_p99_s": 13.635663557355292, "mean_request_tpot_ms": 52.797309879220904, "request_tpot_p95_ms": 103.39530853565607, "e2e_p95_s": 16.243461400270462, "full_stream_e2e_available": true, "token_e2e_p95_s": 16.243452910566702, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c8", "file": "sglang/core/sglang-prefill-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 32.181164343841374, "output_tok_s": 63.63971104705953, "ttft_p50_s": 9.00132921547629, "ttft_p95_s": 13.65014299878385, "ttft_p99_s": 13.650462930160575, "mean_request_tpot_ms": 51.05749745857829, "request_tpot_p95_ms": 95.14606988523155, "e2e_p95_s": 16.264968653908, "full_stream_e2e_available": true, "token_e2e_p95_s": 16.264955582097173, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "prefill-c8", "file": "sglang/core/sglang-prefill-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 32.12950370600447, "output_tok_s": 63.74203656364797, "ttft_p50_s": 8.981774493353441, "ttft_p95_s": 13.690121321124025, "ttft_p99_s": 13.69111354316119, "mean_request_tpot_ms": 45.42219813630777, "request_tpot_p95_ms": 70.54234464381507, "e2e_p95_s": 16.188612060504965, "full_stream_e2e_available": true, "token_e2e_p95_s": 16.18860459048301, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "core", "case": "short-c1", "file": "sglang/core/sglang-short-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 22.450983696151525, "output_tok_s": 45.61047363708748, "ttft_p50_s": 0.23741855542175472, "ttft_p95_s": 0.3275388656649738, "ttft_p99_s": 0.36201195550151166, "mean_request_tpot_ms": 20.114534456598182, "request_tpot_p95_ms": 20.201972841769162, "e2e_p95_s": 2.87286340710707, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.8728486706735565, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.35633182528974594, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.35633182528974594, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.35633182528974594, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.35633182528974594, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "short-c1", "file": "sglang/core/sglang-short-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 22.331683283206075, "output_tok_s": 45.8541341023796, "ttft_p50_s": 0.22814342915080488, "ttft_p95_s": 0.22918487282004207, "ttft_p99_s": 0.22945700164418667, "mean_request_tpot_ms": 20.190707579306434, "request_tpot_p95_ms": 20.21817000817478, "e2e_p95_s": 2.7959118694998324, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.7958978100912644, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.3582354226748406, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.3582354226748406, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.3582354226748406, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.3582354226748406, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "short-c1", "file": "sglang/core/sglang-short-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 22.32139705820009, "output_tok_s": 45.87526476636097, "ttft_p50_s": 0.23785124602727592, "ttft_p95_s": 0.24802749580703676, "ttft_p99_s": 0.2481107349973172, "mean_request_tpot_ms": 20.10400751999792, "request_tpot_p95_ms": 20.18155817375115, "e2e_p95_s": 2.7924669726751743, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.792451668367721, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.3584005059871951, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.3584005059871951, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.3584005059871951, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.3584005059871951, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "short-c32", "file": "sglang/core/sglang-short-c32-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 18.407831029966474, "output_tok_s": 445.02798763548407, "ttft_p50_s": 5.562234603567049, "ttft_p95_s": 5.69418488566298, "ttft_p99_s": 5.699509169100784, "mean_request_tpot_ms": 33.53025147197063, "request_tpot_p95_ms": 49.65762663412634, "e2e_p95_s": 9.245797717547976, "full_stream_e2e_available": true, "token_e2e_p95_s": 9.245790195651352, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.4345976441752774, "pass_fraction_ttft3s_tpot50ms": 0.125, "goodput_ttft5s_tpot50ms_rps": 1.032169404916284, "pass_fraction_ttft5s_tpot50ms": 0.296875 }, { "engine": "sglang", "stage": "core", "case": "short-c32", "file": "sglang/core/sglang-short-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 18.435757387895137, "output_tok_s": 444.35386231426776, "ttft_p50_s": 5.558989139739424, "ttft_p95_s": 5.686285282112658, "ttft_p99_s": 5.691275138184428, "mean_request_tpot_ms": 35.790722174950375, "request_tpot_p95_ms": 49.73272361836624, "e2e_p95_s": 9.24682450077962, "full_stream_e2e_available": true, "token_e2e_p95_s": 9.246816555014812, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.7593938076659849, "pass_fraction_ttft3s_tpot50ms": 0.21875, "goodput_ttft5s_tpot50ms_rps": 1.4103027856654005, "pass_fraction_ttft5s_tpot50ms": 0.40625 }, { "engine": "sglang", "stage": "core", "case": "short-c32", "file": "sglang/core/sglang-short-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 18.356669010128826, "output_tok_s": 446.26832871910614, "ttft_p50_s": 5.556830302579328, "ttft_p95_s": 5.677037351951003, "ttft_p99_s": 5.678323774798773, "mean_request_tpot_ms": 33.86784914968532, "request_tpot_p95_ms": 49.3854750380448, "e2e_p95_s": 9.198331869463436, "full_stream_e2e_available": true, "token_e2e_p95_s": 9.198155375174247, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.5447611434559402, "pass_fraction_ttft3s_tpot50ms": 0.15625, "goodput_ttft5s_tpot50ms_rps": 1.0895222869118804, "pass_fraction_ttft5s_tpot50ms": 0.3125 }, { "engine": "sglang", "stage": "core", "case": "short-c8", "file": "sglang/core/sglang-short-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 8.044648486189544, "output_tok_s": 254.57917813511114, "ttft_p50_s": 1.4450097838416696, "ttft_p95_s": 1.4619857898214832, "ttft_p99_s": 1.4620826938422398, "mean_request_tpot_ms": 20.33349970198882, "request_tpot_p95_ms": 20.864257995061227, "e2e_p95_s": 4.060795152909122, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.060784610686824, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 1.9888998291805557, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.9888998291805557, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.9888998291805557, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "short-c8", "file": "sglang/core/sglang-short-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 8.052535342052579, "output_tok_s": 254.32983687817853, "ttft_p50_s": 1.4499289565719664, "ttft_p95_s": 1.4576065525179729, "ttft_p99_s": 1.4663070224924013, "mean_request_tpot_ms": 20.347812032880714, "request_tpot_p95_ms": 20.913940094462706, "e2e_p95_s": 4.0520477092359215, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.052039989386685, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 1.9869518506107697, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.9869518506107697, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.9869518506107697, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "core", "case": "short-c8", "file": "sglang/core/sglang-short-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 7.970599716994911, "output_tok_s": 256.94427931605384, "ttft_p50_s": 1.4547245716676116, "ttft_p95_s": 1.4722298487322405, "ttft_p99_s": 1.4725899601588026, "mean_request_tpot_ms": 19.92731549965922, "request_tpot_p95_ms": 20.15691651324705, "e2e_p95_s": 4.0051197544671595, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.005113334627822, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 2.0073771821566706, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 2.0073771821566706, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 2.0073771821566706, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-prefill-c4", "file": "sglang/history/sglang-history-prefill-c4-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 20.974545828066766, "output_tok_s": 97.64216192273877, "ttft_p50_s": 2.8959481939673424, "ttft_p95_s": 2.961239076917991, "ttft_p99_s": 2.9613250489812346, "mean_request_tpot_ms": 19.496293524674353, "request_tpot_p95_ms": 24.878936004824936, "e2e_p95_s": 5.259710291633382, "full_stream_e2e_available": true, "token_e2e_p95_s": 5.2597036440856755, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.7628293900213966, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.7628293900213966, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-prefill-c4", "file": "sglang/history/sglang-history-prefill-c4-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 20.920063924975693, "output_tok_s": 97.89645038106066, "ttft_p50_s": 2.945808853255585, "ttft_p95_s": 2.9685806629713625, "ttft_p99_s": 2.9689016557764263, "mean_request_tpot_ms": 18.479667926861676, "request_tpot_p95_ms": 20.072155639411896, "e2e_p95_s": 5.253059729468077, "full_stream_e2e_available": true, "token_e2e_p95_s": 5.253049927647226, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.7648160186020364, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.7648160186020364, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-prefill-c4", "file": "sglang/history/sglang-history-prefill-c4-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 20.9350448991172, "output_tok_s": 97.82639635448601, "ttft_p50_s": 2.944739621365443, "ttft_p95_s": 2.987787603866309, "ttft_p99_s": 2.9882618631236255, "mean_request_tpot_ms": 19.281396660889595, "request_tpot_p95_ms": 24.71601175727308, "e2e_p95_s": 5.253828642191365, "full_stream_e2e_available": true, "token_e2e_p95_s": 5.253820827114396, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.7642687215194219, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.7642687215194219, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c1", "file": "sglang/history/sglang-history-short-c1-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 23.142396475654095, "output_tok_s": 44.24779434909659, "ttft_p50_s": 0.17891228897497058, "ttft_p95_s": 0.19670670328196138, "ttft_p99_s": 0.1976528701139614, "mean_request_tpot_ms": 20.06204863354224, "request_tpot_p95_ms": 20.15972875302569, "e2e_p95_s": 1.448630850063637, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.448617743095383, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.6913717867046342, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.6913717867046342, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.6913717867046342, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.6913717867046342, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c1", "file": "sglang/history/sglang-history-short-c1-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 23.110994894988835, "output_tok_s": 44.307915113686185, "ttft_p50_s": 0.17681982438080013, "ttft_p95_s": 0.18202978558838367, "ttft_p99_s": 0.19256259016692637, "mean_request_tpot_ms": 20.11087003103014, "request_tpot_p95_ms": 20.154927982874806, "e2e_p95_s": 1.4464875527191907, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.446473352611065, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.6923111736513466, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.6923111736513466, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.6923111736513466, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.6923111736513466, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c1", "file": "sglang/history/sglang-history-short-c1-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 23.15841848310083, "output_tok_s": 44.217181788438346, "ttft_p50_s": 0.1786382901482284, "ttft_p95_s": 0.19769891072064638, "ttft_p99_s": 0.1987123006954789, "mean_request_tpot_ms": 20.08726209446433, "request_tpot_p95_ms": 20.174001450724308, "e2e_p95_s": 1.4519307899754494, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.451914850389585, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.6908934654443492, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.6908934654443492, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.6908934654443492, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.6908934654443492, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c4", "file": "sglang/history/sglang-history-short-c4-r1.json", "requests": 32, "completed": 32, "failed": 0, "duration_s": 10.935391953215003, "output_tok_s": 187.2818101776305, "ttft_p50_s": 0.20385263045318425, "ttft_p95_s": 0.3560166540555656, "ttft_p99_s": 0.35640682296361775, "mean_request_tpot_ms": 18.126758095431363, "request_tpot_p95_ms": 18.308938431021357, "e2e_p95_s": 1.4843525264644994, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.4843437930801884, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 2.9262782840254764, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 2.9262782840254764, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 2.9262782840254764, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 2.9262782840254764, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c4", "file": "sglang/history/sglang-history-short-c4-r2.json", "requests": 32, "completed": 32, "failed": 0, "duration_s": 10.948048965074122, "output_tok_s": 187.0652941481555, "ttft_p50_s": 0.2042150830384344, "ttft_p95_s": 0.34221710709389297, "ttft_p99_s": 0.3427451497828588, "mean_request_tpot_ms": 18.174033627257728, "request_tpot_p95_ms": 18.377388413581585, "e2e_p95_s": 1.4957575111882762, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.4957493450725452, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 2.9228952210649295, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 2.9228952210649295, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 2.9228952210649295, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 2.9228952210649295, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c4", "file": "sglang/history/sglang-history-short-c4-r3.json", "requests": 32, "completed": 32, "failed": 0, "duration_s": 10.877106742933393, "output_tok_s": 188.28536378302425, "ttft_p50_s": 0.20470425207167864, "ttft_p95_s": 0.35757704444695265, "ttft_p99_s": 0.3580368464207277, "mean_request_tpot_ms": 18.008358969670972, "request_tpot_p95_ms": 18.2788164325295, "e2e_p95_s": 1.4784948154585436, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.4784845404326916, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 2.941958809109754, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 2.941958809109754, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 2.941958809109754, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 2.941958809109754, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c8", "file": "sglang/history/sglang-history-short-c8-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 13.35008389595896, "output_tok_s": 306.81455127333317, "ttft_p50_s": 0.38688521971926093, "ttft_p95_s": 0.40543246623128654, "ttft_p99_s": 0.41477755485568196, "mean_request_tpot_ms": 20.262917426104348, "request_tpot_p95_ms": 20.5253459679495, "e2e_p95_s": 1.6837959521450103, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.6837816983228548, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 4.793977363645831, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 4.793977363645831, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 4.793977363645831, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 4.793977363645831, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c8", "file": "sglang/history/sglang-history-short-c8-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 13.278860800899565, "output_tok_s": 308.4601956007039, "ttft_p50_s": 0.3880118289962411, "ttft_p95_s": 0.40664501341525466, "ttft_p99_s": 0.4114217031095177, "mean_request_tpot_ms": 20.11594226150053, "request_tpot_p95_ms": 20.384218969145817, "e2e_p95_s": 1.6704298709053547, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.6704242561012506, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 4.819690556260999, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 4.819690556260999, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 4.819690556260999, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 4.819690556260999, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "history", "case": "history-short-c8", "file": "sglang/history/sglang-history-short-c8-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 13.172682314179838, "output_tok_s": 310.94654090236645, "ttft_p50_s": 0.38987168832682073, "ttft_p95_s": 0.40931367964949456, "ttft_p99_s": 0.4145527708623558, "mean_request_tpot_ms": 19.876036265775348, "request_tpot_p95_ms": 20.473347432793133, "e2e_p95_s": 1.681407206552103, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.6813982895575463, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 4.858539701599476, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 4.858539701599476, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 4.858539701599476, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 4.858539701599476, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.5", "file": "sglang/slo/sglang-slo-0.5-r1.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 670.6628576330841, "output_tok_s": 38.17119094733828, "ttft_p50_s": 2.8322165876161307, "ttft_p95_s": 6.132679248508065, "ttft_p99_s": 7.581764451214112, "mean_request_tpot_ms": 41.2842315827555, "request_tpot_p95_ms": 101.55614597216893, "e2e_p95_s": 16.097032101731745, "full_stream_e2e_available": true, "token_e2e_p95_s": 16.097015924588774, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0984101016611065, "pass_fraction_ttft2s_tpot50ms": 0.33, "goodput_ttft3s_tpot50ms_rps": 0.13568665532061655, "pass_fraction_ttft3s_tpot50ms": 0.455, "goodput_ttft5s_tpot50ms_rps": 0.189364892590311, "pass_fraction_ttft5s_tpot50ms": 0.635 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.5", "file": "sglang/slo/sglang-slo-0.5-r2.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 670.8606446017511, "output_tok_s": 38.159937098705726, "ttft_p50_s": 2.9350759850349277, "ttft_p95_s": 5.455989311099984, "ttft_p99_s": 7.630933139948158, "mean_request_tpot_ms": 36.37927335485317, "request_tpot_p95_ms": 89.4282935553883, "e2e_p95_s": 14.162595759076066, "full_stream_e2e_available": true, "token_e2e_p95_s": 14.162583958706817, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0790029947746642, "pass_fraction_ttft2s_tpot50ms": 0.265, "goodput_ttft3s_tpot50ms_rps": 0.1356466514055555, "pass_fraction_ttft3s_tpot50ms": 0.455, "goodput_ttft5s_tpot50ms_rps": 0.20421528837979236, "pass_fraction_ttft5s_tpot50ms": 0.685 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.5", "file": "sglang/slo/sglang-slo-0.5-r3.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 670.863995433785, "output_tok_s": 38.15974649742065, "ttft_p50_s": 2.8334158062934875, "ttft_p95_s": 6.073622179799713, "ttft_p99_s": 7.493577100583352, "mean_request_tpot_ms": 38.822779644676665, "request_tpot_p95_ms": 79.32699507058369, "e2e_p95_s": 14.408885770454072, "full_stream_e2e_available": true, "token_e2e_p95_s": 14.408877224475145, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.09092752095088515, "pass_fraction_ttft2s_tpot50ms": 0.305, "goodput_ttft3s_tpot50ms_rps": 0.14309904936532744, "pass_fraction_ttft3s_tpot50ms": 0.48, "goodput_ttft5s_tpot50ms_rps": 0.18632688719443677, "pass_fraction_ttft5s_tpot50ms": 0.625 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.8", "file": "sglang/slo/sglang-slo-0.8-r1.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 424.1038222173229, "output_tok_s": 60.36257788519018, "ttft_p50_s": 4.452154421014711, "ttft_p95_s": 9.295561882923353, "ttft_p99_s": 10.69542880228254, "mean_request_tpot_ms": 91.73845005503995, "request_tpot_p95_ms": 203.0229543482401, "e2e_p95_s": 34.81513398047536, "full_stream_e2e_available": true, "token_e2e_p95_s": 34.81512695637065, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.04480035077416458, "pass_fraction_ttft2s_tpot50ms": 0.095, "goodput_ttft3s_tpot50ms_rps": 0.07781113555512796, "pass_fraction_ttft3s_tpot50ms": 0.165, "goodput_ttft5s_tpot50ms_rps": 0.13204313912385351, "pass_fraction_ttft5s_tpot50ms": 0.28 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.8", "file": "sglang/slo/sglang-slo-0.8-r2.json", "requests": 200, "completed": 199, "failed": 1, "duration_s": 421.00025494908914, "output_tok_s": 60.50352630565577, "ttft_p50_s": 3.5684973881579936, "ttft_p95_s": 7.354478139197454, "ttft_p99_s": 9.577546174917375, "mean_request_tpot_ms": 101.59122399524495, "request_tpot_p95_ms": 236.5114057117589, "e2e_p95_s": 35.4178314726334, "full_stream_e2e_available": true, "token_e2e_p95_s": 35.41782316383905, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.052256500421028064, "pass_fraction_ttft2s_tpot50ms": 0.11, "goodput_ttft3s_tpot50ms_rps": 0.07600945515785901, "pass_fraction_ttft3s_tpot50ms": 0.16, "goodput_ttft5s_tpot50ms_rps": 0.12351536463152088, "pass_fraction_ttft5s_tpot50ms": 0.26 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.8", "file": "sglang/slo/sglang-slo-0.8-r3.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 420.8270897367038, "output_tok_s": 60.8325856969355, "ttft_p50_s": 4.20940438634716, "ttft_p95_s": 8.746406201086938, "ttft_p99_s": 9.855795736550347, "mean_request_tpot_ms": 130.41608952281, "request_tpot_p95_ms": 358.6681993546212, "e2e_p95_s": 50.654682853701516, "full_stream_e2e_available": true, "token_e2e_p95_s": 50.65467058294452, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.028515274545438517, "pass_fraction_ttft2s_tpot50ms": 0.06, "goodput_ttft3s_tpot50ms_rps": 0.05940682196966358, "pass_fraction_ttft3s_tpot50ms": 0.125, "goodput_ttft5s_tpot50ms_rps": 0.11168482530296753, "pass_fraction_ttft5s_tpot50ms": 0.235 }, { "engine": "sglang", "stage": "slo", "case": "slo-1.1", "file": "sglang/slo/sglang-slo-1.1-r1.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 336.01218932820484, "output_tok_s": 76.18771227074392, "ttft_p50_s": 13.487367060035467, "ttft_p95_s": 26.138306704582646, "ttft_p99_s": 31.07993468972388, "mean_request_tpot_ms": 325.58183168469907, "request_tpot_p95_ms": 372.6496747401578, "e2e_p95_s": 69.1028537699487, "full_stream_e2e_available": true, "token_e2e_p95_s": 69.1028338674223, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.005952165021151868, "pass_fraction_ttft2s_tpot50ms": 0.01, "goodput_ttft3s_tpot50ms_rps": 0.005952165021151868, "pass_fraction_ttft3s_tpot50ms": 0.01, "goodput_ttft5s_tpot50ms_rps": 0.005952165021151868, "pass_fraction_ttft5s_tpot50ms": 0.01 }, { "engine": "sglang", "stage": "slo", "case": "slo-1.1", "file": "sglang/slo/sglang-slo-1.1-r2.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 328.00836237799376, "output_tok_s": 78.04679068059491, "ttft_p50_s": 20.883924887515604, "ttft_p95_s": 30.699214355298317, "ttft_p99_s": 34.49449041138868, "mean_request_tpot_ms": 321.7556111554718, "request_tpot_p95_ms": 371.19225541305707, "e2e_p95_s": 75.24056903971358, "full_stream_e2e_available": true, "token_e2e_p95_s": 75.24055977379902, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "sglang", "stage": "slo", "case": "slo-1.1", "file": "sglang/slo/sglang-slo-1.1-r3.json", "requests": 200, "completed": 200, "failed": 0, "duration_s": 331.2289768769406, "output_tok_s": 77.28792402577449, "ttft_p50_s": 12.170083199627697, "ttft_p95_s": 25.994827236677512, "ttft_p99_s": 28.19680349758826, "mean_request_tpot_ms": 302.259456734338, "request_tpot_p95_ms": 371.23003438276453, "e2e_p95_s": 67.82771018347702, "full_stream_e2e_available": true, "token_e2e_p95_s": 67.82770192525349, "small_tail_sample": false, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.009057178596770448, "pass_fraction_ttft5s_tpot50ms": 0.015 }, { "engine": "vllm", "stage": "core", "case": "decode-c1", "file": "vllm/core/vllm-decode-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 35.59430730016902, "output_tok_s": 115.07458104067528, "ttft_p50_s": 0.19190677837468684, "ttft_p95_s": 0.19711574090179057, "ttft_p99_s": 0.19751932978164405, "mean_request_tpot_ms": 8.328708259155519, "request_tpot_p95_ms": 8.346501350860294, "e2e_p95_s": 4.460156888468191, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.46014322580304, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.2247550410950689, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.2247550410950689, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.2247550410950689, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.2247550410950689, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c1", "file": "vllm/core/vllm-decode-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 35.565328379161656, "output_tok_s": 115.16834475229864, "ttft_p50_s": 0.19066837057471275, "ttft_p95_s": 0.1921028696000576, "ttft_p99_s": 0.1922522659599781, "mean_request_tpot_ms": 8.323614110572375, "request_tpot_p95_ms": 8.335076634537014, "e2e_p95_s": 4.452694016112946, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.449745018221438, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.22493817334433328, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.22493817334433328, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.22493817334433328, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.22493817334433328, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c1", "file": "vllm/core/vllm-decode-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 35.554525919258595, "output_tok_s": 115.2033361182112, "ttft_p50_s": 0.19330018269829452, "ttft_p95_s": 0.19877633501309902, "ttft_p99_s": 0.19933682391885668, "mean_request_tpot_ms": 8.31298277496475, "request_tpot_p95_ms": 8.330485974885013, "e2e_p95_s": 4.453205332998186, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.452945414371788, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.22500651585588125, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.22500651585588125, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.22500651585588125, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.22500651585588125, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c32", "file": "vllm/core/vllm-decode-c32-r1.json", "requests": 64, "completed": 63, "failed": 1, "duration_s": 26.790218219161034, "output_tok_s": 1204.0215475710347, "ttft_p50_s": 3.157293044961989, "ttft_p95_s": 4.451026804978028, "ttft_p99_s": 4.455186772681773, "mean_request_tpot_ms": 20.705067403437475, "request_tpot_p95_ms": 24.071565929093122, "e2e_p95_s": 14.41589999361895, "full_stream_e2e_available": true, "token_e2e_p95_s": 14.415889986651019, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.07465411381268817, "pass_fraction_ttft1s_tpot50ms": 0.03125, "goodput_ttft2s_tpot50ms_rps": 0.5225787966888172, "pass_fraction_ttft2s_tpot50ms": 0.21875, "goodput_ttft3s_tpot50ms_rps": 1.0078305364712903, "pass_fraction_ttft3s_tpot50ms": 0.421875, "goodput_ttft5s_tpot50ms_rps": 2.351604585099677, "pass_fraction_ttft5s_tpot50ms": 0.984375 }, { "engine": "vllm", "stage": "core", "case": "decode-c32", "file": "vllm/core/vllm-decode-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 26.870982750784606, "output_tok_s": 1219.4567018224598, "ttft_p50_s": 3.166381268762052, "ttft_p95_s": 4.45758340882603, "ttft_p99_s": 4.464165972806513, "mean_request_tpot_ms": 20.730916750290973, "request_tpot_p95_ms": 23.859204417422646, "e2e_p95_s": 14.606841707741841, "full_stream_e2e_available": true, "token_e2e_p95_s": 14.606417830148711, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.07442973033584349, "pass_fraction_ttft1s_tpot50ms": 0.03125, "goodput_ttft2s_tpot50ms_rps": 0.5954378426867479, "pass_fraction_ttft2s_tpot50ms": 0.25, "goodput_ttft3s_tpot50ms_rps": 1.0420162247018088, "pass_fraction_ttft3s_tpot50ms": 0.4375, "goodput_ttft5s_tpot50ms_rps": 2.3817513707469917, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c32", "file": "vllm/core/vllm-decode-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 26.45276293484494, "output_tok_s": 1238.736387601928, "ttft_p50_s": 3.149889288470149, "ttft_p95_s": 4.458423188654706, "ttft_p99_s": 4.464812635201961, "mean_request_tpot_ms": 19.836520195827358, "request_tpot_p95_ms": 23.43808249789135, "e2e_p95_s": 14.407223387551493, "full_stream_e2e_available": true, "token_e2e_p95_s": 14.40496928638313, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.07560646896984424, "pass_fraction_ttft1s_tpot50ms": 0.03125, "goodput_ttft2s_tpot50ms_rps": 0.4914420483039876, "pass_fraction_ttft2s_tpot50ms": 0.203125, "goodput_ttft3s_tpot50ms_rps": 0.9828840966079752, "pass_fraction_ttft3s_tpot50ms": 0.40625, "goodput_ttft5s_tpot50ms_rps": 2.4194070070350158, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c8", "file": "vllm/core/vllm-decode-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 13.347630279604346, "output_tok_s": 613.7419023748109, "ttft_p50_s": 1.1495696166530252, "ttft_p95_s": 1.1766843994846568, "ttft_p99_s": 1.1767567037371918, "mean_request_tpot_ms": 11.01532786928068, "request_tpot_p95_ms": 12.596697088854055, "e2e_p95_s": 6.695874874596484, "full_stream_e2e_available": true, "token_e2e_p95_s": 6.695437211543322, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.14983933163447533, "pass_fraction_ttft1s_tpot50ms": 0.125, "goodput_ttft2s_tpot50ms_rps": 1.1987146530758026, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.1987146530758026, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.1987146530758026, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c8", "file": "vllm/core/vllm-decode-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 13.658397993072867, "output_tok_s": 599.7775144753241, "ttft_p50_s": 1.1608486876357347, "ttft_p95_s": 1.175377813517116, "ttft_p99_s": 1.1757533638505264, "mean_request_tpot_ms": 11.307909608355663, "request_tpot_p95_ms": 12.894964216499353, "e2e_p95_s": 6.8920627580955625, "full_stream_e2e_available": true, "token_e2e_p95_s": 6.8920557564124465, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.14643005724495217, "pass_fraction_ttft1s_tpot50ms": 0.125, "goodput_ttft2s_tpot50ms_rps": 1.1714404579596174, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.1714404579596174, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.1714404579596174, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "decode-c8", "file": "vllm/core/vllm-decode-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 13.203079438768327, "output_tok_s": 620.461312680264, "ttft_p50_s": 1.15875782398507, "ttft_p95_s": 1.1833921261131763, "ttft_p99_s": 1.1837312445044517, "mean_request_tpot_ms": 10.864094372735305, "request_tpot_p95_ms": 12.47456373529301, "e2e_p95_s": 6.642328674555756, "full_stream_e2e_available": true, "token_e2e_p95_s": 6.642321534571238, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.15147981266608007, "pass_fraction_ttft1s_tpot50ms": 0.125, "goodput_ttft2s_tpot50ms_rps": 1.2118385013286406, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.2118385013286406, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.2118385013286406, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "long-c1", "file": "vllm/core/vllm-long-c1-r1.json", "requests": 8, "completed": 7, "failed": 1, "duration_s": 36.827968151308596, "output_tok_s": 24.329335691797123, "ttft_p50_s": 3.583831065800041, "ttft_p95_s": 3.6022905338555575, "ttft_p99_s": 3.6047770661115646, "mean_request_tpot_ms": 8.018291399251623, "request_tpot_p95_ms": 8.055036295733348, "e2e_p95_s": 4.620717612840235, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.620705521013588, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.19007293509216502, "pass_fraction_ttft5s_tpot50ms": 0.875 }, { "engine": "vllm", "stage": "core", "case": "long-c1", "file": "vllm/core/vllm-long-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 36.83863146509975, "output_tok_s": 27.796906651380873, "ttft_p50_s": 3.58167616953142, "ttft_p95_s": 3.603658485296182, "ttft_p99_s": 3.604037638953887, "mean_request_tpot_ms": 8.028949084728369, "request_tpot_p95_ms": 8.074000377141994, "e2e_p95_s": 4.62177074230276, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.621756793395616, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.21716333321391307, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "long-c1", "file": "vllm/core/vllm-long-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 36.928813208360225, "output_tok_s": 27.729025414988943, "ttft_p50_s": 3.591873028082773, "ttft_p95_s": 3.6184175213100387, "ttft_p99_s": 3.6205735209072007, "mean_request_tpot_ms": 8.04354656738867, "request_tpot_p95_ms": 8.071679000808734, "e2e_p95_s": 4.637787795555778, "full_stream_e2e_available": true, "token_e2e_p95_s": 4.637775349011645, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.21663301105460112, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "long-c32", "file": "vllm/core/vllm-long-c32-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 225.69256412610412, "output_tok_s": 36.29716393944985, "ttft_p50_s": 12.826002848567441, "ttft_p95_s": 102.62430188418364, "ttft_p99_s": 110.58425515964628, "mean_request_tpot_ms": 606.3053500428374, "request_tpot_p95_ms": 806.9753817038786, "e2e_p95_s": 202.00522467342668, "full_stream_e2e_available": true, "token_e2e_p95_s": 202.00521362377327, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "vllm", "stage": "core", "case": "long-c32", "file": "vllm/core/vllm-long-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 225.51247313199565, "output_tok_s": 36.32615032874526, "ttft_p50_s": 12.7694447201211, "ttft_p95_s": 102.4013492900878, "ttft_p99_s": 110.57750302649569, "mean_request_tpot_ms": 605.6100434321621, "request_tpot_p95_ms": 814.9649418639311, "e2e_p95_s": 201.93572719257787, "full_stream_e2e_available": true, "token_e2e_p95_s": 201.935716194869, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "vllm", "stage": "core", "case": "long-c32", "file": "vllm/core/vllm-long-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 225.69378328276798, "output_tok_s": 36.29696786878875, "ttft_p50_s": 12.755361603340134, "ttft_p95_s": 101.54811689418273, "ttft_p99_s": 109.9483337897528, "mean_request_tpot_ms": 605.477783751058, "request_tpot_p95_ms": 815.346607984425, "e2e_p95_s": 201.99029101619962, "full_stream_e2e_available": true, "token_e2e_p95_s": 201.9902782414341, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "vllm", "stage": "core", "case": "long-c8", "file": "vllm/core/vllm-long-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 58.33737390488386, "output_tok_s": 35.10613973366645, "ttft_p50_s": 15.03520306199789, "ttft_p95_s": 27.004446445731446, "ttft_p99_s": 27.75292531750165, "mean_request_tpot_ms": 110.7999059622814, "request_tpot_p95_ms": 191.38701851608042, "e2e_p95_s": 40.41505611210596, "full_stream_e2e_available": true, "token_e2e_p95_s": 40.4150417977944, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "vllm", "stage": "core", "case": "long-c8", "file": "vllm/core/vllm-long-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 58.37085584271699, "output_tok_s": 35.08600260236773, "ttft_p50_s": 15.075254934839904, "ttft_p95_s": 26.12782807671465, "ttft_p99_s": 27.577070519374683, "mean_request_tpot_ms": 110.99268808593335, "request_tpot_p95_ms": 187.41733556994654, "e2e_p95_s": 40.40727836661972, "full_stream_e2e_available": true, "token_e2e_p95_s": 40.40726501669269, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "vllm", "stage": "core", "case": "long-c8", "file": "vllm/core/vllm-long-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 58.49688603915274, "output_tok_s": 35.01041061620351, "ttft_p50_s": 13.912773853866383, "ttft_p95_s": 26.227974120760337, "ttft_p99_s": 27.676802020194007, "mean_request_tpot_ms": 115.01870134535547, "request_tpot_p95_ms": 185.26405438608452, "e2e_p95_s": 41.68350547377486, "full_stream_e2e_available": true, "token_e2e_p95_s": 41.683492008945905, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.0, "pass_fraction_ttft5s_tpot50ms": 0.0 }, { "engine": "vllm", "stage": "core", "case": "prefill-c1", "file": "vllm/core/vllm-prefill-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 17.731128724757582, "output_tok_s": 57.751540575655035, "ttft_p50_s": 1.1863052379339933, "ttft_p95_s": 1.1974870281293988, "ttft_p99_s": 1.199063700772822, "mean_request_tpot_ms": 8.08807628872753, "request_tpot_p95_ms": 8.112255322534267, "e2e_p95_s": 2.2235585060669107, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.2235430970555172, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.45118391074730496, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.45118391074730496, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.45118391074730496, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "prefill-c1", "file": "vllm/core/vllm-prefill-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 17.745021332986653, "output_tok_s": 57.70632679355879, "ttft_p50_s": 1.188654009718448, "ttft_p95_s": 1.1974325044313445, "ttft_p99_s": 1.1991276847990229, "mean_request_tpot_ms": 8.082117058069072, "request_tpot_p95_ms": 8.12042399536906, "e2e_p95_s": 2.2256309275515376, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.2256072521675376, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.450830678074678, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.450830678074678, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.450830678074678, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "prefill-c1", "file": "vllm/core/vllm-prefill-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 17.77736893389374, "output_tok_s": 57.601324684648674, "ttft_p50_s": 1.1963508452754468, "ttft_p95_s": 1.2043369268300013, "ttft_p99_s": 1.2055022359872236, "mean_request_tpot_ms": 8.07482703207574, "request_tpot_p95_ms": 8.106263262571723, "e2e_p95_s": 2.2309355789329857, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.2309224202297626, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.45001034909881776, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.45001034909881776, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.45001034909881776, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "prefill-c32", "file": "vllm/core/vllm-prefill-c32-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 76.99626041436568, "output_tok_s": 106.39477756339926, "ttft_p50_s": 6.87379388930276, "ttft_p95_s": 34.24326259230729, "ttft_p99_s": 36.54014229060151, "mean_request_tpot_ms": 197.8010022137351, "request_tpot_p95_ms": 272.0928671459967, "e2e_p95_s": 66.62914763563312, "full_stream_e2e_available": true, "token_e2e_p95_s": 66.62913638621103, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.012987643745532136, "pass_fraction_ttft5s_tpot50ms": 0.015625 }, { "engine": "vllm", "stage": "core", "case": "prefill-c32", "file": "vllm/core/vllm-prefill-c32-r2.json", "requests": 64, "completed": 62, "failed": 2, "duration_s": 76.97658709110692, "output_tok_s": 103.09628290751856, "ttft_p50_s": 6.843052109936252, "ttft_p95_s": 34.18993107452988, "ttft_p99_s": 36.56234881227836, "mean_request_tpot_ms": 200.64208972174842, "request_tpot_p95_ms": 265.31210875204636, "e2e_p95_s": 66.80095443504396, "full_stream_e2e_available": true, "token_e2e_p95_s": 66.80094129948411, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.01299096306798369, "pass_fraction_ttft5s_tpot50ms": 0.015625 }, { "engine": "vllm", "stage": "core", "case": "prefill-c32", "file": "vllm/core/vllm-prefill-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 76.85720585286617, "output_tok_s": 106.58727323086131, "ttft_p50_s": 6.838809908367693, "ttft_p95_s": 34.11350479694083, "ttft_p99_s": 36.597290865331885, "mean_request_tpot_ms": 196.82698256404214, "request_tpot_p95_ms": 264.4716606458913, "e2e_p95_s": 66.55927178401033, "full_stream_e2e_available": true, "token_e2e_p95_s": 66.38949690558947, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.013011141751814125, "pass_fraction_ttft5s_tpot50ms": 0.015625 }, { "engine": "vllm", "stage": "core", "case": "prefill-c8", "file": "vllm/core/vllm-prefill-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 20.967769301962107, "output_tok_s": 97.67371867298986, "ttft_p50_s": 5.720112528884783, "ttft_p95_s": 9.193929874454625, "ttft_p99_s": 9.19400312139187, "mean_request_tpot_ms": 36.01764578337408, "request_tpot_p95_ms": 72.57850588607332, "e2e_p95_s": 13.102535630343482, "full_stream_e2e_available": true, "token_e2e_p95_s": 13.102524142945185, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.19076898178318333, "pass_fraction_ttft5s_tpot50ms": 0.25 }, { "engine": "vllm", "stage": "core", "case": "prefill-c8", "file": "vllm/core/vllm-prefill-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 20.97699081012979, "output_tok_s": 97.63078119913274, "ttft_p50_s": 6.88912028959021, "ttft_p95_s": 9.190890473430045, "ttft_p99_s": 9.202502168924548, "mean_request_tpot_ms": 35.409696055273706, "request_tpot_p95_ms": 72.7217607172646, "e2e_p95_s": 13.084489567903802, "full_stream_e2e_available": true, "token_e2e_p95_s": 13.08447510865517, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.047671279882389034, "pass_fraction_ttft5s_tpot50ms": 0.0625 }, { "engine": "vllm", "stage": "core", "case": "prefill-c8", "file": "vllm/core/vllm-prefill-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 21.000171349849552, "output_tok_s": 97.52301378315526, "ttft_p50_s": 5.814751866972074, "ttft_p95_s": 9.252861680695787, "ttft_p99_s": 9.254572435515001, "mean_request_tpot_ms": 34.80779710473258, "request_tpot_p95_ms": 66.089752019295, "e2e_p95_s": 13.09378808317706, "full_stream_e2e_available": true, "token_e2e_p95_s": 13.09377500822302, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.0, "pass_fraction_ttft2s_tpot50ms": 0.0, "goodput_ttft3s_tpot50ms_rps": 0.0, "pass_fraction_ttft3s_tpot50ms": 0.0, "goodput_ttft5s_tpot50ms_rps": 0.04761865907380628, "pass_fraction_ttft5s_tpot50ms": 0.0625 }, { "engine": "vllm", "stage": "core", "case": "short-c1", "file": "vllm/core/vllm-short-c1-r1.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 9.76503406604752, "output_tok_s": 104.86394548897593, "ttft_p50_s": 0.18833609856665134, "ttft_p95_s": 0.19438648449722679, "ttft_p99_s": 0.19495330647099762, "mean_request_tpot_ms": 8.100336528471196, "request_tpot_p95_ms": 8.145341993030893, "e2e_p95_s": 1.2227639746386558, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.2226250279461965, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.8192495741326244, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.8192495741326244, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.8192495741326244, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.8192495741326244, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "short-c1", "file": "vllm/core/vllm-short-c1-r2.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 9.835794522892684, "output_tok_s": 104.10953559640285, "ttft_p50_s": 0.1906620366498828, "ttft_p95_s": 0.2148163946578279, "ttft_p99_s": 0.22470106695313005, "mean_request_tpot_ms": 8.1426015955132, "request_tpot_p95_ms": 8.179851815537146, "e2e_p95_s": 1.2495283655822278, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.2495161704020574, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.8133557468468973, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.8133557468468973, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.8133557468468973, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.8133557468468973, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "short-c1", "file": "vllm/core/vllm-short-c1-r3.json", "requests": 8, "completed": 8, "failed": 0, "duration_s": 9.796815575100482, "output_tok_s": 104.52376000652613, "ttft_p50_s": 0.19477075501345098, "ttft_p95_s": 0.19803353070747107, "ttft_p99_s": 0.1984991964837536, "mean_request_tpot_ms": 8.10413253557773, "request_tpot_p95_ms": 8.153018495617948, "e2e_p95_s": 1.2312646110076457, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.231247127871029, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.8165918750509854, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 0.8165918750509854, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 0.8165918750509854, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 0.8165918750509854, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "short-c32", "file": "vllm/core/vllm-short-c32-r1.json", "requests": 64, "completed": 63, "failed": 1, "duration_s": 13.37099234201014, "output_tok_s": 603.0965984973177, "ttft_p50_s": 3.161648652050644, "ttft_p95_s": 4.455559488805011, "ttft_p99_s": 4.456916702017188, "mean_request_tpot_ms": 30.00502976386484, "request_tpot_p95_ms": 43.429493129356054, "e2e_p95_s": 7.7787515456788245, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.778359147068113, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.8974651763352943, "pass_fraction_ttft2s_tpot50ms": 0.1875, "goodput_ttft3s_tpot50ms_rps": 1.9445078820598043, "pass_fraction_ttft3s_tpot50ms": 0.40625, "goodput_ttft5s_tpot50ms_rps": 4.5621146463710796, "pass_fraction_ttft5s_tpot50ms": 0.953125 }, { "engine": "vllm", "stage": "core", "case": "short-c32", "file": "vllm/core/vllm-short-c32-r2.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 13.35151755809784, "output_tok_s": 613.5632121482298, "ttft_p50_s": 3.163980494486168, "ttft_p95_s": 4.462674967199564, "ttft_p99_s": 4.468984002377838, "mean_request_tpot_ms": 30.372993320771624, "request_tpot_p95_ms": 42.97487018354912, "e2e_p95_s": 7.81973541516345, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.819547306071035, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 1.048569942636135, "pass_fraction_ttft2s_tpot50ms": 0.21875, "goodput_ttft3s_tpot50ms_rps": 1.9473441791813935, "pass_fraction_ttft3s_tpot50ms": 0.40625, "goodput_ttft5s_tpot50ms_rps": 4.643666888817169, "pass_fraction_ttft5s_tpot50ms": 0.96875 }, { "engine": "vllm", "stage": "core", "case": "short-c32", "file": "vllm/core/vllm-short-c32-r3.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 13.206284898798913, "output_tok_s": 620.3107128746743, "ttft_p50_s": 3.1564872628077865, "ttft_p95_s": 4.46526139874477, "ttft_p99_s": 4.46710479256697, "mean_request_tpot_ms": 28.314974083034432, "request_tpot_p95_ms": 42.388605019517655, "e2e_p95_s": 7.752677880530245, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.752152443886734, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.15144304513541854, "pass_fraction_ttft1s_tpot50ms": 0.03125, "goodput_ttft2s_tpot50ms_rps": 0.9086582708125113, "pass_fraction_ttft2s_tpot50ms": 0.1875, "goodput_ttft3s_tpot50ms_rps": 1.968759586760441, "pass_fraction_ttft3s_tpot50ms": 0.40625, "goodput_ttft5s_tpot50ms_rps": 4.846177444333393, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "short-c8", "file": "vllm/core/vllm-short-c8-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 5.066207236144692, "output_tok_s": 404.24718226854407, "ttft_p50_s": 1.1510341125540435, "ttft_p95_s": 1.1754794579464942, "ttft_p99_s": 1.1758746395353228, "mean_request_tpot_ms": 11.710464420194892, "request_tpot_p95_ms": 18.21562979465688, "e2e_p95_s": 2.534013267373666, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.534005962777883, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.39477263893412506, "pass_fraction_ttft1s_tpot50ms": 0.125, "goodput_ttft2s_tpot50ms_rps": 3.1581811114730005, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 3.1581811114730005, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 3.1581811114730005, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "short-c8", "file": "vllm/core/vllm-short-c8-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 5.117432889994234, "output_tok_s": 400.2006560758841, "ttft_p50_s": 1.1633586212992668, "ttft_p95_s": 1.1756080102641135, "ttft_p99_s": 1.1759473806712777, "mean_request_tpot_ms": 11.864500810753364, "request_tpot_p95_ms": 18.38831296630614, "e2e_p95_s": 2.5892059521283954, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.5891990220407024, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.3908209531991056, "pass_fraction_ttft1s_tpot50ms": 0.125, "goodput_ttft2s_tpot50ms_rps": 3.1265676255928447, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 3.1265676255928447, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 3.1265676255928447, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "core", "case": "short-c8", "file": "vllm/core/vllm-short-c8-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 5.033102146815509, "output_tok_s": 406.9061068621047, "ttft_p50_s": 1.160099480068311, "ttft_p95_s": 1.1825663720956072, "ttft_p99_s": 1.183019326510839, "mean_request_tpot_ms": 11.521862637605444, "request_tpot_p95_ms": 18.090126008403992, "e2e_p95_s": 2.536837654421106, "full_stream_e2e_available": true, "token_e2e_p95_s": 2.5368285728618503, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.39736924498252413, "pass_fraction_ttft1s_tpot50ms": 0.125, "goodput_ttft2s_tpot50ms_rps": 3.178953959860193, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 3.178953959860193, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 3.178953959860193, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-prefill-c4", "file": "vllm/history/vllm-history-prefill-c4-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 14.262976380996406, "output_tok_s": 143.5885431829431, "ttft_p50_s": 2.3533574608154595, "ttft_p95_s": 2.3852210904005915, "ttft_p99_s": 2.3855740260798486, "mean_request_tpot_ms": 13.13927584496422, "request_tpot_p95_ms": 22.98532594288663, "e2e_p95_s": 3.57466272264719, "full_stream_e2e_available": true, "token_e2e_p95_s": 3.574652560870163, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.28044637340418577, "pass_fraction_ttft1s_tpot50ms": 0.25, "goodput_ttft2s_tpot50ms_rps": 0.3505579667552322, "pass_fraction_ttft2s_tpot50ms": 0.3125, "goodput_ttft3s_tpot50ms_rps": 1.121785493616743, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.121785493616743, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-prefill-c4", "file": "vllm/history/vllm-history-prefill-c4-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 14.235978255979717, "output_tok_s": 143.86085474243774, "ttft_p50_s": 2.361260599223897, "ttft_p95_s": 2.381105590960942, "ttft_p99_s": 2.38145076863002, "mean_request_tpot_ms": 13.01952903195629, "request_tpot_p95_ms": 22.966835299492118, "e2e_p95_s": 3.561960567254573, "full_stream_e2e_available": true, "token_e2e_p95_s": 3.5618795891059563, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.2809782319188237, "pass_fraction_ttft1s_tpot50ms": 0.25, "goodput_ttft2s_tpot50ms_rps": 0.3512227898985296, "pass_fraction_ttft2s_tpot50ms": 0.3125, "goodput_ttft3s_tpot50ms_rps": 1.1239129276752948, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.1239129276752948, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-prefill-c4", "file": "vllm/history/vllm-history-prefill-c4-r3.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 14.210147986654192, "output_tok_s": 144.12235551124655, "ttft_p50_s": 2.360157538205385, "ttft_p95_s": 2.405993028427474, "ttft_p99_s": 2.4063484711339695, "mean_request_tpot_ms": 12.667486948963974, "request_tpot_p95_ms": 22.898622026747667, "e2e_p95_s": 3.5860501937568188, "full_stream_e2e_available": true, "token_e2e_p95_s": 3.5860377566423267, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.2814889756079034, "pass_fraction_ttft1s_tpot50ms": 0.25, "goodput_ttft2s_tpot50ms_rps": 0.2814889756079034, "pass_fraction_ttft2s_tpot50ms": 0.25, "goodput_ttft3s_tpot50ms_rps": 1.1259559024316137, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.1259559024316137, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-short-c1", "file": "vllm/history/vllm-history-short-c1-r1.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 10.436299157328904, "output_tok_s": 98.11907310848738, "ttft_p50_s": 0.12805861001834273, "ttft_p95_s": 0.14071969245560467, "ttft_p99_s": 0.14196320311166347, "mean_request_tpot_ms": 8.25376461845793, "request_tpot_p95_ms": 8.328779534037624, "e2e_p95_s": 0.6537297149188817, "full_stream_e2e_available": true, "token_e2e_p95_s": 0.6537143609020859, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 1.5331105173201154, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 1.5331105173201154, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.5331105173201154, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.5331105173201154, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-short-c1", "file": "vllm/history/vllm-history-short-c1-r2.json", "requests": 16, "completed": 16, "failed": 0, "duration_s": 10.442569470964372, "output_tok_s": 98.06015682703746, "ttft_p50_s": 0.12798127299174666, "ttft_p95_s": 0.1346695355605334, "ttft_p99_s": 0.14001037417910994, "mean_request_tpot_ms": 8.307687974708628, "request_tpot_p95_ms": 8.337855145024756, "e2e_p95_s": 0.6545965761179104, "full_stream_e2e_available": true, "token_e2e_p95_s": 0.6545847611268982, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 1.5321899504224603, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 1.5321899504224603, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 1.5321899504224603, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 1.5321899504224603, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-short-c1", "file": "vllm/history/vllm-history-short-c1-r3.json", "requests": 16, "completed": 15, "failed": 1, "duration_s": 10.445220414083451, "output_tok_s": 91.90806531048567, "ttft_p50_s": 0.12873740820214152, "ttft_p95_s": 0.13348053600639104, "ttft_p99_s": 0.13923369202762842, "mean_request_tpot_ms": 8.266495612188779, "request_tpot_p95_ms": 8.319672524544691, "e2e_p95_s": 0.6540480660274625, "full_stream_e2e_available": true, "token_e2e_p95_s": 0.6540309075731784, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 1.4360635204763386, "pass_fraction_ttft1s_tpot50ms": 0.9375, "goodput_ttft2s_tpot50ms_rps": 1.4360635204763386, "pass_fraction_ttft2s_tpot50ms": 0.9375, "goodput_ttft3s_tpot50ms_rps": 1.4360635204763386, "pass_fraction_ttft3s_tpot50ms": 0.9375, "goodput_ttft5s_tpot50ms_rps": 1.4360635204763386, "pass_fraction_ttft5s_tpot50ms": 0.9375 }, { "engine": "vllm", "stage": "history", "case": "history-short-c4", "file": "vllm/history/vllm-history-short-c4-r1.json", "requests": 32, "completed": 32, "failed": 0, "duration_s": 7.263893862254918, "output_tok_s": 281.9424455858229, "ttft_p50_s": 0.1957321777008474, "ttft_p95_s": 0.3804752652766183, "ttft_p99_s": 0.4694774786895143, "mean_request_tpot_ms": 10.527712289273502, "request_tpot_p95_ms": 11.49184503481679, "e2e_p95_s": 1.0911690402776002, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.0911586763802916, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 4.405350712278483, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 4.405350712278483, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 4.405350712278483, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 4.405350712278483, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-short-c4", "file": "vllm/history/vllm-history-short-c4-r2.json", "requests": 32, "completed": 32, "failed": 0, "duration_s": 7.6600406798534095, "output_tok_s": 267.36150440903305, "ttft_p50_s": 0.2449947188142687, "ttft_p95_s": 0.3771370127098635, "ttft_p99_s": 0.3845176484063268, "mean_request_tpot_ms": 11.19287471709189, "request_tpot_p95_ms": 13.3567672326333, "e2e_p95_s": 0.9828910409938544, "full_stream_e2e_available": true, "token_e2e_p95_s": 0.9825404049362987, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 4.177523506391141, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 4.177523506391141, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 4.177523506391141, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 4.177523506391141, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-short-c4", "file": "vllm/history/vllm-history-short-c4-r3.json", "requests": 32, "completed": 31, "failed": 1, "duration_s": 7.689951564650983, "output_tok_s": 257.9990242227288, "ttft_p50_s": 0.3618747927248478, "ttft_p95_s": 0.37513421196490526, "ttft_p99_s": 0.38131409604102373, "mean_request_tpot_ms": 10.49479037161804, "request_tpot_p95_ms": 13.249855207663678, "e2e_p95_s": 0.9701046999543905, "full_stream_e2e_available": true, "token_e2e_p95_s": 0.969418773194775, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 4.031234753480137, "pass_fraction_ttft1s_tpot50ms": 0.96875, "goodput_ttft2s_tpot50ms_rps": 4.031234753480137, "pass_fraction_ttft2s_tpot50ms": 0.96875, "goodput_ttft3s_tpot50ms_rps": 4.031234753480137, "pass_fraction_ttft3s_tpot50ms": 0.96875, "goodput_ttft5s_tpot50ms_rps": 4.031234753480137, "pass_fraction_ttft5s_tpot50ms": 0.96875 }, { "engine": "vllm", "stage": "history", "case": "history-short-c8", "file": "vllm/history/vllm-history-short-c8-r1.json", "requests": 64, "completed": 64, "failed": 0, "duration_s": 8.416807833127677, "output_tok_s": 486.64530320848854, "ttft_p50_s": 0.35823564417660236, "ttft_p95_s": 0.3921008318662643, "ttft_p99_s": 0.40604052267502994, "mean_request_tpot_ms": 10.999696149877902, "request_tpot_p95_ms": 14.559169552318512, "e2e_p95_s": 1.0780258276965469, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.0772590407636016, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 7.6038328626326335, "pass_fraction_ttft1s_tpot50ms": 1.0, "goodput_ttft2s_tpot50ms_rps": 7.6038328626326335, "pass_fraction_ttft2s_tpot50ms": 1.0, "goodput_ttft3s_tpot50ms_rps": 7.6038328626326335, "pass_fraction_ttft3s_tpot50ms": 1.0, "goodput_ttft5s_tpot50ms_rps": 7.6038328626326335, "pass_fraction_ttft5s_tpot50ms": 1.0 }, { "engine": "vllm", "stage": "history", "case": "history-short-c8", "file": "vllm/history/vllm-history-short-c8-r2.json", "requests": 64, "completed": 63, "failed": 1, "duration_s": 8.423130135983229, "output_tok_s": 478.6819074272021, "ttft_p50_s": 0.36029482213780284, "ttft_p95_s": 0.3879160741344094, "ttft_p99_s": 0.38898574392311275, "mean_request_tpot_ms": 11.57703463250476, "request_tpot_p95_ms": 14.585373133036589, "e2e_p95_s": 1.069141366612166, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.0691353347618133, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 7.4794048035500325, "pass_fraction_ttft1s_tpot50ms": 0.984375, "goodput_ttft2s_tpot50ms_rps": 7.4794048035500325, "pass_fraction_ttft2s_tpot50ms": 0.984375, "goodput_ttft3s_tpot50ms_rps": 7.4794048035500325, "pass_fraction_ttft3s_tpot50ms": 0.984375, "goodput_ttft5s_tpot50ms_rps": 7.4794048035500325, "pass_fraction_ttft5s_tpot50ms": 0.984375 }, { "engine": "vllm", "stage": "history", "case": "history-short-c8", "file": "vllm/history/vllm-history-short-c8-r3.json", "requests": 64, "completed": 63, "failed": 1, "duration_s": 8.412258065771312, "output_tok_s": 479.300559787369, "ttft_p50_s": 0.3606034149415791, "ttft_p95_s": 0.3876152530312538, "ttft_p99_s": 0.39305714959278704, "mean_request_tpot_ms": 11.195939296326683, "request_tpot_p95_ms": 14.58294540791521, "e2e_p95_s": 1.063924045022577, "full_stream_e2e_available": true, "token_e2e_p95_s": 1.0639127211645245, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 7.489071246677641, "pass_fraction_ttft1s_tpot50ms": 0.984375, "goodput_ttft2s_tpot50ms_rps": 7.489071246677641, "pass_fraction_ttft2s_tpot50ms": 0.984375, "goodput_ttft3s_tpot50ms_rps": 7.489071246677641, "pass_fraction_ttft3s_tpot50ms": 0.984375, "goodput_ttft5s_tpot50ms_rps": 7.489071246677641, "pass_fraction_ttft5s_tpot50ms": 0.984375 }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "file": "vllm/slo/vllm-slo-0.5-r1.json", "requests": 200, "completed": 199, "failed": 1, "duration_s": 669.0113356038928, "output_tok_s": 38.07409328424507, "ttft_p50_s": 1.202807345893234, "ttft_p95_s": 3.2486028635408726, "ttft_p99_s": 3.817685380177573, "mean_request_tpot_ms": 17.48586776547869, "request_tpot_p95_ms": 46.10547348893592, "e2e_p95_s": 7.207136430451646, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.20712829073891, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.22421144757524972, "pass_fraction_ttft2s_tpot50ms": 0.75, "goodput_ttft3s_tpot50ms_rps": 0.26456950813879465, "pass_fraction_ttft3s_tpot50ms": 0.885, "goodput_ttft5s_tpot50ms_rps": 0.2899801388639896, "pass_fraction_ttft5s_tpot50ms": 0.97 }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "file": "vllm/slo/vllm-slo-0.5-r2.json", "requests": 200, "completed": 198, "failed": 2, "duration_s": 668.98949652398, "output_tok_s": 37.88400286056142, "ttft_p50_s": 1.1995770814828575, "ttft_p95_s": 3.250699667632576, "ttft_p99_s": 4.059416490006261, "mean_request_tpot_ms": 16.994079529128303, "request_tpot_p95_ms": 38.643486440313566, "e2e_p95_s": 7.158787277596999, "full_stream_e2e_available": true, "token_e2e_p95_s": 7.148214536532755, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.22720835048947818, "pass_fraction_ttft2s_tpot50ms": 0.76, "goodput_ttft3s_tpot50ms_rps": 0.2645781449778792, "pass_fraction_ttft3s_tpot50ms": 0.885, "goodput_ttft5s_tpot50ms_rps": 0.2899896052299919, "pass_fraction_ttft5s_tpot50ms": 0.97 }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "file": "vllm/slo/vllm-slo-0.5-r3.json", "requests": 200, "completed": 199, "failed": 1, "duration_s": 668.9873620541766, "output_tok_s": 38.07545769143723, "ttft_p50_s": 1.203633830882609, "ttft_p95_s": 3.167877234052866, "ttft_p99_s": 3.594052764996888, "mean_request_tpot_ms": 16.026746525711793, "request_tpot_p95_ms": 36.10068163357851, "e2e_p95_s": 5.939398533385247, "full_stream_e2e_available": true, "token_e2e_p95_s": 5.939385580364615, "small_tail_sample": true, "slo_thresholds_are_experimental": true, "goodput_ttft1s_tpot50ms_rps": 0.0, "pass_fraction_ttft1s_tpot50ms": 0.0, "goodput_ttft2s_tpot50ms_rps": 0.2316934650664561, "pass_fraction_ttft2s_tpot50ms": 0.775, "goodput_ttft3s_tpot50ms_rps": 0.2735477684332998, "pass_fraction_ttft3s_tpot50ms": 0.915, "goodput_ttft5s_tpot50ms_rps": 0.2959697166655375, "pass_fraction_ttft5s_tpot50ms": 0.99 } ], "summary": [ { "engine": "sglang", "stage": "core", "case": "decode-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 48.51652415960526, "output_tok_s_round_min": 48.497939853843306, "output_tok_s_round_max": 48.536096997615374, "ttft_p95_s_round_median": 0.24694492197595536, "ttft_p95_s_round_min": 0.22870647183153778, "ttft_p95_s_round_max": 0.24784173434600235, "mean_request_tpot_ms_round_median": 20.19596505416247, "mean_request_tpot_ms_round_min": 20.18909419976984, "mean_request_tpot_ms_round_max": 20.19936801788694, "request_tpot_p95_ms_round_median": 20.219082154658004, "request_tpot_p95_ms_round_min": 20.211700206976058, "request_tpot_p95_ms_round_max": 20.222119238171107, "e2e_p95_s_round_median": 10.564405523194, "e2e_p95_s_round_min": 10.559498811978846, "e2e_p95_s_round_max": 10.56673616755288, "token_e2e_p95_s_round_median": 10.564387026871554, "token_e2e_p95_s_round_min": 10.559484610427171, "token_e2e_p95_s_round_max": 10.566720018885098 }, { "engine": "sglang", "stage": "core", "case": "decode-c32", "rounds": 3, "requests": 192, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 824.7032600686088, "output_tok_s_round_min": 819.4492130498578, "output_tok_s_round_max": 827.3255400857179, "ttft_p95_s_round_median": 5.679089043894782, "ttft_p95_s_round_min": 5.673246767208911, "ttft_p95_s_round_max": 5.683949015289545, "mean_request_tpot_ms_round_median": 29.412606192380007, "mean_request_tpot_ms_round_min": 29.250298274370913, "mean_request_tpot_ms_round_max": 29.78502103882929, "request_tpot_p95_ms_round_median": 33.473784309163065, "request_tpot_p95_ms_round_min": 33.160563055225396, "request_tpot_p95_ms_round_max": 33.62344485172663, "e2e_p95_s_round_median": 20.029147566552275, "e2e_p95_s_round_min": 19.87822789240163, "e2e_p95_s_round_max": 20.114979301765562, "token_e2e_p95_s_round_median": 20.029140169639142, "token_e2e_p95_s_round_min": 19.878219857113436, "token_e2e_p95_s_round_max": 20.114969361503608 }, { "engine": "sglang", "stage": "core", "case": "decode-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 350.1634969737258, "output_tok_s_round_min": 346.2266408749254, "output_tok_s_round_max": 352.5897233141495, "ttft_p95_s_round_median": 1.465820539277047, "ttft_p95_s_round_min": 1.4581511460710317, "ttft_p95_s_round_max": 1.4701936388155445, "mean_request_tpot_ms_round_median": 20.069967273375795, "mean_request_tpot_ms_round_min": 19.90083777578092, "mean_request_tpot_ms_round_max": 20.328408288163427, "request_tpot_p95_ms_round_median": 20.32843019964571, "request_tpot_p95_ms_round_min": 20.043503221510793, "request_tpot_p95_ms_round_max": 20.548386023828208, "e2e_p95_s_round_median": 11.809660215047188, "e2e_p95_s_round_min": 11.6389586313162, "e2e_p95_s_round_max": 11.931524883024395, "token_e2e_p95_s_round_median": 11.809639187762514, "token_e2e_p95_s_round_min": 11.638950228923932, "token_e2e_p95_s_round_max": 11.931516422890127 }, { "engine": "sglang", "stage": "core", "case": "long-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 16.852339907542344, "output_tok_s_round_min": 16.826806313468847, "output_tok_s_round_max": 16.8545581067932, "ttft_p95_s_round_median": 5.064343739300966, "ttft_p95_s_round_min": 5.060727500542998, "ttft_p95_s_round_max": 5.0802262338344, "mean_request_tpot_ms_round_median": 20.131148656419526, "mean_request_tpot_ms_round_min": 20.123470527409598, "mean_request_tpot_ms_round_max": 20.158018832054633, "request_tpot_p95_ms_round_median": 20.240636005278997, "request_tpot_p95_ms_round_min": 20.234935085553175, "request_tpot_p95_ms_round_max": 20.24613352782115, "e2e_p95_s_round_median": 7.61687787885312, "e2e_p95_s_round_min": 7.61073174267076, "e2e_p95_s_round_max": 7.631990188267082, "token_e2e_p95_s_round_median": 7.61686287720222, "token_e2e_p95_s_round_min": 7.610717630758882, "token_e2e_p95_s_round_max": 7.631974293501116 }, { "engine": "sglang", "stage": "core", "case": "long-c32", "rounds": 3, "requests": 192, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 22.79343004204918, "output_tok_s_round_min": 22.782412556465584, "output_tok_s_round_max": 22.799272904093353, "ttft_p95_s_round_median": 154.68304573709608, "ttft_p95_s_round_min": 149.4713405372342, "ttft_p95_s_round_max": 163.02333678153803, "mean_request_tpot_ms_round_median": 861.0626236304382, "mean_request_tpot_ms_round_min": 831.0106538892009, "mean_request_tpot_ms_round_max": 874.415656850835, "request_tpot_p95_ms_round_median": 1330.0665863047127, "request_tpot_p95_ms_round_min": 1292.964965114223, "request_tpot_p95_ms_round_max": 1331.2565517239636, "e2e_p95_s_round_median": 285.99859518483277, "e2e_p95_s_round_min": 285.8280833825702, "e2e_p95_s_round_max": 286.04614139513575, "token_e2e_p95_s_round_median": 285.9985818383051, "token_e2e_p95_s_round_min": 285.8280466715805, "token_e2e_p95_s_round_max": 286.0461279868614 }, { "engine": "sglang", "stage": "core", "case": "long-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 19.5941789521317, "output_tok_s_round_min": 19.565470313695247, "output_tok_s_round_max": 19.60797536982122, "ttft_p95_s_round_median": 49.71797822928056, "ttft_p95_s_round_min": 47.06338689045515, "ttft_p95_s_round_max": 49.79768008226529, "mean_request_tpot_ms_round_median": 160.45640439127837, "mean_request_tpot_ms_round_min": 150.17712575784066, "mean_request_tpot_ms_round_max": 166.8781289793221, "request_tpot_p95_ms_round_median": 312.1596708633445, "request_tpot_p95_ms_round_min": 295.2013193734181, "request_tpot_p95_ms_round_max": 312.2237857603505, "e2e_p95_s_round_median": 52.29953672760166, "e2e_p95_s_round_min": 52.26222040166613, "e2e_p95_s_round_max": 52.48119758744724, "token_e2e_p95_s_round_median": 52.29952943511307, "token_e2e_p95_s_round_min": 52.26221369160339, "token_e2e_p95_s_round_max": 52.48118782485835 }, { "engine": "sglang", "stage": "core", "case": "prefill-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 31.13579665148616, "output_tok_s_round_min": 31.085417966867993, "output_tok_s_round_max": 31.15314749341019, "ttft_p95_s_round_median": 1.5674305916298181, "ttft_p95_s_round_min": 1.5602353524882346, "ttft_p95_s_round_max": 1.5766864896053447, "mean_request_tpot_ms_round_median": 20.15396488388619, "mean_request_tpot_ms_round_min": 20.098928373130992, "mean_request_tpot_ms_round_max": 20.1766064342874, "request_tpot_p95_ms_round_median": 20.211176678545655, "request_tpot_p95_ms_round_min": 20.191151162015466, "request_tpot_p95_ms_round_max": 20.229295031790894, "e2e_p95_s_round_median": 4.120126041560434, "e2e_p95_s_round_min": 4.117015044391155, "e2e_p95_s_round_max": 4.1264671427896245, "token_e2e_p95_s_round_median": 4.120110503840261, "token_e2e_p95_s_round_min": 4.116997827100567, "token_e2e_p95_s_round_max": 4.126452111685649 }, { "engine": "sglang", "stage": "core", "case": "prefill-c32", "rounds": 3, "requests": 192, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 76.79053095859533, "output_tok_s_round_min": 76.75033754363497, "output_tok_s_round_max": 76.85655420293972, "ttft_p95_s_round_median": 48.04473857576957, "ttft_p95_s_round_min": 47.97562206678556, "ttft_p95_s_round_max": 49.28513757132457, "mean_request_tpot_ms_round_median": 255.60579977728614, "mean_request_tpot_ms_round_min": 254.95234315522387, "mean_request_tpot_ms_round_max": 255.7715372478967, "request_tpot_p95_ms_round_median": 370.4678177221231, "request_tpot_p95_ms_round_min": 369.58377112792147, "request_tpot_p95_ms_round_max": 381.2629947160172, "e2e_p95_s_round_median": 85.95466712096238, "e2e_p95_s_round_min": 85.83662466031963, "e2e_p95_s_round_max": 86.00287257365414, "token_e2e_p95_s_round_median": 85.83661192541474, "token_e2e_p95_s_round_min": 82.07828260697424, "token_e2e_p95_s_round_max": 86.00285777836105 }, { "engine": "sglang", "stage": "core", "case": "prefill-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 63.63971104705953, "output_tok_s_round_min": 63.570639116952684, "output_tok_s_round_max": 63.74203656364797, "ttft_p95_s_round_median": 13.65014299878385, "ttft_p95_s_round_min": 13.624190002330579, "ttft_p95_s_round_max": 13.690121321124025, "mean_request_tpot_ms_round_median": 51.05749745857829, "mean_request_tpot_ms_round_min": 45.42219813630777, "mean_request_tpot_ms_round_max": 52.797309879220904, "request_tpot_p95_ms_round_median": 95.14606988523155, "request_tpot_p95_ms_round_min": 70.54234464381507, "request_tpot_p95_ms_round_max": 103.39530853565607, "e2e_p95_s_round_median": 16.243461400270462, "e2e_p95_s_round_min": 16.188612060504965, "e2e_p95_s_round_max": 16.264968653908, "token_e2e_p95_s_round_median": 16.243452910566702, "token_e2e_p95_s_round_min": 16.18860459048301, "token_e2e_p95_s_round_max": 16.264955582097173 }, { "engine": "sglang", "stage": "core", "case": "short-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 45.8541341023796, "output_tok_s_round_min": 45.61047363708748, "output_tok_s_round_max": 45.87526476636097, "ttft_p95_s_round_median": 0.24802749580703676, "ttft_p95_s_round_min": 0.22918487282004207, "ttft_p95_s_round_max": 0.3275388656649738, "mean_request_tpot_ms_round_median": 20.114534456598182, "mean_request_tpot_ms_round_min": 20.10400751999792, "mean_request_tpot_ms_round_max": 20.190707579306434, "request_tpot_p95_ms_round_median": 20.201972841769162, "request_tpot_p95_ms_round_min": 20.18155817375115, "request_tpot_p95_ms_round_max": 20.21817000817478, "e2e_p95_s_round_median": 2.7959118694998324, "e2e_p95_s_round_min": 2.7924669726751743, "e2e_p95_s_round_max": 2.87286340710707, "token_e2e_p95_s_round_median": 2.7958978100912644, "token_e2e_p95_s_round_min": 2.792451668367721, "token_e2e_p95_s_round_max": 2.8728486706735565 }, { "engine": "sglang", "stage": "core", "case": "short-c32", "rounds": 3, "requests": 192, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 445.02798763548407, "output_tok_s_round_min": 444.35386231426776, "output_tok_s_round_max": 446.26832871910614, "ttft_p95_s_round_median": 5.686285282112658, "ttft_p95_s_round_min": 5.677037351951003, "ttft_p95_s_round_max": 5.69418488566298, "mean_request_tpot_ms_round_median": 33.86784914968532, "mean_request_tpot_ms_round_min": 33.53025147197063, "mean_request_tpot_ms_round_max": 35.790722174950375, "request_tpot_p95_ms_round_median": 49.65762663412634, "request_tpot_p95_ms_round_min": 49.3854750380448, "request_tpot_p95_ms_round_max": 49.73272361836624, "e2e_p95_s_round_median": 9.245797717547976, "e2e_p95_s_round_min": 9.198331869463436, "e2e_p95_s_round_max": 9.24682450077962, "token_e2e_p95_s_round_median": 9.245790195651352, "token_e2e_p95_s_round_min": 9.198155375174247, "token_e2e_p95_s_round_max": 9.246816555014812 }, { "engine": "sglang", "stage": "core", "case": "short-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 254.57917813511114, "output_tok_s_round_min": 254.32983687817853, "output_tok_s_round_max": 256.94427931605384, "ttft_p95_s_round_median": 1.4619857898214832, "ttft_p95_s_round_min": 1.4576065525179729, "ttft_p95_s_round_max": 1.4722298487322405, "mean_request_tpot_ms_round_median": 20.33349970198882, "mean_request_tpot_ms_round_min": 19.92731549965922, "mean_request_tpot_ms_round_max": 20.347812032880714, "request_tpot_p95_ms_round_median": 20.864257995061227, "request_tpot_p95_ms_round_min": 20.15691651324705, "request_tpot_p95_ms_round_max": 20.913940094462706, "e2e_p95_s_round_median": 4.0520477092359215, "e2e_p95_s_round_min": 4.0051197544671595, "e2e_p95_s_round_max": 4.060795152909122, "token_e2e_p95_s_round_median": 4.052039989386685, "token_e2e_p95_s_round_min": 4.005113334627822, "token_e2e_p95_s_round_max": 4.060784610686824 }, { "engine": "sglang", "stage": "history", "case": "history-prefill-c4", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 97.82639635448601, "output_tok_s_round_min": 97.64216192273877, "output_tok_s_round_max": 97.89645038106066, "ttft_p95_s_round_median": 2.9685806629713625, "ttft_p95_s_round_min": 2.961239076917991, "ttft_p95_s_round_max": 2.987787603866309, "mean_request_tpot_ms_round_median": 19.281396660889595, "mean_request_tpot_ms_round_min": 18.479667926861676, "mean_request_tpot_ms_round_max": 19.496293524674353, "request_tpot_p95_ms_round_median": 24.71601175727308, "request_tpot_p95_ms_round_min": 20.072155639411896, "request_tpot_p95_ms_round_max": 24.878936004824936, "e2e_p95_s_round_median": 5.253828642191365, "e2e_p95_s_round_min": 5.253059729468077, "e2e_p95_s_round_max": 5.259710291633382, "token_e2e_p95_s_round_median": 5.253820827114396, "token_e2e_p95_s_round_min": 5.253049927647226, "token_e2e_p95_s_round_max": 5.2597036440856755 }, { "engine": "sglang", "stage": "history", "case": "history-short-c1", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 44.24779434909659, "output_tok_s_round_min": 44.217181788438346, "output_tok_s_round_max": 44.307915113686185, "ttft_p95_s_round_median": 0.19670670328196138, "ttft_p95_s_round_min": 0.18202978558838367, "ttft_p95_s_round_max": 0.19769891072064638, "mean_request_tpot_ms_round_median": 20.08726209446433, "mean_request_tpot_ms_round_min": 20.06204863354224, "mean_request_tpot_ms_round_max": 20.11087003103014, "request_tpot_p95_ms_round_median": 20.15972875302569, "request_tpot_p95_ms_round_min": 20.154927982874806, "request_tpot_p95_ms_round_max": 20.174001450724308, "e2e_p95_s_round_median": 1.448630850063637, "e2e_p95_s_round_min": 1.4464875527191907, "e2e_p95_s_round_max": 1.4519307899754494, "token_e2e_p95_s_round_median": 1.448617743095383, "token_e2e_p95_s_round_min": 1.446473352611065, "token_e2e_p95_s_round_max": 1.451914850389585 }, { "engine": "sglang", "stage": "history", "case": "history-short-c4", "rounds": 3, "requests": 96, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 187.2818101776305, "output_tok_s_round_min": 187.0652941481555, "output_tok_s_round_max": 188.28536378302425, "ttft_p95_s_round_median": 0.3560166540555656, "ttft_p95_s_round_min": 0.34221710709389297, "ttft_p95_s_round_max": 0.35757704444695265, "mean_request_tpot_ms_round_median": 18.126758095431363, "mean_request_tpot_ms_round_min": 18.008358969670972, "mean_request_tpot_ms_round_max": 18.174033627257728, "request_tpot_p95_ms_round_median": 18.308938431021357, "request_tpot_p95_ms_round_min": 18.2788164325295, "request_tpot_p95_ms_round_max": 18.377388413581585, "e2e_p95_s_round_median": 1.4843525264644994, "e2e_p95_s_round_min": 1.4784948154585436, "e2e_p95_s_round_max": 1.4957575111882762, "token_e2e_p95_s_round_median": 1.4843437930801884, "token_e2e_p95_s_round_min": 1.4784845404326916, "token_e2e_p95_s_round_max": 1.4957493450725452 }, { "engine": "sglang", "stage": "history", "case": "history-short-c8", "rounds": 3, "requests": 192, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 308.4601956007039, "output_tok_s_round_min": 306.81455127333317, "output_tok_s_round_max": 310.94654090236645, "ttft_p95_s_round_median": 0.40664501341525466, "ttft_p95_s_round_min": 0.40543246623128654, "ttft_p95_s_round_max": 0.40931367964949456, "mean_request_tpot_ms_round_median": 20.11594226150053, "mean_request_tpot_ms_round_min": 19.876036265775348, "mean_request_tpot_ms_round_max": 20.262917426104348, "request_tpot_p95_ms_round_median": 20.473347432793133, "request_tpot_p95_ms_round_min": 20.384218969145817, "request_tpot_p95_ms_round_max": 20.5253459679495, "e2e_p95_s_round_median": 1.681407206552103, "e2e_p95_s_round_min": 1.6704298709053547, "e2e_p95_s_round_max": 1.6837959521450103, "token_e2e_p95_s_round_median": 1.6813982895575463, "token_e2e_p95_s_round_min": 1.6704242561012506, "token_e2e_p95_s_round_max": 1.6837816983228548 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.5", "rounds": 3, "requests": 600, "failed": 0, "tail_sample_warning": false, "output_tok_s_round_median": 38.159937098705726, "output_tok_s_round_min": 38.15974649742065, "output_tok_s_round_max": 38.17119094733828, "ttft_p95_s_round_median": 6.073622179799713, "ttft_p95_s_round_min": 5.455989311099984, "ttft_p95_s_round_max": 6.132679248508065, "mean_request_tpot_ms_round_median": 38.822779644676665, "mean_request_tpot_ms_round_min": 36.37927335485317, "mean_request_tpot_ms_round_max": 41.2842315827555, "request_tpot_p95_ms_round_median": 89.4282935553883, "request_tpot_p95_ms_round_min": 79.32699507058369, "request_tpot_p95_ms_round_max": 101.55614597216893, "e2e_p95_s_round_median": 14.408885770454072, "e2e_p95_s_round_min": 14.162595759076066, "e2e_p95_s_round_max": 16.097032101731745, "token_e2e_p95_s_round_median": 14.408877224475145, "token_e2e_p95_s_round_min": 14.162583958706817, "token_e2e_p95_s_round_max": 16.097015924588774 }, { "engine": "sglang", "stage": "slo", "case": "slo-0.8", "rounds": 3, "requests": 600, "failed": 1, "tail_sample_warning": true, "output_tok_s_round_median": 60.50352630565577, "output_tok_s_round_min": 60.36257788519018, "output_tok_s_round_max": 60.8325856969355, "ttft_p95_s_round_median": 8.746406201086938, "ttft_p95_s_round_min": 7.354478139197454, "ttft_p95_s_round_max": 9.295561882923353, "mean_request_tpot_ms_round_median": 101.59122399524495, "mean_request_tpot_ms_round_min": 91.73845005503995, "mean_request_tpot_ms_round_max": 130.41608952281, "request_tpot_p95_ms_round_median": 236.5114057117589, "request_tpot_p95_ms_round_min": 203.0229543482401, "request_tpot_p95_ms_round_max": 358.6681993546212, "e2e_p95_s_round_median": 35.4178314726334, "e2e_p95_s_round_min": 34.81513398047536, "e2e_p95_s_round_max": 50.654682853701516, "token_e2e_p95_s_round_median": 35.41782316383905, "token_e2e_p95_s_round_min": 34.81512695637065, "token_e2e_p95_s_round_max": 50.65467058294452 }, { "engine": "sglang", "stage": "slo", "case": "slo-1.1", "rounds": 3, "requests": 600, "failed": 0, "tail_sample_warning": false, "output_tok_s_round_median": 77.28792402577449, "output_tok_s_round_min": 76.18771227074392, "output_tok_s_round_max": 78.04679068059491, "ttft_p95_s_round_median": 26.138306704582646, "ttft_p95_s_round_min": 25.994827236677512, "ttft_p95_s_round_max": 30.699214355298317, "mean_request_tpot_ms_round_median": 321.7556111554718, "mean_request_tpot_ms_round_min": 302.259456734338, "mean_request_tpot_ms_round_max": 325.58183168469907, "request_tpot_p95_ms_round_median": 371.23003438276453, "request_tpot_p95_ms_round_min": 371.19225541305707, "request_tpot_p95_ms_round_max": 372.6496747401578, "e2e_p95_s_round_median": 69.1028537699487, "e2e_p95_s_round_min": 67.82771018347702, "e2e_p95_s_round_max": 75.24056903971358, "token_e2e_p95_s_round_median": 69.1028338674223, "token_e2e_p95_s_round_min": 67.82770192525349, "token_e2e_p95_s_round_max": 75.24055977379902 }, { "engine": "vllm", "stage": "core", "case": "decode-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 115.16834475229864, "output_tok_s_round_min": 115.07458104067528, "output_tok_s_round_max": 115.2033361182112, "ttft_p95_s_round_median": 0.19711574090179057, "ttft_p95_s_round_min": 0.1921028696000576, "ttft_p95_s_round_max": 0.19877633501309902, "mean_request_tpot_ms_round_median": 8.323614110572375, "mean_request_tpot_ms_round_min": 8.31298277496475, "mean_request_tpot_ms_round_max": 8.328708259155519, "request_tpot_p95_ms_round_median": 8.335076634537014, "request_tpot_p95_ms_round_min": 8.330485974885013, "request_tpot_p95_ms_round_max": 8.346501350860294, "e2e_p95_s_round_median": 4.453205332998186, "e2e_p95_s_round_min": 4.452694016112946, "e2e_p95_s_round_max": 4.460156888468191, "token_e2e_p95_s_round_median": 4.452945414371788, "token_e2e_p95_s_round_min": 4.449745018221438, "token_e2e_p95_s_round_max": 4.46014322580304 }, { "engine": "vllm", "stage": "core", "case": "decode-c32", "rounds": 3, "requests": 192, "failed": 1, "tail_sample_warning": true, "output_tok_s_round_median": 1219.4567018224598, "output_tok_s_round_min": 1204.0215475710347, "output_tok_s_round_max": 1238.736387601928, "ttft_p95_s_round_median": 4.45758340882603, "ttft_p95_s_round_min": 4.451026804978028, "ttft_p95_s_round_max": 4.458423188654706, "mean_request_tpot_ms_round_median": 20.705067403437475, "mean_request_tpot_ms_round_min": 19.836520195827358, "mean_request_tpot_ms_round_max": 20.730916750290973, "request_tpot_p95_ms_round_median": 23.859204417422646, "request_tpot_p95_ms_round_min": 23.43808249789135, "request_tpot_p95_ms_round_max": 24.071565929093122, "e2e_p95_s_round_median": 14.41589999361895, "e2e_p95_s_round_min": 14.407223387551493, "e2e_p95_s_round_max": 14.606841707741841, "token_e2e_p95_s_round_median": 14.415889986651019, "token_e2e_p95_s_round_min": 14.40496928638313, "token_e2e_p95_s_round_max": 14.606417830148711 }, { "engine": "vllm", "stage": "core", "case": "decode-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 613.7419023748109, "output_tok_s_round_min": 599.7775144753241, "output_tok_s_round_max": 620.461312680264, "ttft_p95_s_round_median": 1.1766843994846568, "ttft_p95_s_round_min": 1.175377813517116, "ttft_p95_s_round_max": 1.1833921261131763, "mean_request_tpot_ms_round_median": 11.01532786928068, "mean_request_tpot_ms_round_min": 10.864094372735305, "mean_request_tpot_ms_round_max": 11.307909608355663, "request_tpot_p95_ms_round_median": 12.596697088854055, "request_tpot_p95_ms_round_min": 12.47456373529301, "request_tpot_p95_ms_round_max": 12.894964216499353, "e2e_p95_s_round_median": 6.695874874596484, "e2e_p95_s_round_min": 6.642328674555756, "e2e_p95_s_round_max": 6.8920627580955625, "token_e2e_p95_s_round_median": 6.695437211543322, "token_e2e_p95_s_round_min": 6.642321534571238, "token_e2e_p95_s_round_max": 6.8920557564124465 }, { "engine": "vllm", "stage": "core", "case": "long-c1", "rounds": 3, "requests": 24, "failed": 1, "tail_sample_warning": true, "output_tok_s_round_median": 27.729025414988943, "output_tok_s_round_min": 24.329335691797123, "output_tok_s_round_max": 27.796906651380873, "ttft_p95_s_round_median": 3.603658485296182, "ttft_p95_s_round_min": 3.6022905338555575, "ttft_p95_s_round_max": 3.6184175213100387, "mean_request_tpot_ms_round_median": 8.028949084728369, "mean_request_tpot_ms_round_min": 8.018291399251623, "mean_request_tpot_ms_round_max": 8.04354656738867, "request_tpot_p95_ms_round_median": 8.071679000808734, "request_tpot_p95_ms_round_min": 8.055036295733348, "request_tpot_p95_ms_round_max": 8.074000377141994, "e2e_p95_s_round_median": 4.62177074230276, "e2e_p95_s_round_min": 4.620717612840235, "e2e_p95_s_round_max": 4.637787795555778, "token_e2e_p95_s_round_median": 4.621756793395616, "token_e2e_p95_s_round_min": 4.620705521013588, "token_e2e_p95_s_round_max": 4.637775349011645 }, { "engine": "vllm", "stage": "core", "case": "long-c32", "rounds": 3, "requests": 192, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 36.29716393944985, "output_tok_s_round_min": 36.29696786878875, "output_tok_s_round_max": 36.32615032874526, "ttft_p95_s_round_median": 102.4013492900878, "ttft_p95_s_round_min": 101.54811689418273, "ttft_p95_s_round_max": 102.62430188418364, "mean_request_tpot_ms_round_median": 605.6100434321621, "mean_request_tpot_ms_round_min": 605.477783751058, "mean_request_tpot_ms_round_max": 606.3053500428374, "request_tpot_p95_ms_round_median": 814.9649418639311, "request_tpot_p95_ms_round_min": 806.9753817038786, "request_tpot_p95_ms_round_max": 815.346607984425, "e2e_p95_s_round_median": 201.99029101619962, "e2e_p95_s_round_min": 201.93572719257787, "e2e_p95_s_round_max": 202.00522467342668, "token_e2e_p95_s_round_median": 201.9902782414341, "token_e2e_p95_s_round_min": 201.935716194869, "token_e2e_p95_s_round_max": 202.00521362377327 }, { "engine": "vllm", "stage": "core", "case": "long-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 35.08600260236773, "output_tok_s_round_min": 35.01041061620351, "output_tok_s_round_max": 35.10613973366645, "ttft_p95_s_round_median": 26.227974120760337, "ttft_p95_s_round_min": 26.12782807671465, "ttft_p95_s_round_max": 27.004446445731446, "mean_request_tpot_ms_round_median": 110.99268808593335, "mean_request_tpot_ms_round_min": 110.7999059622814, "mean_request_tpot_ms_round_max": 115.01870134535547, "request_tpot_p95_ms_round_median": 187.41733556994654, "request_tpot_p95_ms_round_min": 185.26405438608452, "request_tpot_p95_ms_round_max": 191.38701851608042, "e2e_p95_s_round_median": 40.41505611210596, "e2e_p95_s_round_min": 40.40727836661972, "e2e_p95_s_round_max": 41.68350547377486, "token_e2e_p95_s_round_median": 40.4150417977944, "token_e2e_p95_s_round_min": 40.40726501669269, "token_e2e_p95_s_round_max": 41.683492008945905 }, { "engine": "vllm", "stage": "core", "case": "prefill-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 57.70632679355879, "output_tok_s_round_min": 57.601324684648674, "output_tok_s_round_max": 57.751540575655035, "ttft_p95_s_round_median": 1.1974870281293988, "ttft_p95_s_round_min": 1.1974325044313445, "ttft_p95_s_round_max": 1.2043369268300013, "mean_request_tpot_ms_round_median": 8.082117058069072, "mean_request_tpot_ms_round_min": 8.07482703207574, "mean_request_tpot_ms_round_max": 8.08807628872753, "request_tpot_p95_ms_round_median": 8.112255322534267, "request_tpot_p95_ms_round_min": 8.106263262571723, "request_tpot_p95_ms_round_max": 8.12042399536906, "e2e_p95_s_round_median": 2.2256309275515376, "e2e_p95_s_round_min": 2.2235585060669107, "e2e_p95_s_round_max": 2.2309355789329857, "token_e2e_p95_s_round_median": 2.2256072521675376, "token_e2e_p95_s_round_min": 2.2235430970555172, "token_e2e_p95_s_round_max": 2.2309224202297626 }, { "engine": "vllm", "stage": "core", "case": "prefill-c32", "rounds": 3, "requests": 192, "failed": 2, "tail_sample_warning": true, "output_tok_s_round_median": 106.39477756339926, "output_tok_s_round_min": 103.09628290751856, "output_tok_s_round_max": 106.58727323086131, "ttft_p95_s_round_median": 34.18993107452988, "ttft_p95_s_round_min": 34.11350479694083, "ttft_p95_s_round_max": 34.24326259230729, "mean_request_tpot_ms_round_median": 197.8010022137351, "mean_request_tpot_ms_round_min": 196.82698256404214, "mean_request_tpot_ms_round_max": 200.64208972174842, "request_tpot_p95_ms_round_median": 265.31210875204636, "request_tpot_p95_ms_round_min": 264.4716606458913, "request_tpot_p95_ms_round_max": 272.0928671459967, "e2e_p95_s_round_median": 66.62914763563312, "e2e_p95_s_round_min": 66.55927178401033, "e2e_p95_s_round_max": 66.80095443504396, "token_e2e_p95_s_round_median": 66.62913638621103, "token_e2e_p95_s_round_min": 66.38949690558947, "token_e2e_p95_s_round_max": 66.80094129948411 }, { "engine": "vllm", "stage": "core", "case": "prefill-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 97.63078119913274, "output_tok_s_round_min": 97.52301378315526, "output_tok_s_round_max": 97.67371867298986, "ttft_p95_s_round_median": 9.193929874454625, "ttft_p95_s_round_min": 9.190890473430045, "ttft_p95_s_round_max": 9.252861680695787, "mean_request_tpot_ms_round_median": 35.409696055273706, "mean_request_tpot_ms_round_min": 34.80779710473258, "mean_request_tpot_ms_round_max": 36.01764578337408, "request_tpot_p95_ms_round_median": 72.57850588607332, "request_tpot_p95_ms_round_min": 66.089752019295, "request_tpot_p95_ms_round_max": 72.7217607172646, "e2e_p95_s_round_median": 13.09378808317706, "e2e_p95_s_round_min": 13.084489567903802, "e2e_p95_s_round_max": 13.102535630343482, "token_e2e_p95_s_round_median": 13.09377500822302, "token_e2e_p95_s_round_min": 13.08447510865517, "token_e2e_p95_s_round_max": 13.102524142945185 }, { "engine": "vllm", "stage": "core", "case": "short-c1", "rounds": 3, "requests": 24, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 104.52376000652613, "output_tok_s_round_min": 104.10953559640285, "output_tok_s_round_max": 104.86394548897593, "ttft_p95_s_round_median": 0.19803353070747107, "ttft_p95_s_round_min": 0.19438648449722679, "ttft_p95_s_round_max": 0.2148163946578279, "mean_request_tpot_ms_round_median": 8.10413253557773, "mean_request_tpot_ms_round_min": 8.100336528471196, "mean_request_tpot_ms_round_max": 8.1426015955132, "request_tpot_p95_ms_round_median": 8.153018495617948, "request_tpot_p95_ms_round_min": 8.145341993030893, "request_tpot_p95_ms_round_max": 8.179851815537146, "e2e_p95_s_round_median": 1.2312646110076457, "e2e_p95_s_round_min": 1.2227639746386558, "e2e_p95_s_round_max": 1.2495283655822278, "token_e2e_p95_s_round_median": 1.231247127871029, "token_e2e_p95_s_round_min": 1.2226250279461965, "token_e2e_p95_s_round_max": 1.2495161704020574 }, { "engine": "vllm", "stage": "core", "case": "short-c32", "rounds": 3, "requests": 192, "failed": 1, "tail_sample_warning": true, "output_tok_s_round_median": 613.5632121482298, "output_tok_s_round_min": 603.0965984973177, "output_tok_s_round_max": 620.3107128746743, "ttft_p95_s_round_median": 4.462674967199564, "ttft_p95_s_round_min": 4.455559488805011, "ttft_p95_s_round_max": 4.46526139874477, "mean_request_tpot_ms_round_median": 30.00502976386484, "mean_request_tpot_ms_round_min": 28.314974083034432, "mean_request_tpot_ms_round_max": 30.372993320771624, "request_tpot_p95_ms_round_median": 42.97487018354912, "request_tpot_p95_ms_round_min": 42.388605019517655, "request_tpot_p95_ms_round_max": 43.429493129356054, "e2e_p95_s_round_median": 7.7787515456788245, "e2e_p95_s_round_min": 7.752677880530245, "e2e_p95_s_round_max": 7.81973541516345, "token_e2e_p95_s_round_median": 7.778359147068113, "token_e2e_p95_s_round_min": 7.752152443886734, "token_e2e_p95_s_round_max": 7.819547306071035 }, { "engine": "vllm", "stage": "core", "case": "short-c8", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 404.24718226854407, "output_tok_s_round_min": 400.2006560758841, "output_tok_s_round_max": 406.9061068621047, "ttft_p95_s_round_median": 1.1756080102641135, "ttft_p95_s_round_min": 1.1754794579464942, "ttft_p95_s_round_max": 1.1825663720956072, "mean_request_tpot_ms_round_median": 11.710464420194892, "mean_request_tpot_ms_round_min": 11.521862637605444, "mean_request_tpot_ms_round_max": 11.864500810753364, "request_tpot_p95_ms_round_median": 18.21562979465688, "request_tpot_p95_ms_round_min": 18.090126008403992, "request_tpot_p95_ms_round_max": 18.38831296630614, "e2e_p95_s_round_median": 2.536837654421106, "e2e_p95_s_round_min": 2.534013267373666, "e2e_p95_s_round_max": 2.5892059521283954, "token_e2e_p95_s_round_median": 2.5368285728618503, "token_e2e_p95_s_round_min": 2.534005962777883, "token_e2e_p95_s_round_max": 2.5891990220407024 }, { "engine": "vllm", "stage": "history", "case": "history-prefill-c4", "rounds": 3, "requests": 48, "failed": 0, "tail_sample_warning": true, "output_tok_s_round_median": 143.86085474243774, "output_tok_s_round_min": 143.5885431829431, "output_tok_s_round_max": 144.12235551124655, "ttft_p95_s_round_median": 2.3852210904005915, "ttft_p95_s_round_min": 2.381105590960942, "ttft_p95_s_round_max": 2.405993028427474, "mean_request_tpot_ms_round_median": 13.01952903195629, "mean_request_tpot_ms_round_min": 12.667486948963974, "mean_request_tpot_ms_round_max": 13.13927584496422, "request_tpot_p95_ms_round_median": 22.966835299492118, "request_tpot_p95_ms_round_min": 22.898622026747667, "request_tpot_p95_ms_round_max": 22.98532594288663, "e2e_p95_s_round_median": 3.57466272264719, "e2e_p95_s_round_min": 3.561960567254573, "e2e_p95_s_round_max": 3.5860501937568188, "token_e2e_p95_s_round_median": 3.574652560870163, "token_e2e_p95_s_round_min": 3.5618795891059563, "token_e2e_p95_s_round_max": 3.5860377566423267 }, { "engine": "vllm", "stage": "history", "case": "history-short-c1", "rounds": 3, "requests": 48, "failed": 1, "tail_sample_warning": true, "output_tok_s_round_median": 98.06015682703746, "output_tok_s_round_min": 91.90806531048567, "output_tok_s_round_max": 98.11907310848738, "ttft_p95_s_round_median": 0.1346695355605334, "ttft_p95_s_round_min": 0.13348053600639104, "ttft_p95_s_round_max": 0.14071969245560467, "mean_request_tpot_ms_round_median": 8.266495612188779, "mean_request_tpot_ms_round_min": 8.25376461845793, "mean_request_tpot_ms_round_max": 8.307687974708628, "request_tpot_p95_ms_round_median": 8.328779534037624, "request_tpot_p95_ms_round_min": 8.319672524544691, "request_tpot_p95_ms_round_max": 8.337855145024756, "e2e_p95_s_round_median": 0.6540480660274625, "e2e_p95_s_round_min": 0.6537297149188817, "e2e_p95_s_round_max": 0.6545965761179104, "token_e2e_p95_s_round_median": 0.6540309075731784, "token_e2e_p95_s_round_min": 0.6537143609020859, "token_e2e_p95_s_round_max": 0.6545847611268982 }, { "engine": "vllm", "stage": "history", "case": "history-short-c4", "rounds": 3, "requests": 96, "failed": 1, "tail_sample_warning": true, "output_tok_s_round_median": 267.36150440903305, "output_tok_s_round_min": 257.9990242227288, "output_tok_s_round_max": 281.9424455858229, "ttft_p95_s_round_median": 0.3771370127098635, "ttft_p95_s_round_min": 0.37513421196490526, "ttft_p95_s_round_max": 0.3804752652766183, "mean_request_tpot_ms_round_median": 10.527712289273502, "mean_request_tpot_ms_round_min": 10.49479037161804, "mean_request_tpot_ms_round_max": 11.19287471709189, "request_tpot_p95_ms_round_median": 13.249855207663678, "request_tpot_p95_ms_round_min": 11.49184503481679, "request_tpot_p95_ms_round_max": 13.3567672326333, "e2e_p95_s_round_median": 0.9828910409938544, "e2e_p95_s_round_min": 0.9701046999543905, "e2e_p95_s_round_max": 1.0911690402776002, "token_e2e_p95_s_round_median": 0.9825404049362987, "token_e2e_p95_s_round_min": 0.969418773194775, "token_e2e_p95_s_round_max": 1.0911586763802916 }, { "engine": "vllm", "stage": "history", "case": "history-short-c8", "rounds": 3, "requests": 192, "failed": 2, "tail_sample_warning": true, "output_tok_s_round_median": 479.300559787369, "output_tok_s_round_min": 478.6819074272021, "output_tok_s_round_max": 486.64530320848854, "ttft_p95_s_round_median": 0.3879160741344094, "ttft_p95_s_round_min": 0.3876152530312538, "ttft_p95_s_round_max": 0.3921008318662643, "mean_request_tpot_ms_round_median": 11.195939296326683, "mean_request_tpot_ms_round_min": 10.999696149877902, "mean_request_tpot_ms_round_max": 11.57703463250476, "request_tpot_p95_ms_round_median": 14.58294540791521, "request_tpot_p95_ms_round_min": 14.559169552318512, "request_tpot_p95_ms_round_max": 14.585373133036589, "e2e_p95_s_round_median": 1.069141366612166, "e2e_p95_s_round_min": 1.063924045022577, "e2e_p95_s_round_max": 1.0780258276965469, "token_e2e_p95_s_round_median": 1.0691353347618133, "token_e2e_p95_s_round_min": 1.0639127211645245, "token_e2e_p95_s_round_max": 1.0772590407636016 }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "rounds": 3, "requests": 600, "failed": 4, "tail_sample_warning": true, "output_tok_s_round_median": 38.07409328424507, "output_tok_s_round_min": 37.88400286056142, "output_tok_s_round_max": 38.07545769143723, "ttft_p95_s_round_median": 3.2486028635408726, "ttft_p95_s_round_min": 3.167877234052866, "ttft_p95_s_round_max": 3.250699667632576, "mean_request_tpot_ms_round_median": 16.994079529128303, "mean_request_tpot_ms_round_min": 16.026746525711793, "mean_request_tpot_ms_round_max": 17.48586776547869, "request_tpot_p95_ms_round_median": 38.643486440313566, "request_tpot_p95_ms_round_min": 36.10068163357851, "request_tpot_p95_ms_round_max": 46.10547348893592, "e2e_p95_s_round_median": 7.158787277596999, "e2e_p95_s_round_min": 5.939398533385247, "e2e_p95_s_round_max": 7.207136430451646, "token_e2e_p95_s_round_median": 7.148214536532755, "token_e2e_p95_s_round_min": 5.939385580364615, "token_e2e_p95_s_round_max": 7.20712829073891 } ], "counts": [ { "engine": "sglang", "stage": "core", "executed_rounds": 36, "planned_rounds": 36, "passed_rounds": 36, "failed_rounds": 0, "requests": 1056, "completed": 1056, "failed_requests": 0 }, { "engine": "sglang", "stage": "history", "executed_rounds": 12, "planned_rounds": 12, "passed_rounds": 12, "failed_rounds": 0, "requests": 384, "completed": 384, "failed_requests": 0 }, { "engine": "sglang", "stage": "slo", "executed_rounds": 9, "planned_rounds": 9, "passed_rounds": 8, "failed_rounds": 1, "requests": 1800, "completed": 1799, "failed_requests": 1 }, { "engine": "vllm", "stage": "core", "executed_rounds": 36, "planned_rounds": 36, "passed_rounds": 32, "failed_rounds": 4, "requests": 1056, "completed": 1051, "failed_requests": 5 }, { "engine": "vllm", "stage": "history", "executed_rounds": 12, "planned_rounds": 12, "passed_rounds": 8, "failed_rounds": 4, "requests": 384, "completed": 380, "failed_requests": 4 }, { "engine": "vllm", "stage": "slo", "executed_rounds": 3, "planned_rounds": 9, "passed_rounds": 0, "failed_rounds": 3, "requests": 600, "completed": 596, "failed_requests": 4 } ], "anomalies": [ { "engine": "sglang", "stage": "slo", "case": "slo-0.8", "round_file": "sglang-slo-0.8-r2.json", "sample_sha256": "9bbc2f5cae2666059e9096d61bc468696102d0a135a44cd721a19abf7052d5d8", "http_status": null, "done": false, "usage": null, "visible_text_length": 0, "error": "ServerDisconnectedError: Server disconnected", "root_cause": "not established" }, { "engine": "vllm", "stage": "core", "case": "decode-c32", "round_file": "vllm-decode-c32-r1.json", "sample_sha256": "5182ef48c5391f436da07aae8b9dfb0ea439a77c8bbae88556c4acb605887e3d", "http_status": 200, "done": true, "usage": { "prompt_tokens": 1024, "total_tokens": 1536, "completion_tokens": 512 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "core", "case": "long-c1", "round_file": "vllm-long-c1-r1.json", "sample_sha256": "ff2bf4b77014ac26a69df85efcdc1bc0efa568875bd023e4b74fb23b10e60286", "http_status": 200, "done": true, "usage": { "prompt_tokens": 24576, "total_tokens": 24704, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "core", "case": "prefill-c32", "round_file": "vllm-prefill-c32-r2.json", "sample_sha256": "9ac2982bb582f1d4950796be5430edc0a17022ed5ca73aa2692e03305b3159a8", "http_status": 200, "done": true, "usage": { "prompt_tokens": 8192, "total_tokens": 8320, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "core", "case": "prefill-c32", "round_file": "vllm-prefill-c32-r2.json", "sample_sha256": "ff218e538b44823797a5ec24d4a355b31ec5a2948dabaac196c7b9c575d2f531", "http_status": 200, "done": true, "usage": { "prompt_tokens": 8192, "total_tokens": 8320, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "core", "case": "short-c32", "round_file": "vllm-short-c32-r1.json", "sample_sha256": "dc415ec2b9da5c6618adc98d28c8f8de8a0f23c492d85692bcd9771529ce045b", "http_status": 200, "done": true, "usage": { "prompt_tokens": 1024, "total_tokens": 1152, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "history", "case": "history-short-c1", "round_file": "vllm-history-short-c1-r3.json", "sample_sha256": "4d6a688598eaa2f0c47b8e34dd7d104e86cf32af916028492e804ec93499e5e8", "http_status": 200, "done": true, "usage": { "prompt_tokens": 128, "total_tokens": 192, "completion_tokens": 64 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "history", "case": "history-short-c4", "round_file": "vllm-history-short-c4-r3.json", "sample_sha256": "7c4780924aa5ec4f313275c1ac74d2675c56bebf29ed9d18f66ead1bd15378ff", "http_status": 200, "done": true, "usage": { "prompt_tokens": 128, "total_tokens": 192, "completion_tokens": 64 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "history", "case": "history-short-c8", "round_file": "vllm-history-short-c8-r2.json", "sample_sha256": "5140c7a316583496bf047d5ef11f9dc936ac5b0a9cdfcfe91822f1edb1edf8c7", "http_status": 200, "done": true, "usage": { "prompt_tokens": 128, "total_tokens": 192, "completion_tokens": 64 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "history", "case": "history-short-c8", "round_file": "vllm-history-short-c8-r3.json", "sample_sha256": "0577577f0e244965c15436806eba513db4098d86959b0032f7278f6a816a3c1b", "http_status": 200, "done": true, "usage": { "prompt_tokens": 128, "total_tokens": 192, "completion_tokens": 64 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "round_file": "vllm-slo-0.5-r1.json", "sample_sha256": "1211bac15b83920d6a817b91bb58153393dfd61451faafb9c5d751f1d792f269", "http_status": 200, "done": true, "usage": { "prompt_tokens": 8192, "total_tokens": 8320, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "round_file": "vllm-slo-0.5-r2.json", "sample_sha256": "3f4b875dd6c0b5a829fafb87d32c04b2ca126ec1ec1ca9eee6a9d06fb673f54f", "http_status": 200, "done": true, "usage": { "prompt_tokens": 8192, "total_tokens": 8320, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "round_file": "vllm-slo-0.5-r2.json", "sample_sha256": "b5b93147725d360eae06027441587bb4b4780fdd85e24fd74ca78af333ad052c", "http_status": 200, "done": true, "usage": { "prompt_tokens": 8192, "total_tokens": 8320, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" }, { "engine": "vllm", "stage": "slo", "case": "slo-0.5", "round_file": "vllm-slo-0.5-r3.json", "sample_sha256": "8b40e630d004a021946a1562c39ec0899f31dacf82356f9f2ca51b730167ff9b", "http_status": 200, "done": true, "usage": { "prompt_tokens": 8192, "total_tokens": 8320, "completion_tokens": 128 }, "visible_text_length": 0, "error": "ValueError: Missing DONE, nonempty content, or completion usage", "root_cause": "not established" } ], "history_template": [ { "engine": "sglang", "requests": 36, "errors": 0, "input_min": 535, "input_max": 542, "output_tokens": 256, "rounds": [ { "phase": "sglang-history-template-r1", "route": "direct", "workload": "short", "concurrency": 1, "requests": 12, "success": 12, "errors": 0, "started_at": 1789092581.6400757, "finished_at": 1789092645.7167172, "elapsed": 64.07663896121085, "output_tokens_per_second": 47.94258952720121, "request_rate": 0.18727574034062971, "ttft": { "p50": 0.21295096748508513, "p95": 0.274882310302928, "p99": 0.3258452645037324, "min": 0.17373228445649147, "max": 0.3385860030539334 }, "tpot": { "p50": 0.020044303984910833, "p95": 0.020209201434170643, "p99": 0.02022442204726166, "min": 0.020026912600459423, "max": 0.020228227200534413 }, "e2e": { "p50": 5.330612887861207, "p95": 5.388459849078208, "p99": 5.436487161163241, "min": 5.3176756533794105, "max": 5.448493989184499 }, "input_tokens": { "p50": 538.5, "p95": 542.0, "p99": 542.0, "min": 535, "max": 542 }, "output_tokens": { "p50": 256.0, "p95": 256.0, "p99": 256.0, "min": 256, "max": 256 } }, { "phase": "sglang-history-template-r2", "route": "direct", "workload": "short", "concurrency": 1, "requests": 12, "success": 12, "errors": 0, "started_at": 1789092676.3709972, "finished_at": 1789092740.2944086, "elapsed": 63.92340896697715, "output_tokens_per_second": 48.057512101505665, "request_rate": 0.1877246566465065, "ttft": { "p50": 0.18030743999406695, "p95": 0.21892260157037527, "p99": 0.21902880659792573, "min": 0.16941477870568633, "max": 0.21905535785481334 }, "tpot": { "p50": 0.02018472517420556, "p95": 0.020195514376585684, "p99": 0.020196894150921237, "min": 0.020015141764180917, "max": 0.020197239094505125 }, "e2e": { "p50": 5.32722706743516, "p95": 5.3303051454247905, "p99": 5.33104912030045, "min": 5.318735359702259, "max": 5.331235114019364 }, "input_tokens": { "p50": 538.5, "p95": 540.0, "p99": 540.0, "min": 535, "max": 540 }, "output_tokens": { "p50": 256.0, "p95": 256.0, "p99": 256.0, "min": 256, "max": 256 } }, { "phase": "sglang-history-template-r3", "route": "direct", "workload": "short", "concurrency": 1, "requests": 12, "success": 12, "errors": 0, "started_at": 1789092770.943006, "finished_at": 1789092834.906732, "elapsed": 63.963724229019135, "output_tokens_per_second": 48.02722225805438, "request_rate": 0.18760633694552492, "ttft": { "p50": 0.2142402594909072, "p95": 0.22019795081578195, "p99": 0.22035350936464965, "min": 0.1808181176893413, "max": 0.22039239900186658 }, "tpot": { "p50": 0.02004961727789658, "p95": 0.020202790989230078, "p99": 0.0202047842842545, "min": 0.020032208878546953, "max": 0.020205282608010605 }, "e2e": { "p50": 5.330561584560201, "p95": 5.333405893458985, "p99": 5.333952924269251, "min": 5.32283855881542, "max": 5.334089681971818 }, "input_tokens": { "p50": 538.0, "p95": 542.0, "p99": 542.0, "min": 535, "max": 542 }, "output_tokens": { "p50": 256.0, "p95": 256.0, "p99": 256.0, "min": 256, "max": 256 } } ], "throughput_round_median": 48.02722225805438, "ttft_p95_ms_round_median": 220.19795081578195, "tpot_p95_ms_round_median": 20.202790989230078 }, { "engine": "vllm", "requests": 81, "errors": 0, "input_min": 534, "input_max": 544, "output_tokens": 256, "rounds": [ { "phase": "vllm-history-template-r1", "route": "direct", "workload": "short", "concurrency": 1, "requests": 27, "success": 27, "errors": 0, "started_at": 1789101561.8255627, "finished_at": 1789101622.6663442, "elapsed": 60.84077876666561, "output_tokens_per_second": 113.60801324566631, "request_rate": 0.44378130174088404, "ttft": { "p50": 0.12964869383722544, "p95": 0.14912538067437708, "p99": 0.14934963908046484, "min": 0.12663777125999331, "max": 0.14941685507073998 }, "tpot": { "p50": 0.008314180901894968, "p95": 0.008329391140015978, "p99": 0.008380220323727995, "min": 0.008243819722430963, "max": 0.008398020650972338 }, "e2e": { "p50": 2.251038245856762, "p95": 2.265823009656742, "p99": 2.2678231056965887, "min": 2.2472371007315814, "max": 2.2683264133520424 }, "input_tokens": { "p50": 537.0, "p95": 542.4, "p99": 543.0, "min": 535, "max": 543 }, "output_tokens": { "p50": 256.0, "p95": 256.0, "p99": 256.0, "min": 256, "max": 256 } }, { "phase": "vllm-history-template-r2", "route": "direct", "workload": "short", "concurrency": 1, "requests": 27, "success": 27, "errors": 0, "started_at": 1789101647.1765864, "finished_at": 1789101707.9551544, "elapsed": 60.778565370012075, "output_tokens_per_second": 113.72430326251754, "request_rate": 0.44423555961920913, "ttft": { "p50": 0.1292103542946279, "p95": 0.14844168699346483, "p99": 0.1504345636535436, "min": 0.1272224080748856, "max": 0.1510078296996653 }, "tpot": { "p50": 0.00831679908522204, "p95": 0.008328120176516036, "p99": 0.008329802987658803, "min": 0.008231702026929341, "max": 0.008330390777658014 }, "e2e": { "p50": 2.2503481712192297, "p95": 2.255453480081633, "p99": 2.256313172252849, "min": 2.2460671956650913, "max": 2.2563428478315473 }, "input_tokens": { "p50": 538.0, "p95": 542.7, "p99": 543.74, "min": 535, "max": 544 }, "output_tokens": { "p50": 256.0, "p95": 256.0, "p99": 256.0, "min": 256, "max": 256 } }, { "phase": "vllm-history-template-r3", "route": "direct", "workload": "short", "concurrency": 1, "requests": 27, "success": 27, "errors": 0, "started_at": 1789101732.4744008, "finished_at": 1789101793.2403762, "elapsed": 60.76597346691415, "output_tokens_per_second": 113.74786917160877, "request_rate": 0.44432761395159676, "ttft": { "p50": 0.12890548119321465, "p95": 0.14822980910539627, "p99": 0.15177728364244103, "min": 0.1270715408027172, "max": 0.15297071682289243 }, "tpot": { "p50": 0.008301722709381697, "p95": 0.008328473271217707, "p99": 0.00836097044658427, "min": 0.008234228752553464, "max": 0.008372366238458483 }, "e2e": { "p50": 2.249778133817017, "p95": 2.2540964944288135, "p99": 2.260422699013725, "min": 2.244986359961331, "max": 2.262473361566663 }, "input_tokens": { "p50": 538.0, "p95": 541.0, "p99": 541.0, "min": 534, "max": 541 }, "output_tokens": { "p50": 256.0, "p95": 256.0, "p99": 256.0, "min": 256, "max": 256 } } ], "throughput_round_median": 113.72430326251754, "ttft_p95_ms_round_median": 148.44168699346483, "tpot_p95_ms_round_median": 8.328473271217707 } ], "source_manifest": [ { "file": "sglang/core/sglang-decode-c1-r1.json", "sha256": "2193ce9af7f5efbcc279b20c96388a4b4cd5ce199c13bfc56be5aba2e059221d" }, { "file": "sglang/core/sglang-decode-c1-r2.json", "sha256": "54cbeac188ffc4d9c2c02cfff507be893fbb2e151d88ecf909202069a089e534" }, { "file": "sglang/core/sglang-decode-c1-r3.json", "sha256": "0be760129b1c923d605b618248d08fff498f6002ad0940f58fc48df13f3ceb22" }, { "file": "sglang/core/sglang-decode-c32-r1.json", "sha256": "deae685d4410445c5684e3b667c6406bca0564bc6e25a6dcdabc95d8afa43adb" }, { "file": "sglang/core/sglang-decode-c32-r2.json", "sha256": "4e6eca532f0b801f25647b716433d22f9aa35d3803c84c11402af985595223a2" }, { "file": "sglang/core/sglang-decode-c32-r3.json", "sha256": "00d49decbe3f3fd36bdcea5e2cc74567911a78b773a32ed15ae10865f864341b" }, { "file": "sglang/core/sglang-decode-c8-r1.json", "sha256": "3135f5747c217695fb70e1e9ff8f72c17f72cd92777645164448f5bce3f673cc" }, { "file": "sglang/core/sglang-decode-c8-r2.json", "sha256": "d81d6c3c2fbd1744c99cfa5a55e708d7025cd5661458f0dc794dee8b38fef525" }, { "file": "sglang/core/sglang-decode-c8-r3.json", "sha256": "1ce1cceb87474d0ff52e9fe6271fb3e982e0892edbb3fbe7c39f23048713e0e0" }, { "file": "sglang/core/sglang-long-c1-r1.json", "sha256": "c24cf5c6d5a06e96fe63a5fa6d42d4707739c19152a54fbe76de406b68399cd8" }, { "file": "sglang/core/sglang-long-c1-r2.json", "sha256": "c4bde89ae18c9cb15dfa1bfb1b415116d8774f5711a140f9e30a11e86553803d" }, { "file": "sglang/core/sglang-long-c1-r3.json", "sha256": "dd53265a8c52c9bb002acc6b8d971e23409b9e5a50ca5a6e8923e310be9c3047" }, { "file": "sglang/core/sglang-long-c32-r1.json", "sha256": "66a0555f695bb92fb7e017cef06ed14270f3b7202c9de93469030d3f1b7fee0d" }, { "file": "sglang/core/sglang-long-c32-r2.json", "sha256": "638ab46696667b1ccf4aab75ba041da3d400425f7970fe9fb2cedff80a27716c" }, { "file": "sglang/core/sglang-long-c32-r3.json", "sha256": "c05d5eaf0441893f3bf6591e6dffab7cd5a78c60e3ce10971ab128176c64ff96" }, { "file": "sglang/core/sglang-long-c8-r1.json", "sha256": "b494b2967cb664ba84f59da6ec326de5c8a8b9f83512e496f1047303b0c90eac" }, { "file": "sglang/core/sglang-long-c8-r2.json", "sha256": "b240de17e6c716ed9b67cbf62c5ca236b1375e2c9211698f980f33bce3f8e1c2" }, { "file": "sglang/core/sglang-long-c8-r3.json", "sha256": "beb76619e2dc1d0e6e36705d39cb88f81000f86cc3c1135e92d220991dde7379" }, { "file": "sglang/core/sglang-prefill-c1-r1.json", "sha256": "223c1d01dc8f6c676e1a1961d58a85cadc7ca43f32811397bfeca6778baf5b93" }, { "file": "sglang/core/sglang-prefill-c1-r2.json", "sha256": "83341686707487edf44d05a3b04a24c7d32fd8becd0dd944b38eb77b0ce0bb35" }, { "file": "sglang/core/sglang-prefill-c1-r3.json", "sha256": "569e89daa46df39540ce05e12f6306d38822b805e535d82fd6b54386daba53b0" }, { "file": "sglang/core/sglang-prefill-c32-r1.json", "sha256": "836375c40b1fa9c246261f0bc4d7ba70513d445d3ee6e0759be9c4976f89dfb5" }, { "file": "sglang/core/sglang-prefill-c32-r2.json", "sha256": "0b3455ace33e41239ad974efe984cbff6b788a8ed07c58436c0e53d8b55dc3cb" }, { "file": "sglang/core/sglang-prefill-c32-r3.json", "sha256": "eb8a0974679394db45a91cb470f18759bf3ea8e00384aa3a4b084cdf579a8d0a" }, { "file": "sglang/core/sglang-prefill-c8-r1.json", "sha256": "7e2384620a514a6a37e599b37119a407378fb758a3738a136c5be35cedc7b139" }, { "file": "sglang/core/sglang-prefill-c8-r2.json", "sha256": "1e81ca1f6a1448505ebedd0d108f70bb66fc06333896b29acad2e758aaf1c297" }, { "file": "sglang/core/sglang-prefill-c8-r3.json", "sha256": "edb145126bed097c65beaa264aadd81ab736106387126276b5823e1f14b974b5" }, { "file": "sglang/core/sglang-short-c1-r1.json", "sha256": "eca8fee4d4079ffcbe2aac489e9a0e78dafc290643f3d7522172bfc47694c6d0" }, { "file": "sglang/core/sglang-short-c1-r2.json", "sha256": "ad207cc92626e8c818d93fed4f7d6a8f59bd3d9351119def2613956b36b408dc" }, { "file": "sglang/core/sglang-short-c1-r3.json", "sha256": "37e4e5c21d8eb82eeb13ef7a53a5215b7c6f801900e7a0916b977ad3d1c3f8b1" }, { "file": "sglang/core/sglang-short-c32-r1.json", "sha256": "f62df96982e82169f11e396ab0255bbaf4f40a2da667c7e079ba06901eb4fec3" }, { "file": "sglang/core/sglang-short-c32-r2.json", "sha256": "9e1a8c4335ca965d15dfdac609b7c88d1607295193bbf790e1521b315df275c3" }, { "file": "sglang/core/sglang-short-c32-r3.json", "sha256": "7859729aa5a8a06de66c934c008b574391e4b44d4528d5bb01750c3904293a33" }, { "file": "sglang/core/sglang-short-c8-r1.json", "sha256": "197cf9a2eb1a8f15dff0223387510bb77710fa55b4fc2c847b4c2929190b091d" }, { "file": "sglang/core/sglang-short-c8-r2.json", "sha256": "f516292a8ade2c9eea6549beb8fa7b150ed1243dd9755eab17eb8c76487bc819" }, { "file": "sglang/core/sglang-short-c8-r3.json", "sha256": "386888f4557a5891b1d931b53aa56dc2506328c4b50e67088be7c9f39e5f3a8e" }, { "file": "sglang/history/sglang-history-prefill-c4-r1.json", "sha256": "c34b38f65bad09bc4f8bc96a8e23f562531abc373eeee209840bfae5a45df5fd" }, { "file": "sglang/history/sglang-history-prefill-c4-r2.json", "sha256": "61670d329ae1d0453f2210eb2f8e21332b6a93314070ff2d0efa67a047552771" }, { "file": "sglang/history/sglang-history-prefill-c4-r3.json", "sha256": "51231c9eeb133053f65a0a565903eb8de39246522ea765be9eeffbbf70ac5e2b" }, { "file": "sglang/history/sglang-history-short-c1-r1.json", "sha256": "368d8e8d460ccc288a61167884be3a458af094b808e30fe874304c0e5f83746c" }, { "file": "sglang/history/sglang-history-short-c1-r2.json", "sha256": "9b7907377822ec543d9988373f831f3c4551ab8a08dba83f621eb522b0d6fc07" }, { "file": "sglang/history/sglang-history-short-c1-r3.json", "sha256": "21d201dd123ae8e37144ea8183969a0698e7818424ab1c7817938c929aabf7b2" }, { "file": "sglang/history/sglang-history-short-c4-r1.json", "sha256": "aafe96b699c6b4ff892a3faa67a9fb3982a2e5c7de5c667bb162f132f0738215" }, { "file": "sglang/history/sglang-history-short-c4-r2.json", "sha256": "945d8c6b2b8d339b62223039c13a029ebe23c848cf1f97c295569638909dfd4c" }, { "file": "sglang/history/sglang-history-short-c4-r3.json", "sha256": "0857c4f34d7ffa01bcc5fb826f89fc86725fe1804cd90c952514a6e7ef031534" }, { "file": "sglang/history/sglang-history-short-c8-r1.json", "sha256": "9ba6f14dca5e639896a28989fce9446856793a13cf7376effce6c6b4c5ba4d88" }, { "file": "sglang/history/sglang-history-short-c8-r2.json", "sha256": "2cdb19458f461078191d248757e0285baa4737708994de257a1191131d9227e3" }, { "file": "sglang/history/sglang-history-short-c8-r3.json", "sha256": "6f32cdd3b2bd45e53a39f4eb2ab5928ff3a5fcfaafb672af485b541281413453" }, { "file": "sglang/slo/sglang-slo-0.5-r1.json", "sha256": "9f3de2b469401536f19705544bb1270fb409e396b29e7dd8bc449a4d86480d79" }, { "file": "sglang/slo/sglang-slo-0.5-r2.json", "sha256": "c391a4b674dd4b1cd6a764aeac85948ef99c2624cf5418221aed92ed3cd68fd7" }, { "file": "sglang/slo/sglang-slo-0.5-r3.json", "sha256": "b33ec65d62c1c58062b9b6a8137ebf547f89b744a6626ae580b90aff959747c5" }, { "file": "sglang/slo/sglang-slo-0.8-r1.json", "sha256": "95c2c3b563a0b3335105b6dc1b2ebe69af0c5cfb8acbf003dd936b4f4d175c82" }, { "file": "sglang/slo/sglang-slo-0.8-r2.json", "sha256": "ec94640028d128abf1a2098245a7bc57321dad4b2637f73ed0ae0cb79921f962" }, { "file": "sglang/slo/sglang-slo-0.8-r3.json", "sha256": "f2b3eae3b981991398a3156ea392c2d7042b27aa62710e96e0ed95ef334859b8" }, { "file": "sglang/slo/sglang-slo-1.1-r1.json", "sha256": "912f2d09f47fd693719d916bec72f9a3b5f4bdd8655f2b170160c9ae7e7403a1" }, { "file": "sglang/slo/sglang-slo-1.1-r2.json", "sha256": "a54a3654206c09f41c3fdd6ae7176a10e4c948307ce774b8faa64fe332a618f5" }, { "file": "sglang/slo/sglang-slo-1.1-r3.json", "sha256": "04da43bd075aad0d4eb44bc82600acb20c7143f3e3cc4f63ec8d0234075f9418" }, { "file": "vllm/core/vllm-decode-c1-r1.json", "sha256": "8fab3a3364f41a256620ba3264f00c500c7b7a5e2696373ce7eceb1f5ceddb33" }, { "file": "vllm/core/vllm-decode-c1-r2.json", "sha256": "37693bd96c83ee40618e87527cc4b72549015c42c42a22f3e45cf3fe2a9f73dd" }, { "file": "vllm/core/vllm-decode-c1-r3.json", "sha256": "c3bfa471b3a9358d73549c2252e1eab5584954bb2b3b967f91cf56ee7dafef04" }, { "file": "vllm/core/vllm-decode-c32-r1.json", "sha256": "c8699b151b4b067a8f193c2dabbb6848a01aa5c206e80b17e4707a4a559694e7" }, { "file": "vllm/core/vllm-decode-c32-r2.json", "sha256": "3f364173d4b043bb5dfd7fc356288d88bcd5cb135608aac83b7a1d4d91f174cb" }, { "file": "vllm/core/vllm-decode-c32-r3.json", "sha256": "8a7383278bfe611ffea89bc89a5f744590184216b34073bc3c402caeada0dd69" }, { "file": "vllm/core/vllm-decode-c8-r1.json", "sha256": "71b6dbd5829df7517ab3077aaf708e9ba0060cf6d24d7b38bc22ec1ce168b749" }, { "file": "vllm/core/vllm-decode-c8-r2.json", "sha256": "2fabda43b5c0360f780f7fafeb9e58ef7d8284249c66028c12933e6448f24930" }, { "file": "vllm/core/vllm-decode-c8-r3.json", "sha256": "760f20fcc1948fe1b9d8823a8a69fb6dbad4fc4933f76e6accb5823e5a9293eb" }, { "file": "vllm/core/vllm-long-c1-r1.json", "sha256": "7ad4a24ffc8ed4d06b7b302bd0e9c05a9b4deddd5b08fe8891a8bddc6530b98d" }, { "file": "vllm/core/vllm-long-c1-r2.json", "sha256": "b2be11b920c96b60373e82f1deb5ea8b2de2d890d7b566855220aa57102f7b9a" }, { "file": "vllm/core/vllm-long-c1-r3.json", "sha256": "182bb15fe3d65f27107e3042add5dbc5fc8b109702d1f5e05070e5145d929ef2" }, { "file": "vllm/core/vllm-long-c32-r1.json", "sha256": "ac6a0356e9556e6dba90fbabc106be527e7ef7aa6e5c3b720177be4c0fa66603" }, { "file": "vllm/core/vllm-long-c32-r2.json", "sha256": "64d236839af9e17137f6541984c02f01917e35af06457d6a7621fd19f2062f5e" }, { "file": "vllm/core/vllm-long-c32-r3.json", "sha256": "e72758d0cdbd8ce608135973aa5672afcd79fa48270cbb9efeb2dbb6785f8212" }, { "file": "vllm/core/vllm-long-c8-r1.json", "sha256": "c596647a4e4b376796b738574565ef38d62382139d8ec64d6b2f71d445408129" }, { "file": "vllm/core/vllm-long-c8-r2.json", "sha256": "e60235d82da099702da30cdc0d48281b237afb1e1800f1cf057efc7ff7ba9f09" }, { "file": "vllm/core/vllm-long-c8-r3.json", "sha256": "9b272c6e9af3c4067c30ae2a10acf96162b2c6977869ab8edc8810efb61c557b" }, { "file": "vllm/core/vllm-prefill-c1-r1.json", "sha256": "d06c895953dd94815df474bb1ae2c8a0dd9d636e44369b466fdc8f3bd7ca3393" }, { "file": "vllm/core/vllm-prefill-c1-r2.json", "sha256": "f8da8a172c9b016c8bfe64fc8070091c16a3c8c7f5d38a98cfe94c29e6a5e72f" }, { "file": "vllm/core/vllm-prefill-c1-r3.json", "sha256": "b7e939d5a7dbc6a7884203347141bd4c2dfba5f13b921b46bface22ce4aa6ddb" }, { "file": "vllm/core/vllm-prefill-c32-r1.json", "sha256": "163bb5f6bba3cef9232839d1c1c49d72e9293e7508e1b21fb00307dc783602d9" }, { "file": "vllm/core/vllm-prefill-c32-r2.json", "sha256": "5bb5cf72b9002efefe5cd4977df1c2eaa6fec20af1999ac7b61b3d2ab9eab17a" }, { "file": "vllm/core/vllm-prefill-c32-r3.json", "sha256": "16449be4f92ca13c09b0279aefb1678dad612d6c8be501a9bca4f677a2191b59" }, { "file": "vllm/core/vllm-prefill-c8-r1.json", "sha256": "77d07566f870417ca79941cc40f1e4d8f0cd4caab2c3152f2f1e6e6a31c6d4be" }, { "file": "vllm/core/vllm-prefill-c8-r2.json", "sha256": "40a81a4f637c701b75dbf136142f8c591c5fcd9f73679bb9149a02832e8425f2" }, { "file": "vllm/core/vllm-prefill-c8-r3.json", "sha256": "5a7888a2b2b991f19adb7c85469ce14f8fc7912c37a693ed2c26607f8677b99f" }, { "file": "vllm/core/vllm-short-c1-r1.json", "sha256": "a5460fbb265c2c808394247fd5671d2707294b58f62430ede94402355b77d722" }, { "file": "vllm/core/vllm-short-c1-r2.json", "sha256": "3fb7a840160090f5c293c1748f4c42cb404ffb22801eff5ed1b368018606fae5" }, { "file": "vllm/core/vllm-short-c1-r3.json", "sha256": "e909b32a1202f44db8d8cfa686224b56a20ab5e2b3227dd55c2df17032bd0536" }, { "file": "vllm/core/vllm-short-c32-r1.json", "sha256": "e684da1215d56fe2a05c2654ce92584f71516d6d7b66d5d09f2038d2aab7b8e5" }, { "file": "vllm/core/vllm-short-c32-r2.json", "sha256": "5101bc15213526e65b3ca21a03850167ebafa8c50cacf41579e8aa14d0ba5444" }, { "file": "vllm/core/vllm-short-c32-r3.json", "sha256": "3d63d3e9f64cec5168e2efddb6af6d4ba8fa16ba25e6be124eef925f3a4cac35" }, { "file": "vllm/core/vllm-short-c8-r1.json", "sha256": "169a25a7905f703eae58a59c5c6f40c607c35721d06fa419f0c083a5898a8935" }, { "file": "vllm/core/vllm-short-c8-r2.json", "sha256": "e521a7f9e983335f714505d557887f335d55bcdc59ffc1fa3e2629e1011d7443" }, { "file": "vllm/core/vllm-short-c8-r3.json", "sha256": "a5f89d6d88db9c126bb891ab2458fae35e69128922b53c8bb473b75afb21fa80" }, { "file": "vllm/history/vllm-history-prefill-c4-r1.json", "sha256": "7e6fd2cd43860e91ae86340654fa78615cc60f169eabc24ab4069952c3e6662d" }, { "file": "vllm/history/vllm-history-prefill-c4-r2.json", "sha256": "52199869958e5450f3f86aed2b7db4f58b5b3fbea0b5e1cbdcf068981f0fe06b" }, { "file": "vllm/history/vllm-history-prefill-c4-r3.json", "sha256": "4b4ad6acf8b9282678e23041a332277eedcc759e45133bd3bb021f6956b26252" }, { "file": "vllm/history/vllm-history-short-c1-r1.json", "sha256": "7607850b77bb6299651d0db21f200b641c2fa5ac578be24fb87492b02021ac9b" }, { "file": "vllm/history/vllm-history-short-c1-r2.json", "sha256": "f591150811733512ddf68071efa7a1fe16981ce7507df6d73fa7556005c3d516" }, { "file": "vllm/history/vllm-history-short-c1-r3.json", "sha256": "4543a8e473c39f9c124130d532bfd0f14e74abd95ab9a0946cee99dbafe93412" }, { "file": "vllm/history/vllm-history-short-c4-r1.json", "sha256": "ee224c1c1304958407fbffef44dcfcb25b9513bd18d51c783b2f932169ae30aa" }, { "file": "vllm/history/vllm-history-short-c4-r2.json", "sha256": "34644da907627c47c9d183c15905ba631e68a6f8734e013314e9bacf4c6419c2" }, { "file": "vllm/history/vllm-history-short-c4-r3.json", "sha256": "61ca39e3589a4c95ac00699eba462163c366bbb3a8431ff8362e3794d127b0ef" }, { "file": "vllm/history/vllm-history-short-c8-r1.json", "sha256": "33c8457eb80eaad641b21a0f236b130f153345c2822dc06f144a917e5ff9fc50" }, { "file": "vllm/history/vllm-history-short-c8-r2.json", "sha256": "c8e9135689d9a7152f6c104811f6a1b6203a0bf470a082ac62190e797a4f7292" }, { "file": "vllm/history/vllm-history-short-c8-r3.json", "sha256": "db370f6762672ab8c95a64d5beb0d10e8db32023fdd75cd4ab8b3f9dbf2bad80" }, { "file": "vllm/slo/vllm-slo-0.5-r1.json", "sha256": "a83b82c77c326afdcab2de12be278912db0f92aefa7d6e5f3933c0388ed496d5" }, { "file": "vllm/slo/vllm-slo-0.5-r2.json", "sha256": "e7c55680cd0d23e2b9f8b15fcef295b1135a0eb139724e25927dd083424c1413" }, { "file": "vllm/slo/vllm-slo-0.5-r3.json", "sha256": "860b8b793b414b19a482aa7a36ba87aeda323e712b3fc0fe1e1687d52bd1c162" } ], "public_scope": "Recorded attempt only. No supplemental inference or benchmark. Grafana screenshots retrieved from retained historical metrics. Warmups/calibration excluded from benchmark tables.", "client_adapter_sha256": "1c02882a16a8ce4db7188f418e89339502bbd5e25f2ac36028275e5527988184", "grafana_evidence": "grafana-evidence.json" }