{ "model": "MiniMaxAI/MiniMax-M3.1-Flash", "model_revision": "54e4e69dfc452b1e94a426c24a4fdeeaa8e3ad5e", "status": "COMPLETE", "comparison_design": { "same_model_snapshot": true, "same_gpu_model": "8x H20-3e", "different_physical_nodes": true, "tensor_parallel_size": 8, "prefix_cache": false, "speculative_decoding": false, "kv_cache_storage_dtype_observed_both": "FP8", "client_image_digest": "sha256:f5ec69891fb7b855762f40598eea1aa5cad6e4f2f0c2a2f81d270e2bdcd93632", "interpretation": "Measured deployment configurations; not an isolated engine-only experiment." }, "vllm": { "model": "MiniMaxAI/MiniMax-M3.1-Flash", "model_revision": "54e4e69dfc452b1e94a426c24a4fdeeaa8e3ad5e", "engine": "vllm", "image_source": "vllm/vllm-openai:minimax-m31-flash", "image_digest": "sha256:f5ec69891fb7b855762f40598eea1aa5cad6e4f2f0c2a2f81d270e2bdcd93632", "gpu": "8x H20-3e", "tensor_parallel": 8, "measurement": "median of three round-level metrics; warmups and calibration excluded", "core_rounds": 57, "history_rounds": 9, "formal_requests": 1944, "formal_failed": 0, "core": [ { "case": "short-c1", "input_tokens": 128, "output_tokens": 128, "concurrency": 1, "num_prompts": 8, "rounds": 3, "output_throughput": { "median": 141.62682295450878, "min": 141.04739561675947, "max": 141.6842759149404 }, "median_ttft_ms": { "median": 124.75649872794747, "min": 124.51903335750103, "max": 131.18186173960567 }, "p95_ttft_ms": { "median": 126.21620502322911, "min": 125.37336368113755, "max": 133.72405231930315 }, "p95_tpot_ms": { "median": 6.13511779609569, "min": 6.13327135339262, "max": 6.137426457859165 }, "p95_e2el_ms": { "median": 904.6409559436142, "min": 903.621541056782, "max": 911.3691582810134 } }, { "case": "short-c4", "input_tokens": 128, "output_tokens": 128, "concurrency": 4, "num_prompts": 16, "rounds": 3, "output_throughput": { "median": 392.9423711220021, "min": 370.35047471368756, "max": 395.15748020291755 }, "median_ttft_ms": { "median": 256.29051914438605, "min": 131.86384364962578, "max": 321.64023723453283 }, "p95_ttft_ms": { "median": 393.4230257291347, "min": 305.03305210731924, "max": 396.2191704194993 }, "p95_tpot_ms": { "median": 9.846725983281187, "min": 9.193790543047111, "max": 9.855180542446732 }, "p95_e2el_ms": { "median": 1389.480191282928, "min": 1385.1381831336766, "max": 1391.2165742367506 } }, { "case": "short-c8", "input_tokens": 128, "output_tokens": 128, "concurrency": 8, "num_prompts": 32, "rounds": 3, "output_throughput": { "median": 632.9888419583704, "min": 631.9418874064327, "max": 636.6566193322359 }, "median_ttft_ms": { "median": 389.8476385511458, "min": 385.4289618320763, "max": 391.18376234546304 }, "p95_ttft_ms": { "median": 404.71481708809733, "min": 403.94685505889356, "max": 408.44997107051313 }, "p95_tpot_ms": { "median": 11.612317433551423, "min": 11.604287051852411, "max": 11.684560419727616 }, "p95_e2el_ms": { "median": 1628.0483715701848, "min": 1623.7005817238241, "max": 1644.7363621555269 } }, { "case": "short-c16", "input_tokens": 128, "output_tokens": 128, "concurrency": 16, "num_prompts": 64, "rounds": 3, "output_throughput": { "median": 1074.6405131181675, "min": 1070.5263263353315, "max": 1075.4415946232057 }, "median_ttft_ms": { "median": 350.60447501018643, "min": 348.1759470887482, "max": 352.9473482631147 }, "p95_ttft_ms": { "median": 372.57025414146483, "min": 372.0983864273876, "max": 379.8438461031765 }, "p95_tpot_ms": { "median": 13.504818420651851, "min": 13.489998267288863, "max": 13.510359802070674 }, "p95_e2el_ms": { "median": 1930.2394755650312, "min": 1928.9450229145586, "max": 1957.5945022050291 } }, { "case": "short-c32", "input_tokens": 128, "output_tokens": 128, "concurrency": 32, "num_prompts": 128, "rounds": 3, "output_throughput": { "median": 1585.4454894542205, "min": 1581.1227114951055, "max": 1586.255742381943 }, "median_ttft_ms": { "median": 514.2971337772906, "min": 513.2594336755574, "max": 514.9108972400427 }, "p95_ttft_ms": { "median": 549.7874869965017, "min": 549.5766213629395, "max": 550.3696689382195 }, "p95_tpot_ms": { "median": 16.4237592310652, "min": 16.320307950276558, "max": 16.511528664745214 }, "p95_e2el_ms": { "median": 2626.10334912315, "min": 2596.167112700641, "max": 2632.8399680554867 } }, { "case": "rag4k-c1", "input_tokens": 4096, "output_tokens": 256, "concurrency": 1, "num_prompts": 4, "rounds": 3, "output_throughput": { "median": 123.90122920955163, "min": 123.89517400787076, "max": 123.95350129430386 }, "median_ttft_ms": { "median": 500.88235456496477, "min": 500.6796573288739, "max": 500.91930804774165 }, "p95_ttft_ms": { "median": 502.0938668865711, "min": 501.7043961212039, "max": 503.82344340905547 }, "p95_tpot_ms": { "median": 6.139060927591488, "min": 6.137065379423839, "max": 6.141070855197077 }, "p95_e2el_ms": { "median": 2067.8762378636748, "min": 2066.782518615946, "max": 2067.980425339192 } }, { "case": "rag4k-c4", "input_tokens": 4096, "output_tokens": 256, "concurrency": 4, "num_prompts": 8, "rounds": 3, "output_throughput": { "median": 266.4088208472325, "min": 265.87634432184, "max": 266.94093908880467 }, "median_ttft_ms": { "median": 1596.737185958773, "min": 1595.1698329299688, "max": 1596.7446062713861 }, "p95_ttft_ms": { "median": 1845.1737678144127, "min": 1842.167653888464, "max": 1847.2788562532514 }, "p95_tpot_ms": { "median": 13.095445108106908, "min": 13.048445101571726, "max": 13.107244901395603 }, "p95_e2el_ms": { "median": 3852.727848337963, "min": 3836.6540377028286, "max": 3876.121106557548 } }, { "case": "rag4k-c8", "input_tokens": 4096, "output_tokens": 256, "concurrency": 8, "num_prompts": 16, "rounds": 3, "output_throughput": { "median": 340.80479281082125, "min": 340.47367196086446, "max": 341.799779491642 }, "median_ttft_ms": { "median": 2658.9468736201525, "min": 2658.7329879403114, "max": 2659.3425758183002 }, "p95_ttft_ms": { "median": 3582.5369860976934, "min": 3577.156755141914, "max": 3582.558373454958 }, "p95_tpot_ms": { "median": 21.405001169107123, "min": 21.30312811053705, "max": 21.462718812821837 }, "p95_e2el_ms": { "median": 6451.922165695578, "min": 6417.065714253113, "max": 6520.399931352586 } }, { "case": "rag4k-c16", "input_tokens": 4096, "output_tokens": 256, "concurrency": 16, "num_prompts": 32, "rounds": 3, "output_throughput": { "median": 397.48259843148014, "min": 397.0547730491379, "max": 400.35175899945443 }, "median_ttft_ms": { "median": 3466.1086131818593, "min": 3462.8119585104287, "max": 3467.2704651020467 }, "p95_ttft_ms": { "median": 6787.214458826929, "min": 6786.962133133784, "max": 6789.5977889187625 }, "p95_tpot_ms": { "median": 36.38045915347688, "min": 36.07734268987734, "max": 36.45262991315593 }, "p95_e2el_ms": { "median": 13261.160863982512, "min": 13237.653969973324, "max": 13305.86461564526 } }, { "case": "agent8k-c1", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 1, "num_prompts": 4, "rounds": 3, "output_throughput": { "median": 141.2143322634197, "min": 141.18847984456983, "max": 141.26872618600441 }, "median_ttft_ms": { "median": 934.3557367101312, "min": 930.9955281205475, "max": 935.6515710242093 }, "p95_ttft_ms": { "median": 936.0924287699163, "min": 934.1624373104423, "max": 937.2867114841938 }, "p95_tpot_ms": { "median": 6.174532290710133, "min": 6.174268047821472, "max": 6.1787777258044985 }, "p95_e2el_ms": { "median": 7252.314251707867, "min": 7247.645076923072, "max": 7255.091740097851 } }, { "case": "agent8k-c4", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 4, "num_prompts": 8, "rounds": 3, "output_throughput": { "median": 347.8166819813314, "min": 345.0564539655584, "max": 348.8698047285716 }, "median_ttft_ms": { "median": 3152.3243892006576, "min": 3151.522034779191, "max": 3157.287331763655 }, "p95_ttft_ms": { "median": 3632.8457709867507, "min": 3630.979935778305, "max": 3635.476071946323 }, "p95_tpot_ms": { "median": 10.603174780115834, "min": 10.575740320611663, "max": 10.674973282478714 }, "p95_e2el_ms": { "median": 11851.367546292022, "min": 11843.756904033944, "max": 11907.504843035713 } }, { "case": "agent8k-c8", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 8, "num_prompts": 16, "rounds": 3, "output_throughput": { "median": 474.9745413318611, "min": 470.40036952277103, "max": 479.2172344435182 }, "median_ttft_ms": { "median": 4498.17433161661, "min": 4496.5641484595835, "max": 4502.065575681627 }, "p95_ttft_ms": { "median": 7187.767590396106, "min": 7187.137183267623, "max": 7191.921853693202 }, "p95_tpot_ms": { "median": 15.830224852158688, "min": 15.720319424374422, "max": 16.034068126196836 }, "p95_e2el_ms": { "median": 19161.87181114219, "min": 18750.32052048482, "max": 19388.964678393677 } }, { "case": "agent8k-c16", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 16, "num_prompts": 32, "rounds": 3, "output_throughput": { "median": 591.4484932198878, "min": 590.8662592057296, "max": 594.6847529175591 }, "median_ttft_ms": { "median": 4463.290351908654, "min": 4462.433964479715, "max": 5357.145204208791 }, "p95_ttft_ms": { "median": 13837.3658569064, "min": 13833.062936738132, "max": 13838.446102431042 }, "p95_tpot_ms": { "median": 25.09189950487051, "min": 24.97532999403374, "max": 25.150703881500345 }, "p95_e2el_ms": { "median": 36721.269664820284, "min": 36121.313523175195, "max": 37140.99941854365 } }, { "case": "rag16k-c1", "input_tokens": 16384, "output_tokens": 256, "concurrency": 1, "num_prompts": 4, "rounds": 3, "output_throughput": { "median": 74.06215847147892, "min": 74.02829171604144, "max": 74.09826275691024 }, "median_ttft_ms": { "median": 1884.377155918628, "min": 1883.6494754068553, "max": 1886.029338464141 }, "p95_ttft_ms": { "median": 1889.1413677949458, "min": 1887.076710863039, "max": 1889.948385162279 }, "p95_tpot_ms": { "median": 6.167176055046273, "min": 6.162383847084699, "max": 6.170184647335725 }, "p95_e2el_ms": { "median": 3456.7323146853596, "min": 3455.0892788451165, "max": 3460.205047018826 } }, { "case": "rag16k-c4", "input_tokens": 16384, "output_tokens": 256, "concurrency": 4, "num_prompts": 8, "rounds": 3, "output_throughput": { "median": 109.28704921502572, "min": 109.09483951001238, "max": 109.34713603235328 }, "median_ttft_ms": { "median": 5478.873309213668, "min": 5477.281545754522, "max": 5480.892009567469 }, "p95_ttft_ms": { "median": 7333.353869430721, "min": 7332.830786891282, "max": 7355.14025981538 }, "p95_tpot_ms": { "median": 29.24876368659384, "min": 29.243882537509005, "max": 29.303902632684686 }, "p95_e2el_ms": { "median": 9397.940281452611, "min": 9368.94670710899, "max": 9407.477190997452 } }, { "case": "rag16k-c8", "input_tokens": 16384, "output_tokens": 256, "concurrency": 8, "num_prompts": 16, "rounds": 3, "output_throughput": { "median": 119.8847146232201, "min": 119.54598931014047, "max": 119.94036499354152 }, "median_ttft_ms": { "median": 8718.586056958884, "min": 8165.029070340097, "max": 8721.402467228472 }, "p95_ttft_ms": { "median": 13899.892593501136, "min": 13893.665686016902, "max": 13905.897402670234 }, "p95_tpot_ms": { "median": 56.89551981403401, "min": 56.82653373965592, "max": 56.999091588545085 }, "p95_e2el_ms": { "median": 21072.09875737317, "min": 21061.378204263747, "max": 22974.001393886283 } }, { "case": "decode2k-c1", "input_tokens": 128, "output_tokens": 2048, "concurrency": 1, "num_prompts": 4, "rounds": 3, "output_throughput": { "median": 161.36930928600358, "min": 160.4903615102954, "max": 161.38694305293126 }, "median_ttft_ms": { "median": 125.71988580748439, "min": 124.95411606505513, "max": 202.22873939201236 }, "p95_ttft_ms": { "median": 137.74620397016403, "min": 125.87943770922722, "max": 254.04311101883647 }, "p95_tpot_ms": { "median": 6.140351689292793, "min": 6.139566231988512, "max": 6.141902343358646 }, "p95_e2el_ms": { "median": 12694.102395465598, "min": 12692.195628909394, "max": 12816.53116075322 } }, { "case": "decode2k-c4", "input_tokens": 128, "output_tokens": 2048, "concurrency": 4, "num_prompts": 8, "rounds": 3, "output_throughput": { "median": 499.80628851307125, "min": 499.3658317693937, "max": 500.993383462091 }, "median_ttft_ms": { "median": 264.7086759097874, "min": 262.4966842122376, "max": 265.6393637880683 }, "p95_ttft_ms": { "median": 401.4556393958628, "min": 400.2818755339831, "max": 403.2369466498494 }, "p95_tpot_ms": { "median": 7.938894899753088, "min": 7.930627740800388, "max": 7.9711587289753165 }, "p95_e2el_ms": { "median": 16502.923278324306, "min": 16388.078751647845, "max": 16506.82289004326 } }, { "case": "decode2k-c8", "input_tokens": 128, "output_tokens": 2048, "concurrency": 8, "num_prompts": 16, "rounds": 3, "output_throughput": { "median": 808.388155620451, "min": 805.089404427409, "max": 821.3330297276834 }, "median_ttft_ms": { "median": 385.17776457592845, "min": 260.5486032553017, "max": 385.3724109940231 }, "p95_ttft_ms": { "median": 404.05780472792685, "min": 278.266403125599, "max": 457.56519795395434 }, "p95_tpot_ms": { "median": 9.907889749052236, "min": 9.670689188259683, "max": 9.938211905946078 }, "p95_e2el_ms": { "median": 20482.862630859017, "min": 19969.596231123433, "max": 20510.268366429955 } } ], "core_round_data": [ { "case": "short-c1", "repeat": 1, "source_sha256": "a786283c6c970ec555e6ed35b71e2a921808b4f945accda11636210cc76f05a8", "num_prompts": 8, "max_concurrency": 1, "completed": 8, "failed": 0, "duration": 7.259970987215638, "total_input_tokens": 1024, "total_output_tokens": 1024, "output_throughput": 141.04739561675947, "median_ttft_ms": 131.18186173960567, "p95_ttft_ms": 133.72405231930315, "p95_tpot_ms": 6.137426457859165, "p95_e2el_ms": 911.3691582810134 }, { "case": "short-c1", "repeat": 2, "source_sha256": "f4d3f61f59ae288e5a2cb0994f6118fa12480eb0d59825613f7e432bb089a9fc", "num_prompts": 8, "max_concurrency": 1, "completed": 8, "failed": 0, "duration": 7.227336931973696, "total_input_tokens": 1024, "total_output_tokens": 1024, "output_throughput": 141.6842759149404, "median_ttft_ms": 124.51903335750103, "p95_ttft_ms": 125.37336368113755, "p95_tpot_ms": 6.13327135339262, "p95_e2el_ms": 903.621541056782 }, { "case": "short-c1", "repeat": 3, "source_sha256": "b62477f93f6b6469fd57a162430f542fb733e57fd36e56d31b2795d63aea4ecd", "num_prompts": 8, "max_concurrency": 1, "completed": 8, "failed": 0, "duration": 7.230268805287778, "total_input_tokens": 1024, "total_output_tokens": 1024, "output_throughput": 141.62682295450878, "median_ttft_ms": 124.75649872794747, "p95_ttft_ms": 126.21620502322911, "p95_tpot_ms": 6.13511779609569, "p95_e2el_ms": 904.6409559436142 }, { "case": "short-c4", "repeat": 1, "source_sha256": "fb9fc9e9697bb1d3104b597ea09f8c7f8e7f89636a96d816854aac705bcb3e91", "num_prompts": 16, "max_concurrency": 4, "completed": 16, "failed": 0, "duration": 5.182743849232793, "total_input_tokens": 2048, "total_output_tokens": 2048, "output_throughput": 395.15748020291755, "median_ttft_ms": 256.29051914438605, "p95_ttft_ms": 396.2191704194993, "p95_tpot_ms": 9.193790543047111, "p95_e2el_ms": 1385.1381831336766 }, { "case": "short-c4", "repeat": 2, "source_sha256": "1b408292c1be58daf8b6cc5e584369d79ce4169f52263a65686882a8b75f2ed6", "num_prompts": 16, "max_concurrency": 4, "completed": 16, "failed": 0, "duration": 5.529897056519985, "total_input_tokens": 2048, "total_output_tokens": 2048, "output_throughput": 370.35047471368756, "median_ttft_ms": 321.64023723453283, "p95_ttft_ms": 393.4230257291347, "p95_tpot_ms": 9.846725983281187, "p95_e2el_ms": 1391.2165742367506 }, { "case": "short-c4", "repeat": 3, "source_sha256": "6b00a6f786e204e714d1da5e086506849ead44ab2168bcfbeb34378df3f1a952", "num_prompts": 16, "max_concurrency": 4, "completed": 16, "failed": 0, "duration": 5.211960202082992, "total_input_tokens": 2048, "total_output_tokens": 2048, "output_throughput": 392.9423711220021, "median_ttft_ms": 131.86384364962578, "p95_ttft_ms": 305.03305210731924, "p95_tpot_ms": 9.855180542446732, "p95_e2el_ms": 1389.480191282928 }, { "case": "short-c8", "repeat": 1, "source_sha256": "c7b6a5a6832e4c2b2ba22eb3064c1ec86e628a81499ef6009ed36acc7d7f008e", "num_prompts": 32, "max_concurrency": 8, "completed": 32, "failed": 0, "duration": 6.481608644127846, "total_input_tokens": 4096, "total_output_tokens": 4096, "output_throughput": 631.9418874064327, "median_ttft_ms": 389.8476385511458, "p95_ttft_ms": 403.94685505889356, "p95_tpot_ms": 11.684560419727616, "p95_e2el_ms": 1644.7363621555269 }, { "case": "short-c8", "repeat": 2, "source_sha256": "79e301a7bf439e1022fcaa8539e3ace6d080be01fb90a1fa8f6df78c1ce11fda", "num_prompts": 32, "max_concurrency": 8, "completed": 32, "failed": 0, "duration": 6.4336093831807375, "total_input_tokens": 4096, "total_output_tokens": 4096, "output_throughput": 636.6566193322359, "median_ttft_ms": 391.18376234546304, "p95_ttft_ms": 408.44997107051313, "p95_tpot_ms": 11.604287051852411, "p95_e2el_ms": 1628.0483715701848 }, { "case": "short-c8", "repeat": 3, "source_sha256": "f5179e5c9dc6742c33b4bc709032289a68a332ee18fcf4878d4660c45dcd2d45", "num_prompts": 32, "max_concurrency": 8, "completed": 32, "failed": 0, "duration": 6.470888155512512, "total_input_tokens": 4096, "total_output_tokens": 4096, "output_throughput": 632.9888419583704, "median_ttft_ms": 385.4289618320763, "p95_ttft_ms": 404.71481708809733, "p95_tpot_ms": 11.612317433551423, "p95_e2el_ms": 1623.7005817238241 }, { "case": "short-c16", "repeat": 1, "source_sha256": "e7b85a443b44c714818d5792bca95eaf7fd25c0b2b38e7b9d2126767f54295c6", "num_prompts": 64, "max_concurrency": 16, "completed": 64, "failed": 0, "duration": 7.617336023598909, "total_input_tokens": 8192, "total_output_tokens": 8192, "output_throughput": 1075.4415946232057, "median_ttft_ms": 348.1759470887482, "p95_ttft_ms": 372.0983864273876, "p95_tpot_ms": 13.510359802070674, "p95_e2el_ms": 1928.9450229145586 }, { "case": "short-c16", "repeat": 2, "source_sha256": "0acaad12753655b6eb5e5463a73ec17007a2f2f3c5c21b297d217d39603a8d42", "num_prompts": 64, "max_concurrency": 16, "completed": 64, "failed": 0, "duration": 7.652310642413795, "total_input_tokens": 8192, "total_output_tokens": 8192, "output_throughput": 1070.5263263353315, "median_ttft_ms": 350.60447501018643, "p95_ttft_ms": 379.8438461031765, "p95_tpot_ms": 13.489998267288863, "p95_e2el_ms": 1957.5945022050291 }, { "case": "short-c16", "repeat": 3, "source_sha256": "92788de8e56c93021f43ced5de6ce9938c460a97fba724a1652cb427b8e517eb", "num_prompts": 64, "max_concurrency": 16, "completed": 64, "failed": 0, "duration": 7.62301430106163, "total_input_tokens": 8192, "total_output_tokens": 8192, "output_throughput": 1074.6405131181675, "median_ttft_ms": 352.9473482631147, "p95_ttft_ms": 372.57025414146483, "p95_tpot_ms": 13.504818420651851, "p95_e2el_ms": 1930.2394755650312 }, { "case": "short-c32", "repeat": 1, "source_sha256": "0959e74a7d2a714e0e8584ba2a0958ee9b4d5eca59485c7008f7818cf65497c1", "num_prompts": 128, "max_concurrency": 32, "completed": 128, "failed": 0, "duration": 10.328725414350629, "total_input_tokens": 16384, "total_output_tokens": 16384, "output_throughput": 1586.255742381943, "median_ttft_ms": 514.2971337772906, "p95_ttft_ms": 549.5766213629395, "p95_tpot_ms": 16.4237592310652, "p95_e2el_ms": 2632.8399680554867 }, { "case": "short-c32", "repeat": 2, "source_sha256": "043cd7e205abc18ad4b8de8246d6585cd722781477ec36f0f531a16a3bd9e6ee", "num_prompts": 128, "max_concurrency": 32, "completed": 128, "failed": 0, "duration": 10.36225707270205, "total_input_tokens": 16384, "total_output_tokens": 16384, "output_throughput": 1581.1227114951055, "median_ttft_ms": 514.9108972400427, "p95_ttft_ms": 549.7874869965017, "p95_tpot_ms": 16.511528664745214, "p95_e2el_ms": 2626.10334912315 }, { "case": "short-c32", "repeat": 3, "source_sha256": "79029bfefd7e511cf551d5404f0d72461f191ae82fa5d51b127206dc41e8bd91", "num_prompts": 128, "max_concurrency": 32, "completed": 128, "failed": 0, "duration": 10.33400398120284, "total_input_tokens": 16384, "total_output_tokens": 16384, "output_throughput": 1585.4454894542205, "median_ttft_ms": 513.2594336755574, "p95_ttft_ms": 550.3696689382195, "p95_tpot_ms": 16.320307950276558, "p95_e2el_ms": 2596.167112700641 }, { "case": "rag4k-c1", "repeat": 1, "source_sha256": "04d13aa4b3bf09b3d2279fe09d8ba41037d97e4cfe31bf0b01f8bcbfd141baf1", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 8.26464762724936, "total_input_tokens": 16384, "total_output_tokens": 1024, "output_throughput": 123.90122920955163, "median_ttft_ms": 500.91930804774165, "p95_ttft_ms": 502.0938668865711, "p95_tpot_ms": 6.141070855197077, "p95_e2el_ms": 2067.8762378636748 }, { "case": "rag4k-c1", "repeat": 2, "source_sha256": "46e6b984395c34456ab9102114ba85e385232bc7c6fd86aa5a14cd826a9b7040", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 8.265051550231874, "total_input_tokens": 16384, "total_output_tokens": 1024, "output_throughput": 123.89517400787076, "median_ttft_ms": 500.6796573288739, "p95_ttft_ms": 503.82344340905547, "p95_tpot_ms": 6.137065379423839, "p95_e2el_ms": 2067.980425339192 }, { "case": "rag4k-c1", "repeat": 3, "source_sha256": "7e8fd4b7cf31b4515e64f0062489f11ef846b306c567a2cad0c851bd6239e2ef", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 8.261162365786731, "total_input_tokens": 16384, "total_output_tokens": 1024, "output_throughput": 123.95350129430386, "median_ttft_ms": 500.88235456496477, "p95_ttft_ms": 501.7043961212039, "p95_tpot_ms": 6.139060927591488, "p95_e2el_ms": 2066.782518615946 }, { "case": "rag4k-c4", "repeat": 1, "source_sha256": "7a1410abf828f879aaf22198e71823b740ec04d16e2ee16e794ce6ebcbee8189", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 7.672109070234001, "total_input_tokens": 32768, "total_output_tokens": 2048, "output_throughput": 266.94093908880467, "median_ttft_ms": 1596.737185958773, "p95_ttft_ms": 1845.1737678144127, "p95_tpot_ms": 13.048445101571726, "p95_e2el_ms": 3836.6540377028286 }, { "case": "rag4k-c4", "repeat": 2, "source_sha256": "955dec779a052781cad51b33327156fe799a5558ed7467c55199205ef058c0fa", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 7.7028289418667555, "total_input_tokens": 32768, "total_output_tokens": 2048, "output_throughput": 265.87634432184, "median_ttft_ms": 1596.7446062713861, "p95_ttft_ms": 1847.2788562532514, "p95_tpot_ms": 13.107244901395603, "p95_e2el_ms": 3852.727848337963 }, { "case": "rag4k-c4", "repeat": 3, "source_sha256": "ab0dfa44b1a4fc63420dd80bf03e374a42c398ec8156ba463c0102cea17d38db", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 7.687433146871626, "total_input_tokens": 32768, "total_output_tokens": 2048, "output_throughput": 266.4088208472325, "median_ttft_ms": 1595.1698329299688, "p95_ttft_ms": 1842.167653888464, "p95_tpot_ms": 13.095445108106908, "p95_e2el_ms": 3876.121106557548 }, { "case": "rag4k-c8", "repeat": 1, "source_sha256": "9f4890d9404793e8baf2b40d01d6ecf2b29ee7eda14569e1edeafb1263dfd122", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 11.983623880892992, "total_input_tokens": 65536, "total_output_tokens": 4096, "output_throughput": 341.799779491642, "median_ttft_ms": 2658.9468736201525, "p95_ttft_ms": 3582.558373454958, "p95_tpot_ms": 21.30312811053705, "p95_e2el_ms": 6417.065714253113 }, { "case": "rag4k-c8", "repeat": 2, "source_sha256": "8a751d5ec572c78311c745caff32f1dbea59245cdd41e0db9a51f958bf133f7d", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 12.030298778787255, "total_input_tokens": 65536, "total_output_tokens": 4096, "output_throughput": 340.47367196086446, "median_ttft_ms": 2659.3425758183002, "p95_ttft_ms": 3582.5369860976934, "p95_tpot_ms": 21.462718812821837, "p95_e2el_ms": 6520.399931352586 }, { "case": "rag4k-c8", "repeat": 3, "source_sha256": "47c95d6d207c9936d53054b4fcbd4eeed74002610f76425201cf33d40dbdc553", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 12.018610320053995, "total_input_tokens": 65536, "total_output_tokens": 4096, "output_throughput": 340.80479281082125, "median_ttft_ms": 2658.7329879403114, "p95_ttft_ms": 3577.156755141914, "p95_tpot_ms": 21.405001169107123, "p95_e2el_ms": 6451.922165695578 }, { "case": "rag4k-c16", "repeat": 1, "source_sha256": "3c7f3e0ed3fb8614bd0ce3d2fa48a40f21c4e8d93b3f8986022e7d95913dff43", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 20.631914174184203, "total_input_tokens": 131072, "total_output_tokens": 8192, "output_throughput": 397.0547730491379, "median_ttft_ms": 3467.2704651020467, "p95_ttft_ms": 6789.5977889187625, "p95_tpot_ms": 36.45262991315593, "p95_e2el_ms": 13237.653969973324 }, { "case": "rag4k-c16", "repeat": 2, "source_sha256": "f8187dd5e282cb92f94589f4e000d718ecab9e14bb7362ec202f40b522128aa8", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 20.462005763314664, "total_input_tokens": 131072, "total_output_tokens": 8192, "output_throughput": 400.35175899945443, "median_ttft_ms": 3466.1086131818593, "p95_ttft_ms": 6787.214458826929, "p95_tpot_ms": 36.07734268987734, "p95_e2el_ms": 13261.160863982512 }, { "case": "rag4k-c16", "repeat": 3, "source_sha256": "60e69365f72f54489806151eef5a042feaf5cc862e5f42bdcc47521886e0cab2", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 20.60970727354288, "total_input_tokens": 131072, "total_output_tokens": 8192, "output_throughput": 397.48259843148014, "median_ttft_ms": 3462.8119585104287, "p95_ttft_ms": 6786.962133133784, "p95_tpot_ms": 36.38045915347688, "p95_e2el_ms": 13305.86461564526 }, { "case": "agent8k-c1", "repeat": 1, "source_sha256": "46da08446ff4bb17fb9664ca029e6830c3d37cc4974d3cded5f3e66759135bff", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 29.005554424598813, "total_input_tokens": 32768, "total_output_tokens": 4096, "output_throughput": 141.2143322634197, "median_ttft_ms": 934.3557367101312, "p95_ttft_ms": 936.0924287699163, "p95_tpot_ms": 6.174532290710133, "p95_e2el_ms": 7252.314251707867 }, { "case": "agent8k-c1", "repeat": 2, "source_sha256": "4e6a7946a1bd52fe446ddb40ab4b14442c85ab2d5f39231607c25af5cc79932d", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 29.010865507647395, "total_input_tokens": 32768, "total_output_tokens": 4096, "output_throughput": 141.18847984456983, "median_ttft_ms": 935.6515710242093, "p95_ttft_ms": 937.2867114841938, "p95_tpot_ms": 6.1787777258044985, "p95_e2el_ms": 7255.091740097851 }, { "case": "agent8k-c1", "repeat": 3, "source_sha256": "096108fa8c288a12c3d3fdb54c2002c4b2aab550a89f50f8c1628773f23d93ce", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 28.994386164471507, "total_input_tokens": 32768, "total_output_tokens": 4096, "output_throughput": 141.26872618600441, "median_ttft_ms": 930.9955281205475, "p95_ttft_ms": 934.1624373104423, "p95_tpot_ms": 6.174268047821472, "p95_e2el_ms": 7247.645076923072 }, { "case": "agent8k-c4", "repeat": 1, "source_sha256": "4591fe3108642672c9e38d1653e98e13f23d911c46fde109253460fcc7867e95", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 23.481539213098586, "total_input_tokens": 65536, "total_output_tokens": 8192, "output_throughput": 348.8698047285716, "median_ttft_ms": 3157.287331763655, "p95_ttft_ms": 3632.8457709867507, "p95_tpot_ms": 10.575740320611663, "p95_e2el_ms": 11843.756904033944 }, { "case": "agent8k-c4", "repeat": 2, "source_sha256": "5dc74c35f525dbab146f34fa8db801553d638dc7692a96fa322b7e294f2cea2e", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 23.552636846899986, "total_input_tokens": 65536, "total_output_tokens": 8192, "output_throughput": 347.8166819813314, "median_ttft_ms": 3152.3243892006576, "p95_ttft_ms": 3635.476071946323, "p95_tpot_ms": 10.603174780115834, "p95_e2el_ms": 11851.367546292022 }, { "case": "agent8k-c4", "repeat": 3, "source_sha256": "9e78be97a028c3d5cb941e821faabb77f3ce631cf52ded69ef5f00cbc7b0fea5", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 23.741042678244412, "total_input_tokens": 65536, "total_output_tokens": 8192, "output_throughput": 345.0564539655584, "median_ttft_ms": 3151.522034779191, "p95_ttft_ms": 3630.979935778305, "p95_tpot_ms": 10.674973282478714, "p95_e2el_ms": 11907.504843035713 }, { "case": "agent8k-c8", "repeat": 1, "source_sha256": "74348d0d53a66ec904275517780a4a66a3b90f6c311520ed4ea00cbc0cce0b3f", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 34.49448038637638, "total_input_tokens": 131072, "total_output_tokens": 16384, "output_throughput": 474.9745413318611, "median_ttft_ms": 4502.065575681627, "p95_ttft_ms": 7191.921853693202, "p95_tpot_ms": 15.830224852158688, "p95_e2el_ms": 19388.964678393677 }, { "case": "agent8k-c8", "repeat": 2, "source_sha256": "8b4aeb716529ac0f0a61bf5c28fbac2f698dfc530c56e9b6251eba7400154905", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 34.18908758368343, "total_input_tokens": 131072, "total_output_tokens": 16384, "output_throughput": 479.2172344435182, "median_ttft_ms": 4498.17433161661, "p95_ttft_ms": 7187.137183267623, "p95_tpot_ms": 15.720319424374422, "p95_e2el_ms": 19161.87181114219 }, { "case": "agent8k-c8", "repeat": 3, "source_sha256": "e2d73d084aa4ae2d614b9d2abb4f7ffbe104c316e986b61750d5b22478add0b0", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 34.82990461215377, "total_input_tokens": 131072, "total_output_tokens": 16384, "output_throughput": 470.40036952277103, "median_ttft_ms": 4496.5641484595835, "p95_ttft_ms": 7187.767590396106, "p95_tpot_ms": 16.034068126196836, "p95_e2el_ms": 18750.32052048482 }, { "case": "agent8k-c16", "repeat": 1, "source_sha256": "664a31079a2e5ff5f6f9ca4a5b691598e512fa422b5919225d424317500de9f7", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 55.101463152095675, "total_input_tokens": 262144, "total_output_tokens": 32768, "output_throughput": 594.6847529175591, "median_ttft_ms": 4463.290351908654, "p95_ttft_ms": 13838.446102431042, "p95_tpot_ms": 24.97532999403374, "p95_e2el_ms": 36721.269664820284 }, { "case": "agent8k-c16", "repeat": 2, "source_sha256": "3fa0902f251651597d9eec10bbcba037111712699f15bc7e41eaefbdb76de2bc", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 55.40296471398324, "total_input_tokens": 262144, "total_output_tokens": 32768, "output_throughput": 591.4484932198878, "median_ttft_ms": 5357.145204208791, "p95_ttft_ms": 13833.062936738132, "p95_tpot_ms": 25.09189950487051, "p95_e2el_ms": 36121.313523175195 }, { "case": "agent8k-c16", "repeat": 3, "source_sha256": "9e12a12f9e86d898c183efa022e55d0d4cb83321f97cbb55bf73ce4594f77883", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 55.457558270543814, "total_input_tokens": 262144, "total_output_tokens": 32768, "output_throughput": 590.8662592057296, "median_ttft_ms": 4462.433964479715, "p95_ttft_ms": 13837.3658569064, "p95_tpot_ms": 25.150703881500345, "p95_e2el_ms": 37140.99941854365 }, { "case": "rag16k-c1", "repeat": 1, "source_sha256": "cf352489e6ff2e2c8470d166320b7c7161b37f9f7450798b8744c7b4186835aa", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 13.826224095188081, "total_input_tokens": 65536, "total_output_tokens": 1024, "output_throughput": 74.06215847147892, "median_ttft_ms": 1884.377155918628, "p95_ttft_ms": 1887.076710863039, "p95_tpot_ms": 6.170184647335725, "p95_e2el_ms": 3456.7323146853596 }, { "case": "rag16k-c1", "repeat": 2, "source_sha256": "57e808ab76dfffec3066a3cd7f61ada3226f732e02fe328de8e006d87174785f", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 13.83254937082529, "total_input_tokens": 65536, "total_output_tokens": 1024, "output_throughput": 74.02829171604144, "median_ttft_ms": 1886.029338464141, "p95_ttft_ms": 1889.948385162279, "p95_tpot_ms": 6.162383847084699, "p95_e2el_ms": 3460.205047018826 }, { "case": "rag16k-c1", "repeat": 3, "source_sha256": "a5a9fa9b171ef556a1b4957042f03fcf96e291c882273cb59056c0815ee79a04", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 13.819487284868956, "total_input_tokens": 65536, "total_output_tokens": 1024, "output_throughput": 74.09826275691024, "median_ttft_ms": 1883.6494754068553, "p95_ttft_ms": 1889.1413677949458, "p95_tpot_ms": 6.167176055046273, "p95_e2el_ms": 3455.0892788451165 }, { "case": "rag16k-c4", "repeat": 1, "source_sha256": "59863070004ed1e04ad49756fe2dcfa5f2fb33a481b2d4104725f9538c7cbc43", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 18.72934284619987, "total_input_tokens": 131072, "total_output_tokens": 2048, "output_throughput": 109.34713603235328, "median_ttft_ms": 5478.873309213668, "p95_ttft_ms": 7333.353869430721, "p95_tpot_ms": 29.243882537509005, "p95_e2el_ms": 9368.94670710899 }, { "case": "rag16k-c4", "repeat": 2, "source_sha256": "d883a6c7559801aee1151502f9317c73f8290f9a03375c53aad3d672af68bed0", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 18.739640375599265, "total_input_tokens": 131072, "total_output_tokens": 2048, "output_throughput": 109.28704921502572, "median_ttft_ms": 5477.281545754522, "p95_ttft_ms": 7332.830786891282, "p95_tpot_ms": 29.24876368659384, "p95_e2el_ms": 9407.477190997452 }, { "case": "rag16k-c4", "repeat": 3, "source_sha256": "079649764480f7159dc0c5ddbbc6f5ab1da2695bce63435a23dbc94cdce1015d", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 18.772656976245344, "total_input_tokens": 131072, "total_output_tokens": 2048, "output_throughput": 109.09483951001238, "median_ttft_ms": 5480.892009567469, "p95_ttft_ms": 7355.14025981538, "p95_tpot_ms": 29.303902632684686, "p95_e2el_ms": 9397.940281452611 }, { "case": "rag16k-c8", "repeat": 1, "source_sha256": "e67df096563f0b73cd5e9b82bc7ca4fd3a293ef16e0a80e155570ebd084689d2", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 34.150304613634944, "total_input_tokens": 262144, "total_output_tokens": 4096, "output_throughput": 119.94036499354152, "median_ttft_ms": 8721.402467228472, "p95_ttft_ms": 13899.892593501136, "p95_tpot_ms": 56.82653373965592, "p95_e2el_ms": 21072.09875737317 }, { "case": "rag16k-c8", "repeat": 2, "source_sha256": "f79169e9184f033b5abc17828d9cb508175fddad1f714134625f57edd3804c6a", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 34.16615715250373, "total_input_tokens": 262144, "total_output_tokens": 4096, "output_throughput": 119.8847146232201, "median_ttft_ms": 8718.586056958884, "p95_ttft_ms": 13893.665686016902, "p95_tpot_ms": 56.89551981403401, "p95_e2el_ms": 21061.378204263747 }, { "case": "rag16k-c8", "repeat": 3, "source_sha256": "f22bf54447bc045ecfd3b5661d395d0721814ec1063264256c2d20a4997ccd89", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 34.26296460162848, "total_input_tokens": 262144, "total_output_tokens": 4096, "output_throughput": 119.54598931014047, "median_ttft_ms": 8165.029070340097, "p95_ttft_ms": 13905.897402670234, "p95_tpot_ms": 56.999091588545085, "p95_e2el_ms": 22974.001393886283 }, { "case": "decode2k-c1", "repeat": 1, "source_sha256": "bfe73b267439eed6b97abb18deca2396a64b9125e5ae303c4c506562ecd16b43", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 51.04356375616044, "total_input_tokens": 512, "total_output_tokens": 8192, "output_throughput": 160.4903615102954, "median_ttft_ms": 202.22873939201236, "p95_ttft_ms": 254.04311101883647, "p95_tpot_ms": 6.141902343358646, "p95_e2el_ms": 12816.53116075322 }, { "case": "decode2k-c1", "repeat": 2, "source_sha256": "e957cdd05a16561b4112a05b3ae932a5d9618b5aa4eccf5c0302943aabc3b65e", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 50.759992382489145, "total_input_tokens": 512, "total_output_tokens": 8192, "output_throughput": 161.38694305293126, "median_ttft_ms": 124.95411606505513, "p95_ttft_ms": 125.87943770922722, "p95_tpot_ms": 6.139566231988512, "p95_e2el_ms": 12692.195628909394 }, { "case": "decode2k-c1", "repeat": 3, "source_sha256": "ce79294142914d6ac67d839f36e5e38d25f1adc27915c5b96d21100a76bb588e", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 50.76553922332823, "total_input_tokens": 512, "total_output_tokens": 8192, "output_throughput": 161.36930928600358, "median_ttft_ms": 125.71988580748439, "p95_ttft_ms": 137.74620397016403, "p95_tpot_ms": 6.140351689292793, "p95_e2el_ms": 12694.102395465598 }, { "case": "decode2k-c4", "repeat": 1, "source_sha256": "e44d2a8af6d6646e54c36f82da0a2a5abeb8864b57c9ee586b5bee23d6b4c011", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 32.703026708215475, "total_input_tokens": 1024, "total_output_tokens": 16384, "output_throughput": 500.993383462091, "median_ttft_ms": 264.7086759097874, "p95_ttft_ms": 401.4556393958628, "p95_tpot_ms": 7.938894899753088, "p95_e2el_ms": 16388.078751647845 }, { "case": "decode2k-c4", "repeat": 2, "source_sha256": "75b226e63a8ab6c176e5565024d431ba374fbd37fc755c0e275e2161a07c9b99", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 32.80961362924427, "total_input_tokens": 1024, "total_output_tokens": 16384, "output_throughput": 499.3658317693937, "median_ttft_ms": 262.4966842122376, "p95_ttft_ms": 400.2818755339831, "p95_tpot_ms": 7.9711587289753165, "p95_e2el_ms": 16506.82289004326 }, { "case": "decode2k-c4", "repeat": 3, "source_sha256": "f2ae5c2305e2e1f9f55f171035ac0c5a975fe320bcd0a56d3b2eaabc51ff42fb", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 32.78069999627769, "total_input_tokens": 1024, "total_output_tokens": 16384, "output_throughput": 499.80628851307125, "median_ttft_ms": 265.6393637880683, "p95_ttft_ms": 403.2369466498494, "p95_tpot_ms": 7.930627740800388, "p95_e2el_ms": 16502.923278324306 }, { "case": "decode2k-c8", "repeat": 1, "source_sha256": "48741de02f51c2dcff9ab48f8fe37fdc9c7ad13378f51d26b9e0f624e2d8eaee", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 39.896118643693626, "total_input_tokens": 2048, "total_output_tokens": 32768, "output_throughput": 821.3330297276834, "median_ttft_ms": 385.17776457592845, "p95_ttft_ms": 457.56519795395434, "p95_tpot_ms": 9.670689188259683, "p95_e2el_ms": 19969.596231123433 }, { "case": "decode2k-c8", "repeat": 2, "source_sha256": "5a2554674856c4a2588d98a5e0eb8f0438fb1c189e7bac36d3bb2fbbae40ebf8", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 40.53498282004148, "total_input_tokens": 2048, "total_output_tokens": 32768, "output_throughput": 808.388155620451, "median_ttft_ms": 260.5486032553017, "p95_ttft_ms": 278.266403125599, "p95_tpot_ms": 9.938211905946078, "p95_e2el_ms": 20482.862630859017 }, { "case": "decode2k-c8", "repeat": 3, "source_sha256": "2f481ad732ff868892def22b404fcb79d87edc80d636e9eccb08d6d9d1a613be", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 40.70106974430382, "total_input_tokens": 2048, "total_output_tokens": 32768, "output_throughput": 805.089404427409, "median_ttft_ms": 385.3724109940231, "p95_ttft_ms": 404.05780472792685, "p95_tpot_ms": 9.907889749052236, "p95_e2el_ms": 20510.268366429955 } ], "history_round_data": [ { "case": "vllm-history-short128-64-c1-r1", "source_sha256": "e7eed09abcbfbb471652778f782b76e60188129ded91cd4c5290f4d8a7feb051", "num_prompts": 32, "completed": 32, "failed": 0, "output_throughput": 124.9652262818439, "p95_ttft_ms": 126.94422132335603, "p95_tpot_ms": 6.140456801753432 }, { "case": "vllm-history-short128-64-c1-r2", "source_sha256": "46dbd0109fad6f67ccbc31ab3f675cce127ee8e3749734c1ed2f8b57ee919865", "num_prompts": 32, "completed": 32, "failed": 0, "output_throughput": 125.17436347863814, "p95_ttft_ms": 125.87855025194585, "p95_tpot_ms": 6.134963402938512 }, { "case": "vllm-history-short128-64-c1-r3", "source_sha256": "e8c0557c50750fdfbc347668fbb0398a9b9b79d0ae6c586be5f7cc9763c9f1d4", "num_prompts": 32, "completed": 32, "failed": 0, "output_throughput": 125.11540427269439, "p95_ttft_ms": 125.4135588183999, "p95_tpot_ms": 6.140265789710813 }, { "case": "vllm-history-short128-64-c4-r1", "source_sha256": "b8be96e0ac1ad21d743d2fe1c1aaf8a4beaf696ff92bcca8e990a2899bfa0380", "num_prompts": 64, "completed": 64, "failed": 0, "output_throughput": 289.1945251078987, "p95_ttft_ms": 396.55827088281507, "p95_tpot_ms": 11.928583130920455 }, { "case": "vllm-history-short128-64-c4-r2", "source_sha256": "8dfd1e9b41bdbb2f0d8a6339c6c6fe6aad3c352dd934a11685a04f15fc4c7bd0", "num_prompts": 64, "completed": 64, "failed": 0, "output_throughput": 290.3142167938122, "p95_ttft_ms": 387.3120367527008, "p95_tpot_ms": 11.939077236733976 }, { "case": "vllm-history-short128-64-c4-r3", "source_sha256": "fc9c6fa2c3a3f8dc17361689db0631f1291791ddd2ce1f6bd0bb1a23990b0f96", "num_prompts": 64, "completed": 64, "failed": 0, "output_throughput": 290.7342397882296, "p95_ttft_ms": 393.6838816851377, "p95_tpot_ms": 11.91610245122796 }, { "case": "vllm-history-short128-64-c8-r1", "source_sha256": "10ce982c27cd8632df0714c63c637c6f7f0b066432e836e9b29c8553b3b69bdc", "num_prompts": 128, "completed": 128, "failed": 0, "output_throughput": 513.4286257903584, "p95_ttft_ms": 406.1946322675794, "p95_tpot_ms": 13.667585030345926 }, { "case": "vllm-history-short128-64-c8-r2", "source_sha256": "c2f311df1794b0647c9735d61992b02865440d42a3923dc9408c691a54390c1a", "num_prompts": 128, "completed": 128, "failed": 0, "output_throughput": 516.8819295935665, "p95_ttft_ms": 388.23356702923775, "p95_tpot_ms": 13.589621326398284 }, { "case": "vllm-history-short128-64-c8-r3", "source_sha256": "2abbe76b5d7f1e74dd67bc86bc6dea5233bef8998fcd556004d10041c8388cd8", "num_prompts": 128, "completed": 128, "failed": 0, "output_throughput": 523.9472262008185, "p95_ttft_ms": 400.1070512924343, "p95_tpot_ms": 13.629453013547593 } ], "long_context": [ { "target_context": 65536, "seed": 103101, "fixture_sha256": "b7872b02ceaad62ae5014e894eb6d2fc6d4a1268a311449daa583e165e91fd1e", "prompt_tokens": 57344, "completion_tokens": 82, "reasoning_tokens": 8, "ttft_s": 6.759155843406916, "elapsed_s": 7.293293358758092, "status": "PASS", "needle_fractions": [ 0.10300990513392858, 0.5009940011160714, 0.89892578125 ] }, { "target_context": 131072, "seed": 103101, "fixture_sha256": "3ca1eb63143f10a8c8f4f8a3a297b2948474e8841c9eb86bae7f584dddec2df3", "prompt_tokens": 122881, "completion_tokens": 72, "reasoning_tokens": 0, "ttft_s": 14.617269118316472, "elapsed_s": 15.055812368169427, "status": "PASS", "needle_fractions": [ 0.10140705235146198, 0.5004679323898731, 0.8995043985644648 ] }, { "target_context": 262144, "seed": 103101, "fixture_sha256": "ed3a0c83c1826ab2a0a99df004a53e8f858c8047d1fe9312376089a832e0c1ca", "prompt_tokens": 253952, "completion_tokens": 195, "reasoning_tokens": 106, "ttft_s": 31.7986131478101, "elapsed_s": 33.11156293284148, "status": "PASS", "needle_fractions": [ 0.10067256804435484, 0.5002165763608871, 0.8997448336693549 ] }, { "target_context": 524288, "seed": 103101, "fixture_sha256": "4605f7ede8340a35ba8b3b2366acd2448595579a76b78b72ba414c9bf680aefd", "prompt_tokens": 516095, "completion_tokens": 96, "reasoning_tokens": 10, "ttft_s": 70.96917869336903, "elapsed_s": 71.36111355852336, "status": "PASS", "needle_fractions": [ 0.1003323031612397, 0.500105600713047, 0.8998847111481413 ] }, { "target_context": 1048576, "seed": 103101, "fixture_sha256": "1f7eec8462cf5a45b48ea7ae9eef22494fc7c44c710e6eb44434e3f6c4f33b0c", "prompt_tokens": 1040383, "completion_tokens": 203, "reasoning_tokens": 123, "ttft_s": 168.49122035317123, "elapsed_s": 169.78709274716675, "status": "PASS", "needle_fractions": [ 0.1001659965608819, 0.5000523845545343, 0.8999416561016472 ] }, { "target_context": 1048576, "seed": 103102, "fixture_sha256": "e441b9e81842edbb66a17d7b7d1a660a3c0944d4a040a88c70cf486d1d117457", "prompt_tokens": 1040384, "completion_tokens": 211, "reasoning_tokens": 133, "ttft_s": 168.28992893453687, "elapsed_s": 169.69484428316355, "status": "PASS", "needle_fractions": [ 0.10016590028297244, 0.5000557486466536, 0.8999417522760826 ] }, { "target_context": 1048576, "seed": 103103, "fixture_sha256": "f591fb0fa04c41292b8a1bf924380e24d1d6c2c010a9c0323a767e82beb2bf21", "prompt_tokens": 1040384, "completion_tokens": 174, "reasoning_tokens": 86, "ttft_s": 168.26029389258474, "elapsed_s": 169.29153139982373, "status": "PASS", "needle_fractions": [ 0.10016493909940945, 0.5000519039124016, 0.8999369463582677 ] } ] }, "sglang": { "engine": "sglang", "image_source": "lmsysorg/sglang:dev-minimax-m31-flash", "image_digest": "sha256:c1ad26064c80ea2d628b3575b1803e9df804949e718dd2ee15d63fb21d9dcedb", "version": "0.0.0.dev0", "kv_cache_storage_dtype_observed": "torch.float8_e4m3fn", "core": [ { "case": "short-c1", "input_tokens": 128, "output_tokens": 128, "concurrency": 1, "num_prompts": 8, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 172.49363392282697, "min": 172.41859834068367, "max": 172.63618563668567 }, "median_ttft_ms": { "median": 44.16277166455984, "min": 43.68778597563505, "max": 44.458297081291676 }, "p95_ttft_ms": { "median": 45.5627276096493, "min": 44.82042104937136, "max": 47.456417325884104 }, "p95_tpot_ms": { "median": 5.499668604892305, "min": 5.493460619440815, "max": 5.5024318866373045 }, "p95_e2el_ms": { "median": 743.900159234181, "min": 741.6771683841944, "max": 744.324755249545 } }, { "case": "short-c4", "input_tokens": 128, "output_tokens": 128, "concurrency": 4, "num_prompts": 16, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 492.8887923858154, "min": 492.80857995736915, "max": 493.4284000218911 }, "median_ttft_ms": { "median": 93.46155915409327, "min": 91.79903101176023, "max": 94.94993463158607 }, "p95_ttft_ms": { "median": 98.80888741463423, "min": 97.77835034765303, "max": 100.52074072882533 }, "p95_tpot_ms": { "median": 7.585079808844122, "min": 7.5776663509850195, "max": 7.599370706254455 }, "p95_e2el_ms": { "median": 1044.1892719827592, "min": 1042.6724904682487, "max": 1048.486901447177 } }, { "case": "short-c8", "input_tokens": 128, "output_tokens": 128, "concurrency": 8, "num_prompts": 32, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 775.4615272737518, "min": 773.9916174064034, "max": 778.5176144259877 }, "median_ttft_ms": { "median": 149.0179873071611, "min": 148.98138213902712, "max": 149.82885122299194 }, "p95_ttft_ms": { "median": 152.46935007162392, "min": 152.4163254071027, "max": 153.7201326340437 }, "p95_tpot_ms": { "median": 9.711649798692799, "min": 9.65731562056586, "max": 9.728181429617988 }, "p95_e2el_ms": { "median": 1330.6332176085562, "min": 1323.180090263486, "max": 1342.3432533629239 } }, { "case": "short-c16", "input_tokens": 128, "output_tokens": 128, "concurrency": 16, "num_prompts": 64, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 1132.1552192717052, "min": 1129.8850203484733, "max": 1136.5085483841747 }, "median_ttft_ms": { "median": 262.7600720152259, "min": 260.2427056990564, "max": 266.9516415335238 }, "p95_ttft_ms": { "median": 268.84151357226074, "min": 268.24455610476434, "max": 273.99512380361557 }, "p95_tpot_ms": { "median": 12.48427854198229, "min": 12.47265910561101, "max": 12.520578288744519 }, "p95_e2el_ms": { "median": 1820.5827600322664, "min": 1815.1100737974048, "max": 1820.8843838889152 } }, { "case": "short-c32", "input_tokens": 128, "output_tokens": 128, "concurrency": 32, "num_prompts": 128, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 1688.3636041981686, "min": 1682.5345305437263, "max": 1699.0950047660642 }, "median_ttft_ms": { "median": 413.3862513117492, "min": 390.1862818747759, "max": 425.66580092534423 }, "p95_ttft_ms": { "median": 446.47203516215086, "min": 431.04067030362785, "max": 468.2023205328733 }, "p95_tpot_ms": { "median": 16.39876410504436, "min": 16.33510610792698, "max": 17.001757548945978 }, "p95_e2el_ms": { "median": 2478.8263871800154, "min": 2463.5713007766753, "max": 2486.8842337746173 } }, { "case": "rag4k-c1", "input_tokens": 4096, "output_tokens": 256, "concurrency": 1, "num_prompts": 4, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 133.38294620704738, "min": 133.3547954025137, "max": 133.4434321284477 }, "median_ttft_ms": { "median": 468.84568501263857, "min": 467.865901067853, "max": 469.10622203722596 }, "p95_ttft_ms": { "median": 469.61268223822117, "min": 468.9189343713224, "max": 470.10300452820957 }, "p95_tpot_ms": { "median": 5.68664614750328, "min": 5.686442910638802, "max": 5.6889942898323715 }, "p95_e2el_ms": { "median": 1919.6678414475173, "min": 1918.8826734665781, "max": 1920.409829635173 } }, { "case": "rag4k-c4", "input_tokens": 4096, "output_tokens": 256, "concurrency": 4, "num_prompts": 8, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 281.3201901829796, "min": 280.2076003346688, "max": 288.1122437877945 }, "median_ttft_ms": { "median": 1590.2419504709542, "min": 1533.684334717691, "max": 1591.570264659822 }, "p95_ttft_ms": { "median": 1725.1333188265562, "min": 1657.8835482243448, "max": 1725.3716671839356 }, "p95_tpot_ms": { "median": 10.22558385542795, "min": 8.534145683744082, "max": 10.27245679486762 }, "p95_e2el_ms": { "median": 3661.748126614839, "min": 3588.7990401592106, "max": 3662.4331885017455 } }, { "case": "rag4k-c8", "input_tokens": 4096, "output_tokens": 256, "concurrency": 8, "num_prompts": 16, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 353.81737904799354, "min": 350.2534952338321, "max": 354.19981236254785 }, "median_ttft_ms": { "median": 2666.118375957012, "min": 2664.3777559511364, "max": 2668.1880191899836 }, "p95_ttft_ms": { "median": 3352.9503904283047, "min": 3351.0223887860775, "max": 3353.343572700396 }, "p95_tpot_ms": { "median": 18.593491854000035, "min": 18.584891528292907, "max": 18.82938250771486 }, "p95_e2el_ms": { "median": 5795.752226840705, "min": 5787.144786678255, "max": 5854.554445017129 } }, { "case": "rag4k-c16", "input_tokens": 4096, "output_tokens": 256, "concurrency": 16, "num_prompts": 32, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 411.455257856488, "min": 411.0175996625837, "max": 413.30423932283867 }, "median_ttft_ms": { "median": 4302.710012067109, "min": 4295.139140915126, "max": 4304.791182745248 }, "p95_ttft_ms": { "median": 6545.788497524336, "min": 6535.652985237539, "max": 6610.6509496457875 }, "p95_tpot_ms": { "median": 33.162816337437604, "min": 33.003160120627165, "max": 33.18489848599568 }, "p95_e2el_ms": { "median": 9971.619898779318, "min": 9921.713853953406, "max": 9985.395472170785 } }, { "case": "agent8k-c1", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 1, "num_prompts": 4, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 151.70635076256767, "min": 151.6802139448492, "max": 151.72601992478437 }, "median_ttft_ms": { "median": 888.7754939496517, "min": 888.7001951225102, "max": 889.6221276372671 }, "p95_ttft_ms": { "median": 889.8373300675303, "min": 889.6629661787301, "max": 890.579989599064 }, "p95_tpot_ms": { "median": 5.729406483413129, "min": 5.727920086165837, "max": 5.7296206997133945 }, "p95_e2el_ms": { "median": 6750.254665361717, "min": 6749.277222622186, "max": 6751.704597938806 } }, { "case": "agent8k-c4", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 4, "num_prompts": 8, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 364.1172825026223, "min": 360.5532068427681, "max": 364.236142355617 }, "median_ttft_ms": { "median": 2750.3923224285245, "min": 2744.922410696745, "max": 2752.0297509618104 }, "p95_ttft_ms": { "median": 3432.997280312702, "min": 3428.618638915941, "max": 3433.5632691625506 }, "p95_tpot_ms": { "median": 9.576733368071103, "min": 9.565280099970199, "max": 9.664516629995314 }, "p95_e2el_ms": { "median": 11366.605810215697, "min": 11296.825659042224, "max": 11395.331712579355 } }, { "case": "agent8k-c8", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 8, "num_prompts": 16, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 486.52617352717226, "min": 485.93999446508377, "max": 488.68788052507085 }, "median_ttft_ms": { "median": 4448.089090641588, "min": 4445.697316434234, "max": 4449.902791064233 }, "p95_ttft_ms": { "median": 6819.784871069714, "min": 6813.702237792313, "max": 6830.315636936575 }, "p95_tpot_ms": { "median": 15.0038606383886, "min": 14.845570649283474, "max": 15.014114930774227 }, "p95_e2el_ms": { "median": 16873.628149041906, "min": 16838.896081782877, "max": 16935.64539658837 } }, { "case": "agent8k-c16", "input_tokens": 8192, "output_tokens": 1024, "concurrency": 16, "num_prompts": 32, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 601.2176051188003, "min": 598.9051972415959, "max": 606.1454952849742 }, "median_ttft_ms": { "median": 7850.686855614185, "min": 7838.529746513814, "max": 8269.589121919125 }, "p95_ttft_ms": { "median": 13459.505489934236, "min": 13453.397444169968, "max": 13614.6007088013 }, "p95_tpot_ms": { "median": 24.730357357445566, "min": 24.49711917686519, "max": 24.815388437030965 }, "p95_e2el_ms": { "median": 27367.690696241334, "min": 27050.018274690956, "max": 27389.067482901737 } }, { "case": "rag16k-c1", "input_tokens": 16384, "output_tokens": 256, "concurrency": 1, "num_prompts": 4, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 77.41457208044706, "min": 77.32846680573621, "max": 77.45696276771523 }, "median_ttft_ms": { "median": 1822.7063664235175, "min": 1820.7553923130035, "max": 1826.2194842100143 }, "p95_ttft_ms": { "median": 1824.218468926847, "min": 1822.3110645543784, "max": 1830.2201158832759 }, "p95_tpot_ms": { "median": 5.819052955975719, "min": 5.818760297535097, "max": 5.819530714778048 }, "p95_e2el_ms": { "median": 3307.5869200751185, "min": 3305.66856241785, "max": 3314.0786196570843 } }, { "case": "rag16k-c4", "input_tokens": 16384, "output_tokens": 256, "concurrency": 4, "num_prompts": 8, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 113.5330440434094, "min": 113.08135938296154, "max": 113.70072065809907 }, "median_ttft_ms": { "median": 5021.235513500869, "min": 5020.725603215396, "max": 5024.354773107916 }, "p95_ttft_ms": { "median": 7040.7096181530505, "min": 7039.001921424642, "max": 7042.428741697222 }, "p95_tpot_ms": { "median": 25.92212115104, "min": 25.845547390773014, "max": 26.022184844877497 }, "p95_e2el_ms": { "median": 9056.109406379983, "min": 9014.862985396758, "max": 9057.900490052998 } }, { "case": "rag16k-c8", "input_tokens": 16384, "output_tokens": 256, "concurrency": 8, "num_prompts": 16, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 123.63598877171661, "min": 123.52572385862746, "max": 124.05368075508423 }, "median_ttft_ms": { "median": 8485.748069826514, "min": 8476.816992741078, "max": 8499.057448934764 }, "p95_ttft_ms": { "median": 13980.177571065724, "min": 13959.711564471945, "max": 13992.113258456811 }, "p95_tpot_ms": { "median": 55.41802221069149, "min": 55.235655577488096, "max": 55.56238286525887 }, "p95_e2el_ms": { "median": 16579.84473463148, "min": 16551.10929394141, "max": 16588.759820908308 } }, { "case": "decode2k-c1", "input_tokens": 128, "output_tokens": 2048, "concurrency": 1, "num_prompts": 4, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 180.71335041902194, "min": 180.67529340682853, "max": 180.73660424510993 }, "median_ttft_ms": { "median": 44.617258943617344, "min": 44.01276493445039, "max": 44.86111970618367 }, "p95_ttft_ms": { "median": 44.96118761599064, "min": 44.885131530463696, "max": 45.52065529860556 }, "p95_tpot_ms": { "median": 5.515089424433479, "min": 5.51411819932723, "max": 5.515986695111301 }, "p95_e2el_ms": { "median": 11333.993809716776, "min": 11332.676309905946, "max": 11335.369908902794 } }, { "case": "decode2k-c4", "input_tokens": 128, "output_tokens": 2048, "concurrency": 4, "num_prompts": 8, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 537.2185864485216, "min": 536.9030116496915, "max": 546.6918623807552 }, "median_ttft_ms": { "median": 92.40700304508209, "min": 85.91312356293201, "max": 94.5277251303196 }, "p95_ttft_ms": { "median": 98.29690461046994, "min": 97.80522282235324, "max": 98.67310826666653 }, "p95_tpot_ms": { "median": 7.473062194108555, "min": 7.39443135016944, "max": 7.495040833721648 }, "p95_e2el_ms": { "median": 15377.878010831773, "min": 15222.940224735066, "max": 15427.212808001786 } }, { "case": "decode2k-c8", "input_tokens": 128, "output_tokens": 2048, "concurrency": 8, "num_prompts": 16, "rounds": 3, "executed_rounds": 3, "valid_rounds": 3, "output_throughput": { "median": 857.6480994373153, "min": 849.5902111088446, "max": 861.911923822767 }, "median_ttft_ms": { "median": 151.04826586320996, "min": 150.9172641672194, "max": 152.255492284894 }, "p95_ttft_ms": { "median": 152.36731665208936, "min": 151.9332774914801, "max": 157.7806482091546 }, "p95_tpot_ms": { "median": 9.377934485077896, "min": 9.330358147406411, "max": 9.589670087212198 }, "p95_e2el_ms": { "median": 19288.895815378055, "min": 19233.026472385973, "max": 19766.728955321014 } } ], "core_round_data": [ { "case": "short-c1", "repeat": 1, "source_sha256": "739d7fec2cd397e3e3b26a4f45ba3ee658e9c3eda48b6b0b3d2433fb8d0a8d00", "num_prompts": 8, "max_concurrency": 1, "completed": 8, "failed": 0, "duration": 5.931549033150077, "total_input_tokens": 1024, "total_output_tokens": 1024, "output_throughput": 172.63618563668567, "median_ttft_ms": 43.68778597563505, "p95_ttft_ms": 44.82042104937136, "p95_tpot_ms": 5.493460619440815, "p95_e2el_ms": 741.6771683841944 }, { "case": "short-c1", "repeat": 2, "source_sha256": "ea8c207e9400ce6273256f6b89b7fa83a0a79f14a56151da6321f1c51fa77d6c", "num_prompts": 8, "max_concurrency": 1, "completed": 8, "failed": 0, "duration": 5.936450967565179, "total_input_tokens": 1024, "total_output_tokens": 1024, "output_throughput": 172.49363392282697, "median_ttft_ms": 44.16277166455984, "p95_ttft_ms": 45.5627276096493, "p95_tpot_ms": 5.5024318866373045, "p95_e2el_ms": 744.324755249545 }, { "case": "short-c1", "repeat": 3, "source_sha256": "8294b435c1c0550407b4c65b86285ed35d36a433a933e4d83cf1bd3e41b9574f", "num_prompts": 8, "max_concurrency": 1, "completed": 8, "failed": 0, "duration": 5.939034476876259, "total_input_tokens": 1024, "total_output_tokens": 1024, "output_throughput": 172.41859834068367, "median_ttft_ms": 44.458297081291676, "p95_ttft_ms": 47.456417325884104, "p95_tpot_ms": 5.499668604892305, "p95_e2el_ms": 743.900159234181 }, { "case": "short-c4", "repeat": 1, "source_sha256": "e949063b1db3b1ab2385416f7744ddd4bda0ea919ae62c23ea89854d6bd05c68", "num_prompts": 16, "max_concurrency": 4, "completed": 16, "failed": 0, "duration": 4.1550954934209585, "total_input_tokens": 2048, "total_output_tokens": 2048, "output_throughput": 492.8887923858154, "median_ttft_ms": 93.46155915409327, "p95_ttft_ms": 97.77835034765303, "p95_tpot_ms": 7.599370706254455, "p95_e2el_ms": 1048.486901447177 }, { "case": "short-c4", "repeat": 2, "source_sha256": "7948615f275f02b6c21a485d8c6ff2c0d430c3c31ebe03acdc6fe490af94706e", "num_prompts": 16, "max_concurrency": 4, "completed": 16, "failed": 0, "duration": 4.155771801248193, "total_input_tokens": 2048, "total_output_tokens": 2048, "output_throughput": 492.80857995736915, "median_ttft_ms": 91.79903101176023, "p95_ttft_ms": 98.80888741463423, "p95_tpot_ms": 7.5776663509850195, "p95_e2el_ms": 1042.6724904682487 }, { "case": "short-c4", "repeat": 3, "source_sha256": "b65b7ca2977d48c20a97ba457e8a399a59d77393c298980af5352b262208752f", "num_prompts": 16, "max_concurrency": 4, "completed": 16, "failed": 0, "duration": 4.150551528669894, "total_input_tokens": 2048, "total_output_tokens": 2048, "output_throughput": 493.4284000218911, "median_ttft_ms": 94.94993463158607, "p95_ttft_ms": 100.52074072882533, "p95_tpot_ms": 7.585079808844122, "p95_e2el_ms": 1044.1892719827592 }, { "case": "short-c8", "repeat": 1, "source_sha256": "63f8070d5149f8ea2567fb703915369ef6fbe469457a65a15cda5db2ff52cc68", "num_prompts": 32, "max_concurrency": 8, "completed": 32, "failed": 0, "duration": 5.26128108613193, "total_input_tokens": 4096, "total_output_tokens": 4096, "output_throughput": 778.5176144259877, "median_ttft_ms": 149.0179873071611, "p95_ttft_ms": 152.46935007162392, "p95_tpot_ms": 9.65731562056586, "p95_e2el_ms": 1323.180090263486 }, { "case": "short-c8", "repeat": 2, "source_sha256": "1d50f058fdecde4fe2ea4502e88e5b71e38dccd6ff2a970f29a2618f2acbe968", "num_prompts": 32, "max_concurrency": 8, "completed": 32, "failed": 0, "duration": 5.292046978138387, "total_input_tokens": 4096, "total_output_tokens": 4096, "output_throughput": 773.9916174064034, "median_ttft_ms": 149.82885122299194, "p95_ttft_ms": 153.7201326340437, "p95_tpot_ms": 9.711649798692799, "p95_e2el_ms": 1342.3432533629239 }, { "case": "short-c8", "repeat": 3, "source_sha256": "45ce81e5571039d2327f30de28ae7c0a1e8cab0012a200942da0ce068ea77a45", "num_prompts": 32, "max_concurrency": 8, "completed": 32, "failed": 0, "duration": 5.282015749253333, "total_input_tokens": 4096, "total_output_tokens": 4096, "output_throughput": 775.4615272737518, "median_ttft_ms": 148.98138213902712, "p95_ttft_ms": 152.4163254071027, "p95_tpot_ms": 9.728181429617988, "p95_e2el_ms": 1330.6332176085562 }, { "case": "short-c16", "repeat": 1, "source_sha256": "4dd27fa7399da110d803c21b8f73a39ef0d8a0cb4d347610b49747282e5005c0", "num_prompts": 64, "max_concurrency": 16, "completed": 64, "failed": 0, "duration": 7.2357569532468915, "total_input_tokens": 8192, "total_output_tokens": 8192, "output_throughput": 1132.1552192717052, "median_ttft_ms": 266.9516415335238, "p95_ttft_ms": 273.99512380361557, "p95_tpot_ms": 12.48427854198229, "p95_e2el_ms": 1815.1100737974048 }, { "case": "short-c16", "repeat": 2, "source_sha256": "f925e3fe016d7e4d3c1ebe5261ad744e4de2f36f07c1ae721b08f328ae4cbf53", "num_prompts": 64, "max_concurrency": 16, "completed": 64, "failed": 0, "duration": 7.208040812052786, "total_input_tokens": 8192, "total_output_tokens": 8192, "output_throughput": 1136.5085483841747, "median_ttft_ms": 260.2427056990564, "p95_ttft_ms": 268.24455610476434, "p95_tpot_ms": 12.520578288744519, "p95_e2el_ms": 1820.5827600322664 }, { "case": "short-c16", "repeat": 3, "source_sha256": "2f811ed790761733406024c0d6ebd8a9df000ffe6ec8b658aa9a8b83d14b8fa9", "num_prompts": 64, "max_concurrency": 16, "completed": 64, "failed": 0, "duration": 7.25029525347054, "total_input_tokens": 8192, "total_output_tokens": 8192, "output_throughput": 1129.8850203484733, "median_ttft_ms": 262.7600720152259, "p95_ttft_ms": 268.84151357226074, "p95_tpot_ms": 12.47265910561101, "p95_e2el_ms": 1820.8843838889152 }, { "case": "short-c32", "repeat": 1, "source_sha256": "989247af733e7fa260864379a13b2eddb9fea42dfcc71f417b21b18145f89e64", "num_prompts": 128, "max_concurrency": 32, "completed": 128, "failed": 0, "duration": 9.642780394293368, "total_input_tokens": 16384, "total_output_tokens": 16384, "output_throughput": 1699.0950047660642, "median_ttft_ms": 390.1862818747759, "p95_ttft_ms": 446.47203516215086, "p95_tpot_ms": 17.001757548945978, "p95_e2el_ms": 2478.8263871800154 }, { "case": "short-c32", "repeat": 2, "source_sha256": "cea38fa1708de4089580495213f10294f590b148de0c5465e54dbf8d249cdaba", "num_prompts": 128, "max_concurrency": 32, "completed": 128, "failed": 0, "duration": 9.737690194509923, "total_input_tokens": 16384, "total_output_tokens": 16384, "output_throughput": 1682.5345305437263, "median_ttft_ms": 425.66580092534423, "p95_ttft_ms": 468.2023205328733, "p95_tpot_ms": 16.33510610792698, "p95_e2el_ms": 2486.8842337746173 }, { "case": "short-c32", "repeat": 3, "source_sha256": "bce425cc7aa10b99cf6b2571e4e6f74da747a50a5404dc67674719d0c80bd9f8", "num_prompts": 128, "max_concurrency": 32, "completed": 128, "failed": 0, "duration": 9.704070828855038, "total_input_tokens": 16384, "total_output_tokens": 16384, "output_throughput": 1688.3636041981686, "median_ttft_ms": 413.3862513117492, "p95_ttft_ms": 431.04067030362785, "p95_tpot_ms": 16.39876410504436, "p95_e2el_ms": 2463.5713007766753 }, { "case": "rag4k-c1", "repeat": 1, "source_sha256": "ba3cff39a0ae33fac31d77977990e22bff15113f16fe937b60d00d921a2f3d95", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 7.673663541674614, "total_input_tokens": 16384, "total_output_tokens": 1024, "output_throughput": 133.4434321284477, "median_ttft_ms": 467.865901067853, "p95_ttft_ms": 468.9189343713224, "p95_tpot_ms": 5.686442910638802, "p95_e2el_ms": 1918.8826734665781 }, { "case": "rag4k-c1", "repeat": 2, "source_sha256": "8842ce61bbad94dce346d9ed61b5159061843a6e7cc63f3746f445604173aa93", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 7.678763983771205, "total_input_tokens": 16384, "total_output_tokens": 1024, "output_throughput": 133.3547954025137, "median_ttft_ms": 469.10622203722596, "p95_ttft_ms": 470.10300452820957, "p95_tpot_ms": 5.6889942898323715, "p95_e2el_ms": 1920.409829635173 }, { "case": "rag4k-c1", "repeat": 3, "source_sha256": "2d26039e4fd3afb0275a0eaff64ac37e59eb2cafa90cbfbef33f675aeb4bc299", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 7.677143361419439, "total_input_tokens": 16384, "total_output_tokens": 1024, "output_throughput": 133.38294620704738, "median_ttft_ms": 468.84568501263857, "p95_ttft_ms": 469.61268223822117, "p95_tpot_ms": 5.68664614750328, "p95_e2el_ms": 1919.6678414475173 }, { "case": "rag4k-c4", "repeat": 1, "source_sha256": "7528175a8cced8a40230c1f5e38e0ef84add9f0ca02117a8b59a454dba1ff609", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 7.279960953630507, "total_input_tokens": 32768, "total_output_tokens": 2048, "output_throughput": 281.3201901829796, "median_ttft_ms": 1591.570264659822, "p95_ttft_ms": 1725.3716671839356, "p95_tpot_ms": 10.22558385542795, "p95_e2el_ms": 3662.4331885017455 }, { "case": "rag4k-c4", "repeat": 2, "source_sha256": "ac10b25d2523a52387d83106e0abdad976dfc7c66d5a2f86b78f91205f46db66", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 7.308866702951491, "total_input_tokens": 32768, "total_output_tokens": 2048, "output_throughput": 280.2076003346688, "median_ttft_ms": 1590.2419504709542, "p95_ttft_ms": 1725.1333188265562, "p95_tpot_ms": 10.27245679486762, "p95_e2el_ms": 3661.748126614839 }, { "case": "rag4k-c4", "repeat": 3, "source_sha256": "ff3aa343a7284f4bc41242f0a8347984b3b5e30a4390cb25384eec0effd8faf1", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 7.108340739272535, "total_input_tokens": 32768, "total_output_tokens": 2048, "output_throughput": 288.1122437877945, "median_ttft_ms": 1533.684334717691, "p95_ttft_ms": 1657.8835482243448, "p95_tpot_ms": 8.534145683744082, "p95_e2el_ms": 3588.7990401592106 }, { "case": "rag4k-c8", "repeat": 1, "source_sha256": "0139ce7613f96b711a9c4f15c413ee9315dac751728b9eba01799f2456841b3b", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 11.564094211906195, "total_input_tokens": 65536, "total_output_tokens": 4096, "output_throughput": 354.19981236254785, "median_ttft_ms": 2664.3777559511364, "p95_ttft_ms": 3351.0223887860775, "p95_tpot_ms": 18.584891528292907, "p95_e2el_ms": 5787.144786678255 }, { "case": "rag4k-c8", "repeat": 2, "source_sha256": "02b89bc655048feb07103f2a92f7076f26869d80b7afcbe38ff476c0d65c68b6", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 11.576593583449721, "total_input_tokens": 65536, "total_output_tokens": 4096, "output_throughput": 353.81737904799354, "median_ttft_ms": 2668.1880191899836, "p95_ttft_ms": 3352.9503904283047, "p95_tpot_ms": 18.593491854000035, "p95_e2el_ms": 5795.752226840705 }, { "case": "rag4k-c8", "repeat": 3, "source_sha256": "86eeac6018a1cafd23f3ce90e049f3b6178e4c81de73d5cf632c86529db858d4", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 11.694387224502861, "total_input_tokens": 65536, "total_output_tokens": 4096, "output_throughput": 350.2534952338321, "median_ttft_ms": 2666.118375957012, "p95_ttft_ms": 3353.343572700396, "p95_tpot_ms": 18.82938250771486, "p95_e2el_ms": 5854.554445017129 }, { "case": "rag4k-c16", "repeat": 1, "source_sha256": "3db6f6f00a9ac065a1ad5cee0d49e86061d986e633e58effc34057c5da1c4f6e", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 19.909819703549147, "total_input_tokens": 131072, "total_output_tokens": 8192, "output_throughput": 411.455257856488, "median_ttft_ms": 4302.710012067109, "p95_ttft_ms": 6610.6509496457875, "p95_tpot_ms": 33.162816337437604, "p95_e2el_ms": 9985.395472170785 }, { "case": "rag4k-c16", "repeat": 2, "source_sha256": "0dbf8e94ac19895f0b6218357c3e83ee1b197afa670a5b4835a889d5e5ae4c0d", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 19.820749996230006, "total_input_tokens": 131072, "total_output_tokens": 8192, "output_throughput": 413.30423932283867, "median_ttft_ms": 4295.139140915126, "p95_ttft_ms": 6535.652985237539, "p95_tpot_ms": 33.003160120627165, "p95_e2el_ms": 9921.713853953406 }, { "case": "rag4k-c16", "repeat": 3, "source_sha256": "e662b681c80a925055aff45c1256be36a1af6d624d5d1dc83a13d1367cb2ab22", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 19.931020001880825, "total_input_tokens": 131072, "total_output_tokens": 8192, "output_throughput": 411.0175996625837, "median_ttft_ms": 4304.791182745248, "p95_ttft_ms": 6545.788497524336, "p95_tpot_ms": 33.18489848599568, "p95_e2el_ms": 9971.619898779318 }, { "case": "agent8k-c1", "repeat": 1, "source_sha256": "6ae3f3cece64b4bfcf0b8c22bc573c33bfd3a9cd813afdc0a0c56253980df941", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 26.999528888612986, "total_input_tokens": 32768, "total_output_tokens": 4096, "output_throughput": 151.70635076256767, "median_ttft_ms": 888.7001951225102, "p95_ttft_ms": 889.8373300675303, "p95_tpot_ms": 5.7296206997133945, "p95_e2el_ms": 6750.254665361717 }, { "case": "agent8k-c1", "repeat": 2, "source_sha256": "820ae7c13bff172a671a836eaca5689bb86ba51d0bf53f25bbd8fd8a3c1e57ff", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 27.004181319847703, "total_input_tokens": 32768, "total_output_tokens": 4096, "output_throughput": 151.6802139448492, "median_ttft_ms": 889.6221276372671, "p95_ttft_ms": 890.579989599064, "p95_tpot_ms": 5.729406483413129, "p95_e2el_ms": 6751.704597938806 }, { "case": "agent8k-c1", "repeat": 3, "source_sha256": "41059ff1039d7856e3d8a8914c199dcffa938c321ac06822c21f46a958d4bc5a", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 26.996028776280582, "total_input_tokens": 32768, "total_output_tokens": 4096, "output_throughput": 151.72601992478437, "median_ttft_ms": 888.7754939496517, "p95_ttft_ms": 889.6629661787301, "p95_tpot_ms": 5.727920086165837, "p95_e2el_ms": 6749.277222622186 }, { "case": "agent8k-c4", "repeat": 1, "source_sha256": "70530c155774f01e6cabc92c89af6cabe108e39986572cf21d3665fc9a3a6e94", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 22.490903695113957, "total_input_tokens": 65536, "total_output_tokens": 8192, "output_throughput": 364.236142355617, "median_ttft_ms": 2750.3923224285245, "p95_ttft_ms": 3433.5632691625506, "p95_tpot_ms": 9.576733368071103, "p95_e2el_ms": 11366.605810215697 }, { "case": "agent8k-c4", "repeat": 2, "source_sha256": "07688a9400e36881a71e975e0eaa5aafa675d77a846b67899e28ded943817cf1", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 22.498245465569198, "total_input_tokens": 65536, "total_output_tokens": 8192, "output_throughput": 364.1172825026223, "median_ttft_ms": 2744.922410696745, "p95_ttft_ms": 3428.618638915941, "p95_tpot_ms": 9.565280099970199, "p95_e2el_ms": 11296.825659042224 }, { "case": "agent8k-c4", "repeat": 3, "source_sha256": "843e3cfd047ac269b721e0d5278f4761ff8a007f13172f6c57498d1ff3d1547d", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 22.720641071908176, "total_input_tokens": 65536, "total_output_tokens": 8192, "output_throughput": 360.5532068427681, "median_ttft_ms": 2752.0297509618104, "p95_ttft_ms": 3432.997280312702, "p95_tpot_ms": 9.664516629995314, "p95_e2el_ms": 11395.331712579355 }, { "case": "agent8k-c8", "repeat": 1, "source_sha256": "b9d0dcb0da80ab070ce8cebfb526ea4baffff13d43407bab8b58a307b7e22557", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 33.716097021475434, "total_input_tokens": 131072, "total_output_tokens": 16384, "output_throughput": 485.93999446508377, "median_ttft_ms": 4449.902791064233, "p95_ttft_ms": 6830.315636936575, "p95_tpot_ms": 15.014114930774227, "p95_e2el_ms": 16873.628149041906 }, { "case": "agent8k-c8", "repeat": 2, "source_sha256": "c293e237fcdb1269480ec5331222bba96b22af59ba058bf52199d79a27e8a149", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 33.52651181444526, "total_input_tokens": 131072, "total_output_tokens": 16384, "output_throughput": 488.68788052507085, "median_ttft_ms": 4448.089090641588, "p95_ttft_ms": 6819.784871069714, "p95_tpot_ms": 14.845570649283474, "p95_e2el_ms": 16935.64539658837 }, { "case": "agent8k-c8", "repeat": 3, "source_sha256": "3aebe6bac9b3ad6b3f742d10f5106f9c887ba33c0ca63fa12dc3e5a9ac259064", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 33.67547501344234, "total_input_tokens": 131072, "total_output_tokens": 16384, "output_throughput": 486.52617352717226, "median_ttft_ms": 4445.697316434234, "p95_ttft_ms": 6813.702237792313, "p95_tpot_ms": 15.0038606383886, "p95_e2el_ms": 16838.896081782877 }, { "case": "agent8k-c16", "repeat": 1, "source_sha256": "9a1a4e4e7b8b174f273e9f2ba2eb7326b14c63774443889a15be9970f9e206ef", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 54.50272866431624, "total_input_tokens": 262144, "total_output_tokens": 32768, "output_throughput": 601.2176051188003, "median_ttft_ms": 7838.529746513814, "p95_ttft_ms": 13453.397444169968, "p95_tpot_ms": 24.730357357445566, "p95_e2el_ms": 27367.690696241334 }, { "case": "agent8k-c16", "repeat": 2, "source_sha256": "1bb6e573158246385eebf194301b6462841a3425328fc0f0308a54141f9678e4", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 54.059628018178046, "total_input_tokens": 262144, "total_output_tokens": 32768, "output_throughput": 606.1454952849742, "median_ttft_ms": 8269.589121919125, "p95_ttft_ms": 13614.6007088013, "p95_tpot_ms": 24.49711917686519, "p95_e2el_ms": 27050.018274690956 }, { "case": "agent8k-c16", "repeat": 3, "source_sha256": "d63560253422f25ce26f9f4ea3ddf9121534d1019e6bf4934cd83ee0e39c6586", "num_prompts": 32, "max_concurrency": 16, "completed": 32, "failed": 0, "duration": 54.71316687669605, "total_input_tokens": 262144, "total_output_tokens": 32768, "output_throughput": 598.9051972415959, "median_ttft_ms": 7850.686855614185, "p95_ttft_ms": 13459.505489934236, "p95_tpot_ms": 24.815388437030965, "p95_e2el_ms": 27389.067482901737 }, { "case": "rag16k-c1", "repeat": 1, "source_sha256": "7b4579eb8aba46361c607f46904df8ae45327cd9ec3372056b60f198c858eee3", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 13.227483824826777, "total_input_tokens": 65536, "total_output_tokens": 1024, "output_throughput": 77.41457208044706, "median_ttft_ms": 1822.7063664235175, "p95_ttft_ms": 1824.218468926847, "p95_tpot_ms": 5.818760297535097, "p95_e2el_ms": 3307.5869200751185 }, { "case": "rag16k-c1", "repeat": 2, "source_sha256": "3818d58dd9880db5a9b9afb18b2a290c7692d51963faf1ccaf67ba5526c83455", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 13.242212632670999, "total_input_tokens": 65536, "total_output_tokens": 1024, "output_throughput": 77.32846680573621, "median_ttft_ms": 1826.2194842100143, "p95_ttft_ms": 1830.2201158832759, "p95_tpot_ms": 5.819052955975719, "p95_e2el_ms": 3314.0786196570843 }, { "case": "rag16k-c1", "repeat": 3, "source_sha256": "87f07d7e42bf5f9c3fe32ac66c8b9076ec6127877724cbd89c4d6a4a7baa0e97", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 13.220244680531323, "total_input_tokens": 65536, "total_output_tokens": 1024, "output_throughput": 77.45696276771523, "median_ttft_ms": 1820.7553923130035, "p95_ttft_ms": 1822.3110645543784, "p95_tpot_ms": 5.819530714778048, "p95_e2el_ms": 3305.66856241785 }, { "case": "rag16k-c4", "repeat": 1, "source_sha256": "72c4eb574949e5e0db8eeb2ee603ab60911cdc667ae2d6eb23ecfa275bef3f16", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 18.038801101967692, "total_input_tokens": 131072, "total_output_tokens": 2048, "output_throughput": 113.5330440434094, "median_ttft_ms": 5021.235513500869, "p95_ttft_ms": 7040.7096181530505, "p95_tpot_ms": 25.92212115104, "p95_e2el_ms": 9057.900490052998 }, { "case": "rag16k-c4", "repeat": 2, "source_sha256": "b52bc7a3cd23a586975427afebb35596a9c9ee858876ecd1a21369b6c68241a4", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 18.01219893898815, "total_input_tokens": 131072, "total_output_tokens": 2048, "output_throughput": 113.70072065809907, "median_ttft_ms": 5020.725603215396, "p95_ttft_ms": 7039.001921424642, "p95_tpot_ms": 25.845547390773014, "p95_e2el_ms": 9014.862985396758 }, { "case": "rag16k-c4", "repeat": 3, "source_sha256": "2cab299d7937f06cc9104d5a73ef0e4ef859b8b4dd42670d3fb41b68ec196e6c", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 18.1108540892601, "total_input_tokens": 131072, "total_output_tokens": 2048, "output_throughput": 113.08135938296154, "median_ttft_ms": 5024.354773107916, "p95_ttft_ms": 7042.428741697222, "p95_tpot_ms": 26.022184844877497, "p95_e2el_ms": 9056.109406379983 }, { "case": "rag16k-c8", "repeat": 1, "source_sha256": "2562adaef4043d4332261c9d634ecdb2c50d0d405a3bc77906e03ce6fd6867a0", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 33.12951221317053, "total_input_tokens": 262144, "total_output_tokens": 4096, "output_throughput": 123.63598877171661, "median_ttft_ms": 8499.057448934764, "p95_ttft_ms": 13992.113258456811, "p95_tpot_ms": 55.41802221069149, "p95_e2el_ms": 16579.84473463148 }, { "case": "rag16k-c8", "repeat": 2, "source_sha256": "bdd550d0e425af76259a68bf29664c628e3111fd6764293c97524e4183636fd3", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 33.017964280210435, "total_input_tokens": 262144, "total_output_tokens": 4096, "output_throughput": 124.05368075508423, "median_ttft_ms": 8476.816992741078, "p95_ttft_ms": 13959.711564471945, "p95_tpot_ms": 55.235655577488096, "p95_e2el_ms": 16551.10929394141 }, { "case": "rag16k-c8", "repeat": 3, "source_sha256": "6e460c83ad08dd991ff809d1739886979e653c3dd5739cc70a328e5a07b7cef2", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 33.15908518526703, "total_input_tokens": 262144, "total_output_tokens": 4096, "output_throughput": 123.52572385862746, "median_ttft_ms": 8485.748069826514, "p95_ttft_ms": 13980.177571065724, "p95_tpot_ms": 55.56238286525887, "p95_e2el_ms": 16588.759820908308 }, { "case": "decode2k-c1", "repeat": 1, "source_sha256": "7fb5bbe4a9eb4c579e0ac017c9abc8b9dd6d60834618d04fd577d31053b0a06f", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 45.34100842196494, "total_input_tokens": 512, "total_output_tokens": 8192, "output_throughput": 180.67529340682853, "median_ttft_ms": 44.01276493445039, "p95_ttft_ms": 44.96118761599064, "p95_tpot_ms": 5.515986695111301, "p95_e2el_ms": 11335.369908902794 }, { "case": "decode2k-c1", "repeat": 2, "source_sha256": "22f82037b06b09b8922d9972d225341b830a1b20b64d67288f75e574f308dfbe", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 45.33145991154015, "total_input_tokens": 512, "total_output_tokens": 8192, "output_throughput": 180.71335041902194, "median_ttft_ms": 44.86111970618367, "p95_ttft_ms": 44.885131530463696, "p95_tpot_ms": 5.515089424433479, "p95_e2el_ms": 11333.993809716776 }, { "case": "decode2k-c1", "repeat": 3, "source_sha256": "6fdd4790e93d56eabce00b51df9b6a9d2e734894e61b720c0d66207d9699de01", "num_prompts": 4, "max_concurrency": 1, "completed": 4, "failed": 0, "duration": 45.325627502053976, "total_input_tokens": 512, "total_output_tokens": 8192, "output_throughput": 180.73660424510993, "median_ttft_ms": 44.617258943617344, "p95_ttft_ms": 45.52065529860556, "p95_tpot_ms": 5.51411819932723, "p95_e2el_ms": 11332.676309905946 }, { "case": "decode2k-c4", "repeat": 1, "source_sha256": "d4200693baea92651c1147afa6209d2bb51d0599aadc69fd55e1d637403afb2a", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 29.969350428320467, "total_input_tokens": 1024, "total_output_tokens": 16384, "output_throughput": 546.6918623807552, "median_ttft_ms": 92.40700304508209, "p95_ttft_ms": 98.67310826666653, "p95_tpot_ms": 7.39443135016944, "p95_e2el_ms": 15222.940224735066 }, { "case": "decode2k-c4", "repeat": 2, "source_sha256": "4d2b221edbaef573fdd1cbb0c5d498578d2c4c7f606a21292745d9465778a08c", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 30.49782791081816, "total_input_tokens": 1024, "total_output_tokens": 16384, "output_throughput": 537.2185864485216, "median_ttft_ms": 85.91312356293201, "p95_ttft_ms": 97.80522282235324, "p95_tpot_ms": 7.495040833721648, "p95_e2el_ms": 15427.212808001786 }, { "case": "decode2k-c4", "repeat": 3, "source_sha256": "b37a2c3a872c7659386f48ee470f310c924b178f8c680e782a80f5abbe821763", "num_prompts": 8, "max_concurrency": 4, "completed": 8, "failed": 0, "duration": 30.515753580257297, "total_input_tokens": 1024, "total_output_tokens": 16384, "output_throughput": 536.9030116496915, "median_ttft_ms": 94.5277251303196, "p95_ttft_ms": 98.29690461046994, "p95_tpot_ms": 7.473062194108555, "p95_e2el_ms": 15377.878010831773 }, { "case": "decode2k-c8", "repeat": 1, "source_sha256": "8fe28e7af681d9c60ecda43ff41e47800b4e44654785379197e29af4a29892ba", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 38.01780564151704, "total_input_tokens": 2048, "total_output_tokens": 32768, "output_throughput": 861.911923822767, "median_ttft_ms": 150.9172641672194, "p95_ttft_ms": 151.9332774914801, "p95_tpot_ms": 9.330358147406411, "p95_e2el_ms": 19233.026472385973 }, { "case": "decode2k-c8", "repeat": 2, "source_sha256": "ee2b56db51bdbce43e0ee21e38ba4e31d98f7eb1153b11fb08542ff1e44fd78c", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 38.206812352873385, "total_input_tokens": 2048, "total_output_tokens": 32768, "output_throughput": 857.6480994373153, "median_ttft_ms": 151.04826586320996, "p95_ttft_ms": 152.36731665208936, "p95_tpot_ms": 9.377934485077896, "p95_e2el_ms": 19288.895815378055 }, { "case": "decode2k-c8", "repeat": 3, "source_sha256": "7fcaff89ba78c73b447b07d92067bde2f38a66c6446e6d6fba812c0342ad4747", "num_prompts": 16, "max_concurrency": 8, "completed": 16, "failed": 0, "duration": 38.569182614795864, "total_input_tokens": 2048, "total_output_tokens": 32768, "output_throughput": 849.5902111088446, "median_ttft_ms": 152.255492284894, "p95_ttft_ms": 157.7806482091546, "p95_tpot_ms": 9.589670087212198, "p95_e2el_ms": 19766.728955321014 } ], "history_round_data": [ { "case": "sglang-history-short128-64-c1-r1", "source_sha256": "2362fe9b33fac4343a3350e531aa91f28e4b4ac20a43a273eb91dedd15b59102", "num_prompts": 32, "completed": 32, "failed": 0, "output_throughput": 164.78285669048273, "p95_ttft_ms": 44.72756814211607, "p95_tpot_ms": 5.465134748420308 }, { "case": "sglang-history-short128-64-c1-r2", "source_sha256": "f59ea2831883ba75d6eaad97d9ba264ae12f450d8287a17564d439623462c5c3", "num_prompts": 32, "completed": 32, "failed": 0, "output_throughput": 164.51451484917504, "p95_ttft_ms": 44.999386463314295, "p95_tpot_ms": 5.4669557604938745 }, { "case": "sglang-history-short128-64-c1-r3", "source_sha256": "9dde3899c9cab859c6c971221a80e66aae3b4ce1f4e5463dd08ae4e4b1ea3b8b", "num_prompts": 32, "completed": 32, "failed": 0, "output_throughput": 164.6650353197827, "p95_ttft_ms": 44.931814866140485, "p95_tpot_ms": 5.46779319392665 }, { "case": "sglang-history-short128-64-c4-r1", "source_sha256": "f2741f301d96d2a1d192bcd70037909ab03c43047863335652524ac36f9880d6", "num_prompts": 64, "completed": 64, "failed": 0, "output_throughput": 455.712709708926, "p95_ttft_ms": 98.96190562285483, "p95_tpot_ms": 7.732202677381417 }, { "case": "sglang-history-short128-64-c4-r2", "source_sha256": "771b02f6ff4cb9374cb241a1faef7fe1ef26d396a09d63b6b04b4828de7ecbf7", "num_prompts": 64, "completed": 64, "failed": 0, "output_throughput": 455.08607305128606, "p95_ttft_ms": 101.89272123388946, "p95_tpot_ms": 7.7418212712343255 }, { "case": "sglang-history-short128-64-c4-r3", "source_sha256": "1d29440368471d7c2ae27e65f864489c780859227c94d096d7246950229d8d16", "num_prompts": 64, "completed": 64, "failed": 0, "output_throughput": 455.52362074634755, "p95_ttft_ms": 100.80120512284338, "p95_tpot_ms": 7.706774994435292 }, { "case": "sglang-history-short128-64-c8-r1", "source_sha256": "dc2347f2c193bce7765d024936f77abad5787f62d6518e8d77490357bb574b52", "num_prompts": 128, "completed": 128, "failed": 0, "output_throughput": 707.2591122319451, "p95_ttft_ms": 160.01993417739868, "p95_tpot_ms": 10.18983329247151 }, { "case": "sglang-history-short128-64-c8-r2", "source_sha256": "3937a938127159adb44e1d0845b7f3443678d5bf4e0f5a7803b71f396f6de8df", "num_prompts": 128, "completed": 128, "failed": 0, "output_throughput": 708.2183015051356, "p95_ttft_ms": 151.38186844997108, "p95_tpot_ms": 10.23082100002775 }, { "case": "sglang-history-short128-64-c8-r3", "source_sha256": "355e810590664ebfddf079a339a2d87b64166e5d4cbe35c8ba0ebcff59b4e27a", "num_prompts": 128, "completed": 128, "failed": 0, "output_throughput": 703.8418364655797, "p95_ttft_ms": 152.6752222329378, "p95_tpot_ms": 10.258592695500406 } ], "long_context": [ { "target_context": 65536, "seed": 103101, "fixture_sha256": "b7872b02ceaad62ae5014e894eb6d2fc6d4a1268a311449daa583e165e91fd1e", "prompt_tokens": 57344, "completion_tokens": 73, "ttft_s": 12.093781070783734, "elapsed_s": 12.527812239713967, "status": "PASS", "needle_fractions": [ 0.10300990513392858, 0.5009940011160714, 0.89892578125 ] }, { "target_context": 131072, "seed": 103101, "fixture_sha256": "3ca1eb63143f10a8c8f4f8a3a297b2948474e8841c9eb86bae7f584dddec2df3", "prompt_tokens": 122881, "completion_tokens": 76, "ttft_s": 15.614580172114074, "elapsed_s": 16.102319612167776, "status": "PASS", "needle_fractions": [ 0.10140705235146198, 0.5004679323898731, 0.8995043985644648 ] }, { "target_context": 262144, "seed": 103101, "fixture_sha256": "ed3a0c83c1826ab2a0a99df004a53e8f858c8047d1fe9312376089a832e0c1ca", "prompt_tokens": 253952, "completion_tokens": 207, "ttft_s": 36.76292698830366, "elapsed_s": 38.21698911394924, "status": "PASS", "needle_fractions": [ 0.10067256804435484, 0.5002165763608871, 0.8997448336693549 ] }, { "target_context": 524288, "seed": 103101, "fixture_sha256": "4605f7ede8340a35ba8b3b2366acd2448595579a76b78b72ba414c9bf680aefd", "prompt_tokens": 516095, "completion_tokens": 169, "ttft_s": 94.7944928156212, "elapsed_s": 96.14259034954011, "status": "PASS", "needle_fractions": [ 0.1003323031612397, 0.500105600713047, 0.8998847111481413 ] }, { "target_context": 1048576, "seed": 103101, "fixture_sha256": "1f7eec8462cf5a45b48ea7ae9eef22494fc7c44c710e6eb44434e3f6c4f33b0c", "prompt_tokens": 1040383, "completion_tokens": 247, "ttft_s": 272.49718691501766, "elapsed_s": 274.95116566773504, "status": "PASS", "needle_fractions": [ 0.1001659965608819, 0.5000523845545343, 0.8999416561016472 ] }, { "target_context": 1048576, "seed": 103102, "fixture_sha256": "e441b9e81842edbb66a17d7b7d1a660a3c0944d4a040a88c70cf486d1d117457", "prompt_tokens": 1040384, "completion_tokens": 174, "ttft_s": 272.26041066553444, "elapsed_s": 273.9829167481512, "status": "PASS", "needle_fractions": [ 0.10016590028297244, 0.5000557486466536, 0.8999417522760826 ] }, { "target_context": 1048576, "seed": 103103, "fixture_sha256": "f591fb0fa04c41292b8a1bf924380e24d1d6c2c010a9c0323a767e82beb2bf21", "prompt_tokens": 1040384, "completion_tokens": 203, "ttft_s": 272.1117640398443, "elapsed_s": 274.1249114498496, "status": "PASS", "needle_fractions": [ 0.10016493909940945, 0.5000519039124016, 0.8999369463582677 ] } ], "functional": { "critical": { "passed": 3, "planned": 3 }, "tools": { "passed": 4, "planned": 4 }, "vision": { "passed": 4, "planned": 4 }, "effort": { "passed": 5, "planned": 5 } }, "formal_completed_requests": 1944, "formal_failed_requests": 0 } }