{
  "model": "DeepSeek-V4-Flash-0731",
  "engine": "vLLM 0.26.0b2.dev1+g3b102b576",
  "hardware": "2 replicas x TP8, 16 x NVIDIA H20-3e",
  "precision": "checkpoint as supplied; FP8 KV cache",
  "method": "closed-loop concurrency, per-phase warmup, >=60 seconds measured, 256 output tokens, temperature=0, ignore_eos=true",
  "notes": [
    "First coverage and rechecks are both retained. Some shapes triggered runtime JIT.",
    "Shared-prefix routing comparisons are order-sensitive and are not controlled cold-cache experiments.",
    "TTFT and TPOT percentiles below are computed directly from client request records."
  ],
  "phases": [
    {
      "phase": "direct-c1",
      "route": "direct",
      "workload": "short",
      "concurrency": 1,
      "requests": 35,
      "success": 35,
      "errors": 0,
      "elapsed": 61.0440497584641,
      "output_tokens_per_second": 146.77925261270278,
      "request_rate": 0.5733564555183702,
      "ttft": {
        "p50": 0.1522549595683813,
        "p95": 0.15726048070937396,
        "p99": 0.16076731991022825,
        "min": 0.14664427563548088,
        "max": 0.1625308059155941
      },
      "tpot": {
        "p50": 0.006237084170182546,
        "p95": 0.006249158716231,
        "p99": 0.0062498910628697455,
        "min": 0.006207458711430139,
        "max": 0.0062501316123148975
      },
      "e2e": {
        "p50": 1.7428319416940212,
        "p95": 1.7488937163725495,
        "p99": 1.7497180868685245,
        "min": 1.7372000720351934,
        "max": 1.749933436512947
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 543.3199999999999,
        "min": 533,
        "max": 544
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 0.0,
      "end_offset_seconds": 61.04405450820923
    },
    {
      "phase": "gateway-c1",
      "route": "least-request",
      "workload": "short",
      "concurrency": 1,
      "requests": 35,
      "success": 35,
      "errors": 0,
      "elapsed": 61.39671846665442,
      "output_tokens_per_second": 145.93613834371826,
      "request_rate": 0.5700630404051494,
      "ttft": {
        "p50": 0.15790963917970657,
        "p95": 0.16338340826332567,
        "p99": 0.2640857044607396,
        "min": 0.15127427875995636,
        "max": 0.31523599475622177
      },
      "tpot": {
        "p50": 0.006238634504523932,
        "p95": 0.006241564427082445,
        "p99": 0.006246726821888896,
        "min": 0.0062288852445050785,
        "max": 0.006249346160421184
      },
      "e2e": {
        "p50": 1.748827449977398,
        "p95": 1.7542967477813363,
        "p99": 1.8567742169275874,
        "min": 1.7426403798162937,
        "max": 1.9088536631315947
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 540.3,
        "p99": 541.0,
        "min": 533,
        "max": 541
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 84.55976629257202,
      "end_offset_seconds": 145.9564893245697
    },
    {
      "phase": "gateway-c4",
      "route": "least-request",
      "workload": "short",
      "concurrency": 4,
      "requests": 108,
      "success": 108,
      "errors": 0,
      "elapsed": 61.01261099614203,
      "output_tokens_per_second": 453.1522180184724,
      "request_rate": 1.7701258516346579,
      "ttft": {
        "p50": 0.22746668756008148,
        "p95": 0.30910141272470354,
        "p99": 14.927247753199074,
        "min": 0.15051239915192127,
        "max": 16.022998731583357
      },
      "tpot": {
        "p50": 0.006536463830693095,
        "p95": 0.007100237717888519,
        "p99": 0.007312468116043829,
        "min": 0.006414871451024915,
        "max": 0.007357626415643037
      },
      "e2e": {
        "p50": 1.9640463199466467,
        "p95": 2.019800788536668,
        "p99": 16.56690638840187,
        "min": 1.7972121592611074,
        "max": 17.661510966718197
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 542.0,
        "min": 532,
        "max": 543
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 183.295756816864,
      "end_offset_seconds": 244.30837988853455
    },
    {
      "phase": "gateway-c16",
      "route": "least-request",
      "workload": "short",
      "concurrency": 16,
      "requests": 368,
      "success": 368,
      "errors": 0,
      "elapsed": 60.89702609181404,
      "output_tokens_per_second": 1547.0049367922043,
      "request_rate": 6.042988034344548,
      "ttft": {
        "p50": 0.42528897523880005,
        "p95": 3.0867371472530065,
        "p99": 5.404012444354594,
        "min": 0.15091039426624775,
        "max": 5.404470520094037
      },
      "tpot": {
        "p50": 0.007764453119507023,
        "p95": 0.00895555977435673,
        "p99": 0.009476421822373772,
        "min": 0.007539692754838981,
        "max": 0.028272522153223262
      },
      "e2e": {
        "p50": 2.4027854735031724,
        "p95": 5.054206569865346,
        "p99": 7.395179634485394,
        "min": 2.3351629246026278,
        "max": 7.396997552365065
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 543.0,
        "min": 533,
        "max": 544
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 301.07072830200195,
      "end_offset_seconds": 361.96780347824097
    },
    {
      "phase": "gateway-c32",
      "route": "least-request",
      "workload": "short",
      "concurrency": 32,
      "requests": 672,
      "success": 672,
      "errors": 0,
      "elapsed": 62.43814811296761,
      "output_tokens_per_second": 2755.238667372826,
      "request_rate": 10.762651044425102,
      "ttft": {
        "p50": 0.6633551958948374,
        "p95": 0.8112576629966497,
        "p99": 0.8174408945254982,
        "min": 0.14911848306655884,
        "max": 0.8217105884104967
      },
      "tpot": {
        "p50": 0.008914323096327921,
        "p95": 0.010898885345269067,
        "p99": 0.011082341641845071,
        "min": 0.00861436492379974,
        "max": 0.011702068245001867
      },
      "e2e": {
        "p50": 2.953013598918915,
        "p95": 3.0715057166293263,
        "p99": 3.1750245326571167,
        "min": 2.8654506783932447,
        "max": 3.1795241981744766
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 543.0,
        "min": 533,
        "max": 545
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 409.16277027130127,
      "end_offset_seconds": 471.6009976863861
    },
    {
      "phase": "long-c16",
      "route": "least-request",
      "workload": "long",
      "concurrency": 16,
      "requests": 200,
      "success": 200,
      "errors": 0,
      "elapsed": 62.60468908213079,
      "output_tokens_per_second": 817.8301138566628,
      "request_rate": 3.194648882252589,
      "ttft": {
        "p50": 1.6381223499774933,
        "p95": 2.7381785173900424,
        "p99": 7.197041318006813,
        "min": 0.38324868865311146,
        "max": 7.340347131714225
      },
      "tpot": {
        "p50": 0.012406171007337524,
        "p95": 0.01732048653613995,
        "p99": 0.017341973111398665,
        "min": 0.008149676872234718,
        "max": 0.0367510090198587
      },
      "e2e": {
        "p50": 4.80037048459053,
        "p95": 5.149364618584514,
        "p99": 9.759887247178703,
        "min": 3.8419719617813826,
        "max": 10.376888260245323
      },
      "input_tokens": {
        "p50": 4448.0,
        "p95": 4451.0,
        "p99": 4453.0,
        "min": 4443,
        "max": 4455
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 519.8247821331024,
      "end_offset_seconds": 582.4294996261597
    },
    {
      "phase": "prefix-random",
      "route": "random",
      "workload": "shared-prefix",
      "concurrency": 16,
      "requests": 64,
      "success": 64,
      "errors": 0,
      "elapsed": 100.75551709160209,
      "output_tokens_per_second": 162.6114427570696,
      "request_rate": 0.6352009482698031,
      "ttft": {
        "p50": 19.880066939629614,
        "p95": 33.747209985926744,
        "p99": 33.77495318958536,
        "min": 0.1707962118089199,
        "max": 33.77609150484204
      },
      "tpot": {
        "p50": 0.00965566565373949,
        "p95": 0.15615135514341733,
        "p99": 0.19446949085147625,
        "min": 0.006338875346324023,
        "max": 0.2597045195818532
      },
      "e2e": {
        "p50": 26.407988320104778,
        "p95": 66.41520617315545,
        "p99": 66.42063975403086,
        "min": 1.787243651226163,
        "max": 66.42092901468277
      },
      "input_tokens": {
        "p50": 4447.0,
        "p95": 4450.85,
        "p99": 4451.37,
        "min": 4442,
        "max": 4452
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 673.6417725086212,
      "end_offset_seconds": 774.3972992897034
    },
    {
      "phase": "prefix-aware",
      "route": "prefix-cache",
      "workload": "shared-prefix",
      "concurrency": 16,
      "requests": 288,
      "success": 288,
      "errors": 0,
      "elapsed": 60.858928840607405,
      "output_tokens_per_second": 1211.457404928985,
      "request_rate": 4.732255488003847,
      "ttft": {
        "p50": 0.3458389565348625,
        "p95": 13.728473276179283,
        "p99": 14.074510584622622,
        "min": 0.18079600296914577,
        "max": 14.075016345828772
      },
      "tpot": {
        "p50": 0.008891104293220183,
        "p95": 0.009511293035511877,
        "p99": 0.009664558225548734,
        "min": 0.008846580536634315,
        "max": 0.010350774549970439
      },
      "e2e": {
        "p50": 2.613088581711054,
        "p95": 16.367439928557722,
        "p99": 16.394695203900337,
        "min": 2.5945440996438265,
        "max": 16.396317590028048
      },
      "input_tokens": {
        "p50": 4447.0,
        "p95": 4450.65,
        "p99": 4452.0,
        "min": 4442,
        "max": 4453
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 799.6687657833099,
      "end_offset_seconds": 860.5277307033539
    },
    {
      "phase": "recheck-c4",
      "route": "least-request",
      "workload": "short",
      "concurrency": 4,
      "requests": 124,
      "success": 124,
      "errors": 0,
      "elapsed": 61.017765544354916,
      "output_tokens_per_second": 520.2419281795023,
      "request_rate": 2.032195031951181,
      "ttft": {
        "p50": 0.1690211519598961,
        "p95": 0.317955416161567,
        "p99": 0.32287263218313456,
        "min": 0.1545281857252121,
        "max": 0.32626328617334366
      },
      "tpot": {
        "p50": 0.006535944689576532,
        "p95": 0.007114432873021738,
        "p99": 0.0071257251287792246,
        "min": 0.0064137456756012115,
        "max": 0.007136173480573822
      },
      "e2e": {
        "p50": 1.9727694327011704,
        "p95": 1.9818354664370417,
        "p99": 1.988592721708119,
        "min": 1.7996444888412952,
        "max": 1.992674706503749
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 541.0,
        "p99": 542.0,
        "min": 534,
        "max": 543
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 890.3697805404663,
      "end_offset_seconds": 951.3875622749329
    },
    {
      "phase": "recheck-c16",
      "route": "least-request",
      "workload": "short",
      "concurrency": 16,
      "requests": 400,
      "success": 400,
      "errors": 0,
      "elapsed": 61.03552778810263,
      "output_tokens_per_second": 1677.7113873005676,
      "request_rate": 6.553560106642842,
      "ttft": {
        "p50": 0.3880172437056899,
        "p95": 0.551369393710047,
        "p99": 0.5538026318512856,
        "min": 0.15207703597843647,
        "max": 0.5555348359048367
      },
      "tpot": {
        "p50": 0.00786353401182329,
        "p95": 0.009216543806388098,
        "p99": 0.009348768137395382,
        "min": 0.00759061265225504,
        "max": 0.009464512524359366
      },
      "e2e": {
        "p50": 2.401336809620261,
        "p95": 2.549530057888478,
        "p99": 2.576207752395421,
        "min": 2.3126724418252707,
        "max": 2.578453328460455
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 543.0,
        "min": 533,
        "max": 545
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 997.749715089798,
      "end_offset_seconds": 1058.7852940559387
    },
    {
      "phase": "recheck-c32",
      "route": "least-request",
      "workload": "short",
      "concurrency": 32,
      "requests": 672,
      "success": 672,
      "errors": 0,
      "elapsed": 63.02321139164269,
      "output_tokens_per_second": 2729.6609646079164,
      "request_rate": 10.662738142999673,
      "ttft": {
        "p50": 0.6883548879995942,
        "p95": 0.8165392094291747,
        "p99": 0.8201579416356981,
        "min": 0.15421649627387524,
        "max": 0.8221506550908089
      },
      "tpot": {
        "p50": 0.008975536160755391,
        "p95": 0.011030885574280048,
        "p99": 0.01143797462901064,
        "min": 0.008763488900719905,
        "max": 0.01166558994674215
      },
      "e2e": {
        "p50": 2.9740346176549792,
        "p95": 3.1117103450000285,
        "p99": 3.135245544016361,
        "min": 2.9114415738731623,
        "max": 3.137923615053296
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 543.29,
        "min": 533,
        "max": 545
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 1084.8847839832306,
      "end_offset_seconds": 1147.9080722332
    },
    {
      "phase": "recheck-prefix-random",
      "route": "random",
      "workload": "shared-prefix",
      "concurrency": 16,
      "requests": 66,
      "success": 66,
      "errors": 0,
      "elapsed": 83.21322472579777,
      "output_tokens_per_second": 203.04464892059278,
      "request_rate": 0.7931431598460655,
      "ttft": {
        "p50": 21.95080317929387,
        "p95": 26.76099537871778,
        "p99": 27.277812488377094,
        "min": 0.17103103920817375,
        "max": 27.27853911370039
      },
      "tpot": {
        "p50": 0.0096227956826196,
        "p95": 0.11699177696570462,
        "p99": 0.11719034680512314,
        "min": 0.006328454862038295,
        "max": 0.11755027578157537
      },
      "e2e": {
        "p50": 26.41491323336959,
        "p95": 30.15328128170222,
        "p99": 30.15522043677047,
        "min": 1.786469740793109,
        "max": 30.155326480045915
      },
      "input_tokens": {
        "p50": 4448.0,
        "p95": 4451.0,
        "p99": 4452.0,
        "min": 4443,
        "max": 4452
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 1196.3567621707916,
      "end_offset_seconds": 1279.569997549057
    },
    {
      "phase": "recheck-prefix-aware",
      "route": "prefix-cache",
      "workload": "shared-prefix",
      "concurrency": 16,
      "requests": 369,
      "success": 369,
      "errors": 0,
      "elapsed": 61.77444908954203,
      "output_tokens_per_second": 1529.1759196925332,
      "request_rate": 5.973343436298958,
      "ttft": {
        "p50": 0.33381306007504463,
        "p95": 0.3453005369752645,
        "p99": 0.3679375744611025,
        "min": 0.17445389553904533,
        "max": 0.36859811656177044
      },
      "tpot": {
        "p50": 0.008879127923180075,
        "p95": 0.009467945037519231,
        "p99": 0.009532240944165809,
        "min": 0.006330519542098045,
        "max": 0.00977040532172895
      },
      "e2e": {
        "p50": 2.5995354037731886,
        "p95": 2.6704438112676145,
        "p99": 2.673778206259012,
        "min": 1.7889040131121874,
        "max": 2.680444782599807
      },
      "input_tokens": {
        "p50": 4447.0,
        "p95": 4450.0,
        "p99": 4452.0,
        "min": 4442,
        "max": 4453
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 1304.8297724723816,
      "end_offset_seconds": 1366.6042714118958
    },
    {
      "phase": "eviction-c4",
      "concurrency": 4,
      "route": "least-request",
      "workload": "short",
      "requests": 928,
      "success": 926,
      "errors": 2,
      "elapsed": 600.5887378584594,
      "output_tokens_per_second": 394.70603602271837,
      "request_rate": 1.5418204532137436,
      "ttft": {
        "p50": 0.16632010973989964,
        "p95": 0.31911638705059886,
        "p99": 0.3273481661453843,
        "min": 0.15203458815813065,
        "max": 16.48806388862431
      },
      "tpot": {
        "p50": 0.007611126869040377,
        "p95": 0.008839209858035924,
        "p99": 0.008914938623852588,
        "min": 0.006418513711176667,
        "max": 0.008967727934028589
      },
      "e2e": {
        "p50": 2.1065356181934476,
        "p95": 2.4366822158917785,
        "p99": 2.449187422171235,
        "min": 1.8013265281915665,
        "max": 18.153297301381826
      },
      "input_tokens": {
        "p50": 538.0,
        "p95": 542.0,
        "p99": 543.0,
        "min": 533,
        "max": 545
      },
      "output_tokens": {
        "p50": 256.0,
        "p95": 256.0,
        "p99": 256.0,
        "min": 256,
        "max": 256
      },
      "start_offset_seconds": 1405.0938351154327,
      "end_offset_seconds": 2005.6826040744781
    }
  ]
}