{ "status": "PASS", "runtime": "vLLM 0.31.0 / CUDA 13.0", "gpu": "1 × NVIDIA L20", "context_tokens": 4096, "kv_gib": 4, "models": { "A": { "name": "Qwen3.8-27B-FP8", "restart_initial": { "n": 1, "median": 219.19701036345214, "min": 219.19701036345214, "max": 219.19701036345214, "p95": 219.19701036345214 }, "restart_warm": { "n": 2, "median": 55.62885317578912, "min": 54.133458437398076, "max": 57.12424791418016, "p95": 57.12424791418016 }, "sleep": { "1": { "successful_cycles": 5, "sleep_s": { "n": 5, "median": 1.3361948309466243, "min": 1.3325311271473765, "max": 18.466762921772897, "p95": 18.466762921772897 }, "wake_api_s": { "n": 5, "median": 1.7580801593139768, "min": 1.742534164339304, "max": 1.7689893301576376, "p95": 1.7689893301576376 }, "to_first_token_s": { "n": 5, "median": 1.8620264418423176, "min": 1.8463675044476986, "max": 1.875916856341064, "p95": 1.875916856341064 }, "residual_gib": { "n": 5, "median": 0.5830078125, "min": 0.5830078125, "max": 0.5830078125, "p95": 0.5830078125 } }, "2": { "successful_cycles": 3, "sleep_s": { "n": 3, "median": 0.5871797641739249, "min": 0.5860766898840666, "max": 0.6025053225457668, "p95": 0.6025053225457668 }, "wake_api_s": { "n": 3, "median": 14.80091958772391, "min": 14.64313611201942, "max": 14.804665757343173, "p95": 14.804665757343173 }, "to_first_token_s": { "n": 3, "median": 14.907669715583324, "min": 14.75079134106636, "max": 14.910213840194046, "p95": 14.910213840194046 }, "residual_gib": { "n": 3, "median": 0.5830078125, "min": 0.5830078125, "max": 0.5830078125, "p95": 0.5830078125 } } }, "performance": [ { "stage": "baseline", "concurrency": 1, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 25.345039423871558, "min": 25.334343530277984, "max": 25.355735317465133, "p95": 25.355735317465133 }, "ttft_s": { "n": 16, "median": 0.09779117861762643, "min": 0.0945509746670723, "max": 0.12596794869750738, "p95": 0.12596794869750738 }, "tpot_s": { "n": 16, "median": 0.03895803926307268, "min": 0.03895247855964373, "max": 0.03896221161387333, "p95": 0.03896221161387333 }, "prompt_token_counts": [ 26 ] }, { "stage": "baseline", "concurrency": 4, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 92.63662314906486, "min": 92.56611738509014, "max": 92.7071289130396, "p95": 92.7071289130396 }, "ttft_s": { "n": 16, "median": 0.1998177724890411, "min": 0.18354888632893562, "max": 0.2409528335556388, "p95": 0.2409528335556388 }, "tpot_s": { "n": 16, "median": 0.04192155687226437, "min": 0.041619397164505766, "max": 0.04193431324523499, "p95": 0.04193431324523499 }, "prompt_token_counts": [ 26 ] }, { "stage": "after-wake", "concurrency": 1, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 25.350406377302598, "min": 25.343442873378603, "max": 25.357369881226596, "p95": 25.357369881226596 }, "ttft_s": { "n": 16, "median": 0.09780467301607132, "min": 0.09408002439886332, "max": 0.12442851718515158, "p95": 0.12442851718515158 }, "tpot_s": { "n": 16, "median": 0.038957973465499444, "min": 0.03895208367153885, "max": 0.03896150681534855, "p95": 0.03896150681534855 }, "prompt_token_counts": [ 26 ] }, { "stage": "after-wake", "concurrency": 4, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 92.6835135731726, "min": 92.6486328675364, "max": 92.7183942788088, "p95": 92.7183942788088 }, "ttft_s": { "n": 16, "median": 0.19824444176629186, "min": 0.18486266117542982, "max": 0.22702299524098635, "p95": 0.22702299524098635 }, "tpot_s": { "n": 16, "median": 0.041923967626415135, "min": 0.04161762111035623, "max": 0.04193394579081319, "p95": 0.04193394579081319 }, "prompt_token_counts": [ 26 ] } ] }, "B": { "name": "Qwen3-1.7B", "restart_initial": { "n": 1, "median": 84.059116830118, "min": 84.059116830118, "max": 84.059116830118, "p95": 84.059116830118 }, "restart_warm": { "n": 2, "median": 36.054832791443914, "min": 36.051797261461616, "max": 36.05786832142621, "p95": 36.05786832142621 }, "sleep": { "1": { "successful_cycles": 5, "sleep_s": { "n": 5, "median": 0.18378864973783493, "min": 0.1698524970561266, "max": 2.0744550125673413, "p95": 2.0744550125673413 }, "wake_api_s": { "n": 5, "median": 0.21793049853295088, "min": 0.21592434030026197, "max": 0.24329580832272768, "p95": 0.24329580832272768 }, "to_first_token_s": { "n": 5, "median": 0.23873963579535484, "min": 0.23714125622063875, "max": 0.2664509965106845, "p95": 0.2664509965106845 }, "residual_gib": { "n": 5, "median": 1.02734375, "min": 1.02734375, "max": 1.02734375, "p95": 1.02734375 } }, "2": { "successful_cycles": 3, "sleep_s": { "n": 3, "median": 0.10089693870395422, "min": 0.09138800203800201, "max": 0.10766292456537485, "p95": 0.10766292456537485 }, "wake_api_s": { "n": 3, "median": 2.1528645381331444, "min": 2.1499751107767224, "max": 2.1767852595075965, "p95": 2.1767852595075965 }, "to_first_token_s": { "n": 3, "median": 2.1740154651924968, "min": 2.1732194339856505, "max": 2.198166118003428, "p95": 2.198166118003428 }, "residual_gib": { "n": 3, "median": 1.02734375, "min": 1.02734375, "max": 1.02734375, "p95": 1.02734375 } } }, "performance": [ { "stage": "baseline", "concurrency": 1, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 175.85181172550742, "min": 175.83879256676747, "max": 175.86483088424737, "p95": 175.86483088424737 }, "ttft_s": { "n": 16, "median": 0.016774678137153387, "min": 0.015735766850411892, "max": 0.01965917367488146, "p95": 0.01965917367488146 }, "tpot_s": { "n": 16, "median": 0.005568990766210115, "min": 0.005562372168335389, "max": 0.005575573519165591, "p95": 0.005575573519165591 }, "prompt_token_counts": [ 25 ] }, { "stage": "baseline", "concurrency": 4, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 655.6072737556599, "min": 655.4528385403042, "max": 655.7617089710154, "p95": 655.7617089710154 }, "ttft_s": { "n": 16, "median": 0.033244193997234106, "min": 0.025845522060990334, "max": 0.03464682586491108, "p95": 0.03464682586491108 }, "tpot_s": { "n": 16, "median": 0.005866577075134347, "min": 0.005818273556807379, "max": 0.00589467237490838, "p95": 0.00589467237490838 }, "prompt_token_counts": [ 25 ] }, { "stage": "after-wake", "concurrency": 1, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 175.88590023167149, "min": 175.85870837207568, "max": 175.91309209126732, "p95": 175.91309209126732 }, "ttft_s": { "n": 16, "median": 0.017403739038854837, "min": 0.015094666741788387, "max": 0.0191877419129014, "p95": 0.0191877419129014 }, "tpot_s": { "n": 16, "median": 0.005567922656549009, "min": 0.005562727493563975, "max": 0.005573170325594155, "p95": 0.005573170325594155 }, "prompt_token_counts": [ 25 ] }, { "stage": "after-wake", "concurrency": 4, "rounds": 2, "planned": 16, "success": 16, "rounds_passed": 2, "output_tps": { "n": 2, "median": 657.5999214544754, "min": 655.205672287983, "max": 659.9941706209679, "p95": 659.9941706209679 }, "ttft_s": { "n": 16, "median": 0.029121145140379667, "min": 0.021723573096096516, "max": 0.03929914440959692, "p95": 0.03929914440959692 }, "tpot_s": { "n": 16, "median": 0.005863654808559287, "min": 0.005820392000334938, "max": 0.0058843813337913645, "p95": 0.0058843813337913645 }, "prompt_token_counts": [ 25 ] } ] } }, "requests": { "planned_performance": 128, "executed": 170, "success": 170, "failure": 0 }, "percentile_method": "nearest-rank; small sample", "restart_baseline": "same Pod, warm image and local process cache; not Pod cold start", "dra": "not executed; shared GPU within one container", "switch": { "initial_activation": 1, "handoffs": 19, "success": 19, "directions": { "A→B": { "n": 10, "to_first_token_s": { "n": 10, "median": 1.8169498867355287, "min": 1.7844120059162378, "max": 2.175088402815163, "p95": 2.175088402815163 }, "sleep_s": { "n": 10, "median": 1.3466137093491852, "min": 1.3364814911037683, "max": 1.4008429488167167, "p95": 1.4008429488167167 }, "wake_s": { "n": 10, "median": 0.2191113061271608, "min": 0.21772058587521315, "max": 0.5374723067507148, "p95": 0.5374723067507148 } }, "B→A": { "n": 9, "to_first_token_s": { "n": 9, "median": 2.2567768106237054, "min": 2.2256386652588844, "max": 2.2805262468755245, "p95": 2.2805262468755245 }, "sleep_s": { "n": 9, "median": 0.17176922969520092, "min": 0.17010218743234873, "max": 0.1939093265682459, "p95": 0.1939093265682459 }, "wake_s": { "n": 9, "median": 1.7524940110743046, "min": 1.7443944672122598, "max": 1.7784274388104677, "p95": 1.7784274388104677 } } } }, "memory": { "physical_peak_gib": { "n": 1574, "median": 32.5263671875, "min": 0.0, "max": 33.0888671875, "p95": 33.0546875 }, "rss_peak_gib": { "A": 51.36977005004883, "B": 11.280075073242188 }, "l1_release_fraction": { "A": { "n": 16, "median": 96.89198770976128, "min": 96.89198770976128, "max": 98.21220016171054, "p95": 98.21220016171054 }, "B": { "n": 16, "median": 87.63807285546416, "min": 87.62644083745002, "max": 87.63807285546416, "p95": 87.63807285546416 } } }, "windows": { "A-baseline": { "start": 1791463210.8144903, "end": 1791463317.7292283 }, "A-sleep-level1": { "start": 1791463313.7292283, "end": 1791463367.1176987 }, "A-after-wake": { "start": 1791463363.1176987, "end": 1791463470.003835 }, "B-baseline": { "start": 1791463680.8604085, "end": 1791463699.632152 }, "B-sleep-level1": { "start": 1791463695.632152, "end": 1791463719.8251686 }, "B-after-wake": { "start": 1791463715.8251686, "end": 1791463734.5852547 }, "AB-switch": { "start": 1791463747.3381171, "end": 1791463954.7337406 } } }