{ "schema": "stable-upstream-long-stability-benchmark-v1", "date": "2026-08-10", "topology": { "endpoint_picker_replicas": 1, "model_replicas": 1, "gpus_per_model_replica": 1 }, "hardware": { "gpu_per_model_replica": "NVIDIA H100" }, "model_service": { "model": "GPT-OSS 20B", "runtime": "Red Hat AI Inference Server vLLM", "tensor_parallel_size": 1, "max_model_len": 32768, "max_num_batched_tokens": 8192, "max_num_seqs": 128, "gpu_memory_utilization": 0.9, "prefix_cache": "disabled" }, "endpoint_picker": { "version": "llm-d Endpoint Picker v0.9.0", "flow_control_gate": "enabled", "detector": "request-concurrency", "max_concurrency": 128, "headroom": 0.1, "picker": "random", "priority_bands": [ 100, 50, 0, -10 ], "fairness": "round-robin within each priority band" }, "traffic": { "schema_version": 1, "description": "Thirty-minute cache-off mixed workload with two matched surges.", "scenarios": [ { "name": "mixed-workload-long-stability", "duration_s": 1800, "analysis_windows": [ { "name": "baseline", "start_s": 120, "end_s": 240 }, { "name": "surge-1", "start_s": 420, "end_s": 660 }, { "name": "recovery-1", "start_s": 780, "end_s": 870 }, { "name": "surge-2", "start_s": 1020, "end_s": 1260 }, { "name": "recovery-2", "start_s": 1380, "end_s": 1470 }, { "name": "final", "start_s": 1680, "end_s": 1770 } ], "tenants": [ { "fairness_id": "realtime-chat", "priority": 100, "input_tokens": 1024, "output_tokens": 128, "phases": [ { "start_s": 0, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 4, "rate_amplitude": 1, "period_s": 37, "phase_offset": 0.0, "rate_noise": 0.2, "rate_spike_probability": 0.02 }, { "start_s": 300, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 10, "rate_amplitude": 2, "period_s": 31, "phase_offset": 0.0, "rate_noise": 0.5, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 720, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 3, "rate_amplitude": 0.7, "period_s": 23, "phase_offset": 0.0, "rate_noise": 0.15 }, { "start_s": 900, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 10, "rate_amplitude": 2, "period_s": 31, "phase_offset": 0.0, "rate_noise": 0.5, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 1320, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 3, "rate_amplitude": 0.7, "period_s": 23, "phase_offset": 0.0, "rate_noise": 0.15 }, { "start_s": 1500, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 4, "rate_amplitude": 1, "period_s": 37, "phase_offset": 0.0, "rate_noise": 0.2 } ] }, { "fairness_id": "agentic", "priority": 50, "input_tokens": 4096, "output_tokens": 512, "phases": [ { "start_s": 0, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.5, "rate_amplitude": 0.12, "period_s": 41, "phase_offset": 0.2, "rate_noise": 0.025, "rate_spike_probability": 0.02 }, { "start_s": 300, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 1.8, "rate_amplitude": 0.4, "period_s": 29, "phase_offset": 0.2, "rate_noise": 0.09, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 720, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.3, "rate_amplitude": 0.08, "period_s": 19, "phase_offset": 0.2, "rate_noise": 0.015 }, { "start_s": 900, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 1.8, "rate_amplitude": 0.4, "period_s": 29, "phase_offset": 0.2, "rate_noise": 0.09, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 1320, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.3, "rate_amplitude": 0.08, "period_s": 19, "phase_offset": 0.2, "rate_noise": 0.015 }, { "start_s": 1500, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.5, "rate_amplitude": 0.12, "period_s": 41, "phase_offset": 0.2, "rate_noise": 0.025 } ] }, { "fairness_id": "standard-long-context", "priority": 0, "input_tokens": 20000, "output_tokens": 128, "phases": [ { "start_s": 0, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.05, "rate_amplitude": 0.015, "period_s": 43, "phase_offset": 0.4, "rate_noise": 0.0025, "rate_spike_probability": 0.02 }, { "start_s": 300, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.35, "rate_amplitude": 0.08, "period_s": 33, "phase_offset": 0.4, "rate_noise": 0.0175, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 720, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.05, "rate_amplitude": 0.015, "period_s": 21, "phase_offset": 0.4, "rate_noise": 0.0025 }, { "start_s": 900, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.35, "rate_amplitude": 0.08, "period_s": 33, "phase_offset": 0.4, "rate_noise": 0.0175, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 1320, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.05, "rate_amplitude": 0.015, "period_s": 21, "phase_offset": 0.4, "rate_noise": 0.0025 }, { "start_s": 1500, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.05, "rate_amplitude": 0.015, "period_s": 43, "phase_offset": 0.4, "rate_noise": 0.0025 } ] }, { "fairness_id": "batch-long-context", "priority": -10, "input_tokens": 20000, "output_tokens": 128, "phases": [ { "start_s": 0, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.3, "rate_amplitude": 0.08, "period_s": 47, "phase_offset": 0.6, "rate_noise": 0.015, "rate_spike_probability": 0.02 }, { "start_s": 300, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.6, "rate_amplitude": 0.14, "period_s": 35, "phase_offset": 0.6, "rate_noise": 0.03, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 720, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.1, "rate_amplitude": 0.03, "period_s": 25, "phase_offset": 0.6, "rate_noise": 0.005 }, { "start_s": 900, "duration_s": 420, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.6, "rate_amplitude": 0.14, "period_s": 35, "phase_offset": 0.6, "rate_noise": 0.03, "rate_spike_probability": 0.02, "ramp_s": 15 }, { "start_s": 1320, "duration_s": 180, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.1, "rate_amplitude": 0.03, "period_s": 25, "phase_offset": 0.6, "rate_noise": 0.005 }, { "start_s": 1500, "duration_s": 300, "rate_pattern": "noisy_sinusoidal", "rate_center": 0.3, "rate_amplitude": 0.08, "period_s": 47, "phase_offset": 0.6, "rate_noise": 0.015 } ] } ] } ] }, "execution": { "arrival": "open-loop Poisson replay with noisy sinusoidal phases", "duration_seconds": 1800, "production_runs": 1, "load_generator": "GuideLLM", "cache_mode": "off" }, "decision_limits": { "all_requests_complete": true, "flow_control_engages_in_both_surges": true, "queue_drains_after_each_surge": true, "maximum_second_surge_p95_regression_percent": 20, "premium_surge_p95_ttft_guardrail_ms": 3000, "endpoint_picker_restarts": 0 }, "runner": { "scenario_source": "pipeline/benchmark.py", "sha256": "3811ec26c46bf3a26fa643698ec54bf569bb4bc99c3ea22ca18f805cb077b8e0", "traffic_driver": "GuideLLM", "guidellm_version": "0.7.0", "trace_compiler": "pipeline/guidellm_trace.py", "launcher": "pipeline/run_guidellm_scenario.py", "scenario_file": "scenario.json" } }