diff --git a/docs/hyperparameter_tuning.md b/docs/hyperparameter_tuning.md index b0aa15e8ad..5013a80aba 100644 --- a/docs/hyperparameter_tuning.md +++ b/docs/hyperparameter_tuning.md @@ -11,12 +11,12 @@ When the server is running at full load, look for the following in the log: ### Tune Your Request Submission Speed `#queue-req` indicates the number of requests in the queue. If you frequently see `#queue-req == 0`, it suggests you are bottlenecked by the request submission speed. A healthy range for `#queue-req` is `50 - 500`. -On the other hand, do not make `#queue-req` too large because it will also increase the scheduling overhead on the server. +On the other hand, do not make `#queue-req` too large because it will also increase the scheduling overhead on the server, especially when using the default longest-prefix-match schedule policy (`--schedule-policy lpm`). ### Tune `--schedule-conservativeness` `token usage` indicates the KV cache memory utilization of the server. `token usage > 0.9` means good utilization. If you frequently see `token usage < 0.9` and `#queue-req > 0`, it means the server is too conservative about taking in new requests. You can decrease `--schedule-conservativeness` to a value like 0.3. -The case of serving being too conservative can happen when users send many requests with a large `max_new_tokens` but the requests stop very early due to EOS or stop strings. +The case of server being too conservative can happen when users send many requests with a large `max_new_tokens` but the requests stop very early due to EOS or stop strings. On the other hand, if you see `token usage` very high and you frequently see warnings like `decode out of memory happened, #retracted_reqs: 1, #new_token_ratio: 0.9998 -> 1.0000`, you can increase `--schedule-conservativeness` to a value like 1.3. @@ -25,17 +25,17 @@ If you see `decode out of memory happened` occasionally but not frequently, it i ### Tune `--dp-size` and `--tp-size` Data parallelism is better for throughput. When there is enough GPU memory, always favor data parallelism for throughput. -### Avoid out-of-memory by tuning `--chunked-prefill-size`, `--mem-fraction-static`, `--max-running-requests` +### Avoid out-of-memory by Tuning `--chunked-prefill-size`, `--mem-fraction-static`, `--max-running-requests` If you see out of memory (OOM) errors, you can decrease these parameters. If OOM happens during prefill, try to decrease `--chunked-prefill-size` to `4096` or `2048`. If OOM happens during decoding, try to decrease `--max-running-requests`. You can also try to decrease `--mem-fraction-static`, which reduces the memory usage of the KV cache memory pool and helps both prefill and decoding. -### Try advanced options +### Try Advanced Options - To enable the experimental overlapped scheduler, add `--enable-overlap-scheduler`. It overlaps CPU scheduler with GPU computation and can accelerate almost all workloads. This does not work for constrained decoding currenly. - To enable torch.compile acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. This does not work for FP8 currenly. -### (Minor) Tune `--schedule-policy` -If you have many shared prefixes, use the default `--schedule-policy lpm`. `lpm` stands for longest prefix match. +### Tune `--schedule-policy` +If the workload has many shared prefixes, use the default `--schedule-policy lpm`. `lpm` stands for longest prefix match. When you have no shared prefixes at all or you always send the requests with the shared prefixes together, -you can try `--schedule-policy fcfs`. `fcfs` stands for first come first serve. +you can try `--schedule-policy fcfs`. `fcfs` stands for first come first serve. `fcfs` has a lower scheduling overhead. diff --git a/python/sglang/global_config.py b/python/sglang/global_config.py index 5e7290edc5..d557e6a6e6 100644 --- a/python/sglang/global_config.py +++ b/python/sglang/global_config.py @@ -14,9 +14,15 @@ def __init__(self): self.default_backend = None # Runtime constants: New generation token ratio estimation - self.init_new_token_ratio = 0.7 - self.base_min_new_token_ratio = 0.1 - self.new_token_ratio_decay = 0.001 + self.default_init_new_token_ratio = float( + os.environ.get("SGLANG_INIT_NEW_TOKEN_RATIO", 0.7) + ) + self.default_min_new_token_ratio_factor = float( + os.environ.get("SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR", 0.14) + ) + self.default_new_token_ratio_decay_steps = float( + os.environ.get("SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS", 600) + ) # Runtime constants: others self.retract_decode_steps = 20 diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index c3f679198e..4677568c40 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -254,13 +254,22 @@ def __init__( assert ( server_args.schedule_conservativeness >= 0 ), "Invalid schedule_conservativeness" - self.min_new_token_ratio = min( - global_config.base_min_new_token_ratio + + self.init_new_token_ratio = min( + global_config.default_init_new_token_ratio * server_args.schedule_conservativeness, 1.0, ) - self.new_token_ratio = self.min_new_token_ratio - self.new_token_ratio_decay = global_config.new_token_ratio_decay + self.min_new_token_ratio = min( + self.init_new_token_ratio + * global_config.default_min_new_token_ratio_factor, + 1.0, + ) + self.new_token_ratio_decay = ( + self.init_new_token_ratio - self.min_new_token_ratio + ) / global_config.default_new_token_ratio_decay_steps + self.new_token_ratio = self.init_new_token_ratio + self.batch_is_full = False # Init profiler @@ -307,7 +316,7 @@ def event_loop_normal(self): self.process_batch_result(batch, result) else: self.check_memory() - self.new_token_ratio = global_config.init_new_token_ratio + self.new_token_ratio = self.init_new_token_ratio self.last_batch = batch @@ -334,7 +343,7 @@ def event_loop_overlap(self): self.process_batch_result(tmp_batch, tmp_result) elif batch is None: self.check_memory() - self.new_token_ratio = global_config.init_new_token_ratio + self.new_token_ratio = self.init_new_token_ratio self.last_batch = batch diff --git a/python/sglang/srt/model_executor/cuda_graph_runner.py b/python/sglang/srt/model_executor/cuda_graph_runner.py index 05e2812ebf..e91fbac652 100644 --- a/python/sglang/srt/model_executor/cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/cuda_graph_runner.py @@ -121,13 +121,13 @@ def __init__(self, model_runner: "ModelRunner"): bs for bs in self.capture_bs if bs <= model_runner.req_to_token_pool.size - and bs <= model_runner.server_args.max_cuda_graph_bs + and bs <= model_runner.server_args.cuda_graph_max_bs ] self.compile_bs = ( [ bs for bs in self.capture_bs - if bs <= self.model_runner.server_args.max_torch_compile_bs + if bs <= self.model_runner.server_args.torch_compile_max_bs ] if self.use_torch_compile else [] diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index df632ac782..753debb66d 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -119,8 +119,8 @@ class ServerArgs: enable_overlap_schedule: bool = False enable_mixed_chunk: bool = False enable_torch_compile: bool = False - max_torch_compile_bs: int = 32 - max_cuda_graph_bs: int = 160 + torch_compile_max_bs: int = 32 + cuda_graph_max_bs: int = 160 torchao_config: str = "" enable_p2p_check: bool = False triton_attention_reduce_in_fp32: bool = False @@ -620,15 +620,15 @@ def add_cli_args(parser: argparse.ArgumentParser): help="Optimize the model with torch.compile. Experimental feature.", ) parser.add_argument( - "--max-torch-compile-bs", + "--torch-compile-max-bs", type=int, - default=ServerArgs.max_torch_compile_bs, + default=ServerArgs.torch_compile_max_bs, help="Set the maximum batch size when using torch compile.", ) parser.add_argument( - "--max-cuda-graph-bs", + "--cuda-graph-max-bs", type=int, - default=ServerArgs.max_cuda_graph_bs, + default=ServerArgs.cuda_graph_max_bs, help="Set the maximum batch size for cuda graph.", ) parser.add_argument(