From 609d1f31c09a888b8627ecd8c362173fd455751a Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 2 Apr 2026 05:26:28 +0000 Subject: [PATCH 1/5] SGL qwen3.5 fp8 mi355x performance update Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 10 +++++++--- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 19 ++++++++++++++++--- perf-changelog.yaml | 7 +++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 72ef89c09..5756f60bf 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -186,7 +186,7 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 + image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260330 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x @@ -197,11 +197,15 @@ qwen3.5-fp8-mi355x-sglang: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 2, conc-start: 8, conc-end: 256 } + - { tp: 4, conc-start: 4, conc-end: 32 } + - { tp: 8, conc-start: 4, conc-end: 4 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 2, conc-start: 8, conc-end: 256 } + - { tp: 4, conc-start: 4, conc-end: 16 } + - { tp: 8, conc-start: 4, conc-end: 4 } qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.9-rocm720-mi30x diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index 701695def..a4b27ea74 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -19,6 +19,8 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} +CHUNK_SIZE=8192 EVAL_CONTEXT_ARGS="" if [ "${EVAL_ONLY}" = "true" ]; then @@ -28,20 +30,30 @@ fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor -python3 -m sglang.launch_server \ +sglang serve \ --attention-backend triton \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ - --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & + --mem-fraction-static $MEM_FRAC_STATIC \ + --kv-cache-dtype fp8_e4m3 \ + --cuda-graph-max-bs $CONC \ + --max-running-requests $CONC \ + --chunked-prefill-size $CHUNK_SIZE \ + --max-prefill-tokens $CHUNK_SIZE \ + --disable-radix-cache \ + --num-continuous-decode-steps 2 \ + $EVAL_CONTEXT_ARGS \ + > $SERVER_LOG 2>&1 & SERVER_PID=$! # Wait for server to be ready wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" +export PYTHONDONTWRITEBYTECODE=1 run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ @@ -52,7 +64,8 @@ run_benchmark_serving \ --num-prompts "$((CONC * 10))" \ --max-concurrency "$CONC" \ --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ + --result-dir /workspace/ \ + --trust-remote-code # After throughput, run evaluation only if RUN_EVAL is true if [ "${RUN_EVAL}" = "true" ]; then diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a82882f61..d10bfcb77 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1235,3 +1235,10 @@ - "New model support on ATOM framework" - "Kimi-K2.5 FP4, and MiniMax-M2.5 FP8 configs added for MI355X ATOM" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/963 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + description: + - "Qwen3.5 fp8 mi355x performance update" + - "Relevant Issue: https://github.com/sgl-project/sglang/issues/19633" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/9xx From 7685bb076598433b7d69077444d863548e78ea6a Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 2 Apr 2026 05:29:20 +0000 Subject: [PATCH 2/5] SGL qwen3.5 fp8 mi355x performance update Signed-off-by: seungrokj --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d10bfcb77..5723d9801 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1241,4 +1241,4 @@ description: - "Qwen3.5 fp8 mi355x performance update" - "Relevant Issue: https://github.com/sgl-project/sglang/issues/19633" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/9xx + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/995 From 4f870c835609d47f223fc7c00a5270a9443742ac Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 2 Apr 2026 06:27:13 +0000 Subject: [PATCH 3/5] SGL qwen3.5 fp8 mi355x performance update Signed-off-by: seungrokj --- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 benchmarks/single_node/qwen3.5_fp8_mi355x.sh diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh old mode 100644 new mode 100755 index a4b27ea74..d71d778ff --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -20,7 +20,7 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} -CHUNK_SIZE=8192 +CHUNK_SIZE=32768 EVAL_CONTEXT_ARGS="" if [ "${EVAL_ONLY}" = "true" ]; then @@ -30,6 +30,7 @@ fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor +set -x sglang serve \ --attention-backend triton \ --model-path $MODEL \ From cee748e47541be681275dcfa11fb8d990e8a60aa Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 2 Apr 2026 06:28:27 +0000 Subject: [PATCH 4/5] SGL qwen3.5 fp8 mi355x performance update Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 5756f60bf..584fbcab9 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -197,13 +197,13 @@ qwen3.5-fp8-mi355x-sglang: - isl: 1024 osl: 1024 search-space: - - { tp: 2, conc-start: 8, conc-end: 256 } + - { tp: 2, conc-start: 8, conc-end: 128 } - { tp: 4, conc-start: 4, conc-end: 32 } - { tp: 8, conc-start: 4, conc-end: 4 } - isl: 8192 osl: 1024 search-space: - - { tp: 2, conc-start: 8, conc-end: 256 } + - { tp: 2, conc-start: 8, conc-end: 128 } - { tp: 4, conc-start: 4, conc-end: 16 } - { tp: 8, conc-start: 4, conc-end: 4 } From 4e42d21f8cdf7ca4d09526463efb0745b95a8575 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 2 Apr 2026 08:19:53 +0000 Subject: [PATCH 5/5] SGL qwen3.5 fp8 mi355x performance update Signed-off-by: seungrokj --- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index d71d778ff..6f5565784 100755 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -23,10 +23,10 @@ MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} CHUNK_SIZE=32768 EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi +#if [ "${EVAL_ONLY}" = "true" ]; then +# setup_eval_context +# EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" +#fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor