diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 225992f4d..58469dcc4 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -2695,7 +2695,7 @@ minimaxm3-fp4-mi355x-vllm-mtp: # https://github.com/ROCm/ATOM/blob/5d42d49f9e4292e5b61475917e92e7ec1b1dacb7/recipes/MiniMax-M3.md # block size 128 is mandatory for MSA. TP4 on a single gfx950 node, per the recipe. minimaxm3-fp4-mi355x-atom: - image: rocm/atom-dev:MiniMax-M3-20260623 + image: rocm/atom-dev:MiniMax-M3-20260630 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: mi355x @@ -2714,7 +2714,7 @@ minimaxm3-fp4-mi355x-atom: - { tp: 4, conc-start: 1, conc-end: 256 } minimaxm3-fp4-mi355x-atom-mtp: - image: rocm/atom-dev:MiniMax-M3-20260623 + image: rocm/atom-dev:MiniMax-M3-20260630 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: mi355x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh index a0907f18c..3cfce7937 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh @@ -31,6 +31,7 @@ if [ "$DP_ATTENTION" = "true" ]; then fi SPEC_ARGS=() +OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}' --hf-overrides '{"use_index_cache": true, "index_topk_freq": 4}') # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -38,8 +39,7 @@ MEM_FRAC_STATIC=0.8 set -x export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0 -export ATOM_M3_SPARSE_USE_ASM_PA=1 +export ATOM_FORCE_ATTN_TRITON=1 export MAX_MODEL_LEN=32768 export MAX_NUM_BATCHED_TOKENS=32768 export MAX_NUM_SEQS=256 @@ -48,6 +48,7 @@ python3 -m atom.entrypoints.openai_server \ --server-port $PORT \ "${PARALLEL_ARGS[@]}" \ "${SPEC_ARGS[@]}" \ + "${OPT_ARGS[@]}" \ --block-size 128 \ --gpu-memory-utilization $MEM_FRAC_STATIC \ --max-model-len $MAX_MODEL_LEN \ diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh index fe71c47a0..47de985be 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh @@ -31,6 +31,7 @@ if [ "$DP_ATTENTION" = "true" ]; then fi SPEC_ARGS=(--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens 3 ) +OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}' --hf-overrides '{"use_index_cache": true, "index_topk_freq": 4}') # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -38,8 +39,7 @@ MEM_FRAC_STATIC=0.8 set -x export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0 -export ATOM_M3_SPARSE_USE_ASM_PA=1 +export ATOM_FORCE_ATTN_TRITON=1 export MAX_MODEL_LEN=32768 export MAX_NUM_BATCHED_TOKENS=32768 export MAX_NUM_SEQS=256 @@ -48,6 +48,7 @@ python3 -m atom.entrypoints.openai_server \ --server-port $PORT \ "${PARALLEL_ARGS[@]}" \ "${SPEC_ARGS[@]}" \ + "${OPT_ARGS[@]}" \ --block-size 128 \ --gpu-memory-utilization $MEM_FRAC_STATIC \ --max-model-len $MAX_MODEL_LEN \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 66577badb..06586a614 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4352,6 +4352,15 @@ - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1966 +- config-keys: + - minimaxm3-fp4-mi355x-atom + - minimaxm3-fp4-mi355x-atom-mtp + description: + - "Bump image to rocm/atom-dev:MiniMax-M3-20260630 for both fp4 atom entries" + - "Add OPT_ARGS: pass --online_quant_config '{\"global_quant_config\": \"ptpc_fp8\", \"exclude_layer\": [\"lm_head\", \"model.embed_tokens\", \"vision_tower\", \"multi_modal_projector\", \"patch_merge_mlp\", \"*block_sparse_moe\"]}' and --hf-overrides '{\"use_index_cache\": true, \"index_topk_freq\": 4}' to both scripts" + - "Replace AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0 and ATOM_M3_SPARSE_USE_ASM_PA=1 with ATOM_FORCE_ATTN_TRITON=1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1967 + - config-keys: - minimaxm3-fp8-mi355x-atom - minimaxm3-fp8-mi355x-atom-mtp