Sglang run 120b failed in blackwell gpu * 8

[gengchaogit]

NCCL_P2P_LEVEL=SYS CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7,8 python3 -m sglang.launch_server --model-path /root/windows/gpt-oss-120b --served-model-name sglang --tensor-parallel-size 8 --max-running-requests 20 --trust-remote-code --host 0.0.0.0 --port 8099 --context-length 131072 --dtype auto --quantization mxfp4 --disable-hybrid-swa-memory --mem-fraction-static 0.9 --sleep-on-idle

[gengchaogit]

RuntimeError: No supported CUDA architectures found for major versions [10].

[gengchaogit]

(sglang) root@epyc:~# NCCL_P2P_LEVEL=SYS CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7,8 python3 -m sglang.launch_server --model-path /root/windows/gpt-oss-120b --served-model-name sglang --tensor-parallel-size 8 --max-running-requests 20 --trust-remote-code --host 0.0.0.0 --port 8099 --context-length 131072 --dtype auto --quantization mxfp4 --disable-hybrid-swa-memory --mem-fraction-static 0.9 --sleep-on-idle
[2025-12-19 17:09:28] INFO server_args.py:1143: Enable FlashInfer AllReduce Fusion on sm100 for GptOssForCausalLM
[2025-12-19 17:09:28] WARNING server_args.py:1163: Detected SM100 and MXFP4 quantization format for GPT-OSS model, enabling FlashInfer MXFP4 MOE kernel.
[2025-12-19 17:09:28] INFO model_config.py:913: Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:28] WARNING model_config.py:752: mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:28] server_args=ServerArgs(model_path='/root/windows/gpt-oss-120b', tokenizer_path='/root/windows/gpt-oss-120b', tokenizer_mode='auto', tokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', rl_quant_profile=None, trust_remote_code=True, context_length=131072, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', host='0.0.0.0', port=8099, fastapi_root_path='', grpc_mode=False, skip_server_warmup=False, warmups=None, nccl_port=None, checkpoint_engine_wait_weights_before_ready=False, dtype='bfloat16', quantization='mxfp4', quantization_param_path=None, kv_cache_dtype='auto', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, mem_fraction_static=0.9, max_running_requests=20, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=2048, max_prefill_tokens=16384, schedule_policy='fcfs', enable_priority_scheduling=False, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, schedule_conservativeness=1.0, page_size=1, hybrid_kvcache_ratio=None, swa_full_tokens_ratio=0.8, disable_hybrid_swa_memory=True, radix_eviction_policy='lru', device='cuda', tp_size=8, pp_size=1, pp_max_micro_batch_size=None, stream_interval=1, stream_output=False, random_seed=394611153, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, watchdog_timeout=300, dist_timeout=None, download_dir=None, base_gpu_id=0, gpu_id_step=1, sleep_on_idle=True, mm_process_config={}, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, crash_dump_folder=None, show_time_cost=False, enable_metrics=False, enable_metrics_for_all_schedulers=False, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, collect_tokens_histogram=False, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_trace=False, otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, api_key=None, served_model_name='sglang', weight_version='default', chat_template=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, tool_call_parser=None, tool_server=None, sampling_defaults='model', dp_size=1, load_balance_method='round_robin', load_watch_interval=0.1, prefill_round_robin_balance=False, dist_init_addr=None, nnodes=1, node_rank=0, json_model_override_args='{}', preferred_sampling_params=None, enable_lora=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, attention_backend='triton', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', mm_attention_backend=None, fp8_gemm_runner_backend='auto', nsa_prefill_backend='flashmla_sparse', nsa_decode_backend='fa3', enable_flashinfer_autotune=False, speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_token_map=None, speculative_attention_mode='prefill', speculative_moe_runner_backend=None, speculative_moe_a2a_backend=None, speculative_ngram_min_match_window_size=1, speculative_ngram_max_match_window_size=12, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_branch_length=18, speculative_ngram_capacity=10000000, ep_size=1, moe_a2a_backend='none', moe_runner_backend='flashinfer_mxfp4', flashinfer_mxfp4_moe_precision='default', enable_flashinfer_allreduce_fusion=True, deepep_mode='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, mooncake_ib_device=None, max_mamba_cache_size=None, mamba_ssm_dtype='float32', mamba_full_memory_ratio=0.9, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='layer_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='best_effort', hicache_storage_backend_extra_config=None, enable_lmcache=False, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, enable_double_sparsity=False, ds_channel_config_path=None, ds_heavy_channel_num=32, ds_heavy_token_num=256, ds_heavy_channel_type='qk', ds_sparse_decode_threshold=4096, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', multi_item_scoring_delimiter=None, disable_radix_cache=False, cuda_graph_max_bs=8, cuda_graph_bs=[1, 2, 4, 8], disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, disable_outlines_disk_cache=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, disable_overlap_schedule=False, enable_mixed_chunk=False, enable_dp_attention=False, enable_dp_lm_head=False, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, enable_torch_compile=False, enable_piecewise_cuda_graph=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, piecewise_cuda_graph_max_tokens=4096, piecewise_cuda_graph_tokens=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 640, 768, 896, 1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920, 2048, 2176, 2304, 2432, 2560, 2688, 2816, 2944, 3072, 3200, 3328, 3456, 3584, 3712, 3840, 3968, 4096], piecewise_cuda_graph_compiler='eager', torchao_config='', enable_nan_detection=False, enable_p2p_check=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, num_continuous_decode_steps=1, delete_ckpt_after_loading=False, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, allow_auto_truncate=False, enable_custom_logit_processor=False, flashinfer_mla_disable_ragged=False, disable_shared_experts_fusion=False, disable_chunked_prefix_cache=False, disable_fast_image_processor=False, keep_mm_feature_on_device=False, enable_return_hidden_states=False, scheduler_recv_interval=1, numa_node=None, enable_deterministic_inference=False, rl_on_policy_target=None, enable_attn_tp_input_scattered=False, enable_nsa_prefill_context_parallel=False, enable_fused_qk_norm_rope=False, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, debug_tensor_dump_inject=False, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_decode_tp=None, disaggregation_decode_dp=None, disaggregation_prefill_pp=1, disaggregation_ib_device=None, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_polling_interval=1, custom_weight_loader=[], weight_loader_disable_mmap=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, mm_max_concurrent_calls=32, mm_per_request_timeout=10.0, enable_broadcast_mm_inputs_process=False, decrypted_config_file=None, decrypted_draft_config_file=None, mm_enable_dp_encoder=False, forward_hooks=None)
[2025-12-19 17:09:28] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:28] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:29] Using default HuggingFace chat template with detected content format: string
[2025-12-19 17:09:37 TP0] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:37 TP0] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:37 TP7] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:37 TP7] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:37 TP2] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:37 TP2] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP5] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP5] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP3] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP3] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP2] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP2] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP2] Init torch distributed begin.
[2025-12-19 17:09:38 TP7] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP7] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP7] Init torch distributed begin.
[2025-12-19 17:09:38 TP4] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP4] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP0] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP0] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP0] Init torch distributed begin.
[2025-12-19 17:09:38 TP1] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP1] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:38 TP6] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:38 TP6] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[W1219 17:09:38.873711360 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[2025-12-19 17:09:39 TP5] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:39 TP5] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:39 TP5] Init torch distributed begin.
[W1219 17:09:39.408549241 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[W1219 17:09:39.443721510 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[2025-12-19 17:09:39 TP4] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:39 TP4] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:39 TP4] Init torch distributed begin.
[2025-12-19 17:09:39 TP1] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:39 TP1] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:39 TP1] Init torch distributed begin.
[W1219 17:09:39.723646577 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[2025-12-19 17:09:39 TP6] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:39 TP6] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:39 TP6] Init torch distributed begin.
[2025-12-19 17:09:39 TP3] Downcasting torch.float32 to torch.bfloat16.
[2025-12-19 17:09:39 TP3] mxfp4 quantization is not fully optimized yet. The speed can be slower than non-quantized models.
[2025-12-19 17:09:39 TP3] Init torch distributed begin.
[W1219 17:09:40.003407310 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[W1219 17:09:40.135234979 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[W1219 17:09:40.250499165 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[W1219 17:09:40.277785106 socket.cpp:767] [c10d] The client socket cannot be initialized to connect to [::ffff:127.0.0.1]:8880 (errno: 97 - Address family not supported by protocol).
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[2025-12-19 17:09:40 TP0] sglang is using nccl==2.27.5
[2025-12-19 17:09:41 TP1] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP2] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP0] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP6] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP4] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP7] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP3] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[2025-12-19 17:09:41 TP5] Custom allreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly.
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
[2025-12-19 17:09:41 TP7] Init torch distributed ends. mem usage=0.21 GB
[2025-12-19 17:09:41 TP6] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP0] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP5] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP4] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP1] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP3] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP2] Init torch distributed ends. mem usage=0.23 GB
[2025-12-19 17:09:41 TP5] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP4] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP7] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP1] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP6] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP0] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP3] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:41 TP2] Ignore import error when loading sglang.srt.models.mindspore: name 'ms' is not defined
[2025-12-19 17:09:42 TP7] Load weight begin. avail mem=15.11 GB
[2025-12-19 17:09:42 TP6] Load weight begin. avail mem=15.09 GB
[2025-12-19 17:09:42 TP0] Load weight begin. avail mem=15.09 GB
[2025-12-19 17:09:42 TP5] Load weight begin. avail mem=15.09 GB
[2025-12-19 17:09:42 TP3] Load weight begin. avail mem=15.09 GB
[2025-12-19 17:09:42 TP2] Load weight begin. avail mem=15.09 GB
[2025-12-19 17:09:42 TP1] Load weight begin. avail mem=15.09 GB
torch_dtype is deprecated! Use dtype instead!
[2025-12-19 17:09:42 TP4] Load weight begin. avail mem=15.09 GB
torch_dtype is deprecated! Use dtype instead!
torch_dtype is deprecated! Use dtype instead!
torch_dtype is deprecated! Use dtype instead!
torch_dtype is deprecated! Use dtype instead!
torch_dtype is deprecated! Use dtype instead!
torch_dtype is deprecated! Use dtype instead!
torch_dtype is deprecated! Use dtype instead!
Loading safetensors checkpoint shards: 0% Completed | 0/15 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 53% Completed | 8/15 [00:00<00:00, 77.73it/s]
Loading safetensors checkpoint shards: 100% Completed | 15/15 [00:00<00:00, 74.13it/s]

[2025-12-19 17:11:36 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.0.mlp.experts), it might take a while...
[2025-12-19 17:11:40 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.1.mlp.experts), it might take a while...
[2025-12-19 17:11:43 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.2.mlp.experts), it might take a while...
[2025-12-19 17:11:45 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.3.mlp.experts), it might take a while...
[2025-12-19 17:11:48 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.4.mlp.experts), it might take a while...
[2025-12-19 17:11:51 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.5.mlp.experts), it might take a while...
[2025-12-19 17:11:54 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.6.mlp.experts), it might take a while...
[2025-12-19 17:11:57 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.7.mlp.experts), it might take a while...
[2025-12-19 17:11:59 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.8.mlp.experts), it might take a while...
[2025-12-19 17:12:02 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.9.mlp.experts), it might take a while...
[2025-12-19 17:12:05 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.10.mlp.experts), it might take a while...
[2025-12-19 17:12:08 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.11.mlp.experts), it might take a while...
[2025-12-19 17:12:11 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.12.mlp.experts), it might take a while...
[2025-12-19 17:12:14 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.13.mlp.experts), it might take a while...
[2025-12-19 17:12:16 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.14.mlp.experts), it might take a while...
[2025-12-19 17:12:19 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.15.mlp.experts), it might take a while...
[2025-12-19 17:12:22 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.16.mlp.experts), it might take a while...
[2025-12-19 17:12:25 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.17.mlp.experts), it might take a while...
[2025-12-19 17:12:28 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.18.mlp.experts), it might take a while...
[2025-12-19 17:12:31 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.19.mlp.experts), it might take a while...
[2025-12-19 17:12:33 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.20.mlp.experts), it might take a while...
[2025-12-19 17:12:36 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.21.mlp.experts), it might take a while...
[2025-12-19 17:12:39 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.22.mlp.experts), it might take a while...
[2025-12-19 17:12:42 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.23.mlp.experts), it might take a while...
[2025-12-19 17:12:45 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.24.mlp.experts), it might take a while...
[2025-12-19 17:12:48 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.25.mlp.experts), it might take a while...
[2025-12-19 17:12:50 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.26.mlp.experts), it might take a while...
[2025-12-19 17:12:53 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.27.mlp.experts), it might take a while...
[2025-12-19 17:12:56 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.28.mlp.experts), it might take a while...
[2025-12-19 17:12:59 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.29.mlp.experts), it might take a while...
[2025-12-19 17:13:02 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.30.mlp.experts), it might take a while...
[2025-12-19 17:13:04 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.31.mlp.experts), it might take a while...
[2025-12-19 17:13:07 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.32.mlp.experts), it might take a while...
[2025-12-19 17:13:10 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.33.mlp.experts), it might take a while...
[2025-12-19 17:13:13 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.34.mlp.experts), it might take a while...
[2025-12-19 17:13:16 TP0] Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: model.layers.35.mlp.experts), it might take a while...
[2025-12-19 17:13:19 TP0] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.17 GB, mem usage=9.93 GB.
[2025-12-19 17:13:20 TP4] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.16 GB, mem usage=9.93 GB.
[2025-12-19 17:13:20 TP1] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.16 GB, mem usage=9.93 GB.
[2025-12-19 17:13:22 TP2] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.16 GB, mem usage=9.93 GB.
[2025-12-19 17:13:22 TP3] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.16 GB, mem usage=9.93 GB.
[2025-12-19 17:13:22 TP5] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.16 GB, mem usage=9.93 GB.
[2025-12-19 17:13:23 TP6] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.16 GB, mem usage=9.93 GB.
[2025-12-19 17:13:23 TP7] Load weight end. type=GptOssForCausalLM, dtype=torch.bfloat16, avail mem=5.18 GB, mem usage=9.93 GB.
[2025-12-19 17:13:23 TP0] Using KV cache dtype: torch.bfloat16
[2025-12-19 17:13:23 TP6] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP2] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP4] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP7] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP1] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP0] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP5] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP3] KV Cache is allocated. #tokens: 425405, K size: 1.83 GB, V size: 1.83 GB
[2025-12-19 17:13:23 TP2] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:23 TP0] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:23 TP6] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:23 TP4] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:23 TP1] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:23 TP5] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:23 TP7] Memory pool end. avail mem=2.39 GB
[2025-12-19 17:13:23 TP3] Memory pool end. avail mem=2.37 GB
[2025-12-19 17:13:24 TP0] Capture cuda graph begin. This can take up to several minutes. avail mem=2.37 GB
[2025-12-19 17:13:24 TP0] Capture cuda graph bs [1, 2, 4, 8]
[2025-12-19 17:13:25 TP7] Capture cuda graph begin. This can take up to several minutes. avail mem=2.38 GB
[2025-12-19 17:13:25 TP6] Capture cuda graph begin. This can take up to several minutes. avail mem=2.36 GB
[2025-12-19 17:13:25 TP2] Capture cuda graph begin. This can take up to several minutes. avail mem=2.36 GB
[2025-12-19 17:13:25 TP3] Capture cuda graph begin. This can take up to several minutes. avail mem=2.36 GB
[2025-12-19 17:13:25 TP4] Capture cuda graph begin. This can take up to several minutes. avail mem=2.36 GB
[2025-12-19 17:13:25 TP5] Capture cuda graph begin. This can take up to several minutes. avail mem=2.36 GB
[2025-12-19 17:13:25 TP1] Capture cuda graph begin. This can take up to several minutes. avail mem=2.36 GB
Capturing batches (bs=8 avail_mem=2.36 GB): 0%| | 0/4 [00:03<?, ?it/s]
[2025-12-19 17:13:39 TP0] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[2025-12-19 17:13:39 TP1] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[2025-12-19 17:13:39 TP5] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[2025-12-19 17:13:39 TP4] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[2025-12-19 17:13:39] Received sigquit from a child process. It usually means the child failed.
[2025-12-19 17:13:39 TP7] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[2025-12-19 17:13:39] Received sigquit from a child process. It usually means the child failed.
[2025-12-19 17:13:39 TP3] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[2025-12-19 17:13:39 TP2] Scheduler hit an exception: Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 352, in init
self.capture()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 507, in capture
_capture_one_stream()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 494, in _capture_one_stream
) = self.capture_one_batch_size(bs, forward, stream_idx)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 698, in capture_one_batch_size
run_once()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 685, in run_once
logits_output_or_pp_proxy_tensors = forward(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 611, in forward
hidden_states = self.model(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 542, in forward
hidden_states, residual = layer(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 460, in forward
hidden_states = self.mlp(hidden_states, forward_batch, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 162, in forward
return self.forward_normal(hidden_states, should_allreduce_fusion)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/models/gpt_oss.py", line 182, in forward_normal
final_hidden_states = self.experts(hidden_states, topk_output)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 896, in forward
combine_input = self.run_moe_core(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/moe/fused_moe_triton/layer.py", line 917, in run_moe_core
return self.quant_method.apply(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/layers/quantization/mxfp4.py", line 660, in apply
trtllm_gen_output = trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 2207, in trtllm_fp4_block_scale_moe
return get_trtllm_moe_sm100_module().trtllm_fp4_block_scale_moe(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/fused_moe/core.py", line 912, in get_trtllm_moe_sm100_module
module = gen_trtllm_gen_fused_moe_sm100_module()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/jit/fused_moe.py", line 232, in gen_trtllm_gen_fused_moe_sm100_module
nvcc_flags = current_compilation_context.get_nvcc_flags_list(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/flashinfer/compilation_context.py", line 62, in get_nvcc_flags_list
raise RuntimeError(
RuntimeError: No supported CUDA architectures found for major versions [10].

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2680, in run_scheduler_process
scheduler = Scheduler(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 320, in init
self.tp_worker = TpModelWorker(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/managers/tp_worker.py", line 248, in init
self._model_runner = ModelRunner(
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 359, in init
self.initialize(min_per_gpu_memory)
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 511, in initialize
self.init_device_graphs()
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/model_runner.py", line 2448, in init_device_graphs
self.graph_runner = graph_runnersself.device
File "/root/miniconda3/envs/sglang/lib/python3.10/site-packages/sglang/srt/model_executor/cuda_graph_runner.py", line 354, in init
raise Exception(
Exception: Capture cuda graph failed: No supported CUDA architectures found for major versions [10].
Possible solutions:

1. set --mem-fraction-static to a smaller value (e.g., 0.8 or 0.7)
2. set --cuda-graph-max-bs to a smaller value (e.g., 16)
3. disable torch compile by not using --enable-torch-compile
4. disable CUDA graph by --disable-cuda-graph. (Not recommended. Huge performance loss)
   Open an issue on GitHub https://github.com/sgl-project/sglang/issues/new/choose

[gengchaogit]

--disable-cuda-graph can be startup,but first request will crash