DeepSeek-V3.2-Exp

.gitignore

@@ -42,12 +42,21 @@ lcov-report/
 
 # Build Artifacts
 
+llama.pc
+DartConfiguration.tcl
+CMakeCache.txt
 tags
 .build/
 build*
 release
 debug
 !build-info.cmake
+tests/CTestTestfile.cmake
+tests/cmake_install.cmake
+**/*.cmake
+tests/*.cmake
+**/Makefile
+**/CMakeFiles
 !build-info.cpp.in
 !build-info.sh
 !build.zig

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ggml/src/ggml-cuda/vendors/cutlass"]
+	path = ggml/src/ggml-cuda/vendors/cutlass
+	url = https://github.com/NVIDIA/cutlass

convert_hf_to_gguf.py

@@ -852,6 +852,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
         if chkhsh == "b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5":
             # ref: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
             res = "deepseek-r1-qwen"
+        if chkhsh == "877081d19cf6996e2c4ff0e1236341e9b7bde288f5311a56a937f0afbbb3aeb5":
+            # ref: https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp
+            res = "deepseek-v3.2"
         if chkhsh == "ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352e":
             # ref: https://huggingface.co/Xenova/gpt-4o
             res = "gpt-4o"
@@ -6503,6 +6506,193 @@ def prepare_tensors(self):
                 raise ValueError(f"Unprocessed experts: {experts}")
 
 
+@ModelBase.register(
+    "DeepseekV32ForCausalLM",
+)
+class DeepseekV3_2Model(TextModel):
+    model_arch = gguf.MODEL_ARCH.DEEPSEEK3_2
+
+    def set_vocab(self):
+        try:
+            self._set_vocab_gpt2()
+            return
+        except Exception:
+            pass
+
+        from transformers import AutoTokenizer
+        tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        tokpre = self.get_vocab_base_pre(tokenizer)
+
+        if tokpre == "kimi-k2":
+            # Build merges list using the approach similar to HunYuanMoE
+            merges = []
+            vocab = {}
+            mergeable_ranks = tokenizer.model._mergeable_ranks
+            for token, rank in mergeable_ranks.items():
+                vocab[QwenModel.token_bytes_to_string(token)] = rank
+                if len(token) == 1:
+                    continue
+                merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
+                if len(merged) == 2:
+                    merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
+
+            # Build token list
+            vocab_size = self.hparams["vocab_size"]
+            special_tokens = tokenizer.special_tokens
+            reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
+            tokens: list[str] = []
+            toktypes: list[int] = []
+
+            for i in range(vocab_size):
+                if i not in reverse_vocab:
+                    tokens.append(f"[PAD{i}]")
+                    toktypes.append(gguf.TokenType.UNUSED)
+                else:
+                    token = reverse_vocab[i]
+                    tokens.append(token)
+                    if i in special_tokens.values():
+                        toktypes.append(gguf.TokenType.CONTROL)
+                    else:
+                        toktypes.append(gguf.TokenType.NORMAL)
+
+            self.gguf_writer.add_tokenizer_model("gpt2")
+            self.gguf_writer.add_tokenizer_pre(tokpre)
+            self.gguf_writer.add_token_list(tokens)
+            self.gguf_writer.add_token_types(toktypes)
+            self.gguf_writer.add_token_merges(merges)
+
+            special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
+            special_vocab.add_to_gguf(self.gguf_writer)
+        else:
+            raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")
+
+    def set_gguf_parameters(self):
+
+        # note: deepseek2 using MLA converts into MQA (ie: GQA with 1 group)
+        self.hparams["num_key_value_heads"] = 1
+
+        super().set_gguf_parameters()
+        hparams = self.hparams
+
+        self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
+        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+        if "q_lora_rank" in hparams and hparams["q_lora_rank"] is not None:
+            self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"])
+        self.gguf_writer.add_kv_lora_rank(hparams["kv_lora_rank"])
+
+        # note: deepseek2 using MLA converts into MQA with larger heads, then decompresses to MHA
+        self.gguf_writer.add_key_length(hparams["kv_lora_rank"] + hparams["qk_rope_head_dim"])
+        self.gguf_writer.add_value_length(hparams["kv_lora_rank"])
+        self.gguf_writer.add_key_length_mla(hparams["qk_nope_head_dim"] + hparams["qk_rope_head_dim"])
+        self.gguf_writer.add_value_length_mla(hparams["v_head_dim"])
+
+        self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+        self.gguf_writer.add_expert_count(hparams["n_routed_experts"])
+        self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"])
+        self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
+        self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
+
+        if hparams["scoring_func"] == "sigmoid":
+            self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+        elif hparams["scoring_func"] == "softmax":
+            self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
+        else:
+            raise ValueError(f"Unsupported scoring_func value: {hparams['scoring_func']}")
+
+        self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"])
+
+        rope_scaling = self.hparams.get("rope_scaling") or {}
+        if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
+            self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.YARN)
+            self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
+            self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
+            self.gguf_writer.add_rope_scaling_yarn_log_mul(0.1 * rope_scaling["mscale_all_dim"])
+
+    _experts: list[dict[str, Tensor]] | None = None
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # skip vision tensors and remove "language_model." for Kimi-VL
+        if "vision_tower" in name or "multi_modal_projector" in name:
+            return []
+
+        if name.startswith("language_model."):
+            name = name.replace("language_model.", "")
+
+        # rename e_score_correction_bias tensors
+        if name.endswith("e_score_correction_bias"):
+            name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+
+        # skip Multi-Token Prediction (MTP) layers
+        block_count = self.hparams["num_hidden_layers"]
+        match = re.match(r"model.layers.(\d+)", name)
+        if match and int(match.group(1)) >= block_count:
+            return []
+
+        # process the experts separately
+        if name.find("mlp.experts") != -1:
+            n_experts = self.hparams["n_routed_experts"]
+            assert bid is not None
+
+            if self._experts is None:
+                self._experts = [{} for _ in range(self.block_count)]
+
+            self._experts[bid][name] = data_torch
+
+            if len(self._experts[bid]) >= n_experts * 3:
+                tensors: list[tuple[str, Tensor]] = []
+
+                # merge the experts into a single 3d tensor
+                for w_name in ["down_proj", "gate_proj", "up_proj"]:
+                    datas: list[Tensor] = []
+
+                    for xid in range(n_experts):
+                        ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+                        datas.append(self._experts[bid][ename])
+                        del self._experts[bid][ename]
+
+                    data_torch = torch.stack(datas, dim=0)
+
+                    merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+
+                    new_name = self.map_tensor_name(merged_name)
+
+                    tensors.append((new_name, data_torch))
+                return tensors
+            else:
+                return []
+
+        # note: MLA with the absorption optimization, needs these two split and k_b_proj transposed
+        if name.endswith("kv_b_proj.weight"):
+            name_kb = name.replace("kv_b_proj", "k_b_proj")
+            name_vb = name.replace("kv_b_proj", "v_b_proj")
+
+            n_head_kv = self.hparams["num_key_value_heads"]
+            v_head_dim = self.hparams["v_head_dim"]
+            qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+
+            assert data_torch.shape[0] == n_head_kv * (v_head_dim + qk_nope_head_dim)
+
+            kv_b = data_torch.view(n_head_kv, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
+            k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
+            k_b = k_b.transpose(1, 2)
+
+            return [
+                (self.map_tensor_name(name_kb), k_b),
+                (self.map_tensor_name(name_vb), v_b)
+            ]
+
+        return [(self.map_tensor_name(name), data_torch)]
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+
+        if self._experts is not None:
+            # flatten `list[dict[str, Tensor]]` into `list[str]`
+            experts = [k for d in self._experts for k in d.keys()]
+            if len(experts) > 0:
+                raise ValueError(f"Unprocessed experts: {experts}")
+
+
 @ModelBase.register(
     "DeepseekV2ForCausalLM",
     "DeepseekV3ForCausalLM",

convert_hf_to_gguf_update.py

@@ -127,6 +127,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "megrez",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Infinigence/Megrez-3B-Instruct"},
     {"name": "deepseek-v3",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-V3"},
     {"name": "deepseek-r1-qwen", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"},
+    {"name": "deepseek-v3.2",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp"},
     {"name": "gpt-4o",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Xenova/gpt-4o", },
     {"name": "superbpe",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/UW/OLMo2-8B-SuperBPE-t180k", },
     {"name": "trillion",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/trillionlabs/Trillion-7B-preview", },

ggml/CMakeLists.txt

@@ -106,6 +106,10 @@ if (NOT GGML_LLAMAFILE_DEFAULT)
     set(GGML_LLAMAFILE_DEFAULT OFF)
 endif()
 
+if (NOT GGML_OPENMP_SIMD_DEFAULT)
+    set(GGML_OPENMP_SIMD_DEFAULT OFF)
+endif()
+
 if (NOT GGML_CUDA_GRAPHS_DEFAULT)
     set(GGML_CUDA_GRAPHS_DEFAULT OFF)
 endif()
@@ -169,6 +173,7 @@ option(GGML_RV_ZVFH          "ggml: enable riscv zvfh"       ON)
 option(GGML_RV_ZICBOP        "ggml: enable riscv zicbop"     ON)
 option(GGML_XTHEADVECTOR     "ggml: enable xtheadvector"     OFF)
 option(GGML_VXE              "ggml: enable vxe"              ON)
+option(GGML_OPENMP_SIMD      "ggml: enable OPENMP_SIMD"      ${GGML_OPENMP_SIMD_DEFAULT})
 
 option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
 set(GGML_CPU_ARM_ARCH        "" CACHE STRING "ggml: CPU architecture for ARM")

ggml/include/ggml-cuda-indexer.h

@@ -0,0 +1,84 @@
+#pragma once
+#include "ggml-cuda.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Forward-declare the CUDA context type; definition is in common.cuh
+struct ggml_backend_cuda_context;
+
+
+// DeepSeek V3.2: FP8 indexer K cache quantization helper (CUDA backend)
+void ggml_cuda_indexer_k_cache_fp8_quantize(
+    struct ggml_backend_cuda_context & ctx,
+    const float * dK,           // [num_tokens, head_dim]
+    unsigned char * dKvCache,   // [num_blocks, cache_block_size, cache_stride]
+    const long long * dSlotMap, // [num_tokens]
+    int num_tokens,
+    int head_dim,
+    int quant_bs,
+    int cache_block_size,
+    int cache_stride);
+
+// Gather a contiguous KV range for one stream from the FP8 indexer K cache
+// into a row-major FP8 K matrix plus per-quant-block FP32 scales.
+//
+// Inputs:
+//   dKvCache         : [num_blocks, cache_block_size, cache_stride] (I8)
+//   head_dim         : indexer head dimension D
+//   quant_bs         : quantization block size used when writing the cache
+//   cache_block_size : tokens per block (kv_size)
+//   cache_stride     : bytes per token (D + scales_per_token*4)
+//   stream_id        : which block/stream to gather from
+//   kv_start, kv_len : slice [kv_start, kv_start+kv_len) within that stream
+//
+// Outputs:
+//   dK_fp8_out  : [kv_len, head_dim] row-major FP8 codes
+//   dScale_out  : [kv_len, head_dim/quant_bs] FP32 scales (one per quant block)
+void ggml_cuda_indexer_k_cache_fp8_gather_wmma_hgrp(
+    struct ggml_backend_cuda_context & ctx,
+    const unsigned char * dKvCache,
+    int head_dim,
+    int quant_bs,
+    int cache_block_size,
+    int cache_stride,
+    int stream_id,
+    int kv_start,
+    int kv_len,
+    unsigned char * dK_fp8_out,
+    float * dScale_out);
+
+// Derive per-token KV window ends from device-resident mask [N_kv, T]
+// mask values <= -1e29 are treated as masked; ends[t] = last i where mask[i,t] > -1e29, or 0 if none
+void ggml_cuda_mask_window_ends_device(struct ggml_backend_cuda_context & ctx,
+                                       const float * dMask, int N_kv, int T,
+                                       int * dEnds);
+
+// Device-resident entry: takes device pointers and current CUDA context.
+// Optional FP8 indexer sidecar (DeepSeek V3.2) can be provided via dKvCache
+// plus its layout parameters; if dKvCache is null, the kernel will quantize
+// K from dK on the fly.
+void ggml_cuda_indexer_logits_fused_device(struct ggml_backend_cuda_context & ctx,
+                                           const float * dQ,
+                                           const float * dK,
+                                           const float * dW,
+                                           const float * dKS,
+                                           const int * dStarts, const int * dEnds,
+                                           int D, int H, int Tc, int kv_end,
+                                           const unsigned char * dKvCache,
+                                           int quant_bs,
+                                           int cache_block_size,
+                                           int cache_stride,
+                                           float * dOut);
+
+// Derive per-token KV window ends from device-resident mask and copy to host buffer
+void ggml_cuda_mask_window_ends_device_to_host(struct ggml_backend_cuda_context & ctx,
+                                               const float * dMask, int N_kv, int T, int * hEnds);
+
+// Simple convenience wrappers using current device and default stream
+void ggml_cuda_mask_window_ends_device_to_host_simple(const float * dMask, int N_kv, int T, int * hEnds);
+void ggml_cuda_mask_window_starts_device_to_host_simple(const float * dMask, int N_kv, int T, int * hStarts);
+
+#ifdef __cplusplus
+}
+#endif

ggml/include/ggml-cuda-radix.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Compute top-k indices per column using a CUDA radix-style selection.
+// scores is a row-major 2D array with shape [N, T]: element(i,t) at scores[i + N*t].
+// Writes indices into idx (shape [k, T], same storage rule: idx[i + k*t]).
+void ggml_cuda_topk_radix_indices_host(const float * scores, int N, int T, int k, int * idx);
+
+// Build per-column histogram on the top byte of float->key mapping.
+// scores: [N, T] row-major. Outputs:
+//  - gt_counts: size 256*T, gt_counts[b + 256*t] = sum_{bb>b} counts[bb]
+//  - thr_bins:  size T (currently placeholder; can be 0)
+void ggml_cuda_topk_histogram_host(const float * scores, int N, int T,
+                                   unsigned int * gt_counts, unsigned int * thr_bins);
+
+// Launch equal-bin selection kernel only, given precomputed histogram greater-counts per column
+// scores: [N, T] row-major
+// gt_counts: [256, T] greater-counts per bin
+// idx: [k, T] output indices (row-major leading dimension k)
+void ggml_cuda_topk_select_host(const float * scores, int N, int T, int k,
+                                const unsigned int * gt_counts, int * idx);
+
+#ifdef __cplusplus
+}
+#endif