Promote SGLang MoonViT integration to main

Preserves the previous deployment state on pre-sglang-main-2026-08-04. Adds the validated external SGLang model/processor package, routing-aware DeepSeek embedding patch helper, augmented model metadata/index, launch wrapper, smoke client, and deployment documentation.

config.json

@@ -52,6 +52,83 @@
     0
   ],
   "compress_rope_theta": 160000,
+  "deepseek_vision": {
+    "image_placeholder": "<image>",
+    "image_placeholder_token_id": 129280,
+    "max_image_tokens": 512,
+    "requires_sglang_source_patch": true,
+    "routing_palette": [
+      0,
+      1,
+      2,
+      8,
+      9,
+      10,
+      12,
+      74,
+      81,
+      110,
+      114,
+      240,
+      17081,
+      25312,
+      30711,
+      58279,
+      7637,
+      8936,
+      45556,
+      52073,
+      7743,
+      8347,
+      13203,
+      19795,
+      44418,
+      62970,
+      79038,
+      6381,
+      48025,
+      109859,
+      29629,
+      91213,
+      90662,
+      121562,
+      8570,
+      25568,
+      3685,
+      81916,
+      14638,
+      50590,
+      101211,
+      24832,
+      75337,
+      131,
+      15170,
+      79723,
+      84052,
+      20866,
+      48327,
+      72234,
+      15507,
+      128,
+      16760,
+      34135,
+      36264,
+      59037,
+      3839,
+      29854,
+      109646,
+      64,
+      23442,
+      6584,
+      10255,
+      17173
+    ],
+    "routing_policy": "palette_cycle",
+    "schema_version": 1,
+    "sglang_source_commit": "723c2776401987cba3cfee25830db50f3964b83f",
+    "tower_model_id": "moonshotai/Kimi-K2.6",
+    "tower_revision": "7eb5002f6aadc958aed6a9177b7ed26bb94011bb"
+  },
   "eos_token_id": 1,
   "expert_dtype": "fp4",
   "hc_eps": 1e-06,
@@ -60,11 +137,13 @@
   "head_dim": 512,
   "hidden_act": "silu",
   "hidden_size": 4096,
+  "image_token_id": 129280,
   "index_head_dim": 128,
   "index_n_heads": 64,
   "index_topk": 512,
   "initializer_range": 0.02,
   "max_position_embeddings": 1048576,
+  "media_placeholder_token_id": 129280,
   "model_type": "deepseek_v4",
   "moe_intermediate_size": 2048,
   "n_routed_experts": 256,
@@ -315,5 +394,32 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.57.1",
   "use_cache": true,
+  "vision_config": {
+    "hidden_size": 1152,
+    "init_pos_emb_height": 64,
+    "init_pos_emb_time": 4,
+    "init_pos_emb_width": 64,
+    "intermediate_size": 4304,
+    "merge_kernel_size": [
+      2,
+      2
+    ],
+    "merge_type": "sd2_tpool",
+    "mm_hidden_size": 1152,
+    "mm_projector_type": "patchmerger",
+    "model_type": "kimi_k25",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "pos_emb_type": "divided_fixed",
+    "projector_hidden_act": "gelu",
+    "projector_ln_eps": 1e-05,
+    "text_hidden_size": 4096,
+    "video_attn_type": "spatial_temporal",
+    "vt_hidden_size": 1152,
+    "vt_intermediate_size": 4304,
+    "vt_num_attention_heads": 16,
+    "vt_num_hidden_layers": 27
+  },
   "vocab_size": 129280
 }

configs/routing/deepseek-v4-flash-60d8d707-palette64.json

@@ -0,0 +1,66 @@
+[
+  0,
+  1,
+  2,
+  8,
+  9,
+  10,
+  12,
+  74,
+  81,
+  110,
+  114,
+  240,
+  17081,
+  25312,
+  30711,
+  58279,
+  7637,
+  8936,
+  45556,
+  52073,
+  7743,
+  8347,
+  13203,
+  19795,
+  44418,
+  62970,
+  79038,
+  6381,
+  48025,
+  109859,
+  29629,
+  91213,
+  90662,
+  121562,
+  8570,
+  25568,
+  3685,
+  81916,
+  14638,
+  50590,
+  101211,
+  24832,
+  75337,
+  131,
+  15170,
+  79723,
+  84052,
+  20866,
+  48327,
+  72234,
+  15507,
+  128,
+  16760,
+  34135,
+  36264,
+  59037,
+  3839,
+  29854,
+  109646,
+  64,
+  23442,
+  6584,
+  10255,
+  17173
+]

configs/serve/sglang-moonvit.yaml

@@ -0,0 +1,10 @@
+model-path: webbrain-one/DeepSeek-V4-Flash-Vision-NVFP4
+host: 127.0.0.1
+port: 30000
+tensor-parallel-size: 5
+trust-remote-code: true
+enable-multimodal: true
+limit-mm-data-per-request:
+  image: 1
+disable-cuda-graph: true
+skip-server-warmup: true

docs/SGLANG_DEPLOYMENT.md

@@ -0,0 +1,135 @@
+# SGLang deployment status
+
+## Short answer
+
+The current Hugging Face model directory is **not** a stock-SGLang multimodal model.
+SGLang natively implements the DeepSeek V4 text runtime and Kimi's MoonViT, but it does
+not know that this particular checkpoint combines them.  The integration in
+`sglang_ext/deepseek_vision_sglang` is an external SGLang model/processor package and
+requires one narrow source patch.  Do not describe it as upstream or zero-code support.
+
+The package is pinned to SGLang commit
+`723c2776401987cba3cfee25830db50f3964b83f` (current `main` inspected on 2026-08-04):
+
+- SGLang's [external model registry](https://github.com/sgl-project/sglang/blob/723c2776401987cba3cfee25830db50f3964b83f/python/sglang/srt/models/registry.py)
+  supports `SGLANG_EXTERNAL_MODEL_PACKAGE`.
+- Its [tokenizer manager](https://github.com/sgl-project/sglang/blob/723c2776401987cba3cfee25830db50f3964b83f/python/sglang/srt/managers/tokenizer_manager.py#L459-L468)
+  loads `SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE` through the multimodal processor
+  registry.
+- The native [Kimi K2.5 model](https://github.com/sgl-project/sglang/blob/723c2776401987cba3cfee25830db50f3964b83f/python/sglang/srt/models/kimi_k25.py)
+  supplies MoonViT and PatchMerger kernels.
+- The native [DeepSeek V4 model](https://github.com/sgl-project/sglang/blob/723c2776401987cba3cfee25830db50f3964b83f/python/sglang/srt/models/deepseek_v4.py#L2347-L2360)
+  currently accepts `input_embeds` but ignores it at the embedding site.  The checked-in
+  patch changes that one assignment while keeping `input_ids` for hash routing.
+
+## What the extension does
+
+1. The external processor recognizes one literal `<image>` marker and uses the official,
+   revision-pinned Kimi K2.6 processor for NaViT resize/normalization/patchification.
+2. SGLang's native MoonViT implementation loads `vision_tower.safetensors`.
+3. The native Kimi PatchMerger shape loads the trained `mm_projector.safetensors` and
+   emits 4096-d DeepSeek embeddings.
+4. Image positions are replaced in `inputs_embeds`.  A separate tensor preserves every
+   text token ID and cycles the checked-in 64-ID route palette over image positions.
+5. The routing phase uses absolute image offsets, so chunked-prefill boundaries do not
+   restart the palette.
+
+Keeping the architecture string `DeepseekV4ForCausalLM` is intentional.  SGLang selects
+its V4 attention, memory-pool, and FP4 expert behavior from that exact name; the external
+registry overwrites only the instantiated model class.
+
+## Stage a model directory
+
+Download the private model repository to a local directory that contains the backbone
+shards, `vision_tower.safetensors`, and `mm_projector.safetensors`.  Then run:
+
+```bash
+python scripts/prepare_sglang_model_repo.py /models/deepseek-v4-flash-vision
+```
+
+This makes three reviewable packaging changes:
+
+- adds `vision_config` and `deepseek_vision` metadata to `config.json`;
+- adds both standalone component files to `model.safetensors.index.json` (SGLang ignores
+  safetensor files that are absent from an existing index);
+- copies the standalone extension to `MODEL_DIR/sglang_ext`.
+
+Upload that staged directory to the private deployment repository only after reviewing
+the diff.  The script does not contact Hugging Face.
+
+## Install and launch
+
+Use the exact SGLang commit documented above.  Set the model snapshot's extension path,
+then launch through the checked-in wrapper:
+
+```bash
+export DEEPSEEK_VISION_MODEL_PATH=/models/deepseek-v4-flash-vision
+export DEEPSEEK_VISION_PYTHONPATH="$DEEPSEEK_VISION_MODEL_PATH/sglang_ext"
+export DEEPSEEK_VISION_TP=5
+scripts/launch_sglang_moonvit.sh
+```
+
+For a Hugging Face integration branch, download that revision into the local model
+directory and also set `DEEPSEEK_VISION_REVISION` if `DEEPSEEK_VISION_MODEL_PATH` is a
+Hub model ID.  For example, the pre-merge smoke branch uses:
+
+```bash
+export DEEPSEEK_VISION_REVISION=sglang-integration
+```
+
+The wrapper verifies/applies the one-line SGLang patch before startup and exports all
+three official external-registration variables.  It disables CUDA graphs for the first
+correctness gate.  Re-enable performance features only after text-only parity and image
+parity pass on the pinned build.
+
+## First request
+
+The first gate uses SGLang's native `/generate` endpoint.  It avoids an upstream OpenAI
+chat-rendering limitation: the current DeepSeek V4 chat encoder explicitly flattens
+parts-list content as text-only before multimodal processing.  Calling the OpenAI
+`/v1/chat/completions` route with `image_url` is therefore **not yet supported by this
+extension**.
+
+Use the exact prompt shape used for projector training:
+
+```bash
+curl http://127.0.0.1:30000/generate \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "text":"<｜begin▁of▁sentence｜><｜User｜><image>Describe this image.<｜Assistant｜></think>",
+    "image_data":"data:image/jpeg;base64,...",
+    "sampling_params":{"temperature":0,"max_new_tokens":64}
+  }'
+```
+
+Or use the checked-in smoke client, which builds the data URL and rejects an empty
+generation response:
+
+```bash
+python scripts/smoke_sglang_moonvit.py /path/to/probe.jpg
+```
+
+## Required GPU validation
+
+The Mac can validate packaging, routing math, source anchors, and Python syntax, but it
+cannot instantiate the 168 GB NVFP4 checkpoint or CUDA kernels.  Before calling this
+deployment ready, run these gates on suitable NVIDIA hardware:
+
+1. `python -m deepseek_vision_sglang.patch --check` against the pinned SGLang install.
+2. Loader startup with the staged private model directory.
+3. A text-only prompt compared token-for-token with unmodified SGLang DeepSeek V4.
+4. One cached training image compared against the existing Transformers/BF16 endpoint.
+5. A fresh scene and GUI screenshot, checking image-token count, route-ID count, and
+   projector output shape before judging answer quality.
+
+## Deliberate limitations
+
+- Custom SGLang package plus a one-line source patch; no upstream support claim.
+- Exactly one image per request and at most 512 post-merge image tokens.
+- CUDA image preprocessing only; CPU-only serving is not supported.
+- Tensor parallelism is intended; pipeline parallelism and encoder data parallelism are
+  blocked/unvalidated.
+- CUDA graphs, speculative decoding, OpenAI multimodal chat rendering, streaming parity,
+  and production concurrency are not validated.
+- This serving integration can fix or expose runtime integration errors.  It cannot make
+  a weakly trained projector better; quality still needs the same image benchmark.

model.safetensors.index.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d83d58754cff11724f117d20d95e31803a48512d29f8e00463b2501905d6d72
-size 11205914
+oid sha256:7cc70b0b7838961ca8cbd1d2ca31394302a6d6e84fea8cb270cd33d52af78457
+size 11227454

scripts/launch_sglang_moonvit.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+model_path="${DEEPSEEK_VISION_MODEL_PATH:-webbrain-one/DeepSeek-V4-Flash-Vision-NVFP4}"
+model_revision="${DEEPSEEK_VISION_REVISION:-}"
+tensor_parallel_size="${DEEPSEEK_VISION_TP:-5}"
+host="${DEEPSEEK_VISION_HOST:-127.0.0.1}"
+port="${DEEPSEEK_VISION_PORT:-30000}"
+model_python_path="${DEEPSEEK_VISION_PYTHONPATH:-}"
+
+if [[ -z "$model_python_path" ]]; then
+  echo "Set DEEPSEEK_VISION_PYTHONPATH to MODEL_DIR/sglang_ext." >&2
+  exit 2
+fi
+
+export PYTHONPATH="${model_python_path}${PYTHONPATH:+:${PYTHONPATH}}"
+export SGLANG_EXTERNAL_MODEL_PACKAGE="deepseek_vision_sglang.models"
+export SGLANG_EXTERNAL_MM_MODEL_ARCH="DeepseekV4ForCausalLM"
+export SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE="deepseek_vision_sglang.processors"
+
+python -m deepseek_vision_sglang.patch --apply
+
+launch_args=(
+  --model-path "$model_path"
+  --tp-size "$tensor_parallel_size"
+  --host "$host"
+  --port "$port"
+  --trust-remote-code
+  --enable-multimodal
+  --limit-mm-data-per-request '{"image":1}'
+  --disable-cuda-graph
+  --skip-server-warmup
+)
+if [[ -n "$model_revision" ]]; then
+  launch_args+=(--revision "$model_revision")
+fi
+
+exec python -m sglang.launch_server "${launch_args[@]}"

scripts/prepare_sglang_model_repo.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import struct
+import tempfile
+from pathlib import Path
+from typing import Iterable
+
+
+SGLANG_SOURCE_COMMIT = "723c2776401987cba3cfee25830db50f3964b83f"
+TOWER_MODEL_ID = "moonshotai/Kimi-K2.6"
+TOWER_REVISION = "7eb5002f6aadc958aed6a9177b7ed26bb94011bb"
+
+
+def deepseek_vision_config(*, routing_palette: Iterable[int], vocab_size: int) -> dict:
+    palette = [int(value) for value in routing_palette]
+    if not palette:
+        raise ValueError("routing palette cannot be empty")
+    if min(palette) < 0 or max(palette) >= vocab_size:
+        raise ValueError("routing palette contains an ID outside the model vocabulary")
+    return {
+        "schema_version": 1,
+        "tower_model_id": TOWER_MODEL_ID,
+        "tower_revision": TOWER_REVISION,
+        "image_placeholder": "<image>",
+        "image_placeholder_token_id": vocab_size,
+        "max_image_tokens": 512,
+        "routing_policy": "palette_cycle",
+        "routing_palette": palette,
+        "sglang_source_commit": SGLANG_SOURCE_COMMIT,
+        "requires_sglang_source_patch": True,
+    }
+
+
+def vision_config(*, text_hidden_size: int) -> dict:
+    return {
+        "model_type": "kimi_k25",
+        "patch_size": 14,
+        "init_pos_emb_height": 64,
+        "init_pos_emb_width": 64,
+        "init_pos_emb_time": 4,
+        "pos_emb_type": "divided_fixed",
+        "num_attention_heads": 16,
+        "num_hidden_layers": 27,
+        "hidden_size": 1152,
+        "intermediate_size": 4304,
+        # Current SGLang's K2 projector reads the legacy vt_* aliases while
+        # MoonViT itself reads the canonical names above.  Keep both explicit.
+        "vt_num_attention_heads": 16,
+        "vt_num_hidden_layers": 27,
+        "vt_hidden_size": 1152,
+        "vt_intermediate_size": 4304,
+        "merge_kernel_size": [2, 2],
+        "video_attn_type": "spatial_temporal",
+        "merge_type": "sd2_tpool",
+        "mm_projector_type": "patchmerger",
+        "mm_hidden_size": 1152,
+        "projector_hidden_act": "gelu",
+        "projector_ln_eps": 1e-5,
+        "text_hidden_size": int(text_hidden_size),
+    }
+
+
+def augment_config(config: dict, routing_palette: Iterable[int]) -> dict:
+    architectures = config.get("architectures") or []
+    if "DeepseekV4ForCausalLM" not in architectures:
+        raise ValueError("model architecture must remain DeepseekV4ForCausalLM")
+    vocab_size = int(config["vocab_size"])
+    hidden_size = int(config["hidden_size"])
+    updated = dict(config)
+    updated["vision_config"] = vision_config(text_hidden_size=hidden_size)
+    updated["image_token_id"] = vocab_size
+    updated["media_placeholder_token_id"] = vocab_size
+    updated["deepseek_vision"] = deepseek_vision_config(
+        routing_palette=routing_palette,
+        vocab_size=vocab_size,
+    )
+    return updated
+
+
+def augment_weight_map(
+    index: dict,
+    *,
+    tower_keys: Iterable[str],
+    projector_keys: Iterable[str],
+    tower_size: int = 0,
+    projector_size: int = 0,
+) -> dict:
+    updated = json.loads(json.dumps(index))
+    weight_map = updated.setdefault("weight_map", {})
+    existing_files = set(weight_map.values())
+    additions = {
+        **{str(name): "vision_tower.safetensors" for name in tower_keys},
+        **{str(name): "mm_projector.safetensors" for name in projector_keys},
+    }
+    collisions = sorted(
+        name
+        for name, filename in additions.items()
+        if name in weight_map and weight_map[name] != filename
+    )
+    if collisions:
+        raise ValueError(f"component tensor names collide with backbone keys: {collisions[:8]}")
+    weight_map.update(additions)
+    added_size = sum(
+        size
+        for filename, size in (
+            ("vision_tower.safetensors", int(tower_size)),
+            ("mm_projector.safetensors", int(projector_size)),
+        )
+        if filename not in existing_files
+    )
+    if added_size:
+        metadata = updated.setdefault("metadata", {})
+        metadata["total_size"] = int(metadata.get("total_size", 0)) + added_size
+    return updated
+
+
+def safetensor_keys(path: Path) -> list[str]:
+    with path.open("rb") as handle:
+        prefix = handle.read(8)
+        if len(prefix) != 8:
+            raise ValueError(f"invalid safetensors header prefix: {path}")
+        header_size = struct.unpack("<Q", prefix)[0]
+        if header_size <= 0 or header_size > 100 * 1024 * 1024:
+            raise ValueError(f"invalid safetensors header size {header_size}: {path}")
+        header = json.loads(handle.read(header_size))
+    return [str(name) for name in header if name != "__metadata__"]
+
+
+def write_json_atomic(path: Path, payload: dict) -> None:
+    with tempfile.NamedTemporaryFile(
+        mode="w",
+        encoding="utf-8",
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+        delete=False,
+    ) as handle:
+        json.dump(payload, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+        temporary = Path(handle.name)
+    temporary.replace(path)
+
+
+def stage_extension(model_dir: Path, source_root: Path) -> None:
+    source = source_root / "sglang_ext" / "deepseek_vision_sglang"
+    if not source.is_dir():
+        raise ValueError(f"SGLang extension source is missing: {source}")
+    target_root = model_dir / "sglang_ext"
+    target_root.mkdir(exist_ok=True)
+    target = target_root / source.name
+    if target.exists():
+        shutil.rmtree(target)
+    shutil.copytree(source, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+
+
+def prepare(model_dir: Path, palette_path: Path, source_root: Path) -> None:
+    required = [
+        model_dir / "config.json",
+        model_dir / "model.safetensors.index.json",
+        model_dir / "vision_tower.safetensors",
+        model_dir / "mm_projector.safetensors",
+    ]
+    missing = [str(path) for path in required if not path.is_file()]
+    if missing:
+        raise ValueError(f"model directory is incomplete: {missing}")
+    palette = json.loads(palette_path.read_text(encoding="utf-8"))
+    config = json.loads(required[0].read_text(encoding="utf-8"))
+    index = json.loads(required[1].read_text(encoding="utf-8"))
+    write_json_atomic(required[0], augment_config(config, palette))
+    write_json_atomic(
+        required[1],
+        augment_weight_map(
+            index,
+            tower_keys=safetensor_keys(required[2]),
+            projector_keys=safetensor_keys(required[3]),
+            tower_size=required[2].stat().st_size,
+            projector_size=required[3].stat().st_size,
+        ),
+    )
+    stage_extension(model_dir, source_root)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Stage a DeepSeek V4 + MoonViT model directory for the pinned SGLang fork"
+    )
+    parser.add_argument("model_dir", type=Path)
+    parser.add_argument(
+        "--palette",
+        type=Path,
+        default=Path("configs/routing/deepseek-v4-flash-60d8d707-palette64.json"),
+    )
+    parser.add_argument(
+        "--source-root",
+        type=Path,
+        default=Path(__file__).resolve().parents[1],
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    prepare(args.model_dir.resolve(), args.palette.resolve(), args.source_root.resolve())
+    print(f"prepared {args.model_dir.resolve()} for pinned SGLang {SGLANG_SOURCE_COMMIT}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/smoke_sglang_moonvit.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import mimetypes
+import urllib.request
+from pathlib import Path
+
+
+DEFAULT_PROMPT = (
+    "<｜begin▁of▁sentence｜><｜User｜><image>"
+    "Describe this image.<｜Assistant｜></think>"
+)
+
+
+def image_data_url(path: Path) -> str:
+    mime_type, _ = mimetypes.guess_type(path.name)
+    if mime_type not in {"image/jpeg", "image/png", "image/webp"}:
+        raise ValueError("smoke image must be JPEG, PNG, or WebP")
+    encoded = base64.b64encode(path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
+def build_payload(path: Path, prompt: str, max_new_tokens: int) -> dict:
+    if prompt.count("<image>") != 1:
+        raise ValueError("prompt must contain exactly one literal <image> marker")
+    return {
+        "text": prompt,
+        "image_data": image_data_url(path),
+        "sampling_params": {
+            "temperature": 0,
+            "max_new_tokens": int(max_new_tokens),
+        },
+    }
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Smoke-test the custom SGLang /generate path")
+    parser.add_argument("image", type=Path)
+    parser.add_argument("--url", default="http://127.0.0.1:30000/generate")
+    parser.add_argument("--prompt", default=DEFAULT_PROMPT)
+    parser.add_argument("--max-new-tokens", type=int, default=64)
+    parser.add_argument("--timeout", type=float, default=180.0)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    payload = build_payload(args.image.resolve(), args.prompt, args.max_new_tokens)
+    request = urllib.request.Request(
+        args.url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=args.timeout) as response:
+        result = json.loads(response.read().decode("utf-8"))
+    generated = result.get("text")
+    if not isinstance(generated, str) or not generated.strip():
+        raise RuntimeError(f"SGLang returned no generated text: {result}")
+    print(json.dumps(result, indent=2, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sglang_ext/deepseek_vision_sglang/__init__.py

@@ -0,0 +1,10 @@
+"""External SGLang package for the DeepSeek V4 + MoonViT adapter.
+
+This package is intentionally standalone.  A staged Hugging Face model directory can
+ship ``sglang_ext/deepseek_vision_sglang`` and expose it through ``PYTHONPATH`` without
+installing the training project.
+"""
+
+SGLANG_SOURCE_COMMIT = "723c2776401987cba3cfee25830db50f3964b83f"
+
+__all__ = ["SGLANG_SOURCE_COMMIT"]

sglang_ext/deepseek_vision_sglang/models/__init__.py

@@ -0,0 +1 @@
+"""SGLang external model registry package."""

sglang_ext/deepseek_vision_sglang/models/deepseek_v4_moonvit.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Optional, Tuple
+
+import torch
+import torch.nn as nn
+
+from sglang.srt.configs.kimi_k25 import KimiK25VisionConfig
+from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
+from sglang.srt.managers.mm_utils import (
+    MultiModalityDataPaddingPatternMultimodalTokens,
+    general_mm_embed_routine,
+)
+from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem, MultimodalInputs
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
+from sglang.srt.model_loader.weight_utils import default_weight_loader
+from sglang.srt.models.deepseek_v4 import (
+    DeepseekV4ForCausalLM as SGLangDeepseekV4ForCausalLM,
+)
+from sglang.srt.models.kimi_k25 import (
+    K2VLMultiModalProjector,
+    MoonViT3dPretrainedModel,
+    mm_projection_auto,
+)
+from sglang.srt.multimodal.mm_utils import materialize_multimodal_features
+from sglang.srt.runtime_context import get_mm, get_parallel, get_server_args
+
+from deepseek_vision_sglang.routing import ROUTING_IDS_ATTR, build_sglang_routing_ids
+
+
+def _config_dict(config, name: str) -> dict:
+    value = getattr(config, name, None)
+    if value is None:
+        raise ValueError(f"config.json is missing {name}")
+    if isinstance(value, dict):
+        return dict(value)
+    if hasattr(value, "to_dict"):
+        return value.to_dict()
+    raise TypeError(f"config.json field {name} must be an object")
+
+
+class RoutingAwareDeepseekV4ForCausalLM(SGLangDeepseekV4ForCausalLM):
+    """Recover route IDs when SGLang's multimodal routine supplies embeddings."""
+
+    @torch.no_grad()
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor],
+        positions: torch.Tensor,
+        forward_batch: ForwardBatch,
+        input_embeds: Optional[torch.Tensor] = None,
+        pp_proxy_tensors: Optional[PPProxyTensors] = None,
+    ) -> torch.Tensor:
+        if input_ids is None:
+            input_ids = getattr(forward_batch, ROUTING_IDS_ATTR, None)
+        if input_ids is None:
+            raise RuntimeError("DeepSeek V4 routing IDs were not attached to the forward batch")
+        return super().forward(
+            input_ids=input_ids,
+            positions=positions,
+            forward_batch=forward_batch,
+            input_embeds=input_embeds,
+            pp_proxy_tensors=pp_proxy_tensors,
+        )
+
+
+class DeepseekV4ForCausalLM(nn.Module):
+    """MoonViT wrapper that deliberately overwrites SGLang's text-only V4 entry.
+
+    Keeping the architecture name unchanged is required: SGLang's DeepSeek V4
+    attention, FP4 expert-layout, and memory-pool selection all key off this exact
+    architecture string. ``SGLANG_EXTERNAL_MODEL_PACKAGE`` performs the overwrite.
+    """
+
+    def __init__(self, config, quant_config=None, prefix: str = "") -> None:
+        super().__init__()
+        self.config = config
+        self.quant_config = quant_config
+        adapter = _config_dict(config, "deepseek_vision")
+        vision_config = KimiK25VisionConfig(**_config_dict(config, "vision_config"))
+        self.route_palette = tuple(int(value) for value in adapter["routing_palette"])
+        if not self.route_palette:
+            raise ValueError("deepseek_vision.routing_palette cannot be empty")
+        if min(self.route_palette) < 0 or max(self.route_palette) >= int(config.vocab_size):
+            raise ValueError("deepseek_vision.routing_palette contains an invalid token ID")
+        if int(vision_config.text_hidden_size) != int(config.hidden_size):
+            raise ValueError("vision projector output size differs from DeepSeek hidden size")
+        if get_mm().mm_enable_dp_encoder:
+            raise NotImplementedError(
+                "DeepSeek V4 + MoonViT has not validated SGLang encoder data parallelism"
+            )
+        if get_parallel().pp_size > 1:
+            raise NotImplementedError(
+                "DeepSeek V4 + MoonViT has not validated pipeline parallelism"
+            )
+
+        self.vision_tower = MoonViT3dPretrainedModel(
+            vision_config,
+            use_data_parallel=False,
+            prefix="vision_tower",
+        )
+        self.mm_projector = K2VLMultiModalProjector(vision_config)
+        self.language_model = RoutingAwareDeepseekV4ForCausalLM(
+            config,
+            quant_config,
+            prefix=prefix,
+        )
+        if hasattr(self.language_model, "dtype"):
+            target_dtype = self.language_model.dtype
+            self.vision_tower = self.vision_tower.to(dtype=target_dtype)
+            self.mm_projector = self.mm_projector.to(dtype=target_dtype)
+        self._loaded_vision_params: set[str] = set()
+
+    @property
+    def model(self):
+        return self.language_model
+
+    def __setattr__(self, name, value):
+        if name == "model":
+            return
+        super().__setattr__(name, value)
+
+    @property
+    def pp_group(self):
+        return self.language_model.pp_group
+
+    @property
+    def start_layer(self) -> int:
+        return self.language_model.start_layer
+
+    @property
+    def end_layer(self) -> int:
+        return self.language_model.end_layer
+
+    @property
+    def routed_experts_weights_of_layer(self):
+        return self.language_model.routed_experts_weights_of_layer
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def _materialize_image_features(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        device = self.vision_tower.device
+        dtype = self.vision_tower.patch_embed.proj.weight.dtype
+        parallel = get_parallel()
+        server_args = get_server_args()
+        consumer_count = max(getattr(server_args, "tp_size", parallel.attn_tp_size), 1)
+        device_index = device.index
+        if device.type == "cuda" and device_index is None:
+            device_index = torch.cuda.current_device()
+        features = []
+        for item in items:
+            if device.type == "cuda":
+                item.reconstruct(device_index, ipc_consumer_count=consumer_count)
+            features.append(item.feature)
+        return materialize_multimodal_features(features, device=device, dtype=dtype)
+
+    def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        grid_thws = []
+        for item in items:
+            grid = item.model_specific_data.get("image_grid_thw")
+            if grid is None:
+                grid = item.model_specific_data["grid_thws"]
+            grid_thws.append(grid)
+        grid_thws = torch.concat(grid_thws, dim=0)
+        pixel_values = self._materialize_image_features(items)
+        image_features = self.vision_tower(pixel_values, grid_thws)
+        return mm_projection_auto(self.mm_projector, image_features)
+
+    def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
+        return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
+            input_ids, mm_inputs
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        forward_batch: ForwardBatch,
+        get_embedding: bool = False,
+        pp_proxy_tensors: Optional[PPProxyTensors] = None,
+    ):
+        if get_embedding:
+            raise NotImplementedError("embedding mode is not supported by this generative adapter")
+        routing_ids = build_sglang_routing_ids(input_ids, forward_batch, self.route_palette)
+        setattr(forward_batch, ROUTING_IDS_ATTR, routing_ids)
+        return general_mm_embed_routine(
+            input_ids=input_ids,
+            forward_batch=forward_batch,
+            language_model=self.language_model,
+            multimodal_model=self,
+            data_embedding_funcs={Modality.IMAGE: self.get_image_feature},
+            positions=positions,
+            pp_proxy_tensors=pp_proxy_tensors,
+        )
+
+    def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+        params = dict(self.named_parameters(remove_duplicate=False))
+
+        def load_component(name: str, loaded_weight: torch.Tensor) -> bool:
+            tower_name = name.removeprefix("vision_tower.")
+            tower_name = tower_name.replace("wqkv.", "attn.qkv_proj.").replace(
+                "wo.", "attn.proj."
+            )
+            tower_param = f"vision_tower.{tower_name}"
+            projector_name = name.removeprefix("mm_projector.")
+            projector_name = projector_name.replace("proj.0", "linear_1").replace(
+                "proj.2", "linear_2"
+            )
+            projector_param = f"mm_projector.{projector_name}"
+            target = None
+            if tower_param in params:
+                target = tower_param
+            elif projector_param in params:
+                target = projector_param
+            if target is None:
+                return False
+            param = params[target]
+            loader = getattr(param, "weight_loader", default_weight_loader)
+            loader(param, loaded_weight)
+            self._loaded_vision_params.add(target)
+            return True
+
+        def language_weights():
+            for name, loaded_weight in weights:
+                if load_component(name, loaded_weight):
+                    continue
+                yield name.removeprefix("language_model."), loaded_weight
+
+        self.language_model.load_weights(language_weights())
+
+        required = {
+            name
+            for name in params
+            if name.startswith("vision_tower.") or name.startswith("mm_projector.")
+        }
+        missing = sorted(required - self._loaded_vision_params)
+        if missing:
+            sample = ", ".join(missing[:8])
+            raise RuntimeError(
+                f"vision_tower.safetensors/mm_projector.safetensors did not initialize "
+                f"{len(missing)} parameter(s): {sample}"
+            )
+
+    def post_load_weights(self):
+        self.language_model.post_load_weights()
+
+    @property
+    def stacked_params_mapping(self):
+        return getattr(self.language_model, "stacked_params_mapping", [])
+
+    @property
+    def expert_params_mapping(self):
+        return getattr(self.language_model, "expert_params_mapping", [])
+
+    def mutate_weight_preload(self, name):
+        return self.language_model.mutate_weight_preload(name)
+
+    def custom_scale_remap(self, name):
+        return self.language_model.custom_scale_remap(name)
+
+    @classmethod
+    def get_model_config_for_expert_location(cls, config):
+        return ModelConfigForExpertLocation(
+            num_layers=config.num_hidden_layers,
+            num_logical_experts=config.n_routed_experts,
+            num_groups=None,
+        )
+
+
+EntryClass = [DeepseekV4ForCausalLM]

sglang_ext/deepseek_vision_sglang/patch.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import argparse
+import importlib.util
+from pathlib import Path
+
+from . import SGLANG_SOURCE_COMMIT
+
+
+OLD = "hidden_states = self.embed_tokens(input_ids)"
+NEW = "hidden_states = input_embeds if input_embeds is not None else self.embed_tokens(input_ids)"
+FORWARD_ANCHOR = "class DeepseekV4Model(nn.Module):"
+
+
+def resolve_sglang_deepseek_v4_source() -> Path:
+    spec = importlib.util.find_spec("sglang.srt.models.deepseek_v4")
+    if spec is None or spec.origin is None:
+        raise RuntimeError("could not locate sglang.srt.models.deepseek_v4")
+    return Path(spec.origin).resolve()
+
+
+def patch_deepseek_v4_source(path: str | Path, *, check_only: bool = False) -> bool:
+    """Apply the one-line routing-aware embedding patch.
+
+    Returns ``True`` when the source needed a patch and ``False`` when it was already
+    patched.  ``check_only`` validates the exact source anchor without writing.
+    """
+    source = Path(path)
+    text = source.read_text(encoding="utf-8")
+    if FORWARD_ANCHOR not in text:
+        raise RuntimeError("not a recognized SGLang DeepSeek V4 model source")
+    if text.count(NEW) == 1:
+        return False
+    count = text.count(OLD)
+    if count != 1:
+        raise RuntimeError(
+            "expected exactly one SGLang DeepSeek V4 embedding site; "
+            f"found {count}. The extension is pinned to {SGLANG_SOURCE_COMMIT}."
+        )
+    if not check_only:
+        source.write_text(text.replace(OLD, NEW), encoding="utf-8")
+    return True
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Patch the pinned SGLang DeepSeek V4 loader")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--check", action="store_true")
+    mode.add_argument("--apply", action="store_true")
+    parser.add_argument("source", nargs="?", help="deepseek_v4.py; auto-detected when omitted")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    source = Path(args.source).resolve() if args.source else resolve_sglang_deepseek_v4_source()
+    needed = patch_deepseek_v4_source(source, check_only=args.check)
+    state = "patchable" if needed and args.check else "patched" if needed else "already-patched"
+    print(f"{source}: {state}; pinned SGLang commit {SGLANG_SOURCE_COMMIT}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sglang_ext/deepseek_vision_sglang/processors/__init__.py

@@ -0,0 +1 @@
+"""SGLang external multimodal processor registry package."""

sglang_ext/deepseek_vision_sglang/processors/moonvit.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Union
+
+from transformers import AutoProcessor
+
+from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
+from sglang.srt.multimodal.processors.base_processor import (
+    BaseMultimodalProcessor,
+    MultimodalSpecialTokens,
+)
+from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin
+from sglang.srt.multimodal.processors.kimi_k25 import KimiGPUProcessorWrapper
+
+from deepseek_vision_sglang.models.deepseek_v4_moonvit import DeepseekV4ForCausalLM
+
+
+def _adapter_config(hf_config) -> dict:
+    value = getattr(hf_config, "deepseek_vision", None)
+    if isinstance(value, dict):
+        return value
+    if hasattr(value, "to_dict"):
+        return value.to_dict()
+    raise ValueError("config.json is missing deepseek_vision")
+
+
+class DeepseekV4MoonViTProcessor(KimiGridMMDataMixin, BaseMultimodalProcessor):
+    models = [DeepseekV4ForCausalLM]
+    prefer_tokenized_input = False
+    gpu_image_decode = True
+    precompute_hash_before_cpu_transfer = True
+    preserve_processor_input_ids = True
+    supports_mm_processor_concurrency = False
+
+    def __init__(self, hf_config, server_args, text_processor, *args, **kwargs):
+        adapter = _adapter_config(hf_config)
+        text_tokenizer = getattr(text_processor, "tokenizer", text_processor)
+        tower_processor = AutoProcessor.from_pretrained(
+            adapter["tower_model_id"],
+            revision=adapter["tower_revision"],
+            trust_remote_code=True,
+            use_fast=True,
+        )
+        tower_processor.tokenizer = text_tokenizer
+        media_cfg = tower_processor.media_processor.media_proc_cfg
+        image_token = adapter.get("image_placeholder", "<image>")
+        image_token_id = int(adapter.get("image_placeholder_token_id", hf_config.vocab_size))
+        mm_tokens = MultimodalSpecialTokens(
+            image_token=image_token,
+            image_token_id=image_token_id,
+            image_token_regex=re.compile(re.escape(image_token)),
+        ).build(tower_processor)
+        max_image_tokens = int(adapter.get("max_image_tokens", 512))
+        merge_h, merge_w = media_cfg["merge_kernel_size"]
+        processor = KimiGPUProcessorWrapper(
+            tower_processor,
+            image_token=mm_tokens.image_token,
+            image_token_id=mm_tokens.image_token_id,
+            patch_size=media_cfg["patch_size"],
+            merge_kernel_size=media_cfg["merge_kernel_size"],
+            in_patch_limit=max_image_tokens * int(merge_h) * int(merge_w),
+            patch_limit_on_one_side=media_cfg["patch_limit_on_one_side"],
+            fixed_output_tokens=media_cfg.get("fixed_output_tokens"),
+            image_mean=media_cfg["image_mean"],
+            image_std=media_cfg["image_std"],
+        )
+        super().__init__(hf_config, server_args, processor, *args, **kwargs)
+        self.mm_tokens = mm_tokens
+
+    def _encode_prompt(self, prompt: str, image_count: int) -> list[int]:
+        if not isinstance(prompt, str):
+            raise ValueError("MoonViT SGLang processor requires a string prompt")
+        parts = prompt.split(self.mm_tokens.image_token)
+        if len(parts) - 1 != image_count:
+            raise ValueError(
+                f"prompt contains {len(parts) - 1} image placeholder(s), "
+                f"but request contains {image_count} image(s)"
+            )
+        input_ids: list[int] = []
+        for index, part in enumerate(parts):
+            input_ids.extend(self._tokenizer.encode(part, add_special_tokens=False))
+            if index < image_count:
+                input_ids.append(int(self.mm_tokens.image_token_id))
+        return input_ids
+
+    async def process_mm_data_async(
+        self,
+        image_data: List[Union[str, bytes, Dict]],
+        input_text,
+        request_obj,
+        *args,
+        **kwargs,
+    ):
+        images = image_data or []
+        if len(images) != 1:
+            raise ValueError("DeepSeek V4 + MoonViT currently accepts exactly one image")
+        input_ids = self._encode_prompt(input_text, len(images))
+        base_output = await self.fast_load_mm_data(
+            prompt=input_text,
+            image_data=images,
+            multimodal_tokens=self.mm_tokens,
+            input_ids=input_ids,
+        )
+        mm_items, expanded_ids, _ = await self.process_and_combine_mm_data_async(
+            base_output,
+            self.mm_tokens,
+            sglang_original_input_ids=input_ids,
+        )
+        return MultimodalProcessorOutput(
+            input_ids=expanded_ids.tolist(),
+            mm_items=mm_items,
+            im_token_id=self.mm_tokens.image_token_id,
+        )
+
+
+__all__ = ["DeepseekV4MoonViTProcessor"]

sglang_ext/deepseek_vision_sglang/routing.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+
+ROUTING_IDS_ATTR = "_deepseek_vision_routing_ids"
+
+
+def _as_int_list(values: Any) -> list[int]:
+    if values is None:
+        return []
+    if hasattr(values, "tolist"):
+        values = values.tolist()
+    return [int(value) for value in values]
+
+
+def _item_is_image(item: Any) -> bool:
+    modality = getattr(item, "modality", None)
+    name = getattr(modality, "name", modality)
+    return str(name).upper() == "IMAGE"
+
+
+def _item_offsets(item: Any) -> list[tuple[int, int]]:
+    offsets = getattr(item, "offsets", None) or []
+    return [(int(start), int(end)) for start, end in offsets]
+
+
+def routing_replacements(
+    *,
+    extend_prefix_lens: Sequence[int],
+    extend_seq_lens: Sequence[int],
+    mm_inputs: Sequence[Any],
+    palette: Sequence[int],
+) -> list[tuple[int, int]]:
+    """Return flattened ``(position, route_id)`` replacements for image slots.
+
+    SGLang flattens each request's current extend chunk into one token vector.  An
+    item's offsets remain absolute, inclusive prompt offsets.  Basing the palette
+    phase on ``absolute_position - image_start`` keeps routing identical when a long
+    image prefix is split across chunked-prefill batches.
+    """
+    if not palette:
+        raise ValueError("routing palette cannot be empty")
+    prefixes = _as_int_list(extend_prefix_lens)
+    lengths = _as_int_list(extend_seq_lens)
+    inputs = list(mm_inputs or [])
+    if not (len(prefixes) == len(lengths) == len(inputs)):
+        raise ValueError("SGLang request metadata lengths differ")
+
+    replacements: list[tuple[int, int]] = []
+    flat_request_start = 0
+    for prefix_len, seq_len, request_mm in zip(prefixes, lengths, inputs, strict=True):
+        chunk_start = prefix_len
+        chunk_end = prefix_len + seq_len
+        items = getattr(request_mm, "mm_items", None) if request_mm is not None else None
+        for item in items or []:
+            if not _item_is_image(item):
+                continue
+            for image_start, image_end_inclusive in _item_offsets(item):
+                overlap_start = max(chunk_start, image_start)
+                overlap_end = min(chunk_end, image_end_inclusive + 1)
+                for absolute_position in range(overlap_start, overlap_end):
+                    flat_position = flat_request_start + absolute_position - chunk_start
+                    palette_index = (absolute_position - image_start) % len(palette)
+                    replacements.append((flat_position, int(palette[palette_index])))
+        flat_request_start += seq_len
+    return replacements
+
+
+def build_sglang_routing_ids(input_ids: Any, forward_batch: Any, palette: Sequence[int]):
+    """Clone SGLang token IDs and replace only image slots with palette IDs."""
+    if input_ids is None:
+        raise ValueError("input_ids are required to build DeepSeek routing IDs")
+    result = input_ids.clone() if hasattr(input_ids, "clone") else list(input_ids)
+    mm_inputs = getattr(forward_batch, "mm_inputs", None)
+    if not mm_inputs:
+        return result
+    replacements = routing_replacements(
+        extend_prefix_lens=getattr(forward_batch, "extend_prefix_lens_cpu", None),
+        extend_seq_lens=getattr(forward_batch, "extend_seq_lens_cpu", None),
+        mm_inputs=mm_inputs,
+        palette=palette,
+    )
+    for position, route_id in replacements:
+        result[position] = route_id
+    return result