Skip to content

vllm.models.mistral.mistral_large4.amd.specialized_decode.runtime.weights

One MoE decoder layer's kernel arguments, built from the loaded vLLM modules.

Attention / indexer / router weights are repacked into the kernels' tiled formats (copies). The routed experts are read in place: vLLM's AITER a4w4 MXFP4 storage (aiter shuffle_weight values, e8m0_shuffle scales) is the kernels' ATOM layout. The shared expert, a separate block-FP8 MLP in vLLM, is re-quantized to MXFP4 the way VLLM_FP8_MOE_REQUANT quantizes the routed experts and passed through its own pointers.

Classes:

  • LayerKind –

    Kernel variant of one MoE layer (variants.load_variant + build flags).

  • MoeLayer –

    One MoE layer's kernel inputs (args: launch-order tensors by name).

Functions:

  • check_moe_layer –

    The weight checks of build_moe_layer without building anything.

  • moe_kinds –

    Kernel variant per MoE layer (1..); raises when the layer mix is unsupported.

LayerKind

Kernel variant of one MoE layer (variants.load_variant + build flags).

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
class LayerKind:
    """Kernel variant of one MoE layer (``variants.load_variant`` + build flags)."""

    ROPE = "rope"  # RoPE, own indexer
    ROPE_SHARED = "rope_shared"  # RoPE, top-k of the preceding indexer layer
    NOPE = "nope"  # NoPE, own indexer
    NOPE_KVS = "nope_kvs"  # NoPE, own indexer, reads the source layer's latent cache

MoeLayer dataclass

One MoE layer's kernel inputs (args: launch-order tensors by name).

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
@dataclass
class MoeLayer:
    """One MoE layer's kernel inputs (``args``: launch-order tensors by name)."""

    layer_id: int
    kind: str
    args: dict[str, torch.Tensor]
    params: torch.Tensor  # int64 pointer table (indexer weights, shared expert)
    attn: Any  # MistralMLAAttention (cache bindings are read per step)

_fp8_param(linear)

A block-FP8 linear's [N, K] fp8 weight and fp32 [N/128, K/128] scales.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
def _fp8_param(linear) -> tuple[torch.Tensor, torch.Tensor]:
    """A block-FP8 linear's [N, K] fp8 weight and fp32 [N/128, K/128] scales."""
    w = linear.weight
    s = getattr(linear, "weight_scale_inv", None)
    if s is None:
        s = getattr(linear, "weight_scale", None)
    if w.dtype != FP8 or s is None or s.dtype != torch.float32:
        raise MonoUnsupported(f"{type(linear).__name__}: expected block-FP8 weights")
    n, k = w.shape[-2:]
    if tuple(s.shape) != ((n + 127) // 128, (k + 127) // 128):
        # transposed storage (a kernel that keeps W^T) is not read here
        raise MonoUnsupported(f"block scales {tuple(s.shape)} for weight {(n, k)}")
    return w.contiguous(), s.contiguous()

_q128(w, block_cols=128)

bf16/fp32 [r, c] -> (fp8, fp32 scale per 128 x block_cols block), amax / 448.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
def _q128(w: torch.Tensor, block_cols: int = 128) -> tuple[torch.Tensor, torch.Tensor]:
    """bf16/fp32 [r, c] -> (fp8, fp32 scale per 128 x block_cols block), amax / 448."""
    r, c = w.shape
    w = w.float()
    wp = F.pad(w, (0, (-c) % block_cols, 0, (-r) % 128))
    b = wp.view(wp.shape[0] // 128, 128, wp.shape[1] // block_cols, block_cols)
    s = b.abs().amax((1, 3)).clamp_min(1e-12) / 448.0
    q = (b / s[:, None, :, None]).clamp(-448, 448).view_as(wp)[:r, :c].to(FP8)
    return q.contiguous(), s.contiguous()

_routed_experts(experts)

Views of vLLM's routed-expert storage (no copy).

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
def _routed_experts(experts) -> dict[str, torch.Tensor]:
    """Views of vLLM's routed-expert storage (no copy)."""
    from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend

    experts = getattr(experts, "routed_experts", experts)  # shared-expert wrapper
    method = getattr(experts, "quant_method", None)
    method = getattr(method, "old_quant_method", method)  # modular-kernel wrapper
    backend = getattr(method, "mxfp4_backend", None)
    if backend != Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
        raise MonoUnsupported(
            f"routed experts on {backend} ({type(experts).__name__}/"
            f"{type(method).__name__}); mono reads AITER a4w4 storage "
            "(VLLM_FP8_MOE_REQUANT=mxfp4 VLLM_FP8_MOE_REQUANT_ACT=a4w4)"
        )
    out = {}
    for name, key in (
        ("w13_weight", "w_ug"),
        ("w13_weight_scale", "s_ug"),
        ("w2_weight", "w_dn"),
        ("w2_weight_scale", "s_dn"),
    ):
        t = getattr(experts, name).data
        if t.shape[0] != N_ROUTED or not t.is_contiguous():
            raise MonoUnsupported(f"{name} {tuple(t.shape)}")
        v = t.view(torch.uint8)
        if key.startswith("w"):
            v.is_shuffled = True  # aiter shuffle_weight layout (backend checked above)
        out[key] = v
    return out

_shared_expert_mxfp4(mlp)

The shared expert re-quantized to MXFP4 in vLLM's a4w4 storage layout.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
def _shared_expert_mxfp4(mlp) -> dict[str, torch.Tensor]:
    """The shared expert re-quantized to MXFP4 in vLLM's a4w4 storage layout."""
    from aiter.utility.fp4_utils import e8m0_shuffle

    from vllm._aiter_ops import rocm_aiter_ops

    def mx(linear):
        w, s = _fp8_param(linear)
        q, sc = mxfp4_quantize(dequant_block_fp8_to_bf16(w, s, FP8_BLOCK))
        return q.view(torch.uint8), sc.view(torch.uint8)

    ug, s_ug = mx(mlp.gate_up_proj)  # rows [gate; up], as w13
    dn, s_dn = mx(mlp.down_proj)
    fp4 = torch.float4_e2m1fn_x2
    w13, w2 = rocm_aiter_ops.shuffle_weights(
        ug[None].contiguous().view(fp4), dn[None].contiguous().view(fp4)
    )
    out = {
        "w_ug_sh": w13.view(torch.uint8).contiguous(),
        "w_dn_sh": w2.view(torch.uint8).contiguous(),
        "s_ug_sh": e8m0_shuffle(s_ug.contiguous()).contiguous(),
        "s_dn_sh": e8m0_shuffle(s_dn.contiguous()).contiguous(),
    }
    return out

check_moe_layer(layer, npes=4)

The weight checks of build_moe_layer without building anything.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
def check_moe_layer(layer, npes: int = 4) -> None:
    """The weight checks of ``build_moe_layer`` without building anything."""
    attn = layer.self_attn
    if attn.num_local_heads * npes != 64 or attn.v_head_dim != 128:
        raise MonoUnsupported("attention geometry")
    _fp8_param(attn.q_proj)
    _fp8_param(attn.o_proj)
    w = attn.kv_b_proj.weight
    if w.dtype != torch.bfloat16 or w.shape[-1] != 512:
        raise MonoUnsupported(f"kv_b_proj {w.dtype} {tuple(w.shape)}")
    if attn.ssmax is None:
        raise MonoUnsupported("attention without SSMax")
    ix = attn.indexer
    if ix is not None:
        if ix.ssmax is None or ix.n_head != INDEX_HEADS or ix.head_dim != INDEX_DIM:
            raise MonoUnsupported("indexer geometry")
        _fp8_param(ix.wq_b)
    if layer.mlp.shared_experts is None:
        raise MonoUnsupported("MoE layer without a shared expert")
    _fp8_param(layer.mlp.shared_experts.gate_up_proj)
    _fp8_param(layer.mlp.shared_experts.down_proj)
    _routed_experts(layer.mlp.experts)

moe_kinds(layers)

Kernel variant per MoE layer (1..); raises when the layer mix is unsupported.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
def moe_kinds(layers) -> list[str]:
    """Kernel variant per MoE layer (1..); raises when the layer mix is unsupported."""
    if layers[0].self_attn.spec.is_moe or not all(
        lyr.self_attn.spec.is_moe for lyr in layers[1:]
    ):
        raise MonoUnsupported("layer mix: expected one dense layer, then MoE layers")
    kinds = [layer_kind(lyr.self_attn.spec) for lyr in layers[1:]]
    last_indexer = None
    for i, kind in enumerate(kinds):
        if kind == LayerKind.ROPE_SHARED and last_indexer != LayerKind.ROPE:
            # the top-k is read from the RoPE package's scratch
            raise MonoUnsupported(f"layer {i + 1}: shared top-k after {last_indexer}")
        if KINDS[kind][2]:
            last_indexer = kind
    return kinds