One MoE decoder layer's kernel arguments, built from the loaded vLLM modules.
Attention / indexer / router weights are repacked into the kernels' tiled
formats (copies). The routed experts are read in place: vLLM's AITER a4w4 MXFP4
storage (aiter shuffle_weight values, e8m0_shuffle scales) is the
kernels' ATOM layout. The shared expert, a separate block-FP8 MLP in vLLM, is
re-quantized to MXFP4 the way VLLM_FP8_MOE_REQUANT quantizes the routed
experts and passed through its own pointers.
Classes:
-
LayerKind
–
Kernel variant of one MoE layer (variants.load_variant + build flags).
-
MoeLayer
–
One MoE layer's kernel inputs (args: launch-order tensors by name).
Functions:
-
check_moe_layer
–
The weight checks of build_moe_layer without building anything.
-
moe_kinds
–
Kernel variant per MoE layer (1..); raises when the layer mix is unsupported.
LayerKind
Kernel variant of one MoE layer (variants.load_variant + build flags).
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| class LayerKind:
"""Kernel variant of one MoE layer (``variants.load_variant`` + build flags)."""
ROPE = "rope" # RoPE, own indexer
ROPE_SHARED = "rope_shared" # RoPE, top-k of the preceding indexer layer
NOPE = "nope" # NoPE, own indexer
NOPE_KVS = "nope_kvs" # NoPE, own indexer, reads the source layer's latent cache
|
MoeLayer
dataclass
One MoE layer's kernel inputs (args: launch-order tensors by name).
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| @dataclass
class MoeLayer:
"""One MoE layer's kernel inputs (``args``: launch-order tensors by name)."""
layer_id: int
kind: str
args: dict[str, torch.Tensor]
params: torch.Tensor # int64 pointer table (indexer weights, shared expert)
attn: Any # MistralMLAAttention (cache bindings are read per step)
|
_fp8_param(linear)
A block-FP8 linear's [N, K] fp8 weight and fp32 [N/128, K/128] scales.
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| def _fp8_param(linear) -> tuple[torch.Tensor, torch.Tensor]:
"""A block-FP8 linear's [N, K] fp8 weight and fp32 [N/128, K/128] scales."""
w = linear.weight
s = getattr(linear, "weight_scale_inv", None)
if s is None:
s = getattr(linear, "weight_scale", None)
if w.dtype != FP8 or s is None or s.dtype != torch.float32:
raise MonoUnsupported(f"{type(linear).__name__}: expected block-FP8 weights")
n, k = w.shape[-2:]
if tuple(s.shape) != ((n + 127) // 128, (k + 127) // 128):
# transposed storage (a kernel that keeps W^T) is not read here
raise MonoUnsupported(f"block scales {tuple(s.shape)} for weight {(n, k)}")
return w.contiguous(), s.contiguous()
|
_q128(w, block_cols=128)
bf16/fp32 [r, c] -> (fp8, fp32 scale per 128 x block_cols block), amax / 448.
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| def _q128(w: torch.Tensor, block_cols: int = 128) -> tuple[torch.Tensor, torch.Tensor]:
"""bf16/fp32 [r, c] -> (fp8, fp32 scale per 128 x block_cols block), amax / 448."""
r, c = w.shape
w = w.float()
wp = F.pad(w, (0, (-c) % block_cols, 0, (-r) % 128))
b = wp.view(wp.shape[0] // 128, 128, wp.shape[1] // block_cols, block_cols)
s = b.abs().amax((1, 3)).clamp_min(1e-12) / 448.0
q = (b / s[:, None, :, None]).clamp(-448, 448).view_as(wp)[:r, :c].to(FP8)
return q.contiguous(), s.contiguous()
|
_routed_experts(experts)
Views of vLLM's routed-expert storage (no copy).
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| def _routed_experts(experts) -> dict[str, torch.Tensor]:
"""Views of vLLM's routed-expert storage (no copy)."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
experts = getattr(experts, "routed_experts", experts) # shared-expert wrapper
method = getattr(experts, "quant_method", None)
method = getattr(method, "old_quant_method", method) # modular-kernel wrapper
backend = getattr(method, "mxfp4_backend", None)
if backend != Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
raise MonoUnsupported(
f"routed experts on {backend} ({type(experts).__name__}/"
f"{type(method).__name__}); mono reads AITER a4w4 storage "
"(VLLM_FP8_MOE_REQUANT=mxfp4 VLLM_FP8_MOE_REQUANT_ACT=a4w4)"
)
out = {}
for name, key in (
("w13_weight", "w_ug"),
("w13_weight_scale", "s_ug"),
("w2_weight", "w_dn"),
("w2_weight_scale", "s_dn"),
):
t = getattr(experts, name).data
if t.shape[0] != N_ROUTED or not t.is_contiguous():
raise MonoUnsupported(f"{name} {tuple(t.shape)}")
v = t.view(torch.uint8)
if key.startswith("w"):
v.is_shuffled = True # aiter shuffle_weight layout (backend checked above)
out[key] = v
return out
|
_shared_expert_mxfp4(mlp)
The shared expert re-quantized to MXFP4 in vLLM's a4w4 storage layout.
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| def _shared_expert_mxfp4(mlp) -> dict[str, torch.Tensor]:
"""The shared expert re-quantized to MXFP4 in vLLM's a4w4 storage layout."""
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
def mx(linear):
w, s = _fp8_param(linear)
q, sc = mxfp4_quantize(dequant_block_fp8_to_bf16(w, s, FP8_BLOCK))
return q.view(torch.uint8), sc.view(torch.uint8)
ug, s_ug = mx(mlp.gate_up_proj) # rows [gate; up], as w13
dn, s_dn = mx(mlp.down_proj)
fp4 = torch.float4_e2m1fn_x2
w13, w2 = rocm_aiter_ops.shuffle_weights(
ug[None].contiguous().view(fp4), dn[None].contiguous().view(fp4)
)
out = {
"w_ug_sh": w13.view(torch.uint8).contiguous(),
"w_dn_sh": w2.view(torch.uint8).contiguous(),
"s_ug_sh": e8m0_shuffle(s_ug.contiguous()).contiguous(),
"s_dn_sh": e8m0_shuffle(s_dn.contiguous()).contiguous(),
}
return out
|
check_moe_layer(layer, npes=4)
The weight checks of build_moe_layer without building anything.
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| def check_moe_layer(layer, npes: int = 4) -> None:
"""The weight checks of ``build_moe_layer`` without building anything."""
attn = layer.self_attn
if attn.num_local_heads * npes != 64 or attn.v_head_dim != 128:
raise MonoUnsupported("attention geometry")
_fp8_param(attn.q_proj)
_fp8_param(attn.o_proj)
w = attn.kv_b_proj.weight
if w.dtype != torch.bfloat16 or w.shape[-1] != 512:
raise MonoUnsupported(f"kv_b_proj {w.dtype} {tuple(w.shape)}")
if attn.ssmax is None:
raise MonoUnsupported("attention without SSMax")
ix = attn.indexer
if ix is not None:
if ix.ssmax is None or ix.n_head != INDEX_HEADS or ix.head_dim != INDEX_DIM:
raise MonoUnsupported("indexer geometry")
_fp8_param(ix.wq_b)
if layer.mlp.shared_experts is None:
raise MonoUnsupported("MoE layer without a shared expert")
_fp8_param(layer.mlp.shared_experts.gate_up_proj)
_fp8_param(layer.mlp.shared_experts.down_proj)
_routed_experts(layer.mlp.experts)
|
moe_kinds(layers)
Kernel variant per MoE layer (1..); raises when the layer mix is unsupported.
Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/weights.py
| def moe_kinds(layers) -> list[str]:
"""Kernel variant per MoE layer (1..); raises when the layer mix is unsupported."""
if layers[0].self_attn.spec.is_moe or not all(
lyr.self_attn.spec.is_moe for lyr in layers[1:]
):
raise MonoUnsupported("layer mix: expected one dense layer, then MoE layers")
kinds = [layer_kind(lyr.self_attn.spec) for lyr in layers[1:]]
last_indexer = None
for i, kind in enumerate(kinds):
if kind == LayerKind.ROPE_SHARED and last_indexer != LayerKind.ROPE:
# the top-k is read from the RoPE package's scratch
raise MonoUnsupported(f"layer {i + 1}: shared top-k after {last_indexer}")
if KINDS[kind][2]:
last_indexer = kind
return kinds
|