Skip to content

vllm.models.mistral.amd.ops.ssmax

SSMax query scaling q *= s * log(pos + 1) + b, alone or fused with RoPE.

bf16 only. Every kernel reproduces the unfused rounding: the scale is computed in bf16 (product and sum each rounded), and RoPE output is rounded to bf16 before it is scaled.

Functions:

_round_bf16(x)

Round fp32 to the nearest-even bf16 value, kept in fp32.

Done on the bits: Triton may narrow fp32 math followed by a bf16 cast into native bf16 instructions, which do not match torch on ties.

Source code in vllm/models/mistral/amd/ops/ssmax.py
@triton.jit
def _round_bf16(x):
    """Round fp32 to the nearest-even bf16 value, kept in fp32.

    Done on the bits: Triton may narrow fp32 math followed by a bf16 cast into
    native bf16 instructions, which do not match torch on ties.
    """
    bits = x.to(tl.uint32, bitcast=True)
    bits = (bits + 0x7FFF + ((bits >> 16) & 1)) & 0xFFFF0000
    return bits.to(tl.float32, bitcast=True)

_ssmax_scale(log_pos_ptr, s_ptr, b_ptr, token, head, mask)

bf16 log_pos * s + b (each op rounded), returned in fp32.

Inputs may be fp32 or bf16 (SSMax parameters load in the model dtype).

Source code in vllm/models/mistral/amd/ops/ssmax.py
@triton.jit
def _ssmax_scale(log_pos_ptr, s_ptr, b_ptr, token, head, mask):
    """bf16 ``log_pos * s + b`` (each op rounded), returned in fp32.

    Inputs may be fp32 or bf16 (SSMax parameters load in the model dtype).
    """
    log_pos = _round_bf16(tl.load(log_pos_ptr + token, mask=mask).to(tl.float32))
    s = _round_bf16(tl.load(s_ptr + head, mask=mask).to(tl.float32))
    b = _round_bf16(tl.load(b_ptr + head, mask=mask).to(tl.float32))
    return _round_bf16(_round_bf16(log_pos * s) + b)

fused_rope_ssmax_scaling_(q, k, positions, cos_sin_cache, log_positions, s, b, nope_dim, is_neox_style)

In place: RoPE on q[..., nope_dim:] and k, then SSMax on q.

q is [T, H, nope_dim + rope_dim], k is [T, 1, rope_dim] (the shared MLA rope key); token strides are free.

Source code in vllm/models/mistral/amd/ops/ssmax.py
def fused_rope_ssmax_scaling_(
    q: torch.Tensor,
    k: torch.Tensor,
    positions: torch.Tensor,
    cos_sin_cache: torch.Tensor,
    log_positions: torch.Tensor,
    s: torch.Tensor,
    b: torch.Tensor,
    nope_dim: int,
    is_neox_style: bool,
) -> None:
    """In place: RoPE on ``q[..., nope_dim:]`` and ``k``, then SSMax on ``q``.

    ``q`` is ``[T, H, nope_dim + rope_dim]``, ``k`` is ``[T, 1, rope_dim]``
    (the shared MLA rope key); token strides are free.
    """
    num_tokens, num_heads, head_dim = q.shape
    rope_dim = head_dim - nope_dim
    assert nope_dim > 0 and rope_dim > 0 and rope_dim % 2 == 0
    assert cos_sin_cache.shape[1] == rope_dim
    assert k.shape == (num_tokens, 1, rope_dim)
    assert q.dtype == k.dtype == torch.bfloat16
    assert q.stride(-1) == 1 and k.stride(-1) == 1
    assert positions.shape == log_positions.shape == (num_tokens,)
    assert s.shape == b.shape == (num_heads,)
    if num_tokens == 0:
        return
    _rope_ssmax_scaling_kernel[(num_tokens,)](
        q,
        k,
        positions,
        cos_sin_cache,
        log_positions,
        s,
        b,
        q.stride(0),
        q.stride(1),
        k.stride(0),
        cos_sin_cache.stride(0),
        num_heads=num_heads,
        head_block=triton.next_power_of_2(num_heads),
        nope_dim=nope_dim,
        nope_block=triton.next_power_of_2(nope_dim),
        half_rope_dim=rope_dim // 2,
        rope_block=triton.next_power_of_2(rope_dim // 2),
        is_neox_style=is_neox_style,
        num_warps=4,
    )

fused_ssmax_scaling(q, log_positions, s, b, out_head_dim=None)

Return F.pad(q, (0, out_head_dim - D)) * (log_positions * s + b).

q is [T, H, D] with contiguous head dims; the padding (NoPE layers' zero rope dims) is fused into the same launch.

Source code in vllm/models/mistral/amd/ops/ssmax.py
def fused_ssmax_scaling(
    q: torch.Tensor,
    log_positions: torch.Tensor,
    s: torch.Tensor,
    b: torch.Tensor,
    out_head_dim: int | None = None,
) -> torch.Tensor:
    """Return ``F.pad(q, (0, out_head_dim - D)) * (log_positions * s + b)``.

    ``q`` is ``[T, H, D]`` with contiguous head dims; the padding (NoPE layers'
    zero rope dims) is fused into the same launch.
    """
    num_tokens, num_heads, in_dim = q.shape
    out_dim = out_head_dim or in_dim
    assert out_dim >= in_dim and q.stride(-1) == 1 and q.dtype == torch.bfloat16
    assert log_positions.shape == (num_tokens,)
    assert s.shape == b.shape == (num_heads,)
    out = torch.empty((num_tokens, num_heads, out_dim), dtype=q.dtype, device=q.device)
    if out.numel() == 0:
        return out
    block_size = 1024
    _ssmax_scaling_kernel[(cdiv(out.numel(), block_size),)](
        q,
        log_positions,
        s,
        b,
        out,
        q.stride(0),
        q.stride(1),
        out.numel(),
        num_heads,
        in_dim,
        out_dim,
        block_size,
    )
    return out