Skip to content

vllm.v1.attention.ops.triton_decode_attention

Memory-efficient attention for decoding. It supports page size >= 1.

Functions:

decode_attention_fwd(q, k_buffer, v_buffer, o, lse, req_to_token, b_seq_len, attn_logits, num_kv_splits, sm_scale, page_size=1, logit_cap=0.0, k_scale=None, v_scale=None, is_mla=False, sliding_window=None)

sliding_window limits each row to its last sliding_window keys.

Source code in vllm/v1/attention/ops/triton_decode_attention.py
def decode_attention_fwd(
    q,
    k_buffer,
    v_buffer,
    o,
    lse,
    req_to_token,
    b_seq_len,
    attn_logits,
    num_kv_splits,
    sm_scale,
    page_size=1,
    logit_cap=0.0,
    k_scale=None,
    v_scale=None,
    is_mla=False,
    sliding_window: int | None = None,
):
    """``sliding_window`` limits each row to its last ``sliding_window`` keys."""
    assert num_kv_splits == attn_logits.shape[2]
    sliding_window = sliding_window or 0

    if k_scale is None:
        k_scale = torch.tensor(1.0, dtype=torch.float32, device=q.device)
    if v_scale is None:
        v_scale = torch.tensor(1.0, dtype=torch.float32, device=q.device)

    kv_group_num = q.shape[1] // v_buffer.shape[-2]

    if kv_group_num == 1:
        # MHA
        decode_attention_fwd_normal(
            q,
            k_buffer,
            v_buffer,
            o,
            lse,
            req_to_token,
            b_seq_len,
            attn_logits,
            num_kv_splits,
            sm_scale,
            page_size,
            logit_cap,
            k_scale,
            v_scale,
            sliding_window=sliding_window,
        )
    else:
        # GQA/MQA/MLA
        decode_attention_fwd_grouped(
            q,
            k_buffer,
            v_buffer,
            o,
            lse,
            req_to_token,
            b_seq_len,
            attn_logits,
            num_kv_splits,
            sm_scale,
            page_size,
            logit_cap,
            k_scale,
            v_scale,
            is_mla=is_mla,
            sliding_window=sliding_window,
        )