Skip to content

vllm.model_executor.kernels.attention.dsa.dcp_topk_direct

Direct DCP top-k candidate exchange over PyTorch symmetric memory.

Replaces the NCCL all-gather of the DCP indexer merge for small batches. Each rank packs its local (score, global token id) candidates, stores them into every peer's inbox slice, then publishes an epoch and waits for all peers. The stable top-k selector reads the completed local inbox directly. Two alternating inboxes let the next exchange start without a read acknowledgement: a rank can only overwrite a slot after every peer has published the exchange in between, i.e. after their selectors consumed it.

On SM90+, programmatic dependent launches overlap the dispatch of the pack, push, publish and select kernels; device-side waits keep the data dependencies.

Classes:

Functions:

DirectDCPTopKWorkspace

Bases: DirectCPWorkspace

Symmetric ping-pong inboxes for the DCP top-k candidate exchange.

Source code in vllm/model_executor/kernels/attention/dsa/dcp_topk_direct.py
class DirectDCPTopKWorkspace(DirectCPWorkspace):
    """Symmetric ping-pong inboxes for the DCP top-k candidate exchange."""

    def __init__(
        self,
        group: torch.distributed.ProcessGroup,
        device: torch.device,
        max_rows: int,
        candidates_per_rank: int,
    ) -> None:
        super().__init__(group, device, num_ubatches=1)
        self.max_rows = max_rows
        self.candidates_per_rank = candidates_per_rank
        self.use_pdl = current_platform.is_arch_support_pdl()
        # [ubatch, ping-pong slot, row, source rank, candidate, (score, token id)]
        inbox_shape = (1, 2, max_rows, self.world_size, candidates_per_rank, 2)
        inboxes, peer_inbox_ptrs = self._allocate(inbox_shape, torch.float32)
        signals, peer_signal_ptrs = self._allocate((1, 1), torch.int64)
        self.inboxes = inboxes[0].reshape(
            2, max_rows, self.world_size * candidates_per_rank, 2
        )
        self.peer_inbox_ptrs = peer_inbox_ptrs[0]
        # This rank's published epoch; its parity selects the inbox to write.
        self.write_seq = signals[0]
        self.peer_signal_ptrs = peer_signal_ptrs[0]
        self.packed = torch.empty(
            (max_rows, candidates_per_rank, 2), dtype=torch.float32, device=device
        )

    def merge(
        self,
        logits: torch.Tensor,
        topk_indices: torch.Tensor,
        cp_interleave: int,
        row_starts: torch.Tensor | None,
        score_bits: int = 32,
        id_bits: int = 32,
    ) -> None:
        from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
            pack_dcp_topk_candidates_cutedsl,
        )
        from vllm.model_executor.kernels.attention.dsa.dcp_topk_pingpong_cutedsl import (  # noqa: E501
            stable_topk_from_pingpong_candidates_cutedsl,
        )

        rows = topk_indices.shape[0]
        assert 0 < rows <= self.max_rows
        pack_dcp_topk_candidates_cutedsl(
            logits,
            topk_indices,
            self.packed[:rows],
            self.rank,
            self.world_size,
            cp_interleave,
            row_starts,
        )
        num_elems = rows * self.candidates_per_rank * 2
        block_size = 1024
        _push_candidates_kernel[(triton.cdiv(num_elems, block_size),)](
            self.peer_inbox_ptrs,
            self.write_seq,
            self.packed,
            num_elems,
            my_rank=self.rank,
            world_size=self.world_size,
            local_elems=self.candidates_per_rank * 2,
            slot_elems=self.max_rows * self.world_size * self.candidates_per_rank * 2,
            block_size=block_size,
            launch_pdl=self.use_pdl,
        )
        _publish_and_wait_kernel[(1,)](
            self.peer_signal_ptrs,
            my_rank=self.rank,
            world_size=self.world_size,
            block_size=triton.next_power_of_2(self.world_size),
            max_spins=100_000_000,
            launch_pdl=self.use_pdl,
        )
        stable_topk_from_pingpong_candidates_cutedsl(
            self.inboxes,
            self.write_seq,
            self.candidates_per_rank,
            out=topk_indices,
            use_pdl=self.use_pdl,
            score_bits=score_bits,
            id_bits=id_bits,
        )

get_direct_dcp_topk_workspace(group, device, max_rows, candidates_per_rank, use_direct) cached

Collective on first call (symmetric-memory rendezvous): every DCP rank must call it with the same arguments, outside CUDA graph capture.

Source code in vllm/model_executor/kernels/attention/dsa/dcp_topk_direct.py
@functools.cache
def get_direct_dcp_topk_workspace(
    group: GroupCoordinator,
    device: torch.device,
    max_rows: int,
    candidates_per_rank: int,
    use_direct: bool | None,
) -> DirectDCPTopKWorkspace | None:
    """Collective on first call (symmetric-memory rendezvous): every DCP rank
    must call it with the same arguments, outside CUDA graph capture."""
    if not direct_cp_enabled(group, torch.float32, use_direct):
        return None
    assert group.device_group is not None
    workspace = DirectDCPTopKWorkspace(
        group.device_group, device, max_rows, candidates_per_rank
    )
    logger.info_once(
        "Using symmetric-memory direct DCP top-k candidate exchange "
        "(max_rows=%d, candidates_per_rank=%d).",
        max_rows,
        candidates_per_rank,
    )
    return workspace