Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+build/torch27-cxx11-cu118-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch27-cxx11-cu126-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch27-cxx11-cu128-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu126-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu128-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu129-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,6 @@
+---
+tags:
+  - kernel
+---
+
+RWKV kernel for transformers

build.toml

@@ -0,0 +1,31 @@
+[general]
+name = "rwkv"
+universal = false
+
+[torch]
+src = [
+  "torch-ext/torch_binding.cpp",
+]
+
+[kernel.rwkv]
+depends = ["torch"]
+backend = "cuda"
+cuda-capabilities = [
+    "8.0",
+    "8.9",
+    "9.0",
+    "10.0",
+    "12.0",
+]
+include = ["."]
+src = [
+    "rwkv/wkv_cuda.cu",
+    "rwkv/wkv_cuda_bf16.cu",
+]
+cuda-flags = [
+        "-res-usage",
+        "--use_fast_math",
+        "-O3",
+        "--extra-device-vectorization",
+        "-DTmax=1024",
+]

build/torch27-cxx11-cu118-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch27-cxx11-cu118-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch27-cxx11-cu118-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25632a613591ab66c83b18aeda5bd01f4ba117e34345efdb6191fe501be170cf
+size 2065424

build/torch27-cxx11-cu126-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch27-cxx11-cu126-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch27-cxx11-cu126-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c96331e3a863df3e5fc2eee8fe0916f036a51e5c73a03bba2169556a945db2d7
+size 2106440

build/torch27-cxx11-cu128-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch27-cxx11-cu128-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch27-cxx11-cu128-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b1ec16445438d1b9ec59f7efab20880b58d774f59c06abab5ae964d164d5fa0
+size 2308880

build/torch28-cxx11-cu126-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch28-cxx11-cu126-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch28-cxx11-cu126-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00476bbd1c8b08d8ad3fef06cb12e0a86900830ad4f72dd4ea400131f432d510
+size 2106464

build/torch28-cxx11-cu128-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch28-cxx11-cu128-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch28-cxx11-cu128-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:301bf08ad13b2e382ccd49bd7de6ed8931fd1c9a70eb699728999fa454a52723
+size 2308880

build/torch28-cxx11-cu129-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch28-cxx11-cu129-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch28-cxx11-cu129-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f42c1d67d68e35de1f1140e85011870e61bf78a6c77499e2fc4c8926d3ebe3
+size 2330376

build/torch29-cxx11-cu126-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch29-cxx11-cu126-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch29-cxx11-cu126-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40d078873830f1833a85082fd6d907fd75c7eb3c94605ee77618981fe8cdee4e
+size 2106440

build/torch29-cxx11-cu128-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch29-cxx11-cu128-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch29-cxx11-cu128-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad6856c6dd2c9ed38dc4077f21c09cec11d0de47b870ae348d6576637f9816a2
+size 2308848

build/torch29-cxx11-cu130-x86_64-linux/rwkv/__init__.py

@@ -0,0 +1,170 @@
+from ._ops import ops
+from typing import Tuple, Any
+
+# Use a broad Tensor alias to avoid importing torch at import time.
+from torch import Tensor
+
+def forward(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (float32).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward(w, u, k, v, y)
+
+
+def forward_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor) -> None:
+    """RWKV WKV forward pass (bfloat16 inputs/outputs, float32 ``w``).
+
+    Runs the CUDA kernel and writes the result into ``y`` in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y))
+    ops.forward_bf16(w, u, k, v, y)
+
+
+def forward_with_state(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (float32).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.float32``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state(w, u, k, v, y, s)
+
+
+def forward_with_state_bf16(w: Tensor, u: Tensor, k: Tensor, v: Tensor, y: Tensor, s: Tensor) -> None:
+    """RWKV WKV forward pass with persistent state (bfloat16 inputs/outputs, float32 ``w`` and ``s``).
+
+    Runs the CUDA kernel using and updating state ``s`` and writes the result into ``y``.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u: Input tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        k: Key tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        v: Value tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        y: Output tensor, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+        s: Stateful tensor, shape ``[B, C]``, dtype ``torch.float32`` (updated in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B`` and ``C``; ``y`` shares ``[B, T, C]`` with inputs.
+    """
+    _validate_device_match((w, u, k, v, y, s))
+    ops.forward_with_state_bf16(w, u, k, v, y, s)
+
+
+def backward(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (float32).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.float32``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.float32`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.float32`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def backward_bf16(
+    w: Tensor,
+    u: Tensor,
+    k: Tensor,
+    v: Tensor,
+    y: Tensor,
+    gy: Tensor,
+    gw: Tensor,
+    gu: Tensor,
+    gk: Tensor,
+    gv: Tensor,
+) -> None:
+    """RWKV WKV backward pass (bfloat16 inputs/outputs/gradients, float32 ``w``).
+
+    Writes gradients into the provided tensors in-place.
+
+    Args:
+        w: Decay weights, shape ``[C]``, dtype ``torch.float32``.
+        u, k, v, y: Forward-pass tensors, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gy: Gradient of ``y``, shape ``[B, T, C]``, dtype ``torch.bfloat16``.
+        gw: Gradient for ``w``, shape ``[C]``, dtype ``torch.bfloat16`` (written in-place).
+        gu, gk, gv: Gradients for ``u``, ``k``, ``v`` respectively, shape ``[B, T, C]``, dtype ``torch.bfloat16`` (written in-place).
+
+    Notes:
+        - All tensors must be on the same CUDA device.
+        - Shapes must agree on ``B``, ``T`` and ``C``.
+    """
+    _validate_device_match((w, u, k, v, y, gy, gw, gu, gk, gv))
+    ops.backward_bf16(w, u, k, v, y, gy, gw, gu, gk, gv)
+
+
+def _validate_device_match(tensors: Tuple[Tensor, ...]) -> None:
+    """Minimal runtime validation that all tensors live on the same CUDA device."""
+    if not tensors:
+        return
+    device = tensors[0].device
+    if not device.type == "cuda":
+        raise RuntimeError("RWKV CUDA ops require CUDA tensors")
+    for t in tensors[1:]:
+        if t.device != device:
+            raise RuntimeError("All tensors must be on the same CUDA device")
+
+
+__all__ = [
+    "forward",
+    "forward_bf16",
+    "forward_with_state",
+    "forward_with_state_bf16",
+    "backward",
+    "backward_bf16",
+]

build/torch29-cxx11-cu130-x86_64-linux/rwkv/_ops.py

@@ -0,0 +1,9 @@
+import torch
+from . import _rwkv_eb0e3e5_dirty
+ops = torch.ops._rwkv_eb0e3e5_dirty
+
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_rwkv_eb0e3e5_dirty::{op_name}"

build/torch29-cxx11-cu130-x86_64-linux/rwkv/_rwkv_eb0e3e5_dirty.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbe5a4e59abcaf208a0b7e0c952821d9585431aec58ca2364881d5239ac5819e
+size 2334744

flake.lock

@@ -0,0 +1,168 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "locked": {
+        "lastModified": 1747046372,
+        "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
+      "locked": {
+        "lastModified": 1747046372,
+        "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "hf-nix": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1759851564,
+        "narHash": "sha256-Xybkhm0FM/VzlZ5WndTYq/X/9MAeddd4EQ2Vz8GdkOA=",
+        "owner": "huggingface",
+        "repo": "hf-nix",
+        "rev": "351655d9f124805ed7c1193aa61550ce245f4570",
+        "type": "github"
+      },
+      "original": {
+        "owner": "huggingface",
+        "repo": "hf-nix",
+        "type": "github"
+      }
+    },
+    "kernel-builder": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "hf-nix": "hf-nix",
+        "nixpkgs": [
+          "kernel-builder",
+          "hf-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1760035358,
+        "narHash": "sha256-N5vmCrgwcIluPclf/hmnofLK77EJJYh5PR8SRvw++es=",
+        "owner": "huggingface",
+        "repo": "kernel-builder",
+        "rev": "a48cbd19ae7e425dfc1865188ef06dac43ab9244",
+        "type": "github"
+      },
+      "original": {
+        "owner": "huggingface",
+        "repo": "kernel-builder",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1755963616,
+        "narHash": "sha256-6yD0ww/S8n+U2uPYcJZ3DRURP8Kx036GRpR2uPNZroE=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "73e96df7cff5783f45e21342a75a1540c4eddce4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "kernel-builder": "kernel-builder"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,17 @@
+{
+  description = "Flake for rwkv kernels";
+
+  inputs = {
+    kernel-builder.url = "github:huggingface/kernel-builder";
+  };
+
+  outputs =
+    {
+      self,
+      kernel-builder,
+    }:
+    kernel-builder.lib.genFlakeOutputs {
+      path = ./.;
+      rev = self.shortRev or self.dirtyShortRev or self.lastModifiedDate;
+    };
+}