Kernels
kernels-bot's picture
Uploaded using `kernel-builder`.
3bbbd55 verified
Raw History Blame Contribute Delete
5.39 kB
import operator
import torch
import triton
import triton.language as tl
from .utils import compare_version
from .utils import ensure_contiguous
from .utils import get_npu_core_count
from .utils import infer_device
from .utils import is_npu_available
if compare_version("triton", operator.ge, "3.0.0") and not is_npu_available():
try:
# typical import path with dispatch available
from triton.language.extra.libdevice import tanh
except ModuleNotFoundError:
# for working with NGC containers
from triton.language.extra.cuda.libdevice import tanh
else:
from triton.language.math import tanh
@triton.autotune(
configs=[
triton.Config({"BLOCK_N": bn}, num_stages=ns, num_warps=nw)
for bn in [1024, 2048, 4096]
for ns in [1, 2]
for nw in [4, 8, 16]
],
key=["N"],
)
@triton.jit
def _dyt_fwd_kernel(X, Y, Alpha, Gamma, Beta, HAVE_BETA: tl.constexpr, N: tl.constexpr, BLOCK_N: tl.constexpr):
col = tl.cast(tl.program_id(0), tl.int64) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = col < N
row_id = tl.cast(tl.program_id(1), tl.int64)
X += row_id * N
Y += row_id * N
alpha = tl.load(Alpha).to(tl.float32)
gamma = tl.load(Gamma + col, mask=mask, other=0.0).to(tl.float32)
x = tl.load(X + col, mask=mask, other=0.0).to(tl.float32)
tanh_x = tanh(alpha * x)
y = tanh_x * gamma
if HAVE_BETA:
beta = tl.load(Beta + col, mask=mask, other=0.0).to(tl.float32)
y += beta
tl.store(Y + col, y, mask=mask)
@triton.autotune(
configs=[
triton.Config({"BLOCK_N": bn}, num_stages=ns, num_warps=nw)
for bn in [1024, 2048, 4096]
for ns in [1, 2]
for nw in [4, 8, 16]
],
key=["N"],
# DA is indexed by program_id(0), so different BLOCK_N configs write to
# different slot counts per SM. Autotune trials don't zero outputs between
# runs, so stale slots from a prior trial would leak into da.sum(). Reset
# DA between trials to isolate each config's writes.
reset_to_zero=["DA"],
)
@triton.jit
def _dyt_bwd_kernel(
DY, DX, DA, DG, DB, X, Alpha, Gamma, HAVE_BETA: tl.constexpr, M, N: tl.constexpr, BLOCK_N: tl.constexpr
):
col = tl.cast(tl.program_id(0), tl.int64) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = col < N
start_row_id = tl.cast(tl.program_id(1), tl.int64)
alpha = tl.load(Alpha).to(tl.float32)
da = 0.0
gamma = tl.load(Gamma + col, mask=mask, other=0.0).to(tl.float32)
dg = tl.zeros((BLOCK_N,), dtype=tl.float32)
if HAVE_BETA:
db = tl.zeros((BLOCK_N,), dtype=tl.float32)
for row_id in range(start_row_id, M, tl.num_programs(1)):
x = tl.load(X + row_id * N + col, mask=mask, other=0.0).to(tl.float32)
dy = tl.load(DY + row_id * N + col, mask=mask, other=0.0).to(tl.float32)
tanh_x = tanh(alpha * x)
if HAVE_BETA:
db += dy
dg += dy * tanh_x
tmp = (1 - tanh_x * tanh_x) * dy * gamma
da += tl.sum(x * tmp, 0)
dx = alpha * tmp
tl.store(DX + row_id * N + col, dx, mask=mask)
tl.store(DG + start_row_id * N + col, dg, mask=mask)
if HAVE_BETA:
tl.store(DB + start_row_id * N + col, db, mask=mask)
tl.store(DA + start_row_id * tl.cdiv(N, 512) + tl.program_id(0), da)
def liger_dyt_fwd(x, alpha, gamma, beta):
assert x.is_contiguous()
HAVE_BETA = True if beta is not None else False
input_shape = x.shape
x = x.view(-1, input_shape[-1])
M, N = x.shape
y = torch.empty_like(x)
grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]), M)
_dyt_fwd_kernel[grid](
x,
y,
alpha,
gamma,
beta,
HAVE_BETA,
N,
)
return y.view(input_shape)
def liger_dyt_bwd(dy, x, alpha, gamma, beta):
assert dy.is_contiguous()
input_shape = x.shape
x = x.view(-1, input_shape[-1])
M, N = x.shape
HAVE_BETA = True if beta is not None else False
device = infer_device()
if device == "cuda":
NUM_SMS = torch.cuda.get_device_properties(x.device).multi_processor_count
elif device == "xpu":
NUM_SMS = torch.xpu.get_device_properties(x.device).gpu_subslice_count
elif device == "npu":
NUM_SMS = get_npu_core_count()
da = torch.zeros(NUM_SMS, triton.cdiv(N, 512), dtype=torch.float32, device=x.device)
dg = torch.empty(NUM_SMS, N, dtype=torch.float32, device=x.device)
db = torch.empty(NUM_SMS, N, dtype=torch.float32, device=x.device) if HAVE_BETA else None
dx = torch.empty_like(dy)
grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]), NUM_SMS)
_dyt_bwd_kernel[grid](dy, dx, da, dg, db, x, alpha, gamma, HAVE_BETA, M, N)
if HAVE_BETA:
db = db.sum(0).to(x.dtype)
dg = dg.sum(0).to(gamma.dtype)
da = da.sum().to(x.dtype).unsqueeze(0)
return dx.view(input_shape), da, dg, db
class LigerDyTFunction(torch.autograd.Function):
@staticmethod
@ensure_contiguous
def forward(ctx, x, alpha, gamma, beta):
y = liger_dyt_fwd(x, alpha, gamma, beta)
ctx.save_for_backward(x, alpha, gamma, beta)
return y
@staticmethod
@ensure_contiguous
def backward(ctx, dy):
x, alpha, gamma, beta = ctx.saved_tensors
dx, dalpha, dgamma, dbeta = liger_dyt_bwd(dy, x, alpha, gamma, beta)
return dx, dalpha, dgamma, dbeta