cuda

package
v1.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 14, 2026 License: MIT Imports: 9 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	// UseGemmScratch enables reusable GEMM operand scratch buffers (op0/op1).
	UseGemmScratch = true
)

Functions

func AddBiasLastDim

func AddBiasLastDim(outDev, biasDev DevicePtr, totalElems, lastDim int) error

AddBiasLastDim adds bias[lastDim] to each row-major element out[i].

func AppMemFromBaseline

func AppMemFromBaseline(current MemStats) (appUsed, appFree uint64, ok bool)

AppMemFromBaseline returns GPU bytes used by this app and free left from startup budget.

func CausalMaskSquare added in v1.4.0

func CausalMaskSquare(ioDev DevicePtr, outer, T int, neg float32) error

CausalMaskSquare adds neg to upper triangle of each [T,T] block.

func ClearScratch

func ClearScratch()

ClearScratch frees GEMM scratch buffers.

func DeviceCopyF16 added in v1.4.0

func DeviceCopyF16(dst, src DevicePtr, nElems int) error

DeviceCopyF16 copies nElems FP16 values device-to-device.

func DeviceDownloadF32

func DeviceDownloadF32(p DevicePtr, data []float32, elemOff, nElems int) error

DeviceDownloadF32 copies device FP16 into host float32 at elemOff.

func DeviceDownloadSlice

func DeviceDownloadSlice(dev DevicePtr, data []float32, hostOff, n int) error

DeviceDownloadSlice copies device FP16 at dev base into host float32[hostOff:hostOff+n].

func DeviceFree

func DeviceFree(p DevicePtr)

func DeviceLine added in v1.4.0

func DeviceLine() string

DeviceLine formats the GPU startup banner (device name + SM count).

func DeviceMemLine

func DeviceMemLine() string

DeviceMemLine formats total/available GPU memory for startup logging.

func DeviceMemset

func DeviceMemset(p DevicePtr, nElems int)

func DeviceMemsetAt

func DeviceMemsetAt(p DevicePtr, elemOff, nElems int)

DeviceMemsetAt zeros nElems at elemOff in a device buffer.

func DeviceUploadF32

func DeviceUploadF32(p DevicePtr, data []float32, elemOff, nElems int) error

DeviceUploadF32 copies host float32 into device FP16 at elemOff.

func DeviceUploadSlice

func DeviceUploadSlice(dev DevicePtr, data []float32, hostOff, n int) error

DeviceUploadSlice copies host float32[hostOff:hostOff+n] into device FP16 at dev base.

func ElemKernelsAvailable added in v1.4.0

func ElemKernelsAvailable() bool

ElemKernelsAvailable reports whether mul/mask/softmax device kernels loaded.

func GemmDevice

func GemmDevice(aDev, bDev, cDev DevicePtr, M, N, K int) error

GemmDevice computes C[M,N] = A[M,K] @ B[K,N] on device (all offsets zero).

func GemmDeviceAt

func GemmDeviceAt(aDev, bDev, cDev DevicePtr, aOff, bOff, cOff, M, N, K int) error

GemmDeviceAt computes one row-major GEMM slice on device (FP16).

func GemmF32

func GemmF32(A, B, C []float32, M, N, K int) error

GemmF32 computes C[M,N] = A[M,K] @ B[K,N] using FP16 on the GPU.

func GemmStridedBatchedAt

func GemmStridedBatchedAt(aDev, bDev, cDev DevicePtr, aOff, bOff, cOff, M, N, K, batchCount int) error

GemmStridedBatchedAt runs batchCount row-major GEMMs on device (FP16).

func Init

func Init() error

func MulNum added in v1.4.0

func MulNum(ioDev DevicePtr, nElems int, scale float32) error

MulNum multiplies every FP16 element by scale in-place.

func RecordMemBaseline

func RecordMemBaseline()

RecordMemBaseline snapshots device used/free before this process allocates weights.

func Shutdown

func Shutdown()

func SoftmaxBackwardLastDim added in v1.4.0

func SoftmaxBackwardLastDim(outDev, attDev, dAttDev DevicePtr, rows, lastDim int) error

SoftmaxBackwardLastDim writes softmax grads into out (FP16 device buffers).

func SoftmaxLastDim added in v1.4.0

func SoftmaxLastDim(ioDev DevicePtr, rows, lastDim int) error

SoftmaxLastDim runs in-place softmax over the last dimension (FP16 device buffer).

func ZeroCausalUpperSquare added in v1.4.0

func ZeroCausalUpperSquare(ioDev DevicePtr, outer, T int) error

ZeroCausalUpperSquare zeros upper triangle of each [T,T] block.

Types

type DeviceInfo added in v1.4.0

type DeviceInfo struct {
	Name  string
	SMs   int
	Major int
	Minor int
}

DeviceInfo is a snapshot of the active CUDA device.

func QueryDeviceInfo added in v1.4.0

func QueryDeviceInfo() (DeviceInfo, bool)

QueryDeviceInfo reads name, SM count, and compute capability from the driver.

type DevicePtr

type DevicePtr uintptr

DevicePtr is a GPU allocation (FP16 elements).

func DeviceAlloc

func DeviceAlloc(nElems int) (DevicePtr, error)

DeviceAlloc allocates nbytes on the GPU.

func PtrAt

func PtrAt(p DevicePtr, elemOff int) DevicePtr

PtrAt returns p advanced by elemOff float32 elements.

func ScratchGemmOp

func ScratchGemmOp(which, nElems int) (DevicePtr, error)

ScratchGemmOp returns a reusable operand buffer; which must be 0 or 1.

func ScratchGemmOut

func ScratchGemmOut(nElems int) (DevicePtr, error)

ScratchGemmOut returns a reusable GEMM output buffer.

type MemStats

type MemStats struct {
	Used  uint64
	Total uint64
}

MemStats is device memory usage from cudaMemGetInfo.

func DeviceMemStats

func DeviceMemStats() (MemStats, bool)

DeviceMemStats returns used and total GPU memory in bytes.

func (MemStats) Free

func (m MemStats) Free() uint64

Free returns unused device memory in bytes.

func (MemStats) Ok

func (m MemStats) Ok() bool

Ok reports whether Total was read from the driver.

type TimingStats

type TimingStats struct {
	H2D       time.Duration
	Gemm      time.Duration
	D2H       time.Duration
	AllocWait time.Duration
	OOM       uint64 // bytes that failed to allocate this interval
}

TimingStats is cumulative CUDA timing buckets.

func SnapshotTimings

func SnapshotTimings() TimingStats

SnapshotTimings returns current cumulative CUDA timing counters.

func (TimingStats) Delta

func (s TimingStats) Delta(base TimingStats) TimingStats

Delta returns cumulative counter difference s-base.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL