Documentation
¶
Index ¶
- func AddCausalMaskExtendIn(a *tensor.Tensor, cacheLen int, neg float32)
- func AddCausalMaskIn(a *tensor.Tensor, neg float32)
- func AnyStopInTopK(logits *tensor.Tensor, b int, stopIDs map[int]bool, k int) bool
- func ArgMaxLastToken(logits *tensor.Tensor, b int) int
- func CopyHFLinear(dst *neural.Linear, src *tensor.Tensor, prog *CopyProgress)
- func CopyHFWeight(dst *tensor.Tensor, src *tensor.Tensor, prog *CopyProgress)
- func CopyTensor(dst, src *tensor.Tensor, prog *CopyProgress)
- func CopyTransposed(dst *tensor.Tensor, src *tensor.Tensor, prog *CopyProgress)
- func EnsureFile(remote, local string, size int64, u ui.UI)
- func LoadHFLinear(l *neural.Linear, sd *SafeTensors, keyW, keyB string, prog *CopyProgress) error
- func LoadLinearDirect(l *neural.Linear, sd *SafeTensors, keyW, keyB string, prog *CopyProgress) error
- func LoadTensor(dst *tensor.Tensor, sd *SafeTensors, key string, prog *CopyProgress) error
- func PackQKVLastDimIn(dst, dQ, dK, dV *tensor.Tensor)
- func SampleTopKLast(logits *tensor.Tensor, b int, temperature float32, k int, rng *rand.Rand) int
- func SampleTopKLastChat(logits *tensor.Tensor, b int, temperature float32, k int, stopIDs map[int]bool, ...) int
- func SoftmaxBackward(att, dAtt *tensor.Tensor) *tensor.Tensor
- func ZeroCausalUpperTriangleIn(S *tensor.Tensor)
- type AllocProgress
- type Attention
- type CopyProgress
- type KVCache
- type KVCacheBlock
- type PositionEmbeddings
- type SafeTensors
- type Top
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func AddCausalMaskExtendIn ¶ added in v1.3.0
AddCausalMaskExtendIn masks keys past cacheLen+t for query row t in [*, T, cacheLen+T].
func AddCausalMaskIn ¶
func AnyStopInTopK ¶ added in v1.3.0
AnyStopInTopK reports whether a stop id is among the top-k logits.
func ArgMaxLastToken ¶
ArgMaxLastToken returns the highest logit at the last sequence position.
func CopyHFLinear ¶
func CopyHFLinear(dst *neural.Linear, src *tensor.Tensor, prog *CopyProgress)
CopyHFLinear loads HF nn.Linear weight [out,in] into local [in,out].
func CopyHFWeight ¶
func CopyHFWeight(dst *tensor.Tensor, src *tensor.Tensor, prog *CopyProgress)
CopyHFWeight transposes and copies a HF weight matrix into dst.
func CopyTensor ¶
func CopyTensor(dst, src *tensor.Tensor, prog *CopyProgress)
CopyTensor copies src into dst (shapes must match).
func CopyTransposed ¶
func CopyTransposed(dst *tensor.Tensor, src *tensor.Tensor, prog *CopyProgress)
CopyTransposed copies src^T into dst.
func EnsureFile ¶
EnsureFile downloads remote to local when missing or size mismatch.
func LoadHFLinear ¶
func LoadHFLinear(l *neural.Linear, sd *SafeTensors, keyW, keyB string, prog *CopyProgress) error
LoadHFLinear copies HuggingFace nn.Linear weights with transpose.
func LoadLinearDirect ¶
func LoadLinearDirect(l *neural.Linear, sd *SafeTensors, keyW, keyB string, prog *CopyProgress) error
LoadLinearDirect copies GPT-2 Conv1D weights without transpose.
func LoadTensor ¶
func LoadTensor(dst *tensor.Tensor, sd *SafeTensors, key string, prog *CopyProgress) error
LoadTensor copies one tensor from safetensors into dst.
func PackQKVLastDimIn ¶
func SampleTopKLast ¶
SampleTopKLast samples from the last position with temperature and top-k.
func SampleTopKLastChat ¶ added in v1.3.0
func SampleTopKLastChat( logits *tensor.Tensor, b int, temperature float32, k int, stopIDs map[int]bool, allowStop bool, recent []int, repPenalty float32, rng *rand.Rand, ) int
SampleTopKLastChat samples with optional stop masking and repetition penalty.
func ZeroCausalUpperTriangleIn ¶
ZeroCausalUpperTriangleIn sets gradient entries above diagonal to zero. S shape: [BH, T, T]
Types ¶
type AllocProgress ¶ added in v1.3.0
type AllocProgress struct {
// contains filtered or unexported fields
}
AllocProgress tracks tensor allocation progress during model construction.
func NewAllocProgress ¶ added in v1.3.0
func NewAllocProgress(u ui.UI, totalBytes int64) *AllocProgress
NewAllocProgress creates a progress reporter for allocating totalBytes of model tensors.
func (*AllocProgress) Done ¶ added in v1.3.0
func (p *AllocProgress) Done()
Done finishes the allocation progress line.
func (*AllocProgress) ReportAlloc ¶ added in v1.3.0
func (p *AllocProgress) ReportAlloc(t *tensor.Tensor)
type Attention ¶
type Attention struct {
D int
Heads int
HeadDim int
Causal bool
CAttn *neural.Linear // D -> 3D
CProj *neural.Linear // D -> D
// contains filtered or unexported fields
}
Attention is multi-head self-attention for autoregressive LLM blocks.
B = batch size T = sequence length D = model dimension
func NewAttention ¶
func (*Attention) ForwardNext ¶
ForwardNext runs one decode step and appends K/V into cache.
type CopyProgress ¶ added in v1.3.0
type CopyProgress struct {
// contains filtered or unexported fields
}
CopyProgress tracks weight copy progress for model loading.
func NewCopyProgress ¶ added in v1.3.0
func NewCopyProgress(u ui.UI, totalBytes int64) *CopyProgress
NewCopyProgress creates a progress reporter for copying totalBytes into model tensors.
func (*CopyProgress) Done ¶ added in v1.3.0
func (p *CopyProgress) Done()
Done finishes the progress line.
func (*CopyProgress) ReportTensor ¶ added in v1.3.0
func (p *CopyProgress) ReportTensor(t *tensor.Tensor)
ReportTensor counts t toward copy progress (e.g. after manual weight assembly).
type KVCache ¶
type KVCache struct {
Len int
Blocks []*KVCacheBlock
}
KVCache stores per-layer key/value tensors for autoregressive decoding.
func NewKVCache ¶
NewKVCache keeps old API for GPT-style callers.
func NewKVCacheFromEmbedding ¶
NewKVCacheFromEmbedding creates cache from embedding size and head count.
func NewKVCacheWithHeadDim ¶
NewKVCacheWithHeadDim creates cache with explicit head count and head dim.
type KVCacheBlock ¶
type KVCacheBlock struct {
K *tensor.Tensor // [B,H,ContextSize,hd]
V *tensor.Tensor // [B,H,ContextSize,hd]
// contains filtered or unexported fields
}
func (*KVCacheBlock) Get ¶
func (block *KVCacheBlock) Get(ct *tensor.Tensor) *tensor.Tensor
Get returns cached prefix [BH, Len+1, hd] (single-step decode).
type PositionEmbeddings ¶
type PositionEmbeddings struct {
Emb *neural.Embeddings
}
PositionEmbeddings maps positions 0..T-1 to embedding vectors.
func NewPositionEmbeddings ¶
func NewPositionEmbeddings(count, size int) *PositionEmbeddings
type SafeTensors ¶
type SafeTensors struct {
// contains filtered or unexported fields
}
SafeTensors holds tensors loaded from a HuggingFace safetensors file.
func LoadSafeTensors ¶
func LoadSafeTensors(path string, u ui.UI) (*SafeTensors, error)
LoadSafeTensors reads all tensors from a single safetensors weight file.
func LoadSafeTensorsFromDir ¶ added in v1.3.0
func LoadSafeTensorsFromDir(dir string, u ui.UI) (*SafeTensors, error)
LoadSafeTensorsFromDir loads weights from dir/model.safetensors or sharded index + parts.
func (*SafeTensors) Free ¶
func (s *SafeTensors) Free()
Source Files