Versions in this module Expand all Collapse all v0 v0.3.0 Aug 25, 2025 Changes in this version + type TokenizerNode struct + func NewTokenizerNode(vocab map[string]int32, unkTokenID int32) *TokenizerNode + func (n *TokenizerNode) Attributes() map[string]any + func (n *TokenizerNode) Backward(ctx context.Context, mode types.BackwardMode, outputGradient tensor.Tensor) ([]tensor.Tensor, error) + func (n *TokenizerNode) Forward(ctx context.Context, inputs ...tensor.Tensor) (tensor.Tensor, error) + func (n *TokenizerNode) OpType() string + func (n *TokenizerNode) OutputShape() []int
Go to main page