llama

type Config struct {
	NumCtx       int       // context window in tokens
	NumBatch     int       // prefill batch size
	NumThreads   int       // CPU threads (0 = NumCPU)
	NumGpuLayers int       // layers offloaded to GPU (0 = CPU only)
	TensorSplit  []float32 // multi-GPU split
	FlashAttn    bool
	KVCacheType  string // "", "q8_0", "q4_0"

	PromptFormat         string // profile-declared prompt format, e.g. "chatml" or "llama3"
	PromptTemplateDigest string // digest of the declared/rendered prompt template
	DisableBOS           bool   // false means tokenize the stable prefix with backend BOS handling
}

type ContextManifest = contextasm.ContextManifest

type ContextOverflowError struct {
	Stage            string
	ResidentTokens   int
	AdditionalTokens int
	NumCtx           int
}

type ContextReport struct {
	ResidentTokens  int
	PrefixTokens    int
	NumCtx          int
	AvailableTokens int
	StableByteHash  string
	StableTokenHash string
	ManifestDigest  string
	Manifest        ContextManifest
	Closed          bool
}

type DecodeConfig struct {
	MaxTokens   int
	Temperature *float64
	TopP        *float64
	TopK        int
	Seed        *int
}

type EmbedFunc func(ctx context.Context, modelPath string, cfg Config, input string) ([]float64, error)

type ManifestMismatchError = contextasm.ManifestMismatchError

type ManifestSegment = contextasm.ManifestSegment

type PrefixInput struct {
	Text     string
	Manifest ContextManifest
}

type PrefixStatus struct {
	ReusedTokens    int // tokens kept warm from the resident prefix
	PrefilledTokens int // divergent tail that had to be (re)prefilled
	DroppedTokens   int // old resident tokens removed before prefill
	PrefixTokens    int // total prefix tokens now resident
	ResidentTokens  int // total resident tokens after EnsurePrefix
	AvailableTokens int // remaining context capacity
	StableByteHash  string
	StableTokenHash string
	ManifestDigest  string
}

type Session interface {
	// EnsurePrefix makes the resident KV equal `prefix`, reusing the longest
	// already-resident matching token prefix and prefilling only the divergent
	// tail (this also drops any previous suffix and generated tokens).
	EnsurePrefix(ctx context.Context, prefix PrefixInput) (PrefixStatus, error)

	// PrefillSuffix prefills the volatile suffix (diff / test output / user turn)
	// after the stable prefix, leaving the stable KV untouched.
	PrefillSuffix(ctx context.Context, suffix SuffixInput) (SuffixStatus, error)

	// Decode streams generated text from the current resident state.
	Decode(ctx context.Context, cfg DecodeConfig) (<-chan StreamChunk, error)

	// ExplainContext reports the resident context for observability.
	ExplainContext() ContextReport

	// Close releases the session's resources.
	Close() error
}

type SessionFactory func(modelPath string, cfg Config) (Session, error)

type StreamChunk struct {
	Text  string
	Error error
}

type SuffixInput struct {
	Text     string
	Manifest ContextManifest
}

type SuffixStatus struct {
	SuffixTokens    int
	PrefixTokens    int
	ResidentTokens  int
	AvailableTokens int
	ManifestDigest  string
}

type TokenizeFunc = contextasm.TokenizeFunc

type UnsupportedFeatureError struct {
	Feature string
}