ovsession

type ChatMessage struct {
	Role       string
	Content    string
	ToolCalls  string `json:",omitempty"`
	ToolCallID string `json:",omitempty"`
}

type EmbedSession struct{}

type GenAIConfig struct {
	Device                      string
	KVCachePrecision            string
	CacheSize                   int
	DynamicSplitFuse            *bool
	EnablePrefixCaching         *bool
	UseSparseAttention          *bool
	NumLastDenseTokensInPrefill int
	XAttentionThreshold         float32
	XAttentionBlockSize         int
	XAttentionStride            int
}

type GenAIResult struct {
	Text       string
	ParsedJSON string
	Metrics    PipelineMetrics
}

type GenAISession struct{}

type GenerateOptions struct {
	MaxNewTokens     int
	Temperature      *float64
	TopP             *float64
	StructuredOutput StructuredOutput
	ParserProtocols  []string
}

type PipelineMetrics struct {
	Requests          uint64
	ScheduledRequests uint64
	CacheUsage        float32
	MaxCacheUsage     float32
	AvgCacheUsage     float32
	InferenceDuration float32
	CacheSizeInBytes  uint64
}

type Session struct{}

type StreamChunk struct {
	Text  string
	Error error
}

type StructuredOutput struct {
	Protocol string
	Payload  string
}