llm

type EmbeddingsResult struct {
	// Embeddings are the embeddings generated by the request.
	Embeddings [][]float32
	// Usage is usage related to an embeddings generation request.
	Usage *EmbeddingsUsage
}

type EmbeddingsUsage struct {
	// PromptTokenCount is number of tokens in the prompt.
	PromptTokenCount int
}

type InferencingParams struct {
	// MaxTokens is the maximum tokens that should be inferred.
	// Default: 100
	//
	// Note: the backing implementation may return less tokens.
	MaxTokens int32
	// RepeatPenalty is the amount the model should avoid repeating tokens.
	// Default: 1.1
	RepeatPenalty float32
	// RepeatPenaltyLastNTokenCount the number of tokens the model should
	// apply the repeat penalty to.
	// Default: 64
	RepeatPenaltyLastNTokenCount int32
	// Temperature is the randomness with which the next token is selected.
	// Default: 0.8
	Temperature float32
	// TopK is the number of possible next tokens the model will choose from.
	// Default: 40
	TopK int32
	// TopP is the probability total of next tokens the model will choose
	// from.
	// Default: 0.9
	TopP float32
}

type InferencingResult struct {
	// Text is the text generated by the model.
	Text string
	// Usage is information about the inferencing request.
	Usage *InferencingUsage
}

type InferencingUsage struct {
	// PromptTokenCount is the number of tokens in the prompt.
	PromptTokenCount int
	// GeneratedTokenCount is the number of tokens generated by the
	// inferencing operation.
	GeneratedTokenCount int
}