engine

type Executor struct {
	// contains filtered or unexported fields
}

type InferenceConfig struct {
	ModelPath     string  `json:"model_path"`
	Prompt        string  `json:"prompt"`
	SystemPrompt  string  `json:"system_prompt"`
	Temperature   float64 `json:"temperature"`    // 0.0 to 2.0
	TopP          float64 `json:"top_p"`          // 0.0 to 1.0
	TopK          int     `json:"top_k"`          // 0 to 100
	RepeatPenalty float64 `json:"repeat_penalty"` // 1.0 to 2.0
	MaxTokens     int     `json:"max_tokens"`     // -1 for infinite
	Threads       int     `json:"threads"`        // number of CPU threads
}

type ServerRequest struct {
	Prompt        string  `json:"prompt"`
	NPredict      int     `json:"n_predict"`
	Temperature   float64 `json:"temperature"`
	TopP          float64 `json:"top_p"`
	TopK          int     `json:"top_k"`
	RepeatPenalty float64 `json:"repeat_penalty"`
	Stream        bool    `json:"stream"`
}

type ServerResponse struct {
	Content string `json:"content"`
	Stop    bool   `json:"stop"`
}