Documentation
¶
Index ¶
- Constants
- Variables
- func CanUseGPU(context.Context, string) (bool, error)
- func GetContextSize(modelCfg types.Config, backendCfg *inference.BackendConfiguration) *int32
- func GetDesiredServerVersion() string
- func GetReasoningBudget(backendCfg *inference.BackendConfiguration) *int32
- func New(log logging.Logger, modelManager *models.Manager, serverLog logging.Logger, ...) (inference.Backend, error)
- func SetDesiredServerVersion(version string)
- type Config
Constants ¶
View Source
const (
// Name is the backend name.
Name = "llama.cpp"
)
View Source
const UnlimitedContextSize = -1
Variables ¶
Functions ¶
func GetContextSize ¶
func GetContextSize(modelCfg types.Config, backendCfg *inference.BackendConfiguration) *int32
func GetDesiredServerVersion ¶
func GetDesiredServerVersion() string
func GetReasoningBudget ¶ added in v1.0.7
func GetReasoningBudget(backendCfg *inference.BackendConfiguration) *int32
func New ¶
func New( log logging.Logger, modelManager *models.Manager, serverLog logging.Logger, vendoredServerStoragePath string, updatedServerStoragePath string, conf config.BackendConfig, ) (inference.Backend, error)
New creates a new llama.cpp-based backend.
func SetDesiredServerVersion ¶
func SetDesiredServerVersion(version string)
Types ¶
type Config ¶
type Config struct {
// Args are the base arguments that are always included.
Args []string
}
Config is the configuration for the llama.cpp backend.
func NewDefaultLlamaCppConfig ¶
func NewDefaultLlamaCppConfig() *Config
NewDefaultLlamaCppConfig creates a new LlamaCppConfig with default values.
func (*Config) GetArgs ¶
func (c *Config) GetArgs(bundle types.ModelBundle, socket string, mode inference.BackendMode, config *inference.BackendConfiguration) ([]string, error)
GetArgs implements BackendConfig.GetArgs.
Source Files
Click to show internal directories.
Click to hide internal directories.