llamacpp

package

v0.10.0 Latest Latest Go to latest Published: Nov 14, 2024 License: MIT Imports: 5 Imported by: 3

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/distantmagic/paddler

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
type LlamaCppClient
type LlamaCppCompletionRequest
type LlamaCppConfiguration
- func (self *LlamaCppConfiguration) String() string
type LlamaCppHealthStatus
type LlamaCppHealthStatusCode
type LlamaCppSlotStatus
type LlamaCppSlotsAggregatedStatus
- func (self *LlamaCppSlotsAggregatedStatus) CopyFrom(other *LlamaCppSlotsAggregatedStatus)

Constants ¶

View Source

const (
	CompletionDataPrefix = "data: "
)

Variables ¶

View Source

var (
	ErrorNon200Response = errors.New("Non-200 response from llama.cpp")
)

Functions ¶

This section is empty.

Types ¶

type LlamaCppClient ¶

type LlamaCppClient struct {
	HttpClient            *http.Client
	LlamaCppConfiguration *LlamaCppConfiguration
}

func (*LlamaCppClient) GetHealth ¶

func (self *LlamaCppClient) GetHealth(
	ctx context.Context,
	responseChannel chan<- LlamaCppHealthStatus,
)

func (*LlamaCppClient) GetSlots ¶ added in v0.7.0

func (self *LlamaCppClient) GetSlots(
	ctx context.Context,
	responseChannel chan<- LlamaCppSlotStatus,
)

func (*LlamaCppClient) GetSlotsAggregatedStatus ¶ added in v0.7.0

func (self *LlamaCppClient) GetSlotsAggregatedStatus(
	ctx context.Context,
	responseChannel chan<- LlamaCppSlotsAggregatedStatus,
)

type LlamaCppCompletionRequest ¶

type LlamaCppCompletionRequest struct {
	JsonSchema any    `json:"json_schema"`
	NPredict   int    `json:"n_predict"`
	Prompt     string `json:"prompt"`
	Stream     bool   `json:"stream"`
}

type LlamaCppConfiguration ¶

type LlamaCppConfiguration struct {
	HttpAddress *netcfg.HttpAddressConfiguration `json:"http_address"`
	ApiKey      string
}

func (*LlamaCppConfiguration) String ¶

func (self *LlamaCppConfiguration) String() string

type LlamaCppHealthStatus ¶

type LlamaCppHealthStatus struct {
	Error        error                    `json:"-"`
	ErrorMessage string                   `json:"error_message,omitempty"`
	Status       LlamaCppHealthStatusCode `json:"status"`
}

type LlamaCppHealthStatusCode ¶

type LlamaCppHealthStatusCode string

const (
	Error           LlamaCppHealthStatusCode = "error"
	LoadingModel    LlamaCppHealthStatusCode = "loading model"
	NoSlotAvailable LlamaCppHealthStatusCode = "no slot available"
	Ok              LlamaCppHealthStatusCode = "ok"
)

type LlamaCppSlotStatus ¶ added in v0.7.0

type LlamaCppSlotStatus struct {
	Error        error  `json:"-"`
	ErrorMessage string `json:"error_message,omitempty"`
	IsProcessing bool   `json:"is_processing"`
}

type LlamaCppSlotsAggregatedStatus ¶ added in v0.7.0

type LlamaCppSlotsAggregatedStatus struct {
	Error           error                    `json:"-"`
	ErrorMessage    string                   `json:"error_message,omitempty"`
	Status          LlamaCppHealthStatusCode `json:"status"`
	SlotsIdle       int                      `json:"slots_idle"`
	SlotsProcessing int                      `json:"slots_processing"`
}

func (*LlamaCppSlotsAggregatedStatus) CopyFrom ¶ added in v0.7.1

func (self *LlamaCppSlotsAggregatedStatus) CopyFrom(other *LlamaCppSlotsAggregatedStatus)

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL