executor

type BackpressureWatchdog struct {
	// contains filtered or unexported fields
}

type ChatCompletionRequest struct {
	Model             string                 `json:"model"`
	Messages          []Message              `json:"messages"`
	ExtraFields       map[string]interface{} `json:"extra_fields,omitempty"`
	MaxTokens         int                    `json:"max_tokens,omitempty"`
	SkipSpecialTokens bool                   `json:"skip_special_tokens,omitempty"`
	Temperature       float32                `json:"temperature,omitempty"`
	TopP              float32                `json:"top_p,omitempty"`
	TopK              int                    `json:"top_k,omitempty"`
	Seed              int                    `json:"seed,omitempty"`
	N                 int                    `json:"n,omitempty"`
}

type ChatCompletionResponse struct {
	ID      string   `json:"id"`
	Object  string   `json:"object"`
	Created int      `json:"created"`
	Choices []Choice `json:"choices"`
}

type Choice struct {
	Index        int     `json:"index"`
	Message      Message `json:"message"`
	FinishReason string  `json:"finish_reason"`
}

type ExecutorQueueSingleton struct {
	InputChannel chan ExecutionUnit
	// contains filtered or unexported fields
}

type InferenceQueueSingleton struct {
	InputChannel chan InferenceUnit
	// contains filtered or unexported fields
}

type LimitedBuffer struct {
	// contains filtered or unexported fields
}

type Message struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type MyRoundTripper struct {
	// contains filtered or unexported fields
}

type RoundRobinApiCaller struct {
	// contains filtered or unexported fields
}