batch

type BatchConfig struct {
	MaxBatchSize   int           `json:"max_batch_size"`
	MaxWaitTime    time.Duration `json:"max_wait_time"`
	QueueSize      int           `json:"queue_size"`
	Workers        int           `json:"workers"`
	RetryOnFailure bool          `json:"retry_on_failure"`
}

type BatchHandler func(ctx context.Context, requests []*Request) []*Response

type BatchProcessor struct {
	// contains filtered or unexported fields
}

type BatchStats struct {
	Submitted int64 `json:"submitted"`
	Batched   int64 `json:"batched"`
	Completed int64 `json:"completed"`
	Failed    int64 `json:"failed"`
	Queued    int   `json:"queued"`
}

type Message struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type Request struct {
	ID       string         `json:"id"`
	Model    string         `json:"model"`
	Messages []Message      `json:"messages"`
	Params   map[string]any `json:"params,omitempty"`
}

type Response struct {
	ID      string `json:"id"`
	Content string `json:"content"`
	Error   error  `json:"error,omitempty"`
	Tokens  int    `json:"tokens"`
}