scheduling

type Endpoint interface {
	GetMetadata() *fwkdl.EndpointMetadata
	GetMetrics() *fwkdl.Metrics
	String() string
	Get(string) (fwkdl.Cloneable, bool)
	Put(string, fwkdl.Cloneable)
	Keys() []string
	Clone() fwkdl.AttributeMap
}

type Filter interface {
	plugin.Plugin
	Filter(ctx context.Context, request *InferenceRequest, pods []Endpoint) []Endpoint
}

type InferenceRequest struct {
	// RequestID is the Envoy generated Id for the request being processed
	RequestID string
	// TargetModel is the final target model after traffic split.
	TargetModel string
	// Data contains the request-body fields that we parse out as user input.
	Body *fwkrh.InferenceRequestBody
	// Headers is a map of the request headers.
	Headers map[string]string
	// Request Objective
	Objectives RequestObjectives
	// FairnessID is the identity used by the flow control system to group requests into fairness queues.
	FairnessID string
	// RequestSizeBytes is the size of the raw request body in bytes when available.
	RequestSizeBytes int
	// SchedulingResult captures the scheduling decisions made during the cycle.
	SchedulingResult *SchedulingResult
	// contains filtered or unexported fields
}

type Modality = fwkrh.Modality

type MultiModalFeature = fwkrh.MultiModalFeature

type Picker interface {
	plugin.Plugin
	Pick(ctx context.Context, scoredPods []*ScoredEndpoint) *ProfileRunResult
}

type ProfileHandler interface {
	plugin.Plugin
	// Pick selects the SchedulingProfiles to run from a list of candidate profiles, while taking into consideration the request properties
	// and the previously executed SchedluderProfile cycles along with their results.
	Pick(ctx context.Context, request *InferenceRequest, profiles map[string]SchedulerProfile,
		profileResults map[string]*ProfileRunResult) map[string]SchedulerProfile

	// ProcessResults handles the outcome of the profile runs after all profiles ran.
	// It may aggregate results, log test profile outputs, or apply custom logic. It specifies in the SchedulingResult the
	// key of the primary profile that should be used to get the request selected destination.
	// When a profile run fails, its result in the profileResults map is nil.
	ProcessResults(ctx context.Context, request *InferenceRequest,
		profileResults map[string]*ProfileRunResult) (*SchedulingResult, error)
}

type ProfileRunResult struct {
	TargetEndpoints []Endpoint
}

type RequestObjectives struct {
	Priority int
}

type SchedulerProfile interface {
	Run(ctx context.Context, request *InferenceRequest, candidateEndpoints []Endpoint) (*ProfileRunResult, error)
}

type SchedulingResult struct {
	ProfileResults     map[string]*ProfileRunResult
	PrimaryProfileName string
}

type ScoredEndpoint struct {
	Endpoint
	Score float64
}

type Scorer interface {
	plugin.Plugin
	Category() ScorerCategory
	Score(ctx context.Context, request *InferenceRequest, pods []Endpoint) map[Endpoint]float64
}

type ScorerCategory string

type TokenizedPrompt = fwkrh.TokenizedPrompt