server

type ControllerConfig struct {
	InferenceObjectiveGV    schema.GroupVersion
	InferenceModelRewriteGV schema.GroupVersion
	// contains filtered or unexported fields
}

type ExtProcServerRunner struct {
	GrpcPort                         int
	GKNN                             common.GKNN
	ControllerCfg                    ControllerConfig
	Datastore                        datastore.Datastore
	SecureServing                    bool
	HealthChecking                   bool
	CertPath                         string
	EnableCertReload                 bool
	RefreshPrometheusMetricsInterval time.Duration
	MetricsStalenessThreshold        time.Duration
	Director                         *requestcontrol.Director
	ParserRegistry                   *handlers.ParserRegistry
	SaturationDetector               fwkfc.SaturationDetector
	PriorityBandControlPlane         contracts.PriorityBandControlPlane
	GRPCMaxRecvMsgSize               int
	GRPCMaxSendMsgSize               int
	EnableGRPCStreamMetrics          bool
}

type MetricsHandlerRegistrar interface {
	AddMetricsServerExtraHandler(path string, handler http.Handler) error
}

type Options struct {
	//
	// ext_proc configuration.
	//
	GRPCPort              int    // gRPC port used for communicating with Envoy proxy. (TODO: uint16?)
	EnableLeaderElection  bool   // Enables leader election for high availability
	GRPCMaxRecvMsgSize    int    // Maximum size of a gRPC message to receive (parsed bytes).
	GRPCMaxSendMsgSize    int    // Maximum size of a gRPC message to send (parsed bytes).
	GRPCMaxRecvMsgSizeStr string // Raw string value from CLI flag for receive limit.
	GRPCMaxSendMsgSizeStr string // Raw string value from CLI flag for send limit.
	//
	// InferencePool.
	//
	PoolGroup     string // Kubernetes resource group of the InferencePool this Endpoint Picker is associated with.
	PoolNamespace string // Namespace of the InferencePool this Endpoint Picker is associated with.
	PoolName      string // Name of the InferencePool this Endpoint Picker is associated with.
	//
	// Endpoints (in lieu of using an InferencePool for service discovery).
	//
	EndpointSelector            string // Selector to filter model server pods on, only 'key=value' pairs are supported. (TODO: k8s.Selector, pflag.StringSlice?)
	EndpointTargetPorts         []int  // Target ports of model server pods.
	DisableEndpointSubsetFilter bool   // Disables respecting destination endpoint subset metadata in EPP.
	//
	// MSP metrics scraping.
	//
	ModelServerMetricsScheme         string        // Protocol scheme used in scraping metrics from endpoints.
	ModelServerMetricsPath           string        // URL path used in scraping metrics from endpoints.
	ModelServerMetricsPort           int           // Port to scrape metrics from endpoints. (TODO: Deprecated, uint16)
	ModelServerMetricsHTTPSInsecure  bool          // Disable certificate verification when using 'https' scheme for 'model-server-metrics-scheme'.
	RefreshMetricsInterval           time.Duration // Interval to refresh metrics.
	RefreshPrometheusMetricsInterval time.Duration // Interval to flush Prometheus metrics.
	MetricsStalenessThreshold        time.Duration // Duration after which metrics are considered stale.
	TotalQueuedRequestsMetric        string        // Prometheus metric specification for the number of queued requests.
	TotalRunningRequestsMetric       string        // Prometheus metric specification for the number of running requests.
	KVCacheUsagePercentageMetric     string        // Prometheus metric specification for the fraction of KV-cache blocks currently in use.
	LoRAInfoMetric                   string        // Prometheus metric specification for the LoRA info metrics.
	CacheInfoMetric                  string        // Prometheus metric specification for the cache info metrics.
	//
	// Diagnostics.
	//
	logging.LoggingOptions         // Logging configuration.
	Tracing                 bool   // Enables emitting traces.
	HealthChecking          bool   // Enables health checking.
	MetricsPort             int    // The metrics port exposed by EPP. (TODO: uint16)
	GRPCHealthPort          int    // The port used for gRPC liveness and readiness probes. (TODO: uint16)
	EnablePprof             bool   // Enables pprof handlers.
	CertPath                string // The path to the certificate for secure serving.
	EnableCertReload        bool   // Enables certificate reloading of the certificates specified in --cert-path.
	SecureServing           bool   // Enables secure serving.
	MetricsEndpointAuth     bool   // Enables authentication and authorization of the metrics endpoint.
	EnableGRPCStreamMetrics bool   // Enables ext_proc gRPC stream metrics (in-flight gauge, hold duration, completions counter by code).
	//
	// Configuration.
	//
	ConfigFile string // The path to the configuration file.
	ConfigText string // The configuration specified as text, in lieu of a file.
	// contains filtered or unexported fields
}