proxy

type APIType int

type AllowlistValidator struct {
	// contains filtered or unexported fields
}

type Config struct {
	// Port is the port the sidecar is listening on.
	Port string
	// DecoderURL is the URL of the local decoder (vLLM) instance.
	DecoderURL *url.URL

	// KVConnector is the name of the KV protocol between prefiller and decoder.
	KVConnector string
	// ECConnector is the name of the EC protocol between encoder and prefiller (for EPD mode).
	// If empty, encoder stage is skipped.
	ECConnector string
	// DataParallelSize is the value passed to the vLLM server's --DATA_PARALLEL-SIZE argument.
	DataParallelSize int

	// MaxIdleConnsPerHost controls how many idle keep-alive connections are
	// maintained per host for the reverse proxy transports. Set this to at
	// least the expected concurrency level to avoid connection churn.
	MaxIdleConnsPerHost int

	// EnablePrefillerSampling configures the proxy to randomly choose from the set
	// of provided prefill hosts instead of always using the first one.
	EnablePrefillerSampling bool

	// PrefillMaxRetries is the number of additional attempts when a prefill
	// request fails with a 5xx error (e.g. connection reset → 502).
	// 0 means no retries (original behavior).
	PrefillMaxRetries int
	// PrefillRetryBackoff is the delay between prefill retry attempts.
	PrefillRetryBackoff time.Duration

	// UseTLSForPrefiller indicates whether to use TLS when sending requests to prefillers.
	UseTLSForPrefiller bool
	// UseTLSForDecoder indicates whether to use TLS when sending requests to the decoder.
	UseTLSForDecoder bool
	// UseTLSForEncoder indicates whether to use TLS when sending requests to encoders.
	UseTLSForEncoder bool
	// InsecureSkipVerifyForPrefiller configures the proxy to skip TLS verification for requests to the prefiller.
	InsecureSkipVerifyForPrefiller bool
	// InsecureSkipVerifyForEncoder configures the proxy to skip TLS verification for requests to the encoder.
	InsecureSkipVerifyForEncoder bool
	// InsecureSkipVerifyForDecoder configures the proxy to skip TLS verification for requests to the decoder.
	InsecureSkipVerifyForDecoder bool

	// SecureServing enables TLS for the sidecar server itself.
	SecureServing bool
	// CertPath is the path to TLS certificates for the sidecar server.
	CertPath string

	// MooncakeBootstrapPort is the port used to query the Mooncake bootstrap endpoint on prefill pods.
	MooncakeBootstrapPort int

	// EnableSSRFProtection enables SSRF protection using InferencePool allowlisting.
	EnableSSRFProtection bool
	// InferencePoolNamespace is the Kubernetes namespace of the InferencePool to watch.
	InferencePoolNamespace string
	// InferencePoolName is the name of the InferencePool to watch.
	InferencePoolName string
	// PoolGroup is the API group of the InferencePool resource.
	PoolGroup string

	// DecodeChunkSize is the token budget per decode chunk.
	// Chunked decode is enabled when this value is > 0.
	DecodeChunkSize int

	// Tracing enables OpenTelemetry tracing.
	Tracing bool
	// MoRIIOWriteMode enables MoRI-IO WRITE-mode: the sidecar populates the
	// prefill leg's kv_transfer_params so the prefill engine pushes KV to decode
	// via RDMA Write. Only meaningful with --kv-connector=nixlv2.
	MoRIIOWriteMode bool
	// MoRIIODecodeNotifyPort is the decode pod's base MoRI-IO notify port.
	MoRIIODecodeNotifyPort int
	// MoRIIODecodeHandshakePort is the decode pod's base MoRI-IO handshake port.
	MoRIIODecodeHandshakePort int
	// MoRIIODecodePodIP is decode's routable pod IP, used as the prefill leg's
	// remote_host so prefill handshakes with decode (not itself). Must not be
	// localhost; typically the POD_IP downward-API value.
	MoRIIODecodePodIP string

	// MoRIIOParallelDispatch fires the prefill and decode legs concurrently,
	// synthesising decode's kv_transfer_params from config instead of reading
	// them from the prefill response. Requires MoRIIOWriteMode.
	MoRIIOParallelDispatch bool
	// MoRIIOPrefillHandshakePort is the prefill pod's base MoRI-IO handshake port.
	MoRIIOPrefillHandshakePort int
	// MoRIIOPrefillNotifyPort is the prefill pod's base MoRI-IO notify port.
	MoRIIOPrefillNotifyPort int
	// MoRIIOTPSize is the tensor-parallel size of the engines, echoed into
	// kv_transfer_params[tp_size] in parallel-dispatch mode.
	MoRIIOTPSize int
	// MoRIIODPSize is the data-parallel world size, emitted as remote_dp_size on
	// both legs. Wide-EP (TP=1, DP>1) must set this so the decode connector
	// registers RDMA notifies against every DP rank; 1 leaves the wire unchanged.
	MoRIIODPSize int

	// MoRIIORemoteHosts is the ordered list of prefill-side pod IPs across which
	// vLLM fans out its per-DP-rank handshake, emitted as the decode leg's
	// remote_hosts. host[i] serves DP ranks [i*MoRIIODPSizeLocal, (i+1)*...).
	// Empty disables fan-out (single-host fallback).
	MoRIIORemoteHosts []string
	// MoRIIODPSizeLocal is the per-pod DP size, mapping a global DP rank to a pod
	// via pod_idx = dp_rank / MoRIIODPSizeLocal. 0 means single-pod.
	MoRIIODPSizeLocal int
	// MoRIIODecodeHosts is the decode-side counterpart of MoRIIORemoteHosts,
	// emitted as the prefill leg's remote_hosts. A multi-pod deployment sets
	// both; the lists must use opposite sides or every cross-pod handshake hangs.
	MoRIIODecodeHosts []string
}

type Options struct {
	// Config holds the processed runtime configuration (populated by Complete()).
	// Fields with direct CLI flags are bound here via embedding; derived fields are set in Complete().
	Config
	// contains filtered or unexported fields
}

type Server struct {
	// contains filtered or unexported fields
}