gateway

type AttemptObserver ¶

type AttemptObserver func()

AttemptObserver is invoked after a failed provider attempt is recorded, so the audit/live layer can surface it before the overall request finishes (e.g. a failed primary while failover is still in flight).

type AttemptRecorder ¶

type AttemptRecorder struct {
	// contains filtered or unexported fields
}

func AttemptRecorderFromContext(ctx context.Context) *AttemptRecorder

type BatchConfig ¶

type BatchConfig struct {
	Provider                             core.RoutableProvider
	ModelResolver                        ModelResolver
	ModelAuthorizer                      ModelAuthorizer
	InputFileProviderResolver            BatchInputFileProviderResolver
	WorkflowPolicyResolver               WorkflowPolicyResolver
	BatchRequestPreparer                 BatchRequestPreparer
	BatchStore                           batchstore.Store
	CleanupPreparedBatchInputFile        func(context.Context, string, string)
	CleanupStoredBatchRewrittenInputFile func(context.Context, *batchstore.StoredBatch) bool
	UsageLogger                          usage.LoggerInterface
	PricingResolver                      usage.PricingResolver
	BudgetEnforcer                       func(context.Context) error
}

BatchConfig configures native batch orchestration.

type BatchCreateResult ¶

type BatchCreateResult struct {
	Batch        *core.BatchResponse
	Workflow     *core.Workflow
	ProviderType string
}

BatchCreateResult is the result of creating a native batch.

type BatchExecutionSelection ¶

type BatchExecutionSelection struct {
	ProviderType string
	Selector     core.WorkflowSelector
}

BatchExecutionSelection captures the provider and workflow selector for a native batch.

func DetermineBatchExecutionSelectionWithAuthorizerAndInputFileResolver(
	ctx context.Context,
	provider core.RoutableProvider,
	resolver ModelResolver,
	authorizer ModelAuthorizer,
	inputFileProviderResolver BatchInputFileProviderResolver,
	req *core.BatchRequest,
) (BatchExecutionSelection, error)

type BatchInputFileProviderResolver ¶

type BatchInputFileProviderResolver interface {
	ResolveBatchInputFileProvider(ctx context.Context, fileID string) (providerType string, ok bool, err error)
}

BatchInputFileProviderResolver resolves provider ownership for an uploaded batch input file.

type BatchListParams ¶

type BatchListParams struct {
	Limit int
	After string
}

BatchListParams contains native batch list pagination options.

type BatchMeta ¶

type BatchMeta struct {
	RequestID string
	Endpoint  core.EndpointDescriptor
	Workflow  *core.Workflow
}

BatchMeta carries transport-derived metadata into native batch use cases.

type BatchOrchestrator ¶

type BatchOrchestrator struct {
	// contains filtered or unexported fields
}

BatchOrchestrator owns native batch lifecycle behavior independent of HTTP.

func NewBatchOrchestrator(cfg BatchConfig) *BatchOrchestrator

func (o *BatchOrchestrator) Cancel(ctx context.Context, id string) (*BatchResult, error)

func (o *BatchOrchestrator) Create(ctx context.Context, req *core.BatchRequest, meta BatchMeta) (*BatchCreateResult, error)

func (o *BatchOrchestrator) Get(ctx context.Context, id string) (*BatchResult, error)

func (o *BatchOrchestrator) List(ctx context.Context, params BatchListParams) (*core.BatchListResponse, error)

func (o *BatchOrchestrator) Results(ctx context.Context, id, fallbackRequestID string) (*BatchResultsResult, error)

type BatchRequestPreparer ¶

type BatchRequestPreparer interface {
	PrepareBatchRequest(ctx context.Context, providerType string, req *core.BatchRequest) (*core.BatchRewriteResult, error)
}

BatchRequestPreparer rewrites a native batch request before provider submission. This keeps batch-specific policy out of provider decorators.

type BatchResult ¶

type BatchResult struct {
	Batch        *core.BatchResponse
	ProviderType string
}

BatchResult is the result of a single batch lifecycle operation.

type BatchResultsResult ¶

type BatchResultsResult struct {
	Response     *core.BatchResultsResponse
	ProviderType string
}

BatchResultsResult is the result of retrieving batch output items.

type ChatCompletionResult ¶

type ChatCompletionResult struct {
	Response *core.ChatResponse
	Meta     ExecutionMeta
}

ChatCompletionResult is the non-streaming chat completion result.

type EmbeddingResult ¶

type EmbeddingResult struct {
	Response *core.EmbeddingResponse
	Meta     ExecutionMeta
}

EmbeddingResult is the embeddings result.

type ExecutionMeta ¶

type ExecutionMeta struct {
	ProviderType  string
	ProviderName  string
	Model         string
	FailoverModel string
	UsedFailover  bool
}

ExecutionMeta describes the concrete route used for provider execution.

type FailoverResolver ¶

type FailoverResolver interface {
	ResolveFailovers(resolution *core.RequestModelResolution, op core.Operation) []core.ModelSelector
}

FailoverResolver resolves alternate concrete model selectors for a translated request after the primary selector has already been resolved.

type InferenceConfig ¶

type InferenceConfig struct {
	Provider                 core.RoutableProvider
	ModelResolver            ModelResolver
	ModelAuthorizer          ModelAuthorizer
	WorkflowPolicyResolver   WorkflowPolicyResolver
	FailoverResolver         FailoverResolver
	TranslatedRequestPatcher TranslatedRequestPatcher
	UsageLogger              usage.LoggerInterface
	PricingResolver          usage.PricingResolver
	RouteGate                RouteGate
	GuardrailsHash           string
}

InferenceConfig configures translated inference orchestration.

type InferenceOrchestrator ¶

type InferenceOrchestrator struct {
	// contains filtered or unexported fields
}

InferenceOrchestrator owns translated inference workflow resolution, request patching, provider dispatch, failover, usage logging, and cache metadata.

func NewInferenceOrchestrator(cfg InferenceConfig) *InferenceOrchestrator

func (o *InferenceOrchestrator) CanFastPathStreamingChatPassthrough(workflow *core.Workflow, req *core.ChatRequest) bool

func (o *InferenceOrchestrator) DispatchChatCompletion(
	ctx context.Context,
	workflow *core.Workflow,
	req *core.ChatRequest,
) (*core.ChatResponse, string, string, string, bool, error)

func (o *InferenceOrchestrator) DispatchEmbeddings(
	ctx context.Context,
	workflow *core.Workflow,
	req *core.EmbeddingRequest,
) (*core.EmbeddingResponse, string, string, error)

func (o *InferenceOrchestrator) DispatchResponses(
	ctx context.Context,
	workflow *core.Workflow,
	req *core.ResponsesRequest,
) (*core.ResponsesResponse, string, string, string, bool, error)

func (o *InferenceOrchestrator) ExecuteChatCompletion(ctx context.Context, workflow *core.Workflow, req *core.ChatRequest, requestID, endpoint string) (*ChatCompletionResult, error)

func (o *InferenceOrchestrator) ExecuteEmbeddings(ctx context.Context, workflow *core.Workflow, req *core.EmbeddingRequest, requestID, endpoint string) (*EmbeddingResult, error)

func (o *InferenceOrchestrator) ExecuteResponses(ctx context.Context, workflow *core.Workflow, req *core.ResponsesRequest, requestID, endpoint string) (*ResponsesResult, error)

func (o *InferenceOrchestrator) FailoverSelectors(workflow *core.Workflow) []core.ModelSelector

func (o *InferenceOrchestrator) LogUsage(
	ctx context.Context,
	workflow *core.Workflow,
	model, providerType, providerName string,
	extractFn func(*core.ModelPricing) *usage.UsageEntry,
)

func (o *InferenceOrchestrator) PrepareChatRequest(ctx context.Context, req *core.ChatRequest, meta RequestMeta) (*PreparedChatRequest, error)

func (o *InferenceOrchestrator) PrepareEmbeddingRequest(ctx context.Context, req *core.EmbeddingRequest, meta RequestMeta) (*PreparedEmbeddingRequest, error)

func (o *InferenceOrchestrator) PrepareResponsesRequest(ctx context.Context, req *core.ResponsesRequest, meta RequestMeta) (*PreparedResponsesRequest, error)

func (o *InferenceOrchestrator) ProviderTypeForSelector(selector core.ModelSelector, fallback string) string

func (o *InferenceOrchestrator) ResolveChatRoute(workflow *core.Workflow, req *core.ChatRequest) (*core.ChatRequest, string, string, string)

func (o *InferenceOrchestrator) ShouldEnforceReturningUsageData() bool

func (o *InferenceOrchestrator) StreamChatCompletion(ctx context.Context, workflow *core.Workflow, req *core.ChatRequest) (*StreamResult, error)

func (o *InferenceOrchestrator) StreamResponses(ctx context.Context, workflow *core.Workflow, req *core.ResponsesRequest) (*StreamResult, error)

func (o *InferenceOrchestrator) WithCacheRequestContext(ctx context.Context, workflow *core.Workflow) context.Context

type ModelAuthorizer ¶

type ModelAuthorizer interface {
	ValidateModelAccess(ctx context.Context, selector core.ModelSelector) error
	AllowsModel(ctx context.Context, selector core.ModelSelector) bool
	FilterPublicModels(ctx context.Context, models []core.Model) []core.Model
}

ModelAuthorizer validates request-scoped access to concrete models.

type ModelResolver ¶

type ModelResolver interface {
	ResolveModel(requested core.RequestedModelSelector) (core.ModelSelector, bool, error)
}

ModelResolver resolves raw request selectors into concrete model selectors before provider execution.

type PreparedChatRequest ¶

type PreparedChatRequest struct {
	Context  context.Context
	Request  *core.ChatRequest
	Workflow *core.Workflow
}

PreparedChatRequest is a translated chat request ready for cache lookup or execution.

type PreparedEmbeddingRequest ¶

type PreparedEmbeddingRequest struct {
	Context  context.Context
	Request  *core.EmbeddingRequest
	Workflow *core.Workflow
}

PreparedEmbeddingRequest is a translated embeddings request ready for execution.

type PreparedResponsesRequest ¶

type PreparedResponsesRequest struct {
	Context  context.Context
	Request  *core.ResponsesRequest
	Workflow *core.Workflow
}

PreparedResponsesRequest is a translated Responses request ready for cache lookup or execution.

type ProviderAttempt ¶

type ProviderAttempt struct {
	Seq          int
	Kind         string
	ProviderType string
	ProviderName string
	Model        string
	StatusCode   int
	Success      bool
	ErrorType    string
	ErrorCode    string
	ErrorMessage string
	StartedAt    time.Time
	DurationNs   int64
	// ResponseBody and ResponseHeaders hold the raw upstream error response for
	// a failed attempt. They are persisted only when audit body/header logging
	// is enabled; the audit layer parses and redacts them before storage.
	ResponseBody    []byte
	ResponseHeaders http.Header
}

ProviderAttempt describes one external provider call made while serving a logical request. It is intentionally storage-agnostic; server/audit layers decide how to persist it.

func AttemptsFromContext(ctx context.Context) []ProviderAttempt

type RequestMeta ¶

type RequestMeta struct {
	RequestID string
	Endpoint  core.EndpointDescriptor
	Workflow  *core.Workflow
}

RequestMeta carries transport-derived metadata into gateway use cases.

type ResponsesResult ¶

type ResponsesResult struct {
	Response *core.ResponsesResponse
	Meta     ExecutionMeta
}

ResponsesResult is the non-streaming Responses API result.

type RouteGate ¶

type RouteGate interface {
	RouteAvailable(providerName, model string) bool
}

RouteGate reports whether a provider/model route currently has rate-limit capacity. Failover uses it to skip saturated targets; admission at the handler layer stays the authoritative check.

type StreamResult ¶

type StreamResult struct {
	Stream io.ReadCloser
	Meta   ExecutionMeta
}

StreamResult is a provider SSE stream plus route metadata for observers.

type TranslatedRequestPatcher ¶

type TranslatedRequestPatcher interface {
	PatchChatRequest(ctx context.Context, req *core.ChatRequest) (*core.ChatRequest, error)
	PatchResponsesRequest(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesRequest, error)
}

TranslatedRequestPatcher applies request-level transforms for translated routes after workflow resolution has resolved the concrete execution selector.

type UserPathModelResolver ¶

type UserPathModelResolver interface {
	ResolveModelForUserPath(ctx context.Context, requested core.RequestedModelSelector) (core.ModelSelector, bool, error)
}

UserPathModelResolver is an optional ModelResolver that resolves with awareness of the effective request user path, so a redirect (alias) can be scoped to specific user_paths and fall through to the literal model name for callers that do not match. Resolvers that do not implement it are resolved unscoped via ResolveModel.

type WorkflowPolicyResolver ¶

type WorkflowPolicyResolver interface {
	Match(selector core.WorkflowSelector) (*core.ResolvedWorkflowPolicy, error)
}

WorkflowPolicyResolver matches persisted workflow versions for requests.