Documentation
¶
Index ¶
- Constants
- Variables
- func ConfigureContainerRequestMounts(containerId string, stub *types.StubWithRelated, workspace *types.Workspace, ...) ([]types.Mount, error)
- func ConfigureContainerRequestNetwork(request *types.ContainerRequest, stubConfig types.StubConfigV1) error
- func ConfigureDurableDiskPlacement(ctx context.Context, repos DurableDiskPlacementRepos, ...) error
- func ConfigureUnavailablePrivatePoolFallback(ctx context.Context, repos DurableDiskPlacementRepos, ...) error
- func CopyWithProxyBuffer(dst io.Writer, src io.Reader) (int64, error)
- func CopyWithProxyBufferActivity(dst io.Writer, src io.Reader, onActivity func()) (int64, error)
- func CopyWithProxyBufferFlush(dst io.Writer, src io.Reader, flush func()) (int64, error)
- func EmptyStubObjectHash() string
- func EnsureEmptyStubObject(ctx context.Context, backendRepo repository.BackendRepository, ...) (types.Object, error)
- func GetOrCreateEmptyStubObject(ctx context.Context, backendRepo repository.BackendRepository, ...) (types.Object, error)
- func IsEmptyStubObject(object types.Object) bool
- func ListenForTaskSchedulingFailures(ctx context.Context, rdb *common.RedisClient, ...)
- func NewDeploymentStubCache() *expirable.LRU[string, string]
- func ParseAndValidateDeploymentStubId(ctx context.Context, deploymentStubCache *expirable.LRU[string, string], ...) (string, error)
- func ParseTunnelRequest(c echo.Context, containerID string, port uint32) (*pb.ContainerTunnelRequest, error)
- func ProxyConn(dst io.Writer, src io.Reader, done <-chan struct{}, bufferSize int) error
- func ServeTunnel(c echo.Context, worker TunnelClient, base *pb.ContainerTunnelRequest) error
- func SetConnOptions(conn net.Conn, keepAlive bool, keepAliveInterval time.Duration, ...)
- func StubSchedulable(s GPUPoolChecker, stubConfig *types.StubConfigV1) (bool, string)
- func ValidateVolume(v *pb.Volume) error
- type AutoscaledInstance
- func (i *AutoscaledInstance) CheckConcurrencyLimit() error
- func (i *AutoscaledInstance) ConsumeContainerEvent(event types.ContainerEvent)
- func (i *AutoscaledInstance) ConsumeScaleResult(result *AutoscalerResult)
- func (i *AutoscaledInstance) HandleScalingEvent(desiredContainers int) error
- func (i *AutoscaledInstance) Monitor() error
- func (i *AutoscaledInstance) State() (*AutoscaledInstanceState, error)
- func (i *AutoscaledInstance) Sync() error
- func (i *AutoscaledInstance) UnschedulableReason() string
- func (i *AutoscaledInstance) WaitForContainer(ctx context.Context, duration time.Duration) (*types.ContainerState, error)
- type AutoscaledInstanceConfig
- type AutoscaledInstanceState
- type Autoscaler
- type AutoscalerResult
- type AutoscalerSample
- type ConnIdleDeadline
- type ContainerEventManager
- type ContainerLeaseManager
- type ContainerSecrets
- type ContainerStopper
- type ContainerStream
- type ContainerStreamOpts
- type DurableDiskPlacementRepos
- type GPUPoolChecker
- type IAutoscaledInstance
- type IAutoscaler
- type InstanceController
- type ProxyBufferPool
- type RingBuffer
- func (rb *RingBuffer[T]) Capacity() int
- func (rb *RingBuffer[T]) Len() int
- func (rb *RingBuffer[T]) Overwrites() uint64
- func (rb *RingBuffer[T]) Pop() (T, bool)
- func (rb *RingBuffer[T]) Push(request T, priority bool) bool
- func (rb *RingBuffer[T]) PushWithOverwrite(request T, priority bool) (T, bool)
- type TunnelClient
Constants ¶
const (
IgnoreScalingEventInterval = 10 * time.Second
)
Variables ¶
var ErrWorkerUnreachable = errors.New("worker unreachable")
ErrWorkerUnreachable wraps a failure to connect to the worker running a container.
Functions ¶
func ConfigureContainerRequestMounts ¶
func ConfigureContainerRequestMounts(containerId string, stub *types.StubWithRelated, workspace *types.Workspace, config types.StubConfigV1) ([]types.Mount, error)
func ConfigureContainerRequestNetwork ¶
func ConfigureContainerRequestNetwork(request *types.ContainerRequest, stubConfig types.StubConfigV1) error
func ConfigureDurableDiskPlacement ¶
func ConfigureDurableDiskPlacement(ctx context.Context, repos DurableDiskPlacementRepos, workspace *types.Workspace, stubConfig *types.StubConfigV1) error
func ConfigureUnavailablePrivatePoolFallback ¶
func ConfigureUnavailablePrivatePoolFallback(ctx context.Context, repos DurableDiskPlacementRepos, workspace *types.Workspace, stubConfig *types.StubConfigV1) error
func CopyWithProxyBufferActivity ¶
CopyWithProxyBufferActivity copies with the shared proxy buffer and calls onActivity after every successful read.
func EmptyStubObjectHash ¶
func EmptyStubObjectHash() string
func EnsureEmptyStubObject ¶
func EnsureEmptyStubObject(ctx context.Context, backendRepo repository.BackendRepository, workspace *types.Workspace) (types.Object, error)
EnsureEmptyStubObject returns the workspace's canonical empty build-context object, creating and uploading it if needed. Stubs created without user code (dashboard launches, gateway-orchestrated workloads) point at it.
func GetOrCreateEmptyStubObject ¶
func GetOrCreateEmptyStubObject(ctx context.Context, backendRepo repository.BackendRepository, workspace *types.Workspace) (types.Object, error)
GetOrCreateEmptyStubObject registers the canonical empty context without uploading it. Sandbox workers recognize its hash and create an empty workspace locally, so launching a sandbox does not depend on object storage.
func IsEmptyStubObject ¶
func ListenForTaskSchedulingFailures ¶
func ListenForTaskSchedulingFailures( ctx context.Context, rdb *common.RedisClient, backendRepo repository.BackendRepository, dispatcher *task.Dispatcher, )
func ParseTunnelRequest ¶
func ServeTunnel ¶
func ServeTunnel(c echo.Context, worker TunnelClient, base *pb.ContainerTunnelRequest) error
ServeTunnel requires the caller to authorize the container and port first. Frames carry request IDs; the worker's request journal handles lost replies.
func SetConnOptions ¶
func StubSchedulable ¶
func StubSchedulable(s GPUPoolChecker, stubConfig *types.StubConfigV1) (bool, string)
StubSchedulable reports whether a stub's containers could ever be placed by the scheduler right now, based on registered pool configs (not live worker capacity). It is in-memory and cheap enough for the request path. When the stub is unschedulable it returns a user-facing reason.
Pool-selector-bound stubs are always treated as schedulable: their pools are registered lazily during scheduling, so absence here proves nothing.
func ValidateVolume ¶
ValidateVolume rejects ids and mount paths that would carry a bind mount source, or the symlink to it, outside the workspace's own directories. Bucket mounts have no volume record, so their id may be empty.
Types ¶
type AutoscaledInstance ¶
type AutoscaledInstance struct {
Ctx context.Context
AppConfig types.AppConfig
CancelFunc context.CancelFunc
Name string
Rdb *common.RedisClient
Lock *common.RedisLock
IsActive bool
FailedContainerThreshold int
// DB objects
Workspace *types.Workspace
Stub *types.StubWithRelated
StubConfig *types.StubConfigV1
Object *types.Object
Token *types.Token
// Scheduling
Scheduler *scheduler.Scheduler
ContainerEventChan chan types.ContainerEvent
Containers map[string]bool
ScaleEventChan chan int
EntryPoint []string
Autoscaler IAutoscaler
// Repositories
ContainerRepo repository.ContainerRepository
BackendRepo repository.BackendRepository
TaskRepo repository.TaskRepository
EventRepo repository.EventRepository
UsageMetricsRepo repository.UsageMetricsRepository
// Keys
InstanceLockKey string
// Callbacks
StartContainersFunc func(containersToRun int) error
StopContainersFunc func(containersToStop int) error
}
func NewAutoscaledInstance ¶
func NewAutoscaledInstance(ctx context.Context, cfg *AutoscaledInstanceConfig) (*AutoscaledInstance, error)
func (*AutoscaledInstance) CheckConcurrencyLimit ¶
func (i *AutoscaledInstance) CheckConcurrencyLimit() error
func (*AutoscaledInstance) ConsumeContainerEvent ¶
func (i *AutoscaledInstance) ConsumeContainerEvent(event types.ContainerEvent)
func (*AutoscaledInstance) ConsumeScaleResult ¶
func (i *AutoscaledInstance) ConsumeScaleResult(result *AutoscalerResult)
func (*AutoscaledInstance) HandleScalingEvent ¶
func (i *AutoscaledInstance) HandleScalingEvent(desiredContainers int) error
func (*AutoscaledInstance) Monitor ¶
func (i *AutoscaledInstance) Monitor() error
func (*AutoscaledInstance) State ¶
func (i *AutoscaledInstance) State() (*AutoscaledInstanceState, error)
func (*AutoscaledInstance) Sync ¶
func (i *AutoscaledInstance) Sync() error
Sync updates any persistent state that can be changed on the instance. If a stub has a deployment associated with it, we update the IsActive field.
func (*AutoscaledInstance) UnschedulableReason ¶
func (i *AutoscaledInstance) UnschedulableReason() string
UnschedulableReason returns a non-empty user-facing reason when this instance's containers can never be placed by the scheduler and none are currently running or pending. Callers should fail incoming requests fast with this reason instead of holding them until the queue timeout: the moment capacity joins (a pool is added or a machine comes online) the check passes again with no redeploy needed.
func (*AutoscaledInstance) WaitForContainer ¶
func (i *AutoscaledInstance) WaitForContainer(ctx context.Context, duration time.Duration) (*types.ContainerState, error)
type AutoscaledInstanceConfig ¶
type AutoscaledInstanceConfig struct {
Name string
AppConfig types.AppConfig
Workspace *types.Workspace
Stub *types.StubWithRelated
StubConfig *types.StubConfigV1
Object *types.Object
Token *types.Token
Scheduler *scheduler.Scheduler
Rdb *common.RedisClient
InstanceLockKey string
ContainerRepo repository.ContainerRepository
BackendRepo repository.BackendRepository
EventRepo repository.EventRepository
TaskRepo repository.TaskRepository
UsageMetricsRepo repository.UsageMetricsRepository
StartContainersFunc func(containersToRun int) error
StopContainersFunc func(containersToStop int) error
}
type AutoscaledInstanceState ¶
type Autoscaler ¶
type Autoscaler[I IAutoscaledInstance, S AutoscalerSample] struct { // contains filtered or unexported fields }
func NewAutoscaler ¶
func NewAutoscaler[I IAutoscaledInstance, S AutoscalerSample](instance I, sampleFunc func(I) (S, error), scaleFunc func(I, S) *AutoscalerResult) *Autoscaler[I, S]
func (*Autoscaler[I, S]) Start ¶
func (as *Autoscaler[I, S]) Start(ctx context.Context)
Start the autoscaler
func (*Autoscaler[I, S]) Trigger ¶
func (as *Autoscaler[I, S]) Trigger()
type AutoscalerResult ¶
type AutoscalerSample ¶
type AutoscalerSample interface{}
type ConnIdleDeadline ¶
type ConnIdleDeadline struct {
// contains filtered or unexported fields
}
func NewConnIdleDeadline ¶
func NewConnIdleDeadline(timeout time.Duration, conns ...net.Conn) *ConnIdleDeadline
NewConnIdleDeadline applies one shared idle deadline to every connection. Call Refresh whenever bytes move in either direction.
func (*ConnIdleDeadline) Clear ¶
func (d *ConnIdleDeadline) Clear()
Clear removes the shared idle deadline from all tracked connections.
func (*ConnIdleDeadline) Refresh ¶
func (d *ConnIdleDeadline) Refresh()
Refresh extends the shared idle deadline from now.
type ContainerEventManager ¶
type ContainerEventManager struct {
// contains filtered or unexported fields
}
func NewContainerEventManager ¶
func NewContainerEventManager(ctx context.Context, containerPrefixes []string, keyEventManager *common.KeyEventManager, instanceFactory func(ctx context.Context, stubId string, options ...func(IAutoscaledInstance)) (IAutoscaledInstance, error)) (*ContainerEventManager, error)
func (*ContainerEventManager) Listen ¶
func (em *ContainerEventManager) Listen()
type ContainerLeaseManager ¶
type ContainerLeaseManager struct {
// contains filtered or unexported fields
}
func NewContainerLeaseManager ¶
func NewContainerLeaseManager( redisClient *common.RedisClient, stopper ContainerStopper, containerPrefix string, leaseKey func(string) string, ) *ContainerLeaseManager
type ContainerSecrets ¶
type ContainerSecrets struct {
Env []string
Bindings []types.SecretBinding
}
func ConfigureContainerRequestSecrets ¶
func ConfigureContainerRequestSecrets( ctx context.Context, backend repository.BackendRepository, workspace *types.Workspace, stubConfig types.StubConfigV1, brokered bool, ) (ContainerSecrets, error)
type ContainerStopper ¶
type ContainerStopper interface {
Stop(*types.StopContainerArgs) error
}
type ContainerStream ¶
type ContainerStream struct {
// contains filtered or unexported fields
}
func NewContainerStream ¶
func NewContainerStream(opts ContainerStreamOpts) (*ContainerStream, error)
type ContainerStreamOpts ¶
type ContainerStreamOpts struct {
SendCallback func(o common.OutputMsg) error
ExitCallback func(exitCode int32) error
ContainerRepo repository.ContainerRepository
Tailscale *network.Tailscale
Config types.AppConfig
KeyEventManager *common.KeyEventManager
SyncQueue chan *pb.SyncContainerWorkspaceRequest
}
type DurableDiskPlacementRepos ¶
type DurableDiskPlacementRepos struct {
BackendRepo repository.BackendRepository
ComputeRepo repository.ComputeRepository
WorkerRepo repository.WorkerRepository
WorkerPoolRepo repository.WorkerPoolRepository
}
type GPUPoolChecker ¶
GPUPoolChecker answers whether any serverless pool config supports a GPU type. Implemented by *scheduler.Scheduler.
type IAutoscaledInstance ¶
type IAutoscaledInstance interface {
ConsumeScaleResult(*AutoscalerResult)
ConsumeContainerEvent(types.ContainerEvent)
HandleScalingEvent(int) error
Sync() error
}
type IAutoscaler ¶
type InstanceController ¶
type InstanceController struct {
// contains filtered or unexported fields
}
func NewInstanceController ¶
func NewInstanceController( ctx context.Context, getOrCreateInstance func(ctx context.Context, stubId string, options ...func(IAutoscaledInstance)) (IAutoscaledInstance, error), getInstance func(stubId string) (IAutoscaledInstance, bool), stubTypes []string, backendRepo repository.BackendRepository, containerRepo repository.ContainerRepository, redisClient *common.RedisClient, ) *InstanceController
func (*InstanceController) Init ¶
func (c *InstanceController) Init() error
func (*InstanceController) Load ¶
func (c *InstanceController) Load(filter *types.DeploymentFilter) error
type ProxyBufferPool ¶
type ProxyBufferPool struct{}
ProxyBufferPool adapts the shared proxy copy buffers for httputil.ReverseProxy.
func (ProxyBufferPool) Get ¶
func (ProxyBufferPool) Get() []byte
func (ProxyBufferPool) Put ¶
func (ProxyBufferPool) Put(buf []byte)
type RingBuffer ¶
type RingBuffer[T any] struct { // contains filtered or unexported fields }
func NewRingBuffer ¶
func NewRingBuffer[T any](size int) *RingBuffer[T]
func (*RingBuffer[T]) Capacity ¶
func (rb *RingBuffer[T]) Capacity() int
func (*RingBuffer[T]) Len ¶
func (rb *RingBuffer[T]) Len() int
func (*RingBuffer[T]) Overwrites ¶
func (rb *RingBuffer[T]) Overwrites() uint64
func (*RingBuffer[T]) Pop ¶
func (rb *RingBuffer[T]) Pop() (T, bool)
Pop retrieves and removes the oldest request from the buffer
func (*RingBuffer[T]) Push ¶
func (rb *RingBuffer[T]) Push(request T, priority bool) bool
Push adds a new request to the buffer. If priority is true, it gets inserted at the front of the buffer. It returns true when an existing item was overwritten.
func (*RingBuffer[T]) PushWithOverwrite ¶
func (rb *RingBuffer[T]) PushWithOverwrite(request T, priority bool) (T, bool)
PushWithOverwrite adds a new request to the buffer and returns the item that was displaced, if any.
type TunnelClient ¶
type TunnelClient interface {
Tunnel(context.Context, *pb.ContainerTunnelRequest) (*pb.ContainerTunnelResponse, error)
}
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package llmroute contains the LLM-aware request inspection, replica scoring and affinity primitives used to route OpenAI-compatible inference traffic.
|
Package llmroute contains the LLM-aware request inspection, replica scoring and affinity primitives used to route OpenAI-compatible inference traffic. |