abstractions

package
v0.0.0-...-0186d81 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 10, 2026 License: AGPL-3.0 Imports: 39 Imported by: 0

Documentation

Index

Constants

View Source
const (
	IgnoreScalingEventInterval = 10 * time.Second
)

Variables

View Source
var ErrWorkerUnreachable = errors.New("worker unreachable")

ErrWorkerUnreachable wraps a failure to connect to the worker running a container.

Functions

func ConfigureContainerRequestMounts

func ConfigureContainerRequestMounts(containerId string, stub *types.StubWithRelated, workspace *types.Workspace, config types.StubConfigV1) ([]types.Mount, error)

func ConfigureContainerRequestNetwork

func ConfigureContainerRequestNetwork(request *types.ContainerRequest, stubConfig types.StubConfigV1) error

func ConfigureDurableDiskPlacement

func ConfigureDurableDiskPlacement(ctx context.Context, repos DurableDiskPlacementRepos, workspace *types.Workspace, stubConfig *types.StubConfigV1) error

func ConfigureUnavailablePrivatePoolFallback

func ConfigureUnavailablePrivatePoolFallback(ctx context.Context, repos DurableDiskPlacementRepos, workspace *types.Workspace, stubConfig *types.StubConfigV1) error

func CopyWithProxyBuffer

func CopyWithProxyBuffer(dst io.Writer, src io.Reader) (int64, error)

func CopyWithProxyBufferActivity

func CopyWithProxyBufferActivity(dst io.Writer, src io.Reader, onActivity func()) (int64, error)

CopyWithProxyBufferActivity copies with the shared proxy buffer and calls onActivity after every successful read.

func CopyWithProxyBufferFlush

func CopyWithProxyBufferFlush(dst io.Writer, src io.Reader, flush func()) (int64, error)

func EmptyStubObjectHash

func EmptyStubObjectHash() string

func EnsureEmptyStubObject

func EnsureEmptyStubObject(ctx context.Context, backendRepo repository.BackendRepository, workspace *types.Workspace) (types.Object, error)

EnsureEmptyStubObject returns the workspace's canonical empty build-context object, creating and uploading it if needed. Stubs created without user code (dashboard launches, gateway-orchestrated workloads) point at it.

func GetOrCreateEmptyStubObject

func GetOrCreateEmptyStubObject(ctx context.Context, backendRepo repository.BackendRepository, workspace *types.Workspace) (types.Object, error)

GetOrCreateEmptyStubObject registers the canonical empty context without uploading it. Sandbox workers recognize its hash and create an empty workspace locally, so launching a sandbox does not depend on object storage.

func IsEmptyStubObject

func IsEmptyStubObject(object types.Object) bool

func ListenForTaskSchedulingFailures

func ListenForTaskSchedulingFailures(
	ctx context.Context,
	rdb *common.RedisClient,
	backendRepo repository.BackendRepository,
	dispatcher *task.Dispatcher,
)

func NewDeploymentStubCache

func NewDeploymentStubCache() *expirable.LRU[string, string]

func ParseAndValidateDeploymentStubId

func ParseAndValidateDeploymentStubId(
	ctx context.Context,
	deploymentStubCache *expirable.LRU[string, string],
	authInfo *auth.AuthInfo,
	stubId string,
	deploymentName string,
	version string,
	stubType string,
	backendRepo repository.BackendRepository,
) (string, error)

func ParseTunnelRequest

func ParseTunnelRequest(c echo.Context, containerID string, port uint32) (*pb.ContainerTunnelRequest, error)

func ProxyConn

func ProxyConn(dst io.Writer, src io.Reader, done <-chan struct{}, bufferSize int) error

func ServeTunnel

func ServeTunnel(c echo.Context, worker TunnelClient, base *pb.ContainerTunnelRequest) error

ServeTunnel requires the caller to authorize the container and port first. Frames carry request IDs; the worker's request journal handles lost replies.

func SetConnOptions

func SetConnOptions(conn net.Conn, keepAlive bool, keepAliveInterval time.Duration, readDeadline time.Duration)

func StubSchedulable

func StubSchedulable(s GPUPoolChecker, stubConfig *types.StubConfigV1) (bool, string)

StubSchedulable reports whether a stub's containers could ever be placed by the scheduler right now, based on registered pool configs (not live worker capacity). It is in-memory and cheap enough for the request path. When the stub is unschedulable it returns a user-facing reason.

Pool-selector-bound stubs are always treated as schedulable: their pools are registered lazily during scheduling, so absence here proves nothing.

func ValidateVolume

func ValidateVolume(v *pb.Volume) error

ValidateVolume rejects ids and mount paths that would carry a bind mount source, or the symlink to it, outside the workspace's own directories. Bucket mounts have no volume record, so their id may be empty.

Types

type AutoscaledInstance

type AutoscaledInstance struct {
	Ctx                      context.Context
	AppConfig                types.AppConfig
	CancelFunc               context.CancelFunc
	Name                     string
	Rdb                      *common.RedisClient
	Lock                     *common.RedisLock
	IsActive                 bool
	FailedContainerThreshold int

	// DB objects
	Workspace  *types.Workspace
	Stub       *types.StubWithRelated
	StubConfig *types.StubConfigV1
	Object     *types.Object
	Token      *types.Token

	// Scheduling
	Scheduler          *scheduler.Scheduler
	ContainerEventChan chan types.ContainerEvent
	Containers         map[string]bool
	ScaleEventChan     chan int
	EntryPoint         []string
	Autoscaler         IAutoscaler

	// Repositories
	ContainerRepo    repository.ContainerRepository
	BackendRepo      repository.BackendRepository
	TaskRepo         repository.TaskRepository
	EventRepo        repository.EventRepository
	UsageMetricsRepo repository.UsageMetricsRepository

	// Keys
	InstanceLockKey string

	// Callbacks
	StartContainersFunc func(containersToRun int) error
	StopContainersFunc  func(containersToStop int) error
}

func (*AutoscaledInstance) CheckConcurrencyLimit

func (i *AutoscaledInstance) CheckConcurrencyLimit() error

func (*AutoscaledInstance) ConsumeContainerEvent

func (i *AutoscaledInstance) ConsumeContainerEvent(event types.ContainerEvent)

func (*AutoscaledInstance) ConsumeScaleResult

func (i *AutoscaledInstance) ConsumeScaleResult(result *AutoscalerResult)

func (*AutoscaledInstance) HandleScalingEvent

func (i *AutoscaledInstance) HandleScalingEvent(desiredContainers int) error

func (*AutoscaledInstance) Monitor

func (i *AutoscaledInstance) Monitor() error

func (*AutoscaledInstance) State

func (*AutoscaledInstance) Sync

func (i *AutoscaledInstance) Sync() error

Sync updates any persistent state that can be changed on the instance. If a stub has a deployment associated with it, we update the IsActive field.

func (*AutoscaledInstance) UnschedulableReason

func (i *AutoscaledInstance) UnschedulableReason() string

UnschedulableReason returns a non-empty user-facing reason when this instance's containers can never be placed by the scheduler and none are currently running or pending. Callers should fail incoming requests fast with this reason instead of holding them until the queue timeout: the moment capacity joins (a pool is added or a machine comes online) the check passes again with no redeploy needed.

func (*AutoscaledInstance) WaitForContainer

func (i *AutoscaledInstance) WaitForContainer(ctx context.Context, duration time.Duration) (*types.ContainerState, error)

type AutoscaledInstanceConfig

type AutoscaledInstanceConfig struct {
	Name                string
	AppConfig           types.AppConfig
	Workspace           *types.Workspace
	Stub                *types.StubWithRelated
	StubConfig          *types.StubConfigV1
	Object              *types.Object
	Token               *types.Token
	Scheduler           *scheduler.Scheduler
	Rdb                 *common.RedisClient
	InstanceLockKey     string
	ContainerRepo       repository.ContainerRepository
	BackendRepo         repository.BackendRepository
	EventRepo           repository.EventRepository
	TaskRepo            repository.TaskRepository
	UsageMetricsRepo    repository.UsageMetricsRepository
	StartContainersFunc func(containersToRun int) error
	StopContainersFunc  func(containersToStop int) error
}

type AutoscaledInstanceState

type AutoscaledInstanceState struct {
	RunningContainers  int
	PendingContainers  int
	StoppingContainers int
	FailedContainers   []string
}

type Autoscaler

type Autoscaler[I IAutoscaledInstance, S AutoscalerSample] struct {
	// contains filtered or unexported fields
}

func NewAutoscaler

func NewAutoscaler[I IAutoscaledInstance, S AutoscalerSample](instance I, sampleFunc func(I) (S, error), scaleFunc func(I, S) *AutoscalerResult) *Autoscaler[I, S]

func (*Autoscaler[I, S]) Start

func (as *Autoscaler[I, S]) Start(ctx context.Context)

Start the autoscaler

func (*Autoscaler[I, S]) Trigger

func (as *Autoscaler[I, S]) Trigger()

type AutoscalerResult

type AutoscalerResult struct {
	DesiredContainers int
	ResultValid       bool
}

type AutoscalerSample

type AutoscalerSample interface{}

type ConnIdleDeadline

type ConnIdleDeadline struct {
	// contains filtered or unexported fields
}

func NewConnIdleDeadline

func NewConnIdleDeadline(timeout time.Duration, conns ...net.Conn) *ConnIdleDeadline

NewConnIdleDeadline applies one shared idle deadline to every connection. Call Refresh whenever bytes move in either direction.

func (*ConnIdleDeadline) Clear

func (d *ConnIdleDeadline) Clear()

Clear removes the shared idle deadline from all tracked connections.

func (*ConnIdleDeadline) Refresh

func (d *ConnIdleDeadline) Refresh()

Refresh extends the shared idle deadline from now.

type ContainerEventManager

type ContainerEventManager struct {
	// contains filtered or unexported fields
}

func NewContainerEventManager

func NewContainerEventManager(ctx context.Context, containerPrefixes []string, keyEventManager *common.KeyEventManager, instanceFactory func(ctx context.Context, stubId string, options ...func(IAutoscaledInstance)) (IAutoscaledInstance, error)) (*ContainerEventManager, error)

func (*ContainerEventManager) Listen

func (em *ContainerEventManager) Listen()

type ContainerLeaseManager

type ContainerLeaseManager struct {
	// contains filtered or unexported fields
}

func NewContainerLeaseManager

func NewContainerLeaseManager(
	redisClient *common.RedisClient,
	stopper ContainerStopper,
	containerPrefix string,
	leaseKey func(string) string,
) *ContainerLeaseManager

func (*ContainerLeaseManager) Run

type ContainerSecrets

type ContainerSecrets struct {
	Env      []string
	Bindings []types.SecretBinding
}

func ConfigureContainerRequestSecrets

func ConfigureContainerRequestSecrets(
	ctx context.Context,
	backend repository.BackendRepository,
	workspace *types.Workspace,
	stubConfig types.StubConfigV1,
	brokered bool,
) (ContainerSecrets, error)

type ContainerStopper

type ContainerStopper interface {
	Stop(*types.StopContainerArgs) error
}

type ContainerStream

type ContainerStream struct {
	// contains filtered or unexported fields
}

func NewContainerStream

func NewContainerStream(opts ContainerStreamOpts) (*ContainerStream, error)

func (*ContainerStream) Stream

func (l *ContainerStream) Stream(ctx context.Context, authInfo *auth.AuthInfo, containerId string) error

type ContainerStreamOpts

type ContainerStreamOpts struct {
	SendCallback    func(o common.OutputMsg) error
	ExitCallback    func(exitCode int32) error
	ContainerRepo   repository.ContainerRepository
	Tailscale       *network.Tailscale
	Config          types.AppConfig
	KeyEventManager *common.KeyEventManager
	SyncQueue       chan *pb.SyncContainerWorkspaceRequest
}

type DurableDiskPlacementRepos

type DurableDiskPlacementRepos struct {
	BackendRepo    repository.BackendRepository
	ComputeRepo    repository.ComputeRepository
	WorkerRepo     repository.WorkerRepository
	WorkerPoolRepo repository.WorkerPoolRepository
}

type GPUPoolChecker

type GPUPoolChecker interface {
	HasManagedPoolForGPU(gpuType string) bool
}

GPUPoolChecker answers whether any serverless pool config supports a GPU type. Implemented by *scheduler.Scheduler.

type IAutoscaledInstance

type IAutoscaledInstance interface {
	ConsumeScaleResult(*AutoscalerResult)
	ConsumeContainerEvent(types.ContainerEvent)
	HandleScalingEvent(int) error
	Sync() error
}

type IAutoscaler

type IAutoscaler interface {
	Start(ctx context.Context)
	Trigger()
}

type InstanceController

type InstanceController struct {
	// contains filtered or unexported fields
}

func NewInstanceController

func NewInstanceController(
	ctx context.Context,
	getOrCreateInstance func(ctx context.Context, stubId string, options ...func(IAutoscaledInstance)) (IAutoscaledInstance, error),
	getInstance func(stubId string) (IAutoscaledInstance, bool),
	stubTypes []string,
	backendRepo repository.BackendRepository,
	containerRepo repository.ContainerRepository,
	redisClient *common.RedisClient,
) *InstanceController

func (*InstanceController) Init

func (c *InstanceController) Init() error

func (*InstanceController) Load

func (*InstanceController) Warmup

func (c *InstanceController) Warmup(
	ctx context.Context,
	stubId string,
) error

type ProxyBufferPool

type ProxyBufferPool struct{}

ProxyBufferPool adapts the shared proxy copy buffers for httputil.ReverseProxy.

func (ProxyBufferPool) Get

func (ProxyBufferPool) Get() []byte

func (ProxyBufferPool) Put

func (ProxyBufferPool) Put(buf []byte)

type RingBuffer

type RingBuffer[T any] struct {
	// contains filtered or unexported fields
}

func NewRingBuffer

func NewRingBuffer[T any](size int) *RingBuffer[T]

func (*RingBuffer[T]) Capacity

func (rb *RingBuffer[T]) Capacity() int

func (*RingBuffer[T]) Len

func (rb *RingBuffer[T]) Len() int

func (*RingBuffer[T]) Overwrites

func (rb *RingBuffer[T]) Overwrites() uint64

func (*RingBuffer[T]) Pop

func (rb *RingBuffer[T]) Pop() (T, bool)

Pop retrieves and removes the oldest request from the buffer

func (*RingBuffer[T]) Push

func (rb *RingBuffer[T]) Push(request T, priority bool) bool

Push adds a new request to the buffer. If priority is true, it gets inserted at the front of the buffer. It returns true when an existing item was overwritten.

func (*RingBuffer[T]) PushWithOverwrite

func (rb *RingBuffer[T]) PushWithOverwrite(request T, priority bool) (T, bool)

PushWithOverwrite adds a new request to the buffer and returns the item that was displaced, if any.

type TunnelClient

type TunnelClient interface {
	Tunnel(context.Context, *pb.ContainerTunnelRequest) (*pb.ContainerTunnelResponse, error)
}

Directories

Path Synopsis
Package llmroute contains the LLM-aware request inspection, replica scoring and affinity primitives used to route OpenAI-compatible inference traffic.
Package llmroute contains the LLM-aware request inspection, replica scoring and affinity primitives used to route OpenAI-compatible inference traffic.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL