Documentation
¶
Overview ¶
Package managedendpoint hosts inference endpoints deployed from the endpoints repo on spare GPU capacity and serves them under an OpenAI-compatible /v1 route.
Index ¶
- Constants
- func Usage(ctx context.Context, meter repository.UsageMetricsRepository, ...) (*types.UsageReport, error)
- type EndpointState
- type Opts
- type Service
- func (s *Service) AckConfig(ctx context.Context, in *pb.HarnessAckConfigRequest) (*pb.HarnessAckConfigResponse, error)
- func (s *Service) AdminWorkspace(ctx context.Context) (*types.Workspace, error)
- func (s *Service) ApplyRepo(ctx context.Context, in *pb.ApplyRepoRequest) (*pb.ApplyRepoResponse, error)
- func (s *Service) Enabled() bool
- func (s *Service) GetEndpoint(ctx context.Context, in *pb.GetEndpointRequest) (*pb.GetEndpointResponse, error)
- func (s *Service) GetGitOpsStatus(ctx context.Context, _ *pb.GetGitOpsStatusRequest) (*pb.GetGitOpsStatusResponse, error)
- func (s *Service) GetMetrics(ctx context.Context, in *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error)
- func (s *Service) Heartbeat(ctx context.Context, in *pb.HarnessHeartbeatRequest) (*pb.HarnessHeartbeatResponse, error)
- func (s *Service) ListEndpoints(ctx context.Context, _ *pb.ListEndpointsRequest) (*pb.ListEndpointsResponse, error)
- func (s *Service) ListReplicas(ctx context.Context, in *pb.ListReplicasRequest) (*pb.ListReplicasResponse, error)
- func (s *Service) PublishEvents(ctx context.Context, in *pb.HarnessPublishEventsRequest) (*pb.HarnessPublishEventsResponse, error)
- func (s *Service) Register(ctx context.Context, in *pb.HarnessRegisterRequest) (*pb.HarnessRegisterResponse, error)
- func (s *Service) SetReplicaConfig(ctx context.Context, in *pb.SetReplicaConfigRequest) (*pb.SetReplicaConfigResponse, error)
- func (s *Service) StopReplica(ctx context.Context, in *pb.StopReplicaRequest) (*pb.StopReplicaResponse, error)
- func (s *Service) WatchConfig(in *pb.HarnessWatchConfigRequest, ...) error
Constants ¶
View Source
const ( // Environment injected into every replica container. EnvEndpointID = "BEAM_ENDPOINT_ID" EnvReplicaID = "BEAM_REPLICA_ID" EnvReplicaSecret = "BEAM_REPLICA_SECRET" // presented on harness RPCs as x-beam-replica-secret EnvGpu = "BEAM_GPU" EnvLocality = "BEAM_LOCALITY" EnvEndpointPort = "BEAM_ENDPOINT_PORT" EnvHarnessEnabled = "BEAM_HARNESS_ENABLED" EnvHarnessConfig = "BEAM_HARNESS_CONFIG" EnvDrainSeconds = "BEAM_DRAIN_SECONDS" // grace after SIGTERM before the worker kills the engine )
Variables ¶
This section is empty.
Functions ¶
func Usage ¶
func Usage(ctx context.Context, meter repository.UsageMetricsRepository, kind types.UsageKind, workspaceID string, from, to time.Time) (*types.UsageReport, error)
Usage reads one workspace's spend or earnings back from the billing meter as the dashboard shows it: totals, per UTC day and per model.
Types ¶
type EndpointState ¶
type EndpointState string
EndpointState is the one word shown to operators. Deploy problems come from the repo's last CI run; everything after that from the fleet and the replicas.
const ( StateDeployFailed EndpointState = "deploy_failed" StateRetired EndpointState = "retired" StateDisabled EndpointState = "disabled" // deployed but not placed by config.yaml StateWaitingForCapacity EndpointState = "waiting_for_capacity" StateIdle EndpointState = "idle" // serverless placement, scaled to zero StateLoading EndpointState = "loading" StateReady EndpointState = "ready" StateFailed EndpointState = "failed" )
type Opts ¶
type Opts struct {
Config types.AppConfig
BackendRepo repository.BackendRepository
ContainerRepo repository.ContainerRepository
WorkerRepo repository.WorkerRepository
WorkspaceRepo repository.WorkspaceRepository
EndpointRepo repository.ManagedEndpointRepository
EventRepo repository.EventRepository
UsageMetricsRepo repository.UsageMetricsRepository
Scheduler *scheduler.Scheduler
RedisClient *common.RedisClient
Tailscale *network.Tailscale
// RouteGroup is the root echo group (/v1 is mounted under it).
RouteGroup *echo.Group
// AdminRouteGroup receives the /api/v1/endpoints REST mirror.
AdminRouteGroup *echo.Group
// DrainContext stops admission and queued /v1 requests during shutdown.
// Active generations run until the service context ends after HTTP draining.
DrainContext context.Context
}
Opts wires the service into the gateway.
type Service ¶
type Service struct {
pb.UnimplementedEndpointHarnessServiceServer
pb.UnimplementedEndpointAdminServiceServer
// contains filtered or unexported fields
}
Service is the managed endpoints control plane inside one gateway replica.
func New ¶
New constructs the service. When disabled it registers no routes and every RPC fails with FailedPrecondition.
func (*Service) AckConfig ¶
func (s *Service) AckConfig(ctx context.Context, in *pb.HarnessAckConfigRequest) (*pb.HarnessAckConfigResponse, error)
func (*Service) AdminWorkspace ¶
AdminWorkspace is the cluster admin workspace that owns every managed stub and replica.
func (*Service) ApplyRepo ¶
func (s *Service) ApplyRepo(ctx context.Context, in *pb.ApplyRepoRequest) (*pb.ApplyRepoResponse, error)
ApplyRepo validates one commit of the endpoints repo and, unless dry_run, records deploy outcomes, applies config.yaml and retires apps whose directory is gone. Validation problems are returned in errors. Commits apply one at a time across gateways; a concurrent CI run waits its turn.
func (*Service) GetEndpoint ¶
func (s *Service) GetEndpoint(ctx context.Context, in *pb.GetEndpointRequest) (*pb.GetEndpointResponse, error)
func (*Service) GetGitOpsStatus ¶
func (s *Service) GetGitOpsStatus(ctx context.Context, _ *pb.GetGitOpsStatusRequest) (*pb.GetGitOpsStatusResponse, error)
func (*Service) GetMetrics ¶
func (s *Service) GetMetrics(ctx context.Context, in *pb.GetMetricsRequest) (*pb.GetMetricsResponse, error)
func (*Service) Heartbeat ¶
func (s *Service) Heartbeat(ctx context.Context, in *pb.HarnessHeartbeatRequest) (*pb.HarnessHeartbeatResponse, error)
func (*Service) ListEndpoints ¶
func (s *Service) ListEndpoints(ctx context.Context, _ *pb.ListEndpointsRequest) (*pb.ListEndpointsResponse, error)
func (*Service) ListReplicas ¶
func (s *Service) ListReplicas(ctx context.Context, in *pb.ListReplicasRequest) (*pb.ListReplicasResponse, error)
func (*Service) PublishEvents ¶
func (s *Service) PublishEvents(ctx context.Context, in *pb.HarnessPublishEventsRequest) (*pb.HarnessPublishEventsResponse, error)
func (*Service) Register ¶
func (s *Service) Register(ctx context.Context, in *pb.HarnessRegisterRequest) (*pb.HarnessRegisterResponse, error)
func (*Service) SetReplicaConfig ¶
func (s *Service) SetReplicaConfig(ctx context.Context, in *pb.SetReplicaConfigRequest) (*pb.SetReplicaConfigResponse, error)
SetReplicaConfig pushes a live config to one replica and waits for the harness to ack it.
func (*Service) StopReplica ¶
func (s *Service) StopReplica(ctx context.Context, in *pb.StopReplicaRequest) (*pb.StopReplicaResponse, error)
StopReplica drains and stops a replica; the controller refills on its next tick.
func (*Service) WatchConfig ¶
func (s *Service) WatchConfig(in *pb.HarnessWatchConfigRequest, stream pb.EndpointHarnessService_WatchConfigServer) error
WatchConfig streams the replica's live config on every new revision.
Source Files
Click to show internal directories.
Click to hide internal directories.