Documentation
¶
Index ¶
- Constants
- Variables
- func GetDefaultRegistry() map[string]*InjectionPoint
- func ListInjectionPoints() []string
- func MergePlaceholders(defaults, overrides map[string]string) map[string]string
- func RenderMessage(templateStr string, data map[string]string) (string, error)
- func ValidateOverrides(template *InjectionTemplate, overrides map[string]string) error
- func ValidatePointID(pointID string) bool
- func WithInjectionContext(ctx context.Context, cfg *InjectionConfig) context.Context
- type ErrorType
- type ExecutionTrace
- type GlobalInjectionConfig
- type InMemoryTraceStore
- func (s *InMemoryTraceStore) AppendPoint(jobID string, point PointRecord) error
- func (s *InMemoryTraceStore) Clear()
- func (s *InMemoryTraceStore) Count() int
- func (s *InMemoryTraceStore) Delete(jobID string) error
- func (s *InMemoryTraceStore) Get(jobID string) (*ExecutionTrace, error)
- func (s *InMemoryTraceStore) List(limit int) ([]*ExecutionTrace, error)
- type InjectedError
- type InjectionConfig
- type InjectionContextKey
- type InjectionPoint
- type InjectionRule
- type InjectionTemplate
- type Injector
- type InjectorImpl
- func (i *InjectorImpl) CheckPoint(ctx context.Context, pointID string) error
- func (i *InjectorImpl) GetGlobalConfig() *GlobalInjectionConfig
- func (i *InjectorImpl) GetPoint(pointID string) *InjectionPoint
- func (i *InjectorImpl) GetTrace(ctx context.Context, jobID string) *ExecutionTrace
- func (i *InjectorImpl) ListPoints() []*InjectionPoint
- func (i *InjectorImpl) RenderTemplate(pointID string, errorType ErrorType, overrides map[string]string) (*InjectedError, error)
- func (i *InjectorImpl) SetGlobalConfig(config *GlobalInjectionConfig) error
- type PointRecord
- type TraceStore
Constants ¶
const ( POINT_CONSOLE_CREATE_JOB = "console.create_job" POINT_CONSOLE_GET_JOB = "console.get_job" POINT_CONSOLE_CANCEL_JOB = "console.cancel_job" POINT_CONSOLE_LIST_JOBS = "console.list_jobs" POINT_CONSOLE_UPLOAD_FILE = "console.upload_file" POINT_CONSOLE_DOWNLOAD_FILE = "console.download_file" )
Injection point constants for the console.
const ( POINT_PLANNER_ENQUEUE = "planner.enqueue" POINT_PLANNER_PLAN = "planner.plan" POINT_PLANNER_SUBMIT_BATCH = "planner.submit_batch" POINT_PLANNER_CANCEL = "planner.cancel" POINT_PLANNER_PERSIST = "planner.persist" POINT_PLANNER_RECOVER = "planner.recover" )
Injection point constants for the planner.
const ( POINT_RM_PROVISION = "rm.provision" POINT_RM_GET_STATUS = "rm.get_status" POINT_RM_RELEASE = "rm.release" POINT_RM_CATALOG_LOOKUP = "rm.catalog_lookup" POINT_RM_LIST = "rm.list" )
Injection point constants for the resource manager.
const ( POINT_BATCH_CLIENT_CREATE_BATCH = "batch_client.create_batch" POINT_BATCH_CLIENT_GET_BATCH = "batch_client.get_batch" POINT_BATCH_CLIENT_CANCEL_BATCH = "batch_client.cancel_batch" POINT_BATCH_CLIENT_LIST_BATCHES = "batch_client.list_batches" )
Injection point constants for the batch client.
const ( POINT_STORE_UPSERT_JOB = "store.upsert_job" POINT_STORE_GET_JOB = "store.get_job" POINT_STORE_LIST_JOBS = "store.list_jobs" POINT_STORE_UPSERT_PROVISION = "store.upsert_provision" POINT_STORE_GET_PROVISION = "store.get_provision" )
Injection point constants for the store.
const ( CodeDeadlineExceeded = "DEADLINE_EXCEEDED" CodeInvalidArgument = "INVALID_ARGUMENT" CodeNotFound = "NOT_FOUND" CodePermissionDenied = "PERMISSION_DENIED" CodeResourceExhausted = "RESOURCE_EXHAUSTED" CodeInternal = "INTERNAL" CodeCrash = "CRASH" CodeUnknown = "UNKNOWN" )
Default gRPC/HTTP error codes for error types.
Variables ¶
var ( // ErrInjectorDisabled is returned when the injector is disabled. ErrInjectorDisabled = errors.New("error injection is disabled") // ErrInvalidMode is returned when an invalid mode is specified. ErrInvalidMode = errors.New("invalid injector mode") // ErrPointNotFound is returned when an injection point is not found. ErrPointNotFound = errors.New("injection point not found") // ErrInvalidProbability is returned when probability is out of range. ErrInvalidProbability = errors.New("probability must be between 0.0 and 1.0") // ErrPointAlreadyRegistered is returned when registering a duplicate point. ErrPointAlreadyRegistered = errors.New("injection point already registered") )
Sentinel errors for the injector.
var DefaultRegistry map[string]*InjectionPoint
DefaultRegistry holds all pre-defined injection points
var (
ErrTraceNotFound = errors.New("trace not found")
)
Common errors for trace store operations
Functions ¶
func GetDefaultRegistry ¶
func GetDefaultRegistry() map[string]*InjectionPoint
GetDefaultRegistry returns the default registry containing all pre-defined injection points
func ListInjectionPoints ¶
func ListInjectionPoints() []string
ListInjectionPoints returns all available injection point IDs
func MergePlaceholders ¶
MergePlaceholders merges two placeholder maps with overrides taking precedence. Returns a new map without mutating the inputs.
func RenderMessage ¶
RenderMessage renders a single template string with the provided data. It uses Go's text/template package with {{.variable}} syntax.
func ValidateOverrides ¶
func ValidateOverrides(template *InjectionTemplate, overrides map[string]string) error
ValidateOverrides checks that all override keys exist in the template placeholders. Returns an error listing all invalid keys if any are found.
func ValidatePointID ¶
ValidatePointID checks if the given point ID exists in the default registry
func WithInjectionContext ¶
func WithInjectionContext(ctx context.Context, cfg *InjectionConfig) context.Context
WithInjectionContext creates a new context with injection config
Types ¶
type ErrorType ¶
type ErrorType string
ErrorType represents the category of error to inject.
const ( // ErrorTypeTimeout indicates a timeout error. ErrorTypeTimeout ErrorType = "timeout" ErrorTypeUnavailable ErrorType = "unavailable" // ErrorTypeInvalidArgument indicates an invalid argument error. ErrorTypeInvalidArgument ErrorType = "invalid_argument" // ErrorTypeNotFound indicates a resource not found error. ErrorTypeNotFound ErrorType = "not_found" // ErrorTypePermissionDenied indicates a permission denied error. ErrorTypePermissionDenied ErrorType = "permission_denied" // ErrorTypeResourceExhausted indicates a resource exhausted error. ErrorTypeResourceExhausted ErrorType = "resource_exhausted" // ErrorTypeInternal indicates an internal error. ErrorTypeInternal ErrorType = "internal" // ErrorTypeCrash indicates a simulated crash/panic. ErrorTypeCrash ErrorType = "crash" )
type ExecutionTrace ¶
type ExecutionTrace struct {
// JobID is the identifier of the job being traced.
JobID string `json:"job_id"`
// StartTime is when the job execution began.
StartTime time.Time `json:"start_time"`
// EndTime is when the job execution completed.
EndTime time.Time `json:"end_time,omitempty"`
// Points contains records for all injection points evaluated during execution.
Points []PointRecord `json:"points,omitempty"`
}
ExecutionTrace captures the complete trace of error injection for a job execution.
func (*ExecutionTrace) AppendPoint ¶
func (t *ExecutionTrace) AppendPoint(record PointRecord)
AppendPoint adds a new point record to the execution trace.
func (*ExecutionTrace) Clone ¶
func (t *ExecutionTrace) Clone() *ExecutionTrace
type GlobalInjectionConfig ¶
type GlobalInjectionConfig struct {
// Enabled determines whether global error injection is active.
Enabled bool `json:"enabled"`
// Rules contains the global injection rules to apply.
Rules []InjectionRule `json:"rules,omitempty"`
// ExcludedPoints lists injection points that should never have errors injected.
ExcludedPoints []string `json:"excluded_points,omitempty"`
// GlobalProbability is the default probability for all injection points in chaos mode.
GlobalProbability float64 `json:"global_probability,omitempty"`
// PointWeights overrides probability per injection point in chaos mode.
// Keys are point IDs (e.g., "rm.provision"), values are probabilities (0.0-1.0).
// Point-specific probability takes precedence over GlobalProbability.
PointWeights map[string]float64 `json:"point_weights,omitempty"`
}
GlobalInjectionConfig contains global error injection settings.
type InMemoryTraceStore ¶
type InMemoryTraceStore struct {
// contains filtered or unexported fields
}
InMemoryTraceStore implements TraceStore with an in-memory map This is for debugging only and does not persist to any database
func NewInMemoryTraceStore ¶
func NewInMemoryTraceStore() *InMemoryTraceStore
NewInMemoryTraceStore creates a new in-memory trace store
func NewInMemoryTraceStoreWithLimit ¶
func NewInMemoryTraceStoreWithLimit(maxTraces int) *InMemoryTraceStore
NewInMemoryTraceStoreWithLimit creates a new in-memory trace store with a maximum trace limit When the limit is exceeded, the oldest traces are evicted (LRU)
func (*InMemoryTraceStore) AppendPoint ¶
func (s *InMemoryTraceStore) AppendPoint(jobID string, point PointRecord) error
AppendPoint adds a point record to the trace for the given job ID Thread-safe operation that creates a new trace if one doesn't exist
func (*InMemoryTraceStore) Clear ¶
func (s *InMemoryTraceStore) Clear()
Clear removes all traces from the store
func (*InMemoryTraceStore) Count ¶
func (s *InMemoryTraceStore) Count() int
Count returns the number of traces currently stored
func (*InMemoryTraceStore) Delete ¶
func (s *InMemoryTraceStore) Delete(jobID string) error
Delete removes the execution trace for the given job ID No error is returned if the trace doesn't exist
func (*InMemoryTraceStore) Get ¶
func (s *InMemoryTraceStore) Get(jobID string) (*ExecutionTrace, error)
Get retrieves the execution trace for the given job ID Returns ErrTraceNotFound if the trace doesn't exist
func (*InMemoryTraceStore) List ¶
func (s *InMemoryTraceStore) List(limit int) ([]*ExecutionTrace, error)
List returns the most recent execution traces, sorted by StartTime (newest first) Limited to the specified count
type InjectedError ¶
type InjectedError struct {
// Type is the category of the injected error.
Type ErrorType `json:"type"`
// Code is the gRPC/HTTP status code.
Code string `json:"code"`
// Message is the rendered error description.
Message string `json:"message"`
// Details contains additional structured error information.
Details map[string]string `json:"details,omitempty"`
}
InjectedError represents an error that was injected into the pipeline.
func RenderError ¶
func RenderError(template *InjectionTemplate, overrides map[string]string) (*InjectedError, error)
RenderError generates an InjectedError from a template with placeholder overrides. It merges default placeholders with overrides, then renders the message and details templates.
func (*InjectedError) Error ¶
func (e *InjectedError) Error() string
Error implements the error interface for InjectedError.
func (*InjectedError) ToError ¶
func (e *InjectedError) ToError() error
ToError converts the InjectedError to a standard Go error.
type InjectionConfig ¶
type InjectionConfig struct {
// JobID is the identifier of the job this config applies to.
JobID string `json:"job_id"`
// Enabled determines whether error injection is active for this job.
Enabled bool `json:"enabled"`
// Rules contains the injection rules to apply.
Rules []InjectionRule `json:"rules,omitempty"`
// GlobalProbability is the default probability for rules without explicit probability.
GlobalProbability float64 `json:"global_probability,omitempty"`
}
InjectionConfig contains per-job error injection configuration.
func GetInjectionConfigFromContext ¶
func GetInjectionConfigFromContext(ctx context.Context) *InjectionConfig
GetInjectionConfigFromContext retrieves the per-job config from context Returns nil if the config is not present or context is nil
type InjectionContextKey ¶
type InjectionContextKey string
InjectionContextKey is the type for context keys used in error injection
const (
InjectionKeyConfig InjectionContextKey = "injection-config"
)
Context key constants for storing injection-related values
type InjectionPoint ¶
type InjectionPoint struct {
// ID is a unique identifier in the format "component.action".
ID string `json:"id"`
// Component is the pipeline component name.
Component string `json:"component"`
// Action is the specific operation within the component.
Action string `json:"action"`
// Description provides a human-readable explanation of the injection point.
Description string `json:"description"`
// Templates contains pre-defined error templates for this injection point.
Templates map[ErrorType]*InjectionTemplate `json:"templates,omitempty"`
}
InjectionPoint represents a specific point in the pipeline where errors can be injected.
func GetInjectionPoint ¶
func GetInjectionPoint(pointID string) (*InjectionPoint, bool)
GetInjectionPoint retrieves an injection point by ID from the default registry
type InjectionRule ¶
type InjectionRule struct {
// PointRef references the injection point by ID.
PointRef string `json:"point_ref"`
// ErrorType specifies which template to use from the injection point.
ErrorType ErrorType `json:"error_type"`
// Probability is the chance (0.0-1.0) that the error will trigger.
Probability float64 `json:"probability"`
// Overrides allows customization of template placeholder values.
Overrides map[string]string `json:"overrides,omitempty"`
}
InjectionRule defines a rule for injecting errors at specific points.
type InjectionTemplate ¶
type InjectionTemplate struct {
// Type is the category of error this template produces.
Type ErrorType `json:"type"`
// Code is the gRPC/HTTP status code for the error.
Code string `json:"code"`
// MessageTemplate is a Go template string with {{.variable}} syntax for error message.
MessageTemplate string `json:"message_template"`
// Placeholders contains template variables with their default values.
Placeholders map[string]string `json:"placeholders,omitempty"`
// DetailsTemplate contains optional structured error details templates.
DetailsTemplate map[string]string `json:"details_template,omitempty"`
}
InjectionTemplate defines a template for generating specific error types.
func GetInjectionTemplate ¶
func GetInjectionTemplate(pointID string, errorType ErrorType) (*InjectionTemplate, bool)
GetInjectionTemplate retrieves a specific injection template from an injection point
type Injector ¶
type Injector interface {
// CheckPoint checks if an error should be injected at the given point.
// Returns nil if no error should be injected, or an InjectedError if one should be.
CheckPoint(ctx context.Context, pointID string) error
// GetTrace retrieves the execution trace for a job.
GetTrace(ctx context.Context, jobID string) *ExecutionTrace
// GetGlobalConfig returns the current global configuration.
GetGlobalConfig() *GlobalInjectionConfig
// SetGlobalConfig updates the global configuration.
SetGlobalConfig(config *GlobalInjectionConfig) error
// RenderTemplate renders an error from the template for a given point.
RenderTemplate(pointID string, errorType ErrorType, overrides map[string]string) (*InjectedError, error)
}
Injector defines the interface for error injection.
type InjectorImpl ¶
type InjectorImpl struct {
// contains filtered or unexported fields
}
InjectorImpl implements the Injector interface.
func NewInjector ¶
func NewInjector() (*InjectorImpl, error)
NewInjector creates a new InjectorImpl instance.
func (*InjectorImpl) CheckPoint ¶
func (i *InjectorImpl) CheckPoint(ctx context.Context, pointID string) error
CheckPoint checks if an error should be injected at the given point.
func (*InjectorImpl) GetGlobalConfig ¶
func (i *InjectorImpl) GetGlobalConfig() *GlobalInjectionConfig
GetGlobalConfig returns the current global configuration.
func (*InjectorImpl) GetPoint ¶
func (i *InjectorImpl) GetPoint(pointID string) *InjectionPoint
GetPoint retrieves a registered injection point by ID.
func (*InjectorImpl) GetTrace ¶
func (i *InjectorImpl) GetTrace(ctx context.Context, jobID string) *ExecutionTrace
func (*InjectorImpl) ListPoints ¶
func (i *InjectorImpl) ListPoints() []*InjectionPoint
ListPoints returns all registered injection points.
func (*InjectorImpl) RenderTemplate ¶
func (i *InjectorImpl) RenderTemplate(pointID string, errorType ErrorType, overrides map[string]string) (*InjectedError, error)
RenderTemplate renders an error from the template for a given point.
func (*InjectorImpl) SetGlobalConfig ¶
func (i *InjectorImpl) SetGlobalConfig(config *GlobalInjectionConfig) error
SetGlobalConfig updates the global configuration.
type PointRecord ¶
type PointRecord struct {
// PointID is the identifier of the injection point that was evaluated.
PointID string `json:"point_id"`
// Timestamp is when the injection point was evaluated.
Timestamp time.Time `json:"timestamp"`
// Triggered indicates whether an error was actually injected.
Triggered bool `json:"triggered"`
// ContextSnapshot captures the relevant context at the time of evaluation.
ContextSnapshot map[string]string `json:"context_snapshot,omitempty"`
// Error contains the injected error if one was triggered.
Error *InjectedError `json:"error,omitempty"`
// TemplateUsed is the error type of the template that was applied.
TemplateUsed ErrorType `json:"template_used,omitempty"`
// OverridesApplied contains the placeholder overrides that were used.
OverridesApplied map[string]string `json:"overrides_applied,omitempty"`
// ProbabilityRoll is the random value that was rolled against the probability.
ProbabilityRoll float64 `json:"probability_roll"`
}
PointRecord tracks the execution and outcome of an injection point evaluation.
type TraceStore ¶
type TraceStore interface {
// AppendPoint adds a point record to the trace for the given job ID
AppendPoint(jobID string, point PointRecord) error
// Get retrieves the execution trace for the given job ID
Get(jobID string) (*ExecutionTrace, error)
// Delete removes the execution trace for the given job ID
Delete(jobID string) error
// List returns the most recent execution traces, limited to the specified count
List(limit int) ([]*ExecutionTrace, error)
}
TraceStore defines the interface for storing and retrieving execution traces