Documentation
¶
Index ¶
- func GetAvailableSchemaProviderTypes() []string
- func GetAvailableSourceTypes() []string
- func RegisterSchemaProvider(providerType string, factory SchemaProviderFactory) error
- func RegisterSourceType(sourceType string, factory SourceFactory) error
- func ResolveSchemaPath(config SchemaSourceConfig, path string) string
- func SetGlobalMetrics(metrics SourceMetrics)
- type AcknowledgementBarrier
- type ConfigProperty
- type ConfigSchema
- type DefaultSchemaProviderRegistry
- func (r *DefaultSchemaProviderRegistry) CreateProvider(config SchemaSourceConfig) (SchemaProvider, error)
- func (r *DefaultSchemaProviderRegistry) GetAvailableTypes() []string
- func (r *DefaultSchemaProviderRegistry) GetTypeInfo(providerType string) SchemaProviderTypeInfo
- func (r *DefaultSchemaProviderRegistry) RegisterSchemaProvider(providerType string, factory SchemaProviderFactory) error
- type DefaultSourceMetrics
- func (d *DefaultSourceMetrics) RecordError(sourceID, operation string, err error)
- func (d *DefaultSourceMetrics) RecordFactProcessed(sourceID, factType string)
- func (d *DefaultSourceMetrics) RecordHealthCheck(sourceID string, healthy bool)
- func (d *DefaultSourceMetrics) RecordLatency(sourceID string, duration time.Duration)
- type DefaultSourceRegistry
- func (r *DefaultSourceRegistry) CreateSource(config SourceConfig) (FactSource, error)
- func (r *DefaultSourceRegistry) GetAvailableTypes() []string
- func (r *DefaultSourceRegistry) GetTypeInfo(sourceType string) SourceTypeInfo
- func (r *DefaultSourceRegistry) RegisterSourceType(sourceType string, factory SourceFactory) error
- type FactMapping
- type FactSource
- type FormatConverter
- type Schema
- type SchemaDefinition
- type SchemaFormat
- type SchemaProvider
- type SchemaProviderFactory
- type SchemaProviderRegistry
- type SchemaProviderTypeInfo
- type SchemaSourceConfig
- type SourceConfig
- type SourceError
- type SourceFactory
- type SourceMetadata
- type SourceMetrics
- type SourceRegistry
- type SourceTypeInfo
- type Transformation
- type TypedFact
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GetAvailableSchemaProviderTypes ¶
func GetAvailableSchemaProviderTypes() []string
GetAvailableSchemaProviderTypes returns all registered schema provider types.
func GetAvailableSourceTypes ¶
func GetAvailableSourceTypes() []string
GetAvailableSourceTypes returns all registered source types
func RegisterSchemaProvider ¶
func RegisterSchemaProvider(providerType string, factory SchemaProviderFactory) error
RegisterSchemaProvider registers a schema provider type globally.
func RegisterSourceType ¶
func RegisterSourceType(sourceType string, factory SourceFactory) error
RegisterSourceType registers a source type globally
func ResolveSchemaPath ¶
func ResolveSchemaPath(config SchemaSourceConfig, path string) string
ResolveSchemaPath resolves a path relative to the schema source config's base directory.
func SetGlobalMetrics ¶
func SetGlobalMetrics(metrics SourceMetrics)
SetGlobalMetrics sets the global metrics implementation
Types ¶
type AcknowledgementBarrier ¶
type AcknowledgementBarrier struct {
// contains filtered or unexported fields
}
AcknowledgementBarrier commits a source checkpoint only after every fact in one source record has crossed the caller's durable boundary.
func NewAcknowledgementBarrier ¶
func NewAcknowledgementBarrier(count int, commit func(context.Context) error) *AcknowledgementBarrier
NewAcknowledgementBarrier creates one idempotent callback per emitted fact. The commit function must itself be idempotent because a failed response can leave its durable outcome unknown.
type ConfigProperty ¶
type ConfigProperty struct {
Type string `json:"type"` // "string", "int", "bool", "array", "object"
Description string `json:"description"`
Default interface{} `json:"default,omitempty"`
Examples []string `json:"examples,omitempty"`
}
ConfigProperty describes a single configuration property
type ConfigSchema ¶
type ConfigSchema struct {
Properties map[string]ConfigProperty `json:"properties"`
Required []string `json:"required"`
}
ConfigSchema describes the configuration schema for a source type
type DefaultSchemaProviderRegistry ¶
type DefaultSchemaProviderRegistry struct {
// contains filtered or unexported fields
}
DefaultSchemaProviderRegistry provides a simple in-memory registry.
func NewDefaultSchemaProviderRegistry ¶
func NewDefaultSchemaProviderRegistry() *DefaultSchemaProviderRegistry
NewDefaultSchemaProviderRegistry creates a new registry.
func (*DefaultSchemaProviderRegistry) CreateProvider ¶
func (r *DefaultSchemaProviderRegistry) CreateProvider(config SchemaSourceConfig) (SchemaProvider, error)
func (*DefaultSchemaProviderRegistry) GetAvailableTypes ¶
func (r *DefaultSchemaProviderRegistry) GetAvailableTypes() []string
func (*DefaultSchemaProviderRegistry) GetTypeInfo ¶
func (r *DefaultSchemaProviderRegistry) GetTypeInfo(providerType string) SchemaProviderTypeInfo
func (*DefaultSchemaProviderRegistry) RegisterSchemaProvider ¶
func (r *DefaultSchemaProviderRegistry) RegisterSchemaProvider(providerType string, factory SchemaProviderFactory) error
type DefaultSourceMetrics ¶
type DefaultSourceMetrics struct{}
DefaultSourceMetrics provides a no-op implementation
func (*DefaultSourceMetrics) RecordError ¶
func (d *DefaultSourceMetrics) RecordError(sourceID, operation string, err error)
func (*DefaultSourceMetrics) RecordFactProcessed ¶
func (d *DefaultSourceMetrics) RecordFactProcessed(sourceID, factType string)
func (*DefaultSourceMetrics) RecordHealthCheck ¶
func (d *DefaultSourceMetrics) RecordHealthCheck(sourceID string, healthy bool)
func (*DefaultSourceMetrics) RecordLatency ¶
func (d *DefaultSourceMetrics) RecordLatency(sourceID string, duration time.Duration)
type DefaultSourceRegistry ¶
type DefaultSourceRegistry struct {
// contains filtered or unexported fields
}
DefaultSourceRegistry provides a simple in-memory registry
func NewDefaultSourceRegistry ¶
func NewDefaultSourceRegistry() *DefaultSourceRegistry
NewDefaultSourceRegistry creates a new default registry
func (*DefaultSourceRegistry) CreateSource ¶
func (r *DefaultSourceRegistry) CreateSource(config SourceConfig) (FactSource, error)
func (*DefaultSourceRegistry) GetAvailableTypes ¶
func (r *DefaultSourceRegistry) GetAvailableTypes() []string
func (*DefaultSourceRegistry) GetTypeInfo ¶
func (r *DefaultSourceRegistry) GetTypeInfo(sourceType string) SourceTypeInfo
func (*DefaultSourceRegistry) RegisterSourceType ¶
func (r *DefaultSourceRegistry) RegisterSourceType(sourceType string, factory SourceFactory) error
type FactMapping ¶
type FactMapping struct {
SourceKey string `json:"source_key" yaml:"source_key"` // Source-specific key (topic, table, etc.)
EffectusType string `json:"effectus_type" yaml:"effectus_type"` // Target Effectus fact type
SchemaVersion string `json:"schema_version" yaml:"schema_version"` // Target schema version
}
FactMapping maps source-specific identifiers to Effectus fact types
type FactSource ¶
type FactSource interface {
// Subscribe to facts with schema validation
Subscribe(ctx context.Context, factTypes []string) (<-chan *TypedFact, error)
// Start the source (for sources that need initialization)
Start(ctx context.Context) error
// Stop the source gracefully
Stop(ctx context.Context) error
// Get schema information for this source
GetSourceSchema() *Schema
// Health check
HealthCheck() error
// Source metadata
GetMetadata() SourceMetadata
}
FactSource represents any source that can provide typed facts to Effectus
func CreateSource ¶
func CreateSource(config SourceConfig) (FactSource, error)
CreateSource creates a source from configuration using the global registry
type FormatConverter ¶
type FormatConverter interface {
// Convert raw data to proto message using target schema
Convert(rawData []byte, targetSchema *Schema) (proto.Message, error)
// Check if converter can handle the source->target conversion
CanConvert(sourceFormat, targetFormat string) bool
// Get supported formats
GetSupportedFormats() []string
}
FormatConverter handles conversion from external formats to proto
type SchemaDefinition ¶
type SchemaDefinition struct {
Name string `json:"name" yaml:"name"`
Version string `json:"version" yaml:"version"`
Format SchemaFormat `json:"format" yaml:"format"`
Data []byte `json:"-" yaml:"-"`
Source string `json:"source,omitempty" yaml:"source,omitempty"`
}
SchemaDefinition represents a typed schema payload returned by a provider.
type SchemaFormat ¶
type SchemaFormat string
SchemaFormat describes how to interpret a schema definition payload.
const ( SchemaFormatAuto SchemaFormat = "auto" SchemaFormatJSONSchema SchemaFormat = "jsonschema" SchemaFormatEffectus SchemaFormat = "effectus" SchemaFormatProto SchemaFormat = "proto" )
type SchemaProvider ¶
type SchemaProvider interface {
LoadSchemas(ctx context.Context) ([]SchemaDefinition, error)
Close() error
}
SchemaProvider loads schema definitions from an external system.
func CreateSchemaProvider ¶
func CreateSchemaProvider(config SchemaSourceConfig) (SchemaProvider, error)
CreateSchemaProvider creates a provider from configuration.
type SchemaProviderFactory ¶
type SchemaProviderFactory interface {
Create(config SchemaSourceConfig) (SchemaProvider, error)
ValidateConfig(config SchemaSourceConfig) error
GetConfigSchema() ConfigSchema
}
SchemaProviderFactory constructs schema providers.
type SchemaProviderRegistry ¶
type SchemaProviderRegistry interface {
RegisterSchemaProvider(providerType string, factory SchemaProviderFactory) error
CreateProvider(config SchemaSourceConfig) (SchemaProvider, error)
GetAvailableTypes() []string
GetTypeInfo(providerType string) SchemaProviderTypeInfo
}
SchemaProviderRegistry manages available schema providers.
type SchemaProviderTypeInfo ¶
type SchemaProviderTypeInfo struct {
Type string `json:"type"`
Description string `json:"description"`
Capabilities []string `json:"capabilities"`
ConfigSchema ConfigSchema `json:"config_schema"`
Examples []string `json:"examples"`
}
SchemaProviderTypeInfo provides information about a schema provider type.
type SchemaSourceConfig ¶
type SchemaSourceConfig struct {
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
Namespace string `json:"namespace" yaml:"namespace"`
Version string `json:"version" yaml:"version"`
Config map[string]interface{} `json:"config" yaml:"config"`
BaseDir string `json:"-" yaml:"-"`
}
SchemaSourceConfig represents configuration for a schema source provider.
type SourceConfig ¶
type SourceConfig struct {
SourceID string `json:"source_id" yaml:"source_id"`
Type string `json:"type" yaml:"type"`
Config map[string]interface{} `json:"config" yaml:"config"`
Mappings []FactMapping `json:"mappings" yaml:"mappings"`
Transforms []Transformation `json:"transforms" yaml:"transforms"`
Tags []string `json:"tags" yaml:"tags"`
}
SourceConfig represents configuration for any source
type SourceError ¶
type SourceError struct {
SourceID string `json:"source_id"`
Operation string `json:"operation"`
Message string `json:"message"`
Cause error `json:"-"`
}
SourceError represents errors from source operations
func NewSourceError ¶
func NewSourceError(sourceID, operation, message string, cause error) *SourceError
NewSourceError creates a new source error
func (*SourceError) Error ¶
func (e *SourceError) Error() string
func (*SourceError) Unwrap ¶
func (e *SourceError) Unwrap() error
type SourceFactory ¶
type SourceFactory interface {
// Create a new source instance
Create(config SourceConfig) (FactSource, error)
// Validate configuration before creating
ValidateConfig(config SourceConfig) error
// Get configuration schema for this source type
GetConfigSchema() ConfigSchema
}
SourceFactory creates instances of a specific source type
type SourceMetadata ¶
type SourceMetadata struct {
SourceID string `json:"source_id"`
SourceType string `json:"source_type"` // "kafka", "http", "database", "file"
Version string `json:"version"`
Capabilities []string `json:"capabilities"` // ["streaming", "batch", "realtime"]
SchemaFormats []string `json:"schema_formats"` // ["protobuf", "json", "avro"]
Config map[string]string `json:"config,omitempty"`
Tags []string `json:"tags,omitempty"`
}
SourceMetadata provides information about fact sources
type SourceMetrics ¶
type SourceMetrics interface {
// Record facts processed
RecordFactProcessed(sourceID, factType string)
// Record processing errors
RecordError(sourceID, operation string, err error)
// Record processing latency
RecordLatency(sourceID string, duration time.Duration)
// Record source health
RecordHealthCheck(sourceID string, healthy bool)
}
SourceMetrics provides observability for sources
func GetGlobalMetrics ¶
func GetGlobalMetrics() SourceMetrics
GetGlobalMetrics returns the global metrics implementation
type SourceRegistry ¶
type SourceRegistry interface {
// Register a source type with factory function
RegisterSourceType(sourceType string, factory SourceFactory) error
// Create a source instance from configuration
CreateSource(config SourceConfig) (FactSource, error)
// List available source types
GetAvailableTypes() []string
// Get source type information
GetTypeInfo(sourceType string) SourceTypeInfo
}
SourceRegistry manages available fact sources
type SourceTypeInfo ¶
type SourceTypeInfo struct {
Type string `json:"type"`
Description string `json:"description"`
Capabilities []string `json:"capabilities"`
ConfigSchema ConfigSchema `json:"config_schema"`
Examples []string `json:"examples"`
}
SourceTypeInfo provides information about a source type
type Transformation ¶
type Transformation struct {
SourcePath string `json:"source_path" yaml:"source_path"` // JSONPath or similar
TargetType string `json:"target_type" yaml:"target_type"` // Target fact type
Mapping map[string]string `json:"mapping" yaml:"mapping"` // Field mappings
}
Transformation defines how to transform data from source format
type TypedFact ¶
type TypedFact struct {
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
Data proto.Message `json:"-"` // Proto message data
RawData []byte `json:"raw_data,omitempty"` // Original raw data
Timestamp time.Time `json:"timestamp"`
SourceID string `json:"source_id"`
TraceID string `json:"trace_id,omitempty"`
SpanID string `json:"span_id,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
// Acknowledge confirms that a source record reached the caller's durable
// processing boundary. Sources that support acknowledgements leave the
// record pending until this callback succeeds. The callback is safe to
// retry; callers that do not invoke it intentionally request redelivery.
Acknowledge func(context.Context) error `json:"-"`
}
TypedFact represents a fact with full schema information