schema

package
v0.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 14, 2025 License: MIT Imports: 14 Imported by: 0

Documentation

Overview

Package schema provides advanced schema evolution and type inference capabilities

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ChangeType 

type ChangeType string

ChangeType represents the type of schema change 

const (
	ChangeTypeAddField       ChangeType = "ADD_FIELD"
	ChangeTypeRemoveField    ChangeType = "REMOVE_FIELD"
	ChangeTypeRenameField    ChangeType = "RENAME_FIELD"
	ChangeTypeModifyType     ChangeType = "MODIFY_TYPE"
	ChangeTypeModifyRequired ChangeType = "MODIFY_REQUIRED"
)

type CompatibilityMode 

type CompatibilityMode string

CompatibilityMode defines how schema changes are validated 

const (
	// CompatibilityNone allows any schema change
	CompatibilityNone CompatibilityMode = "NONE"
	// CompatibilityBackward ensures new schema can read old data
	CompatibilityBackward CompatibilityMode = "BACKWARD"
	// CompatibilityForward ensures old schema can read new data
	CompatibilityForward CompatibilityMode = "FORWARD"
	// CompatibilityFull ensures bidirectional compatibility
	CompatibilityFull CompatibilityMode = "FULL"
	// CompatibilityBackwardTransitive ensures compatibility with all previous versions
	CompatibilityBackwardTransitive CompatibilityMode = "BACKWARD_TRANSITIVE"
)

type DefaultEvolutionStrategy 

type DefaultEvolutionStrategy struct {
	// contains filtered or unexported fields
}

DefaultEvolutionStrategy allows most schema changes

func (*DefaultEvolutionStrategy) CanEvolve 

func (s *DefaultEvolutionStrategy) CanEvolve(old, new *models.Schema) bool

func (*DefaultEvolutionStrategy) Evolve 

func (s *DefaultEvolutionStrategy) Evolve(old, new *models.Schema) (*models.Schema, error)

func (*DefaultEvolutionStrategy) GetMigrationPlan 

func (s *DefaultEvolutionStrategy) GetMigrationPlan(old, new *models.Schema) *MigrationPlan

type EvolutionManager 

type EvolutionManager struct {
	// contains filtered or unexported fields
}

EvolutionManager handles schema evolution and compatibility

func NewEvolutionManager 

func NewEvolutionManager(registry *Registry, logger *zap.Logger) *EvolutionManager

NewEvolutionManager creates a new evolution manager

func (*EvolutionManager) CheckCompatibility 

func (em *EvolutionManager) CheckCompatibility(old, new *models.Schema, mode CompatibilityMode) error

CheckCompatibility checks if two schemas are compatible

func (*EvolutionManager) EvolveSchema 

func (em *EvolutionManager) EvolveSchema(current *models.Schema, newData []map[string]interface{}) (*models.Schema, error)

EvolveSchema evolves a schema based on new data

func (*EvolutionManager) GetMigrationPlan 

func (em *EvolutionManager) GetMigrationPlan(fromVersion, toVersion *SchemaVersion) (*MigrationPlan, error)

GetMigrationPlan creates a migration plan between schema versions

func (*EvolutionManager) RegisterStrategy 

func (em *EvolutionManager) RegisterStrategy(name string, strategy EvolutionStrategy)

RegisterStrategy registers a custom evolution strategy

type EvolutionStrategy 

type EvolutionStrategy interface {
	CanEvolve(old, new *models.Schema) bool
	Evolve(old, new *models.Schema) (*models.Schema, error)
	GetMigrationPlan(old, new *models.Schema) *MigrationPlan
}

EvolutionStrategy defines how schema changes are handled

type FlexibleEvolutionStrategy 

type FlexibleEvolutionStrategy struct {
	// contains filtered or unexported fields
}

FlexibleEvolutionStrategy allows all changes with migration support

func (*FlexibleEvolutionStrategy) CanEvolve 

func (s *FlexibleEvolutionStrategy) CanEvolve(old, new *models.Schema) bool

func (*FlexibleEvolutionStrategy) Evolve 

func (s *FlexibleEvolutionStrategy) Evolve(old, new *models.Schema) (*models.Schema, error)

func (*FlexibleEvolutionStrategy) GetMigrationPlan 

func (s *FlexibleEvolutionStrategy) GetMigrationPlan(old, new *models.Schema) *MigrationPlan

type InferredType 

type InferredType struct {
	Type          string         `json:"type"`
	Format        string         `json:"format,omitempty"`
	Confidence    float64        `json:"confidence"`
	Nullable      bool           `json:"nullable"`
	Cardinality   int            `json:"cardinality"`
	Examples      []interface{}  `json:"examples,omitempty"`
	Distribution  map[string]int `json:"distribution,omitempty"`
	NumericStats  *NumericStats  `json:"numeric_stats,omitempty"`
	StringStats   *StringStats   `json:"string_stats,omitempty"`
	TemporalStats *TemporalStats `json:"temporal_stats,omitempty"`
}

InferredType represents a type inference result with confidence

type MigrationEngine 

type MigrationEngine struct {
	// contains filtered or unexported fields
}

MigrationEngine handles data migration between schema versions

func NewMigrationEngine 

func NewMigrationEngine(logger *zap.Logger, registry *Registry) *MigrationEngine

NewMigrationEngine creates a new migration engine

func (*MigrationEngine) MigrateBatch 

func (me *MigrationEngine) MigrateBatch(records []map[string]interface{}, plan *MigrationPlan) ([]map[string]interface{}, error)

MigrateBatch migrates a batch of records

func (*MigrationEngine) MigrateRecord 

func (me *MigrationEngine) MigrateRecord(record map[string]interface{}, plan *MigrationPlan) (map[string]interface{}, error)

MigrateRecord migrates a single record according to a migration plan

func (*MigrationEngine) ValidateRecord 

func (me *MigrationEngine) ValidateRecord(record map[string]interface{}, schema *models.Schema) []ValidationError

ValidateRecord validates a record against a schema

type MigrationPlan 

type MigrationPlan struct {
	FromVersion int                    `json:"from_version"`
	ToVersion   int                    `json:"to_version"`
	Changes     []SchemaChange         `json:"changes"`
	Strategy    string                 `json:"strategy"`
	Metadata    map[string]interface{} `json:"metadata"`
}

MigrationPlan describes how to migrate data between schema versions

type MigrationRule 

type MigrationRule struct {
	Type           string                        `json:"type"`
	DefaultValue   interface{}                   `json:"default_value,omitempty"`
	Transformation string                        `json:"transformation,omitempty"`
	CustomLogic    func(interface{}) interface{} `json:"-"`
}

MigrationRule defines how to migrate data for a field

type NumericStats 

type NumericStats struct {
	Min       float64 `json:"min"`
	Max       float64 `json:"max"`
	Mean      float64 `json:"mean"`
	Median    float64 `json:"median"`
	StdDev    float64 `json:"std_dev"`
	Precision int     `json:"precision"`
	Scale     int     `json:"scale"`
}

NumericStats holds statistics for numeric types

type Registry 

type Registry struct {
	// contains filtered or unexported fields
}

Registry manages schema versions and evolution

func NewRegistry 

func NewRegistry(logger *zap.Logger) *Registry

NewRegistry creates a new schema registry

func (*Registry) EvolveSchema 

func (r *Registry) EvolveSchema(ctx context.Context, subject string, newData []map[string]interface{}) (*SchemaVersion, error)

EvolveSchema attempts to evolve a schema based on new data

func (*Registry) Export 

func (r *Registry) Export() ([]byte, error)

Export exports the registry state

func (*Registry) GetEvolutionManager 

func (r *Registry) GetEvolutionManager() *EvolutionManager

GetEvolutionManager returns the evolution manager

func (*Registry) GetLatestSchema 

func (r *Registry) GetLatestSchema(subject string) (*SchemaVersion, error)

GetLatestSchema retrieves the latest schema version

func (*Registry) GetSchema 

func (r *Registry) GetSchema(subject string, version int) (*SchemaVersion, error)

GetSchema retrieves a specific schema version

func (*Registry) GetSchemaHistory 

func (r *Registry) GetSchemaHistory(subject string) ([]*SchemaVersion, error)

GetSchemaHistory returns all versions of a schema

func (*Registry) Import 

func (r *Registry) Import(data []byte) error

Import imports registry state

func (*Registry) InferSchema 

func (r *Registry) InferSchema(ctx context.Context, subject string, samples []map[string]interface{}) (*models.Schema, error)

InferSchema uses advanced type inference to detect schema from data

func (*Registry) OnSchemaChange 

func (r *Registry) OnSchemaChange(callback func(subject string, old, new *SchemaVersion))

OnSchemaChange registers a callback for schema changes

func (*Registry) RegisterSchema 

func (r *Registry) RegisterSchema(ctx context.Context, subject string, schema *models.Schema) (*SchemaVersion, error)

RegisterSchema registers a new schema version

func (*Registry) SetCompatibilityMode 

func (r *Registry) SetCompatibilityMode(subject string, mode CompatibilityMode)

SetCompatibilityMode sets the compatibility mode for a subject

type SchemaChange 

type SchemaChange struct {
	Type      ChangeType    `json:"type"`
	Field     string        `json:"field"`
	OldField  *models.Field `json:"old_field,omitempty"`
	NewField  *models.Field `json:"new_field,omitempty"`
	Migration MigrationRule `json:"migration,omitempty"`
}

SchemaChange represents a single change in a schema

type SchemaVersion 

type SchemaVersion struct {
	Version       int                    `json:"version"`
	Schema        *models.Schema         `json:"schema"`
	CreatedAt     time.Time              `json:"created_at"`
	UpdatedAt     time.Time              `json:"updated_at"`
	Fingerprint   string                 `json:"fingerprint"`
	Compatibility CompatibilityMode      `json:"compatibility"`
	Metadata      map[string]interface{} `json:"metadata"`
}

SchemaVersion represents a version of a schema

type StrictEvolutionStrategy 

type StrictEvolutionStrategy struct {
	// contains filtered or unexported fields
}

StrictEvolutionStrategy allows only safe schema changes

func (*StrictEvolutionStrategy) CanEvolve 

func (s *StrictEvolutionStrategy) CanEvolve(old, new *models.Schema) bool

func (*StrictEvolutionStrategy) Evolve 

func (s *StrictEvolutionStrategy) Evolve(old, new *models.Schema) (*models.Schema, error)

func (*StrictEvolutionStrategy) GetMigrationPlan 

func (s *StrictEvolutionStrategy) GetMigrationPlan(old, new *models.Schema) *MigrationPlan

type StringStats 

type StringStats struct {
	MinLength    int      `json:"min_length"`
	MaxLength    int      `json:"max_length"`
	AvgLength    float64  `json:"avg_length"`
	Pattern      string   `json:"pattern,omitempty"`
	CommonValues []string `json:"common_values,omitempty"`
}

StringStats holds statistics for string types

type TemporalStats 

type TemporalStats struct {
	MinDate  time.Time `json:"min_date"`
	MaxDate  time.Time `json:"max_date"`
	Format   string    `json:"format"`
	Timezone string    `json:"timezone,omitempty"`
}

TemporalStats holds statistics for temporal types

type TransformationFunc 

type TransformationFunc func(record map[string]interface{}, change *SchemaChange) error

TransformationFunc applies custom transformations

type TypeConverter 

type TypeConverter func(value interface{}, targetType string) (interface{}, error)

TypeConverter converts values between types

type TypeInferenceEngine 

type TypeInferenceEngine struct {
	// contains filtered or unexported fields
}

TypeInferenceEngine provides advanced type detection capabilities

func NewTypeInferenceEngine 

func NewTypeInferenceEngine(logger *zap.Logger) *TypeInferenceEngine

NewTypeInferenceEngine creates a new type inference engine

func (*TypeInferenceEngine) InferSchema 

func (e *TypeInferenceEngine) InferSchema(name string, samples []map[string]interface{}) (*models.Schema, error)

InferSchema infers a complete schema from sample data

func (*TypeInferenceEngine) InferType 

func (e *TypeInferenceEngine) InferType(fieldName string, values []interface{}) *InferredType

InferType infers the type of a field from sample values

type ValidationError 

type ValidationError struct {
	Field   string              `json:"field"`
	VType   ValidationErrorType `json:"type"`
	Message string              `json:"message"`
}

ValidationError represents a validation error

type ValidationErrorType 

type ValidationErrorType string

ValidationErrorType represents the type of validation error 

const (
	ValidationErrorRequired   ValidationErrorType = "REQUIRED"
	ValidationErrorTypeError  ValidationErrorType = "TYPE"
	ValidationErrorFormat     ValidationErrorType = "FORMAT"
	ValidationErrorConstraint ValidationErrorType = "CONSTRAINT"
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.