Documentation
¶
Index ¶
- type BatchProcessor
- type CacheItem
- type ConsoleProgressReporter
- type DeduplicationStats
- type Deduplicator
- type DefaultProgressReporter
- func (pr *DefaultProgressReporter) AddError(err error)
- func (pr *DefaultProgressReporter) AddWarning(message string)
- func (pr *DefaultProgressReporter) Complete(success bool)
- func (pr *DefaultProgressReporter) FinishPhase(phase ProcessingPhase, success bool, message string)
- func (pr *DefaultProgressReporter) GetSummary() ProgressSummary
- func (pr *DefaultProgressReporter) SetOverallProgress(current, total int)
- func (pr *DefaultProgressReporter) StartPhase(phase ProcessingPhase, total int)
- func (pr *DefaultProgressReporter) UpdateProgress(current int, message string)
- type FileStreamReader
- type LRUCache
- type MemoryManager
- func (mm *MemoryManager) GetBuffer() []byte
- func (mm *MemoryManager) GetMemoryStats() MemoryStats
- func (mm *MemoryManager) GetStatement() *types.Statement
- func (mm *MemoryManager) PutBuffer(buf []byte)
- func (mm *MemoryManager) PutStatement(stmt *types.Statement)
- func (mm *MemoryManager) ReleaseMemory(size int64)
- func (mm *MemoryManager) TrackMemoryUsage(size int64) bool
- type MemoryOptimizedStatement
- type MemoryStats
- type MigrationFile
- type PhaseInfo
- type ProcessedFile
- type ProcessingPhase
- type ProcessingStats
- type ProgressAggregator
- type ProgressReporter
- type ProgressSummary
- type ProgressTracker
- type StreamingProcessor
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BatchProcessor ¶
type BatchProcessor struct {
// contains filtered or unexported fields
}
BatchProcessor handles batch processing with configurable batch sizes
func NewBatchProcessor ¶
func NewBatchProcessor(batchSize int, maxBatchSizeMB int, memManager *MemoryManager) *BatchProcessor
NewBatchProcessor creates a new batch processor
func (*BatchProcessor) AddFile ¶
func (bp *BatchProcessor) AddFile(file *MigrationFile) (bool, error)
AddFile adds a file to the current batch
func (*BatchProcessor) GetCurrentBatch ¶
func (bp *BatchProcessor) GetCurrentBatch() []*MigrationFile
GetCurrentBatch returns the current batch and resets it
func (*BatchProcessor) HasPendingBatch ¶
func (bp *BatchProcessor) HasPendingBatch() bool
HasPendingBatch returns true if there are files in the current batch
type CacheItem ¶
type CacheItem[K comparable, V any] struct { Key K Value V }
CacheItem represents a cached item
type ConsoleProgressReporter ¶
type ConsoleProgressReporter struct {
*DefaultProgressReporter
// contains filtered or unexported fields
}
ConsoleProgressReporter reports progress to console output
func NewConsoleProgressReporter ¶
func NewConsoleProgressReporter(ctx context.Context, printInterval time.Duration) *ConsoleProgressReporter
NewConsoleProgressReporter creates a console progress reporter
type DeduplicationStats ¶
type DeduplicationStats struct {
UniqueItems int `json:"unique_items"`
}
DeduplicationStats provides deduplication information
type Deduplicator ¶
type Deduplicator[T any] struct { // contains filtered or unexported fields }
Deduplicator handles duplicate statement detection and removal
func NewDeduplicator ¶
func NewDeduplicator[T any](hashFunc func(T) string) *Deduplicator[T]
NewDeduplicator creates a new deduplicator with the specified hash function
func (*Deduplicator[T]) GetStats ¶
func (d *Deduplicator[T]) GetStats() DeduplicationStats
GetStats returns deduplication statistics
func (*Deduplicator[T]) IsDuplicate ¶
func (d *Deduplicator[T]) IsDuplicate(item T) bool
IsDuplicate checks if an item has been seen before
func (*Deduplicator[T]) Reset ¶
func (d *Deduplicator[T]) Reset()
Reset clears the deduplicator state
type DefaultProgressReporter ¶
type DefaultProgressReporter struct {
// contains filtered or unexported fields
}
DefaultProgressReporter is a concrete implementation of ProgressReporter
func NewProgressReporter ¶
func NewProgressReporter(ctx context.Context, updateCallback func(ProgressSummary)) *DefaultProgressReporter
NewProgressReporter creates a new progress reporter
func (*DefaultProgressReporter) AddError ¶
func (pr *DefaultProgressReporter) AddError(err error)
AddError adds an error
func (*DefaultProgressReporter) AddWarning ¶
func (pr *DefaultProgressReporter) AddWarning(message string)
AddWarning adds a warning message
func (*DefaultProgressReporter) Complete ¶
func (pr *DefaultProgressReporter) Complete(success bool)
Complete marks the overall process as complete
func (*DefaultProgressReporter) FinishPhase ¶
func (pr *DefaultProgressReporter) FinishPhase(phase ProcessingPhase, success bool, message string)
FinishPhase completes the current phase
func (*DefaultProgressReporter) GetSummary ¶
func (pr *DefaultProgressReporter) GetSummary() ProgressSummary
GetSummary returns the current progress summary
func (*DefaultProgressReporter) SetOverallProgress ¶
func (pr *DefaultProgressReporter) SetOverallProgress(current, total int)
SetOverallProgress sets the overall progress
func (*DefaultProgressReporter) StartPhase ¶
func (pr *DefaultProgressReporter) StartPhase(phase ProcessingPhase, total int)
StartPhase starts a new processing phase
func (*DefaultProgressReporter) UpdateProgress ¶
func (pr *DefaultProgressReporter) UpdateProgress(current int, message string)
UpdateProgress updates the progress of the current phase
type FileStreamReader ¶
type FileStreamReader struct {
// contains filtered or unexported fields
}
FileStreamReader provides streaming file reading capabilities
func NewFileStreamReader ¶
func NewFileStreamReader(reader io.Reader, bufferSizeKB int, memManager *MemoryManager) *FileStreamReader
NewFileStreamReader creates a new file stream reader
func (*FileStreamReader) GetPosition ¶
func (fsr *FileStreamReader) GetPosition() int64
GetPosition returns the current read position
func (*FileStreamReader) ReadChunk ¶
func (fsr *FileStreamReader) ReadChunk() ([]byte, error)
ReadChunk reads the next chunk of data
type LRUCache ¶
type LRUCache[K comparable, V any] struct { // contains filtered or unexported fields }
LRUCache provides memory-efficient caching with least-recently-used eviction
func NewLRUCache ¶
func NewLRUCache[K comparable, V any](capacity int) *LRUCache[K, V]
NewLRUCache creates a new LRU cache with the specified capacity
func (*LRUCache[K, V]) Clear ¶
func (c *LRUCache[K, V]) Clear()
Clear removes all items from the cache
type MemoryManager ¶
type MemoryManager struct {
// contains filtered or unexported fields
}
MemoryManager handles memory-efficient processing with deduplication and size limits
func NewMemoryManager ¶
func NewMemoryManager(maxMemoryMB int) *MemoryManager
NewMemoryManager creates a memory manager with the specified limits
func (*MemoryManager) GetBuffer ¶
func (mm *MemoryManager) GetBuffer() []byte
GetBuffer retrieves a pooled byte buffer
func (*MemoryManager) GetMemoryStats ¶
func (mm *MemoryManager) GetMemoryStats() MemoryStats
GetMemoryStats returns current memory usage statistics
func (*MemoryManager) GetStatement ¶
func (mm *MemoryManager) GetStatement() *types.Statement
GetStatement retrieves a pooled statement object
func (*MemoryManager) PutBuffer ¶
func (mm *MemoryManager) PutBuffer(buf []byte)
PutBuffer returns a buffer to the pool
func (*MemoryManager) PutStatement ¶
func (mm *MemoryManager) PutStatement(stmt *types.Statement)
PutStatement returns a statement to the pool
func (*MemoryManager) ReleaseMemory ¶
func (mm *MemoryManager) ReleaseMemory(size int64)
ReleaseMemory decreases tracked memory usage
func (*MemoryManager) TrackMemoryUsage ¶
func (mm *MemoryManager) TrackMemoryUsage(size int64) bool
TrackMemoryUsage updates current memory usage
type MemoryOptimizedStatement ¶
MemoryOptimizedStatement wraps a statement with memory optimization features
func NewMemoryOptimizedStatement ¶
func NewMemoryOptimizedStatement(mm *MemoryManager, stmt *types.Statement) *MemoryOptimizedStatement
NewMemoryOptimizedStatement creates a statement with memory tracking
func (*MemoryOptimizedStatement) Release ¶
func (mos *MemoryOptimizedStatement) Release()
Release frees the memory used by this statement
type MemoryStats ¶
type MemoryStats struct {
MaxMemoryBytes int64 `json:"max_memory_bytes"`
CurrentMemoryBytes int64 `json:"current_memory_bytes"`
SystemMemoryBytes int64 `json:"system_memory_bytes"`
CacheSize int `json:"cache_size"`
CacheHitRate float64 `json:"cache_hit_rate"`
}
MemoryStats provides memory usage information
type MigrationFile ¶
MigrationFile represents a migration file to be processed
type PhaseInfo ¶
type PhaseInfo struct {
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Duration time.Duration `json:"duration"`
Total int `json:"total"`
Current int `json:"current"`
Progress float64 `json:"progress"`
Message string `json:"message"`
Success bool `json:"success"`
IsActive bool `json:"is_active"`
IsComplete bool `json:"is_complete"`
}
PhaseInfo contains information about a processing phase
type ProcessedFile ¶
type ProcessedFile struct {
OriginalFile *MigrationFile
Migration *types.Migration
ProcessTime time.Duration
MemoryUsed int64
Errors []error
}
ProcessedFile represents a processed migration file
type ProcessingPhase ¶
type ProcessingPhase string
ProcessingPhase represents different phases of migration processing
const ( PhaseDiscovery ProcessingPhase = "DISCOVERY" PhaseValidation ProcessingPhase = "VALIDATION" PhaseParsing ProcessingPhase = "PARSING" PhaseAnalysis ProcessingPhase = "ANALYSIS" PhaseDependencies ProcessingPhase = "DEPENDENCIES" PhaseSquashing ProcessingPhase = "SQUASHING" PhaseOptimization ProcessingPhase = "OPTIMIZATION" PhaseGeneration ProcessingPhase = "GENERATION" PhaseOutput ProcessingPhase = "OUTPUT" PhaseCompletion ProcessingPhase = "COMPLETION" )
type ProcessingStats ¶
type ProcessingStats struct {
FilesProcessed int64 `json:"files_processed"`
FilesSkipped int64 `json:"files_skipped"`
FilesErrored int64 `json:"files_errored"`
TotalBytes int64 `json:"total_bytes"`
ProcessingTime int64 `json:"processing_time_ms"`
PeakMemoryUsage int64 `json:"peak_memory_usage"`
AverageFileSize int64 `json:"average_file_size"`
ThroughputMBps float64 `json:"throughput_mbps"`
}
ProcessingStats tracks processing statistics
type ProgressAggregator ¶
type ProgressAggregator struct {
// contains filtered or unexported fields
}
ProgressAggregator aggregates progress from multiple sources
func NewProgressAggregator ¶
func NewProgressAggregator() *ProgressAggregator
NewProgressAggregator creates a new progress aggregator
func (*ProgressAggregator) AddSource ¶
func (pa *ProgressAggregator) AddSource(name string, reporter ProgressReporter, weight float64)
AddSource adds a progress source with a weight
func (*ProgressAggregator) GetAggregatedProgress ¶
func (pa *ProgressAggregator) GetAggregatedProgress() ProgressSummary
GetAggregatedProgress returns the weighted average progress
type ProgressReporter ¶
type ProgressReporter interface {
StartPhase(phase ProcessingPhase, total int)
UpdateProgress(current int, message string)
FinishPhase(phase ProcessingPhase, success bool, message string)
SetOverallProgress(current, total int)
AddWarning(message string)
AddError(err error)
GetSummary() ProgressSummary
}
ProgressReporter interface for reporting progress
type ProgressSummary ¶
type ProgressSummary struct {
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Duration time.Duration `json:"duration"`
CurrentPhase ProcessingPhase `json:"current_phase"`
OverallProgress float64 `json:"overall_progress"`
PhaseProgress map[ProcessingPhase]float64 `json:"phase_progress"`
PhaseDetails map[ProcessingPhase]PhaseInfo `json:"phase_details"`
Warnings []string `json:"warnings"`
Errors []string `json:"errors"`
IsComplete bool `json:"is_complete"`
Success bool `json:"success"`
}
ProgressSummary provides a summary of the progress
type ProgressTracker ¶
type ProgressTracker struct {
// contains filtered or unexported fields
}
ProgressTracker tracks processing progress for large operations
func NewProgressTracker ¶
func NewProgressTracker(total int64, updateFreq time.Duration, callback func(int64, int64, float64)) *ProgressTracker
NewProgressTracker creates a new progress tracker
func (*ProgressTracker) GetProgress ¶
func (pt *ProgressTracker) GetProgress() (current, total int64, percentage float64)
GetProgress returns current progress statistics
func (*ProgressTracker) Update ¶
func (pt *ProgressTracker) Update(increment int64)
Update updates the progress counter
type StreamingProcessor ¶
type StreamingProcessor struct {
// contains filtered or unexported fields
}
StreamingProcessor handles memory-efficient processing of large migration sets
func NewStreamingProcessor ¶
func NewStreamingProcessor(batchSize, workerCount int, memManager *MemoryManager) *StreamingProcessor
NewStreamingProcessor creates a new streaming processor
func (*StreamingProcessor) GetResults ¶
func (sp *StreamingProcessor) GetResults() <-chan *ProcessedFile
GetResults returns a channel of processed files
func (*StreamingProcessor) GetStats ¶
func (sp *StreamingProcessor) GetStats() ProcessingStats
GetStats returns current processing statistics
func (*StreamingProcessor) ProcessDirectory ¶
func (sp *StreamingProcessor) ProcessDirectory(dir string) error
ProcessDirectory processes all migration files in a directory
func (*StreamingProcessor) Start ¶
func (sp *StreamingProcessor) Start()
Start begins the streaming processing
func (*StreamingProcessor) Stop ¶
func (sp *StreamingProcessor) Stop() error
Stop gracefully stops the streaming processor