performance

package
v0.11.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 9, 2026 License: MIT Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BatchProcessor

type BatchProcessor struct {
	// contains filtered or unexported fields
}

BatchProcessor handles batch processing with configurable batch sizes

func NewBatchProcessor

func NewBatchProcessor(batchSize int, maxBatchSizeMB int, memManager *MemoryManager) *BatchProcessor

NewBatchProcessor creates a new batch processor

func (*BatchProcessor) AddFile

func (bp *BatchProcessor) AddFile(file *MigrationFile) (bool, error)

AddFile adds a file to the current batch

func (*BatchProcessor) GetCurrentBatch

func (bp *BatchProcessor) GetCurrentBatch() []*MigrationFile

GetCurrentBatch returns the current batch and resets it

func (*BatchProcessor) HasPendingBatch

func (bp *BatchProcessor) HasPendingBatch() bool

HasPendingBatch returns true if there are files in the current batch

type CacheItem

type CacheItem[K comparable, V any] struct {
	Key   K
	Value V
}

CacheItem represents a cached item

type ConsoleProgressReporter

type ConsoleProgressReporter struct {
	*DefaultProgressReporter
	// contains filtered or unexported fields
}

ConsoleProgressReporter reports progress to console output

func NewConsoleProgressReporter

func NewConsoleProgressReporter(ctx context.Context, printInterval time.Duration) *ConsoleProgressReporter

NewConsoleProgressReporter creates a console progress reporter

type DeduplicationStats

type DeduplicationStats struct {
	UniqueItems int `json:"unique_items"`
}

DeduplicationStats provides deduplication information

type Deduplicator

type Deduplicator[T any] struct {
	// contains filtered or unexported fields
}

Deduplicator handles duplicate statement detection and removal

func NewDeduplicator

func NewDeduplicator[T any](hashFunc func(T) string) *Deduplicator[T]

NewDeduplicator creates a new deduplicator with the specified hash function

func (*Deduplicator[T]) GetStats

func (d *Deduplicator[T]) GetStats() DeduplicationStats

GetStats returns deduplication statistics

func (*Deduplicator[T]) IsDuplicate

func (d *Deduplicator[T]) IsDuplicate(item T) bool

IsDuplicate checks if an item has been seen before

func (*Deduplicator[T]) Reset

func (d *Deduplicator[T]) Reset()

Reset clears the deduplicator state

type DefaultProgressReporter

type DefaultProgressReporter struct {
	// contains filtered or unexported fields
}

DefaultProgressReporter is a concrete implementation of ProgressReporter

func NewProgressReporter

func NewProgressReporter(ctx context.Context, updateCallback func(ProgressSummary)) *DefaultProgressReporter

NewProgressReporter creates a new progress reporter

func (*DefaultProgressReporter) AddError

func (pr *DefaultProgressReporter) AddError(err error)

AddError adds an error

func (*DefaultProgressReporter) AddWarning

func (pr *DefaultProgressReporter) AddWarning(message string)

AddWarning adds a warning message

func (*DefaultProgressReporter) Complete

func (pr *DefaultProgressReporter) Complete(success bool)

Complete marks the overall process as complete

func (*DefaultProgressReporter) FinishPhase

func (pr *DefaultProgressReporter) FinishPhase(phase ProcessingPhase, success bool, message string)

FinishPhase completes the current phase

func (*DefaultProgressReporter) GetSummary

func (pr *DefaultProgressReporter) GetSummary() ProgressSummary

GetSummary returns the current progress summary

func (*DefaultProgressReporter) SetOverallProgress

func (pr *DefaultProgressReporter) SetOverallProgress(current, total int)

SetOverallProgress sets the overall progress

func (*DefaultProgressReporter) StartPhase

func (pr *DefaultProgressReporter) StartPhase(phase ProcessingPhase, total int)

StartPhase starts a new processing phase

func (*DefaultProgressReporter) UpdateProgress

func (pr *DefaultProgressReporter) UpdateProgress(current int, message string)

UpdateProgress updates the progress of the current phase

type FileStreamReader

type FileStreamReader struct {
	// contains filtered or unexported fields
}

FileStreamReader provides streaming file reading capabilities

func NewFileStreamReader

func NewFileStreamReader(reader io.Reader, bufferSizeKB int, memManager *MemoryManager) *FileStreamReader

NewFileStreamReader creates a new file stream reader

func (*FileStreamReader) GetPosition

func (fsr *FileStreamReader) GetPosition() int64

GetPosition returns the current read position

func (*FileStreamReader) ReadChunk

func (fsr *FileStreamReader) ReadChunk() ([]byte, error)

ReadChunk reads the next chunk of data

type LRUCache

type LRUCache[K comparable, V any] struct {
	// contains filtered or unexported fields
}

LRUCache provides memory-efficient caching with least-recently-used eviction

func NewLRUCache

func NewLRUCache[K comparable, V any](capacity int) *LRUCache[K, V]

NewLRUCache creates a new LRU cache with the specified capacity

func (*LRUCache[K, V]) Clear

func (c *LRUCache[K, V]) Clear()

Clear removes all items from the cache

func (*LRUCache[K, V]) Get

func (c *LRUCache[K, V]) Get(key K) (V, bool)

Get retrieves an item from the cache

func (*LRUCache[K, V]) HitRate

func (c *LRUCache[K, V]) HitRate() float64

HitRate returns the cache hit rate

func (*LRUCache[K, V]) Put

func (c *LRUCache[K, V]) Put(key K, value V)

Put adds an item to the cache

func (*LRUCache[K, V]) Size

func (c *LRUCache[K, V]) Size() int

Size returns the current number of items in the cache

type MemoryManager

type MemoryManager struct {
	// contains filtered or unexported fields
}

MemoryManager handles memory-efficient processing with deduplication and size limits

func NewMemoryManager

func NewMemoryManager(maxMemoryMB int) *MemoryManager

NewMemoryManager creates a memory manager with the specified limits

func (*MemoryManager) GetBuffer

func (mm *MemoryManager) GetBuffer() []byte

GetBuffer retrieves a pooled byte buffer

func (*MemoryManager) GetMemoryStats

func (mm *MemoryManager) GetMemoryStats() MemoryStats

GetMemoryStats returns current memory usage statistics

func (*MemoryManager) GetStatement

func (mm *MemoryManager) GetStatement() *types.Statement

GetStatement retrieves a pooled statement object

func (*MemoryManager) PutBuffer

func (mm *MemoryManager) PutBuffer(buf []byte)

PutBuffer returns a buffer to the pool

func (*MemoryManager) PutStatement

func (mm *MemoryManager) PutStatement(stmt *types.Statement)

PutStatement returns a statement to the pool

func (*MemoryManager) ReleaseMemory

func (mm *MemoryManager) ReleaseMemory(size int64)

ReleaseMemory decreases tracked memory usage

func (*MemoryManager) TrackMemoryUsage

func (mm *MemoryManager) TrackMemoryUsage(size int64) bool

TrackMemoryUsage updates current memory usage

type MemoryOptimizedStatement

type MemoryOptimizedStatement struct {
	*types.Statement
	// contains filtered or unexported fields
}

MemoryOptimizedStatement wraps a statement with memory optimization features

func NewMemoryOptimizedStatement

func NewMemoryOptimizedStatement(mm *MemoryManager, stmt *types.Statement) *MemoryOptimizedStatement

NewMemoryOptimizedStatement creates a statement with memory tracking

func (*MemoryOptimizedStatement) Release

func (mos *MemoryOptimizedStatement) Release()

Release frees the memory used by this statement

type MemoryStats

type MemoryStats struct {
	MaxMemoryBytes     int64   `json:"max_memory_bytes"`
	CurrentMemoryBytes int64   `json:"current_memory_bytes"`
	SystemMemoryBytes  int64   `json:"system_memory_bytes"`
	CacheSize          int     `json:"cache_size"`
	CacheHitRate       float64 `json:"cache_hit_rate"`
}

MemoryStats provides memory usage information

type MigrationFile

type MigrationFile struct {
	Path     string
	Content  []byte
	Sequence int
	Size     int64
}

MigrationFile represents a migration file to be processed

type PhaseInfo

type PhaseInfo struct {
	StartTime  time.Time     `json:"start_time"`
	EndTime    time.Time     `json:"end_time"`
	Duration   time.Duration `json:"duration"`
	Total      int           `json:"total"`
	Current    int           `json:"current"`
	Progress   float64       `json:"progress"`
	Message    string        `json:"message"`
	Success    bool          `json:"success"`
	IsActive   bool          `json:"is_active"`
	IsComplete bool          `json:"is_complete"`
}

PhaseInfo contains information about a processing phase

type ProcessedFile

type ProcessedFile struct {
	OriginalFile *MigrationFile
	Migration    *types.Migration
	ProcessTime  time.Duration
	MemoryUsed   int64
	Errors       []error
}

ProcessedFile represents a processed migration file

type ProcessingPhase

type ProcessingPhase string

ProcessingPhase represents different phases of migration processing

const (
	PhaseDiscovery    ProcessingPhase = "DISCOVERY"
	PhaseValidation   ProcessingPhase = "VALIDATION"
	PhaseParsing      ProcessingPhase = "PARSING"
	PhaseAnalysis     ProcessingPhase = "ANALYSIS"
	PhaseDependencies ProcessingPhase = "DEPENDENCIES"
	PhaseSquashing    ProcessingPhase = "SQUASHING"
	PhaseOptimization ProcessingPhase = "OPTIMIZATION"
	PhaseGeneration   ProcessingPhase = "GENERATION"
	PhaseOutput       ProcessingPhase = "OUTPUT"
	PhaseCompletion   ProcessingPhase = "COMPLETION"
)

type ProcessingStats

type ProcessingStats struct {
	FilesProcessed  int64   `json:"files_processed"`
	FilesSkipped    int64   `json:"files_skipped"`
	FilesErrored    int64   `json:"files_errored"`
	TotalBytes      int64   `json:"total_bytes"`
	ProcessingTime  int64   `json:"processing_time_ms"`
	PeakMemoryUsage int64   `json:"peak_memory_usage"`
	AverageFileSize int64   `json:"average_file_size"`
	ThroughputMBps  float64 `json:"throughput_mbps"`
}

ProcessingStats tracks processing statistics

type ProgressAggregator

type ProgressAggregator struct {
	// contains filtered or unexported fields
}

ProgressAggregator aggregates progress from multiple sources

func NewProgressAggregator

func NewProgressAggregator() *ProgressAggregator

NewProgressAggregator creates a new progress aggregator

func (*ProgressAggregator) AddSource

func (pa *ProgressAggregator) AddSource(name string, reporter ProgressReporter, weight float64)

AddSource adds a progress source with a weight

func (*ProgressAggregator) GetAggregatedProgress

func (pa *ProgressAggregator) GetAggregatedProgress() ProgressSummary

GetAggregatedProgress returns the weighted average progress

type ProgressReporter

type ProgressReporter interface {
	StartPhase(phase ProcessingPhase, total int)
	UpdateProgress(current int, message string)
	FinishPhase(phase ProcessingPhase, success bool, message string)
	SetOverallProgress(current, total int)
	AddWarning(message string)
	AddError(err error)
	GetSummary() ProgressSummary
}

ProgressReporter interface for reporting progress

type ProgressSummary

type ProgressSummary struct {
	StartTime       time.Time                     `json:"start_time"`
	EndTime         time.Time                     `json:"end_time"`
	Duration        time.Duration                 `json:"duration"`
	CurrentPhase    ProcessingPhase               `json:"current_phase"`
	OverallProgress float64                       `json:"overall_progress"`
	PhaseProgress   map[ProcessingPhase]float64   `json:"phase_progress"`
	PhaseDetails    map[ProcessingPhase]PhaseInfo `json:"phase_details"`
	Warnings        []string                      `json:"warnings"`
	Errors          []string                      `json:"errors"`
	IsComplete      bool                          `json:"is_complete"`
	Success         bool                          `json:"success"`
}

ProgressSummary provides a summary of the progress

type ProgressTracker

type ProgressTracker struct {
	// contains filtered or unexported fields
}

ProgressTracker tracks processing progress for large operations

func NewProgressTracker

func NewProgressTracker(total int64, updateFreq time.Duration, callback func(int64, int64, float64)) *ProgressTracker

NewProgressTracker creates a new progress tracker

func (*ProgressTracker) GetProgress

func (pt *ProgressTracker) GetProgress() (current, total int64, percentage float64)

GetProgress returns current progress statistics

func (*ProgressTracker) Update

func (pt *ProgressTracker) Update(increment int64)

Update updates the progress counter

type StreamingProcessor

type StreamingProcessor struct {
	// contains filtered or unexported fields
}

StreamingProcessor handles memory-efficient processing of large migration sets

func NewStreamingProcessor

func NewStreamingProcessor(batchSize, workerCount int, memManager *MemoryManager) *StreamingProcessor

NewStreamingProcessor creates a new streaming processor

func (*StreamingProcessor) GetResults

func (sp *StreamingProcessor) GetResults() <-chan *ProcessedFile

GetResults returns a channel of processed files

func (*StreamingProcessor) GetStats

func (sp *StreamingProcessor) GetStats() ProcessingStats

GetStats returns current processing statistics

func (*StreamingProcessor) ProcessDirectory

func (sp *StreamingProcessor) ProcessDirectory(dir string) error

ProcessDirectory processes all migration files in a directory

func (*StreamingProcessor) Start

func (sp *StreamingProcessor) Start()

Start begins the streaming processing

func (*StreamingProcessor) Stop

func (sp *StreamingProcessor) Stop() error

Stop gracefully stops the streaming processor

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL