progress

package
v0.1.0-alpha.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 12, 2026 License: MIT Imports: 8 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Checkpoint

type Checkpoint struct {
	ID                int64     `json:"id" db:"id"`
	ProgressID        int64     `json:"progress_id" db:"progress_id"`
	CheckpointTime    time.Time `json:"checkpoint_time" db:"checkpoint_time"`
	TarPosition       int64     `json:"tar_position" db:"tar_position"`
	BytesProcessed    int64     `json:"bytes_processed" db:"bytes_processed"`
	RecordsProcessed  int64     `json:"records_processed" db:"records_processed"`
	LastTransactionID string    `json:"last_transaction_id" db:"last_transaction_id"`
}

Checkpoint represents a checkpoint in the ingestion process

type ProcessedFile

type ProcessedFile struct {
	ID           int64     `json:"id" db:"id"`
	ProgressID   int64     `json:"progress_id" db:"progress_id"`
	FileName     string    `json:"file_name" db:"file_name"`
	FileSize     int64     `json:"file_size" db:"file_size"`
	RecordsCount int       `json:"records_count" db:"records_count"`
	ProcessedAt  time.Time `json:"processed_at" db:"processed_at"`
	Checksum     string    `json:"checksum" db:"checksum"`
}

ProcessedFile represents a file that has been processed

type Progress

type Progress struct {
	ID               int64      `json:"id" db:"id"`
	SourceURL        string     `json:"source_url" db:"source_url"`
	SourceHash       string     `json:"source_hash" db:"source_hash"`
	TotalBytes       int64      `json:"total_bytes" db:"total_bytes"`
	DownloadedBytes  int64      `json:"downloaded_bytes" db:"downloaded_bytes"`
	ProcessedBytes   int64      `json:"processed_bytes" db:"processed_bytes"`
	LastTarPosition  int64      `json:"last_tar_position" db:"last_tar_position"`
	LastXMLFile      string     `json:"last_xml_file" db:"last_xml_file"`
	RecordsProcessed int64      `json:"records_processed" db:"records_processed"`
	State            State      `json:"state" db:"state"`
	StartedAt        time.Time  `json:"started_at" db:"started_at"`
	UpdatedAt        time.Time  `json:"updated_at" db:"updated_at"`
	CompletedAt      *time.Time `json:"completed_at,omitempty" db:"completed_at"`
	ErrorMessage     string     `json:"error_message,omitempty" db:"error_message"`
}

Progress represents the ingestion progress for a source

type ResumeInfo

type ResumeInfo struct {
	DownloadedBytes  int64    `json:"downloaded_bytes"`
	TarPosition      int64    `json:"tar_position"`
	LastFile         string   `json:"last_file"`
	RecordsProcessed int64    `json:"records_processed"`
	ProcessedFiles   []string `json:"processed_files"`
}

ResumeInfo contains information needed to resume processing

type State

type State string

State represents the current state of an ingestion

const (
	StateDownloading State = "downloading"
	StateProcessing  State = "processing"
	StateCompleted   State = "completed"
	StateFailed      State = "failed"
	StatePaused      State = "paused"
)

type Statistics

type Statistics struct {
	TotalBytes             int64         `json:"total_bytes"`
	DownloadedBytes        int64         `json:"downloaded_bytes"`
	ProcessedBytes         int64         `json:"processed_bytes"`
	RecordsProcessed       int64         `json:"records_processed"`
	FilesProcessed         int           `json:"files_processed"`
	Duration               time.Duration `json:"duration"`
	BytesPerSecond         float64       `json:"bytes_per_second"`
	RecordsPerSecond       float64       `json:"records_per_second"`
	PercentComplete        float64       `json:"percent_complete"`
	EstimatedTimeRemaining time.Duration `json:"estimated_time_remaining"`
}

Statistics contains ingestion statistics

type Tracker

type Tracker struct {
	// contains filtered or unexported fields
}

Tracker manages ingestion progress tracking

func NewTracker

func NewTracker(db *sql.DB) (*Tracker, error)

NewTracker creates a new progress tracker

func (*Tracker) CleanupOldProgress

func (t *Tracker) CleanupOldProgress(olderThan time.Duration) error

CleanupOldProgress removes old completed or failed progress records

func (*Tracker) GetProgress

func (t *Tracker) GetProgress(sourceHash string) (*Progress, error)

GetProgress retrieves progress for a source hash

func (*Tracker) GetResumeInfo

func (t *Tracker) GetResumeInfo() (*ResumeInfo, error)

GetResumeInfo returns information needed to resume an ingestion

func (*Tracker) GetStatistics

func (t *Tracker) GetStatistics() (*Statistics, error)

GetStatistics returns ingestion statistics

func (*Tracker) IsFileProcessed

func (t *Tracker) IsFileProcessed(fileName string) bool

IsFileProcessed checks if a file has already been processed

func (*Tracker) MarkCompleted

func (t *Tracker) MarkCompleted() error

MarkCompleted marks the ingestion as completed

func (*Tracker) MarkFailed

func (t *Tracker) MarkFailed(errorMsg string) error

MarkFailed marks the ingestion as failed

func (*Tracker) RecordFileProcessed

func (t *Tracker) RecordFileProcessed(fileName string, fileSize int64, recordsCount int, checksum string) error

RecordFileProcessed records that a file has been processed

func (*Tracker) SaveCheckpointFile

func (t *Tracker) SaveCheckpointFile(dir string) error

SaveCheckpointFile saves a checkpoint to a JSON file for backup

func (*Tracker) SetCheckpointInterval

func (t *Tracker) SetCheckpointInterval(interval time.Duration)

SetCheckpointInterval sets the checkpoint interval

func (*Tracker) StartOrResume

func (t *Tracker) StartOrResume(sourceURL string, forceRestart bool) (*Progress, error)

StartOrResume starts a new ingestion or resumes an existing one

func (*Tracker) UpdateDownloadProgress

func (t *Tracker) UpdateDownloadProgress(downloadedBytes, totalBytes int64) error

UpdateDownloadProgress updates download progress

func (*Tracker) UpdateProcessingProgress

func (t *Tracker) UpdateProcessingProgress(tarPosition int64, processedBytes int64, lastFile string, records int64) error

UpdateProcessingProgress updates processing progress

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL