processor

type ComprehensiveExtractor struct {
	// contains filtered or unexported fields
}

type Database interface {
	InsertStudy(study *database.Study) error
	InsertExperiment(exp *database.Experiment) error
	InsertSample(sample *database.Sample) error
	InsertRun(run *database.Run) error
	InsertSubmission(submission *database.Submission) error
	InsertAnalysis(analysis *database.Analysis) error
	BatchInsertExperiments(experiments []database.Experiment) error

	// Pool/multiplex support
	InsertSamplePool(pool *database.SamplePool) error
	GetSamplePools(parentSample string) ([]database.SamplePool, error)
	CountSamplePools() (int, error)
	GetAveragePoolSize() (float64, error)
	GetMaxPoolSize() (int, error)

	// Identifier and link support
	InsertIdentifier(identifier *database.Identifier) error
	GetIdentifiers(recordType, recordAccession string) ([]database.Identifier, error)
	FindRecordsByIdentifier(idValue string) ([]database.Identifier, error)
	InsertLink(link *database.Link) error
	GetLinks(recordType, recordAccession string) ([]database.Link, error)
}

type ExtractionOptions struct {
	ExtractAttributes     bool // Extract all custom attributes
	ExtractLinks          bool // Extract all external links
	NormalizeOrganism     bool // Normalize organism names
	ExtractFromAttributes bool // Extract known fields from attributes
	BatchSize             int  // Batch size for database operations
	ValidateXML           bool // Validate XML against schemas
	StrictValidation      bool // Fail on validation errors (vs warnings only)
	LogValidationIssues   bool // Log validation issues
}

type ExtractionStats struct {
	StudiesProcessed     int
	ExperimentsProcessed int
	SamplesProcessed     int
	RunsProcessed        int
	AnalysesProcessed    int
	SubmissionsProcessed int
	StudiesExtracted     int
	ExperimentsExtracted int
	SamplesExtracted     int
	RunsExtracted        int
	AnalysesExtracted    int
	SubmissionsExtracted int
	ValidationErrors     int
	ValidationWarnings   int
	Errors               []string
	StartTime            time.Time
}

type FilterOptions struct {
	// Taxonomy filters
	TaxonomyIDs   []int // List of NCBI taxonomy IDs to include
	ExcludeTaxIDs []int // Taxonomy IDs to exclude

	// Date filters
	DateFrom  time.Time // Start date (submission/published date)
	DateTo    time.Time // End date
	DateField string    // "submission", "published", or "last_updated"

	// Organism filters
	Organisms        []string // Scientific names to include
	ExcludeOrganisms []string // Scientific names to exclude

	// Technical filters
	Platforms        []string // Sequencing platforms (ILLUMINA, OXFORD_NANOPORE, etc.)
	Strategies       []string // Library strategies (RNA-Seq, WGS, WES, etc.)
	StudyTypes       []string // Study types
	InstrumentModels []string // Specific instrument models

	// Quality filters
	MinReads int64 // Minimum read count (total_spots)
	MaxReads int64 // Maximum read count
	MinBases int64 // Minimum base count (total_bases)
	MaxBases int64 // Maximum base count

	// Source filters
	Centers   []string // Submission centers
	Countries []string // Geographic origin (from attributes)

	// Control flags
	SkipIfNoMatch bool // Skip entire file if no matches
	StatsOnly     bool // Just count matches without inserting
	Verbose       bool // Print detailed filtering information
}

type FilterStats struct {
	// Overall counts
	TotalProcessed int64
	TotalMatched   int64
	TotalSkipped   int64

	// Skip reasons
	SkippedByTaxonomy int64
	SkippedByDate     int64
	SkippedByOrganism int64
	SkippedByPlatform int64
	SkippedByStrategy int64
	SkippedByReads    int64
	SkippedByCenter   int64

	// Unique record tracking
	UniqueStudies     map[string]bool
	UniqueExperiments map[string]bool
	UniqueSamples     map[string]bool
	UniqueRuns        map[string]bool

	// Performance metrics
	StartTime      time.Time
	ProcessingTime time.Duration
}

type FilteredProcessor struct {
	*StreamProcessor
	// contains filtered or unexported fields
}

type IdentifierHandler struct {
	// contains filtered or unexported fields
}

type OptimizedStreamProcessor struct {
	*StreamProcessor
	// contains filtered or unexported fields
}

type PerformanceMetrics struct {
	RecordsPerSecond  float64
	BytesPerSecond    float64
	AvgBatchTime      float64
	DatabaseWriteTime float64
	XMLParseTime      float64
	NetworkTime       float64
	TotalMemoryUsed   int64
	GCPauses          int
	CPUUtilization    float64
}

type PlatformHandler struct {
	// contains filtered or unexported fields
}

type PoolHandler struct {
	// contains filtered or unexported fields
}

type PoolStats struct {
	TotalPools        int
	AveragePoolSize   float64
	MaxPoolSize       int
	PoolsWithBarcodes int
}

type ProcessorOptions struct {
	BatchSize         int  // Records per batch (default: 5000)
	UseTransactions   bool // Use DB transactions (default: true)
	EnableCompression bool // Enable response compression (default: true)
	WorkerCount       int  // Parallel workers (default: 4)
	BufferSize        int  // Read buffer size (default: 64KB)
}

type Progress struct {
	BytesProcessed         int64
	TotalBytes             int64
	RecordsProcessed       int64
	CurrentFile            string
	PercentComplete        float64
	BytesPerSecond         float64
	TimeElapsed            time.Duration
	EstimatedTimeRemaining time.Duration
}

type ProgressFunc func(progress Progress)

type ResumableProcessor struct {
	*StreamProcessor
	// contains filtered or unexported fields
}

type ResumeOptions struct {
	ForceRestart    bool          // Force fresh start even if progress exists
	Interactive     bool          // Ask user about resume
	CheckpointEvery time.Duration // How often to checkpoint
	MaxRetries      int           // Maximum retry attempts
	RetryDelay      time.Duration // Initial retry delay
}

type SamplePool struct {
	PoolID       int
	ParentSample string
	MemberSample string
	MemberName   string
	Proportion   float64
	ReadLabel    string
	Barcode      string
	BarcodeRead  int
}

type StreamProcessor struct {
	// contains filtered or unexported fields
}

type StructuredIdentifier struct {
	RecordType      string // study, sample, experiment, run, analysis
	RecordAccession string
	IDType          string // primary, secondary, external, submitter, uuid
	IDNamespace     string
	IDValue         string
	IDLabel         string
}

type StructuredLink struct {
	RecordType      string
	RecordAccession string
	LinkType        string // url, xref, entrez
	DB              string
	ID              string
	Label           string
	URL             string
	Query           string
}