federated

package
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 20, 2026 License: MIT Imports: 29 Imported by: 0

Documentation

Overview

Package federated provides custom assertions for E2E testing.

Package federated provides test fixtures and data generators for E2E testing.

Package federated provides E2E test harness for federated query testing. It extends the base e2e_harness with capabilities specific to the three-tier architecture: S3 Base files, S3 Delta files, and Postgres Hot Buffer.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func AssertAllRecordsHaveSchemaID

func AssertAllRecordsHaveSchemaID(t *testing.T, records []*internal.PersistentRecord, schemaID int16)

AssertAllRecordsHaveSchemaID verifies all records have the expected schema ID.

func AssertChecksumMatch

func AssertChecksumMatch(t *testing.T, report *ComparisonReport)

AssertChecksumMatch verifies that checksums match.

func AssertCompactionMerged

func AssertCompactionMerged(t *testing.T, result *CompactionResult)

AssertCompactionMerged verifies that records were merged during compaction.

func AssertCompactionReducedFiles

func AssertCompactionReducedFiles(t *testing.T, result *CompactionResult)

AssertCompactionReducedFiles verifies that compaction reduced the number of files.

func AssertComparisonMatch

func AssertComparisonMatch(t *testing.T, report *ComparisonReport)

AssertComparisonMatch verifies that a comparison report shows a match.

func AssertDirtyIDsExcluded

func AssertDirtyIDsExcluded(t *testing.T, plan *internal.ExecutionPlan, expectedCount int)

AssertDirtyIDsExcluded verifies that dirty IDs were excluded in the execution plan.

func AssertFilesCreated

func AssertFilesCreated(t *testing.T, result *FlushResult, minFiles int)

AssertFilesCreated verifies that files were created during flush.

func AssertFlushNotTriggered

func AssertFlushNotTriggered(t *testing.T, result *FlushResult)

AssertFlushNotTriggered verifies that a CDC flush was not triggered.

func AssertFlushTriggered

func AssertFlushTriggered(t *testing.T, result *FlushResult)

AssertFlushTriggered verifies that a CDC flush was triggered.

func AssertLatencyUnder

func AssertLatencyUnder(t *testing.T, duration time.Duration, threshold time.Duration, msgAndArgs ...any)

AssertLatencyUnder verifies query latency is under the specified threshold.

func AssertNoDeleted

func AssertNoDeleted(t *testing.T, records []*internal.PersistentRecord)

AssertNoDeleted verifies that no records are soft-deleted.

func AssertNoDuplicates

func AssertNoDuplicates(t *testing.T, records []*internal.PersistentRecord)

AssertNoDuplicates verifies that all records have unique row_ids.

func AssertP95Latency

func AssertP95Latency(t *testing.T, latencies []time.Duration, threshold time.Duration)

AssertP95Latency verifies that the p95 latency is under threshold.

func AssertParquetFileExists

func AssertParquetFileExists(t *testing.T, files []string, expectedPath string)

AssertParquetFileExists verifies that a parquet file exists in S3.

func AssertParquetRowCount

func AssertParquetRowCount(t *testing.T, meta *ParquetMetadata, expected int64)

AssertParquetRowCount verifies the row count in a parquet file.

func AssertParquetRowCountInRange

func AssertParquetRowCountInRange(t *testing.T, meta *ParquetMetadata, min, max int64)

AssertParquetRowCountInRange verifies the row count is within range.

func AssertPlanContainsNote

func AssertPlanContainsNote(t *testing.T, plan *internal.ExecutionPlan, noteSubstring string)

AssertPlanContainsNote verifies that the execution plan contains a specific note.

func AssertPlanContainsSource

func AssertPlanContainsSource(t *testing.T, plan *internal.ExecutionPlan, source string)

AssertPlanContainsSource verifies that the execution plan includes a specific source.

func AssertQueryResultMatch

func AssertQueryResultMatch(t *testing.T, expected, actual *QueryResult)

AssertQueryResultMatch verifies that two query results match.

func AssertRecordCount

func AssertRecordCount(t *testing.T, records []*internal.PersistentRecord, expected int, msgAndArgs ...any)

AssertRecordCount verifies the expected number of records.

func AssertRecordExists

func AssertRecordExists(t *testing.T, records []*internal.PersistentRecord, rowID uuid.UUID)

AssertRecordExists verifies that a record with the given row_id exists.

func AssertRecordNotExists

func AssertRecordNotExists(t *testing.T, records []*internal.PersistentRecord, rowID uuid.UUID)

AssertRecordNotExists verifies that no record with the given row_id exists.

func AssertRowsFlushed

func AssertRowsFlushed(t *testing.T, result *FlushResult, expected int64)

AssertRowsFlushed verifies the number of rows flushed.

func AssertThroughput

func AssertThroughput(t *testing.T, requestCount int, totalDuration time.Duration, minQPS float64)

AssertThroughput verifies that throughput is above minimum QPS.

func AssertTotalRecordsInRange

func AssertTotalRecordsInRange(t *testing.T, result *QueryResult, min, max int64)

AssertTotalRecordsInRange verifies total records is within expected range.

func AvgDuration

func AvgDuration(latencies []time.Duration) time.Duration

AvgDuration returns the average duration from a slice.

func DeterministicAttributeID added in v0.1.0

func DeterministicAttributeID(schemaID int16, name string) int

DeterministicAttributeID returns a stable test-only attribute ID for a schema/name pair. Deprecated: benchmark-backed seeding should use schema metadata IDs via FederatedTestHarness.Registry.

func GroupByRowID

func GroupByRowID(records []TestRecord) map[uuid.UUID][]TestRecord

GroupByRowID groups records by row_id.

func MaxDuration

func MaxDuration(latencies []time.Duration) time.Duration

MaxDuration returns the maximum duration from a slice.

func MinDuration

func MinDuration(latencies []time.Duration) time.Duration

MinDuration returns the minimum duration from a slice.

func Percentile

func Percentile(latencies []time.Duration, p int) time.Duration

Percentile calculates the p-th percentile of durations.

func RequireComparisonMatch

func RequireComparisonMatch(t *testing.T, report *ComparisonReport)

RequireComparisonMatch verifies that a comparison report shows a match, failing immediately if not.

func RequireLatencyUnder

func RequireLatencyUnder(t *testing.T, duration time.Duration, threshold time.Duration, msgAndArgs ...any)

RequireLatencyUnder verifies query latency is under the specified threshold, failing immediately if not.

func RequireNoDeleted

func RequireNoDeleted(t *testing.T, records []*internal.PersistentRecord)

RequireNoDeleted verifies that no records are soft-deleted, failing immediately if found.

func RequireNoDuplicates

func RequireNoDuplicates(t *testing.T, records []*internal.PersistentRecord)

RequireNoDuplicates verifies that all records have unique row_ids, failing immediately if not.

func RequireP95Latency

func RequireP95Latency(t *testing.T, latencies []time.Duration, threshold time.Duration)

RequireP95Latency verifies that the p95 latency is under threshold, failing immediately if not.

func RequireRecordCount

func RequireRecordCount(t *testing.T, records []*internal.PersistentRecord, expected int, msgAndArgs ...any)

RequireRecordCount verifies the expected number of records and fails immediately if not met.

func SortByChangedAt

func SortByChangedAt(records []TestRecord)

SortByChangedAt sorts records by changed_at in ascending order.

func SortByChangedAtDesc

func SortByChangedAtDesc(records []TestRecord)

SortByChangedAtDesc sorts records by changed_at in descending order.

Types

type AttributeMismatch

type AttributeMismatch struct {
	RowID         uuid.UUID
	AttributeName string
	FederatedVal  any
	PostgresVal   any
}

AttributeMismatch describes a single attribute value mismatch.

type CompactionResult

type CompactionResult struct {
	FilesCompacted int
	FilesCreated   int
	RowsMerged     int64
	Duration       time.Duration
}

CompactionResult contains compaction operation results.

type ComparisonReport

type ComparisonReport struct {
	Match               bool
	FederatedCount      int64
	PostgresCount       int64
	MissingInFed        []uuid.UUID
	MissingInPG         []uuid.UUID
	AttributeMismatches []AttributeMismatch
	FederatedChecksum   string
	PostgresChecksum    string
}

ComparisonReport contains data comparison results.

type FederatedTestHarness

type FederatedTestHarness struct {
	*e2e_harness.TestHarness

	// Configuration
	SchemaID  int16
	S3Bucket  string
	S3Prefix  string
	CDCConfig cdc.CDCConfig
	Registry  forma.SchemaRegistry

	// Postgres connection info for DuckDB postgres_scan
	PGHost     string
	PGPort     string
	PGUser     string
	PGPassword string
	PGDatabase string
	PGSSLMode  string

	// S3 connection info shared by DuckDB and direct client access.
	S3Region    string
	S3AccessKey string
	S3SecretKey string
	// contains filtered or unexported fields
}

FederatedTestHarness extends the base TestHarness with federated query testing capabilities.

func NewFederatedTestHarness

func NewFederatedTestHarness(ctx context.Context) (*FederatedTestHarness, error)

NewFederatedTestHarness creates a new federated test harness with all dependencies.

func (*FederatedTestHarness) CalculateChecksum

func (h *FederatedTestHarness) CalculateChecksum(records []*internal.PersistentRecord) string

CalculateChecksum calculates a simple checksum of records.

func (*FederatedTestHarness) Cleanup

func (h *FederatedTestHarness) Cleanup(ctx context.Context) error

Cleanup releases all resources.

func (*FederatedTestHarness) ClearAllData

func (h *FederatedTestHarness) ClearAllData(ctx context.Context) error

ClearAllData removes all test data.

func (*FederatedTestHarness) ClearChangeLog

func (h *FederatedTestHarness) ClearChangeLog(ctx context.Context) error

ClearChangeLog removes all records from change_log table.

func (*FederatedTestHarness) CompareResults

func (h *FederatedTestHarness) CompareResults(federated, postgres *QueryResult) *ComparisonReport

CompareResults compares federated and postgres query results.

func (*FederatedTestHarness) CountUnflushedRecords

func (h *FederatedTestHarness) CountUnflushedRecords(ctx context.Context) int

CountUnflushedRecords returns the count of unflushed records.

func (*FederatedTestHarness) DownloadS3File

func (h *FederatedTestHarness) DownloadS3File(ctx context.Context, key string) ([]byte, error)

DownloadS3File downloads a file from S3.

func (*FederatedTestHarness) ExecuteFederatedQuery

func (h *FederatedTestHarness) ExecuteFederatedQuery(ctx context.Context, opts *QueryOptions) (*QueryResult, error)

ExecuteFederatedQuery executes a federated query using DuckDB.

func (*FederatedTestHarness) ExecutePostgresQuery

func (h *FederatedTestHarness) ExecutePostgresQuery(ctx context.Context, opts *QueryOptions) (*QueryResult, error)

ExecutePostgresQuery executes a direct Postgres query (no DuckDB).

func (*FederatedTestHarness) GetChangeLogStats

func (h *FederatedTestHarness) GetChangeLogStats(ctx context.Context) (count int64, oldestTs int64, err error)

GetChangeLogStats returns change_log statistics.

func (*FederatedTestHarness) GetDuckDB

func (h *FederatedTestHarness) GetDuckDB() *internal.DuckDBClient

GetDuckDB returns the DuckDB client for direct access.

func (*FederatedTestHarness) GetPostgresDB

func (h *FederatedTestHarness) GetPostgresDB() *sql.DB

GetPostgresDB returns the Postgres connection for direct access.

func (*FederatedTestHarness) GetS3Client

func (h *FederatedTestHarness) GetS3Client() *s3.Client

GetS3Client returns the S3 client for direct access.

func (*FederatedTestHarness) InsertOverlappingRecords

func (h *FederatedTestHarness) InsertOverlappingRecords(ctx context.Context, rowID uuid.UUID, versions int) error

InsertOverlappingRecords creates multiple versions of the same row_id across tiers.

func (*FederatedTestHarness) ListParquetFiles

func (h *FederatedTestHarness) ListParquetFiles(ctx context.Context, tier string) ([]string, error)

ListParquetFiles lists parquet files in a tier.

func (*FederatedTestHarness) MarkRecordDeleted

func (h *FederatedTestHarness) MarkRecordDeleted(ctx context.Context, rowID uuid.UUID) error

MarkRecordDeleted marks a record as soft-deleted.

func (*FederatedTestHarness) ReadParquetMetadata

func (h *FederatedTestHarness) ReadParquetMetadata(ctx context.Context, s3Key string) (*ParquetMetadata, error)

ReadParquetMetadata reads metadata from a parquet file.

func (*FederatedTestHarness) RunCDCFlush

func (h *FederatedTestHarness) RunCDCFlush(ctx context.Context) (*FlushResult, error)

RunCDCFlush triggers a CDC flush operation.

func (*FederatedTestHarness) RunCompaction

func (h *FederatedTestHarness) RunCompaction(ctx context.Context) (*CompactionResult, error)

RunCompaction triggers a compaction operation.

func (*FederatedTestHarness) SeedAllTiers

func (h *FederatedTestHarness) SeedAllTiers(ctx context.Context, base, delta, hot int) error

SeedAllTiers seeds data across all three tiers.

func (*FederatedTestHarness) SeedBaseRecords

func (h *FederatedTestHarness) SeedBaseRecords(ctx context.Context, count int) ([]TestRecord, error)

SeedBaseRecords creates records directly in S3 base files.

func (*FederatedTestHarness) SeedDeltaRecords

func (h *FederatedTestHarness) SeedDeltaRecords(ctx context.Context, count int) ([]TestRecord, error)

SeedDeltaRecords creates records in S3 delta files.

func (*FederatedTestHarness) SeedHotRecords

func (h *FederatedTestHarness) SeedHotRecords(ctx context.Context, count int) ([]TestRecord, error)

SeedHotRecords creates records in Postgres hot buffer (unflushed).

func (*FederatedTestHarness) SeedHotRecordsWithData

func (h *FederatedTestHarness) SeedHotRecordsWithData(ctx context.Context, records []TestRecord) error

SeedHotRecordsWithData inserts specific test records into hot buffer.

func (*FederatedTestHarness) SetupSchema

func (h *FederatedTestHarness) SetupSchema(schemaID int16, schemaName string) error

SetupSchema configures the schema for testing.

func (*FederatedTestHarness) SimulateS3Failure

func (h *FederatedTestHarness) SimulateS3Failure() func()

SimulateS3Failure disables S3 operations and returns a restore function.

func (*FederatedTestHarness) StreamFederatedQuery

func (h *FederatedTestHarness) StreamFederatedQuery(ctx context.Context, opts *QueryOptions, handler func(*internal.PersistentRecord) error) error

StreamFederatedQuery streams query results with a handler callback.

func (*FederatedTestHarness) ValidateDeduplication

func (h *FederatedTestHarness) ValidateDeduplication(records []*internal.PersistentRecord) error

ValidateDeduplication verifies that records are properly deduplicated.

func (*FederatedTestHarness) WriteParquet

func (h *FederatedTestHarness) WriteParquet(ctx context.Context, tier, filename string, records []TestRecord) error

WriteParquet writes records to a parquet file in S3.

type Filter

type Filter struct {
	RowID      uuid.UUID
	Conditions map[string]any
}

Filter defines query filter conditions.

type FlushResult

type FlushResult struct {
	Flushed      bool
	RowsFlushed  int64
	FilesCreated []string
	Duration     time.Duration
}

FlushResult contains CDC flush operation results.

type GeneratorOptions

type GeneratorOptions struct {
	SchemaID       int16
	TimeRangeHours int     // Range of time for records (hours ago)
	TimeOffset     int     // Offset from now (hours ago)
	DeletedRatio   float64 // Ratio of records to mark as deleted (0.0-1.0)
	Seed           int64   // Random seed for reproducibility (0 = use time)
}

GeneratorOptions configures test record generation.

func DefaultGeneratorOptions

func DefaultGeneratorOptions() *GeneratorOptions

DefaultGeneratorOptions returns sensible defaults for test generation.

type LatencyStats

type LatencyStats struct {
	Min   time.Duration
	Max   time.Duration
	Avg   time.Duration
	P50   time.Duration
	P95   time.Duration
	P99   time.Duration
	Count int
}

LatencyStats holds statistical information about latencies.

func CalculateLatencyStats

func CalculateLatencyStats(latencies []time.Duration) LatencyStats

CalculateLatencyStats computes statistics from a slice of durations.

type ParquetMetadata

type ParquetMetadata struct {
	RowCount   int64
	RowIDMin   string
	RowIDMax   string
	CreatedMin int64
	CreatedMax int64
	SizeBytes  int64
}

ParquetMetadata contains parquet file metadata.

type PerformanceDataset

type PerformanceDataset struct {
	Base  []TestRecord
	Delta []TestRecord
	Hot   []TestRecord
}

PerformanceScenario returns large datasets for performance testing.

type PresetScenarios

type PresetScenarios struct{}

PresetScenarios provides predefined test scenarios.

func (PresetScenarios) CDCFlushScenario

func (PresetScenarios) CDCFlushScenario(schemaID int16, count int, ageHours int) []TestRecord

CDCFlushScenario returns records for CDC flush threshold testing.

func (PresetScenarios) DeduplicationScenario

func (PresetScenarios) DeduplicationScenario(schemaID int16) (base, delta, hot []TestRecord)

DeduplicationScenario returns records with overlapping row_ids for dedup testing.

func (PresetScenarios) LargeScale

func (PresetScenarios) LargeScale(schemaID int16) *PerformanceDataset

LargeScale returns a 500K total record dataset.

func (PresetScenarios) MediumScale

func (PresetScenarios) MediumScale(schemaID int16) *PerformanceDataset

MediumScale returns a 100K total record dataset.

func (PresetScenarios) SoftDeleteScenario

func (PresetScenarios) SoftDeleteScenario(schemaID int16) []TestRecord

SoftDeleteScenario returns records for soft delete testing.

func (PresetScenarios) ThreeTierNoOverlap

func (PresetScenarios) ThreeTierNoOverlap(schemaID int16) (base, delta, hot []TestRecord)

ThreeTierNoOverlap returns records for a scenario with data in all tiers but no overlap.

type QueryOptions

type QueryOptions struct {
	Limit          int
	Offset         int
	Filter         *Filter
	SortBy         string
	SortDesc       bool
	PreferHot      bool
	TradeTimeStart int64
	TradeTimeEnd   int64
	CountOnly      bool
	KeysetCursor   *internal.KeysetCursor
}

QueryOptions configures federated query execution.

type QueryResult

type QueryResult struct {
	Records      []*internal.PersistentRecord
	TotalRecords int64
	Plan         *internal.ExecutionPlan
	Duration     time.Duration
}

QueryResult wraps query results with metadata.

type TestRecord

type TestRecord struct {
	RowID      uuid.UUID
	SchemaID   int16
	Attributes map[string]any
	ChangedAt  int64
	DeletedAt  int64
	FlushedAt  int64
}

TestRecord represents a record for testing across all tiers.

func DeduplicateByRowID

func DeduplicateByRowID(records []TestRecord) []TestRecord

DeduplicateByRowID keeps only the latest version of each row_id.

func FilterDeleted

func FilterDeleted(records []TestRecord) []TestRecord

FilterDeleted returns only non-deleted records.

func GenerateBulkRecords

func GenerateBulkRecords(count int, schemaID int16, duplicateRatio float64) []TestRecord

GenerateBulkRecords creates a large number of records for performance testing.

func GenerateOverlappingRecords

func GenerateOverlappingRecords(rowID uuid.UUID, versions int, schemaID int16) []TestRecord

GenerateOverlappingRecords creates multiple versions of the same row_id.

func GenerateTestRecords

func GenerateTestRecords(count int, opts *GeneratorOptions) []TestRecord

GenerateTestRecords creates test records with realistic data distribution.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL