manifest

package
v0.0.24 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 17, 2026 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func AppendFile

func AppendFile(ctx context.Context, st Store, path string, schemaID int16, entry FileEntry) error

AppendFile adds a FileEntry to the manifest and saves it. Uses optimistic locking via etag to handle concurrent updates.

func AppendFiles

func AppendFiles(ctx context.Context, st Store, path string, schemaID int16, entries []FileEntry) error

AppendFiles adds multiple FileEntry items to the manifest and saves it. Uses optimistic locking via etag to handle concurrent updates.

func ListPaths

func ListPaths(m *Manifest, tier string) []string

ListPaths returns paths for the given tier.

func Save

func Save(ctx context.Context, st Store, path string, m *Manifest, etag string) (string, error)

Save writes manifest with updated timestamp and optional optimistic etag.

Types

type FSStore

type FSStore struct {
	Root fs.FS
}

FSStore implements Store on local filesystem.

func (*FSStore) Load

func (f *FSStore) Load(ctx context.Context, path string) ([]byte, string, error)

func (*FSStore) Save

func (f *FSStore) Save(ctx context.Context, path string, data []byte, etag string) (string, error)

type FileEntry

type FileEntry struct {
	Tier       string `json:"tier"`
	Path       string `json:"path"`
	RowIDMin   string `json:"row_id_min"`
	RowIDMax   string `json:"row_id_max"`
	CreatedMin int64  `json:"created_min"`
	CreatedMax int64  `json:"created_max"`
	SizeBytes  int64  `json:"size_bytes"`
	RowCount   int64  `json:"row_count"`
	Checksum   string `json:"checksum,omitempty"`
}

FileEntry describes a single parquet file tracked by the manifest. Tier is typically "base" or "delta".

func FilterByTier

func FilterByTier(m *Manifest, tier string) []FileEntry

FilterBySchema returns files matching the tier (use "" for all tiers).

type Manifest

type Manifest struct {
	SchemaID    int16       `json:"schema_id"`
	Version     int64       `json:"version"`
	UpdatedAtMs int64       `json:"updated_at_ms"`
	Files       []FileEntry `json:"files"`
}

Manifest holds per-schema parquet inventory.

func Decode

func Decode(r io.Reader) (*Manifest, error)

Decode reads from reader.

func Load

func Load(ctx context.Context, st Store, path string) (*Manifest, string, error)

Load reads manifest from store.

func LoadOrCreate

func LoadOrCreate(ctx context.Context, st Store, path string, schemaID int16) (*Manifest, string, error)

LoadOrCreate loads an existing manifest or creates a new empty one for the schema. Returns the manifest, etag (empty if new), and any error.

func Parse

func Parse(data []byte) (*Manifest, error)

Parse decodes manifest JSON bytes.

type PathResolver

type PathResolver struct {
	Prefix       string
	PathTemplate string
}

PathResolver builds manifest paths per schema using a template. Defaults to "manifest/{{.SchemaID}}.json" under an optional prefix.

func (PathResolver) Resolve

func (r PathResolver) Resolve(schemaID int16) (string, error)

Resolve returns the full path for a schema manifest.

type S3Client added in v0.0.24

type S3Client interface {
	GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
}

type S3Store

type S3Store struct {
	Client S3Client
	Bucket string
}

S3Store implements Store using AWS S3-compatible APIs.

func (*S3Store) Load

func (s *S3Store) Load(ctx context.Context, path string) ([]byte, string, error)

func (*S3Store) Save

func (s *S3Store) Save(ctx context.Context, path string, data []byte, etag string) (string, error)

type Store

type Store interface {
	Load(ctx context.Context, path string) (data []byte, etag string, err error)
	Save(ctx context.Context, path string, data []byte, etag string) (newETag string, err error)
}

Store abstracts load/save operations (could be S3 or local FS).

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL