zipstream

package
v0.10.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 4, 2025 License: BSD-3-Clause-Clear Imports: 10 Imported by: 0

Documentation

Index

Constants

View Source
const (
	TDFManifestFileName = "0.manifest.json"
	TDFPayloadFileName  = "0.payload"
)

Variables

View Source
var (
	ErrWriterClosed     = errors.New("archive writer closed")
	ErrInvalidSegment   = errors.New("invalid segment index")
	ErrOutOfOrder       = errors.New("segment out of order")
	ErrDuplicateSegment = errors.New("duplicate segment already written")
	ErrSegmentMissing   = errors.New("segment missing")
	ErrInvalidSize      = errors.New("invalid size")
	ErrZip64Required    = errors.New("ZIP64 required but disabled (Zip64Never)")
)

Common errors

Functions

func CRC32CombineIEEE

func CRC32CombineIEEE(crc1, crc2 uint32, len2 int64) uint32

CRC32CombineIEEE combines two CRC-32 (IEEE) checksums as if the data were concatenated. crc1 is the CRC of the first part, crc2 of the second part, and len2 is the byte length of the second part. This uses the standard reflected IEEE polynomial 0xEDB88320 as used by ZIP.

Types

type CDFileHeader

type CDFileHeader struct {
	Signature              uint32
	VersionCreated         uint16
	VersionNeeded          uint16
	GeneralPurposeBitFlag  uint16
	CompressionMethod      uint16
	LastModifiedTime       uint16
	LastModifiedDate       uint16
	Crc32                  uint32
	CompressedSize         uint32
	UncompressedSize       uint32
	FilenameLength         uint16
	ExtraFieldLength       uint16
	FileCommentLength      uint16
	DiskNumberStart        uint16
	InternalFileAttributes uint16
	ExternalFileAttributes uint32
	LocalHeaderOffset      uint32
}

type CentralDirectory

type CentralDirectory struct {
	Entries []FileEntry // File entries in the archive
	Offset  uint64      // Offset where central directory starts
	Size    uint64      // Size of central directory
}

CentralDirectory manages the ZIP central directory structure

func NewCentralDirectory

func NewCentralDirectory() *CentralDirectory

NewCentralDirectory creates a new central directory

func (*CentralDirectory) AddFile

func (cd *CentralDirectory) AddFile(entry FileEntry)

AddFile adds a file entry to the central directory

func (*CentralDirectory) GenerateBytes

func (cd *CentralDirectory) GenerateBytes(isZip64 bool) ([]byte, error)

GenerateBytes generates the central directory bytes

type Config

type Config struct {
	Zip64         Zip64Mode
	MaxSegments   int
	EnableLogging bool
}

Config holds configuration options for writers

type EndOfCDRecord

type EndOfCDRecord struct {
	Signature               uint32
	DiskNumber              uint16
	StartDiskNumber         uint16
	NumberOfCDRecordEntries uint16
	TotalCDRecordEntries    uint16
	SizeOfCentralDirectory  uint32
	CentralDirectoryOffset  uint32
	CommentLength           uint16
}

type Error

type Error struct {
	Op   string // Operation that failed
	Type string // Writer type: "sequential", "streaming", "segment"
	Err  error  // Underlying error
}

Error provides detailed error information for archive operations

func (*Error) Error

func (e *Error) Error() string

func (*Error) Unwrap

func (e *Error) Unwrap() error

type FileEntry

type FileEntry struct {
	Name           string    // Filename in the archive
	Offset         uint64    // Offset of local file header in archive
	Size           uint64    // Uncompressed size
	CompressedSize uint64    // Compressed size (same as Size for no compression)
	CRC32          uint32    // CRC32 checksum of uncompressed data
	ModTime        time.Time // Last modification time
	IsStreaming    bool      // Whether this uses data descriptor pattern
}

FileEntry represents a file in the ZIP archive with metadata

type LocalFileHeader

type LocalFileHeader struct {
	Signature             uint32
	Version               uint16
	GeneralPurposeBitFlag uint16
	CompressionMethod     uint16
	LastModifiedTime      uint16
	LastModifiedDate      uint16
	Crc32                 uint32
	CompressedSize        uint32
	UncompressedSize      uint32
	FilenameLength        uint16
	ExtraFieldLength      uint16
}

type Option

type Option func(*Config)

Option is a functional option for configuring writers

func WithLogging

func WithLogging() Option

WithLogging enables debug logging

func WithMaxSegments

func WithMaxSegments(maxSegments int) Option

WithMaxSegments sets the maximum number of segments for SegmentWriter

func WithZip64

func WithZip64() Option

WithZip64 enables ZIP64 format support for large files WithZip64 forces ZIP64 mode; kept for backward compatibility. Equivalent to WithZip64Mode(Zip64Always).

func WithZip64Mode

func WithZip64Mode(mode Zip64Mode) Option

WithZip64Mode sets the ZIP64 mode (Auto/Always/Never).

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func NewReader

func NewReader(readSeeker io.ReadSeeker) (Reader, error)

NewReader Create archive reader instance.

func (Reader) ReadAllFileData

func (reader Reader) ReadAllFileData(filename string, maxSize int64) ([]byte, error)

ReadAllFileData Return all the data of the file if the file is available and below the specified size. NOTE: Use this method for small file sizes.

func (Reader) ReadFileData

func (reader Reader) ReadFileData(filename string, index int64, length int64) ([]byte, error)

ReadFileData Read data from file of given length of size.

func (Reader) ReadFileSize

func (reader Reader) ReadFileSize(filename string) (int64, error)

ReadFileSize Return the file size of the filename.

type SegmentEntry

type SegmentEntry struct {
	Index   int       // Segment index (0-based)
	Size    uint64    // Size of stored segment bytes (no compression)
	CRC32   uint32    // CRC32 of stored segment bytes
	Written time.Time // When this segment was written
}

SegmentEntry represents a single segment in out-of-order writing

type SegmentMetadata

type SegmentMetadata struct {
	ExpectedCount int                   // Total number of expected segments (unused when Order set)
	Segments      map[int]*SegmentEntry // Map of segments by index
	TotalSize     uint64                // Cumulative size of all segments

	TotalCRC32 uint32 // Final CRC32 when all segments are processed
	// Order, when set, defines the exact logical order of segments for
	// completeness checks and CRC computation. Indices may be sparse.
	Order []int
	// contains filtered or unexported fields
}

SegmentMetadata tracks per-segment metadata for out-of-order writing. It stores only plaintext size and CRC for each index and computes the final CRC via CRC32-combine at finalize time (no payload buffering).

func NewSegmentMetadata

func NewSegmentMetadata(expectedCount int) *SegmentMetadata

NewSegmentMetadata creates metadata for tracking segments using combine-based CRC.

func (*SegmentMetadata) AddSegment

func (sm *SegmentMetadata) AddSegment(index int, _ []byte, originalSize uint64, originalCRC32 uint32) error

AddSegment records metadata for a segment (size + CRC) without retaining payload bytes.

func (*SegmentMetadata) FinalizeCRC

func (sm *SegmentMetadata) FinalizeCRC()

FinalizeCRC computes the total CRC32 by combining per-segment CRCs in index order.

func (*SegmentMetadata) GetMissingSegments

func (sm *SegmentMetadata) GetMissingSegments() []int

GetMissingSegments returns a list of missing segment indices

func (*SegmentMetadata) GetTotalCRC32

func (sm *SegmentMetadata) GetTotalCRC32() uint32

GetTotalCRC32 returns the final CRC32 value (only valid when IsComplete() is true)

func (*SegmentMetadata) IsComplete

func (sm *SegmentMetadata) IsComplete() bool

IsComplete returns true if all expected segments have been processed

func (*SegmentMetadata) SetOrder

func (sm *SegmentMetadata) SetOrder(order []int) error

SetOrder defines the exact logical order of segments. Duplicates are not allowed. When set, completeness/CRC use this order; ExpectedCount is ignored.

type SegmentWriter

type SegmentWriter interface {
	Writer
	WriteSegment(ctx context.Context, index int, data []byte) ([]byte, error)
	Finalize(ctx context.Context, manifest []byte) ([]byte, error)
	// CleanupSegment removes the presence marker for a segment index.
	// Calling this before Finalize will cause IsComplete() to fail for that index.
	CleanupSegment(index int) error
}

SegmentWriter handles out-of-order segments with deterministic output

func NewSegmentTDFWriter

func NewSegmentTDFWriter(expectedSegments int, opts ...Option) SegmentWriter

NewSegmentTDFWriter creates a new SegmentWriter for out-of-order segment writing

type TDFReader

type TDFReader struct {
	// contains filtered or unexported fields
}

func NewTDFReader

func NewTDFReader(readSeeker io.ReadSeeker) (TDFReader, error)

NewTDFReader Create tdf reader instance.

func (TDFReader) Manifest

func (tdfReader TDFReader) Manifest() (string, error)

Manifest Return the manifest of the tdf.

func (TDFReader) PayloadSize

func (tdfReader TDFReader) PayloadSize() (int64, error)

PayloadSize Return the size of the payload.

func (TDFReader) ReadPayload

func (tdfReader TDFReader) ReadPayload(index, length int64) ([]byte, error)

ReadPayload Return the payload of given length from index.

type Writer

type Writer interface {
	io.Closer
}

Writer is the base interface for all archive writers

type Zip32DataDescriptor

type Zip32DataDescriptor struct {
	Signature        uint32
	Crc32            uint32
	CompressedSize   uint32
	UncompressedSize uint32
}

type Zip64DataDescriptor

type Zip64DataDescriptor struct {
	Signature        uint32
	Crc32            uint32
	CompressedSize   uint64
	UncompressedSize uint64
}

type Zip64EndOfCDRecord

type Zip64EndOfCDRecord struct {
	Signature                          uint32
	RecordSize                         uint64
	VersionMadeBy                      uint16
	VersionToExtract                   uint16
	DiskNumber                         uint32
	StartDiskNumber                    uint32
	NumberOfCDRecordEntries            uint64
	TotalCDRecordEntries               uint64
	CentralDirectorySize               uint64
	StartingDiskCentralDirectoryOffset uint64
}

type Zip64EndOfCDRecordLocator

type Zip64EndOfCDRecordLocator struct {
	Signature         uint32
	CDStartDiskNumber uint32
	CDOffset          uint64
	NumberOfDisks     uint32
}

type Zip64ExtendedInfoExtraField

type Zip64ExtendedInfoExtraField struct {
	Signature             uint16
	Size                  uint16
	OriginalSize          uint64
	CompressedSize        uint64
	LocalFileHeaderOffset uint64
}

type Zip64ExtendedLocalInfoExtraField

type Zip64ExtendedLocalInfoExtraField struct {
	Signature      uint16
	Size           uint16
	OriginalSize   uint64
	CompressedSize uint64
}

type Zip64Mode

type Zip64Mode int

Zip64Mode controls when ZIP64 structures are used.

const (
	Zip64Auto   Zip64Mode = iota // Use ZIP64 only when needed
	Zip64Always                  // Force ZIP64 even for small archives
	Zip64Never                   // Forbid ZIP64; error if limits exceeded
)

type ZipFileEntry

type ZipFileEntry struct {
	// contains filtered or unexported fields
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL