manifester

package
v0.0.0-...-ed487d1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 19, 2026 License: GPL-2.0 Imports: 46 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// Build modes
	ModeNar            = "" // default
	ModeGenericTarball = "generic-tarball"
)
View Source
const (
	ChunkDiffMaxDigests = 256
	ChunkDiffMaxBytes   = 16 << 20
)
View Source
const (
	DefaultSmallFileCutoff = 224

	SmallManifestCutoff = 32 * 1024
)

Variables

View Source
var (
	ErrReq      = errors.New("request err")
	ErrNotFound = wrapNotFoundErr{errors.New("not found")}
	ErrInternal = errors.New("internal err")
)
View Source
var (
	// protocol is (mostly) json over http
	ManifestPath  = "/manifest"
	ChunkDiffPath = "/chunkdiff"

	// chunk read protocol
	ChunkReadPath     = "/chunk/"    // digest as final path component
	ManifestCachePath = "/manifest/" // cache key as final path component

	BuildRootPath = "/buildroot/" // written by manifester, read only by gc

	ExpandGz = "gz"
	ExpandXz = "xz"

	// header value is base64(proto-encoded pb.Lengths)
	LengthsHeader = "x-styx-lengths"
)

Functions

func IsS3NotFound

func IsS3NotFound(err error) bool

func NewManifestServer

func NewManifestServer(cfg Config, mb *ManifestBuilder) (*server, error)

Types

type BuildArgs

type BuildArgs struct {
	SmallFileCutoff int

	ShardTotal int
	ShardIndex int
	// contains filtered or unexported fields
}

type ChunkDiffStats

type ChunkDiffStats struct {
	Reqs       int   `json:"reqs"`
	Expands    int   `json:"exps"`
	BaseChunks int   `json:"baseC"`
	BaseBytes  int   `json:"baseB"`
	ReqChunks  int   `json:"reqC"`
	ReqBytes   int   `json:"reqB"`
	DiffBytes  int   `json:"diffB"`
	DlTotalMs  int64 `json:"dlMs"`
	ZstdMs     int64 `json:"zstdMs"`
}

ChunkDiffStats must contain only integers! (For now, since we sometimes scan backwards to find the start of the stats. We can relax this requirement if we write a reverse json parser.)

type ChunkStoreRead

type ChunkStoreRead interface {
	// Data will be appended to dst and returned.
	// If dst is given, it must be big enough to hold the full chunk!
	Get(ctx context.Context, key string, dst []byte) ([]byte, error)
}

func NewChunkStoreReadUrl

func NewChunkStoreReadUrl(url, path string) ChunkStoreRead

path should be either ChunkReadPath or ManifestCachePath

type ChunkStoreWrite

type ChunkStoreWrite interface {
	PutIfNotExists(ctx context.Context, path, key string, data []byte) ([]byte, error)
	Get(ctx context.Context, path, key string, dst []byte) ([]byte, error)
}

func NewChunkStoreWrite

func NewChunkStoreWrite(cfg ChunkStoreWriteConfig) (ChunkStoreWrite, error)

type ChunkStoreWriteConfig

type ChunkStoreWriteConfig struct {
	// One of these is required:
	ChunkBucket      string
	ChunkLocalDir    string
	ZstdEncoderLevel int
}

type Config

type Config struct {
	Bind             string
	AllowedUpstreams []string

	ChunkDiffZstdLevel int
	ChunkDiffParallel  int
}

type DeprecatedChunkDiffReq

type DeprecatedChunkDiffReq struct {
	Bases []byte
	Reqs  []byte

	// If set: Bases and Reqs each comprise one single file in the given compression
	// format. Pass each one through this decompressor before diffing.
	ExpandBeforeDiff string `json:",omitempty"`
}

deprecated: use pb.ManifesterChunkDiffReq

type ManifestBuildRes

type ManifestBuildRes struct {
	CacheKey     string // path relative to ManifestCachePath
	EtagCacheKey string
	Sph          string
	Bytes        []byte
}

type ManifestBuilder

type ManifestBuilder struct {
	// contains filtered or unexported fields
}

func (*ManifestBuilder) Build

func (b *ManifestBuilder) Build(
	ctx context.Context,
	buildMode, upstream, storePathHash string,
	shardTotal, shardIndex int,
	useLocalStoreDump string,
	writeBuildRoot bool,
) (*ManifestBuildRes, error)

func (*ManifestBuilder) BuildFromNar

func (b *ManifestBuilder) BuildFromNar(
	ctx context.Context,
	upstream, storePathHash string,
	shardTotal, shardIndex int,
	useLocalStoreDump string,
	writeBuildRoot bool,
) (*ManifestBuildRes, error)

func (*ManifestBuilder) BuildFromTarball

func (b *ManifestBuilder) BuildFromTarball(
	ctx context.Context,
	upstream string,
	shardTotal, shardIndex int,
	useLocalStoreDump string,
	writeBuildRoot bool,
) (*ManifestBuildRes, error)

func (*ManifestBuilder) ClearStats

func (b *ManifestBuilder) ClearStats()

func (*ManifestBuilder) ManifestAsEntry

func (b *ManifestBuilder) ManifestAsEntry(ctx context.Context, args *BuildArgs, path string, manifest *pb.Manifest) (*pb.Entry, error)

func (*ManifestBuilder) Stats

func (b *ManifestBuilder) Stats() Stats

If errors occur during building, stats might not be exactly right.

type ManifestBuilderConfig

type ManifestBuilderConfig struct {
	ConcurrentChunkOps int
	ChunkSizer         func(int64) shift.Shift

	// Verify loaded narinfo against these keys. Nil means don't verify.
	PublicKeys []signature.PublicKey
	// Sign manifests with these keys.
	SigningKeys []signature.SecretKey
}

type ManifestReq

type ManifestReq struct {
	Upstream      string // url of nix binary cache or generic file
	StorePathHash string

	// TODO: this should really be a new request type
	BuildMode string `json:",omitempty"`

	// TODO: move this to pb and embed a GlobalParams?
	DigestAlgo string
	DigestBits int

	// used for tarball cache lookups only
	ETag string

	// sharded manifesting (not in cache key, only shard 0 writes to cache)
	ShardTotal int `json:",omitempty"`
	ShardIndex int `json:",omitempty"`
}

func (*ManifestReq) CacheKey

func (r *ManifestReq) CacheKey() string

type Stats

type Stats struct {
	Manifests       int64
	Shards          int64
	TotalChunks     int64
	TotalUncmpBytes int64
	NewChunks       int64
	NewUncmpBytes   int64
	NewCmpBytes     int64
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL