filereader

package
v1.6.0-alpha.7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 31, 2023 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func CreateWorkers

func CreateWorkers(ctx context.Context, n int, opt WorkersOpt) ([]chan<- tasks.ReadFile, *errgroup.Group)

func RoundRobin

func RoundRobin[T any](ts []T) func() T

func TrySync

func TrySync(workers []chan<- tasks.ReadFile)

func WarmUpFooters

func WarmUpFooters(glob string, sendToWorker func(tasks.ReadFile)) error

func WarmUpWholeFile

func WarmUpWholeFile(glob string, sendToWorker func(tasks.ReadFile)) error

func WarmUpWholeFileAfter

func WarmUpWholeFileAfter(glob string, after time.Time, sendToWorker func(tasks.ReadFile)) error

Types

type Config

type Config struct {
	Files           string
	Type            string
	RateLimit       float64
	NWorkers        int
	Direct          bool
	CheckpointEvery uint64
	CheckpointFile  string
}

type ExecContext

type ExecContext struct {
	// contains filtered or unexported fields
}

func New

func New(masterCtx context.Context, config Config) *ExecContext

func (*ExecContext) Run

func (execCtx *ExecContext) Run()

type StatedFile

type StatedFile struct {
	Info os.FileInfo
	Path string
}

func StatFilesByGlob

func StatFilesByGlob(glob string) ([]StatedFile, error)

type WorkersOpt

type WorkersOpt struct {
	ObserveTotalSize *uint64
	RateLimitInMiB   float64
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL