buf

package
v0.30.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 18, 2025 License: MIT Imports: 11 Imported by: 2

Documentation

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Buffer 

type Buffer struct {
	// contains filtered or unexported fields
}

Buffer implements both StatsReadCloser and StatsWriteCloser interfaces.

Buffer is meant to abstract away the details of writing and reading to either a file buffer or in-memory buffer.

Buffer will: - compress writes if Options.Compress is true. - keep track of buffer statistics - calculate MD5 checksum on calling Close() - calculate the buffer size - provide the tmp file path in the file stats. - clean up tmp file if Abort() or Cleanup() are called.

func NewBuffer 

func NewBuffer(opt *Options) (b *Buffer, err error)
Example 
bfr, err := NewBuffer(nil)
if bfr == nil {
	return
}

fmt.Println(err)
fmt.Println(bfr.sts.Created() != "")
fmt.Println(bfr.w != nil)
fmt.Println(bfr.wGzip)
fmt.Println(bfr.wSize != nil)
fmt.Println(bfr.bBuf != nil)
fmt.Println(bfr.fBuf)
fmt.Println(bfr.r != nil)
fmt.Println(bfr.hshr != nil)

Output:

<nil>
true
true
<nil>
true
true
<nil>
true
true
Example (WithCompression) 
opt := NewOptions()
opt.Compress = true

bfr, err := NewBuffer(opt)
if bfr == nil {
	return
}

fmt.Println(err)
fmt.Println(bfr.sts.Created() != "")
fmt.Println(bfr.w != nil)
fmt.Println(bfr.wGzip != nil)
fmt.Println(bfr.wSize != nil)
fmt.Println(bfr.bBuf != nil)
fmt.Println(bfr.fBuf)
fmt.Println(bfr.r != nil)
fmt.Println(bfr.hshr != nil)

Output:

<nil>
true
true
true
true
true
<nil>
true
true
Example (WithTmpFile) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"

bfr, err := NewBuffer(opt)
if bfr == nil {
	return
}

fmt.Println(err)
fmt.Println(bfr.sts.Created() != "")
fmt.Println(strings.Contains(bfr.sts.Path(), "/tmp/tmpprefix_"))
fmt.Println(bfr.w != nil)
fmt.Println(bfr.wGzip)
fmt.Println(bfr.wSize != nil)
fmt.Println(bfr.bBuf)
fmt.Println(bfr.fBuf != nil)
fmt.Println(bfr.r != nil)
fmt.Println(bfr.hshr != nil)

os.Remove(bfr.sts.Path()) // cleanup tmp file
os.Remove("./tmp")        // remove dir

Output:

<nil>
true
true
true
<nil>
true
<nil>
true
true
true
Example (WithTmpFileErr) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "/private"
opt.FileBufPrefix = "tmpprefix_"

bfr, err := NewBuffer(opt)
if err == nil {
	return
}

fmt.Println(bfr)
fmt.Println(strings.Contains(err.Error(), "permission denied"))

Output:

<nil>
true

func (*Buffer) Abort 

func (bfr *Buffer) Abort() (err error)

Abort will clear the buffer (remove tmp file if exists) and prevent further buffer writes.

Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
err := bfr.Abort()

fmt.Println(err)

Output:

<nil>
Example (Compression) 
opt := NewOptions()
opt.Compress = true

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))

err := bfr.Abort()
fmt.Println(err)

Output:

<nil>

func (*Buffer) Cleanup 

func (bfr *Buffer) Cleanup() (err error)

Cleanup will remove the tmp file (if exists) or reset the in-memory buffer (if used).

Cleanup should not be used until the user is done with the contents of the buffer.

Cleanup is called automatically as part of the abort process but since the user may wish to read from the buffer after closing, Cleanup will need to be called after Close, especially if using compression since Close flushes the compression buffer and finalizes writing.

Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))

err := bfr.Cleanup()

// bytes should be re-set
// so reading from bBuf should return EOF
b := make([]byte, 1)
n, bErr := bfr.bBuf.Read(b)

fmt.Println(err)
fmt.Println(n)
fmt.Println(bErr)

Output:

<nil>
0
EOF
Example (TmpFile) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))

err := bfr.Cleanup()

// check if tmp file exists
f, oErr := os.Open(bfr.sts.Path())
if oErr == nil {
	return
}

fmt.Println(err)
fmt.Println(f)
fmt.Println(strings.Contains(
	oErr.Error(),
	"no such file or directory"))

os.Remove("./tmp") // cleanup dir

Output:

<nil>
<nil>
true

func (*Buffer) Close 

func (bfr *Buffer) Close() (err error)

Close prevents further writing and flushes writes to the underlying buffer.

Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))

fmt.Println(bfr.sts.Checksum() == "")
fmt.Println(bfr.sts.Size)

err := bfr.Close()

fmt.Println(err)
fmt.Println(bfr.sts.Checksum() != "")
fmt.Println(bfr.sts.Size)

// closing again has no effect
err = bfr.Close()

fmt.Println(err)
fmt.Println(bfr.sts.Checksum() != "")
fmt.Println(bfr.sts.Size)

Output:

true
0
<nil>
true
20
<nil>
true
20
Example (Checksum) 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
bfr.Close()

fmt.Println(bfr.sts.Checksum())

Output:

54f30d75cf7374c7e524a4530dbc93c2
Example (ChecksumCompressed) 
opt := NewOptions()
opt.Compress = true

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
bfr.Close()

fmt.Println(bfr.sts.Checksum())

Output:

42e649f9834028184ec21940d13a300f
Example (ChecksumTmpFile) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
bfr.Close()

fmt.Println(bfr.sts.Checksum())

os.Remove(bfr.sts.Path()) // cleanup tmp file
os.Remove("./tmp")        // remove dir

Output:

54f30d75cf7374c7e524a4530dbc93c2
Example (ChecksumTmpFileCompressed) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"
opt.Compress = true

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
bfr.Close()

fmt.Println(bfr.sts.Checksum())

os.Remove(bfr.sts.Path()) // cleanup tmp file
os.Remove("./tmp")        // remove dir

Output:

42e649f9834028184ec21940d13a300f
Example (SizeCompressed) 
opt := NewOptions()
opt.Compress = true
bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
bfr.Close()

// ByteCnt and Size are different
// because of compression.
fmt.Println(bfr.sts.ByteCnt)
fmt.Println(bfr.sts.Size)

Output:

20
48
Example (TmpFileSizeCompressed) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"
opt.Compress = true
bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
bfr.Close()

// ByteCnt and Size are different
// because of compression.
fmt.Println(bfr.sts.ByteCnt)
fmt.Println(bfr.sts.Size)

os.Remove(bfr.sts.Path()) // cleanup tmp file
os.Remove("./tmp")        // remove dir

Output:

20
48

func (*Buffer) Read 

func (bfr *Buffer) Read(p []byte) (n int, err error)

Read will read the raw underlying buffer bytes. If the buffer is writing with compression it will not decompress on reads. Read is made for reading the final written bytes and copying them to the final location.

Close should be called before Read as Close will sync the underlying buffer. This is especially important when using compression and/or a tmp file.

Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.Write([]byte("test line\n"))
bfr.Write([]byte("test line\n"))

// read
b := make([]byte, 20)
bfr.Read(b)

fmt.Print(string(b))

Output:

test line
test line
Example (AfterCleanup) 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.Write([]byte("test line\n"))
bfr.Write([]byte("test line\n"))
bfr.Close()

// resets/removes the underlying buffer.
// after calling Cleanup there is nothing
// left to read. So read always returns EOF.
bfr.Cleanup()

// read
b := make([]byte, 1)
n, err := bfr.Read(b) // nothing should be read in

fmt.Println(n)
fmt.Println(err)
fmt.Println(b[0])

// read again - same result
b = make([]byte, 1)
n, err = bfr.Read(b) // nothing should be read in

fmt.Println(n)
fmt.Println(err)
fmt.Println(b[0])

Output:

0
EOF
0
0
EOF
0
Example (Compressed) 
opt := NewOptions()
opt.Compress = true
bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.Write([]byte("test line\n"))
bfr.Write([]byte("test line\n"))
bfr.Close()

// read with decompressor
b := make([]byte, 20)
r, _ := gzip.NewReader(bfr)
r.Read(b)

fmt.Print(string(b))

Output:

test line
test line
Example (TmpFile) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.Write([]byte("test line\n"))
bfr.Write([]byte("test line\n"))
bfr.Close()

// read
b := make([]byte, 20)
bfr.Read(b)

fmt.Print(string(b))

os.Remove(bfr.sts.Path()) // cleanup tmp file
os.Remove("./tmp")        // remove dir

Output:

test line
test line
Example (TmpFileCompressed) 
opt := NewOptions()
opt.UseFileBuf = true
opt.FileBufDir = "./tmp"
opt.FileBufPrefix = "tmpprefix_"
opt.Compress = true

bfr, _ := NewBuffer(opt)
if bfr == nil {
	return
}

// write first
bfr.Write([]byte("test line\n"))
bfr.Write([]byte("test line\n"))
bfr.Close()

// read with decompressor
b := make([]byte, 20)
r, _ := gzip.NewReader(bfr)
r.Read(b)

fmt.Print(string(b))

os.Remove(bfr.sts.Path()) // cleanup tmp file
os.Remove("./tmp")        // remove dir

Output:

test line
test line

func (*Buffer) Reset added in v0.4.0 

func (bfr *Buffer) Reset()

Reset will reset the in-memory buffer (if used) and remove the reference to the tmp file (if exists)

Reset does not verify that the tmp file is closed

func (*Buffer) Stats 

func (bfr *Buffer) Stats() stat.Stats
Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
bfr.WriteLine([]byte("test line"))
bfr.WriteLine([]byte("test line"))
sts := bfr.Stats()

fmt.Println(sts.ByteCnt)
fmt.Println(sts.LineCnt)

Output:

20
2

func (*Buffer) Write 

func (bfr *Buffer) Write(p []byte) (n int, err error)

Write will write to the underlying buffer. The underlying bytes writing will be compressed if compression was specified on buffer initialization.

Write is thread safe.

Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
n1, err1 := bfr.Write([]byte("test line\n"))
n2, err2 := bfr.Write([]byte("test line\n"))

// read
b := make([]byte, 20)
bfr.Read(b)

fmt.Println(n1)
fmt.Println(err1)
fmt.Println(n2)
fmt.Println(err2)
fmt.Print(string(b))
fmt.Println(bfr.sts.ByteCnt)
fmt.Println(bfr.sts.LineCnt)
fmt.Println(bfr.wSize.Size())

Output:

10
<nil>
10
<nil>
test line
test line
20
0
20

func (*Buffer) WriteLine 

func (bfr *Buffer) WriteLine(ln []byte) (err error)

WriteLine will write to the underlying buffer. The underlying bytes writing will be compressed if compression was specified on buffer initialization.

WriteLine is thread safe.

Example 
bfr, _ := NewBuffer(nil)
if bfr == nil {
	return
}

// write first
err1 := bfr.WriteLine([]byte("test line"))
err2 := bfr.WriteLine([]byte("test line"))

// read
b := make([]byte, 20)
bfr.Read(b)

fmt.Println(err1)
fmt.Println(err2)
fmt.Print(string(b))
fmt.Println(bfr.sts.ByteCnt)
fmt.Println(bfr.sts.LineCnt)
fmt.Println(bfr.wSize.Size())

Output:

<nil>
<nil>
test line
test line
20
2
20

type Options 

type Options struct {
	// UseFileBuf specifies to use a tmp file for the delayed writing.
	// Can optionally also specify the tmp directory and tmp name
	// prefix.
	UseFileBuf bool

	// FileBufDir optionally specifies the temp directory. If not specified then
	// the os default temp dir is used.
	FileBufDir string

	// FileBufPrefix optionally specifies the temp file prefix.
	// The full tmp file name is randomly generated and guaranteed
	// not to conflict with existing files. A prefix can help one find
	// the tmp file.
	FileBufPrefix string

	// Compress set to true will turn on gzip compression.
	// Writes will be compressed but reads will read raw
	// compressed bytes.
	Compress bool

	// CompressLevel type of compression used on the file
	// gzip.BestSpeed = 1 // quick but very little compression
	// gzip.BestCompression = 9 // smallest size but takes longer
	// gzip.DefaultCompression = -1 // balance between speed and size
	CompressLevel int

	// KeepFailed files when using a file buffer and the
	// copy commands fails
	KeepFailed bool
}

func NewOptions 

func NewOptions() *Options
Example 
opt := NewOptions()
if opt == nil {
	return
}

fmt.Println(opt.FileBufDir)
fmt.Println(opt.UseFileBuf)
fmt.Println(opt.FileBufPrefix)
fmt.Println(opt.Compress)

Output:


false

false

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.