marshaller

ExampleVTproto_MarshalStream demonstrates how to use streaming marshaling

// Create test data
testData := &StoreData{
	Kv: map[string][]byte{
		"account:001": []byte(`{"balance":1000,"nonce":1}`),
		"account:002": []byte(`{"balance":2000,"nonce":5}`),
		"account:003": []byte(`{"balance":500,"nonce":2}`),
	},
	DeletePrefixes: []string{"temp:", "cache:"},
}

marshaller := &VTproto{}

// Stream marshal - generates data on-the-fly without loading everything in memory
reader := marshaller.MarshalStream(testData, 0) // 0 for auto-sizing
defer reader.Close()

// Read data in chunks (simulating writing to file or network)
buffer := make([]byte, 1024)
totalBytes := 0

for {
	n, err := reader.Read(buffer)
	if n > 0 {
		totalBytes += n
		// Process chunk (e.g., write to file, send over network)
		fmt.Printf("Read chunk of %d bytes\n", n)
	}
	if err == io.EOF {
		break
	}
	if err != nil {
		panic(fmt.Sprintf("Read error: %v", err))
	}
}

fmt.Printf("Total bytes streamed: %d\n", totalBytes)

// Note: Actual output will vary based on data structure and marshaling

ExampleVTproto_MarshalStream_fileWrite demonstrates streaming to a file

testData := &StoreData{
	Kv: map[string][]byte{
		"large_data_1": []byte(strings.Repeat("data", 1000)),
		"large_data_2": []byte(strings.Repeat("more", 1000)),
	},
	DeletePrefixes: []string{"cleanup:"},
}

marshaller := &VTproto{}

// Create temporary file
tmpFile, err := os.CreateTemp("", "stream_marshal_*.bin")
if err != nil {
	panic(err)
}
defer os.Remove(tmpFile.Name())
defer tmpFile.Close()

// Stream directly to file without loading all data in memory
reader := marshaller.MarshalStream(testData, 0)
defer reader.Close()

// Copy from stream to file efficiently
bytesWritten, err := io.Copy(tmpFile, reader)
if err != nil {
	panic(err)
}

fmt.Printf("Streamed %d bytes to file\n", bytesWritten)

// Verify by reading back and unmarshaling
tmpFile.Seek(0, 0) // Reset to beginning
result, _, err := marshaller.UnmarshalStream(tmpFile, bytesWritten)
if err != nil {
	panic(err)
}

fmt.Printf("Successfully round-tripped %d KV pairs\n", len(result.Kv))

// Note: Actual byte count and file paths will vary

// Create test data
testData := &StoreData{
	Kv: map[string][]byte{
		"key1": []byte("value1"),
		"key2": []byte("value2"),
	},
	DeletePrefixes: []string{"prefix1", "prefix2"},
}

// Marshal the data first
marshaller := &VTproto{}
serialized, err := marshaller.Marshal(testData)
if err != nil {
	panic(fmt.Sprintf("failed to marshal: %v", err))
}

// Create a reader from the serialized data
reader := bytes.NewReader(serialized)

// Unmarshal using the streaming function
result, dataSize, err := marshaller.UnmarshalStream(reader, 0)
if err != nil {
	panic(fmt.Sprintf("failed to unmarshal stream: %v", err))
}

fmt.Printf("Data size: %d\n", dataSize)
fmt.Printf("KV pairs: %d\n", len(result.Kv))
fmt.Printf("Delete prefixes: %d\n", len(result.DeletePrefixes))

Output:

Data size: 20
KV pairs: 2
Delete prefixes: 2