Documentation
¶
Index ¶
- func ArraysBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Arrays) <-chan error
- func ComplexBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Complex) <-chan error
- func RepeatsBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Repeats) <-chan error
- func SimpleBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Simple) <-chan error
- func TimesBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Times) <-chan error
- type Arrays
- type Complex
- type Repeats
- type Simple
- type Times
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ArraysBulkAvroWriter ¶ added in v0.12.2
func ArraysBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Arrays) <-chan error
ArraysBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.
Example ¶
input := []*Arrays{{}, {}, {}}
inputChan := make(chan *Arrays)
devnull, _ := os.Open("/dev/null")
defer devnull.Close()
errChan := ArraysBulkAvroWriter(devnull, time.Now(), inputChan)
for _, item := range input {
select {
case err := <-errChan:
fmt.Print(err)
return
case inputChan <- item:
}
}
// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
if err != nil {
fmt.Print(err)
return
}
}
func ComplexBulkAvroWriter ¶ added in v0.12.2
func ComplexBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Complex) <-chan error
ComplexBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.
Example ¶
input := []*Complex{{}, {}, {}}
inputChan := make(chan *Complex)
devnull, _ := os.Open("/dev/null")
defer devnull.Close()
errChan := ComplexBulkAvroWriter(devnull, time.Now(), inputChan)
for _, item := range input {
select {
case err := <-errChan:
fmt.Print(err)
return
case inputChan <- item:
}
}
// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
if err != nil {
fmt.Print(err)
return
}
}
func RepeatsBulkAvroWriter ¶ added in v0.12.2
func RepeatsBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Repeats) <-chan error
RepeatsBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.
Example ¶
input := []*Repeats{{}, {}, {}}
inputChan := make(chan *Repeats)
devnull, _ := os.Open("/dev/null")
defer devnull.Close()
errChan := RepeatsBulkAvroWriter(devnull, time.Now(), inputChan)
for _, item := range input {
select {
case err := <-errChan:
fmt.Print(err)
return
case inputChan <- item:
}
}
// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
if err != nil {
fmt.Print(err)
return
}
}
func SimpleBulkAvroWriter ¶ added in v0.12.2
func SimpleBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Simple) <-chan error
SimpleBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.
Example ¶
input := []*Simple{{}, {}, {}}
inputChan := make(chan *Simple)
devnull, _ := os.Open("/dev/null")
defer devnull.Close()
errChan := SimpleBulkAvroWriter(devnull, time.Now(), inputChan)
for _, item := range input {
select {
case err := <-errChan:
fmt.Print(err)
return
case inputChan <- item:
}
}
// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
if err != nil {
fmt.Print(err)
return
}
}
func TimesBulkAvroWriter ¶ added in v0.12.14
TimesBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.
Example ¶
input := []*Times{{}, {}, {}}
inputChan := make(chan *Times)
devnull, _ := os.Open("/dev/null")
defer devnull.Close()
errChan := TimesBulkAvroWriter(devnull, time.Now(), inputChan)
for _, item := range input {
select {
case err := <-errChan:
fmt.Print(err)
return
case inputChan <- item:
}
}
// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
if err != nil {
fmt.Print(err)
return
}
}
Types ¶
type Arrays ¶ added in v0.12.2
type Arrays struct {
Heights []int64 `json:"heights,omitempty"`
Parents []struct {
Count int64 `json:"count"`
Children []string `json:"children"`
Date time.Time `json:"date"`
Info struct {
Name string `json:"name"`
Age int64 `json:"age"`
} `json:"info"`
} `json:"parents"`
}
func (*Arrays) WriteAvroCF ¶ added in v0.12.2
WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.
type Complex ¶
type Complex struct {
Simple
Caption string `json:"caption"`
Credit string `json:"credit"`
Crops []struct {
Height float64 `json:"height"`
Name string `json:"name"`
Path string `json:"path" description:"full path to the cropped image file"`
RelativePath string `json:"relativePath" description:"a long"`
Width float64 `json:"width"`
} `json:"crops"`
Cutline string `json:"cutline,omitempty"`
DatePhotoTaken time.Time `json:"datePhotoTaken"`
Orientation string `json:"orientation"`
OriginalSize struct {
Height float64 `json:"height"`
Width float64 `json:"width"`
} `json:"originalSize"`
Type string `json:"type" description:"a type"`
URL struct {
Absolute string `json:"absolute"`
Meta struct {
Description string `json:"description"`
SiteName string `json:"siteName"`
} `json:"meta,omitempty"`
Publish string `json:"publish"`
} `json:"URL"`
}
func (*Complex) WriteAvroCF ¶
WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.
type Repeats ¶
type Repeats struct {
Height int64 `json:"height,omitempty"`
SomeDateObj struct {
Type string `json:"type"`
Visible bool `json:"visible,omitempty"`
} `json:"someDateObj,omitempty"`
Type string `json:"type"`
Visible bool `json:"visible,omitempty"`
Width float64 `json:"width,omitempty"`
}
func (*Repeats) WriteAvroCF ¶
WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.
type Simple ¶
type Simple struct {
Height int64 `json:"height,omitempty"`
SomeDateObj struct {
Dates []time.Time `json:"dates,omitempty"`
} `json:"someDateObj,omitempty"`
Type string `json:"type"`
Visible bool `json:"visible,omitempty"`
Width float64 `json:"width,omitempty"`
}
func (*Simple) WriteAvroCF ¶
WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.
type Times ¶ added in v0.12.14
type Times struct {
NonRequiredDate time.Time `json:"nonRequiredDate,omitempty"`
RequiredDate time.Time `json:"requiredDate"`
}
func (*Times) WriteAvroCF ¶ added in v0.12.14
WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.
Source Files
Directories