Documentation
¶
Index ¶
- Variables
- func Execute() error
- func GenerateHashFromIDs(rowData []string, variantID string, ids []int) string
- func LoadConfig()
- func LoadSchemaFromFileOrRepo(ctx *ls.Context, compiledSchema, repoDir, schemaName, typeName string, ...) (*ls.Layer, error)
- func LoadValuesetFiles(ctx *ls.Context, env map[string]string, vs *Valuesets, ...) error
- func NewReadGraphStep(cmd *cobra.Command) pipeline.ReadGraphStep
- func OutputIngestedGraph(cmd *cobra.Command, outFormat string, target *lpg.Graph, wr io.Writer, ...) error
- func ReadLayers(input []byte, interner ls.Interner) ([]*ls.Layer, error)
- func RecalculatePaths(bnd *bundle.Bundle, dir string)
- type Attribute
- type BaseIngestParams
- type BundlesSchemaLoader
- type CSVExport
- type CSVIngester
- type CSVJoinConfig
- type CSVJoinIngester
- type ImportJSONSchemaRequest
- type JSONExport
- type JSONIngester
- type LS
- type Layer
- type MeasureStep
- type OCStep
- type Options
- type ReshapeStep
- type SliceByTermsSpec
- type Valueset
- type ValuesetStep
- type ValuesetValue
- type Valuesets
- type WriteGraphStep
- type XMLIngester
Constants ¶
This section is empty.
Variables ¶
View Source
var Environment = make(map[string]string)
View Source
var ExportTarget = io.Writer(os.Stdout)
Functions ¶
func GenerateHashFromIDs ¶
func LoadConfig ¶
func LoadConfig()
func LoadValuesetFiles ¶
func NewReadGraphStep ¶
func NewReadGraphStep(cmd *cobra.Command) pipeline.ReadGraphStep
func OutputIngestedGraph ¶
func ReadLayers ¶
ReadLayers reads layer(s) from jsongraph, jsonld
func RecalculatePaths ¶
Types ¶
type Attribute ¶
type Attribute struct {
ID string `json:"@id"`
AttributeName string `json:"attributeName"`
Types string `json:"@type"`
}
Following structs are used to define a Layered Schema
type BaseIngestParams ¶
type BaseIngestParams struct {
Repo string `json:"repo" yaml:"repo"`
Schema string `json:"schema" yaml:"schema"`
Type string `json:"type" yaml:"type"`
Bundle []string `json:"bundle" yaml:"bundle"`
CompiledSchema string `json:"compiledSchema" yaml:"compiledSchema"`
EmbedSchemaNodes bool `json:"embedSchemaNodes" yaml:"embedSchemaNodes"`
OnlySchemaAttributes bool `json:"onlySchemaAttributes" yaml:"onlySchemaAttributes"`
IngestNullValues bool `json:"ingestNullValues" yaml:"ingestNullValues"`
}
func (BaseIngestParams) IsEmptySchema ¶
func (b BaseIngestParams) IsEmptySchema() bool
IsEmptySchema returns true if none of the schema properties are set
type BundlesSchemaLoader ¶
type BundlesSchemaLoader struct {
Bundles []*bundle.Bundle
// contains filtered or unexported fields
}
func LoadBundle ¶
func LoadBundle(ctx *ls.Context, file []string) (BundlesSchemaLoader, error)
func (BundlesSchemaLoader) LoadSchema ¶
func (b BundlesSchemaLoader) LoadSchema(ref string) (*ls.Layer, error)
type CSVExport ¶
type CSVIngester ¶
type CSVIngester struct {
BaseIngestParams
StartRow int `json:"startRow" yaml:"startRow"`
EndRow int `json:"endRow" yaml:"endRow"`
HeaderRow int `json:"headerRow" yaml:"headerRow"`
ID string `json:"id" yaml:"id"`
IngestByRows bool `json:"ingestByRows" yaml:"ingestByRows"`
Delimiter string `json:"delimiter" yaml:"delimiter"`
// contains filtered or unexported fields
}
func (*CSVIngester) Flush ¶
func (ci *CSVIngester) Flush(ctx *pipeline.PipelineContext) error
func (CSVIngester) Help ¶
func (CSVIngester) Help()
func (CSVIngester) Name ¶
func (CSVIngester) Name() string
func (*CSVIngester) Run ¶
func (ci *CSVIngester) Run(pipeline *pipeline.PipelineContext) error
type CSVJoinConfig ¶
type CSVJoinConfig struct {
VariantID string `json:"variantId" yaml:"variantId"`
Cols []int `json:"cols" yaml:"cols"`
IDCols []int `json:"idCols" yaml:"idCols"`
// Any one of these columns must be full to assume this is valid entry
Any []int `json:"any" yaml:"any"`
// contains filtered or unexported fields
}
type CSVJoinIngester ¶
type CSVJoinIngester struct {
BaseIngestParams
StartRow int `json:"startRow" yaml:"startRow"`
EndRow int `json:"endRow" yaml:"endRow"`
HeaderRow int `json:"headerRow" yaml:"headerRow"`
ID string `json:"id" yaml:"id"`
Delimiter string `json:"delimiter" yaml:"delimiter"`
Entities []CSVJoinConfig `json:"entities" yaml:"entities"`
// contains filtered or unexported fields
}
func (*CSVJoinIngester) Flush ¶
func (cji *CSVJoinIngester) Flush(pipeline *pipeline.PipelineContext) error
func (CSVJoinIngester) Name ¶
func (CSVJoinIngester) Name() string
func (*CSVJoinIngester) Run ¶
func (cji *CSVJoinIngester) Run(pipeline *pipeline.PipelineContext) error
type ImportJSONSchemaRequest ¶
type ImportJSONSchemaRequest struct {
Entities []jsonsch.Entity `json:"entities"`
SchemaID string `json:"schemaId"`
Schema string `json:"schemaVariant"`
Layers []SliceByTermsSpec `json:"layers"`
}
func (*ImportJSONSchemaRequest) CompileAndImport ¶
func (req *ImportJSONSchemaRequest) CompileAndImport() (*lpg.Graph, []jsonsch.EntityLayer, error)
func (*ImportJSONSchemaRequest) Slice ¶
func (req *ImportJSONSchemaRequest) Slice(index int, item jsonsch.EntityLayer) ([]*ls.Layer, error)
type JSONExport ¶
type JSONExport struct{}
func (*JSONExport) Flush ¶
func (*JSONExport) Flush(*pipeline.PipelineContext) error
func (JSONExport) Help ¶
func (JSONExport) Help()
func (JSONExport) Name ¶
func (JSONExport) Name() string
func (*JSONExport) Run ¶
func (*JSONExport) Run(pipeline *pipeline.PipelineContext) error
type JSONIngester ¶
type JSONIngester struct {
BaseIngestParams
ID string
// contains filtered or unexported fields
}
func (*JSONIngester) Flush ¶
func (ji *JSONIngester) Flush(pipeline *pipeline.PipelineContext) error
func (JSONIngester) Help ¶
func (JSONIngester) Help()
func (JSONIngester) Name ¶
func (JSONIngester) Name() string
func (*JSONIngester) Run ¶
func (ji *JSONIngester) Run(pipeline *pipeline.PipelineContext) error
type MeasureStep ¶
type MeasureStep struct {
BaseIngestParams
SchemaNodeIDs []string `json:"schemaNodeIds" yaml:"schemaNodeIds"`
// contains filtered or unexported fields
}
func (*MeasureStep) Flush ¶
func (ms *MeasureStep) Flush(pipeline *pipeline.PipelineContext) error
func (MeasureStep) Help ¶
func (MeasureStep) Help()
func (MeasureStep) Name ¶
func (MeasureStep) Name() string
func (*MeasureStep) Run ¶
func (ms *MeasureStep) Run(pipeline *pipeline.PipelineContext) error
type Options ¶
type Options struct {
// Order in which to search valueset
LookupOrder []string `json:"lookupOrder" yaml:"lookupOrder"`
// Which value to return
Output []string `json:"output" yaml:"output"`
// Types of string separation i.e. ";", "|", ",", " "
Separator map[string]string `json:"separator" yaml:"separator"`
}
type ReshapeStep ¶
type ReshapeStep struct {
BaseIngestParams
ScriptFile string `json:"scriptFile" yaml:"scriptFile"`
// contains filtered or unexported fields
}
func (*ReshapeStep) Flush ¶
func (rs *ReshapeStep) Flush(pipeline *pipeline.PipelineContext) error
func (ReshapeStep) Help ¶
func (ReshapeStep) Help()
func (ReshapeStep) Name ¶
func (ReshapeStep) Name() string
func (*ReshapeStep) Run ¶
func (rs *ReshapeStep) Run(pipeline *pipeline.PipelineContext) error
type SliceByTermsSpec ¶
type Valueset ¶
type Valueset struct {
ID string `json:"id" yaml:"id"`
Values []ValuesetValue `json:"values" yaml:"values"`
Options Options `json:"options" yaml:"options"`
}
func (Valueset) Lookup ¶
func (vs Valueset) Lookup(req ls.ValuesetLookupRequest) (ls.ValuesetLookupResponse, error)
type ValuesetStep ¶
type ValuesetStep struct {
BaseIngestParams
ValuesetFiles []string `json:"valuesetFiles" yaml:"valuesetFiles"`
Tables []string `json:"tables" yaml:"tables"`
// contains filtered or unexported fields
}
func (*ValuesetStep) Flush ¶
func (vs *ValuesetStep) Flush(pipeline *pipeline.PipelineContext) error
func (ValuesetStep) Help ¶
func (ValuesetStep) Help()
func (ValuesetStep) Name ¶
func (ValuesetStep) Name() string
func (*ValuesetStep) Run ¶
func (vs *ValuesetStep) Run(pipeline *pipeline.PipelineContext) error
type ValuesetValue ¶
type ValuesetValue struct {
// Possible input values
Values []string `json:"values" yaml:"values"`
// Possible input value as key-value pairs
KeyValues map[string]string `json:"keyValues" yaml:"keyValues"`
CaseSensitive bool `json:"caseSensitive" yaml:"caseSensitive"`
// Result output value
Result string `json:"result" yaml:"result"`
// Result output values as key-value pairs
ResultValues map[string]string `json:"results" yaml:"results"`
}
func (ValuesetValue) IsDefault ¶
func (v ValuesetValue) IsDefault() bool
func (ValuesetValue) Match ¶
func (v ValuesetValue) Match(req ls.ValuesetLookupRequest) (*ls.ValuesetLookupResponse, error)
type Valuesets ¶
type Valuesets struct {
Services map[string]string `json:"services" yaml:"services"`
Spreadsheets []string `json:"spreadsheets" yaml:"spreadsheets"`
Sets map[string]Valueset `json:"valuesets" yaml:"valuesets"`
// contains filtered or unexported fields
}
func (*Valuesets) LoadSpreadsheets ¶
func (Valuesets) Lookup ¶
func (vsets Valuesets) Lookup(ctx *ls.Context, req ls.ValuesetLookupRequest) (ls.ValuesetLookupResponse, error)
Lookup can be used as the external lookup func of LookupProcessor
type WriteGraphStep ¶
func NewWriteGraphStep ¶
func NewWriteGraphStep(cmd *cobra.Command) WriteGraphStep
func (WriteGraphStep) Flush ¶
func (WriteGraphStep) Flush(*pipeline.PipelineContext) error
func (WriteGraphStep) Help ¶
func (WriteGraphStep) Help()
func (WriteGraphStep) Name ¶
func (WriteGraphStep) Name() string
func (WriteGraphStep) Run ¶
func (wr WriteGraphStep) Run(pipeline *pipeline.PipelineContext) error
type XMLIngester ¶
type XMLIngester struct {
BaseIngestParams
ID string
// contains filtered or unexported fields
}
func (*XMLIngester) Flush ¶
func (xml *XMLIngester) Flush(pipeline *pipeline.PipelineContext) error
func (XMLIngester) Help ¶
func (XMLIngester) Help()
func (XMLIngester) Name ¶
func (XMLIngester) Name() string
func (*XMLIngester) Run ¶
func (xml *XMLIngester) Run(pipeline *pipeline.PipelineContext) error
Source Files
¶
- applylayer.go
- compact.go
- compile.go
- compose.go
- config.go
- dot.go
- expand.go
- export.go
- export_csv.go
- export_json.go
- flatten.go
- frame.go
- getschema.go
- getschema_csv.go
- getschema_json.go
- import.go
- import_csv.go
- import_json.go
- ingest.go
- ingest_csv.go
- ingest_json.go
- ingest_xml.go
- list.go
- oc.go
- pipeline.go
- reshape.go
- root.go
- schema.go
- slice.go
- template.go
- units.go
- validate.go
- validate_json.go
- valueset.go
- writegraph.go
Click to show internal directories.
Click to hide internal directories.