manifest

package
v1.15.7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 30, 2025 License: Apache-2.0 Imports: 55 Imported by: 18

Documentation

Index

Constants

View Source
const (
	ModuleKindStore      = "store"
	ModuleKindMap        = "map"
	ModuleKindBlockIndex = "blockIndex"
)
View Source
const (
	OutputValueTypeInt64      = "int64"
	OutputValueTypeFloat64    = "float64"
	OutputValueTypeBigInt     = "bigint"
	OutputValueTypeBigDecimal = "bigdecimal"

	// Deprecated: bigfloat value type replaced with bigdecimal
	OutputValueTypeBigFloat = "bigfloat"
	OutputValueTypeString   = "string"
)
View Source
const (
	UpdatePolicySet            = "set"
	UpdatePolicySetIfNotExists = "set_if_not_exists"
	UpdatePolicyAdd            = "add"
	UpdatePolicyMax            = "max"
	UpdatePolicyMin            = "min"
	UpdatePolicyAppend         = "append"
	UpdatePolicySetSum         = "set_sum"
)
View Source
const PrefixSeparator = ":"
View Source
const UNSET = math.MaxUint64

Variables

View Source
var IPFSTimeout time.Duration
View Source
var IPFSURL string
View Source
var JPGHeader = []byte{0xff, 0xd8, 0xff}
View Source
var PNGHeader = []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}
View Source
var TestUseSimpleHash = false
View Source
var WebPHeader = []byte{0x52, 0x49, 0x46, 0x46}

Functions

func ApplyNetwork added in v1.3.0

func ApplyNetwork(network string, pkg *pbsubstreams.Package) error

func ApplyParams added in v1.1.1

func ApplyParams(params map[string]string, pkg *pbsubstreams.Package) error

func BuildMessageDescriptors added in v1.0.2

func BuildMessageDescriptors(pkg *pbsubstreams.Package) (out map[string]*ModuleDescriptor, err error)

func DescribeSinkConfigs added in v1.1.15

func DescribeSinkConfigs(pkg *pbssv1.Package) (desc string, files map[string][]byte, err error)

DescribeSinkConfigs returns a human-readable description of the sinkconfigs. Fields that were imported from files are returned as bytes in a map

func ExtendedModuleHash added in v1.15.5

func ExtendedModuleHash(module *pbsubstreams.Module, hash string) string

ExtendedModuleHash adds the missing input store 'mode' and store 'valueType' and 'updatePolicy' they are not required for the cached data to be consistent, but they can make it fail, we don't want those failures to persist after the user fixes his substreams.yaml

func ExtractNetworkEndpoint added in v1.1.15

func ExtractNetworkEndpoint(networkFromManifest, fromFlag string, logger *zap.Logger) (string, error)

func GenerateMermaidLiveURL added in v1.0.1

func GenerateMermaidLiveURL(mods *pbsubstreams.Modules) string

func IsLikelyManifestInput added in v1.1.12

func IsLikelyManifestInput(in string) bool

IsLikelyManifestInput determines if the input is likely a manifest input, which is determined by checking:

  • If the input starts with remote prefix ("https://", "http://", "ipfs://", "gs://", "s3://", "az://")
  • If the input ends with `.yaml`
  • If the input is a directory (we check for path separator)

func NewSimpleTestModules added in v1.1.1

func NewSimpleTestModules() []*pbsubstreams.Module

NewSimpleTestModules can be used in foreign packages for their test suite

func NewTestModules

func NewTestModules() []*pbsubstreams.Module

NewTestModules can be used in foreign packages for their test suite

func ParseParams added in v1.3.0

func ParseParams(paramsString []string) (map[string]string, error)

func PrintMermaid

func PrintMermaid(mods *pbsubstreams.Modules)

func SortModuleNamesByGraphTopology added in v0.1.0

func SortModuleNamesByGraphTopology(mods []string, g *ModuleGraph) []string

func SplitBinaryType added in v1.7.1

func SplitBinaryType(in string) (typeID string, rawExtensions string)

SplitBinaryType splits a binary type in two components: the type ID and the raw extensions.

The format is `<id>+<extensions>`, where `<id>` is the binary type ID and `<extensions>` is a comma-separated list of runtime extensions that are type specific. The `+<extensions>` part is optional and can be omitted.

So we accept the following formats:

  • wasm/rust-v1
  • wasm/rust-v1+wasm-bindgen-shims
  • wasm/rust-v1+wasm-bindgen-shims,other-extension=value

This method returns the ID and the raw unsplitted extensions string. The input `wasm/rust-v1+wasm-bindgen-shims,other-extension=value` would result in ("wasm/rust-v1", "wasm-bindgen-shims,other-extension=value") being returned to your

func TestReadManifest added in v0.1.0

func TestReadManifest(t testing.T, manifestPath string) *pbsubstreams.Package

func ValidateModules

func ValidateModules(mods *pbsubstreams.Modules) error

ValidateModules is run both by the client _and_ the server.

Types

type Binary

type Binary struct {
	File                string            `yaml:"file,omitempty"`
	Type                string            `yaml:"type,omitempty"`
	Native              string            `yaml:"native,omitempty"`
	Content             []byte            `yaml:"-"`
	Entrypoint          string            `yaml:"entrypoint,omitempty"`
	ProtoPackageMapping map[string]string `yaml:"protoPackageMapping,omitempty"`
	Build               string            `yaml:"build,omitempty"`
}

type BlockFilter added in v1.4.0

type BlockFilter struct {
	Module string           `yaml:"module,omitempty"`
	Query  BlockFilterQuery `yaml:"query,omitempty"`
}

func (*BlockFilter) IsEmpty added in v1.7.2

func (bf *BlockFilter) IsEmpty() bool

type BlockFilterQuery added in v1.6.0

type BlockFilterQuery struct {
	String string `yaml:"string,omitempty"`
	Params bool   `yaml:"params,omitempty"`
}

type BufImport added in v1.6.1

type BufImport struct {
	LocalPath string   `yaml:"localPath,omitempty"`
	Module    string   `yaml:"module"`
	Version   string   `yaml:"version"`
	Symbols   []string `yaml:"symbols"`
}

type Input

type Input struct {
	Source string `yaml:"source,omitempty"`
	Store  string `yaml:"store,omitempty"`
	Map    string `yaml:"map,omitempty"`
	Params string `yaml:"params,omitempty"`

	Mode string `yaml:"mode,omitempty"`
}

func (*Input) IsMap added in v0.1.0

func (i *Input) IsMap() bool

func (*Input) IsParams added in v1.0.0

func (i *Input) IsParams() bool

func (*Input) IsSource added in v0.1.0

func (i *Input) IsSource() bool

func (*Input) IsStore added in v0.1.0

func (i *Input) IsStore() bool

type Manifest

type Manifest struct {
	SpecVersion  string                    `yaml:"specVersion,omitempty"` // check that it equals v0.1.0
	Package      PackageMeta               `yaml:"package,omitempty"`
	Protobuf     Protobuf                  `yaml:"protobuf,omitempty"`
	Imports      mapSlice                  `yaml:"imports,omitempty"`
	Binaries     map[string]Binary         `yaml:"binaries,omitempty"`
	Modules      []*Module                 `yaml:"modules,omitempty"`
	Params       map[string]string         `yaml:"params,omitempty"`
	BlockFilters map[string]string         `yaml:"blockFilters,omitempty"`
	Network      string                    `yaml:"network,omitempty"`
	Networks     map[string]*NetworkParams `yaml:"networks,omitempty"`
	Sink         *Sink                     `yaml:"sink,omitempty"`

	Graph   *ModuleGraph `yaml:"-"`
	Workdir string       `yaml:"-"`
}

Manifest is a YAML structure used to create a Package and its list of Modules. The notion of a manifest does not live in protobuf definitions.

func LoadManifestFile added in v0.1.0

func LoadManifestFile(inputPath, workingDir string) (*Manifest, error)

type Module

type Module struct {
	Name         string       `yaml:"name,omitempty"`
	Doc          string       `yaml:"doc,omitempty"`
	Kind         string       `yaml:"kind,omitempty"`
	InitialBlock *uint64      `yaml:"initialBlock,omitempty"`
	BlockFilter  *BlockFilter `yaml:"blockFilter,omitempty"`

	UpdatePolicy string `yaml:"updatePolicy,omitempty"`
	ValueType    string `yaml:"valueType,omitempty"`
	Binary       string `yaml:"binary,omitempty"`

	Inputs []*Input     `yaml:"inputs,omitempty"`
	Output StreamOutput `yaml:"output,omitempty"`
	Use    string       `yaml:"use,omitempty"`
}

func (*Module) String

func (m *Module) String() string

func (*Module) ToProtoWASM

func (m *Module) ToProtoWASM(codeIndex uint32) (*pbsubstreams.Module, error)

type ModuleDescriptor added in v1.0.2

type ModuleDescriptor struct {
	// Either or
	StoreValueType string
	MapOutputType  string

	ProtoMessageType  string
	MessageDescriptor *desc.MessageDescriptor
}

type ModuleGraph

type ModuleGraph struct {
	*graph.Mutable
	// contains filtered or unexported fields
}

func NewModuleGraph

func NewModuleGraph(modules []*pbsubstreams.Module) (*ModuleGraph, error)

func (*ModuleGraph) AncestorStoresOf

func (g *ModuleGraph) AncestorStoresOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) AncestorsOf

func (g *ModuleGraph) AncestorsOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) ChildrenOf added in v1.0.2

func (g *ModuleGraph) ChildrenOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) Context added in v1.0.2

func (g *ModuleGraph) Context(moduleName string) (parents []string, children []string, err error)

func (*ModuleGraph) GetSources

func (g *ModuleGraph) GetSources() []string

func (*ModuleGraph) GroupedAncestorStores

func (g *ModuleGraph) GroupedAncestorStores(moduleName string) ([][]*pbsubstreams.Module, error)

func (*ModuleGraph) HasStatefulDependencies added in v1.1.20

func (g *ModuleGraph) HasStatefulDependencies(moduleName string) (bool, error)

func (*ModuleGraph) InDegree added in v1.10.0

func (g *ModuleGraph) InDegree(moduleName string) int

func (*ModuleGraph) LeafNames added in v1.10.0

func (g *ModuleGraph) LeafNames() []string

func (*ModuleGraph) Leafs added in v1.10.0

func (g *ModuleGraph) Leafs() []*pbsubstreams.Module

func (*ModuleGraph) MapModules added in v1.10.0

func (g *ModuleGraph) MapModules() []string

func (*ModuleGraph) Module added in v0.0.14

func (g *ModuleGraph) Module(moduleName string) (*pbsubstreams.Module, error)

func (*ModuleGraph) ModuleIndexFromName added in v1.1.1

func (g *ModuleGraph) ModuleIndexFromName(name string) (int, bool)

func (*ModuleGraph) ModuleInitialBlock

func (g *ModuleGraph) ModuleInitialBlock(moduleName string) (uint64, error)

func (*ModuleGraph) ModuleNameFromIndex added in v1.1.1

func (g *ModuleGraph) ModuleNameFromIndex(index int) string

func (*ModuleGraph) Modules added in v1.1.1

func (g *ModuleGraph) Modules() []string

func (*ModuleGraph) ModulesDownTo

func (g *ModuleGraph) ModulesDownTo(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) MustChildrenOf added in v1.0.2

func (g *ModuleGraph) MustChildrenOf(moduleName string) []*pbsubstreams.Module

func (*ModuleGraph) MustParentsOf added in v1.0.2

func (g *ModuleGraph) MustParentsOf(moduleName string) []*pbsubstreams.Module

func (*ModuleGraph) OutDegree added in v1.10.0

func (g *ModuleGraph) OutDegree(moduleName string) int

func (*ModuleGraph) ParentStoresOf

func (g *ModuleGraph) ParentStoresOf(moduleName string) ([]*pbsubstreams.Modules, error)

func (*ModuleGraph) ParentsOf

func (g *ModuleGraph) ParentsOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) ResetGraphHashes added in v0.0.21

func (graph *ModuleGraph) ResetGraphHashes()

ResetGraphHashes is to be called when you want to force a recomputation of the module hashes.

func (*ModuleGraph) RootNames added in v1.10.0

func (g *ModuleGraph) RootNames() []string

func (*ModuleGraph) Roots added in v1.10.0

func (g *ModuleGraph) Roots() []*pbsubstreams.Module

func (*ModuleGraph) StoresDownTo

func (g *ModuleGraph) StoresDownTo(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) TopologicalSort

func (g *ModuleGraph) TopologicalSort() ([]*pbsubstreams.Module, bool)

func (*ModuleGraph) TopologicalSortKnownModules added in v1.1.1

func (g *ModuleGraph) TopologicalSortKnownModules(known map[string]bool) ([]*pbsubstreams.Module, bool)

type ModuleHash

type ModuleHash []byte

type ModuleHashes added in v0.0.21

type ModuleHashes struct {
	// contains filtered or unexported fields
}

func NewModuleHashes added in v0.0.21

func NewModuleHashes() *ModuleHashes

func (*ModuleHashes) Get added in v0.0.21

func (m *ModuleHashes) Get(moduleName string) string

func (*ModuleHashes) HashModule added in v0.0.21

func (m *ModuleHashes) HashModule(modules *pbsubstreams.Modules, module *pbsubstreams.Module, graph *ModuleGraph) (ModuleHash, error)

func (*ModuleHashes) Iter added in v0.0.21

func (m *ModuleHashes) Iter(cb func(hash, name string) error) error

type ModuleMarshaler

type ModuleMarshaler []*pbsubstreams.Module

func (ModuleMarshaler) MarshalJSON

func (m ModuleMarshaler) MarshalJSON() ([]byte, error)

type NetworkParams added in v1.3.0

type NetworkParams struct {
	InitialBlocks map[string]uint64 `yaml:"initialBlock,omitempty" json:"initialBlock,omitempty"`
	Params        map[string]string `yaml:"params,omitempty" json:"params,omitempty"`
}

type Option added in v1.1.12

type Option func(r *Reader) *Reader

func SkipModuleOutputTypeValidationReader added in v0.1.0

func SkipModuleOutputTypeValidationReader() Option

func SkipPackageValidationReader added in v1.1.8

func SkipPackageValidationReader() Option

func SkipSourceCodeReader

func SkipSourceCodeReader() Option

func WithCollectProtoDefinitions added in v0.0.21

func WithCollectProtoDefinitions(f func(protoDefinitions []*desc.FileDescriptor)) Option

func WithOverrideNetwork added in v1.3.0

func WithOverrideNetwork(network string) Option

func WithOverrideOutputModule added in v1.3.0

func WithOverrideOutputModule(outputModule string) Option

func WithParams added in v1.3.0

func WithParams(params map[string]string) Option

func WithRegistryURL added in v1.11.0

func WithRegistryURL(url string) Option

type PackageBundle added in v1.10.1

type PackageBundle struct {
	Package      *pbsubstreams.Package
	Manifest     *Manifest
	ManifestPath string
	Graph        *ModuleGraph
}

type PackageMeta

type PackageMeta struct {
	Name        string `yaml:"name,omitempty"`
	Version     string `yaml:"version,omitempty"` // Semver for package authors
	URL         string `yaml:"url,omitempty"`
	Doc         string `yaml:"doc,omitempty"`
	Description string `yaml:"description,omitempty"`
	Image       string `yaml:"image,omitempty"`
}

type Protobuf

type Protobuf struct {
	DescriptorSets []*BufImport `yaml:"descriptorSets,omitempty"`
	Files          []string     `yaml:"files,omitempty"`
	ImportPaths    []string     `yaml:"importPaths,omitempty"`
	ExcludePaths   []string     `yaml:"excludePaths,omitempty"`
}

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func MustNewReader added in v1.1.3

func MustNewReader(input string, opts ...Option) *Reader

func NewReader

func NewReader(input string, opts ...Option) (*Reader, error)

func (*Reader) IsLocalManifest added in v0.0.21

func (r *Reader) IsLocalManifest() bool

IsLocalManifest determines if reader's input to read the manifest is a local manifest file, which is determined by ensure it's not a remote package and if the file end with `.yaml` or `.yml`.

func (*Reader) IsRemotePackage added in v0.0.21

func (r *Reader) IsRemotePackage(input string) bool

IsRemotePackage determines if reader's input to read the manifest is a remote file accessible over HTTP/HTTPS, Google Cloud Storage, S3 or Azure Storage.

func (*Reader) ParseStandardPackageAndVersion added in v1.11.0

func (r *Reader) ParseStandardPackageAndVersion(input string) (packageName, version string, err error)

func (*Reader) Read

func (r *Reader) Read() (*PackageBundle, error)

type ReaderValidation added in v1.14.0

type ReaderValidation struct {
	SkipSourceCodeImportValidation bool
	SkipModuleOutputTypeValidation bool
	SkipPackageValidation          bool
}

type Sink added in v1.0.0

type Sink struct {
	Type   string      `yaml:"type,omitempty"`
	Module string      `yaml:"module,omitempty"`
	Config interface{} `yaml:"config,omitempty"`
}

type StreamOutput

type StreamOutput struct {
	// For 'map'
	Type string `yaml:"type,omitempty"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL