manifest

package
v1.17.11 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 21, 2026 License: Apache-2.0 Imports: 56 Imported by: 18

Documentation

Index

Constants

View Source
const (
	ModuleKindStore      = "store"
	ModuleKindMap        = "map"
	ModuleKindBlockIndex = "blockIndex"
)
View Source
const (
	OutputValueTypeInt64      = "int64"
	OutputValueTypeFloat64    = "float64"
	OutputValueTypeBigInt     = "bigint"
	OutputValueTypeBigDecimal = "bigdecimal"

	// Deprecated: bigfloat value type replaced with bigdecimal
	OutputValueTypeBigFloat = "bigfloat"
	OutputValueTypeString   = "string"
)
View Source
const (
	UpdatePolicySet            = "set"
	UpdatePolicySetIfNotExists = "set_if_not_exists"
	UpdatePolicyAdd            = "add"
	UpdatePolicyMax            = "max"
	UpdatePolicyMin            = "min"
	UpdatePolicyAppend         = "append"
	UpdatePolicySetSum         = "set_sum"
)
View Source
const PrefixSeparator = ":"
View Source
const UNSET = math.MaxUint64

Variables

View Source
var IPFSTimeout time.Duration
View Source
var IPFSURL string
View Source
var JPGHeader = []byte{0xff, 0xd8, 0xff}
View Source
var PNGHeader = []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}
View Source
var TestUseSimpleHash = false
View Source
var WebPHeader = []byte{0x52, 0x49, 0x46, 0x46}

Functions

func ApplyNetwork added in v1.3.0

func ApplyNetwork(network string, pkg *pbsubstreams.Package) error

func ApplyParams added in v1.1.1

func ApplyParams(params map[string]string, pkg *pbsubstreams.Package) error

func BuildMessageDescriptors added in v1.0.2

func BuildMessageDescriptors(pkg *pbsubstreams.Package) (out map[string]*ModuleDescriptor, err error)

func DescribeSinkConfigs added in v1.1.15

func DescribeSinkConfigs(pkg *pbssv1.Package) (desc string, files map[string][]byte, err error)

DescribeSinkConfigs returns a human-readable description of the sinkconfigs. Fields that were imported from files are returned as bytes in a map

func ExtendedModuleHash added in v1.15.5

func ExtendedModuleHash(module *pbsubstreams.Module, hash string) string

ExtendedModuleHash adds the missing input store 'mode' and store 'valueType' and 'updatePolicy' they are not required for the cached data to be consistent, but they can make it fail, we don't want those failures to persist after the user fixes his substreams.yaml

func ExtractNetworkEndpoint added in v1.1.15

func ExtractNetworkEndpoint(networkFromManifest, fromFlag string, logger *zap.Logger) (string, error)

func GenerateMermaidLiveURL added in v1.0.1

func GenerateMermaidLiveURL(mods *pbsubstreams.Modules) string

func IsLikelyManifestInput added in v1.1.12

func IsLikelyManifestInput(in string) bool

IsLikelyManifestInput determines if the input is likely a manifest input, which is determined by checking:

  • If the input starts with remote prefix ("https://", "http://", "ipfs://", "gs://", "s3://", "az://")
  • If the input is reading from standard input (i.e., `-`)
  • If the input ends with `.yaml`
  • If the input is a directory (we check for path separator)

func NewSimpleTestModules added in v1.1.1

func NewSimpleTestModules() []*pbsubstreams.Module

NewSimpleTestModules can be used in foreign packages for their test suite

func NewTestModules

func NewTestModules() []*pbsubstreams.Module

NewTestModules can be used in foreign packages for their test suite

func ParseParams added in v1.3.0

func ParseParams(paramsString []string) (map[string]string, error)

func ParseShortPackageIdentifier added in v1.16.3

func ParseShortPackageIdentifier(input string) (packageName, version string, likelyShortcut bool, validationError error)

ParseShortPackageIdentifier parses a package identifier in the format "package@version" and returns the package name and version. If the version is not specified, it defaults to "latest". Once deemed valid, we return validation errors as a multierror.

The `likelyShortcut` boolean indicates if the input is likely a shortcut package identifier which is when a @ is found in the input.

The parser accepts "latest" as a valid version, and it will be returned as such even if it's not a semver version.

func PrintMermaid

func PrintMermaid(mods *pbsubstreams.Modules)

func SortModuleNamesByGraphTopology added in v0.1.0

func SortModuleNamesByGraphTopology(mods []string, g *ModuleGraph) []string

func SplitBinaryType added in v1.7.1

func SplitBinaryType(in string) (typeID string, rawExtensions string)

SplitBinaryType splits a binary type in two components: the type ID and the raw extensions.

The format is `<id>+<extensions>`, where `<id>` is the binary type ID and `<extensions>` is a comma-separated list of runtime extensions that are type specific. The `+<extensions>` part is optional and can be omitted.

So we accept the following formats:

  • wasm/rust-v1
  • wasm/rust-v1+wasm-bindgen-shims
  • wasm/rust-v1+wasm-bindgen-shims,other-extension=value

This method returns the ID and the raw unsplitted extensions string. The input `wasm/rust-v1+wasm-bindgen-shims,other-extension=value` would result in ("wasm/rust-v1", "wasm-bindgen-shims,other-extension=value") being returned to your

func TestReadManifest added in v0.1.0

func TestReadManifest(t testing.T, manifestPath string) *pbsubstreams.Package

func ValidateModules

func ValidateModules(mods *pbsubstreams.Modules) error

ValidateModules is run both by the client _and_ the server.

func ValidateProtoOutputTypes added in v1.15.9

func ValidateProtoOutputTypes(pkg *pbsubstreams.Package) error

ValidateProtoOutputTypes validates that all modules with proto:... output types have corresponding protobuf definitions in the package.

Types

type Binary

type Binary struct {
	File                string            `yaml:"file,omitempty"`
	Type                string            `yaml:"type,omitempty"`
	Native              string            `yaml:"native,omitempty"`
	Content             []byte            `yaml:"-"`
	Entrypoint          string            `yaml:"entrypoint,omitempty"`
	ProtoPackageMapping map[string]string `yaml:"protoPackageMapping,omitempty"`
	Build               string            `yaml:"build,omitempty"`
}

type BlockFilter added in v1.4.0

type BlockFilter struct {
	Module string           `yaml:"module,omitempty"`
	Query  BlockFilterQuery `yaml:"query,omitempty"`
}

func (*BlockFilter) IsEmpty added in v1.7.2

func (bf *BlockFilter) IsEmpty() bool

type BlockFilterQuery added in v1.6.0

type BlockFilterQuery struct {
	String string `yaml:"string,omitempty"`
	Params bool   `yaml:"params,omitempty"`
}

type BufImport deprecated added in v1.6.1

type BufImport = DescriptorSetEntry

Deprecated: BufImport type replaced with DescriptorSetEntry This type alias is maintained for backward compatibility

type DescriptorCache added in v1.17.0

type DescriptorCache struct {
	// contains filtered or unexported fields
}

type DescriptorSetEntry added in v1.17.0

type DescriptorSetEntry struct {
	LocalPath string   `yaml:"localPath,omitempty"`
	Module    string   `yaml:"module"`
	Version   string   `yaml:"version,omitempty"`
	Symbols   []string `yaml:"symbols,omitempty"`
}

func (*DescriptorSetEntry) UnmarshalYAML added in v1.17.0

func (d *DescriptorSetEntry) UnmarshalYAML(n *yaml.Node) error

UnmarshalYAML implements custom YAML unmarshaling for DescriptorSetEntry Supports two formats: 1. Full path: buf.build/org/pkg 2. With version: buf.build/org/pkg@version

type Input

type Input struct {
	Source            string `yaml:"source,omitempty"`
	Store             string `yaml:"store,omitempty"`
	Map               string `yaml:"map,omitempty"`
	Params            string `yaml:"params,omitempty"`
	FoundationalStore string `yaml:"foundational-store,omitempty"`

	Mode string `yaml:"mode,omitempty"`
}

func (*Input) IsFoundationalStore added in v1.16.6

func (i *Input) IsFoundationalStore() bool

func (*Input) IsMap added in v0.1.0

func (i *Input) IsMap() bool

func (*Input) IsParams added in v1.0.0

func (i *Input) IsParams() bool

func (*Input) IsSource added in v0.1.0

func (i *Input) IsSource() bool

func (*Input) IsStore added in v0.1.0

func (i *Input) IsStore() bool

type Manifest

type Manifest struct {
	SpecVersion  string                    `yaml:"specVersion,omitempty"` // check that it equals v0.1.0
	Package      PackageMeta               `yaml:"package,omitempty"`
	Protobuf     Protobuf                  `yaml:"protobuf,omitempty"`
	Imports      mapSlice                  `yaml:"imports,omitempty"`
	Binaries     map[string]Binary         `yaml:"binaries,omitempty"`
	Modules      []*Module                 `yaml:"modules,omitempty"`
	Params       map[string]string         `yaml:"params,omitempty"`
	BlockFilters map[string]string         `yaml:"blockFilters,omitempty"`
	Network      string                    `yaml:"network,omitempty"`
	Networks     map[string]*NetworkParams `yaml:"networks,omitempty"`
	Sink         *Sink                     `yaml:"sink,omitempty"`

	Graph   *ModuleGraph `yaml:"-"`
	Workdir string       `yaml:"-"`
}

Manifest is a YAML structure used to create a Package and its list of Modules. The notion of a manifest does not live in protobuf definitions.

func LoadManifestFile added in v0.1.0

func LoadManifestFile(inputPath, workingDir string) (*Manifest, error)

type Module

type Module struct {
	Name         string       `yaml:"name,omitempty"`
	Doc          string       `yaml:"doc,omitempty"`
	Kind         string       `yaml:"kind,omitempty"`
	InitialBlock *uint64      `yaml:"initialBlock,omitempty"`
	BlockFilter  *BlockFilter `yaml:"blockFilter,omitempty"`

	UpdatePolicy string `yaml:"updatePolicy,omitempty"`
	ValueType    string `yaml:"valueType,omitempty"`
	Binary       string `yaml:"binary,omitempty"`

	Inputs []*Input     `yaml:"inputs,omitempty"`
	Output StreamOutput `yaml:"output,omitempty"`
	Use    string       `yaml:"use,omitempty"`
}

func (*Module) String

func (m *Module) String() string

func (*Module) ToProtoWASM

func (m *Module) ToProtoWASM(codeIndex uint32) (*pbsubstreams.Module, error)

type ModuleDescriptor added in v1.0.2

type ModuleDescriptor struct {
	// Either or
	StoreValueType string
	MapOutputType  string

	ProtoMessageType  string
	MessageDescriptor *desc.MessageDescriptor
}

type ModuleGraph

type ModuleGraph struct {
	*graph.Mutable
	// contains filtered or unexported fields
}

func ApplyPackageTransformations added in v1.17.0

func ApplyPackageTransformations(pkg *pbsubstreams.Package, skipPackageValidation bool, overrideNetwork string, overrideOutputModule string, overrideParams map[string]string) (*ModuleGraph, error)

ApplyPackageTransformations applies validation, network overrides, params, and computes initial blocks on a package. It returns the module graph for the package.

func NewModuleGraph

func NewModuleGraph(modules []*pbsubstreams.Module) (*ModuleGraph, error)

func (*ModuleGraph) AncestorStoresOf

func (g *ModuleGraph) AncestorStoresOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) AncestorsOf

func (g *ModuleGraph) AncestorsOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) ChildrenOf added in v1.0.2

func (g *ModuleGraph) ChildrenOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) Context added in v1.0.2

func (g *ModuleGraph) Context(moduleName string) (parents []string, children []string, err error)

func (*ModuleGraph) GetSources

func (g *ModuleGraph) GetSources() []string

func (*ModuleGraph) GroupedAncestorStores

func (g *ModuleGraph) GroupedAncestorStores(moduleName string) ([][]*pbsubstreams.Module, error)

func (*ModuleGraph) HasStatefulDependencies added in v1.1.20

func (g *ModuleGraph) HasStatefulDependencies(moduleName string) (bool, error)

func (*ModuleGraph) InDegree added in v1.10.0

func (g *ModuleGraph) InDegree(moduleName string) int

func (*ModuleGraph) LeafNames added in v1.10.0

func (g *ModuleGraph) LeafNames() []string

func (*ModuleGraph) Leafs added in v1.10.0

func (g *ModuleGraph) Leafs() []*pbsubstreams.Module

func (*ModuleGraph) MapModules added in v1.10.0

func (g *ModuleGraph) MapModules() []string

func (*ModuleGraph) Module added in v0.0.14

func (g *ModuleGraph) Module(moduleName string) (*pbsubstreams.Module, error)

func (*ModuleGraph) ModuleIndexFromName added in v1.1.1

func (g *ModuleGraph) ModuleIndexFromName(name string) (int, bool)

func (*ModuleGraph) ModuleInitialBlock

func (g *ModuleGraph) ModuleInitialBlock(moduleName string) (uint64, error)

func (*ModuleGraph) ModuleNameFromIndex added in v1.1.1

func (g *ModuleGraph) ModuleNameFromIndex(index int) string

func (*ModuleGraph) Modules added in v1.1.1

func (g *ModuleGraph) Modules() []string

func (*ModuleGraph) ModulesDownTo

func (g *ModuleGraph) ModulesDownTo(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) MustChildrenOf added in v1.0.2

func (g *ModuleGraph) MustChildrenOf(moduleName string) []*pbsubstreams.Module

func (*ModuleGraph) MustParentsOf added in v1.0.2

func (g *ModuleGraph) MustParentsOf(moduleName string) []*pbsubstreams.Module

func (*ModuleGraph) OutDegree added in v1.10.0

func (g *ModuleGraph) OutDegree(moduleName string) int

func (*ModuleGraph) ParentStoresOf

func (g *ModuleGraph) ParentStoresOf(moduleName string) ([]*pbsubstreams.Modules, error)

func (*ModuleGraph) ParentsOf

func (g *ModuleGraph) ParentsOf(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) ResetGraphHashes added in v0.0.21

func (graph *ModuleGraph) ResetGraphHashes()

ResetGraphHashes is to be called when you want to force a recomputation of the module hashes.

func (*ModuleGraph) RootNames added in v1.10.0

func (g *ModuleGraph) RootNames() []string

func (*ModuleGraph) Roots added in v1.10.0

func (g *ModuleGraph) Roots() []*pbsubstreams.Module

func (*ModuleGraph) StoresDownTo

func (g *ModuleGraph) StoresDownTo(moduleName string) ([]*pbsubstreams.Module, error)

func (*ModuleGraph) TopologicalSort

func (g *ModuleGraph) TopologicalSort() ([]*pbsubstreams.Module, bool)

func (*ModuleGraph) TopologicalSortKnownModules added in v1.1.1

func (g *ModuleGraph) TopologicalSortKnownModules(known map[string]bool) ([]*pbsubstreams.Module, bool)

type ModuleHash

type ModuleHash []byte

type ModuleHashes added in v0.0.21

type ModuleHashes struct {
	// contains filtered or unexported fields
}

func NewModuleHashes added in v0.0.21

func NewModuleHashes() *ModuleHashes

func (*ModuleHashes) Get added in v0.0.21

func (m *ModuleHashes) Get(moduleName string) string

func (*ModuleHashes) HashModule added in v0.0.21

func (m *ModuleHashes) HashModule(modules *pbsubstreams.Modules, module *pbsubstreams.Module, graph *ModuleGraph) (ModuleHash, error)

func (*ModuleHashes) Iter added in v0.0.21

func (m *ModuleHashes) Iter(cb func(hash, name string) error) error

type ModuleMarshaler

type ModuleMarshaler []*pbsubstreams.Module

func (ModuleMarshaler) MarshalJSON

func (m ModuleMarshaler) MarshalJSON() ([]byte, error)

type NetworkParams added in v1.3.0

type NetworkParams struct {
	InitialBlocks map[string]uint64 `yaml:"initialBlock,omitempty" json:"initialBlock,omitempty"`
	Params        map[string]string `yaml:"params,omitempty" json:"params,omitempty"`
}

type Option added in v1.1.12

type Option func(r *Reader) *Reader

func SkipModuleOutputTypeValidationReader added in v0.1.0

func SkipModuleOutputTypeValidationReader() Option

func SkipPackageValidationReader added in v1.1.8

func SkipPackageValidationReader() Option

func SkipSourceCodeReader

func SkipSourceCodeReader() Option

func WithCollectProtoDefinitions added in v0.0.21

func WithCollectProtoDefinitions(f func(protoDefinitions []*desc.FileDescriptor)) Option

func WithOverrideNetwork added in v1.3.0

func WithOverrideNetwork(network string) Option

func WithOverrideOutputModule added in v1.3.0

func WithOverrideOutputModule(outputModule string) Option

func WithParams added in v1.3.0

func WithParams(params map[string]string) Option

func WithProtoDescriptorSet added in v1.15.9

func WithProtoDescriptorSet(protoDescriptorSet string) Option

func WithProtoPath added in v1.15.9

func WithProtoPath(protoPath string) Option

func WithRegistryURL added in v1.11.0

func WithRegistryURL(url string) Option

type PackageBundle added in v1.10.1

type PackageBundle struct {
	Package      *pbsubstreams.Package
	Manifest     *Manifest
	ManifestPath string
	Graph        *ModuleGraph
}

type PackageMeta

type PackageMeta struct {
	Name        string `yaml:"name,omitempty"`
	Version     string `yaml:"version,omitempty"` // Semver for package authors
	URL         string `yaml:"url,omitempty"`
	Doc         string `yaml:"doc,omitempty"`
	Description string `yaml:"description,omitempty"`
	Image       string `yaml:"image,omitempty"`
}

type Protobuf

type Protobuf struct {
	DescriptorSets []*DescriptorSetEntry `yaml:"descriptorSets,omitempty"`
	Files          []string              `yaml:"files,omitempty"`
	ImportPaths    []string              `yaml:"importPaths,omitempty"`
	ExcludePaths   []string              `yaml:"excludePaths,omitempty"`
}

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func MustNewReader added in v1.1.3

func MustNewReader(input string, opts ...Option) *Reader

func NewReader

func NewReader(input string, opts ...Option) (*Reader, error)

func (*Reader) IsLocalManifest added in v0.0.21

func (r *Reader) IsLocalManifest() bool

IsLocalManifest determines if reader's input to read the manifest is a local manifest file, which is determined by ensure it's not a remote package and if the file end with `.yaml` or `.yml`.

func (*Reader) IsRemotePackage added in v0.0.21

func (r *Reader) IsRemotePackage(input string) bool

IsRemotePackage determines if reader's input to read the manifest is a remote file accessible over HTTP/HTTPS, Google Cloud Storage, S3 or Azure Storage.

func (*Reader) ParseStandardPackageAndVersion deprecated added in v1.11.0

func (r *Reader) ParseStandardPackageAndVersion(input string) (packageName, version string, err error)

Deprecated: ParseStandardPackageAndVersion is deprecated, use manifest.ParseShortPackageIdentifier instead.

func (*Reader) Read

func (r *Reader) Read() (*PackageBundle, error)

type ReaderValidation added in v1.14.0

type ReaderValidation struct {
	SkipSourceCodeImportValidation bool
	SkipModuleOutputTypeValidation bool
	SkipPackageValidation          bool
}

type Sink added in v1.0.0

type Sink struct {
	Type   string      `yaml:"type,omitempty"`
	Module string      `yaml:"module,omitempty"`
	Config interface{} `yaml:"config,omitempty"`
}

type StreamOutput

type StreamOutput struct {
	// For 'map'
	Type string `yaml:"type,omitempty"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL