Documentation
¶
Index ¶
- Constants
- Variables
- func ApplyNetwork(network string, pkg *pbsubstreams.Package) error
- func ApplyParams(params map[string]string, pkg *pbsubstreams.Package) error
- func BuildMessageDescriptors(pkg *pbsubstreams.Package) (out map[string]*ModuleDescriptor, err error)
- func DescribeSinkConfigs(pkg *pbssv1.Package) (desc string, files map[string][]byte, err error)
- func ExtendedModuleHash(module *pbsubstreams.Module, hash string) string
- func ExtractNetworkEndpoint(networkFromManifest, fromFlag string, logger *zap.Logger) (string, error)
- func GenerateMermaidLiveURL(mods *pbsubstreams.Modules) string
- func IsLikelyManifestInput(in string) bool
- func NewSimpleTestModules() []*pbsubstreams.Module
- func NewTestModules() []*pbsubstreams.Module
- func ParseParams(paramsString []string) (map[string]string, error)
- func PrintMermaid(mods *pbsubstreams.Modules)
- func SortModuleNamesByGraphTopology(mods []string, g *ModuleGraph) []string
- func SplitBinaryType(in string) (typeID string, rawExtensions string)
- func TestReadManifest(t testing.T, manifestPath string) *pbsubstreams.Package
- func ValidateModules(mods *pbsubstreams.Modules) error
- type Binary
- type BlockFilter
- type BlockFilterQuery
- type BufImport
- type Input
- type Manifest
- type Module
- type ModuleDescriptor
- type ModuleGraph
- func (g *ModuleGraph) AncestorStoresOf(moduleName string) ([]*pbsubstreams.Module, error)
- func (g *ModuleGraph) AncestorsOf(moduleName string) ([]*pbsubstreams.Module, error)
- func (g *ModuleGraph) ChildrenOf(moduleName string) ([]*pbsubstreams.Module, error)
- func (g *ModuleGraph) Context(moduleName string) (parents []string, children []string, err error)
- func (g *ModuleGraph) GetSources() []string
- func (g *ModuleGraph) GroupedAncestorStores(moduleName string) ([][]*pbsubstreams.Module, error)
- func (g *ModuleGraph) HasStatefulDependencies(moduleName string) (bool, error)
- func (g *ModuleGraph) InDegree(moduleName string) int
- func (g *ModuleGraph) LeafNames() []string
- func (g *ModuleGraph) Leafs() []*pbsubstreams.Module
- func (g *ModuleGraph) MapModules() []string
- func (g *ModuleGraph) Module(moduleName string) (*pbsubstreams.Module, error)
- func (g *ModuleGraph) ModuleIndexFromName(name string) (int, bool)
- func (g *ModuleGraph) ModuleInitialBlock(moduleName string) (uint64, error)
- func (g *ModuleGraph) ModuleNameFromIndex(index int) string
- func (g *ModuleGraph) Modules() []string
- func (g *ModuleGraph) ModulesDownTo(moduleName string) ([]*pbsubstreams.Module, error)
- func (g *ModuleGraph) MustChildrenOf(moduleName string) []*pbsubstreams.Module
- func (g *ModuleGraph) MustParentsOf(moduleName string) []*pbsubstreams.Module
- func (g *ModuleGraph) OutDegree(moduleName string) int
- func (g *ModuleGraph) ParentStoresOf(moduleName string) ([]*pbsubstreams.Modules, error)
- func (g *ModuleGraph) ParentsOf(moduleName string) ([]*pbsubstreams.Module, error)
- func (graph *ModuleGraph) ResetGraphHashes()
- func (g *ModuleGraph) RootNames() []string
- func (g *ModuleGraph) Roots() []*pbsubstreams.Module
- func (g *ModuleGraph) StoresDownTo(moduleName string) ([]*pbsubstreams.Module, error)
- func (g *ModuleGraph) TopologicalSort() ([]*pbsubstreams.Module, bool)
- func (g *ModuleGraph) TopologicalSortKnownModules(known map[string]bool) ([]*pbsubstreams.Module, bool)
- type ModuleHash
- type ModuleHashes
- type ModuleMarshaler
- type NetworkParams
- type Option
- func SkipModuleOutputTypeValidationReader() Option
- func SkipPackageValidationReader() Option
- func SkipSourceCodeReader() Option
- func WithCollectProtoDefinitions(f func(protoDefinitions []*desc.FileDescriptor)) Option
- func WithOverrideNetwork(network string) Option
- func WithOverrideOutputModule(outputModule string) Option
- func WithParams(params map[string]string) Option
- func WithRegistryURL(url string) Option
- type PackageBundle
- type PackageMeta
- type Protobuf
- type Reader
- type ReaderValidation
- type Sink
- type StreamOutput
Constants ¶
const ( ModuleKindStore = "store" ModuleKindMap = "map" ModuleKindBlockIndex = "blockIndex" )
const ( OutputValueTypeInt64 = "int64" OutputValueTypeFloat64 = "float64" OutputValueTypeBigInt = "bigint" OutputValueTypeBigDecimal = "bigdecimal" // Deprecated: bigfloat value type replaced with bigdecimal OutputValueTypeBigFloat = "bigfloat" OutputValueTypeString = "string" )
const ( UpdatePolicySet = "set" UpdatePolicySetIfNotExists = "set_if_not_exists" UpdatePolicyAdd = "add" UpdatePolicyMax = "max" UpdatePolicyMin = "min" UpdatePolicyAppend = "append" UpdatePolicySetSum = "set_sum" )
const PrefixSeparator = ":"
const UNSET = math.MaxUint64
Variables ¶
var HardcodedEndpoints = map[string]string{
"mainnet": "mainnet.eth.streamingfast.io:443",
"matic": "polygon.streamingfast.io:443",
"polygon": "polygon.streamingfast.io:443",
"amoy": "amoy.substreams.pinax.network:443",
"polygon-amoy": "amoy.substreams.pinax.network:443",
"goerli": "goerli.eth.streamingfast.io:443",
"mumbai": "mumbai.streamingfast.io:443",
"bnb": "bnb.streamingfast.io:443",
"bsc": "bnb.streamingfast.io:443",
"base": "base-mainnet.streamingfast.io:443",
"sepolia": "sepolia.eth.streamingfast.io:443",
"holesky": "holesky.eth.streamingfast.io:443",
"near": "mainnet.near.streamingfast.io:443",
"near-mainnet": "mainnet.near.streamingfast.io:443",
"arbitrum": "arb-one.streamingfast.io:443",
"arb": "arb-one.streamingfast.io:443",
"arb-one": "arb-one.streamingfast.io:443",
"arbitrum-one": "arb-one.streamingfast.io:443",
"solana": "mainnet.sol.streamingfast.io:443",
"sol": "mainnet.sol.streamingfast.io:443",
"solana-accounts-mainnet": "accounts.mainnet.sol.streamingfast.io:443",
"solana-mainnet": "mainnet.sol.streamingfast.io:443",
"solana-mainnet-beta": "mainnet.sol.streamingfast.io:443",
"solana-devnet": "devnet.sol.streamingfast.io:443",
"optimism": "optimism.streamingfast.io:443",
"bitcoin": "btc-mainnet.streamingfast.io:443",
"chapel": "chapel.substreams.pinax.network:443",
"injective-mainnet": "mainnet.injective.streamingfast.io:443",
"injective-testnet": "testnet.injective.streamingfast.io:443",
"sei": "evm-mainnet.sei.streamingfast.io:443",
"sei-mainnet": "evm-mainnet.sei.streamingfast.io:443",
"sei-evm-mainnet": "evm-mainnet.sei.streamingfast.io:443",
"starknet-mainnet": "mainnet.starknet.streamingfast.io:443",
"starknet": "mainnet.starknet.streamingfast.io:443",
"starknet-testnet": "testnet.starknet.streamingfast.io:443",
"mantra-mainnet": "mainnet.mantra.streamingfast.io:443",
"mantra-testnet": "testnet.mantra.streamingfast.io:443",
"avalanche-mainnet": "avalanche-mainnet.streamingfast.io:443",
"stellar-testnet": "testnet.stellar.streamingfast.io:443",
"stellar": "mainnet.stellar.streamingfast.io:443",
"unichain": "mainnet.unichain.streamingfast.io:443",
"eos": "eos.substreams.pinax.network:443",
"jungle4": "jungle4.substreams.pinax.network:443",
"kylin": "kylin.substreams.pinax.network:443",
"wax": "wax.substreams.pinax.network:443",
"waxtest": "waxtest.substreams.pinax.network:443",
"telos": "telos.substreams.pinax.network:443",
"telostest": "telostest.substreams.pinax.network:443",
"ore": "ore.substreams.pinax.network:443",
"orestage": "orestage.substreams.pinax.network:443",
"ux": "ux.substreams.pinax.network:443",
}
TODO: replace by the blockchain-based discovery when available
var IPFSTimeout time.Duration
var IPFSURL string
var JPGHeader = []byte{0xff, 0xd8, 0xff}
var PNGHeader = []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}
var TestUseSimpleHash = false
var WebPHeader = []byte{0x52, 0x49, 0x46, 0x46}
Functions ¶
func ApplyNetwork ¶ added in v1.3.0
func ApplyNetwork(network string, pkg *pbsubstreams.Package) error
func ApplyParams ¶ added in v1.1.1
func ApplyParams(params map[string]string, pkg *pbsubstreams.Package) error
func BuildMessageDescriptors ¶ added in v1.0.2
func BuildMessageDescriptors(pkg *pbsubstreams.Package) (out map[string]*ModuleDescriptor, err error)
func DescribeSinkConfigs ¶ added in v1.1.15
DescribeSinkConfigs returns a human-readable description of the sinkconfigs. Fields that were imported from files are returned as bytes in a map
func ExtendedModuleHash ¶ added in v1.15.5
func ExtendedModuleHash(module *pbsubstreams.Module, hash string) string
ExtendedModuleHash adds the missing input store 'mode' and store 'valueType' and 'updatePolicy' they are not required for the cached data to be consistent, but they can make it fail, we don't want those failures to persist after the user fixes his substreams.yaml
func ExtractNetworkEndpoint ¶ added in v1.1.15
func GenerateMermaidLiveURL ¶ added in v1.0.1
func GenerateMermaidLiveURL(mods *pbsubstreams.Modules) string
func IsLikelyManifestInput ¶ added in v1.1.12
IsLikelyManifestInput determines if the input is likely a manifest input, which is determined by checking:
- If the input starts with remote prefix ("https://", "http://", "ipfs://", "gs://", "s3://", "az://")
- If the input ends with `.yaml`
- If the input is a directory (we check for path separator)
func NewSimpleTestModules ¶ added in v1.1.1
func NewSimpleTestModules() []*pbsubstreams.Module
NewSimpleTestModules can be used in foreign packages for their test suite
func NewTestModules ¶
func NewTestModules() []*pbsubstreams.Module
NewTestModules can be used in foreign packages for their test suite
func ParseParams ¶ added in v1.3.0
func PrintMermaid ¶
func PrintMermaid(mods *pbsubstreams.Modules)
func SortModuleNamesByGraphTopology ¶ added in v0.1.0
func SortModuleNamesByGraphTopology(mods []string, g *ModuleGraph) []string
func SplitBinaryType ¶ added in v1.7.1
SplitBinaryType splits a binary type in two components: the type ID and the raw extensions.
The format is `<id>+<extensions>`, where `<id>` is the binary type ID and `<extensions>` is a comma-separated list of runtime extensions that are type specific. The `+<extensions>` part is optional and can be omitted.
So we accept the following formats:
- wasm/rust-v1
- wasm/rust-v1+wasm-bindgen-shims
- wasm/rust-v1+wasm-bindgen-shims,other-extension=value
This method returns the ID and the raw unsplitted extensions string. The input `wasm/rust-v1+wasm-bindgen-shims,other-extension=value` would result in ("wasm/rust-v1", "wasm-bindgen-shims,other-extension=value") being returned to your
func TestReadManifest ¶ added in v0.1.0
func TestReadManifest(t testing.T, manifestPath string) *pbsubstreams.Package
func ValidateModules ¶
func ValidateModules(mods *pbsubstreams.Modules) error
ValidateModules is run both by the client _and_ the server.
Types ¶
type Binary ¶
type Binary struct {
File string `yaml:"file,omitempty"`
Type string `yaml:"type,omitempty"`
Native string `yaml:"native,omitempty"`
Content []byte `yaml:"-"`
Entrypoint string `yaml:"entrypoint,omitempty"`
ProtoPackageMapping map[string]string `yaml:"protoPackageMapping,omitempty"`
Build string `yaml:"build,omitempty"`
}
type BlockFilter ¶ added in v1.4.0
type BlockFilter struct {
Module string `yaml:"module,omitempty"`
Query BlockFilterQuery `yaml:"query,omitempty"`
}
func (*BlockFilter) IsEmpty ¶ added in v1.7.2
func (bf *BlockFilter) IsEmpty() bool
type BlockFilterQuery ¶ added in v1.6.0
type Input ¶
type Manifest ¶
type Manifest struct {
SpecVersion string `yaml:"specVersion,omitempty"` // check that it equals v0.1.0
Package PackageMeta `yaml:"package,omitempty"`
Protobuf Protobuf `yaml:"protobuf,omitempty"`
Imports mapSlice `yaml:"imports,omitempty"`
Binaries map[string]Binary `yaml:"binaries,omitempty"`
Modules []*Module `yaml:"modules,omitempty"`
Params map[string]string `yaml:"params,omitempty"`
BlockFilters map[string]string `yaml:"blockFilters,omitempty"`
Network string `yaml:"network,omitempty"`
Networks map[string]*NetworkParams `yaml:"networks,omitempty"`
Sink *Sink `yaml:"sink,omitempty"`
Graph *ModuleGraph `yaml:"-"`
Workdir string `yaml:"-"`
}
Manifest is a YAML structure used to create a Package and its list of Modules. The notion of a manifest does not live in protobuf definitions.
func LoadManifestFile ¶ added in v0.1.0
type Module ¶
type Module struct {
Name string `yaml:"name,omitempty"`
Doc string `yaml:"doc,omitempty"`
Kind string `yaml:"kind,omitempty"`
InitialBlock *uint64 `yaml:"initialBlock,omitempty"`
BlockFilter *BlockFilter `yaml:"blockFilter,omitempty"`
UpdatePolicy string `yaml:"updatePolicy,omitempty"`
ValueType string `yaml:"valueType,omitempty"`
Binary string `yaml:"binary,omitempty"`
Inputs []*Input `yaml:"inputs,omitempty"`
Output StreamOutput `yaml:"output,omitempty"`
Use string `yaml:"use,omitempty"`
}
func (*Module) ToProtoWASM ¶
func (m *Module) ToProtoWASM(codeIndex uint32) (*pbsubstreams.Module, error)
type ModuleDescriptor ¶ added in v1.0.2
type ModuleDescriptor struct {
// Either or
StoreValueType string
MapOutputType string
ProtoMessageType string
MessageDescriptor *desc.MessageDescriptor
}
type ModuleGraph ¶
func NewModuleGraph ¶
func NewModuleGraph(modules []*pbsubstreams.Module) (*ModuleGraph, error)
func (*ModuleGraph) AncestorStoresOf ¶
func (g *ModuleGraph) AncestorStoresOf(moduleName string) ([]*pbsubstreams.Module, error)
func (*ModuleGraph) AncestorsOf ¶
func (g *ModuleGraph) AncestorsOf(moduleName string) ([]*pbsubstreams.Module, error)
func (*ModuleGraph) ChildrenOf ¶ added in v1.0.2
func (g *ModuleGraph) ChildrenOf(moduleName string) ([]*pbsubstreams.Module, error)
func (*ModuleGraph) Context ¶ added in v1.0.2
func (g *ModuleGraph) Context(moduleName string) (parents []string, children []string, err error)
func (*ModuleGraph) GetSources ¶
func (g *ModuleGraph) GetSources() []string
func (*ModuleGraph) GroupedAncestorStores ¶
func (g *ModuleGraph) GroupedAncestorStores(moduleName string) ([][]*pbsubstreams.Module, error)
func (*ModuleGraph) HasStatefulDependencies ¶ added in v1.1.20
func (g *ModuleGraph) HasStatefulDependencies(moduleName string) (bool, error)
func (*ModuleGraph) InDegree ¶ added in v1.10.0
func (g *ModuleGraph) InDegree(moduleName string) int
func (*ModuleGraph) LeafNames ¶ added in v1.10.0
func (g *ModuleGraph) LeafNames() []string
func (*ModuleGraph) Leafs ¶ added in v1.10.0
func (g *ModuleGraph) Leafs() []*pbsubstreams.Module
func (*ModuleGraph) MapModules ¶ added in v1.10.0
func (g *ModuleGraph) MapModules() []string
func (*ModuleGraph) Module ¶ added in v0.0.14
func (g *ModuleGraph) Module(moduleName string) (*pbsubstreams.Module, error)
func (*ModuleGraph) ModuleIndexFromName ¶ added in v1.1.1
func (g *ModuleGraph) ModuleIndexFromName(name string) (int, bool)
func (*ModuleGraph) ModuleInitialBlock ¶
func (g *ModuleGraph) ModuleInitialBlock(moduleName string) (uint64, error)
func (*ModuleGraph) ModuleNameFromIndex ¶ added in v1.1.1
func (g *ModuleGraph) ModuleNameFromIndex(index int) string
func (*ModuleGraph) Modules ¶ added in v1.1.1
func (g *ModuleGraph) Modules() []string
func (*ModuleGraph) ModulesDownTo ¶
func (g *ModuleGraph) ModulesDownTo(moduleName string) ([]*pbsubstreams.Module, error)
func (*ModuleGraph) MustChildrenOf ¶ added in v1.0.2
func (g *ModuleGraph) MustChildrenOf(moduleName string) []*pbsubstreams.Module
func (*ModuleGraph) MustParentsOf ¶ added in v1.0.2
func (g *ModuleGraph) MustParentsOf(moduleName string) []*pbsubstreams.Module
func (*ModuleGraph) OutDegree ¶ added in v1.10.0
func (g *ModuleGraph) OutDegree(moduleName string) int
func (*ModuleGraph) ParentStoresOf ¶
func (g *ModuleGraph) ParentStoresOf(moduleName string) ([]*pbsubstreams.Modules, error)
func (*ModuleGraph) ParentsOf ¶
func (g *ModuleGraph) ParentsOf(moduleName string) ([]*pbsubstreams.Module, error)
func (*ModuleGraph) ResetGraphHashes ¶ added in v0.0.21
func (graph *ModuleGraph) ResetGraphHashes()
ResetGraphHashes is to be called when you want to force a recomputation of the module hashes.
func (*ModuleGraph) RootNames ¶ added in v1.10.0
func (g *ModuleGraph) RootNames() []string
func (*ModuleGraph) Roots ¶ added in v1.10.0
func (g *ModuleGraph) Roots() []*pbsubstreams.Module
func (*ModuleGraph) StoresDownTo ¶
func (g *ModuleGraph) StoresDownTo(moduleName string) ([]*pbsubstreams.Module, error)
func (*ModuleGraph) TopologicalSort ¶
func (g *ModuleGraph) TopologicalSort() ([]*pbsubstreams.Module, bool)
func (*ModuleGraph) TopologicalSortKnownModules ¶ added in v1.1.1
func (g *ModuleGraph) TopologicalSortKnownModules(known map[string]bool) ([]*pbsubstreams.Module, bool)
type ModuleHash ¶
type ModuleHash []byte
type ModuleHashes ¶ added in v0.0.21
type ModuleHashes struct {
// contains filtered or unexported fields
}
func NewModuleHashes ¶ added in v0.0.21
func NewModuleHashes() *ModuleHashes
func (*ModuleHashes) Get ¶ added in v0.0.21
func (m *ModuleHashes) Get(moduleName string) string
func (*ModuleHashes) HashModule ¶ added in v0.0.21
func (m *ModuleHashes) HashModule(modules *pbsubstreams.Modules, module *pbsubstreams.Module, graph *ModuleGraph) (ModuleHash, error)
type ModuleMarshaler ¶
type ModuleMarshaler []*pbsubstreams.Module
func (ModuleMarshaler) MarshalJSON ¶
func (m ModuleMarshaler) MarshalJSON() ([]byte, error)
type NetworkParams ¶ added in v1.3.0
type Option ¶ added in v1.1.12
func SkipModuleOutputTypeValidationReader ¶ added in v0.1.0
func SkipModuleOutputTypeValidationReader() Option
func SkipPackageValidationReader ¶ added in v1.1.8
func SkipPackageValidationReader() Option
func SkipSourceCodeReader ¶
func SkipSourceCodeReader() Option
func WithCollectProtoDefinitions ¶ added in v0.0.21
func WithCollectProtoDefinitions(f func(protoDefinitions []*desc.FileDescriptor)) Option
func WithOverrideNetwork ¶ added in v1.3.0
func WithOverrideOutputModule ¶ added in v1.3.0
func WithParams ¶ added in v1.3.0
func WithRegistryURL ¶ added in v1.11.0
type PackageBundle ¶ added in v1.10.1
type PackageBundle struct {
Package *pbsubstreams.Package
Manifest *Manifest
ManifestPath string
Graph *ModuleGraph
}
type PackageMeta ¶
type Reader ¶
type Reader struct {
// contains filtered or unexported fields
}
func MustNewReader ¶ added in v1.1.3
func (*Reader) IsLocalManifest ¶ added in v0.0.21
IsLocalManifest determines if reader's input to read the manifest is a local manifest file, which is determined by ensure it's not a remote package and if the file end with `.yaml` or `.yml`.
func (*Reader) IsRemotePackage ¶ added in v0.0.21
IsRemotePackage determines if reader's input to read the manifest is a remote file accessible over HTTP/HTTPS, Google Cloud Storage, S3 or Azure Storage.
func (*Reader) ParseStandardPackageAndVersion ¶ added in v1.11.0
func (*Reader) Read ¶
func (r *Reader) Read() (*PackageBundle, error)
type ReaderValidation ¶ added in v1.14.0
type StreamOutput ¶
type StreamOutput struct {
// For 'map'
Type string `yaml:"type,omitempty"`
}