Documentation
¶
Index ¶
- Variables
- func CalculateCommp(t *testing.T, content []byte, targetPieceSize uint64) string
- func CompareDirectories(t *testing.T, dir1, dir2 string)
- func Download(ctx context.Context, url string, nThreads int) ([]byte, error)
- func GetAllPieceCIDs(content string) []string
- func GetFirstCID(content string) string
- func SetVersionJSON(versionJSON []byte) error
- func SetupErrorHandler()
- func SetupHelpPager()
- func WaitForServerReady(ctx context.Context, url string) error
- type MockAdmin
- type MockDataPrep
- func (m *MockDataPrep) AddOutputStorageHandler(ctx context.Context, db *gorm.DB, id string, output string) (*model.Preparation, error)
- func (m *MockDataPrep) AddPieceHandler(ctx context.Context, db *gorm.DB, id string, request dataprep.AddPieceRequest) (*model.Car, error)
- func (m *MockDataPrep) AddSourceStorageHandler(ctx context.Context, db *gorm.DB, id string, source string) (*model.Preparation, error)
- func (m *MockDataPrep) CreatePreparationHandler(ctx context.Context, db *gorm.DB, request dataprep.CreateRequest) (*model.Preparation, error)
- func (m *MockDataPrep) ExploreHandler(ctx context.Context, db *gorm.DB, id string, name string, path string) (*dataprep.ExploreResult, error)
- func (m *MockDataPrep) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Preparation, error)
- func (m *MockDataPrep) ListPiecesHandler(ctx context.Context, db *gorm.DB, id string) ([]dataprep.PieceList, error)
- func (m *MockDataPrep) ListSchedulesHandler(ctx context.Context, db *gorm.DB, id string) ([]model.Schedule, error)
- func (m *MockDataPrep) RemoveOutputStorageHandler(ctx context.Context, db *gorm.DB, id string, output string) (*model.Preparation, error)
- type MockDeal
- type MockFile
- func (m *MockFile) GetFileDealsHandler(ctx context.Context, db *gorm.DB, id string) ([]model.Deal, error)
- func (m *MockFile) GetFileHandler(ctx context.Context, db *gorm.DB, id uint64) (*model.File, error)
- func (m *MockFile) PrepareToPackFileHandler(ctx context.Context, db *gorm.DB, fileID uint64) (int64, error)
- func (m *MockFile) PushFileHandler(ctx context.Context, db *gorm.DB, preparation string, source string, ...) (*model.File, error)
- type MockJob
- func (m *MockJob) GetStatusHandler(ctx context.Context, db *gorm.DB, id string) ([]job.SourceStatus, error)
- func (m *MockJob) PackHandler(ctx context.Context, db *gorm.DB, jobID int64) (*model.Car, error)
- func (m *MockJob) PauseDagGenHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)
- func (m *MockJob) PausePackHandler(ctx context.Context, db *gorm.DB, id string, name string, jobID int64) ([]model.Job, error)
- func (m *MockJob) PauseScanHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)
- func (m *MockJob) PrepareToPackSourceHandler(ctx context.Context, db *gorm.DB, id string, name string) error
- func (m *MockJob) StartDagGenHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)
- func (m *MockJob) StartPackHandler(ctx context.Context, db *gorm.DB, id string, name string, jobID int64) ([]model.Job, error)
- func (m *MockJob) StartScanHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)
- type MockSchedule
- func (m *MockSchedule) CreateHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, ...) (*model.Schedule, error)
- func (m *MockSchedule) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Schedule, error)
- func (m *MockSchedule) PauseHandler(ctx context.Context, db *gorm.DB, scheduleID uint32) (*model.Schedule, error)
- func (m *MockSchedule) ResumeHandler(ctx context.Context, db *gorm.DB, scheduleID uint32) (*model.Schedule, error)
- type MockStorage
- func (m *MockStorage) CreateStorageHandler(ctx context.Context, db *gorm.DB, storageType string, ...) (*model.Storage, error)
- func (m *MockStorage) ExploreHandler(ctx context.Context, db *gorm.DB, name string, path string) ([]storage.DirEntry, error)
- func (m *MockStorage) ListStoragesHandler(ctx context.Context, db *gorm.DB) ([]model.Storage, error)
- func (m *MockStorage) RemoveHandler(ctx context.Context, db *gorm.DB, name string) error
- func (m *MockStorage) UpdateStorageHandler(ctx context.Context, db *gorm.DB, name string, config map[string]string) (*model.Storage, error)
- type MockWallet
- func (m *MockWallet) AttachHandler(ctx context.Context, db *gorm.DB, preparation string, wallet string) (*model.Preparation, error)
- func (m *MockWallet) DetachHandler(ctx context.Context, db *gorm.DB, preparation string, wallet string) (*model.Preparation, error)
- func (m *MockWallet) ImportHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, ...) (*model.Wallet, error)
- func (m *MockWallet) ListAttachedHandler(ctx context.Context, db *gorm.DB, preparation string) ([]model.Wallet, error)
- func (m *MockWallet) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Wallet, error)
- func (m *MockWallet) RemoveHandler(ctx context.Context, db *gorm.DB, address string) error
- type Runner
- type RunnerMode
Constants ¶
This section is empty.
Variables ¶
View Source
var App = &cli.App{ Name: "singularity", Usage: "A tool for large-scale clients with PB-scale data onboarding to Filecoin network", Description: `Database Backend Support: Singularity supports multiple database backend: sqlite3, postgres, mysql5.7+ Use '--database-connection-string' or $DATABASE_CONNECTION_STRING to specify the database connection string. Example for postgres - postgres://user:pass@example.com:5432/dbname Example for mysql - mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true Example for sqlite3 - sqlite:/absolute/path/to/database.db or - sqlite:relative/path/to/database.db Network Support: Default settings in Singularity are for Mainnet. You may set below environment values for other network: For Calibration network: * Set LOTUS_API to https://api.calibration.node.glif.io/rpc/v1 * Set MARKET_DEAL_URL to https://marketdeals-calibration.s3.amazonaws.com/StateMarketDeals.json.zst * Set LOTUS_TEST to 1 For all other networks: * Set LOTUS_API to your network's Lotus API endpoint * Set MARKET_DEAL_URL to empty string * Set LOTUS_TEST to 0 or 1 based on whether the network address starts with 'f' or 't' Switching between different networks with the same database instance is not recommended.`, EnableBashCompletion: true, Flags: []cli.Flag{ &cli.StringFlag{ Name: "database-connection-string", Usage: "Connection string to the database", DefaultText: "sqlite:" + "./singularity.db", Value: "sqlite:" + "./singularity.db", EnvVars: []string{"DATABASE_CONNECTION_STRING"}, }, &cli.BoolFlag{ Name: "json", Usage: "Enable JSON output", Value: false, }, &cli.BoolFlag{ Name: "verbose", Usage: "Enable verbose output. This will print more columns for the result as well as full error trace", Value: false, }, &cli.StringFlag{ Name: "lotus-api", Category: "Lotus", Usage: "Lotus RPC API endpoint", Value: "https://api.node.glif.io/rpc/v1", EnvVars: []string{"LOTUS_API"}, }, &cli.StringFlag{ Name: "lotus-token", Category: "Lotus", Usage: "Lotus RPC API token", Value: "", EnvVars: []string{"LOTUS_TOKEN"}, }, &cli.BoolFlag{ Name: "lotus-test", Category: "Lotus", Usage: "Whether the runtime environment is using Testnet.", EnvVars: []string{"LOTUS_TEST"}, Action: func(c *cli.Context, testnet bool) error { if testnet { address.CurrentNetwork = address.Testnet logger.Infow("Current network is set to Testnet") } return nil }, }, }, Commands: []*cli.Command{ ez.PrepCmd, VersionCmd, { Name: "admin", Usage: "Admin commands", Category: "Operations", Subcommands: []*cli.Command{ admin.InitCmd, admin.ResetCmd, admin.MigrateDatasetCmd, admin.MigrateScheduleCmd, }, }, DownloadCmd, tool.ExtractCarCmd, { Name: "deal", Usage: "Replication / Deal making management", Category: "Operations", Subcommands: []*cli.Command{ { Name: "schedule", Usage: "Schedule deals", Subcommands: []*cli.Command{ schedule.CreateCmd, schedule.ListCmd, schedule.PauseCmd, schedule.ResumeCmd, }, }, deal.SendManualCmd, deal.ListCmd, }, }, { Name: "run", Category: "Daemons", Usage: "run different singularity components", Subcommands: []*cli.Command{ run.APICmd, run.DatasetWorkerCmd, run.ContentProviderCmd, run.DealTrackerCmd, run.DealPusherCmd, }, }, { Name: "wallet", Category: "Operations", Usage: "Wallet management", Subcommands: []*cli.Command{ wallet.ImportCmd, wallet.ListCmd, wallet.RemoveCmd, }, }, { Name: "storage", Category: "Operations", Usage: "Create and manage storage system connections", Subcommands: []*cli.Command{ storage.CreateCmd, storage.ExploreCmd, storage.ListCmd, storage.RemoveCmd, storage.UpdateCmd, }, }, { Name: "prep", Category: "Operations", Usage: "Create and manage dataset preparations", Subcommands: []*cli.Command{ dataprep.CreateCmd, dataprep.ListCmd, dataprep.StatusCmd, dataprep.AttachSourceCmd, dataprep.AttachOutputCmd, dataprep.DetachOutputCmd, dataprep.StartScanCmd, dataprep.PauseScanCmd, dataprep.StartPackCmd, dataprep.PausePackCmd, dataprep.StartDagGenCmd, dataprep.PauseDagGenCmd, dataprep.ListPiecesCmd, dataprep.AddPieceCmd, dataprep.ExploreCmd, dataprep.AttachWalletCmd, dataprep.ListWalletsCmd, dataprep.DetachWalletCmd, }, }, }, }
View Source
var DownloadCmd = &cli.Command{ Name: "download", Usage: "Download a CAR file from the metadata API", Category: "Utility", Before: cliutil.CheckNArgs, ArgsUsage: "<piece_cid>", Flags: func() []cli.Flag { flags := []cli.Flag{ &cli.StringFlag{ Name: "api", Usage: "URL of the metadata API", Value: "http://127.0.0.1:7777", Category: "General Config", }, &cli.StringFlag{ Name: "out-dir", Usage: "Directory to write CAR files to", Value: ".", Category: "General Config", }, &cli.IntFlag{ Name: "concurrency", Usage: "Number of concurrent downloads", Value: 10, Category: "General Config", }, &cli.BoolFlag{ Name: "quiet", Usage: "Suppress all output", Category: "General Config", Value: false, }, } keys := make(map[string]struct{}) for _, backend := range storagesystem.Backends { var providers []string for _, providerOptions := range backend.ProviderOptions { providers = append(providers, providerOptions.Provider) for _, option := range providerOptions.Options { flag := option.ToCLIFlag(backend.Prefix+"-", false, backend.Description) if _, ok := keys[flag.Names()[0]]; ok { continue } keys[flag.Names()[0]] = struct{}{} flags = append(flags, flag) } } if len(providers) > 1 { providerFlag := &cli.StringFlag{ Name: backend.Prefix + "-provider", Usage: strings.Join(providers, " | "), EnvVars: []string{ strings.ToUpper(backend.Prefix) + "_PROVIDER", }, Category: backend.Description, Value: providers[0], } flags = append(flags, providerFlag) } } return flags }(), Action: func(c *cli.Context) error { api := c.String("api") outDir := c.String("out-dir") concurrency := c.Int("concurrency") piece := c.Args().First() config := map[string]string{} for _, key := range c.LocalFlagNames() { if c.IsSet(key) { if slices.Contains([]string{"api", "out-dir", "concurrency", "quiet"}, key) { continue } value := c.String(key) config[key] = value } } err := handler.DownloadHandler(c, piece, api, config, outDir, concurrency) if err == nil { log.Logger("Download").Info("Download complete") return nil } return errors.WithStack(err) }, }
View Source
var Version string
View Source
var VersionCmd = &cli.Command{ Name: "version", Usage: "Print version information", Aliases: []string{"v"}, Action: func(context *cli.Context) error { buildInfo, ok := debug.ReadBuildInfo() if !ok { fmt.Println("unknown version") } version := buildInfo.Main.Version if version == "(devel)" || version == "" { version = Version } var revision string var modified string for _, setting := range buildInfo.Settings { switch setting.Key { case "vcs.revision": revision = setting.Value[:7] case "vcs.modified": modified = setting.Value } } if revision == "" { revision = "-unknown" } else { revision = "-" + revision } switch modified { case "true": modified = "-dirty" case "false": modified = "" case "": modified = "-unknown" default: modified = "-" + modified } v := fmt.Sprintf("singularity %s%s%s\n", version, revision, modified) _, err := context.App.Writer.Write([]byte(v)) return errors.WithStack(err) }, }
Functions ¶
func CalculateCommp ¶ added in v0.4.0
func CompareDirectories ¶ added in v0.4.0
func GetAllPieceCIDs ¶ added in v0.4.0
func GetFirstCID ¶ added in v0.4.0
func SetVersionJSON ¶ added in v0.4.0
func SetupErrorHandler ¶ added in v0.4.0
func SetupErrorHandler()
func SetupHelpPager ¶ added in v0.3.0
func SetupHelpPager()
Types ¶
type MockAdmin ¶ added in v0.4.0
func (*MockAdmin) InitHandler ¶ added in v0.4.0
type MockDataPrep ¶ added in v0.4.0
func (*MockDataPrep) AddOutputStorageHandler ¶ added in v0.4.0
func (m *MockDataPrep) AddOutputStorageHandler(ctx context.Context, db *gorm.DB, id string, output string) (*model.Preparation, error)
func (*MockDataPrep) AddPieceHandler ¶ added in v0.4.0
func (*MockDataPrep) AddSourceStorageHandler ¶ added in v0.4.0
func (m *MockDataPrep) AddSourceStorageHandler(ctx context.Context, db *gorm.DB, id string, source string) (*model.Preparation, error)
func (*MockDataPrep) CreatePreparationHandler ¶ added in v0.4.0
func (m *MockDataPrep) CreatePreparationHandler(ctx context.Context, db *gorm.DB, request dataprep.CreateRequest) (*model.Preparation, error)
func (*MockDataPrep) ExploreHandler ¶ added in v0.4.0
func (*MockDataPrep) ListHandler ¶ added in v0.4.0
func (m *MockDataPrep) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Preparation, error)
func (*MockDataPrep) ListPiecesHandler ¶ added in v0.4.0
func (*MockDataPrep) ListSchedulesHandler ¶ added in v0.4.0
func (*MockDataPrep) RemoveOutputStorageHandler ¶ added in v0.4.0
func (m *MockDataPrep) RemoveOutputStorageHandler(ctx context.Context, db *gorm.DB, id string, output string) (*model.Preparation, error)
type MockDeal ¶ added in v0.4.0
func (*MockDeal) ListHandler ¶ added in v0.4.0
type MockFile ¶ added in v0.4.0
func (*MockFile) GetFileDealsHandler ¶ added in v0.4.0
func (*MockFile) GetFileHandler ¶ added in v0.4.0
func (*MockFile) PrepareToPackFileHandler ¶ added in v0.4.0
type MockJob ¶ added in v0.4.0
func (*MockJob) GetStatusHandler ¶ added in v0.4.0
func (*MockJob) PackHandler ¶ added in v0.4.0
func (*MockJob) PauseDagGenHandler ¶ added in v0.4.0
func (*MockJob) PausePackHandler ¶ added in v0.4.0
func (*MockJob) PauseScanHandler ¶ added in v0.4.0
func (*MockJob) PrepareToPackSourceHandler ¶ added in v0.4.0
func (*MockJob) StartDagGenHandler ¶ added in v0.4.0
func (*MockJob) StartPackHandler ¶ added in v0.4.0
type MockSchedule ¶ added in v0.4.0
func (*MockSchedule) CreateHandler ¶ added in v0.4.0
func (m *MockSchedule) CreateHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request schedule.CreateRequest) (*model.Schedule, error)
func (*MockSchedule) ListHandler ¶ added in v0.4.0
func (*MockSchedule) PauseHandler ¶ added in v0.4.0
type MockStorage ¶ added in v0.4.0
func (*MockStorage) CreateStorageHandler ¶ added in v0.4.0
func (*MockStorage) ExploreHandler ¶ added in v0.4.0
func (*MockStorage) ListStoragesHandler ¶ added in v0.4.0
func (*MockStorage) RemoveHandler ¶ added in v0.4.0
type MockWallet ¶ added in v0.4.0
func (*MockWallet) AttachHandler ¶ added in v0.4.0
func (m *MockWallet) AttachHandler(ctx context.Context, db *gorm.DB, preparation string, wallet string) (*model.Preparation, error)
func (*MockWallet) DetachHandler ¶ added in v0.4.0
func (m *MockWallet) DetachHandler(ctx context.Context, db *gorm.DB, preparation string, wallet string) (*model.Preparation, error)
func (*MockWallet) ImportHandler ¶ added in v0.4.0
func (m *MockWallet) ImportHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request wallet.ImportRequest) (*model.Wallet, error)
func (*MockWallet) ListAttachedHandler ¶ added in v0.4.0
func (*MockWallet) ListHandler ¶ added in v0.4.0
func (*MockWallet) RemoveHandler ¶ added in v0.4.0
type Runner ¶ added in v0.4.0
type Runner struct {
// contains filtered or unexported fields
}
func NewRunner ¶ added in v0.4.0
func NewRunner() *Runner
NewRunner creates a new Runner to capture CLI args Note: tests invoking this function should stay in cmd.Test package because this function relies on environment variables to set database connection string so it won't work with parallel execution of different test packages.
func (*Runner) WithMode ¶ added in v0.4.0
func (r *Runner) WithMode(mode RunnerMode) *Runner
type RunnerMode ¶ added in v0.4.0
type RunnerMode string
const ( Normal RunnerMode = "normal" Verbose RunnerMode = "verbose" JSON RunnerMode = "json" )
Click to show internal directories.
Click to hide internal directories.