cmd

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 8, 2023 License: Apache-2.0, MIT Imports: 55 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

View Source
var App = &cli.App{
	Name:  "singularity",
	Usage: "A tool for large-scale clients with PB-scale data onboarding to Filecoin network",
	Description: `Database Backend Support:
  Singularity supports multiple database backend: sqlite3, postgres, mysql5.7+
  Use '--database-connection-string' or $DATABASE_CONNECTION_STRING to specify the database connection string.
    Example for postgres  - postgres://user:pass@example.com:5432/dbname
    Example for mysql     - mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true
    Example for sqlite3   - sqlite:/absolute/path/to/database.db
                or        - sqlite:relative/path/to/database.db

Network Support:
  Default settings in Singularity are for Mainnet. You may set below environment values for other network:
    For Calibration network:
      * Set LOTUS_API to https://api.calibration.node.glif.io/rpc/v1
      * Set MARKET_DEAL_URL to https://marketdeals-calibration.s3.amazonaws.com/StateMarketDeals.json.zst
      * Set LOTUS_TEST to 1
    For all other networks:
      * Set LOTUS_API to your network's Lotus API endpoint
      * Set MARKET_DEAL_URL to empty string
      * Set LOTUS_TEST to 0 or 1 based on whether the network address starts with 'f' or 't'
    Switching between different networks with the same database instance is not recommended.`,
	EnableBashCompletion: true,
	Flags: []cli.Flag{
		&cli.StringFlag{
			Name:        "database-connection-string",
			Usage:       "Connection string to the database",
			DefaultText: "sqlite:" + "./singularity.db",
			Value:       "sqlite:" + "./singularity.db",
			EnvVars:     []string{"DATABASE_CONNECTION_STRING"},
		},
		&cli.BoolFlag{
			Name:  "json",
			Usage: "Enable JSON output",
			Value: false,
		},
		&cli.BoolFlag{
			Name:  "verbose",
			Usage: "Enable verbose output. This will print more columns for the result as well as full error trace",
			Value: false,
		},
		&cli.StringFlag{
			Name:     "lotus-api",
			Category: "Lotus",
			Usage:    "Lotus RPC API endpoint",
			Value:    "https://api.node.glif.io/rpc/v1",
			EnvVars:  []string{"LOTUS_API"},
		},
		&cli.StringFlag{
			Name:     "lotus-token",
			Category: "Lotus",
			Usage:    "Lotus RPC API token",
			Value:    "",
			EnvVars:  []string{"LOTUS_TOKEN"},
		},
		&cli.BoolFlag{
			Name:     "lotus-test",
			Category: "Lotus",
			Usage:    "Whether the runtime environment is using Testnet.",
			EnvVars:  []string{"LOTUS_TEST"},
			Action: func(c *cli.Context, testnet bool) error {
				if testnet {
					address.CurrentNetwork = address.Testnet
					logger.Infow("Current network is set to Testnet")
				}
				return nil
			},
		},
	},
	Commands: []*cli.Command{
		ez.PrepCmd,
		VersionCmd,
		{
			Name:     "admin",
			Usage:    "Admin commands",
			Category: "Operations",
			Subcommands: []*cli.Command{
				admin.InitCmd,
				admin.ResetCmd,
				admin.MigrateDatasetCmd,
				admin.MigrateScheduleCmd,
			},
		},
		DownloadCmd,
		tool.ExtractCarCmd,
		{
			Name:     "deal",
			Usage:    "Replication / Deal making management",
			Category: "Operations",
			Subcommands: []*cli.Command{
				{
					Name:  "schedule",
					Usage: "Schedule deals",
					Subcommands: []*cli.Command{
						schedule.CreateCmd,
						schedule.ListCmd,
						schedule.PauseCmd,
						schedule.ResumeCmd,
					},
				},
				deal.SendManualCmd,
				deal.ListCmd,
			},
		},
		{
			Name:     "run",
			Category: "Daemons",
			Usage:    "run different singularity components",
			Subcommands: []*cli.Command{
				run.APICmd,
				run.DatasetWorkerCmd,
				run.ContentProviderCmd,
				run.DealTrackerCmd,
				run.DealPusherCmd,
			},
		},
		{
			Name:     "wallet",
			Category: "Operations",
			Usage:    "Wallet management",
			Subcommands: []*cli.Command{
				wallet.ImportCmd,
				wallet.ListCmd,
				wallet.RemoveCmd,
			},
		},
		{
			Name:     "storage",
			Category: "Operations",
			Usage:    "Create and manage storage system connections",
			Subcommands: []*cli.Command{
				storage.CreateCmd,
				storage.ExploreCmd,
				storage.ListCmd,
				storage.RemoveCmd,
				storage.UpdateCmd,
			},
		},
		{
			Name:     "prep",
			Category: "Operations",
			Usage:    "Create and manage dataset preparations",
			Subcommands: []*cli.Command{
				dataprep.CreateCmd,
				dataprep.ListCmd,
				dataprep.StatusCmd,
				dataprep.AttachSourceCmd,
				dataprep.AttachOutputCmd,
				dataprep.DetachOutputCmd,
				dataprep.StartScanCmd,
				dataprep.PauseScanCmd,
				dataprep.StartPackCmd,
				dataprep.PausePackCmd,
				dataprep.StartDagGenCmd,
				dataprep.PauseDagGenCmd,
				dataprep.ListPiecesCmd,
				dataprep.AddPieceCmd,
				dataprep.ExploreCmd,
				dataprep.AttachWalletCmd,
				dataprep.ListWalletsCmd,
				dataprep.DetachWalletCmd,
			},
		},
	},
}
View Source
var DownloadCmd = &cli.Command{
	Name:      "download",
	Usage:     "Download a CAR file from the metadata API",
	Category:  "Utility",
	Before:    cliutil.CheckNArgs,
	ArgsUsage: "<piece_cid>",
	Flags: func() []cli.Flag {
		flags := []cli.Flag{
			&cli.StringFlag{
				Name:     "api",
				Usage:    "URL of the metadata API",
				Value:    "http://127.0.0.1:7777",
				Category: "General Config",
			},
			&cli.StringFlag{
				Name:     "out-dir",
				Usage:    "Directory to write CAR files to",
				Value:    ".",
				Category: "General Config",
			},
			&cli.IntFlag{
				Name:     "concurrency",
				Usage:    "Number of concurrent downloads",
				Value:    10,
				Category: "General Config",
			},
			&cli.BoolFlag{
				Name:     "quiet",
				Usage:    "Suppress all output",
				Category: "General Config",
				Value:    false,
			},
		}

		keys := make(map[string]struct{})
		for _, backend := range storagesystem.Backends {
			var providers []string
			for _, providerOptions := range backend.ProviderOptions {
				providers = append(providers, providerOptions.Provider)
				for _, option := range providerOptions.Options {
					flag := option.ToCLIFlag(backend.Prefix+"-", false, backend.Description)
					if _, ok := keys[flag.Names()[0]]; ok {
						continue
					}
					keys[flag.Names()[0]] = struct{}{}
					flags = append(flags, flag)
				}
			}
			if len(providers) > 1 {
				providerFlag := &cli.StringFlag{
					Name:  backend.Prefix + "-provider",
					Usage: strings.Join(providers, " | "),
					EnvVars: []string{
						strings.ToUpper(backend.Prefix) + "_PROVIDER",
					},
					Category: backend.Description,
					Value:    providers[0],
				}
				flags = append(flags, providerFlag)
			}
		}
		return flags
	}(),
	Action: func(c *cli.Context) error {
		api := c.String("api")
		outDir := c.String("out-dir")
		concurrency := c.Int("concurrency")
		piece := c.Args().First()
		config := map[string]string{}
		for _, key := range c.LocalFlagNames() {
			if c.IsSet(key) {
				if slices.Contains([]string{"api", "out-dir", "concurrency", "quiet"}, key) {
					continue
				}
				value := c.String(key)
				config[key] = value
			}
		}
		err := handler.DownloadHandler(c, piece, api, config, outDir, concurrency)
		if err == nil {
			log.Logger("Download").Info("Download complete")
			return nil
		}
		return errors.WithStack(err)
	},
}
View Source
var Version string
View Source
var VersionCmd = &cli.Command{
	Name:    "version",
	Usage:   "Print version information",
	Aliases: []string{"v"},
	Action: func(context *cli.Context) error {
		buildInfo, ok := debug.ReadBuildInfo()
		if !ok {
			fmt.Println("unknown version")
		}

		version := buildInfo.Main.Version
		if version == "(devel)" || version == "" {
			version = Version
		}
		var revision string
		var modified string
		for _, setting := range buildInfo.Settings {
			switch setting.Key {
			case "vcs.revision":
				revision = setting.Value[:7]
			case "vcs.modified":
				modified = setting.Value
			}
		}
		if revision == "" {
			revision = "-unknown"
		} else {
			revision = "-" + revision
		}
		switch modified {
		case "true":
			modified = "-dirty"
		case "false":
			modified = ""
		case "":
			modified = "-unknown"
		default:
			modified = "-" + modified
		}
		v := fmt.Sprintf("singularity %s%s%s\n", version, revision, modified)
		_, err := context.App.Writer.Write([]byte(v))

		return errors.WithStack(err)
	},
}

Functions

func CalculateCommp added in v0.4.0

func CalculateCommp(t *testing.T, content []byte, targetPieceSize uint64) string

func CompareDirectories added in v0.4.0

func CompareDirectories(t *testing.T, dir1, dir2 string)

func Download added in v0.4.0

func Download(ctx context.Context, url string, nThreads int) ([]byte, error)

func GetAllPieceCIDs added in v0.4.0

func GetAllPieceCIDs(content string) []string

func GetFirstCID added in v0.4.0

func GetFirstCID(content string) string

func SetVersionJSON added in v0.4.0

func SetVersionJSON(versionJSON []byte) error

func SetupErrorHandler added in v0.4.0

func SetupErrorHandler()

func SetupHelpPager added in v0.3.0

func SetupHelpPager()

func WaitForServerReady added in v0.4.0

func WaitForServerReady(ctx context.Context, url string) error

Types

type MockAdmin added in v0.4.0

type MockAdmin struct {
	mock.Mock
}

func (*MockAdmin) InitHandler added in v0.4.0

func (m *MockAdmin) InitHandler(ctx context.Context, db *gorm.DB) error

func (*MockAdmin) ResetHandler added in v0.4.0

func (m *MockAdmin) ResetHandler(ctx context.Context, db *gorm.DB) error

type MockDataPrep added in v0.4.0

type MockDataPrep struct {
	mock.Mock
}

func (*MockDataPrep) AddOutputStorageHandler added in v0.4.0

func (m *MockDataPrep) AddOutputStorageHandler(ctx context.Context, db *gorm.DB, id string, output string) (*model.Preparation, error)

func (*MockDataPrep) AddPieceHandler added in v0.4.0

func (m *MockDataPrep) AddPieceHandler(ctx context.Context, db *gorm.DB, id string, request dataprep.AddPieceRequest) (*model.Car, error)

func (*MockDataPrep) AddSourceStorageHandler added in v0.4.0

func (m *MockDataPrep) AddSourceStorageHandler(ctx context.Context, db *gorm.DB, id string, source string) (*model.Preparation, error)

func (*MockDataPrep) CreatePreparationHandler added in v0.4.0

func (m *MockDataPrep) CreatePreparationHandler(ctx context.Context, db *gorm.DB, request dataprep.CreateRequest) (*model.Preparation, error)

func (*MockDataPrep) ExploreHandler added in v0.4.0

func (m *MockDataPrep) ExploreHandler(ctx context.Context, db *gorm.DB, id string, name string, path string) (*dataprep.ExploreResult, error)

func (*MockDataPrep) ListHandler added in v0.4.0

func (m *MockDataPrep) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Preparation, error)

func (*MockDataPrep) ListPiecesHandler added in v0.4.0

func (m *MockDataPrep) ListPiecesHandler(ctx context.Context, db *gorm.DB, id string) ([]dataprep.PieceList, error)

func (*MockDataPrep) ListSchedulesHandler added in v0.4.0

func (m *MockDataPrep) ListSchedulesHandler(ctx context.Context, db *gorm.DB, id string) ([]model.Schedule, error)

func (*MockDataPrep) RemoveOutputStorageHandler added in v0.4.0

func (m *MockDataPrep) RemoveOutputStorageHandler(ctx context.Context, db *gorm.DB, id string, output string) (*model.Preparation, error)

type MockDeal added in v0.4.0

type MockDeal struct {
	mock.Mock
}

func (*MockDeal) ListHandler added in v0.4.0

func (m *MockDeal) ListHandler(ctx context.Context, db *gorm.DB, request deal.ListDealRequest) ([]model.Deal, error)

func (*MockDeal) SendManualHandler added in v0.4.0

func (m *MockDeal) SendManualHandler(ctx context.Context, db *gorm.DB, dealMaker replication.DealMaker, request deal.Proposal) (*model.Deal, error)

type MockFile added in v0.4.0

type MockFile struct {
	mock.Mock
}

func (*MockFile) GetFileDealsHandler added in v0.4.0

func (m *MockFile) GetFileDealsHandler(
	ctx context.Context,
	db *gorm.DB,
	id string,
) ([]model.Deal, error)

func (*MockFile) GetFileHandler added in v0.4.0

func (m *MockFile) GetFileHandler(ctx context.Context, db *gorm.DB, id uint64) (*model.File, error)

func (*MockFile) PrepareToPackFileHandler added in v0.4.0

func (m *MockFile) PrepareToPackFileHandler(ctx context.Context, db *gorm.DB, fileID uint64) (int64, error)

func (*MockFile) PushFileHandler added in v0.4.0

func (m *MockFile) PushFileHandler(ctx context.Context, db *gorm.DB, preparation string, source string, fileInfo file.Info) (*model.File, error)

type MockJob added in v0.4.0

type MockJob struct {
	mock.Mock
}

func (*MockJob) GetStatusHandler added in v0.4.0

func (m *MockJob) GetStatusHandler(ctx context.Context, db *gorm.DB, id string) ([]job.SourceStatus, error)

func (*MockJob) PackHandler added in v0.4.0

func (m *MockJob) PackHandler(ctx context.Context, db *gorm.DB, jobID int64) (*model.Car, error)

func (*MockJob) PauseDagGenHandler added in v0.4.0

func (m *MockJob) PauseDagGenHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)

func (*MockJob) PausePackHandler added in v0.4.0

func (m *MockJob) PausePackHandler(ctx context.Context, db *gorm.DB, id string, name string, jobID int64) ([]model.Job, error)

func (*MockJob) PauseScanHandler added in v0.4.0

func (m *MockJob) PauseScanHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)

func (*MockJob) PrepareToPackSourceHandler added in v0.4.0

func (m *MockJob) PrepareToPackSourceHandler(ctx context.Context, db *gorm.DB, id string, name string) error

func (*MockJob) StartDagGenHandler added in v0.4.0

func (m *MockJob) StartDagGenHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)

func (*MockJob) StartPackHandler added in v0.4.0

func (m *MockJob) StartPackHandler(ctx context.Context, db *gorm.DB, id string, name string, jobID int64) ([]model.Job, error)

func (*MockJob) StartScanHandler added in v0.4.0

func (m *MockJob) StartScanHandler(ctx context.Context, db *gorm.DB, id string, name string) (*model.Job, error)

type MockSchedule added in v0.4.0

type MockSchedule struct {
	mock.Mock
}

func (*MockSchedule) CreateHandler added in v0.4.0

func (m *MockSchedule) CreateHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request schedule.CreateRequest) (*model.Schedule, error)

func (*MockSchedule) ListHandler added in v0.4.0

func (m *MockSchedule) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Schedule, error)

func (*MockSchedule) PauseHandler added in v0.4.0

func (m *MockSchedule) PauseHandler(ctx context.Context, db *gorm.DB, scheduleID uint32) (*model.Schedule, error)

func (*MockSchedule) ResumeHandler added in v0.4.0

func (m *MockSchedule) ResumeHandler(ctx context.Context, db *gorm.DB, scheduleID uint32) (*model.Schedule, error)

type MockStorage added in v0.4.0

type MockStorage struct {
	mock.Mock
}

func (*MockStorage) CreateStorageHandler added in v0.4.0

func (m *MockStorage) CreateStorageHandler(ctx context.Context, db *gorm.DB, storageType string, request storage.CreateRequest) (*model.Storage, error)

func (*MockStorage) ExploreHandler added in v0.4.0

func (m *MockStorage) ExploreHandler(ctx context.Context, db *gorm.DB, name string, path string) ([]storage.DirEntry, error)

func (*MockStorage) ListStoragesHandler added in v0.4.0

func (m *MockStorage) ListStoragesHandler(ctx context.Context, db *gorm.DB) ([]model.Storage, error)

func (*MockStorage) RemoveHandler added in v0.4.0

func (m *MockStorage) RemoveHandler(ctx context.Context, db *gorm.DB, name string) error

func (*MockStorage) UpdateStorageHandler added in v0.4.0

func (m *MockStorage) UpdateStorageHandler(ctx context.Context, db *gorm.DB, name string, config map[string]string) (*model.Storage, error)

type MockWallet added in v0.4.0

type MockWallet struct {
	mock.Mock
}

func (*MockWallet) AttachHandler added in v0.4.0

func (m *MockWallet) AttachHandler(ctx context.Context, db *gorm.DB, preparation string, wallet string) (*model.Preparation, error)

func (*MockWallet) DetachHandler added in v0.4.0

func (m *MockWallet) DetachHandler(ctx context.Context, db *gorm.DB, preparation string, wallet string) (*model.Preparation, error)

func (*MockWallet) ImportHandler added in v0.4.0

func (m *MockWallet) ImportHandler(ctx context.Context, db *gorm.DB, lotusClient jsonrpc.RPCClient, request wallet.ImportRequest) (*model.Wallet, error)

func (*MockWallet) ListAttachedHandler added in v0.4.0

func (m *MockWallet) ListAttachedHandler(ctx context.Context, db *gorm.DB, preparation string) ([]model.Wallet, error)

func (*MockWallet) ListHandler added in v0.4.0

func (m *MockWallet) ListHandler(ctx context.Context, db *gorm.DB) ([]model.Wallet, error)

func (*MockWallet) RemoveHandler added in v0.4.0

func (m *MockWallet) RemoveHandler(ctx context.Context, db *gorm.DB, address string) error

type Runner added in v0.4.0

type Runner struct {
	// contains filtered or unexported fields
}

func NewRunner added in v0.4.0

func NewRunner() *Runner

NewRunner creates a new Runner to capture CLI args Note: tests invoking this function should stay in cmd.Test package because this function relies on environment variables to set database connection string so it won't work with parallel execution of different test packages.

func (*Runner) Run added in v0.4.0

func (r *Runner) Run(ctx context.Context, args string) (string, string, error)

func (*Runner) Save added in v0.4.0

func (r *Runner) Save(t *testing.T, tempDirs ...string)

func (*Runner) WithMode added in v0.4.0

func (r *Runner) WithMode(mode RunnerMode) *Runner

type RunnerMode added in v0.4.0

type RunnerMode string
const (
	Normal  RunnerMode = "normal"
	Verbose RunnerMode = "verbose"
	JSON    RunnerMode = "json"
)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL