datasource

package
v0.0.5 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 26, 2023 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var AddCmd = &cli.Command{
	Name:  "add",
	Usage: "Add a new data source to the dataset",
	Subcommands: underscore.Map(fs.Registry, func(r *fs.RegInfo) *cli.Command {
		cmd := datasource.OptionsToCLIFlags(r)
		cmd.Action = func(c *cli.Context) error {
			datasetName := c.Args().Get(0)
			path := c.Args().Get(1)
			db := database.MustOpenFromCLI(c)
			dataset, err := database.FindDatasetByName(db, datasetName)
			if err != nil {
				return handler.NewBadRequestError(err)
			}
			if path == "" {
				return handler.NewBadRequestString("path is required")
			}
			if r.Prefix == "local" {
				path, err = filepath.Abs(path)
				if err != nil {
					return errors.Wrap(err, "failed to get absolute path")
				}
			}
			deleteAfterExport := c.Bool("delete-after-export")
			result := map[string]string{}
			for _, flag := range c.Command.Flags {
				flagName := flag.Names()[0]
				if strings.HasPrefix(flagName, r.Prefix) && c.String(flagName) != "" {
					optionName := strings.SplitN(strings.ReplaceAll(flagName, "-", "_"), "_", 2)[1]
					reg, err := fs.Find(r.Prefix)
					if err != nil {
						return errors.Wrap(err, "failed to find fs")
					}
					option, err := underscore.Find(reg.Options, func(o fs.Option) bool {
						return o.Name == optionName
					})
					if err != nil {
						return errors.Wrap(err, "failed to find option")
					}
					result[option.Name] = c.String(flagName)
				}
			}
			source := model.Source{
				DatasetID:           dataset.ID,
				Type:                r.Prefix,
				Path:                path,
				Metadata:            model.Metadata(result),
				PushOnly:            false,
				ScanIntervalSeconds: 0,
				ScanningState:       model.Ready,
				DeleteAfterExport:   deleteAfterExport,
			}

			handler, err := datasource.DefaultHandlerResolver{}.Resolve(c.Context, source)
			if err != nil {
				return errors.Wrap(err, "failed to resolve handler")
			}

			_, err = handler.List(c.Context, "")
			if err != nil {
				return errors.Wrap(err, "failed to check source")
			}

			dir := model.Directory{
				Name: path,
			}
			err = database.DoRetry(func() error { return db.Create(&dir).Error })
			if err != nil {
				return errors.Wrap(err, "failed to create directory")
			}

			source.RootDirectoryID = dir.ID
			err = database.DoRetry(func() error { return db.Create(&source).Error })
			if err != nil {
				return errors.Wrap(err, "failed to create source")
			}

			cliutil.PrintToConsole(source, c.Bool("json"))
			return nil
		}
		return cmd
	}),
}
View Source
var CheckCmd = &cli.Command{
	Name:      "check",
	Usage:     "Check a data source by listing its entries",
	ArgsUsage: "<source_id> [sub_path]",
	Description: "This command will list entries in a data source under <sub_path>. " +
		"If <sub_path> is not provided, it will use the root directory",
	Action: func(c *cli.Context) error {
		db := database.MustOpenFromCLI(c)
		entries, err := datasource.CheckSourceHandler(
			db,
			c.Context,
			c.Args().Get(0),
			datasource.CheckSourceRequest{
				Path: c.Args().Get(1),
			},
		)
		if err != nil {
			return err.CliError()
		}

		cliutil.PrintToConsole(entries, c.Bool("json"))
		return nil
	},
}
View Source
var ListCmd = &cli.Command{
	Name:  "list",
	Usage: "List all sources",
	Flags: []cli.Flag{
		&cli.StringFlag{
			Name:  "dataset",
			Usage: "Filter by dataset name",
		},
	},
	Action: func(c *cli.Context) error {
		db := database.MustOpenFromCLI(c)
		datasetName := c.String("dataset")
		sources, err := datasource.ListSourceHandler(
			db,
			datasetName,
		)
		if err != nil {
			return err.CliError()
		}
		cliutil.PrintToConsole(sources, c.Bool("json"))
		return nil
	},
}
View Source
var RemoveCmd = &cli.Command{
	Name:      "remove",
	Usage:     "Remove a data source",
	ArgsUsage: "<source_id>",
	Action: func(c *cli.Context) error {
		db := database.MustOpenFromCLI(c)
		err := datasource.RemoveSourceHandler(
			db,
			c.Args().Get(0),
		)
		return err.CliError()
	},
}
View Source
var RescanCmd = &cli.Command{
	Name:        "rescan",
	Usage:       "Rescan a data source",
	ArgsUsage:   "<source_id>",
	Description: "This command will clear any error of a data source and rescan it",
	Action: func(c *cli.Context) error {
		db := database.MustOpenFromCLI(c)
		entries, err := datasource.RescanSourceHandler(
			db,
			c.Args().Get(0),
		)
		if err != nil {
			return err.CliError()
		}

		cliutil.PrintToConsole(entries, c.Bool("json"))
		return nil
	},
}
View Source
var UpdateCmd = &cli.Command{
	Name:      "update",
	Usage:     "Update the config options of a source",
	ArgsUsage: "<source_id>",
	Flags: func() []cli.Flag {
		var flags []cli.Flag
		for _, cmd := range AddCmd.Subcommands {
			cmdFlags := underscore.Map(cmd.Flags, func(flag cli.Flag) cli.Flag {
				stringFlag, ok := flag.(*cli.StringFlag)
				if !ok {
					return flag
				}
				stringFlag.Required = false
				return stringFlag
			})
			flags = append(flags, cmdFlags...)
		}
		return flags
	}(),
	Action: func(c *cli.Context) error {
		db := database.MustOpenFromCLI(c)
		config := map[string]string{}
		for _, name := range c.LocalFlagNames() {
			if c.IsSet(name) {
				value := c.String(name)
				config[name] = value
			}
		}

		source, err := datasource.UpdateSourceHandler(
			db,
			context.Background(),
			c.Args().Get(0),
			config,
		)
		if err != nil {
			return err.CliError()
		}

		cliutil.PrintToConsole(source, c.Bool("json"))
		return nil
	},
}

Functions

This section is empty.

Types

This section is empty.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL