crawl

var CrawlCmd = &cobra.Command{
	Use:   "crawl [nodes file]",
	Short: "Crawl a network",
	Long:  `This is a basic function to crawl a network.`,
	Args:  cobra.MinimumNArgs(1),
	PreRun: func(cmd *cobra.Command, args []string) {
		inputCrawlParams.NodesFile = args[0]
	},
	RunE: func(cmd *cobra.Command, args []string) error {
		inputSet, err := p2p.LoadNodesJSON(inputCrawlParams.NodesFile)
		if err != nil {
			return err
		}

		var cfg discover.Config
		cfg.PrivateKey, _ = crypto.GenerateKey()
		bn, err := p2p.ParseBootnodes(inputCrawlParams.Bootnodes)
		if err != nil {
			log.Error().Err(err).Msg("Unable to parse bootnodes")
			return err
		}
		cfg.Bootnodes = bn

		db, err := enode.OpenDB(inputCrawlParams.Database)
		if err != nil {
			return err
		}

		ln := enode.NewLocalNode(db, cfg.PrivateKey)
		socket, err := listen(ln)
		if err != nil {
			return err
		}

		disc, err := discover.ListenV4(socket, ln, cfg)
		if err != nil {
			return err
		}
		defer disc.Close()

		c := newCrawler(inputSet, disc, disc.RandomNodes())
		c.revalidateInterval = 10 * time.Minute

		timeout, err := time.ParseDuration(inputCrawlParams.Timeout)
		if err != nil {
			return err
		}

		log.Info().Msg("Starting crawl")

		output := c.run(timeout, inputCrawlParams.Threads)
		return p2p.WriteNodesJSON(inputCrawlParams.NodesFile, output)
	},
}