scrapecmd

package
v0.13.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 16, 2025 License: MIT Imports: 16 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var ScrapeCmd = &cobra.Command{
	Use:   "scrape [repository|developer]",
	Short: "Scrape trending repositories or trending developers form GitHub trending page.",
	Args:  cobra.ExactArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		action := args[0]
		config.Init()

		search := search.NewSearch()
		ctx, stop := context.WithCancel(context.Background())
		db := database.GetInstance(ctx)
		repositories := global.InitRepositories(db)

		tokenPool := github.NewTokenPool(config.GitHubTokens)
		gh := github.NewClient(tokenPool)
		handler := scrape.NewScrapeHandler(repositories, search, gh)

		defer func() {
			err := db.Close()

			if err != nil {
				slog.Error("failed to close db", slog.Any("error", err))
				sentry.CaptureException(err)
			}

			stop()
			sentry.Flush(2 * time.Second)
		}()

		appSignal := make(chan os.Signal, 3)
		signal.Notify(appSignal, os.Interrupt, syscall.SIGTERM)

		go func() {
			<-appSignal
			stop()
		}()

		err := handler.Handle(ctx, action)
		if err != nil {
			slog.Error("failed to handle action", slog.Any("error", err))
			sentry.CaptureException(err)
		}
	},
}

Functions

This section is empty.

Types

This section is empty.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL