crawlers

package

v0.1.2 Latest Latest Go to latest Published: Sep 30, 2025 License: GPL-3.0 Imports: 17 Imported by: 0

Documentation ¶

Index ¶

Constants
Variables
func GenerateObjects(image string) []*v1beta1.Crawler
type CrawledTarget
type Crawler
type GitHubOrg
type GitLab
type StaticList

Constants ¶

const (
	GitLabGroupsParamName          = "GITLAB_GROUPS"
	GitlabInstanceURLParamName     = "GITLAB_INSTANCE_URL"
	GitlabIncludeSubgroupParamName = "INCLUDE_SUBGROUPS"
)

View Source

const (
	ProfileParamName            = "PROFILE"
	SleepDurationParamName      = "SLEEP_DURATION"
	SleepDurationDefaultValue   = "1m"
	PipelineTTLParamName        = "PIPELINE_TTL"
	PipelineTTLDefaultValue     = "168h" // 7 days
	DownloaderOverrideParamName = "DOWNLOADER_OVERRIDE"
)

View Source

const (
	GitHubOrgsParamName = "GITHUB_ORGS"
)

View Source

const (
	GitHubTokenSecretEnvVar = "GITHUB_TOKEN"
)

View Source

const (
	GitlabTokenSecretEnvVar = "GITLAB_TOKEN"
)

View Source

const (
	StaticTargetIdentifierList = "TARGET_IDENTIFIERS"
)

Variables ¶

View Source

var AllCrawlers = []Crawler{
	GitHubOrg{},
	GitLab{},
	StaticList{},
}

View Source

var DefaultParameters = []v1beta1.ParameterDefinition{
	{
		Name:        ProfileParamName,
		Description: "Profile to use for the pipelines created from this crawler.",
		Required:    true,
	},
	{
		Name:        SleepDurationParamName,
		Description: "Duration to sleep between requests. Will be parsed as a time.Duration.",
		Required:    false,
		Default:     ptr.To(SleepDurationDefaultValue),
	},
	{
		Name:        PipelineTTLParamName,
		Description: "TTL for the pipelines created by this crawler. Will be parsed as a time.Duration.",
		Required:    false,
		Default:     ptr.To(PipelineTTLDefaultValue),
	},
	{
		Name:        DownloaderOverrideParamName,
		Description: "Override the downloader for the crawler. By default, it will be chosen based on the crawler type.",
		Required:    false,
	},
}

Functions ¶

func GenerateObjects ¶ added in v0.1.2

func GenerateObjects(image string) []*v1beta1.Crawler

Types ¶

type CrawledTarget ¶ added in v0.1.2

type CrawledTarget struct {
	Target            v1beta1.Target
	DefaultDownloader string
}

CrawledTarget represents a target that has been crawled that should have a pipeline run created for it. It includes the target itself and the default downloader to use for it (if not overridden setting the Downloader parameter).

type Crawler ¶

type Crawler interface {
	GetParameters() []v1beta1.ParameterDefinition
	GetName() string
	Crawl(ctx context.Context, params map[string]string, queue chan CrawledTarget) error
	GetEnvSecrets() []definitions.EnvironmentSecret
	GetFileSecrets() []definitions.FileSecret
	EnvironmentVariables() []corev1.EnvVar
}

type GitHubOrg ¶

type GitHubOrg struct{}

func (GitHubOrg) Crawl ¶

func (GitHubOrg) Crawl(
	ctx context.Context,
	params map[string]string,
	queue chan CrawledTarget,
) error

Crawl retrieves all repositories from a specified GitHub organization and sends their clone URLs to the provided queue channel. By default, the downloader used is "git", but this can be overridden by setting the parameter variable [DownloaderParamName] to a different value. The GitHub token can be set by setting the secret [GithubTokenParam].

func (GitHubOrg) EnvironmentVariables ¶ added in v0.1.2

func (o GitHubOrg) EnvironmentVariables() []corev1.EnvVar

func (GitHubOrg) GetEnvSecrets ¶ added in v0.1.2

func (o GitHubOrg) GetEnvSecrets() []definitions.EnvironmentSecret

func (GitHubOrg) GetFileSecrets ¶ added in v0.1.2

func (o GitHubOrg) GetFileSecrets() []definitions.FileSecret

func (GitHubOrg) GetName ¶ added in v0.1.2

func (GitHubOrg) GetName() string

func (GitHubOrg) GetParameters ¶ added in v0.1.2

func (GitHubOrg) GetParameters() []v1beta1.ParameterDefinition

type GitLab ¶

type GitLab struct{}

func (GitLab) Crawl ¶

func (g GitLab) Crawl(
	ctx context.Context,
	params map[string]string,
	queue chan CrawledTarget,
) error

Crawl retrieves all repositories from a specified GitLab groups and sends their clone URLs to the provided queue channel. By default, the downloader used is "git", but this can be overridden by setting the parameter variable [DownloaderParamName] to a different value.

func (GitLab) EnvironmentVariables ¶ added in v0.1.2

func (GitLab) EnvironmentVariables() []corev1.EnvVar

func (GitLab) GetEnvSecrets ¶ added in v0.1.2

func (GitLab) GetEnvSecrets() []definitions.EnvironmentSecret

func (GitLab) GetFileSecrets ¶ added in v0.1.2

func (GitLab) GetFileSecrets() []definitions.FileSecret

func (GitLab) GetName ¶ added in v0.1.2

func (g GitLab) GetName() string

func (GitLab) GetParameters ¶ added in v0.1.2

func (g GitLab) GetParameters() []v1beta1.ParameterDefinition

type StaticList ¶ added in v0.1.1

type StaticList struct{}

func (StaticList) Crawl ¶ added in v0.1.1

func (s StaticList) Crawl(
	_ context.Context,
	params map[string]string,
	queue chan CrawledTarget,
) error

func (StaticList) EnvironmentVariables ¶ added in v0.1.2

func (StaticList) EnvironmentVariables() []corev1.EnvVar

func (StaticList) GetEnvSecrets ¶ added in v0.1.2

func (StaticList) GetEnvSecrets() []definitions.EnvironmentSecret

func (StaticList) GetFileSecrets ¶ added in v0.1.2

func (StaticList) GetFileSecrets() []definitions.FileSecret

func (StaticList) GetName ¶ added in v0.1.2

func (s StaticList) GetName() string

func (StaticList) GetParameters ¶ added in v0.1.2

func (s StaticList) GetParameters() []v1beta1.ParameterDefinition

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL