regexrover

type CSVWriter struct {
	// contains filtered or unexported fields
}

type Config struct {
	ApiKey                string
	Cx                    string
	OutputDriver          string
	KeywordsFile          string
	CrawlIgnoreDomains    []string
	CrawlAllowedUrlsRegex string
	CrawlUserAgent        string
	CrawlTag              string
	CrawlMatchRegex       string
	CrawlUrlsFile         string
	CrawlCacheDir         string
	CrawlDepth            int
	CrawlThreads          int
	CrawlLog              bool
	MatchOutputChunks     int
	MaxSearchResults      int
}

type Crawler struct {
	// contains filtered or unexported fields
}

type CrawlerStats struct {
	Scraped  uint32
	Requests uint32
	Pending  int64
	Matches  uint32
}

type Logger func(format string, v ...interface{})

type SQLiteWriter struct {
	// contains filtered or unexported fields
}

type SearchResults struct {
	Links []string
	// contains filtered or unexported fields
}

type Writer interface {
	WriterInit
	WriteWithCache(key string, value string, forceWrite bool) error
	WriteAll(records [][]string) error
	GetFileName() string
	Close() error
}

type WriterInit interface {
	Init(cacheMaxSize int) error
}