types

type ContentType string

type DynamicHTMLFetcher interface {
	//Fetch the proxy info
	Fetch(parent context.Context, urlIdx int, url string) (ps []*ProxyServer, e error)
	//Headless specifies whether the web driver should be run in headless mode
	Headless() bool
	//HomePageTimeout specifies how many seconds to wait before home page navigation is timed out
	HomePageTimeout() int
}

type FetcherSpec interface {
	//UID returns the unique identifier for this spec.
	UID() string
	//Urls return the server urls that provide the free proxy server lists.
	Urls() []string
	//ProxyMode returns whether to use the following:
	//Master: the fetcher needs a master proxy server to access the free proxy list provider.
	//Rotate: the fetcher will use available proxy from database by random
	//Direct: the fetcher will not use any proxy to access the provider website.
	ProxyMode() ProxyMode
	//RefreshInterval determines how often the list should be refreshed, in minutes.
	RefreshInterval() int
	//Retry specifies how many times should be retried
	Retry() int
}

type JSONFetcher interface {
	//ParseJSON parses JSON payload and extracts proxy information
	ParseJSON(payload []byte) (ps []*ProxyServer)
}

type NetworkTraffic struct {
	ID                    uint      `gorm:"primaryKey;autoIncrement"`
	Timestamp             time.Time `gorm:"not null"`
	SourceIP              string    `gorm:"not null"`
	DestinationIP         string    `gorm:"not null"`
	SourcePort            int       `gorm:"size:16"`
	DestinationPort       int       `gorm:"size:16"`
	Protocol              string    `gorm:"size:10"`
	Method                string    `gorm:"size:10"`
	URL                   string    `gorm:"not null"`
	RequestHeaders        string    `gorm:"type:text"`
	RequestBody           []byte    `gorm:"type:blob"`
	ResponseHeaders       string    `gorm:"type:text"`
	ResponseBody          []byte    `gorm:"type:blob"`
	StatusCode            uint      `gorm:"size:16"`
	ResponseContentLength uint      `gorm:"size:32"`
	MIMEType              string    `gorm:"size:50"`
	gorm.Model
}

type PlainTextFetcher interface {
	//ParsePlainText parses plain text payload and extracts proxy information
	ParsePlainText(payload []byte) (ps []*ProxyServer)
}

type ProxyMode string

type ProxyServer struct {
	gorm.Model

	ID          uint
	Source      string `gorm:"index:idx_source"`
	Host        string `gorm:"uniqueIndex:idx_host_port;index:idx_status"`
	Port        string `gorm:"uniqueIndex:idx_host_port;index:idx_status"`
	Type        string `gorm:"index:idx_status"`
	Loc         string
	Status      string `gorm:"index:idx_last_check;index:idx_status"`
	Suc         int
	Fail        int `gorm:"index:idx_status"`
	Score       float64
	StatusG     string  `db:"status_g"`
	SucG        int     `db:"suc_g"`
	FailG       int     `db:"fail_g"`
	ScoreG      float64 `db:"score_g"`
	LastCheck   string  `db:"last_check" gorm:"index:idx_last_check"`
	LastScanned string  `db:"last_scanned" gorm:"index:idx_source"`
}

type StaticHTMLFetcher interface {
	//IsGBK returns wheter the web page is GBK encoded.
	IsGBK() bool
	//ListSelector returns the jQuery selectors for searching the proxy server list/table.
	ListSelector() []string
	//ScanItem process each item found in the table determined by ListSelector().
	ScanItem(itemIdx, urlIdx int, s *goquery.Selection) (ps *ProxyServer)
}

type UserAgent struct {
	gorm.Model

	ID                   int             `db:"id"`
	Source               sql.NullString  `db:"source"`
	UserAgent            sql.NullString  `db:"user_agent"`
	TimesSeen            sql.NullInt64   `db:"times_seen"`
	Percent              sql.NullFloat64 `db:"percent"`
	SimpleSoftwareString sql.NullString  `db:"simple_software_string"`
	SoftwareName         sql.NullString  `db:"software_name"`
	SoftwareVersion      sql.NullString  `db:"software_version"`
	SoftwareType         sql.NullString  `db:"software_type"`
	SoftwareSubType      sql.NullString  `db:"software_sub_type"`
	HardwareType         sql.NullString  `db:"hardware_type"`
	FirstSeenAt          sql.NullString  `db:"first_seen_at"`
	LastSeenAt           sql.NullString  `db:"last_seen_at"`
	UpdatedAt            sql.NullString  `db:"updated_at"`
}