model

type Domain struct {
	ID      string `json:"id,omitempty" elastic_meta:"_id"`
	Host    string
	Port    string
	Favicon string
	Enabled bool
}

type FetchTask struct {
	Url Url

	CurrentStatus int
	CurrentStage  int
	StageStatus   map[int]int
}

type Host struct {
	Host        string        `json:"host,omitempty" elastic_meta:"_id" elastic_mapping:"host: { type: keyword, ignore_above: 256 }"`
	Favicon     string        `json:"favicon,omitempty"`
	Enabled     bool          `json:"enabled"`
	HostConfigs *[]HostConfig `json:"host_configs,omitempty"`
	Created     time.Time     `json:"created,omitempty"`
	Updated     time.Time     `json:"updated,omitempty"`
}

type HostConfig struct {
	ID         string `json:"id,omitempty" elastic_meta:"_id"`
	Host       string `json:"host"`
	UrlPattern string `json:"url_pattern"`
	Runner     string `json:"runner"`
	SortOrder  int    `json:"sort_order"`

	PipelineID string `json:"pipeline_id"`
	Cookies    string `json:"cookies,omitempty"`

	Created time.Time `json:"created,omitempty"`
	Updated time.Time `json:"updated,omitempty"`
}

type Index struct {
	Host     string    `json:"host,omitempty"`
	Task     *Task     `json:"task,omitempty" elastic_mapping:"task:{type:object}"`
	Snapshot *Snapshot `json:"snapshot,omitempty" elastic_mapping:"snapshot:{type:object}"`
}

type KV struct {
	Key   string   `json:"key,omitempty"`
	Value []string `json:"value,omitempty"`
}

type LinkGroup struct {
	Internal []PageLink `json:"internal,omitempty" elastic_mapping:"internal:{type:object}"`
	External []PageLink `json:"external,omitempty" elastic_mapping:"external:{type:object}"`
}

type PageLink struct {
	Url   string `json:"url,omitempty" elastic_mapping:"url: { type: keyword }"`
	Label string `json:"label,omitempty" elastic_mapping:"label: { type: text }"`
}

type Project struct {
	ID          string    `json:"id,omitempty" elastic_meta:"_id"`
	Name        string    `json:"name,omitempty"`
	Description string    `json:"description,omitempty"`
	Enabled     bool      `json:"enabled"`
	Created     time.Time `json:"created,omitempty"`
	Updated     time.Time `json:"updated,omitempty"`
	Banner      string    `json:"banner,omitempty"`
	Favicon     string    `json:"favicon,omitempty"`

	DomainRules config.Rules `json:"domain_rules,omitempty"`
	UrlRules    config.Rules `json:"url_rules,omitempty"`
}

type Snapshot struct {
	ID      string `json:"id,omitempty" elastic_meta:"_id"`
	Version int    `json:"version,omitempty"`
	Url     string `json:"url,omitempty"`
	TaskID  string `json:"task_id,omitempty"`
	Path    string `json:"path,omitempty"` //path of this file
	File    string `json:"file,omitempty"` //filename of this page
	Ext     string `json:"ext,omitempty"`  //extension of filename

	StatusCode int    `json:"-"`
	Payload    []byte `json:"-"`
	Size       uint64 `json:"size,omitempty"`

	ScreenshotID string `json:"screenshot_id,omitempty"`

	Headers    map[string][]string     `json:"-"`
	Metadata   *map[string]interface{} `json:"-"`
	Parameters []KV                    `json:"-"`

	Language string `json:"lang,omitempty"`

	Title       string `json:"title,omitempty" elastic_mapping:"title: { type: text, fields: { keyword: { type: keyword } } }"`
	Summary     string `json:"summary,omitempty"`
	Text        string `json:"text,omitempty" elastic_mapping:"text: { type: text }"`
	ContentType string `json:"content_type,omitempty"`

	Tags []string `json:"tags,omitempty"`

	Links LinkGroup `json:"links,omitempty" elastic_mapping:"links:{type:object}"`

	Images struct {
		Internal []PageLink `json:"internal,omitempty" elastic_mapping:"internal:{type:object}"`
		External []PageLink `json:"external,omitempty" elastic_mapping:"external:{type:object}"`
	} `json:"images,omitempty" elastic_mapping:"images:{type:object}"`

	H1     []string `json:"h1,omitempty" elastic_mapping:"h1: { type: text }"`
	H2     []string `json:"h2,omitempty" elastic_mapping:"h2: { type: text }"`
	H3     []string `json:"h3,omitempty" elastic_mapping:"h3: { type: text }"`
	H4     []string `json:"h4,omitempty" elastic_mapping:"h4: { type: text }"`
	H5     []string `json:"h5,omitempty" elastic_mapping:"h5: { type: text }"`
	Bold   []string `json:"bold,omitempty" elastic_mapping:"bold: { type: text }"`
	Italic []string `json:"italic,omitempty"`

	Classifications  []string                `json:"classifications,omitempty"`
	EnrichedFeatures *map[string]interface{} `json:"enriched_features,omitempty"`

	Hash    string `json:"hash,omitempty"`
	SimHash string `json:"sim_hash,omitempty"`

	Created time.Time `json:"created,omitempty"`
}

type Task struct {
	ID string `json:"id" elastic_meta:"_id"`
	// the url may not cleaned, may miss the host part, need reference to provide the complete url information
	Url         string    `json:"url,omitempty"`
	Reference   string    `json:"reference_url,omitempty"`
	Depth       int       `json:"depth"`
	Breadth     int       `json:"breadth"`
	Host        string    `json:"host"`
	Schema      string    `json:"schema,omitempty"`
	OriginalUrl string    `json:"original_url,omitempty"`
	Status      int       `json:"status"`
	Message     string    `json:"message,omitempty"`
	Created     time.Time `json:"created,omitempty" elastic_mapping:"created: { type: date }"`
	Updated     time.Time `json:"updated,omitempty" elastic_mapping:"updated: { type: date }"`
	LastFetch   time.Time `json:"last_fetch,omitempty" elastic_mapping:"last_fetch: { type: date }"`
	LastCheck   time.Time `json:"last_check,omitempty" elastic_mapping:"last_check: { type: date }"`
	NextCheck   time.Time `json:"next_check,omitempty" elastic_mapping:"next_check: { type: date }"`

	SnapshotVersion  int       `json:"snapshot_version,omitempty"`
	SnapshotID       string    `json:"snapshot_id,omitempty"`
	SnapshotHash     string    `json:"snapshot_hash,omitempty"`
	SnapshotSimHash  string    `json:"snapshot_simhash,omitempty"`
	SnapshotCreated  time.Time `json:"snapshot_created,omitempty" elastic_mapping:"snapshot_created: { type: date }"`
	LastScreenshotID string    `json:"last_screenshot_id,omitempty"`

	PipelineConfigID string      `json:"pipline_config_id,omitempty"`
	HostConfig       *HostConfig `json:"host_config,omitempty"`

	// transient properties
	Snapshots     []Snapshot `json:"-"`
	SnapshotCount int        `json:"-"`
}

type Url struct {
	Domain Domain

	FirstPath  string
	SecondPath string
	ThirdPath  string

	FileExt string
}