model

type Aggregation struct {
	Buckets []Bucket `json:"buckets,omitempty"`
}

type Bucket struct {
	Key      string `json:"key,omitempty"`
	DocCount int    `json:"doc_count,omitempty"`
}

type Context struct {
	Parameters

	SequenceID   int64       `json:"sequence"`
	Simulate     bool        `json:"simulate"`
	IgnoreBroken bool        `json:"ignore_broken"`
	Payload      interface{} `json:"-"`
	// contains filtered or unexported fields
}

type Host struct {
	Host        string        `json:"host,omitempty" gorm:"not null;unique;primary_key" index:"id"`
	Favicon     string        `json:"favicon,omitempty"`
	Enabled     bool          `json:"enabled"`
	HostConfigs *[]HostConfig `json:"host_configs,omitempty"`
	Created     *time.Time    `json:"created,omitempty"`
	Updated     *time.Time    `json:"updated,omitempty"`
}

type HostConfig struct {
	ID         string `json:"id,omitempty" gorm:"not null;unique;primary_key" index:"id"`
	Host       string `gorm:"index" json:"host"`
	UrlPattern string `gorm:"index" json:"url_pattern"`
	Runner     string `gorm:"index" json:"runner"`
	SortOrder  int    `gorm:"index" json:"sort_order"`

	PipelineID string `gorm:"index" json:"pipeline_id"`
	Cookies    string `json:"cookies,omitempty"`

	Created time.Time `gorm:"index" json:"created,omitempty"`
	Updated time.Time `gorm:"index" json:"updated,omitempty"`
}

type IndexDocument struct {
	Index     string                   `json:"_index,omitempty"`
	ID        string                   `json:"_id,omitempty"`
	Source    map[string]interface{}   `json:"_source,omitempty"`
	Highlight map[string][]interface{} `json:"highlight,omitempty"`
}

type Joint interface {
	Name() string
	Process(s *Context) error
}

type JointConfig struct {
	JointName  string                 `json:"joint" config:"joint"`                     //the joint name
	Parameters map[string]interface{} `json:"parameters,omitempty" config:"parameters"` //kv parameters for this joint
	Enabled    bool                   `json:"enabled" config:"enabled"`
}

type KV struct {
	Key   string   `json:"key,omitempty"`
	Value []string `storm:"inline" json:"value,omitempty"`
}

type LinkGroup struct {
	Internal []PageLink `json:"internal,omitempty"`
	External []PageLink `json:"external,omitempty"`
}

type PageLink struct {
	Url   string `json:"url,omitempty"`
	Label string `json:"label,omitempty"`
}

type ParaKey string

type Parameters struct {
	Data map[string]interface{} `json:"data,omitempty"`
	// contains filtered or unexported fields
}

type Pipeline struct {
	// contains filtered or unexported fields
}

type PipelineConfig struct {
	ID            string         `json:"id,omitempty" index:"id"`
	Name          string         `json:"name,omitempty" config:"name"`
	StartJoint    *JointConfig   `gorm:"-" json:"start,omitempty" config:"start"`
	ProcessJoints []*JointConfig `gorm:"-" json:"process,omitempty" config:"process"`
	EndJoint      *JointConfig   `gorm:"-" json:"end,omitempty" config:"end"`
	Created       *time.Time     `json:"created,omitempty"`
	Updated       *time.Time     `json:"updated,omitempty"`
	Tags          []string       `gorm:"-" json:"tags,omitempty" config:"tags"`
}

type Project struct {
	ID          string    `storm:"id,unique" json:"id,omitempty" gorm:"not null;unique;primary_key" index:"id"`
	Name        string    `json:"name,omitempty"`
	Description string    `json:"description,omitempty"`
	Enabled     bool      `json:"enabled"`
	Created     time.Time `json:"created,omitempty"`
	Updated     time.Time `json:"updated,omitempty"`
}

type Snapshot struct {
	ID      string `json:"id,omitempty" gorm:"not null;unique;primary_key" index:"id"`
	Version int    `json:"version,omitempty"`
	Url     string `json:"url,omitempty"`
	TaskID  string `json:"task_id,omitempty"`
	Path    string `json:"path,omitempty"  gorm:"-"` //path of this file
	File    string `json:"file,omitempty"  gorm:"-"` //filename of this page
	Ext     string `json:"ext,omitempty"  gorm:"-"`  //extension of filename

	StatusCode int    `json:"-" gorm:"-"`
	Payload    []byte `json:"-" gorm:"-"`
	Size       uint64 `json:"size,omitempty"`

	ScreenshotID string `json:"screenshot_id,omitempty"`

	Headers    map[string][]string     `json:"-" gorm:"-"`
	Metadata   *map[string]interface{} `json:"-" gorm:"-"`
	Parameters []KV                    `json:"-" gorm:"-"`

	Language string `json:"lang,omitempty" gorm:"-"`

	Title       string `json:"title,omitempty"`
	Summary     string `json:"summary,omitempty" gorm:"-"`
	Text        string `json:"text,omitempty" gorm:"-"`
	ContentType string `json:"content_type,omitempty"`

	Tags []string `json:"tags,omitempty" gorm:"-"`

	Links LinkGroup `json:"links,omitempty" gorm:"-"`

	Images struct {
		Internal []PageLink `json:"internal,omitempty"`
		External []PageLink `json:"external,omitempty"`
	} `json:"images,omitempty" gorm:"-"`

	H1     []string `json:"h1,omitempty" gorm:"-"`
	H2     []string `json:"h2,omitempty" gorm:"-"`
	H3     []string `json:"h3,omitempty" gorm:"-"`
	H4     []string `json:"h4,omitempty" gorm:"-"`
	H5     []string `json:"h5,omitempty" gorm:"-"`
	Bold   []string `json:"bold,omitempty" gorm:"-"`
	Italic []string `json:"italic,omitempty" gorm:"-"`

	Classifications  []string                `json:"classifications,omitempty" gorm:"-"`
	EnrichedFeatures *map[string]interface{} `json:"enriched_features,omitempty" gorm:"-"`

	Hash    string `json:"hash,omitempty"`
	SimHash string `json:"sim_hash,omitempty"`

	Created time.Time `json:"created,omitempty"`
}

type Task struct {
	ID string `gorm:"not null;unique;primary_key" json:"id" index:"id"`
	// the url may not cleaned, may miss the host part, need reference to provide the complete url information
	Url         string    `storm:"index" json:"url,omitempty" gorm:"type:varchar(500)"`
	Reference   string    `json:"reference_url,omitempty"`
	Depth       int       `storm:"index" json:"depth"`
	Breadth     int       `storm:"index" json:"breadth"`
	Host        string    `gorm:"index" json:"host"`
	Schema      string    `json:"schema,omitempty"`
	OriginalUrl string    `json:"original_url,omitempty"`
	Status      int       `gorm:"index" json:"status"`
	Message     string    `json:"message,omitempty"`
	Created     time.Time `gorm:"index" json:"created,omitempty"`
	Updated     time.Time `gorm:"index" json:"updated,omitempty"`
	LastFetch   time.Time `gorm:"index" json:"last_fetch,omitempty"`
	LastCheck   time.Time `gorm:"index" json:"last_check,omitempty"`
	NextCheck   time.Time `gorm:"index" json:"next_check,omitempty"`

	SnapshotVersion  int       `json:"snapshot_version,omitempty"`
	SnapshotID       string    `json:"snapshot_id,omitempty"`
	SnapshotHash     string    `json:"snapshot_hash,omitempty"`
	SnapshotSimHash  string    `json:"snapshot_simhash,omitempty"`
	SnapshotCreated  time.Time `json:"snapshot_created,omitempty"`
	LastScreenshotID string    `json:"last_screenshot_id,omitempty"`

	PipelineConfigID string      `json:"pipline_config_id,omitempty"`
	HostConfig       *HostConfig `json:"host_config,omitempty"`

	// transient properties
	Snapshots     []Snapshot `json:"-"`
	SnapshotCount int        `json:"-"`
}