model

package

Go to main page

Versions in this module

v0

v0.10.0

Feb 20, 2018

Changes in this version

+ const PERMISSION_ADMIN_MINIMAL

+ const PERMISSION_SNAPSHOT_VIEW

+ const PipelineConfigBucket

+ const ROLE_ADMIN

+ const ROLE_GUEST

+ const TaskPendingFetch

+ func CreateHostConfig(config *HostConfig) error

+ func CreatePipelineConfig(cfg *PipelineConfig) error

+ func CreateProject(project *Project) error

+ func DeleteHostConfig(id string) error

+ func DeletePipelineConfig(id string) error

+ func DeleteProject(id string) error

+ func GetAllRegisteredJoints() map[string]interface

+ func GetHostStatus(status int) (error, map[string]interface{})

+ func GetPermissionsByRole(role string) (*hashset.Set, error)

+ func GetTaskStatus(host string) (error, map[string]interface{})

+ func RegisterPipeJoint(joint Joint)

+ func RegisterPipeJointWithName(jointName string, joint Joint)

+ func UpdateHostConfig(config *HostConfig) error

+ func UpdatePipelineConfig(id string, cfg *PipelineConfig) error

+ func UpdateProject(project *Project) error

+ type Context struct

+ IgnoreBroken bool

+ Payload interface{}

+ SequenceID int64

+ Simulate bool

+ func (context *Context) End(msg interface{})

+ func (context *Context) Exit(msg interface{})

+ func (context *Context) IsEnd() bool

+ func (context *Context) IsExit() bool

+ type Host struct

+ Created *time.Time

+ Enabled bool

+ Favicon string

+ Host string

+ HostConfigs *[]HostConfig

+ Updated *time.Time

+ func CreateHost(host string) Host

+ func GetHost(host string) (Host, error)

+ func GetHostList(from, size int, host string) (int, []Host, error)

+ type HostConfig struct

+ Cookies string

+ Created time.Time

+ Host string

+ ID string

+ PipelineID string

+ Runner string

+ SortOrder int

+ Updated time.Time

+ UrlPattern string

+ func GetHostConfig(runner, host string) []HostConfig

+ func GetHostConfigByHostAndUrl(runner, host, url string) (*HostConfig, error)

+ func GetHostConfigByID(id string) (HostConfig, error)

+ func GetHostConfigList(from, size int, host string) (int, []HostConfig, error)

type IndexDocument

+ ID string

+ type Joint interface

+ Name func() string

+ Process func(s *Context) error

+ func GetJointInstance(cfg *JointConfig) Joint

+ type JointConfig struct

+ Enabled bool

+ JointName string

+ Parameters map[string]interface{}

+ type ParaKey string

+ const CONTEXT_PAGE_LINKS

+ const CONTEXT_SNAPSHOT

+ const CONTEXT_SNAPSHOT_ContentType

+ const CONTEXT_TASK_Breadth

+ const CONTEXT_TASK_Cookies

+ const CONTEXT_TASK_Created

+ const CONTEXT_TASK_Depth

+ const CONTEXT_TASK_Host

+ const CONTEXT_TASK_ID

+ const CONTEXT_TASK_LastCheck

+ const CONTEXT_TASK_LastFetch

+ const CONTEXT_TASK_LastScreenshotID

+ const CONTEXT_TASK_Message

+ const CONTEXT_TASK_NextCheck

+ const CONTEXT_TASK_OriginalUrl

+ const CONTEXT_TASK_PipelineConfigID

+ const CONTEXT_TASK_Reference

+ const CONTEXT_TASK_Schema

+ const CONTEXT_TASK_SnapshotCreated

+ const CONTEXT_TASK_SnapshotHash

+ const CONTEXT_TASK_SnapshotID

+ const CONTEXT_TASK_SnapshotSimHash

+ const CONTEXT_TASK_SnapshotVersion

+ const CONTEXT_TASK_Status

+ const CONTEXT_TASK_URL

+ const CONTEXT_TASK_Updated

+ type Parameters struct

+ Data map[string]interface{}

+ func (para *Parameters) Get(key ParaKey) interface{}

+ func (para *Parameters) GetBool(key ParaKey, defaultV bool) bool

+ func (para *Parameters) GetInt(key ParaKey, defaultV int) (int, bool)

+ func (para *Parameters) GetInt64(key ParaKey, defaultV int64) (int64, bool)

+ func (para *Parameters) GetInt64OrDefault(key ParaKey, defaultV int64) int64

+ func (para *Parameters) GetIntOrDefault(key ParaKey, defaultV int) int

+ func (para *Parameters) GetMap(key ParaKey) (map[string]interface{}, bool)

+ func (para *Parameters) GetOrDefault(key ParaKey, val interface{}) interface{}

+ func (para *Parameters) GetString(key ParaKey) (string, bool)

+ func (para *Parameters) GetStringArray(key ParaKey) ([]string, bool)

+ func (para *Parameters) GetStringOrDefault(key ParaKey, val string) string

+ func (para *Parameters) GetTime(key ParaKey) (time.Time, bool)

+ func (para *Parameters) Has(key ParaKey) bool

+ func (para *Parameters) MustGet(key ParaKey) interface{}

+ func (para *Parameters) MustGetBytes(key ParaKey) []byte

+ func (para *Parameters) MustGetInt(key ParaKey) int

+ func (para *Parameters) MustGetInt64(key ParaKey) int64

+ func (para *Parameters) MustGetMap(key ParaKey) map[string]interface{}

+ func (para *Parameters) MustGetString(key ParaKey) string

+ func (para *Parameters) MustGetTime(key ParaKey) time.Time

+ func (para *Parameters) Set(key ParaKey, value interface{})

+ type Pipeline struct

+ func NewPipeline(name string) *Pipeline

+ func NewPipelineFromConfig(name string, config *PipelineConfig, context *Context) *Pipeline

+ func (pipe *Pipeline) Context(s *Context) *Pipeline

+ func (pipe *Pipeline) End(s Joint) *Pipeline

+ func (pipe *Pipeline) GetContext() *Context

+ func (pipe *Pipeline) GetCurrentJoint() string

+ func (pipe *Pipeline) GetID() string

+ func (pipe *Pipeline) Join(s Joint) *Pipeline

+ func (pipe *Pipeline) Run() *Context

+ func (pipe *Pipeline) Start(s Joint) *Pipeline

+ type PipelineConfig struct

+ Created *time.Time

+ EndJoint *JointConfig

+ ID string

+ Name string

+ ProcessJoints []*JointConfig

+ StartJoint *JointConfig

+ Tags []string

+ Updated *time.Time

+ func GetPipelineConfig(id string) (*PipelineConfig, error)

+ func GetPipelineList(from, size int) (int, []PipelineConfig, error)

+ type Project struct

+ Created time.Time

+ Description string

+ Enabled bool

+ ID string

+ Name string

+ Updated time.Time

+ func GetProject(id string) (Project, error)

+ func GetProjectList(from, size int) (int, []Project, error)

type Snapshot

+ Ext string

+ ScreenshotID string

type Task

+ Breadth int

+ Depth int

+ HostConfig *HostConfig

+ LastScreenshotID string

+ PipelineConfigID string

+ Reference string

+ SnapshotCount int

+ Snapshots []Snapshot

+ Url string

+ func GetFailedTasks(offset time.Time) (int, []Task, error)

+ func NewTask(url, ref string, depth int, breadth int) *Task

v0.9.0

Oct 20, 2017

Changes in this version

+ func DeleteSnapshot(snapshot *Snapshot) error

+ func GetTaskStatusText(status TaskStatus) string

+ type Aggregation struct

+ Buckets []Bucket

+ type Bucket struct

+ DocCount int

+ Key string

type Domain

+ Created *time.Time

+ Updated *time.Time

type IndexDocument

+ Highlight map[string][]interface{}

type Snapshot

+ Created *time.Time

+ func GetSnapshot(id string) (Snapshot, error)

+ func GetSnapshotByField(k, v string) ([]Snapshot, error)

+ func GetSnapshotList(from, size int, taskId string) (int, []Snapshot, error)

type Task

+ Created *time.Time

+ LastCheck *time.Time

+ LastFetch *time.Time

+ NextCheck *time.Time

+ SnapshotCreated *time.Time

+ Updated *time.Time

+ type TaskSetting struct

+ EnabledJoints []string

type TaskStatus

+ const Task404

+ const TaskDuplicated

+ const TaskFailed

+ const TaskInterrupted

+ const TaskRedirected

+ const TaskSuccess

+ const TaskTimeout

v0.8.0

Jun 24, 2017

Changes in this version

+ func CreateSnapshot(snapshot *Snapshot) error

+ func CreateTask(task *Task) error

+ func DeleteTask(id string) error

+ func IncrementDomainLinkCount(host string) error

+ func UpdateTask(task *Task)

+ type Domain struct

+ CreateTime *time.Time

+ Favicon string

+ Host string

+ LinksCount int64

+ Settings *DomainSetting

+ UpdateTime *time.Time

+ func CreateDomain(host string) Domain

+ func GetDomain(domain string) (Domain, error)

+ func GetDomainList(from, size int, domain string) (int, []Domain, error)

+ type DomainSetting struct

+ type IndexDocument struct

+ Id string

+ Index string

+ Source map[string]interface{}

+ Type string

+ type KV struct

+ Key string

+ Value []string

+ type LinkGroup struct

+ External []PageLink

+ Internal []PageLink

+ type PageLink struct

+ Label string

+ Url string

+ type Seed struct

+ Breadth int

+ Depth int

+ Reference string

+ Url string

+ func NewTaskSeed(url, ref string, depth int, breadth int) Seed

+ func TaskSeedFromBytes(b []byte) Seed

+ func (this Seed) Get(url string) Seed

+ func (this Seed) GetBytes() ([]byte, error)

+ func (this Seed) MustGetBytes() []byte

+ type Snapshot struct

+ Bold []string

+ Classifications []string

+ ContentType string

+ CreateTime *time.Time

+ EnrichedFeatures *map[string]interface{}

+ File string

+ H1 []string

+ H2 []string

+ H3 []string

+ H4 []string

+ H5 []string

+ Hash string

+ Headers map[string][]string

+ ID string

+ Images struct{ ... }

+ Italic []string

+ Language string

+ Links LinkGroup

+ Metadata *map[string]interface{}

+ Parameters []KV

+ Path string

+ Payload []byte

+ SimHash string

+ Size uint64

+ StatusCode int

+ Summary string

+ Tags []string

+ TaskID string

+ Text string

+ Title string

+ Url string

+ Version int

+ type Task struct

+ CreateTime *time.Time

+ Host string

+ ID string

+ LastCheckTime *time.Time

+ LastFetchTime *time.Time

+ Message string

+ NextCheckTime *time.Time

+ OriginalUrl string

+ Phrase pipeline.Phrase

+ Schema string

+ SnapshotCreateTime *time.Time

+ SnapshotHash string

+ SnapshotID string

+ SnapshotSimHash string

+ SnapshotVersion int

+ Status TaskStatus

+ UpdateTime *time.Time

+ func GetPendingNewFetchTasks() (int, []Task, error)

+ func GetPendingUpdateFetchTasks(offset *time.Time) (int, []Task, error)

+ func GetTask(id string) (Task, error)

+ func GetTaskByField(k, v string) (Task, error)

+ func GetTaskList(from, size int, domain string) (int, []Task, error)

+ type TaskStatus int

+ const Task404Ignore

+ const TaskCreated

+ const TaskFetchFailed

+ const TaskFetchSuccess

+ const TaskFetchTimeout

+ const TaskRedirectedIgnore

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL