Documentation
¶
Index ¶
- Variables
- func URLToString(URL *url.URL) string
- type Item
- func (i *Item) AddChild(child *Item, from ItemState) error
- func (i *Item) CheckConsistency() error
- func (i *Item) Close()
- func (i *Item) CompleteAndCheck() bool
- func (i *Item) DedupeItems() error
- func (i *Item) DrawTree() string
- func (i *Item) DrawTreeWithStatus() string
- func (i *Item) GetChildren() []*Item
- func (i *Item) GetDepth() int64
- func (i *Item) GetDepthWithoutRedirections() int64
- func (i *Item) GetError() error
- func (i *Item) GetID() string
- func (i *Item) GetMaxDepth() int64
- func (i *Item) GetNodesAtLevel(targetLevel int64) ([]*Item, error)
- func (i *Item) GetParent() *Item
- func (i *Item) GetSeed() *Item
- func (i *Item) GetSeedVia() string
- func (i *Item) GetShortID() string
- func (i *Item) GetSource() ItemSource
- func (i *Item) GetStatus() ItemState
- func (i *Item) GetURL() *URL
- func (i *Item) HasChildren() bool
- func (i *Item) HasRedirection() bool
- func (i *Item) HasWork() bool
- func (i *Item) IsChild() bool
- func (i *Item) IsRedirection() bool
- func (i *Item) IsSeed() bool
- func (parent *Item) RemoveChild(child *Item)
- func (i *Item) SetError(err error)
- func (i *Item) SetSource(source ItemSource) error
- func (i *Item) SetStatus(status ItemState)
- func (i *Item) Traverse(fn func(*Item))
- type ItemSource
- type ItemState
- type URL
- func (u *URL) GetBase() *url.URL
- func (u *URL) GetBody() spooledtempfile.ReadSeekCloser
- func (u *URL) GetDocumentCache() *goquery.Document
- func (u *URL) GetDocumentEncoding() encoding.Encoding
- func (u *URL) GetHops() int
- func (u *URL) GetMIMEType() *mimetype.MIME
- func (u *URL) GetParsed() *url.URL
- func (u *URL) GetRedirects() int
- func (u *URL) GetRequest() *http.Request
- func (u *URL) GetResponse() *http.Response
- func (u *URL) Parse() (err error)
- func (u *URL) RewindBody()
- func (u *URL) SetBase(base *url.URL)
- func (u *URL) SetBody(body spooledtempfile.ReadSeekCloser)
- func (u *URL) SetDocumentCache(doc *goquery.Document)
- func (u *URL) SetDocumentEncoding(enc encoding.Encoding)
- func (u *URL) SetHops(hops int)
- func (u *URL) SetMIMEType(mimetype *mimetype.MIME)
- func (u *URL) SetRequest(r *http.Request)
- func (u *URL) SetResponse(r *http.Response)
- func (u *URL) String() string
Constants ¶
This section is empty.
Variables ¶
var ( ErrNotASeed = errors.New("item is not a seed") ErrFailedAtPreprocessor = errors.New("item failed at preprocessor") ErrFailedAtArchiver = errors.New("item failed at archiver") ErrFailedAtPostprocessor = errors.New("item failed at postprocessor") ErrItemNil = errors.New("item is nil") ErrItemURLNil = errors.New("url is nil") ErrItemIDEmpty = errors.New("id is empty") ErrItemChildHasSeedVia = errors.New("item is a child but has a seedVia") ErrItemFreshHasChildren = errors.New("item is fresh but has children") ErrItemFreshParentInvalid = errors.New("item is not a seed and fresh but parent is not ItemGotChildren or ItemGotRedirected") ErrItemRedirectedHasMultipleChildren = errors.New("item has more than one child but is ItemGotRedirected") ErrItemChildrenStateInvalid = errors.New("item has children but is not ItemGotChildren, ItemGotRedirected, ItemCompleted or ItemFailed") )
Errors definition
Functions ¶
func URLToString ¶
URLToString exists to apply some custom stuff, in opposition of simply using the u.parsed.String() method
Types ¶
type Item ¶
type Item struct {
// contains filtered or unexported fields
}
Item represents a URL, it's children (e.g. discovered assets) and it's state in the pipeline The children follow a tree structure where the seed is the root and the children are the leaves, this is to keep track of the hops and the origin of the children
func NewItemWithID ¶
NewItem creates a new item with the given ID, URL and seedVia
func (*Item) AddChild ¶
AddChild adds a child to the item The from parameter is used to set the status of the parent (either ItemGotRedirected or ItemGotChildren)
func (*Item) CheckConsistency ¶
CheckConsistency checks if the item is consistent with the constraints of the model Developers should add more constraints as needed Ideally this function should be called after every mutation of an item object to ensure consistency and throw a panic if consistency is broken
func (*Item) CompleteAndCheck ¶
CompleteAndCheck traverse the seed's tree to complete the items and returns true if the seed is completed
func (*Item) DedupeItems ¶
DedupeItems dedupes items from any level, keeping in priority a Completed item
func (*Item) DrawTreeWithStatus ¶
DrawTree generates the ASCII representation of the tree
func (*Item) GetChildren ¶
GetChildren returns the children of the item
func (*Item) GetDepthWithoutRedirections ¶
func (*Item) GetMaxDepth ¶
GetMaxDepth returns the maxDepth of the item by traversing the tree
func (*Item) GetNodesAtLevel ¶
GetNodesAtLevel returns all the nodes at a given level in the seed
Can be paired with item.GetMaxDepth() to get all the items at the max depth (i.e.: all the items that potentially need work)
Returns ErrNotASeed as error if the item is not a seed
func (*Item) GetSeedVia ¶
GetSeedVia returns the seedVia of the item
func (*Item) GetShortID ¶
GetShortID returns the short ID of the item
func (*Item) GetSource ¶
func (i *Item) GetSource() ItemSource
GetSource returns the source of the item
func (*Item) HasChildren ¶
HasChildren returns true if the item has children
func (*Item) HasRedirection ¶
HasRedirection returns true if the item has a redirection
func (*Item) IsRedirection ¶
IsRedirection returns true if the item is from a redirection
func (*Item) RemoveChild ¶
RemoveChild removes a child from the item
func (*Item) SetSource ¶
func (i *Item) SetSource(source ItemSource) error
SetSource sets the source of the item
type ItemSource ¶
type ItemSource int64
ItemSource qualifies the source of a item in the pipeline
const ( // ItemSourceInsert is for items which source is not defined when inserted on reactor ItemSourceInsert ItemSource = iota // ItemSourceQueue is for items that are from the Queue ItemSourceQueue // ItemSourceHQ is for items that are from the HQ ItemSourceHQ // ItemSourcePostprocess is for items generated from redirections ItemSourcePostprocess // ItemSourceFeedback is for items that are from the Feedback ItemSourceFeedback )
type ItemState ¶
type ItemState int64
ItemState qualifies the state of a item in the pipeline
const ( // ItemFresh is the initial state of a item either it's from HQ, the Queue or Feedback ItemFresh ItemState = iota // ItemPreProcessed is the state after the item has been pre-processed ItemPreProcessed // ItemArchived is the state after the item has been archived ItemArchived // ItemFailed is the state after the item has failed ItemFailed // ItemCompleted is the state after the item has been completed ItemCompleted // ItemSeen is the state given to an item that has been seen before and is not going to be processed ItemSeen // ItemGotRedirected is the state after the item has been redirected ItemGotRedirected // ItemGotChildren is the state after the item has got children ItemGotChildren )
type URL ¶
type URL struct {
Raw string
Hops int // This determines the number of hops this item is the result of, a hop is a "jump" from 1 page to another page
Redirects int
// contains filtered or unexported fields
}
func NewURL ¶
NewURL parses a raw URL string and returns a URL object. If the URL is invalid, it returns a URL object with the raw string and an error.
func (*URL) GetBody ¶
func (u *URL) GetBody() spooledtempfile.ReadSeekCloser
func (*URL) GetDocumentCache ¶
func (*URL) GetDocumentEncoding ¶
func (*URL) GetMIMEType ¶
if mimetype is not set, try to get it from Content-Type header and cache it.
WARN: Remember to check for nil return values
func (*URL) GetRedirects ¶
func (*URL) GetRequest ¶
func (*URL) GetResponse ¶
NOTE:
In normal mode, GetResponse() always returns a valid *http.Response In headless mode, GetResponse() always returns nil
func (*URL) RewindBody ¶
func (u *URL) RewindBody()
func (*URL) SetBody ¶
func (u *URL) SetBody(body spooledtempfile.ReadSeekCloser)
Source Files