models

package

v0.8.9 Latest Latest Go to latest Published: Oct 23, 2024 License: MIT Imports: 7 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/0x00f00bar/webcrawlerGo

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func PageDelete(id int, query string, db *sql.DB) error
func PageGetLatestPageCount(ctx context.Context, baseurl *url.URL, markedURL string, cutoffDate time.Time, ...) (int, error)
func PageInsert(m *Page, query string, db *sql.DB) error
func URLDelete(id int, query string, db *sql.DB) error
func URLInsert(m *URL, query string, db *sql.DB) error
func URLUpdate(m *URL, query string, db *sql.DB) error
func ValidOrderBy(orderBy string, validFields []string) bool
type Models
type Page
type PageContent
- func PageGetLatestPagesPaginated(ctx context.Context, baseurl *url.URL, markedURL string, cutoffDate time.Time, ...) ([]*PageContent, error)
type PageModel
type URL
type URLModel

Constants ¶

View Source

const (
	QuerySelectPage          = "SELECT id, url_id, added_at, content FROM pages"
	QueryGetPageById         = QuerySelectPage + " WHERE id = __ARG__"
	QueryGetAllPageByURL     = QuerySelectPage + " WHERE url_id = __ARG__ ORDER BY __ARG__"
	QueryInsertPage          = `INSERT INTO pages (url_id, content) VALUES (__ARG__, __ARG__) RETURNING id, added_at`
	QueryDeletePage          = `DELETE from pages WHERE id = __ARG__`
	QueryGetLatestPagesCount = `` /* 352-byte string literal not displayed */

	QueryGetLatestPagesPaginated = `` /* 397-byte string literal not displayed */

)

Queries related to pages table

View Source

const (
	QuerySelectURL   = "SELECT id, url, first_encountered, last_checked, last_saved, is_monitored, is_alive, version FROM urls "
	QueryGetURLById  = QuerySelectURL + "WHERE id = __ARG__"
	QueryGetURLByURL = QuerySelectURL + "WHERE url = __ARG__"
	QueryInsertURL   = `` /* 151-byte string literal not displayed */

	QueryUpdateURL = `` /* 191-byte string literal not displayed */

	QueryDeleteURL          = `DELETE from urls WHERE id = __ARG__`
	QueryGetAllURL          = QuerySelectURL + "ORDER BY __ARG__"
	QueryGetAllMonitoredURL = QuerySelectURL + `
	WHERE is_monitored = true AND is_alive = true
	ORDER BY __ARG__`
)

Queries related to urls table

View Source

const QueryArgStr = "__ARG__"

QueryArgStr is the substring used in sql queries as placeholder for query arguments

Variables ¶

View Source

var (
	ErrRecordNotFound = errors.New("models: record not found")
	ErrNullURL        = errors.New("models: url cannot be empty or null")
	ErrEditConflict   = errors.New("models: edit conflict")
	ErrInvalidOrderBy = errors.New("models: invalid order by")
)

View Source

var DefaultDBTimeout = 5 * time.Second

DefaultDBTimeout for sql queries

View Source

var PageColumns = []string{"id", "url_id", "added_at", "content"}

View Source

var URLColumns = []string{
	"id", "url", "first_encountered", "last_checked",
	"last_saved", "is_monitored", "is_alive", "version",
}

Functions ¶

func PageDelete ¶

func PageDelete(id int, query string, db *sql.DB) error

PageDelete delete page row by id

func PageGetLatestPageCount ¶

func PageGetLatestPageCount(
	ctx context.Context,
	baseurl *url.URL,
	markedURL string,
	cutoffDate time.Time,
	query string,
	db *sql.DB,
) (int, error)

PageGetLatestPageCount returns count of the latest pages filtered by baseurl, markedURL and by cutoff date

func PageInsert ¶

func PageInsert(m *Page, query string, db *sql.DB) error

PageInsert writes a page to pages table

func URLDelete ¶

func URLDelete(id int, query string, db *sql.DB) error

URLDelete url row by id

func URLInsert ¶

func URLInsert(m *URL, query string, db *sql.DB) error

URLInsert writes a url to urls table

func URLUpdate ¶

func URLUpdate(m *URL, query string, db *sql.DB) error

URLUpdate updates a url with provided values. Optimistic locking enabled: if version change detected return ErrEditConflict

func ValidOrderBy ¶

func ValidOrderBy(orderBy string, validFields []string) bool

ValidOrderBy tells if the orderBy is present in validFields

Types ¶

type Models ¶

type Models struct {
	URLs  URLModel
	Pages PageModel
}

Models embeds URLModel and PageModel interface

type Page ¶

type Page struct {
	ID      uint
	URLID   uint
	AddedAt time.Time
	Content string
}

Page type holds the information of URL content saved in model

func NewPage ¶

func NewPage(urlId uint, content string) *Page

NewPage returns new Page type with AddedAt set to current time.

func PageGetAllByURL ¶

func PageGetAllByURL(urlID uint, orderBy string, query string, db *sql.DB) ([]*Page, error)

PageGetAllByURL fetches a row from pages table by urlId and order by orderBy

func PageGetById ¶

func PageGetById(id int, query string, db *sql.DB) (*Page, error)

PageGetById fetches a row from pages table by id

type PageContent ¶

type PageContent struct {
	URL     string
	AddedAt time.Time
	Content string
	// contains filtered or unexported fields
}

PageContent type contains feilds required for saving page contents to disk

func PageGetLatestPagesPaginated ¶

func PageGetLatestPagesPaginated(
	ctx context.Context,
	baseurl *url.URL,
	markedURL string,
	cutoffDate time.Time,
	pageNum int,
	pageSize int,
	query string,
	db *sql.DB,
) ([]*PageContent, error)

PageGetLatestPagesPaginated returns PageContent of the latest pages filtered by baseurl, markedURL and by cutoff date

type PageModel ¶

type PageModel interface {
	GetById(id int) (*Page, error)
	GetAllByURL(urlId uint, orderBy string) ([]*Page, error)
	GetLatestPageCount(
		ctx context.Context,
		baseURL *url.URL,
		markedURL string,
		cutoffDate time.Time,
	) (int, error)
	GetLatestPagesPaginated(
		ctx context.Context,
		baseURL *url.URL,
		markedURL string,
		cutoffDate time.Time,
		pageNum int,
		pageSize int,
	) ([]*PageContent, error)
	Insert(*Page) error
	// Update method is not required, yet
	// Update(*Page) error
	Delete(id int) error
}

type URL ¶

type URL struct {
	ID               uint
	URL              string
	FirstEncountered time.Time
	LastChecked      time.Time
	LastSaved        time.Time
	IsMonitored      bool
	IsAlive          bool
	Version          uint
}

URL type holds the information of URL saved in model

func NewURL ¶

func NewURL(url string, lastChecked, lastSaved time.Time, isMonitored bool) *URL

NewURL returns new URL type with FirstEncountered set to time.Now

func URLGetAll ¶

func URLGetAll(orderBy string, query string, db *sql.DB) ([]*URL, error)

URLGetAll fetches all rows from urls table in orderBy order

func URLGetById ¶

func URLGetById(id int, query string, db *sql.DB) (*URL, error)

URLGetById fetches a row from urls table by id

func URLGetByURL ¶

func URLGetByURL(urlStr string, query string, db *sql.DB) (*URL, error)

URLGetByURL fetches a row from urls table by url string

type URLModel ¶

type URLModel interface {
	GetAll(orderBy string) ([]*URL, error)
	GetAllMonitored(orderBy string) ([]*URL, error)
	GetById(id int) (*URL, error)
	GetByURL(url string) (*URL, error)
	Insert(*URL) error
	Update(*URL) error
	Delete(id int) error
}

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
psql
sqlite

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL