Documentation
¶
Overview ¶
Package goodread is a library for reading public Goodreads data: books, authors, series, lists, quotes, users, shelves, genres, reviews, and search. Every page is normalized into a rich struct (JSON-LD first, HTML fallback), with explicit empty/zero/[] for genuinely absent fields. It carries no CLI knowledge and depends only on goquery and the standard library.
Index ¶
- Constants
- Variables
- func AuthorURL(id string) string
- func BookURL(id string) string
- func Classify(s string) (entity, id string)
- func DefaultDataDir() string
- func DocFromBytes(body []byte) (*goquery.Document, error)
- func ExtractID(rawurl, prefix string) string
- func GenreURL(slug string) string
- func InferEntityType(u string) string
- func ListURL(id string) string
- func LoadCookies(path string) ([]*http.Cookie, error)
- func ParseList(doc *goquery.Document, listID, pageURL string) (*List, []ListBook, error)
- func ParseSearchHTMLNextPage(doc *goquery.Document) string
- func ParseSeries(doc *goquery.Document, seriesID, pageURL string) (*Series, []SeriesBook, error)
- func ParseShelf(doc *goquery.Document, userID, shelfName, pageURL string) (*Shelf, []ShelfBook, error)
- func ParseShelfNextPage(doc *goquery.Document) string
- func ParseShelfRSS(body []byte, userID, shelfName, pageURL string) (*Shelf, []ShelfBook, error)
- func QuoteURL(id string) string
- func SearchAutocompleteURL(q string) string
- func SearchHTMLURL(q string, page int) string
- func SeriesURL(id string) string
- func ShelfRSSURL(userID, shelf string) string
- func ShelfURL(userID, shelf string, page int) string
- func UserURL(id string) string
- type Author
- type Book
- type Cache
- type Client
- func (c *Client) CachingFetch(ctx context.Context, cache *Cache, cfg Config, url string) ([]byte, error)
- func (c *Client) CachingFetchHTML(ctx context.Context, cache *Cache, cfg Config, url string) (*goquery.Document, error)
- func (c *Client) Fetch(ctx context.Context, rawurl string) ([]byte, int, error)
- func (c *Client) FetchHTML(ctx context.Context, rawurl string) (*goquery.Document, int, error)
- func (c *Client) GetAuthor(ctx context.Context, id string) (*Author, error)
- func (c *Client) GetBook(ctx context.Context, id string) (*Book, error)
- func (c *Client) GetGenre(ctx context.Context, slug string) (*Genre, error)
- func (c *Client) GetList(ctx context.Context, id string) (*List, []ListBook, error)
- func (c *Client) GetQuotes(ctx context.Context, pageURL string) ([]Quote, error)
- func (c *Client) GetReviews(ctx context.Context, bookID string) ([]Review, error)
- func (c *Client) GetSeries(ctx context.Context, id string) (*Series, []SeriesBook, error)
- func (c *Client) GetShelf(ctx context.Context, userID, shelfName string, maxPages int) (*Shelf, []ShelfBook, error)
- func (c *Client) GetShelfRSS(ctx context.Context, userID, shelfName string) (*Shelf, []ShelfBook, error)
- func (c *Client) GetUser(ctx context.Context, id string) (*User, error)
- func (c *Client) RobotsSitemaps(ctx context.Context) ([]SitemapCategory, error)
- func (c *Client) Search(ctx context.Context, query string, limit int) ([]SearchResult, error)
- func (c *Client) SearchBooks(ctx context.Context, query string, limit int) ([]Book, error)
- func (c *Client) SearchHTML(ctx context.Context, query string, limit int) ([]SearchResult, error)
- func (c *Client) SitemapIndex(ctx context.Context, indexURL string) ([]string, error)
- func (c *Client) SitemapURLs(ctx context.Context, sitemapURL string) ([]string, error)
- type Config
- type Domain
- type Genre
- type List
- type ListBook
- type QueueItem
- type Quote
- type Review
- type SearchResult
- type Series
- type SeriesBook
- type Shelf
- type ShelfBook
- type SitemapCategory
- type Store
- func (s *Store) Close() error
- func (s *Store) Count(entityType string) (int, error)
- func (s *Store) CountsByType() (map[string]int, error)
- func (s *Store) Each(entityType string, fn func(id string, data []byte) error) error
- func (s *Store) Enqueue(ctx context.Context, url, entityType string, priority int) error
- func (s *Store) Get(entityType, id string) ([]byte, error)
- func (s *Store) Info() (string, error)
- func (s *Store) MarkFailed(ctx context.Context, id int64) error
- func (s *Store) MarkFetched(ctx context.Context, id int64, htmlPath string) error
- func (s *Store) NextPending(ctx context.Context, n int) ([]QueueItem, error)
- func (s *Store) Put(entityType, id, url string, record any) error
- func (s *Store) QueueStats() (map[string]int, error)
- func (s *Store) ResetActive() error
- func (s *Store) Vacuum() error
- type User
Constants ¶
const ( BaseURL = "https://www.goodreads.com" RobotsTxtURL = "https://www.goodreads.com/robots.txt" DefaultDelay = 2 * time.Second DefaultWorkers = 2 DefaultTimeout = 30 * time.Second DefaultRetries = 3 DefaultCacheTTL = 24 * time.Hour )
Site constants.
Variables ¶
var ErrBlocked = errors.New("blocked: page requires sign-in")
ErrBlocked signals that Goodreads served a sign-in wall (or otherwise refused the page). Callers map it to the "blocked" exit code and print a hint.
var ErrNotFound = fmt.Errorf("not found")
ErrNotFound is returned when a page 404s.
var ErrRateLimited = errors.New("rate limited (HTTP 429)")
ErrRateLimited signals a sustained HTTP 429 after retries.
Functions ¶
func Classify ¶
Classify inspects any Goodreads URL or bare id and returns (entity, id). A bare numeric id is treated as a book. Returns ("","") when unrecognizable.
func DefaultDataDir ¶
func DefaultDataDir() string
DefaultDataDir resolves $XDG_DATA_HOME/goodread (or ~/.local/share/goodread).
func DocFromBytes ¶
DocFromBytes parses raw HTML bytes into a goquery document.
func InferEntityType ¶
InferEntityType maps a URL to an entity type, defaulting to book.
func LoadCookies ¶
LoadCookies reads a Netscape/Mozilla cookies.txt file (the format exported by most browser extensions and by curl) into http.Cookie values.
func ParseList ¶
ParseList parses a Listopia list page. The list title, books count, voters, creator, and tags live in the ".stacked" footer block; each book row is a schema.org/Book table row carrying rank, rating, score, and votes.
func ParseSearchHTMLNextPage ¶
ParseSearchHTMLNextPage returns the next search page URL, or "".
func ParseSeries ¶
ParseSeries parses a Goodreads series page. The page is a legacy React view whose data lives in data-react-props attributes (SeriesHeader for the title, subtitle, and description; one or more SeriesList blocks for the books), with a visible-text fallback for the rare page that lacks them.
func ParseShelf ¶
func ParseShelf(doc *goquery.Document, userID, shelfName, pageURL string) (*Shelf, []ShelfBook, error)
ParseShelf parses a user's shelf page into ShelfBook rows.
func ParseShelfNextPage ¶
ParseShelfNextPage returns the next shelf page URL, or "".
func ParseShelfRSS ¶
ParseShelfRSS parses a shelf RSS feed into a shelf header and its rows.
func SearchAutocompleteURL ¶
SearchAutocompleteURL builds the JSON autocomplete endpoint URL.
func SearchHTMLURL ¶
SearchHTMLURL builds the full HTML search URL for a page.
func ShelfRSSURL ¶
ShelfRSSURL builds the RSS-feed form of a user's shelf.
Types ¶
type Author ¶
type Author struct {
AuthorID string `json:"author_id" kit:"id"`
Name string `json:"name"`
Bio string `json:"bio" kit:"body"`
PhotoURL string `json:"photo_url"`
Website string `json:"website"`
BornDate string `json:"born_date"`
DiedDate string `json:"died_date"`
Hometown string `json:"hometown"`
Influences []string `json:"influences"`
Genres []string `json:"genres"`
AvgRating float64 `json:"avg_rating"`
RatingsCount int64 `json:"ratings_count"`
ReviewsCount int64 `json:"reviews_count"`
BooksCount int `json:"books_count"`
FollowersCount int `json:"followers_count"`
NotableBookIDs []string `json:"notable_book_ids" kit:"link,kind=goodreads/book,optional"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
Author is a Goodreads author page (/author/show/<id>).
type Book ¶
type Book struct {
BookID string `json:"book_id" kit:"id"`
WorkID string `json:"work_id"`
Title string `json:"title"`
TitleWithoutSeries string `json:"title_without_series"`
Description string `json:"description" kit:"body"`
AuthorID string `json:"author_id" kit:"link,kind=goodreads/author"`
AuthorName string `json:"author_name"`
ISBN string `json:"isbn"`
ISBN13 string `json:"isbn13"`
ASIN string `json:"asin"`
AvgRating float64 `json:"avg_rating"`
RatingsCount int64 `json:"ratings_count"`
ReviewsCount int64 `json:"reviews_count"`
PublishedYear int `json:"published_year"`
Publisher string `json:"publisher"`
Language string `json:"language"`
Pages int `json:"pages"`
Format string `json:"format"`
SeriesID string `json:"series_id" kit:"link,kind=goodreads/series,optional"`
SeriesName string `json:"series_name"`
SeriesPosition string `json:"series_position"`
Genres []string `json:"genres"`
CoverURL string `json:"cover_url"`
URL string `json:"url"`
SimilarBookIDs []string `json:"similar_book_ids" kit:"link,kind=goodreads/book,optional"`
FetchedAt time.Time `json:"fetched_at"`
}
Book is a Goodreads book page (/book/show/<id>).
func ParseAutocompleteBooks ¶
ParseAutocompleteBooks parses the rich JSON autocomplete response into Book records. The autocomplete endpoint is open (not WAF-challenged), so it is the primary way to obtain structured book data without a lent session.
type Cache ¶
type Cache struct {
// contains filtered or unexported fields
}
Cache is a content-addressed, gzipped on-disk page cache. A captured page is stored under cache/<ab>/<sha256>.html.gz, keyed by its URL.
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client performs polite, retrying HTTP GETs against goodreads.com.
func NewClientWithCookies ¶
NewClientWithCookies builds a client pre-loaded with a lent session.
func (*Client) CachingFetch ¶
func (c *Client) CachingFetch(ctx context.Context, cache *Cache, cfg Config, url string) ([]byte, error)
CachingFetch returns page bytes, reading the cache unless NoCache/Refresh is set, and populating it on a miss.
func (*Client) CachingFetchHTML ¶
func (c *Client) CachingFetchHTML(ctx context.Context, cache *Cache, cfg Config, url string) (*goquery.Document, error)
CachingFetchHTML is CachingFetch plus goquery parsing.
func (*Client) Fetch ¶
Fetch returns the raw body and status for a URL, retrying transient failures. A 404 returns (nil, 404, nil). A sign-in redirect returns ErrBlocked.
func (*Client) GetQuotes ¶
GetQuotes fetches and parses a quotes page (an author's or a book's quotes URL).
func (*Client) GetReviews ¶
GetReviews fetches a book page and returns the reviews embedded on it.
func (*Client) GetShelf ¶
func (c *Client) GetShelf(ctx context.Context, userID, shelfName string, maxPages int) (*Shelf, []ShelfBook, error)
GetShelf walks a user's shelf, following pagination up to maxPages (maxPages <= 0 means all pages). It returns the shelf header and every row.
func (*Client) GetShelfRSS ¶
func (c *Client) GetShelfRSS(ctx context.Context, userID, shelfName string) (*Shelf, []ShelfBook, error)
GetShelfRSS fetches a user's shelf via the open RSS feed (no WAF challenge).
func (*Client) RobotsSitemaps ¶
func (c *Client) RobotsSitemaps(ctx context.Context) ([]SitemapCategory, error)
RobotsSitemaps fetches robots.txt and returns the advertised siteindex URLs.
func (*Client) SearchBooks ¶
SearchBooks returns rich Book records from the open autocomplete endpoint.
func (*Client) SearchHTML ¶
SearchHTML walks the full HTML search results, following pagination until limit results are collected (limit <= 0 means a single page).
func (*Client) SitemapIndex ¶
SitemapIndex fetches a <sitemapindex> document and returns its child URLs.
type Config ¶
type Config struct {
DataDir string // root for cache/store; defaults to XDG data dir
Workers int // concurrency for multi-page/bulk work
Delay time.Duration // minimum spacing between requests
Timeout time.Duration // per-request timeout
Retries int // retry attempts on 429/5xx
CacheTTL time.Duration // on-disk page-cache freshness window
NoCache bool // bypass the on-disk page cache entirely
Refresh bool // force re-fetch and overwrite the cache
CookiePath string // optional Netscape cookie jar for lent sessions
}
Config holds runtime configuration for the library.
func DefaultConfig ¶
func DefaultConfig() Config
DefaultConfig returns sensible, polite defaults rooted at the XDG data dir.
type Domain ¶ added in v0.2.0
type Domain struct{}
Domain is the Goodreads driver. It carries no state; the per-run client is built by the factory Register hands kit.
func (Domain) Classify ¶ added in v0.2.0
Classify turns any accepted input into the canonical (type, id), so `ant resolve` and `ant url` need no network.
func (Domain) Info ¶ added in v0.2.0
func (Domain) Info() kit.DomainInfo
Info describes the scheme, the hostnames a pasted link is matched against, and the identity a host reuses for help and version.
type Genre ¶
type Genre struct {
Slug string `json:"slug" kit:"id"`
Name string `json:"name"`
Description string `json:"description" kit:"body"`
BooksCount int `json:"books_count"`
BookIDs []string `json:"book_ids" kit:"link,kind=goodreads/book,optional"`
RelatedGenres []string `json:"related_genres" kit:"link,kind=goodreads/genre,optional"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
Genre is a Goodreads genre/taxonomy node (/genres/<slug>).
func ParseGenre ¶
ParseGenre parses a Goodreads genre page. The full description sits in a hidden "freeText" span (the visible "freeTextContainer" is truncated), and the genuine related genres are scoped to the "Related Genres" box so the global genre nav does not leak in.
type List ¶
type List struct {
ListID string `json:"list_id" kit:"id"`
Name string `json:"name"`
Description string `json:"description" kit:"body"`
BooksCount int `json:"books_count"`
VotersCount int `json:"voters_count"`
LikesCount int `json:"likes_count"`
Tags []string `json:"tags"`
CreatedByUser string `json:"created_by_user"`
CreatedByUserID string `json:"created_by_user_id" kit:"link,kind=goodreads/user,optional"`
CreatedDate string `json:"created_date"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
List is a Goodreads Listopia list (/list/show/<id>).
type ListBook ¶
type ListBook struct {
ListID string `json:"list_id"`
BookID string `json:"book_id"`
Rank int `json:"rank"`
Score int `json:"score"`
Votes int `json:"votes"`
AvgRating float64 `json:"avg_rating"`
RatingsCount int64 `json:"ratings_count"`
}
ListBook links a list to a book with its rank, score, votes, and the book's aggregate rating as shown on the list row.
type QueueItem ¶
type QueueItem struct {
ID int64 `json:"id"`
URL string `json:"url"`
EntityType string `json:"entity_type"`
Priority int `json:"priority"`
HTMLPath string `json:"html_path"` // set when status='fetched'
}
QueueItem is one row of the crawl queue.
type Quote ¶
type Quote struct {
QuoteID string `json:"quote_id" kit:"id"`
Text string `json:"text" kit:"body"`
AuthorID string `json:"author_id" kit:"link,kind=goodreads/author,optional"`
AuthorName string `json:"author_name"`
BookID string `json:"book_id" kit:"link,kind=goodreads/book,optional"`
BookTitle string `json:"book_title"`
WorkID string `json:"work_id"`
LikesCount int `json:"likes_count"`
Tags []string `json:"tags"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
Quote is a Goodreads quote.
func ParseQuotes ¶
ParseQuotes parses one or more quotes from a quotes page. The .quoteText and .quoteFooter blocks are flat siblings (not nested per quote), so attribution, the book link, likes, and tags are read from each quote's own subtree and its immediately following footer, never from a shared parent.
type Review ¶
type Review struct {
ReviewID string `json:"review_id" kit:"id"`
BookID string `json:"book_id" kit:"link,kind=goodreads/book"`
UserID string `json:"user_id" kit:"link,kind=goodreads/user,optional"`
UserName string `json:"user_name"`
Rating int `json:"rating"`
Text string `json:"text" kit:"body"`
DateAdded time.Time `json:"date_added"`
LikesCount int `json:"likes_count"`
IsSpoiler bool `json:"is_spoiler"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
Review is a Goodreads book review.
type SearchResult ¶
type SearchResult struct {
Position int `json:"position"`
URL string `json:"url"`
EntityType string `json:"entity_type"` // book or author
Title string `json:"title"`
}
SearchResult is one result from autocomplete or full search.
func ParseSearchAutocomplete ¶
func ParseSearchAutocomplete(body []byte) []SearchResult
ParseSearchAutocomplete parses the JSON autocomplete response.
func ParseSearchHTML ¶
func ParseSearchHTML(doc *goquery.Document) []SearchResult
ParseSearchHTML parses book/author result rows from a full-search page.
type Series ¶
type Series struct {
SeriesID string `json:"series_id" kit:"id"`
Name string `json:"name"`
Description string `json:"description" kit:"body"`
AuthorID string `json:"author_id" kit:"link,kind=goodreads/author,optional"`
AuthorName string `json:"author_name"`
TotalBooks int `json:"total_books"`
PrimaryWorkCount int `json:"primary_work_count"`
BookIDs []string `json:"book_ids" kit:"link,kind=goodreads/book,optional"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
Series is a Goodreads book series (/series/<id>).
type SeriesBook ¶
type SeriesBook struct {
SeriesID string `json:"series_id"`
BookID string `json:"book_id"`
Position int `json:"position"`
PositionLabel string `json:"position_label"`
}
SeriesBook links a series to a book at a position. PositionLabel keeps the raw Goodreads label ("Book 1", "Book 0.5") since series positions are not always whole numbers.
type Shelf ¶
type Shelf struct {
ShelfID string `json:"shelf_id" kit:"id"` // "<user_id>/<shelf_name>"
UserID string `json:"user_id" kit:"link,kind=goodreads/user"`
Name string `json:"name"`
BooksCount int `json:"books_count"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
Shelf is a user's named reading shelf.
type ShelfBook ¶
type ShelfBook struct {
ShelfID string `json:"shelf_id"`
UserID string `json:"user_id"`
BookID string `json:"book_id"`
Title string `json:"title"`
AuthorName string `json:"author_name,omitempty"`
ISBN string `json:"isbn,omitempty"`
NumPages int `json:"num_pages,omitempty"`
AvgRating float64 `json:"avg_rating,omitempty"`
Rating int `json:"rating"`
Shelves []string `json:"shelves,omitempty"`
ReviewID string `json:"review_id,omitempty"`
ReviewText string `json:"review_text,omitempty"`
BookDescription string `json:"book_description,omitempty"`
BookPublished int `json:"book_published,omitempty"`
CoverURL string `json:"cover_url,omitempty"`
DateAdded time.Time `json:"date_added"`
DateCreated time.Time `json:"date_created"`
DateRead time.Time `json:"date_read"`
}
ShelfBook links a shelf to a book with reading metadata. The RSS feed populates the richer fields (author/isbn/review/...); the HTML fallback fills only the basics.
type SitemapCategory ¶
SitemapCategory pairs a siteindex category (author, list, quote, genre, ...) with its siteindex URL.
type Store ¶
type Store struct {
// contains filtered or unexported fields
}
Store is a pure-Go SQLite store for parsed records and a crawl queue.
func (*Store) CountsByType ¶
CountsByType returns a map of entity type to record count.
func (*Store) MarkFailed ¶
MarkFailed records a queue item as failed.
func (*Store) MarkFetched ¶
MarkFetched records a queue item as fetched with its cache path.
func (*Store) NextPending ¶
NextPending claims and returns up to n pending queue items (status -> 'active').
func (*Store) QueueStats ¶
QueueStats returns counts of queue items grouped by status.
func (*Store) ResetActive ¶
ResetActive returns any 'active' items to 'pending' (recover from a crash).
type User ¶
type User struct {
UserID string `json:"user_id" kit:"id"`
Name string `json:"name"`
Username string `json:"username"`
Location string `json:"location"`
JoinedDate time.Time `json:"joined_date"`
FriendsCount int `json:"friends_count"`
BooksCount int `json:"books_count"`
BooksReadCount int `json:"books_read_count"`
RatingsCount int `json:"ratings_count"`
ReviewsCount int `json:"reviews_count"`
AvgRating float64 `json:"avg_rating"`
Bio string `json:"bio" kit:"body"`
Website string `json:"website"`
AvatarURL string `json:"avatar_url"`
FavoriteBookIDs []string `json:"favorite_book_ids" kit:"link,kind=goodreads/book,optional"`
CurrentlyReadingBookIDs []string `json:"currently_reading_book_ids" kit:"link,kind=goodreads/book,optional"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetched_at"`
}
User is a public Goodreads reader profile (/user/show/<id>).