Documentation
¶
Index ¶
- Variables
- func AllMediaTypes() []string
- func AvailableSystems() []string
- func BaseName(filename string) string
- func DefaultMediaTypes() []string
- func DoTyped[T any](d *Deduplicator, key string, fn func() (T, error)) (T, error)
- func LookupSystemID(platform string) (string, error)
- type Config
- type Deduplicator
- type Filter
- type FilterConfig
- type FilterContext
- type Hashes
- type LookupEntry
- type LookupSource
- type MissingContext
- type Model
- type ProgressUpdate
- type RateLimiter
- type RateLimiterStats
- type ScrapeResult
- type ScrapeResults
- type Scraper
- type Status
- type UpdateType
- type Worker
Constants ¶
This section is empty.
Variables ¶
var MediaExtensions = map[string]string{
"ss": "png",
"sstitle": "png",
"box-2D": "png",
"box-3D": "png",
"wheel-hd": "png",
"wheel": "png",
"fanart": "jpg",
"video-normalized": "mp4",
"video": "mp4",
"support-2D": "png",
"box-2D-back": "png",
}
MediaExtensions maps media types to expected file extensions
var MediaTypeMapping = map[string][]string{
"screenshots": {"ss"},
"titlescreens": {"sstitle"},
"covers": {"box-2D"},
"3dboxes": {"box-3D"},
"marquees": {"wheel-hd", "wheel"},
"fanart": {"fanart"},
"videos": {"video-normalized", "video"},
"physicalmedia": {"support-2D"},
"backcovers": {"box-2D-back"},
}
MediaTypeMapping maps ES-DE folder names to Screenscraper media types Some have fallbacks (e.g., marquees tries wheel-hd first, then wheel)
var SystemMapping = map[string]string{
"nes": "3",
"famicom": "3",
"snes": "4",
"superfamicom": "4",
"n64": "14",
"nintendo64": "14",
"gc": "13",
"gamecube": "13",
"ngc": "13",
"wii": "16",
"wiiu": "18",
"fds": "106",
"gb": "9",
"gameboy": "9",
"gbc": "10",
"gameboycolor": "10",
"gba": "12",
"gameboyadvance": "12",
"nds": "15",
"ds": "15",
"dsi": "15",
"3ds": "17",
"virtualboy": "11",
"vb": "11",
"megadrive": "1",
"md": "1",
"genesis": "1",
"mastersystem": "2",
"sms": "2",
"sega32x": "19",
"32x": "19",
"segacd": "20",
"megacd": "20",
"gamegear": "21",
"gg": "21",
"saturn": "22",
"dreamcast": "23",
"dc": "23",
"psx": "57",
"ps1": "57",
"playstation": "57",
"ps2": "58",
"playstation2": "58",
"ps3": "59",
"playstation3": "59",
"psp": "61",
"psvita": "62",
"vita": "62",
"xbox": "32",
"xbox360": "33",
"xboxone": "34",
"pcengine": "31",
"pce": "31",
"turbografx16": "31",
"tg16": "31",
"supergrafx": "105",
"sgx": "105",
"pcfx": "72",
"neogeo": "142",
"ng": "142",
"neogeocd": "70",
"ngcd": "70",
"ngp": "25",
"neogeopocket": "25",
"ngpc": "82",
"atari2600": "26",
"2600": "26",
"atari5200": "40",
"5200": "40",
"atari7800": "41",
"7800": "41",
"lynx": "28",
"atarilynx": "28",
"jaguar": "27",
"atarijaguar": "27",
"wonderswan": "45",
"ws": "45",
"wonderswancolor": "46",
"wsc": "46",
"colecovision": "48",
"intellivision": "115",
"vectrex": "102",
"3do": "29",
}
SystemMapping maps platform names to Screenscraper system IDs. Platform names can be romident Platform values, recalbox names, or common aliases.
Functions ¶
func AvailableSystems ¶
func AvailableSystems() []string
AvailableSystems returns a sorted list of commonly used system names.
func DefaultMediaTypes ¶
func DefaultMediaTypes() []string
DefaultMediaTypes returns the default media types to download
func DoTyped ¶
func DoTyped[T any](d *Deduplicator, key string, fn func() (T, error)) (T, error)
DoTyped is a typed wrapper around Do for convenience
func LookupSystemID ¶
LookupSystemID converts a platform name to a Screenscraper system ID. Accepts romident Platform values, recalbox names, or common aliases. Returns error if the platform is not recognized.
Types ¶
type Config ¶
type Config struct {
// System
SystemID string
// Media selection
MediaTypes []string // ES-DE media type names
// Region preferences
PreferredRegions []string
// Output directory for media files
MediaOutputDir string
// Cache settings
SkipCacheRead bool // --no-cache
SkipCacheWrite bool // --cache-only
// Overwrite behavior
Overwrite bool // --overwrite
// Rate limiting (from user info)
MaxThreads int
MaxRequestsPerMin int
// Filter for which entries to scrape
Filter *Filter
FilterConfig *FilterConfig
}
Config holds scraper configuration
type Deduplicator ¶
type Deduplicator struct {
// contains filtered or unexported fields
}
Deduplicator prevents duplicate in-flight requests When multiple goroutines request the same key, only one performs the work and others wait for the result
func NewDeduplicator ¶
func NewDeduplicator() *Deduplicator
NewDeduplicator creates a new deduplicator
type Filter ¶
type Filter struct {
// contains filtered or unexported fields
}
Filter evaluates filter expressions against entries
func NewFilter ¶
NewFilter creates a new filter from an expression string Example expressions:
- "true" (scrape all)
- "missing.metadata" (only scrape if not in gamelist)
- "missing.covers or missing.videos" (scrape if missing cover OR video)
func (*Filter) Expression ¶
Expression returns the original expression string
func (*Filter) ShouldScrape ¶
func (f *Filter) ShouldScrape(ctx FilterContext) (bool, error)
ShouldScrape evaluates the filter for a given context Returns true if the entry should be scraped
type FilterConfig ¶
type FilterConfig struct {
GamelistPath string // Path to existing gamelist.xml (for metadata checks)
MediaDir string // Path to media directory (for media file checks)
GamelistMap map[string]bool // Map of ROM paths that exist in gamelist
}
FilterConfig holds configuration for filtering entries
func NewFilterConfig ¶
func NewFilterConfig(gamelistPath, mediaDir string) *FilterConfig
NewFilterConfig creates a FilterConfig from paths, loading gamelist if present
type FilterContext ¶
type FilterContext struct {
Missing MissingContext `expr:"missing"`
}
FilterContext contains the variables available in filter expressions
func BuildFilterContext ¶
func BuildFilterContext(baseName string, config *FilterConfig) FilterContext
BuildFilterContext creates a FilterContext for a given entry
type Hashes ¶
Hashes contains ROM hash values
type LookupEntry ¶
type LookupEntry struct {
// Identification
Name string // Display name (from DAT or filename)
FileName string // ROM filename with extension (for API)
Hashes Hashes // SHA1, MD5, CRC32
Serial string // Game code (from DAT serial or ROM header)
Size int64 // File size in bytes
// Region info (parsed from name or ROM header)
Regions []string // e.g., ["us", "eu"]
// Output path (for media naming)
BaseName string // Filename without extension
// Source info
Source LookupSource
ROMPath string // Only for ROM source
}
LookupEntry is the unified input for Screenscraper lookups
type LookupSource ¶
type LookupSource int
LookupSource indicates where the lookup entry came from
const ( SourceDAT LookupSource = iota SourceROM )
type MissingContext ¶
type MissingContext struct {
Metadata bool `expr:"metadata"`
Screenshots bool `expr:"screenshots"`
Titlescreens bool `expr:"titlescreens"`
Covers bool `expr:"covers"`
Boxes3D bool `expr:"3dboxes"`
Marquees bool `expr:"marquees"`
Fanart bool `expr:"fanart"`
Videos bool `expr:"videos"`
Physicalmedia bool `expr:"physicalmedia"`
Backcovers bool `expr:"backcovers"`
}
MissingContext tracks what's missing for a given entry
type Model ¶
type Model struct {
// contains filtered or unexported fields
}
Model is the bubbletea model for the progress display
func NewModel ¶
func NewModel(total, maxThreads, mediaTypesCount int, updatesCh <-chan ProgressUpdate, getStats func() RateLimiterStats) Model
NewModel creates a new progress model
type ProgressUpdate ¶
type ProgressUpdate struct {
Type UpdateType
EntryName string
WorkerID int
MediaTotal int // total media types to download
MediaDone int // media types downloaded successfully
CacheHits int // API calls avoided due to cache (game info + media)
MediaFailed int // media types that failed (error/timeout)
MediaMissing int // media types not available
CurrentMedia string // currently downloading (for display)
Error error
}
ProgressUpdate represents a progress update from a worker
type RateLimiter ¶
type RateLimiter struct {
// contains filtered or unexported fields
}
RateLimiter controls request rate to respect API limits
func NewRateLimiter ¶
func NewRateLimiter(maxThreads, maxPerMinute int) *RateLimiter
NewRateLimiter creates a new rate limiter
func (*RateLimiter) Acquire ¶
func (rl *RateLimiter) Acquire(ctx context.Context) error
Acquire waits for permission to make a request Returns an error if the context is cancelled
func (*RateLimiter) Release ¶
func (rl *RateLimiter) Release()
Release returns a thread slot after a request completes
func (*RateLimiter) ResetBackoff ¶
func (rl *RateLimiter) ResetBackoff()
ResetBackoff resets the backoff level (call after successful request)
func (*RateLimiter) Stats ¶
func (rl *RateLimiter) Stats() RateLimiterStats
Stats returns current statistics
func (*RateLimiter) TriggerBackoff ¶
func (rl *RateLimiter) TriggerBackoff()
TriggerBackoff triggers exponential backoff (call when receiving 429)
type RateLimiterStats ¶
type RateLimiterStats struct {
ActiveThreads int
MaxThreads int
RequestsLastMin int
MaxPerMinute int
BackoffLevel int
BackoffRemaining time.Duration
TotalRequests int // total API calls completed
RequestsPerSecond float64 // completed API calls per second (30s sliding window)
InFlightRequests int // requests started but not yet completed
}
RateLimiterStats contains current rate limiter statistics
type ScrapeResult ¶
type ScrapeResult struct {
Entry *LookupEntry
Game *screenscraper.Game // nil if not found
Media map[string]string // mediaType -> local path (downloaded)
Error error
Cached bool // true if game info from cache
Skipped bool // true if skipped (BIOS, etc.)
Reason string // reason for skip/error
CacheHits int // total API calls avoided due to cache
}
ScrapeResult contains the result of looking up a single entry
type ScrapeResults ¶
type ScrapeResults struct {
Results []*ScrapeResult
TotalEntries int
Found int
NotFound int
Skipped int
Errors int
MediaTotal int
MediaDownloaded int
CacheHits int
FilteredOut int // entries excluded by --filter expression
}
ScrapeResults contains the results of a scrape operation
type Scraper ¶
type Scraper struct {
// contains filtered or unexported fields
}
Scraper orchestrates the scraping process
func New ¶
func New(client *screenscraper.ScreenscraperClient, diskCache *cache.DiskCache, config *Config) *Scraper
New creates a new scraper
func (*Scraper) RateLimiterStats ¶
func (s *Scraper) RateLimiterStats() RateLimiterStats
RateLimiterStats returns current rate limiter statistics
func (*Scraper) ScrapeFromDAT ¶
ScrapeFromDAT scrapes games from a DAT file
func (*Scraper) Updates ¶
func (s *Scraper) Updates() <-chan ProgressUpdate
Updates returns the progress update channel
type UpdateType ¶
type UpdateType int
UpdateType represents the type of progress update
const ( UpdateTypeStarted UpdateType = iota UpdateTypeProgress // media download progress UpdateTypeFound UpdateTypeNotFound UpdateTypeSkipped UpdateTypeError )
type Worker ¶
type Worker struct {
// contains filtered or unexported fields
}
Worker handles scraping tasks
func NewWorker ¶
func NewWorker(id int, client *screenscraper.ScreenscraperClient, cache *cache.DiskCache, config *Config, rateLimiter *RateLimiter, dedup *Deduplicator, updates chan<- ProgressUpdate) *Worker
NewWorker creates a new worker
func (*Worker) Process ¶
func (w *Worker) Process(ctx context.Context, entry *LookupEntry) *ScrapeResult
Process handles a single lookup entry