scraper

package
v0.0.0-...-7691f75 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 25, 2026 License: BSD-2-Clause Imports: 22 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var MediaExtensions = map[string]string{
	"ss":               "png",
	"sstitle":          "png",
	"box-2D":           "png",
	"box-3D":           "png",
	"wheel-hd":         "png",
	"wheel":            "png",
	"fanart":           "jpg",
	"video-normalized": "mp4",
	"video":            "mp4",
	"support-2D":       "png",
	"box-2D-back":      "png",
}

MediaExtensions maps media types to expected file extensions

View Source
var MediaTypeMapping = map[string][]string{
	"screenshots":   {"ss"},
	"titlescreens":  {"sstitle"},
	"covers":        {"box-2D"},
	"3dboxes":       {"box-3D"},
	"marquees":      {"wheel-hd", "wheel"},
	"fanart":        {"fanart"},
	"videos":        {"video-normalized", "video"},
	"physicalmedia": {"support-2D"},
	"backcovers":    {"box-2D-back"},
}

MediaTypeMapping maps ES-DE folder names to Screenscraper media types Some have fallbacks (e.g., marquees tries wheel-hd first, then wheel)

View Source
var SystemMapping = map[string]string{

	"nes":          "3",
	"famicom":      "3",
	"snes":         "4",
	"superfamicom": "4",
	"n64":          "14",
	"nintendo64":   "14",
	"gc":           "13",
	"gamecube":     "13",
	"ngc":          "13",
	"wii":          "16",
	"wiiu":         "18",
	"fds":          "106",

	"gb":             "9",
	"gameboy":        "9",
	"gbc":            "10",
	"gameboycolor":   "10",
	"gba":            "12",
	"gameboyadvance": "12",
	"nds":            "15",
	"ds":             "15",
	"dsi":            "15",
	"3ds":            "17",
	"virtualboy":     "11",
	"vb":             "11",

	"megadrive":    "1",
	"md":           "1",
	"genesis":      "1",
	"mastersystem": "2",
	"sms":          "2",
	"sega32x":      "19",
	"32x":          "19",
	"segacd":       "20",
	"megacd":       "20",
	"gamegear":     "21",
	"gg":           "21",
	"saturn":       "22",
	"dreamcast":    "23",
	"dc":           "23",

	"psx":          "57",
	"ps1":          "57",
	"playstation":  "57",
	"ps2":          "58",
	"playstation2": "58",
	"ps3":          "59",
	"playstation3": "59",

	"psp":    "61",
	"psvita": "62",
	"vita":   "62",

	"xbox":    "32",
	"xbox360": "33",
	"xboxone": "34",

	"pcengine":     "31",
	"pce":          "31",
	"turbografx16": "31",
	"tg16":         "31",
	"supergrafx":   "105",
	"sgx":          "105",
	"pcfx":         "72",

	"neogeo":       "142",
	"ng":           "142",
	"neogeocd":     "70",
	"ngcd":         "70",
	"ngp":          "25",
	"neogeopocket": "25",
	"ngpc":         "82",

	"atari2600":   "26",
	"2600":        "26",
	"atari5200":   "40",
	"5200":        "40",
	"atari7800":   "41",
	"7800":        "41",
	"lynx":        "28",
	"atarilynx":   "28",
	"jaguar":      "27",
	"atarijaguar": "27",

	"wonderswan":      "45",
	"ws":              "45",
	"wonderswancolor": "46",
	"wsc":             "46",

	"colecovision":  "48",
	"intellivision": "115",
	"vectrex":       "102",
	"3do":           "29",
}

SystemMapping maps platform names to Screenscraper system IDs. Platform names can be romident Platform values, recalbox names, or common aliases.

Functions

func AllMediaTypes

func AllMediaTypes() []string

AllMediaTypes returns all supported media types

func AvailableSystems

func AvailableSystems() []string

AvailableSystems returns a sorted list of commonly used system names.

func BaseName

func BaseName(filename string) string

BaseName returns the filename without extension

func DefaultMediaTypes

func DefaultMediaTypes() []string

DefaultMediaTypes returns the default media types to download

func DoTyped

func DoTyped[T any](d *Deduplicator, key string, fn func() (T, error)) (T, error)

DoTyped is a typed wrapper around Do for convenience

func LookupSystemID

func LookupSystemID(platform string) (string, error)

LookupSystemID converts a platform name to a Screenscraper system ID. Accepts romident Platform values, recalbox names, or common aliases. Returns error if the platform is not recognized.

Types

type Config

type Config struct {
	// System
	SystemID string

	// Media selection
	MediaTypes []string // ES-DE media type names

	// Region preferences
	PreferredRegions []string

	// Output directory for media files
	MediaOutputDir string

	// Cache settings
	SkipCacheRead  bool // --no-cache
	SkipCacheWrite bool // --cache-only

	// Overwrite behavior
	Overwrite bool // --overwrite

	// Rate limiting (from user info)
	MaxThreads        int
	MaxRequestsPerMin int

	// Filter for which entries to scrape
	Filter       *Filter
	FilterConfig *FilterConfig
}

Config holds scraper configuration

type Deduplicator

type Deduplicator struct {
	// contains filtered or unexported fields
}

Deduplicator prevents duplicate in-flight requests When multiple goroutines request the same key, only one performs the work and others wait for the result

func NewDeduplicator

func NewDeduplicator() *Deduplicator

NewDeduplicator creates a new deduplicator

func (*Deduplicator) Do

func (d *Deduplicator) Do(key string, fn func() (interface{}, error)) (interface{}, error)

Do ensures only one request is made for a given key. Concurrent callers with the same key will wait for the first request to complete.

type Filter

type Filter struct {
	// contains filtered or unexported fields
}

Filter evaluates filter expressions against entries

func NewFilter

func NewFilter(expression string) (*Filter, error)

NewFilter creates a new filter from an expression string Example expressions:

  • "true" (scrape all)
  • "missing.metadata" (only scrape if not in gamelist)
  • "missing.covers or missing.videos" (scrape if missing cover OR video)

func (*Filter) Expression

func (f *Filter) Expression() string

Expression returns the original expression string

func (*Filter) ShouldScrape

func (f *Filter) ShouldScrape(ctx FilterContext) (bool, error)

ShouldScrape evaluates the filter for a given context Returns true if the entry should be scraped

type FilterConfig

type FilterConfig struct {
	GamelistPath string          // Path to existing gamelist.xml (for metadata checks)
	MediaDir     string          // Path to media directory (for media file checks)
	GamelistMap  map[string]bool // Map of ROM paths that exist in gamelist
}

FilterConfig holds configuration for filtering entries

func NewFilterConfig

func NewFilterConfig(gamelistPath, mediaDir string) *FilterConfig

NewFilterConfig creates a FilterConfig from paths, loading gamelist if present

type FilterContext

type FilterContext struct {
	Missing MissingContext `expr:"missing"`
}

FilterContext contains the variables available in filter expressions

func BuildFilterContext

func BuildFilterContext(baseName string, config *FilterConfig) FilterContext

BuildFilterContext creates a FilterContext for a given entry

type Hashes

type Hashes struct {
	SHA1  string
	MD5   string
	CRC32 string
}

Hashes contains ROM hash values

func (Hashes) CacheKey

func (h Hashes) CacheKey() string

CacheKey returns the primary key for cache lookups

func (Hashes) IsEmpty

func (h Hashes) IsEmpty() bool

IsEmpty returns true if no hashes are set

type LookupEntry

type LookupEntry struct {
	// Identification
	Name     string // Display name (from DAT or filename)
	FileName string // ROM filename with extension (for API)
	Hashes   Hashes // SHA1, MD5, CRC32
	Serial   string // Game code (from DAT serial or ROM header)
	Size     int64  // File size in bytes

	// Region info (parsed from name or ROM header)
	Regions []string // e.g., ["us", "eu"]

	// Output path (for media naming)
	BaseName string // Filename without extension

	// Source info
	Source  LookupSource
	ROMPath string // Only for ROM source
}

LookupEntry is the unified input for Screenscraper lookups

type LookupSource

type LookupSource int

LookupSource indicates where the lookup entry came from

const (
	SourceDAT LookupSource = iota
	SourceROM
)

type MissingContext

type MissingContext struct {
	Metadata      bool `expr:"metadata"`
	Screenshots   bool `expr:"screenshots"`
	Titlescreens  bool `expr:"titlescreens"`
	Covers        bool `expr:"covers"`
	Boxes3D       bool `expr:"3dboxes"`
	Marquees      bool `expr:"marquees"`
	Fanart        bool `expr:"fanart"`
	Videos        bool `expr:"videos"`
	Physicalmedia bool `expr:"physicalmedia"`
	Backcovers    bool `expr:"backcovers"`
}

MissingContext tracks what's missing for a given entry

type Model

type Model struct {
	// contains filtered or unexported fields
}

Model is the bubbletea model for the progress display

func NewModel

func NewModel(total, maxThreads, mediaTypesCount int, updatesCh <-chan ProgressUpdate, getStats func() RateLimiterStats) Model

NewModel creates a new progress model

func (Model) Init

func (m Model) Init() tea.Cmd

Init initializes the model

func (Model) Update

func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd)

Update handles messages

func (Model) View

func (m Model) View() string

View renders the live UI (in-progress items + stats)

type ProgressUpdate

type ProgressUpdate struct {
	Type         UpdateType
	EntryName    string
	WorkerID     int
	MediaTotal   int    // total media types to download
	MediaDone    int    // media types downloaded successfully
	CacheHits    int    // API calls avoided due to cache (game info + media)
	MediaFailed  int    // media types that failed (error/timeout)
	MediaMissing int    // media types not available
	CurrentMedia string // currently downloading (for display)
	Error        error
}

ProgressUpdate represents a progress update from a worker

type RateLimiter

type RateLimiter struct {
	// contains filtered or unexported fields
}

RateLimiter controls request rate to respect API limits

func NewRateLimiter

func NewRateLimiter(maxThreads, maxPerMinute int) *RateLimiter

NewRateLimiter creates a new rate limiter

func (*RateLimiter) Acquire

func (rl *RateLimiter) Acquire(ctx context.Context) error

Acquire waits for permission to make a request Returns an error if the context is cancelled

func (*RateLimiter) Release

func (rl *RateLimiter) Release()

Release returns a thread slot after a request completes

func (*RateLimiter) ResetBackoff

func (rl *RateLimiter) ResetBackoff()

ResetBackoff resets the backoff level (call after successful request)

func (*RateLimiter) Stats

func (rl *RateLimiter) Stats() RateLimiterStats

Stats returns current statistics

func (*RateLimiter) TriggerBackoff

func (rl *RateLimiter) TriggerBackoff()

TriggerBackoff triggers exponential backoff (call when receiving 429)

type RateLimiterStats

type RateLimiterStats struct {
	ActiveThreads     int
	MaxThreads        int
	RequestsLastMin   int
	MaxPerMinute      int
	BackoffLevel      int
	BackoffRemaining  time.Duration
	TotalRequests     int     // total API calls completed
	RequestsPerSecond float64 // completed API calls per second (30s sliding window)
	InFlightRequests  int     // requests started but not yet completed
}

RateLimiterStats contains current rate limiter statistics

type ScrapeResult

type ScrapeResult struct {
	Entry     *LookupEntry
	Game      *screenscraper.Game // nil if not found
	Media     map[string]string   // mediaType -> local path (downloaded)
	Error     error
	Cached    bool   // true if game info from cache
	Skipped   bool   // true if skipped (BIOS, etc.)
	Reason    string // reason for skip/error
	CacheHits int    // total API calls avoided due to cache
}

ScrapeResult contains the result of looking up a single entry

type ScrapeResults

type ScrapeResults struct {
	Results         []*ScrapeResult
	TotalEntries    int
	Found           int
	NotFound        int
	Skipped         int
	Errors          int
	MediaTotal      int
	MediaDownloaded int
	CacheHits       int
	FilteredOut     int // entries excluded by --filter expression
}

ScrapeResults contains the results of a scrape operation

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

Scraper orchestrates the scraping process

func New

func New(client *screenscraper.ScreenscraperClient, diskCache *cache.DiskCache, config *Config) *Scraper

New creates a new scraper

func (*Scraper) GetConfig

func (s *Scraper) GetConfig() *Config

GetConfig returns the scraper configuration

func (*Scraper) RateLimiterStats

func (s *Scraper) RateLimiterStats() RateLimiterStats

RateLimiterStats returns current rate limiter statistics

func (*Scraper) ScrapeFromDAT

func (s *Scraper) ScrapeFromDAT(ctx context.Context, datPath string) (*ScrapeResults, error)

ScrapeFromDAT scrapes games from a DAT file

func (*Scraper) Updates

func (s *Scraper) Updates() <-chan ProgressUpdate

Updates returns the progress update channel

type Status

type Status int

Status represents the status of a scrape operation

const (
	StatusPending Status = iota
	StatusInProgress
	StatusFound
	StatusNotFound
	StatusSkipped
	StatusError
)

func (Status) String

func (s Status) String() string

String returns a string representation of the status

type UpdateType

type UpdateType int

UpdateType represents the type of progress update

const (
	UpdateTypeStarted  UpdateType = iota
	UpdateTypeProgress            // media download progress
	UpdateTypeFound
	UpdateTypeNotFound
	UpdateTypeSkipped
	UpdateTypeError
)

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

Worker handles scraping tasks

func NewWorker

func NewWorker(id int, client *screenscraper.ScreenscraperClient, cache *cache.DiskCache, config *Config, rateLimiter *RateLimiter, dedup *Deduplicator, updates chan<- ProgressUpdate) *Worker

NewWorker creates a new worker

func (*Worker) Process

func (w *Worker) Process(ctx context.Context, entry *LookupEntry) *ScrapeResult

Process handles a single lookup entry

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL