Documentation
¶
Overview ¶
Package sde imports CCP's Static Data Export into Postgres.
The archive is ~99 MB compressed and ~575 MB across 79 JSONL members once expanded (mapMoons alone is 224 MB), so nothing here materialises a whole member. Members are read line by line with one decoded row in flight, and the zip is streamed to disk rather than held in memory because archive/zip needs random access.
Index ¶
- Variables
- func LoadedBuild(ctx context.Context, pool *pgxpool.Pool) (int64, string, error)
- func RecordBuild(ctx context.Context, pool *pgxpool.Pool, build int64, release string) error
- func SeedInvFlags(ctx context.Context, pool *pgxpool.Pool) (int64, string, error)
- type Derivation
- type DeriveResult
- type ExternalResult
- func GenerateSupercapPrices(ctx context.Context, pool *pgxpool.Pool) (ExternalResult, error)
- func ImportInsurancePrices(ctx context.Context, pool *pgxpool.Pool, userAgent string) (ExternalResult, error)
- func ImportStructures(ctx context.Context, pool *pgxpool.Pool, userAgent string) (ExternalResult, error)
- func SeedManualCustomPrices(ctx context.Context, pool *pgxpool.Pool) (ExternalResult, error)
- type ImportOptions
- type ImportResult
- type LoadResult
- type Manifest
- type Row
- func (r Row) Bool(field string) *bool
- func (r Row) Float(field string) *float64
- func (r Row) Int(field string) *int32
- func (r Row) Int64(field string) *int64
- func (r Row) Key() (int64, bool)
- func (r Row) Lang(field string) *string
- func (r Row) List(field string) []any
- func (r Row) Map(field string) map[string]any
- func (r Row) Str(field string) *string
- type SeedResult
- type Source
- type Table
Constants ¶
This section is empty.
Variables ¶
var Derivations = []Derivation{
{
Name: "inv_types.category_id",
Description: "Denormalise category from the type's group",
SQL: `
UPDATE inv_types t
SET category_id = (
SELECT g.category_id FROM inv_groups g WHERE g.group_id = t.group_id
)
WHERE t.category_id IS DISTINCT FROM (
SELECT g.category_id FROM inv_groups g WHERE g.group_id = t.group_id
)
`,
},
{
Name: "stations.region_id / constellation_id",
Description: "Inherit region and constellation from the station's system",
SQL: `
UPDATE stations s
SET region_id = (
SELECT sys.region_id FROM solar_systems sys
WHERE sys.solar_system_id = s.solar_system_id
),
constellation_id = (
SELECT sys.constellation_id FROM solar_systems sys
WHERE sys.solar_system_id = s.solar_system_id
)
WHERE s.region_id IS DISTINCT FROM (
SELECT sys.region_id FROM solar_systems sys
WHERE sys.solar_system_id = s.solar_system_id
)
OR s.constellation_id IS DISTINCT FROM (
SELECT sys.constellation_id FROM solar_systems sys
WHERE sys.solar_system_id = s.solar_system_id
)
`,
},
}
Derivations are ordered: solar_systems must be loaded before stations can inherit region and constellation from it.
var NestedTables = []Table{ { Member: "typeDogma", Name: "type_dogma_attributes", PK: []string{"type_id", "attribute_id"}, PruneAbsent: true, Columns: []string{"type_id", "attribute_id", "value"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } list := r.List("dogmaAttributes") out := make([][]any, 0, len(list)) for _, item := range list { m, ok := item.(map[string]any) if !ok { continue } a := Row(m) attrID := a.Int("attributeID") if attrID == nil { continue } out = append(out, []any{int32(id), *attrID, a.Float("value")}) } return out }, }, { Member: "typeDogma", Name: "type_dogma_effects", PK: []string{"type_id", "effect_id"}, PruneAbsent: true, Columns: []string{"type_id", "effect_id", "is_default"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } list := r.List("dogmaEffects") out := make([][]any, 0, len(list)) for _, item := range list { m, ok := item.(map[string]any) if !ok { continue } e := Row(m) effectID := e.Int("effectID") if effectID == nil { continue } out = append(out, []any{int32(id), *effectID, e.Bool("isDefault")}) } return out }, }, { Member: "typeMaterials", Name: "type_materials", PK: []string{"type_id", "material_type_id"}, PruneAbsent: true, Columns: []string{"type_id", "material_type_id", "quantity"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } list := r.List("materials") out := make([][]any, 0, len(list)) for _, item := range list { m, ok := item.(map[string]any) if !ok { continue } mat := Row(m) matID := mat.Int("materialTypeID") if matID == nil { continue } out = append(out, []any{int32(id), *matID, mat.Int("quantity")}) } return out }, }, { Member: "blueprints", Name: "blueprints", PK: []string{"blueprint_type_id"}, PruneAbsent: true, Columns: []string{"blueprint_type_id", "max_production_limit"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } return [][]any{{int32(id), r.Int("maxProductionLimit")}} }, }, { Member: "blueprints", Name: "blueprint_activities", PK: []string{"blueprint_type_id", "activity"}, PruneAbsent: true, Columns: []string{"blueprint_type_id", "activity", "time"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } var out [][]any forEachActivity(r, func(name string, a Row) { out = append(out, []any{int32(id), name, a.Int("time")}) }) return out }, }, { Member: "blueprints", Name: "blueprint_activity_materials", PK: []string{"blueprint_type_id", "activity", "material_type_id"}, PruneAbsent: true, Columns: []string{"blueprint_type_id", "activity", "material_type_id", "quantity"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } var out [][]any forEachActivity(r, func(name string, a Row) { for _, item := range a.List("materials") { m, ok := item.(map[string]any) if !ok { continue } mat := Row(m) typeID := mat.Int("typeID") if typeID == nil { continue } out = append(out, []any{int32(id), name, *typeID, mat.Int("quantity")}) } }) return out }, }, { Member: "blueprints", Name: "blueprint_activity_products", PK: []string{"blueprint_type_id", "activity", "product_type_id"}, PruneAbsent: true, Columns: []string{"blueprint_type_id", "activity", "product_type_id", "quantity", "probability"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } var out [][]any forEachActivity(r, func(name string, a Row) { for _, item := range a.List("products") { m, ok := item.(map[string]any) if !ok { continue } p := Row(m) typeID := p.Int("typeID") if typeID == nil { continue } out = append(out, []any{ int32(id), name, *typeID, p.Int("quantity"), p.Float("probability"), }) } }) return out }, }, { Member: "blueprints", Name: "blueprint_activity_skills", PK: []string{"blueprint_type_id", "activity", "skill_type_id"}, PruneAbsent: true, Columns: []string{"blueprint_type_id", "activity", "skill_type_id", "level"}, Expand: func(r Row) [][]any { id, ok := r.Key() if !ok { return nil } var out [][]any forEachActivity(r, func(name string, a Row) { for _, item := range a.List("skills") { m, ok := item.(map[string]any) if !ok { continue } s := Row(m) typeID := s.Int("typeID") if typeID == nil { continue } out = append(out, []any{int32(id), name, *typeID, s.Int("level")}) } }) return out }, }, }
Members whose payload is nested, so one archive record becomes many rows.
These use Table.Expand rather than Table.Values. The row counts are wildly different from the record counts: typeDogma has one record per type but produces 645 k attribute rows, and one blueprint record produces rows across five tables.
These are registered separately from Tables so the simple one-to-one mappings stay readable, but they load through exactly the same COPY-to- staging-then-merge path.
var StationNameDerivation = Derivation{
Name: "stations.station_name",
Description: "Copy the composed name from the station's celestial row",
SQL: `
UPDATE stations s
SET station_name = (
SELECT c.item_name FROM celestials c WHERE c.item_id = s.station_id
)
WHERE s.station_name IS DISTINCT FROM (
SELECT c.item_name FROM celestials c WHERE c.item_id = s.station_id
)
`,
}
StationNameDerivation is separate from Derivations because it depends on celestials, which are imported after the declarative tables. Verified against production: stations.station_name is identical to the station's celestial name for all 5,210 rows.
var Tables = []Table{ { Member: "types", Name: "inv_types", PK: []string{"type_id"}, Columns: []string{ "type_id", "group_id", "name", "description", "mass", "volume", "capacity", "portion_size", "packaged_volume", "radius", "published", "market_group_id", "icon_id", "sound_id", "graphic_id", "meta_group_id", "race_id", "faction_id", "base_price", "variation_parent_type_id", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Int("groupID"), r.Lang("name"), r.Lang("description"), r.Float("mass"), r.Float("volume"), r.Float("capacity"), r.Int("portionSize"), r.Float("packagedVolume"), r.Float("radius"), r.Bool("published"), r.Int("marketGroupID"), r.Int("iconID"), r.Int("soundID"), r.Int("graphicID"), r.Int("metaGroupID"), r.Int("raceID"), r.Int("factionID"), r.Float("basePrice"), r.Int("variationParentTypeID"), }, true }, }, { Member: "groups", Name: "inv_groups", PK: []string{"group_id"}, Columns: []string{"group_id", "category_id", "name", "published", "icon_id"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Int("categoryID"), r.Lang("name"), r.Bool("published"), r.Int("iconID"), }, true }, }, { Member: "categories", Name: "inv_categories", PK: []string{"category_id"}, Columns: []string{"category_id", "name", "published", "icon_id"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{int32(id), r.Lang("name"), r.Bool("published"), r.Int("iconID")}, true }, }, { Member: "metaGroups", Name: "inv_meta_groups", PK: []string{"meta_group_id"}, PruneAbsent: true, Columns: []string{"meta_group_id", "name", "icon_id", "icon_suffix"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{int32(id), r.Lang("name"), r.Int("iconID"), r.Str("iconSuffix")}, true }, }, { Member: "marketGroups", Name: "inv_market_groups", PK: []string{"market_group_id"}, Columns: []string{"market_group_id", "parent_group_id", "name", "description", "icon_id", "has_types"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Int("parentGroupID"), r.Lang("name"), r.Lang("description"), r.Int("iconID"), r.Bool("hasTypes"), }, true }, }, { Member: "mapRegions", Name: "regions", PK: []string{"region_id"}, PruneAbsent: true, Columns: []string{ "region_id", "name", "description", "center_x", "center_y", "center_z", "faction_id", "nebula_id", "wormhole_class_id", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } x, y, z := xyz(r, "position") return []any{ int32(id), r.Lang("name"), r.Lang("description"), x, y, z, r.Int("factionID"), r.Int("nebulaID"), r.Int("wormholeClassID"), }, true }, }, { Member: "mapConstellations", Name: "constellations", PK: []string{"constellation_id"}, PruneAbsent: true, Columns: []string{"constellation_id", "constellation_name", "region_id", "x", "y", "z", "faction_id", "radius"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } x, y, z := xyz(r, "position") return []any{ int32(id), r.Lang("name"), r.Int("regionID"), x, y, z, r.Int("factionID"), r.Float("radius"), }, true }, }, { Member: "mapSolarSystems", Name: "solar_systems", PK: []string{"solar_system_id"}, PruneAbsent: true, Columns: []string{ "solar_system_id", "system_name", "constellation_id", "region_id", "x", "y", "z", "x2d", "z2d", "security", "security_class", "faction_id", "border", "fringe", "corridor", "hub", "international", "regional", "luminosity", "radius", "sun_type_id", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } x, y, z := xyz(r, "position") // The 2D map projection stores the source y as z2d — it is a // top-down projection, so the vertical axis is dropped and what the // SDE calls y is the second screen axis. var x2d, z2d *float64 if p := r.Map("position2D"); p != nil { x2d = Row(p).Float("x") z2d = Row(p).Float("y") } return []any{ int32(id), r.Lang("name"), r.Int("constellationID"), r.Int("regionID"), x, y, z, x2d, z2d, r.Float("securityStatus"), r.Str("securityClass"), r.Int("factionID"), r.Bool("border"), r.Bool("fringe"), r.Bool("corridor"), r.Bool("hub"), r.Bool("international"), r.Bool("regional"), r.Float("luminosity"), r.Float("radius"), r.Int("sunTypeID"), }, true }, }, { Member: "factions", Name: "factions", PK: []string{"faction_id"}, PruneAbsent: true, Columns: []string{ "faction_id", "name", "description", "corporation_id", "militia_corporation_id", "solar_system_id", "icon_id", "size_factor", "race_ids", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Lang("name"), r.Lang("description"), r.Int("corporationID"), r.Int("militiaCorporationID"), r.Int("solarSystemID"), r.Int("iconID"), r.Float("sizeFactor"), int32Slice(r.List("memberRaces")), }, true }, }, { Member: "bloodlines", Name: "bloodlines", PK: []string{"bloodline_id"}, PruneAbsent: true, Columns: []string{ "bloodline_id", "bloodline_name", "race_id", "description", "male_description", "female_description", "short_description", "short_male_description", "short_female_description", "ship_type_id", "corporation_id", "perception", "willpower", "charisma", "memory", "intelligence", "icon_id", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Lang("name"), r.Int("raceID"), r.Lang("description"), r.Lang("maleDescription"), r.Lang("femaleDescription"), r.Lang("shortDescription"), r.Lang("shortMaleDescription"), r.Lang("shortFemaleDescription"), r.Int("shipTypeID"), r.Int("corporationID"), r.Int("perception"), r.Int("willpower"), r.Int("charisma"), r.Int("memory"), r.Int("intelligence"), r.Int("iconID"), }, true }, }, { Member: "races", Name: "races", PK: []string{"race_id"}, PruneAbsent: true, Columns: []string{"race_id", "race_name", "description", "short_description", "icon_id"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Lang("name"), r.Lang("description"), r.Lang("shortDescription"), r.Int("iconID"), }, true }, }, { Member: "npcCorporations", Name: "npc_corporations", PK: []string{"corporation_id"}, PruneAbsent: true, Columns: []string{ "corporation_id", "name", "description", "ticker_name", "ceo_id", "station_id", "size", "extent", "tax_rate", "deleted", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Lang("name"), r.Lang("description"), r.Str("tickerName"), r.Int("ceoID"), r.Int("stationID"), r.Str("size"), r.Str("extent"), r.Float("taxRate"), r.Bool("deleted"), }, true }, }, { Member: "stationOperations", Name: "station_operations", PK: []string{"operation_id"}, PruneAbsent: true, Columns: []string{"operation_id", "operation_name", "description", "activity_id", "services"}, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } return []any{ int32(id), r.Lang("operationName"), r.Lang("description"), r.Int("activityID"), int32Slice(r.List("services")), }, true }, }, { Member: "npcStations", Name: "stations", PK: []string{"station_id"}, PruneAbsent: true, Columns: []string{ "station_id", "type_id", "corporation_id", "solar_system_id", "operation_id", "celestial_index", "orbit_id", "orbit_index", "x", "y", "z", "reprocessing_efficiency", "reprocessing_stations_take", }, Values: func(r Row) ([]any, bool) { id, ok := r.Key() if !ok { return nil, false } x, y, z := xyz(r, "position") return []any{ int32(id), r.Int("typeID"), r.Int("ownerID"), r.Int("solarSystemID"), r.Int("operationID"), r.Int("celestialIndex"), r.Int("orbitID"), r.Int("orbitIndex"), x, y, z, r.Float("reprocessingEfficiency"), r.Float("reprocessingStationsTake"), }, true }, }, }
Tables declares the straightforward one-member-to-one-table imports, transcribed from the importTable() calls in backend/src/commands/UpdateSdeCommand.ts.
Column types come from internal/dbgen (generated from the schema), which is what decides the accessor used for each field: Int for integer columns, Float for double precision, Lang for localised text, Str for bare strings.
Not covered here, because they are not one-to-one and get their own passes: typeDogma (two tables), typeMaterials, blueprints (five tables), the map celestial members (five members into one table), stargate-derived system jumps, inv_flags (a hardcoded list, not in the archive), and the EVE Ref feeds.
Functions ¶
func LoadedBuild ¶
LoadedBuild reads back what RecordBuild stored. Returns 0 when nothing has been imported yet.
func RecordBuild ¶
RecordBuild stores the imported build number in the config table so sde:status can report what is loaded and the cron can skip an unchanged build.
Types ¶
type Derivation ¶
Derivation is a post-import fill for a column the archive does not carry directly but that can be computed from data already loaded.
These run after every table is in place, because they read across tables.
type DeriveResult ¶
type DeriveResult struct {
Name string `json:"name"`
Rows int64 `json:"rows"`
Elapsed string `json:"elapsed"`
}
DeriveResult reports one derivation.
func DeriveOne ¶
func DeriveOne(ctx context.Context, pool *pgxpool.Pool, d Derivation) (DeriveResult, error)
DeriveOne runs a single derivation.
type ExternalResult ¶
type ExternalResult struct {
Name string `json:"name"`
Rows int64 `json:"rows"`
Elapsed string `json:"elapsed"`
}
ExternalResult reports one external feed.
func GenerateSupercapPrices ¶
GenerateSupercapPrices derives supercapital values from blueprint materials.
Requires the prices table, which is populated by the market-history importer rather than by the SDE. With no prices loaded every hull costs zero and nothing is written — that is the correct outcome, not a failure, so this reports zero rows instead of erroring.
func ImportInsurancePrices ¶
func ImportInsurancePrices(ctx context.Context, pool *pgxpool.Pool, userAgent string) (ExternalResult, error)
ImportInsurancePrices loads ship insurance costs and payouts.
The feed nests six levels per type — Basic through Platinum — which flatten into one row each.
func ImportStructures ¶
func ImportStructures(ctx context.Context, pool *pgxpool.Pool, userAgent string) (ExternalResult, error)
ImportStructures loads player-owned structures observed by EVE Ref.
Unlike everything else here this is not static data — structures are built and destroyed constantly — but killmails reference them as locations, so the names have to come from somewhere.
func SeedManualCustomPrices ¶
SeedManualCustomPrices writes the hand-maintained price list.
type ImportOptions ¶
type ImportOptions struct {
CacheDir string
UserAgent string
Progress func(string)
SkipExternal bool
}
ImportOptions controls a complete published-build import.
type ImportResult ¶
type ImportResult struct {
BuildNumber int64
ReleaseDate string
Read int64
Written int64
Pruned int64
Duration time.Duration
Elapsed string
Tables []LoadResult
Celestials LoadResult
SystemJumps LoadResult
InvFlags int64
Dust514 SeedResult
External []ExternalResult
Derived []DeriveResult
}
ImportResult summarises a complete SDE import. The detailed per-source results are retained so both the CLI and scheduled worker can report useful progress without duplicating the orchestration.
func ImportBuild ¶
func ImportBuild( ctx context.Context, pool *pgxpool.Pool, manifest *Manifest, opts ImportOptions, ) (ImportResult, error)
ImportBuild performs the complete import for a published SDE build.
RecordBuild deliberately runs last. Every operation before it is idempotent, so an interrupted import can safely converge on its next run, while the build marker never claims a partial database is current.
type LoadResult ¶
type LoadResult struct {
Table string `json:"table"`
Member string `json:"member"`
Read int64 `json:"read"`
Written int64 `json:"written"`
Skipped int64 `json:"skipped"`
Pruned int64 `json:"pruned"`
Duration time.Duration `json:"-"`
Elapsed string `json:"elapsed"`
Missing bool `json:"missing,omitempty"`
}
LoadResult reports what a single table import did.
func ImportCelestials ¶
ImportCelestials runs every pass and merges the result.
func ImportSystemJumps ¶
ImportSystemJumps builds the adjacency table.
func Load ¶
Load streams a member into its table.
The write path is COPY into a TEMP table followed by INSERT ... SELECT ... ON CONFLICT DO UPDATE, rather than batched upserts. Two reasons: COPY is several times faster than even large multi-row INSERTs, and the merge then happens in one statement so the destination table is never half-updated. The import is idempotent, which matters because it runs daily from a cron.
Everything runs on a single dedicated connection because a TEMP table is session-scoped — with a pool, the COPY and the merge could otherwise land on different connections and the merge would not see the staged rows.
type Manifest ¶
type Manifest struct {
Key string `json:"_key"`
BuildNumber int64 `json:"buildNumber"`
ReleaseDate string `json:"releaseDate"`
}
Manifest is the content of latest.jsonl.
type Row ¶
Row is one decoded JSONL object.
SDE JSON is loose in ways that matter: every number decodes as float64, some numeric fields arrive as strings, booleans are sometimes 0/1, and localised text is either a bare string or a {lang: text} map. The accessors below are the single place those quirks are handled, so table definitions stay declarative.
All of them return pointers, because "absent" and "zero" are different things here — a missing volume must become SQL NULL, not 0.0, or downstream arithmetic silently treats unknown as free.
func (Row) Int ¶
Int returns an integer field as int32, the width sqlc generates for Postgres `integer` columns. Values beyond int32 are dropped rather than silently wrapped — a wrapped type ID would corrupt a join.
func (Row) Key ¶
Key returns the row's primary identifier. Every SDE record carries "_key".
Zero is rejected. The archive ships placeholder records at _key 0 — a "#System" entry appears in types, groups and categories — and importing them is actively harmful, not merely untidy: EVE data uses 0 as the "no entity" sentinel, so a real inv_types row with type_id 0 would make every join against an absent ship or item spuriously resolve to "#System".
This also matches the TypeScript importer, whose `if (!id) continue` drops zero as falsy.
func (Row) Lang ¶
Lang extracts localised text, preferring English.
Falls back to any other non-empty translation rather than returning nothing: a name in Japanese is more useful than NULL, and a handful of SDE records genuinely lack "en".
func (Row) List ¶
List returns an array field for callers that need to walk nested structures (blueprint activities, dogma attribute lists).
type SeedResult ¶
type SeedResult struct {
Categories int64 `json:"categories"`
Groups int64 `json:"groups"`
MarketGroups int64 `json:"market_groups"`
Types int64 `json:"types"`
Elapsed string `json:"elapsed"`
}
SeedResult reports what the Dust 514 seed wrote.
func SeedDust514 ¶
SeedDust514 upserts the bundled data.
Runs after the archive import so that where CCP still ships a record, the live archive wins on the first pass and this only fills what is missing — both write the same rows, so the order is about provenance, not conflict.
type Source ¶
type Source struct {
Build int64
Release string
Path string
// contains filtered or unexported fields
}
Source is an opened SDE archive.
func Open ¶
func Open(ctx context.Context, cacheDir string, m *Manifest, userAgent string, progress func(string)) (*Source, error)
Open returns the archive for a build, downloading it into cacheDir only when not already present. Caching by build number matters during development: a re-import otherwise re-fetches 99 MB every run, and the archive for a given build is immutable.
func (*Source) Has ¶
Has reports whether a member exists. CCP adds and removes members between builds, so a caller may legitimately skip a missing one.
type Table ¶
type Table struct {
// Member is the archive member without its .jsonl suffix.
Member string
// Name is the destination table.
Name string
// PK is the conflict target for the upsert.
PK []string
// Columns are the columns written, in the order Values returns them.
Columns []string
// Values converts a row to column values. Returning false skips the row —
// used for records with no usable "_key".
//
// Exactly one of Values or Expand must be set.
Values func(Row) ([]any, bool)
// Expand is the one-to-many form: a single archive record yields any number
// of destination rows. Several members nest their real payload in an array
// or object — one typeDogma record carries every attribute for a type, one
// blueprint carries every activity — so the row count in the archive bears
// no relation to the row count in the table.
Expand func(Row) [][]any
// Optional marks members that may legitimately be absent from an archive.
// CCP adds and drops members between builds.
Optional bool
// PruneAbsent makes the archive member authoritative for this table.
//
// An upsert alone leaves rows behind when CCP removes a nested dogma
// attribute, material, blueprint activity, or other record from a later
// build. Those rows are no longer static data; they are stale data. Tables
// set to true are merged and pruned in one transaction so readers never see
// a half-updated snapshot.
//
// Leave this false for tables augmented from another source. In particular,
// inv_types, inv_groups, inv_categories, and inv_market_groups also contain
// bundled Dust 514 records which are intentionally absent from CCP's
// current archive.
PruneAbsent bool
}
Table declares how one JSONL member maps onto one Postgres table.
Deliberately data, not code: adding a table is a literal in tables.go rather than a new function, which keeps the 14 straightforward imports readable side by side and makes them diffable against the TypeScript they came from.
func TableByName ¶
TableByName finds a declaration, for --only. Matching on member name is ambiguous for nested members (typeDogma feeds two tables, blueprints five), so table name wins and a member match returns the first declaration using it.