sde

package
v0.1.13 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 30, 2026 License: MIT Imports: 16 Imported by: 0

Documentation

Overview

Package sde imports CCP's Static Data Export into Postgres.

The archive is ~99 MB compressed and ~575 MB across 79 JSONL members once expanded (mapMoons alone is 224 MB), so nothing here materialises a whole member. Members are read line by line with one decoded row in flight, and the zip is streamed to disk rather than held in memory because archive/zip needs random access.

Index

Constants

This section is empty.

Variables

View Source
var Derivations = []Derivation{
	{
		Name:        "inv_types.category_id",
		Description: "Denormalise category from the type's group",

		SQL: `
            UPDATE inv_types t
            SET category_id = (
                SELECT g.category_id FROM inv_groups g WHERE g.group_id = t.group_id
            )
            WHERE t.category_id IS DISTINCT FROM (
                SELECT g.category_id FROM inv_groups g WHERE g.group_id = t.group_id
            )
        `,
	},
	{
		Name:        "stations.region_id / constellation_id",
		Description: "Inherit region and constellation from the station's system",

		SQL: `
            UPDATE stations s
            SET region_id = (
                    SELECT sys.region_id FROM solar_systems sys
                    WHERE sys.solar_system_id = s.solar_system_id
                ),
                constellation_id = (
                    SELECT sys.constellation_id FROM solar_systems sys
                    WHERE sys.solar_system_id = s.solar_system_id
                )
            WHERE s.region_id IS DISTINCT FROM (
                    SELECT sys.region_id FROM solar_systems sys
                    WHERE sys.solar_system_id = s.solar_system_id
                )
               OR s.constellation_id IS DISTINCT FROM (
                    SELECT sys.constellation_id FROM solar_systems sys
                    WHERE sys.solar_system_id = s.solar_system_id
                )
        `,
	},
}

Derivations are ordered: solar_systems must be loaded before stations can inherit region and constellation from it.

View Source
var NestedTables = []Table{
	{

		Member:      "typeDogma",
		Name:        "type_dogma_attributes",
		PK:          []string{"type_id", "attribute_id"},
		PruneAbsent: true,
		Columns:     []string{"type_id", "attribute_id", "value"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			list := r.List("dogmaAttributes")
			out := make([][]any, 0, len(list))
			for _, item := range list {
				m, ok := item.(map[string]any)
				if !ok {
					continue
				}
				a := Row(m)
				attrID := a.Int("attributeID")
				if attrID == nil {
					continue
				}
				out = append(out, []any{int32(id), *attrID, a.Float("value")})
			}
			return out
		},
	},
	{

		Member:      "typeDogma",
		Name:        "type_dogma_effects",
		PK:          []string{"type_id", "effect_id"},
		PruneAbsent: true,
		Columns:     []string{"type_id", "effect_id", "is_default"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			list := r.List("dogmaEffects")
			out := make([][]any, 0, len(list))
			for _, item := range list {
				m, ok := item.(map[string]any)
				if !ok {
					continue
				}
				e := Row(m)
				effectID := e.Int("effectID")
				if effectID == nil {
					continue
				}
				out = append(out, []any{int32(id), *effectID, e.Bool("isDefault")})
			}
			return out
		},
	},
	{

		Member:      "typeMaterials",
		Name:        "type_materials",
		PK:          []string{"type_id", "material_type_id"},
		PruneAbsent: true,
		Columns:     []string{"type_id", "material_type_id", "quantity"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			list := r.List("materials")
			out := make([][]any, 0, len(list))
			for _, item := range list {
				m, ok := item.(map[string]any)
				if !ok {
					continue
				}
				mat := Row(m)
				matID := mat.Int("materialTypeID")
				if matID == nil {
					continue
				}
				out = append(out, []any{int32(id), *matID, mat.Int("quantity")})
			}
			return out
		},
	},
	{
		Member:      "blueprints",
		Name:        "blueprints",
		PK:          []string{"blueprint_type_id"},
		PruneAbsent: true,
		Columns:     []string{"blueprint_type_id", "max_production_limit"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			return [][]any{{int32(id), r.Int("maxProductionLimit")}}
		},
	},
	{

		Member:      "blueprints",
		Name:        "blueprint_activities",
		PK:          []string{"blueprint_type_id", "activity"},
		PruneAbsent: true,
		Columns:     []string{"blueprint_type_id", "activity", "time"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			var out [][]any
			forEachActivity(r, func(name string, a Row) {
				out = append(out, []any{int32(id), name, a.Int("time")})
			})
			return out
		},
	},
	{
		Member:      "blueprints",
		Name:        "blueprint_activity_materials",
		PK:          []string{"blueprint_type_id", "activity", "material_type_id"},
		PruneAbsent: true,
		Columns:     []string{"blueprint_type_id", "activity", "material_type_id", "quantity"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			var out [][]any
			forEachActivity(r, func(name string, a Row) {
				for _, item := range a.List("materials") {
					m, ok := item.(map[string]any)
					if !ok {
						continue
					}
					mat := Row(m)

					typeID := mat.Int("typeID")
					if typeID == nil {
						continue
					}
					out = append(out, []any{int32(id), name, *typeID, mat.Int("quantity")})
				}
			})
			return out
		},
	},
	{
		Member:      "blueprints",
		Name:        "blueprint_activity_products",
		PK:          []string{"blueprint_type_id", "activity", "product_type_id"},
		PruneAbsent: true,
		Columns:     []string{"blueprint_type_id", "activity", "product_type_id", "quantity", "probability"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			var out [][]any
			forEachActivity(r, func(name string, a Row) {
				for _, item := range a.List("products") {
					m, ok := item.(map[string]any)
					if !ok {
						continue
					}
					p := Row(m)
					typeID := p.Int("typeID")
					if typeID == nil {
						continue
					}
					out = append(out, []any{
						int32(id), name, *typeID, p.Int("quantity"), p.Float("probability"),
					})
				}
			})
			return out
		},
	},
	{
		Member:      "blueprints",
		Name:        "blueprint_activity_skills",
		PK:          []string{"blueprint_type_id", "activity", "skill_type_id"},
		PruneAbsent: true,
		Columns:     []string{"blueprint_type_id", "activity", "skill_type_id", "level"},
		Expand: func(r Row) [][]any {
			id, ok := r.Key()
			if !ok {
				return nil
			}
			var out [][]any
			forEachActivity(r, func(name string, a Row) {
				for _, item := range a.List("skills") {
					m, ok := item.(map[string]any)
					if !ok {
						continue
					}
					s := Row(m)
					typeID := s.Int("typeID")
					if typeID == nil {
						continue
					}
					out = append(out, []any{int32(id), name, *typeID, s.Int("level")})
				}
			})
			return out
		},
	},
}

Members whose payload is nested, so one archive record becomes many rows.

These use Table.Expand rather than Table.Values. The row counts are wildly different from the record counts: typeDogma has one record per type but produces 645 k attribute rows, and one blueprint record produces rows across five tables.

These are registered separately from Tables so the simple one-to-one mappings stay readable, but they load through exactly the same COPY-to- staging-then-merge path.

View Source
var StationNameDerivation = Derivation{
	Name:        "stations.station_name",
	Description: "Copy the composed name from the station's celestial row",
	SQL: `
        UPDATE stations s
        SET station_name = (
            SELECT c.item_name FROM celestials c WHERE c.item_id = s.station_id
        )
        WHERE s.station_name IS DISTINCT FROM (
            SELECT c.item_name FROM celestials c WHERE c.item_id = s.station_id
        )
    `,
}

StationNameDerivation is separate from Derivations because it depends on celestials, which are imported after the declarative tables. Verified against production: stations.station_name is identical to the station's celestial name for all 5,210 rows.

View Source
var Tables = []Table{
	{
		Member: "types",
		Name:   "inv_types",
		PK:     []string{"type_id"},

		Columns: []string{
			"type_id", "group_id", "name", "description", "mass", "volume",
			"capacity", "portion_size", "packaged_volume", "radius", "published",
			"market_group_id", "icon_id", "sound_id", "graphic_id", "meta_group_id",
			"race_id", "faction_id", "base_price", "variation_parent_type_id",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Int("groupID"), r.Lang("name"), r.Lang("description"),
				r.Float("mass"), r.Float("volume"), r.Float("capacity"),
				r.Int("portionSize"), r.Float("packagedVolume"), r.Float("radius"),
				r.Bool("published"), r.Int("marketGroupID"), r.Int("iconID"),
				r.Int("soundID"), r.Int("graphicID"), r.Int("metaGroupID"),
				r.Int("raceID"), r.Int("factionID"), r.Float("basePrice"),
				r.Int("variationParentTypeID"),
			}, true
		},
	},
	{
		Member:  "groups",
		Name:    "inv_groups",
		PK:      []string{"group_id"},
		Columns: []string{"group_id", "category_id", "name", "published", "icon_id"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Int("categoryID"), r.Lang("name"),
				r.Bool("published"), r.Int("iconID"),
			}, true
		},
	},
	{
		Member:  "categories",
		Name:    "inv_categories",
		PK:      []string{"category_id"},
		Columns: []string{"category_id", "name", "published", "icon_id"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{int32(id), r.Lang("name"), r.Bool("published"), r.Int("iconID")}, true
		},
	},
	{
		Member:      "metaGroups",
		Name:        "inv_meta_groups",
		PK:          []string{"meta_group_id"},
		PruneAbsent: true,
		Columns:     []string{"meta_group_id", "name", "icon_id", "icon_suffix"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{int32(id), r.Lang("name"), r.Int("iconID"), r.Str("iconSuffix")}, true
		},
	},
	{
		Member:  "marketGroups",
		Name:    "inv_market_groups",
		PK:      []string{"market_group_id"},
		Columns: []string{"market_group_id", "parent_group_id", "name", "description", "icon_id", "has_types"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Int("parentGroupID"), r.Lang("name"),
				r.Lang("description"), r.Int("iconID"), r.Bool("hasTypes"),
			}, true
		},
	},
	{
		Member:      "mapRegions",
		Name:        "regions",
		PK:          []string{"region_id"},
		PruneAbsent: true,

		Columns: []string{
			"region_id", "name", "description", "center_x", "center_y", "center_z",
			"faction_id", "nebula_id", "wormhole_class_id",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			x, y, z := xyz(r, "position")
			return []any{
				int32(id), r.Lang("name"), r.Lang("description"), x, y, z,
				r.Int("factionID"), r.Int("nebulaID"), r.Int("wormholeClassID"),
			}, true
		},
	},
	{
		Member:      "mapConstellations",
		Name:        "constellations",
		PK:          []string{"constellation_id"},
		PruneAbsent: true,
		Columns:     []string{"constellation_id", "constellation_name", "region_id", "x", "y", "z", "faction_id", "radius"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			x, y, z := xyz(r, "position")
			return []any{
				int32(id), r.Lang("name"), r.Int("regionID"),
				x, y, z, r.Int("factionID"), r.Float("radius"),
			}, true
		},
	},
	{
		Member:      "mapSolarSystems",
		Name:        "solar_systems",
		PK:          []string{"solar_system_id"},
		PruneAbsent: true,
		Columns: []string{
			"solar_system_id", "system_name", "constellation_id", "region_id",
			"x", "y", "z", "x2d", "z2d", "security", "security_class", "faction_id",
			"border", "fringe", "corridor", "hub", "international", "regional",
			"luminosity", "radius", "sun_type_id",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			x, y, z := xyz(r, "position")
			// The 2D map projection stores the source y as z2d — it is a
			// top-down projection, so the vertical axis is dropped and what the
			// SDE calls y is the second screen axis.
			var x2d, z2d *float64
			if p := r.Map("position2D"); p != nil {
				x2d = Row(p).Float("x")
				z2d = Row(p).Float("y")
			}
			return []any{
				int32(id), r.Lang("name"), r.Int("constellationID"), r.Int("regionID"),
				x, y, z, x2d, z2d, r.Float("securityStatus"), r.Str("securityClass"),
				r.Int("factionID"), r.Bool("border"), r.Bool("fringe"), r.Bool("corridor"),
				r.Bool("hub"), r.Bool("international"), r.Bool("regional"),
				r.Float("luminosity"), r.Float("radius"), r.Int("sunTypeID"),
			}, true
		},
	},
	{
		Member:      "factions",
		Name:        "factions",
		PK:          []string{"faction_id"},
		PruneAbsent: true,
		Columns: []string{
			"faction_id", "name", "description", "corporation_id",
			"militia_corporation_id", "solar_system_id", "icon_id", "size_factor",

			"race_ids",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Lang("name"), r.Lang("description"),
				r.Int("corporationID"), r.Int("militiaCorporationID"),
				r.Int("solarSystemID"), r.Int("iconID"), r.Float("sizeFactor"),
				int32Slice(r.List("memberRaces")),
			}, true
		},
	},
	{
		Member:      "bloodlines",
		Name:        "bloodlines",
		PK:          []string{"bloodline_id"},
		PruneAbsent: true,
		Columns: []string{
			"bloodline_id", "bloodline_name", "race_id", "description",
			"male_description", "female_description", "short_description",
			"short_male_description", "short_female_description", "ship_type_id",
			"corporation_id", "perception", "willpower", "charisma", "memory",
			"intelligence", "icon_id",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Lang("name"), r.Int("raceID"), r.Lang("description"),
				r.Lang("maleDescription"), r.Lang("femaleDescription"),
				r.Lang("shortDescription"), r.Lang("shortMaleDescription"),
				r.Lang("shortFemaleDescription"), r.Int("shipTypeID"),
				r.Int("corporationID"), r.Int("perception"), r.Int("willpower"),
				r.Int("charisma"), r.Int("memory"), r.Int("intelligence"), r.Int("iconID"),
			}, true
		},
	},
	{
		Member:      "races",
		Name:        "races",
		PK:          []string{"race_id"},
		PruneAbsent: true,
		Columns:     []string{"race_id", "race_name", "description", "short_description", "icon_id"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Lang("name"), r.Lang("description"),
				r.Lang("shortDescription"), r.Int("iconID"),
			}, true
		},
	},
	{
		Member:      "npcCorporations",
		Name:        "npc_corporations",
		PK:          []string{"corporation_id"},
		PruneAbsent: true,
		Columns: []string{
			"corporation_id", "name", "description", "ticker_name", "ceo_id",
			"station_id", "size", "extent", "tax_rate", "deleted",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Lang("name"), r.Lang("description"), r.Str("tickerName"),
				r.Int("ceoID"), r.Int("stationID"), r.Str("size"), r.Str("extent"),
				r.Float("taxRate"), r.Bool("deleted"),
			}, true
		},
	},
	{
		Member:      "stationOperations",
		Name:        "station_operations",
		PK:          []string{"operation_id"},
		PruneAbsent: true,
		Columns:     []string{"operation_id", "operation_name", "description", "activity_id", "services"},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			return []any{
				int32(id), r.Lang("operationName"), r.Lang("description"),
				r.Int("activityID"), int32Slice(r.List("services")),
			}, true
		},
	},
	{
		Member:      "npcStations",
		Name:        "stations",
		PK:          []string{"station_id"},
		PruneAbsent: true,

		Columns: []string{
			"station_id", "type_id", "corporation_id", "solar_system_id",
			"operation_id", "celestial_index", "orbit_id", "orbit_index",
			"x", "y", "z", "reprocessing_efficiency", "reprocessing_stations_take",
		},
		Values: func(r Row) ([]any, bool) {
			id, ok := r.Key()
			if !ok {
				return nil, false
			}
			x, y, z := xyz(r, "position")
			return []any{
				int32(id), r.Int("typeID"), r.Int("ownerID"), r.Int("solarSystemID"),
				r.Int("operationID"), r.Int("celestialIndex"),
				r.Int("orbitID"), r.Int("orbitIndex"), x, y, z,
				r.Float("reprocessingEfficiency"), r.Float("reprocessingStationsTake"),
			}, true
		},
	},
}

Tables declares the straightforward one-member-to-one-table imports, transcribed from the importTable() calls in backend/src/commands/UpdateSdeCommand.ts.

Column types come from internal/dbgen (generated from the schema), which is what decides the accessor used for each field: Int for integer columns, Float for double precision, Lang for localised text, Str for bare strings.

Not covered here, because they are not one-to-one and get their own passes: typeDogma (two tables), typeMaterials, blueprints (five tables), the map celestial members (five members into one table), stargate-derived system jumps, inv_flags (a hardcoded list, not in the archive), and the EVE Ref feeds.

Functions

func LoadedBuild

func LoadedBuild(ctx context.Context, pool *pgxpool.Pool) (int64, string, error)

LoadedBuild reads back what RecordBuild stored. Returns 0 when nothing has been imported yet.

func RecordBuild

func RecordBuild(ctx context.Context, pool *pgxpool.Pool, build int64, release string) error

RecordBuild stores the imported build number in the config table so sde:status can report what is loaded and the cron can skip an unchanged build.

func SeedInvFlags

func SeedInvFlags(ctx context.Context, pool *pgxpool.Pool) (int64, string, error)

SeedInvFlags upserts the bundled flag list.

Types

type Derivation

type Derivation struct {
	Name        string
	Description string
	SQL         string
}

Derivation is a post-import fill for a column the archive does not carry directly but that can be computed from data already loaded.

These run after every table is in place, because they read across tables.

type DeriveResult

type DeriveResult struct {
	Name    string `json:"name"`
	Rows    int64  `json:"rows"`
	Elapsed string `json:"elapsed"`
}

DeriveResult reports one derivation.

func Derive

func Derive(ctx context.Context, pool *pgxpool.Pool) ([]DeriveResult, error)

Derive runs every derivation in order.

func DeriveOne

func DeriveOne(ctx context.Context, pool *pgxpool.Pool, d Derivation) (DeriveResult, error)

DeriveOne runs a single derivation.

type ExternalResult

type ExternalResult struct {
	Name    string `json:"name"`
	Rows    int64  `json:"rows"`
	Elapsed string `json:"elapsed"`
}

ExternalResult reports one external feed.

func GenerateSupercapPrices

func GenerateSupercapPrices(ctx context.Context, pool *pgxpool.Pool) (ExternalResult, error)

GenerateSupercapPrices derives supercapital values from blueprint materials.

Requires the prices table, which is populated by the market-history importer rather than by the SDE. With no prices loaded every hull costs zero and nothing is written — that is the correct outcome, not a failure, so this reports zero rows instead of erroring.

func ImportInsurancePrices

func ImportInsurancePrices(ctx context.Context, pool *pgxpool.Pool, userAgent string) (ExternalResult, error)

ImportInsurancePrices loads ship insurance costs and payouts.

The feed nests six levels per type — Basic through Platinum — which flatten into one row each.

func ImportStructures

func ImportStructures(ctx context.Context, pool *pgxpool.Pool, userAgent string) (ExternalResult, error)

ImportStructures loads player-owned structures observed by EVE Ref.

Unlike everything else here this is not static data — structures are built and destroyed constantly — but killmails reference them as locations, so the names have to come from somewhere.

func SeedManualCustomPrices

func SeedManualCustomPrices(ctx context.Context, pool *pgxpool.Pool) (ExternalResult, error)

SeedManualCustomPrices writes the hand-maintained price list.

type ImportOptions

type ImportOptions struct {
	CacheDir     string
	UserAgent    string
	Progress     func(string)
	SkipExternal bool
}

ImportOptions controls a complete published-build import.

type ImportResult

type ImportResult struct {
	BuildNumber int64
	ReleaseDate string
	Read        int64
	Written     int64
	Pruned      int64
	Duration    time.Duration
	Elapsed     string

	Tables      []LoadResult
	Celestials  LoadResult
	SystemJumps LoadResult
	InvFlags    int64
	Dust514     SeedResult
	External    []ExternalResult
	Derived     []DeriveResult
}

ImportResult summarises a complete SDE import. The detailed per-source results are retained so both the CLI and scheduled worker can report useful progress without duplicating the orchestration.

func ImportBuild

func ImportBuild(
	ctx context.Context,
	pool *pgxpool.Pool,
	manifest *Manifest,
	opts ImportOptions,
) (ImportResult, error)

ImportBuild performs the complete import for a published SDE build.

RecordBuild deliberately runs last. Every operation before it is idempotent, so an interrupted import can safely converge on its next run, while the build marker never claims a partial database is current.

type LoadResult

type LoadResult struct {
	Table    string        `json:"table"`
	Member   string        `json:"member"`
	Read     int64         `json:"read"`
	Written  int64         `json:"written"`
	Skipped  int64         `json:"skipped"`
	Pruned   int64         `json:"pruned"`
	Duration time.Duration `json:"-"`
	Elapsed  string        `json:"elapsed"`
	Missing  bool          `json:"missing,omitempty"`
}

LoadResult reports what a single table import did.

func ImportCelestials

func ImportCelestials(ctx context.Context, pool *pgxpool.Pool, src *Source) (LoadResult, error)

ImportCelestials runs every pass and merges the result.

func ImportSystemJumps

func ImportSystemJumps(ctx context.Context, pool *pgxpool.Pool, src *Source) (LoadResult, error)

ImportSystemJumps builds the adjacency table.

func Load

func Load(ctx context.Context, pool *pgxpool.Pool, t Table, src *Source) (LoadResult, error)

Load streams a member into its table.

The write path is COPY into a TEMP table followed by INSERT ... SELECT ... ON CONFLICT DO UPDATE, rather than batched upserts. Two reasons: COPY is several times faster than even large multi-row INSERTs, and the merge then happens in one statement so the destination table is never half-updated. The import is idempotent, which matters because it runs daily from a cron.

Everything runs on a single dedicated connection because a TEMP table is session-scoped — with a pool, the COPY and the merge could otherwise land on different connections and the merge would not see the staged rows.

type Manifest

type Manifest struct {
	Key         string `json:"_key"`
	BuildNumber int64  `json:"buildNumber"`
	ReleaseDate string `json:"releaseDate"`
}

Manifest is the content of latest.jsonl.

func FetchManifest

func FetchManifest(ctx context.Context, userAgent string) (*Manifest, error)

FetchManifest reports the currently published build without downloading it.

type Row

type Row map[string]any

Row is one decoded JSONL object.

SDE JSON is loose in ways that matter: every number decodes as float64, some numeric fields arrive as strings, booleans are sometimes 0/1, and localised text is either a bare string or a {lang: text} map. The accessors below are the single place those quirks are handled, so table definitions stay declarative.

All of them return pointers, because "absent" and "zero" are different things here — a missing volume must become SQL NULL, not 0.0, or downstream arithmetic silently treats unknown as free.

func (Row) Bool

func (r Row) Bool(field string) *bool

Bool returns a boolean field, accepting 0/1 and "true"/"false".

func (Row) Float

func (r Row) Float(field string) *float64

Float returns a numeric field, accepting numeric strings.

func (Row) Int

func (r Row) Int(field string) *int32

Int returns an integer field as int32, the width sqlc generates for Postgres `integer` columns. Values beyond int32 are dropped rather than silently wrapped — a wrapped type ID would corrupt a join.

func (Row) Int64

func (r Row) Int64(field string) *int64

Int64 returns an integer field at full width, for bigint columns.

func (Row) Key

func (r Row) Key() (int64, bool)

Key returns the row's primary identifier. Every SDE record carries "_key".

Zero is rejected. The archive ships placeholder records at _key 0 — a "#System" entry appears in types, groups and categories — and importing them is actively harmful, not merely untidy: EVE data uses 0 as the "no entity" sentinel, so a real inv_types row with type_id 0 would make every join against an absent ship or item spuriously resolve to "#System".

This also matches the TypeScript importer, whose `if (!id) continue` drops zero as falsy.

func (Row) Lang

func (r Row) Lang(field string) *string

Lang extracts localised text, preferring English.

Falls back to any other non-empty translation rather than returning nothing: a name in Japanese is more useful than NULL, and a handful of SDE records genuinely lack "en".

func (Row) List

func (r Row) List(field string) []any

List returns an array field for callers that need to walk nested structures (blueprint activities, dogma attribute lists).

func (Row) Map

func (r Row) Map(field string) map[string]any

Map returns an object field, used the same way as List.

func (Row) Str

func (r Row) Str(field string) *string

Str returns a bare string field.

type SeedResult

type SeedResult struct {
	Categories   int64  `json:"categories"`
	Groups       int64  `json:"groups"`
	MarketGroups int64  `json:"market_groups"`
	Types        int64  `json:"types"`
	Elapsed      string `json:"elapsed"`
}

SeedResult reports what the Dust 514 seed wrote.

func SeedDust514

func SeedDust514(ctx context.Context, pool *pgxpool.Pool) (SeedResult, error)

SeedDust514 upserts the bundled data.

Runs after the archive import so that where CCP still ships a record, the live archive wins on the first pass and this only fills what is missing — both write the same rows, so the order is about provenance, not conflict.

type Source

type Source struct {
	Build   int64
	Release string
	Path    string
	// contains filtered or unexported fields
}

Source is an opened SDE archive.

func Open

func Open(ctx context.Context, cacheDir string, m *Manifest, userAgent string, progress func(string)) (*Source, error)

Open returns the archive for a build, downloading it into cacheDir only when not already present. Caching by build number matters during development: a re-import otherwise re-fetches 99 MB every run, and the archive for a given build is immutable.

func (*Source) Close

func (s *Source) Close() error

func (*Source) Has

func (s *Source) Has(member string) bool

Has reports whether a member exists. CCP adds and removes members between builds, so a caller may legitimately skip a missing one.

func (*Source) Members

func (s *Source) Members() []string

Members lists every JSONL member, for diagnostics.

func (*Source) Stream

func (s *Source) Stream(ctx context.Context, member string, fn func(Row) error) error

Stream decodes each line of a member and hands it to fn. Iteration stops at the first error fn returns, and at ctx cancellation.

fn must not retain the Row: it is reused between lines to avoid an allocation per record, which matters across the ~1.5 M rows of a full import.

type Table

type Table struct {
	// Member is the archive member without its .jsonl suffix.
	Member string

	// Name is the destination table.
	Name string

	// PK is the conflict target for the upsert.
	PK []string

	// Columns are the columns written, in the order Values returns them.
	Columns []string

	// Values converts a row to column values. Returning false skips the row —
	// used for records with no usable "_key".
	//
	// Exactly one of Values or Expand must be set.
	Values func(Row) ([]any, bool)

	// Expand is the one-to-many form: a single archive record yields any number
	// of destination rows. Several members nest their real payload in an array
	// or object — one typeDogma record carries every attribute for a type, one
	// blueprint carries every activity — so the row count in the archive bears
	// no relation to the row count in the table.
	Expand func(Row) [][]any

	// Optional marks members that may legitimately be absent from an archive.
	// CCP adds and drops members between builds.
	Optional bool

	// PruneAbsent makes the archive member authoritative for this table.
	//
	// An upsert alone leaves rows behind when CCP removes a nested dogma
	// attribute, material, blueprint activity, or other record from a later
	// build. Those rows are no longer static data; they are stale data. Tables
	// set to true are merged and pruned in one transaction so readers never see
	// a half-updated snapshot.
	//
	// Leave this false for tables augmented from another source. In particular,
	// inv_types, inv_groups, inv_categories, and inv_market_groups also contain
	// bundled Dust 514 records which are intentionally absent from CCP's
	// current archive.
	PruneAbsent bool
}

Table declares how one JSONL member maps onto one Postgres table.

Deliberately data, not code: adding a table is a literal in tables.go rather than a new function, which keeps the 14 straightforward imports readable side by side and makes them diffable against the TypeScript they came from.

func AllTables

func AllTables() []Table

AllTables is every declarative import, simple and nested.

func TableByName

func TableByName(name string) *Table

TableByName finds a declaration, for --only. Matching on member name is ambiguous for nested members (typeDogma feeds two tables, blueprints five), so table name wins and a member match returns the first declaration using it.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL