dat

package
v1.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 20, 2025 License: MIT Imports: 14 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// NullRowSentinel is the sentinel value indicating a null row reference
	NullRowSentinel uint32 = 0xfefe_fefe

	// LongIDNullSentinel is the 64-bit sentinel value for null LongID references
	LongIDNullSentinel uint64 = 0xfefe_fefe_fefe_fefe

	// SentinelValue32 is the 32-bit sentinel value for null strings/arrays
	SentinelValue32 uint32 = 0xFEFEFEFE

	// SentinelValue64 is the 64-bit sentinel value for null strings/arrays
	SentinelValue64 uint64 = 0xFEFEFEFEFEFEFEFE

	// MaxReasonableForeignKeyIndex is the maximum reasonable value for foreign key indices
	MaxReasonableForeignKeyIndex uint32 = 100_000_000

	// MinDATFileSize is the minimum size for a valid DAT file (4 bytes for row count + 8 bytes boundary)
	MinDATFileSize = 12

	// MaxRowCount is the maximum reasonable number of rows in a DAT file
	MaxRowCount = 10_000_000

	// MinOffsetForArraysAndStrings is the minimum offset value for arrays and strings in dynamic data
	MinOffsetForArraysAndStrings = 8

	// ElementSize64BitForeignRow is the element size for foreign/enum row arrays in 64-bit width
	ElementSize64BitForeignRow = 16

	// UTF-16 surrogate pair constants for Unicode decoding
	UTF16HighSurrogateStart      = 0xD800
	UTF16HighSurrogateEnd        = 0xDBFF
	UTF16LowSurrogateStart       = 0xDC00
	UTF16SupplementaryPlaneStart = 0x10000

	// Default configuration values
	DefaultMaxStringLength           = 65536 // 64KB max string length
	DefaultMaxArrayCount             = 65536 // Match reference implementations
	DefaultArraySizeWarningThreshold = 1000  // Warn when arrays exceed 1000 elements
)
View Source
const CommunitySchemaURL = "https://github.com/poe-tool-dev/dat-schema/releases/download/latest/schema.min.json"

CommunitySchemaURL is the official URL for the community schema

Variables

View Source
var BoundaryMarker = []byte{0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb}

Functions

This section is empty.

Types

type ColumnReference

type ColumnReference struct {
	Table  string  `json:"table"`            // Referenced table name
	Column *string `json:"column,omitempty"` // Referenced column name (optional)
}

ColumnReference represents a foreign key relationship from the community schema

type CommunitySchema

type CommunitySchema struct {
	SchemaMetadata
	Tables       []TableSchema       `json:"tables"`       // Table definitions
	Enumerations []SchemaEnumeration `json:"enumerations"` // Enumeration definitions
}

CommunitySchema represents the complete community schema file structure

func (*CommunitySchema) FilterTablesForVersion

func (schema *CommunitySchema) FilterTablesForVersion(tableNames []string, version string) ([]string, error)

FilterTablesForVersion filters a list of table names to only include those valid for the given version

func (*CommunitySchema) GetSchemaTableNames

func (schema *CommunitySchema) GetSchemaTableNames() []string

GetSchemaTableNames returns all table names from the schema

func (*CommunitySchema) GetTableSchema

func (cs *CommunitySchema) GetTableSchema(tableName string, gameVersion string) (*TableSchema, error)

GetTableSchema finds a table schema by name filtered by game version compatibility

func (*CommunitySchema) GetValidTables

func (cs *CommunitySchema) GetValidTables(gameVersion int) []TableSchema

GetValidTables returns all tables that are valid for the given game version

type DATParser

type DATParser struct {
	// contains filtered or unexported fields
}

DATParser parses Path of Exile DAT files

func NewDATParser

func NewDATParser() *DATParser

func (*DATParser) CalculateRowSize

func (p *DATParser) CalculateRowSize(schema *TableSchema, width ParserWidth) int

func (*DATParser) ParseDATFile

func (p *DATParser) ParseDATFile(ctx context.Context, r io.Reader, schema *TableSchema) (*ParsedTable, error)

func (*DATParser) ParseDATFileWithFilename

func (p *DATParser) ParseDATFileWithFilename(ctx context.Context, r io.Reader, filename string, schema *TableSchema) (*ParsedTable, error)

func (*DATParser) ReadArray

func (p *DATParser) ReadArray(dynamicData []byte, offset uint64, count uint64, elementType FieldType, state *parseState) (interface{}, error)

func (*DATParser) ReadString

func (p *DATParser) ReadString(dynamicData []byte, offset uint64, state *parseState) (string, error)

type DatFile

type DatFile struct {
	// RowCount is the number of rows in the DAT file (first 4 bytes)
	RowCount int

	// RowLength is the fixed size of each row in bytes
	RowLength int

	// FixedData contains the fixed-size row data section
	FixedData []byte

	// DynamicData contains the variable-length data section (strings, arrays)
	// Starts after the boundary marker (see BoundaryMarker in parser.go)
	DynamicData []byte
}

DatFile represents the structure of a parsed DAT file

type FieldType

type FieldType string

FieldType represents a DAT file field type from the community schema

const (
	TypeBool    FieldType = "bool"
	TypeString  FieldType = "string"
	TypeInt16   FieldType = "i16"
	TypeUint16  FieldType = "u16"
	TypeInt32   FieldType = "i32"
	TypeUint32  FieldType = "u32"
	TypeInt64   FieldType = "i64"
	TypeUint64  FieldType = "u64"
	TypeFloat32 FieldType = "f32"
	TypeFloat64 FieldType = "f64"

	// Reference types
	TypeRow        FieldType = "row"        // row index (references column in same table)
	TypeForeignRow FieldType = "foreignrow" // row index (references column in foreign table)
	TypeEnumRow    FieldType = "enumrow"    // row index (references foreign table with no columns)
	TypeLongID     FieldType = "longid"     // 64-bit row reference (foreign table)

	// Array type (used when array: true and type is one of the above)
	TypeArray FieldType = "array" // column is an array of unknown type
)

FieldType constants matching the community schema specification

func (FieldType) Size

func (ft FieldType) Size(width ...ParserWidth) int

Size returns the fixed size in bytes for this field type in DAT files Returns 0 for variable-length types (strings, arrays) If width is provided, it uses that width, otherwise defaults to 64-bit for backward compatibility

func (FieldType) Valid

func (ft FieldType) Valid() bool

Valid checks if the FieldType is a valid type from the community schema

type ParseMetadata

type ParseMetadata struct {
	FixedDataSize   int `json:"fixedDataSize"`   // Size of fixed data section
	DynamicDataSize int `json:"dynamicDataSize"` // Size of dynamic data section
	TotalFileSize   int `json:"totalFileSize"`   // Total DAT file size
	MaxFieldsParsed int `json:"maxFieldsParsed"` // Maximum number of fields parsed in any row (for partial schema)
}

ParseMetadata contains metadata about the parsing operation

type ParsedRow

type ParsedRow struct {
	Index        int                    `json:"index"`        // Row index (0-based)
	Fields       map[string]interface{} `json:"fields"`       // Field name to parsed value mapping
	FieldsParsed int                    `json:"fieldsParsed"` // Number of fields successfully parsed (useful for partial schema)
}

ParsedRow represents a single parsed row from a DAT table

type ParsedTable

type ParsedTable struct {
	Schema   *TableSchema   `json:"schema"`   // Original table schema
	RowCount int            `json:"rowCount"` // Number of rows parsed
	Rows     []ParsedRow    `json:"rows"`     // All parsed row data
	Metadata *ParseMetadata `json:"metadata"` // Parsing metadata
}

ParsedTable represents a completely parsed DAT table with all rows and metadata

type ParserOptions

type ParserOptions struct {
	// StrictMode enables strict validation of DAT file format
	StrictMode bool

	// ValidateReferences checks that foreign key references point to valid rows
	ValidateReferences bool

	// MaxStringLength limits the maximum length of strings to prevent memory issues
	MaxStringLength int

	// MaxArrayCount limits the maximum number of array elements
	MaxArrayCount int

	// ArraySizeWarningThreshold sets the threshold for logging warnings about large arrays
	ArraySizeWarningThreshold int
}

ParserOptions configures DAT parsing behavior

type ParserWidth

type ParserWidth int

ParserWidth represents the bit width of the DAT file parser

const (
	// Width32 represents 32-bit DAT files (.dat, .datl)
	Width32 ParserWidth = 32
	// Width64 represents 64-bit DAT files (.dat64, .datl64)
	Width64 ParserWidth = 64
)

func WidthForFilename

func WidthForFilename(filename string) ParserWidth

WidthForFilename determines the parser width based on the DAT file extension

type SchemaEnumeration

type SchemaEnumeration struct {
	ValidFor    ValidFor  `json:"validFor"`    // Game version compatibility
	Name        string    `json:"name"`        // Enumeration name
	Indexing    int       `json:"indexing"`    // 0 or 1 based indexing
	Enumerators []*string `json:"enumerators"` // Enumeration values (nullable)
}

SchemaEnumeration represents an enumeration definition from the community schema

type SchemaManager

type SchemaManager struct {
	// contains filtered or unexported fields
}

SchemaManager implements SchemaManager using a cached schema file

func NewSchemaManager

func NewSchemaManager() (*SchemaManager, error)

LoadCachedSchema loads the schema from cache or downloads it if not available

func (*SchemaManager) GetTableSchema

func (sm *SchemaManager) GetTableSchema(tableName string, gameVersion string) (*TableSchema, error)

GetTableSchema retrieves a table schema by name filtered by game version compatibility

func (*SchemaManager) GetValidTablesForVersion

func (sm *SchemaManager) GetValidTablesForVersion(version string) ([]TableSchema, error)

GetValidTablesForVersion returns all tables valid for the given game version

func (*SchemaManager) IsTableValidForVersion

func (sm *SchemaManager) IsTableValidForVersion(tableName, version string) (bool, error)

IsTableValidForVersion checks if a table is valid for the given game version

func (*SchemaManager) LoadSchema

func (sm *SchemaManager) LoadSchema() (*CommunitySchema, error)

LoadSchema returns the cached schema

type SchemaMetadata

type SchemaMetadata struct {
	Version   int `json:"version"`   // Schema version number
	CreatedAt int `json:"createdAt"` // Unix timestamp when schema was created
}

SchemaMetadata contains metadata about the community schema

type TableColumn

type TableColumn struct {
	Name        *string          `json:"name"`
	Description *string          `json:"description"`
	Array       bool             `json:"array"`
	Type        FieldType        `json:"type"`
	Unique      bool             `json:"unique"`
	Localized   bool             `json:"localized"`
	Until       *string          `json:"until"`      // Version when this column was removed
	References  *ColumnReference `json:"references"` // Foreign key reference
	File        *string          `json:"file"`       // File extension for asset files
	Files       []string         `json:"files"`      // Multiple file extensions
	Interval    bool             `json:"interval"`   // Whether this is an interval field
}

TableColumn represents a column definition from the community schema

type TableSchema

type TableSchema struct {
	ValidFor ValidFor      `json:"validFor"` // Game version compatibility
	Name     string        `json:"name"`     // Table name (matches DAT filename)
	Columns  []TableColumn `json:"columns"`  // Column definitions
	Tags     []string      `json:"tags"`     // Metadata tags
}

TableSchema represents a complete table definition from the community schema

func (*TableSchema) GetArrayColumns

func (table *TableSchema) GetArrayColumns() []TableColumn

GetArrayColumns returns all columns in a table that are arrays

func (*TableSchema) GetColumnByName

func (table *TableSchema) GetColumnByName(columnName string) (*TableColumn, bool)

GetColumnByName finds a column by name in a table schema

func (*TableSchema) GetColumnNames

func (table *TableSchema) GetColumnNames() []string

GetColumnNames returns all column names for a given table

func (*TableSchema) GetForeignKeyColumns

func (table *TableSchema) GetForeignKeyColumns() []TableColumn

GetForeignKeyColumns returns all columns in a table that are foreign key references

func (*TableSchema) GetLocalizedColumns

func (table *TableSchema) GetLocalizedColumns() []TableColumn

GetLocalizedColumns returns all columns in a table that are localized

func (*TableSchema) HasColumn

func (table *TableSchema) HasColumn(columnName string) bool

HasColumn checks if a table has a column with the given name

type ValidFor

type ValidFor int

ValidFor represents game version compatibility flags

const (
	ValidForPoE1 ValidFor = 0x01 // Path of Exile 1 only
	ValidForPoE2 ValidFor = 0x02 // Path of Exile 2 only
	ValidForBoth ValidFor = 0x03 // Both games (PoE1 | PoE2)
)

func (ValidFor) IsValidForGame

func (vf ValidFor) IsValidForGame(gameVersion int) bool

IsValidForGame checks if a ValidFor flag is compatible with the given game version

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL