migrate

package
v0.2.7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 29, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func MigrationRecordToInternalObject

func MigrationRecordToInternalObject(record MigrationRecord) (models.InternalObject, error)

func TransformBatch

func TransformBatch(records []IndexdRecord, defaultAuthz []string, now time.Time) ([]MigrationRecord, []TransformError)

func Validate

func Validate(record MigrationRecord) error

Types

type AuthConfig

type AuthConfig struct {
	BearerToken string
	Basic       *BasicAuth
}

type BasicAuth

type BasicAuth struct {
	Username string
	Password string
}

type Config

type Config struct {
	BatchSize    int
	Limit        int
	DryRun       bool
	DefaultAuthz []string
	Sweeps       int
}

type DumpReader

type DumpReader interface {
	ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error
}

type HTTPClient

type HTTPClient struct {
	// contains filtered or unexported fields
}

func NewHTTPClient

func NewHTTPClient(baseURL string, auth AuthConfig, httpClient *http.Client) (*HTTPClient, error)

func (*HTTPClient) ListPage

func (c *HTTPClient) ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error)

func (*HTTPClient) LoadBatch

func (c *HTTPClient) LoadBatch(ctx context.Context, records []MigrationRecord) error

type IndexdPage

type IndexdPage struct {
	Records []IndexdRecord `json:"records"`
	IDs     []string       `json:"ids"`
	Start   string         `json:"start"`
	Page    *int           `json:"page,omitempty"`
	Limit   *int           `json:"limit,omitempty"`
}

type IndexdRecord

type IndexdRecord struct {
	DID                string            `json:"did"`
	Size               *int64            `json:"size,omitempty"`
	FileName           *string           `json:"file_name,omitempty"`
	Version            *string           `json:"version,omitempty"`
	Description        *string           `json:"description,omitempty"`
	URLs               []string          `json:"urls,omitempty"`
	Hashes             map[string]string `json:"hashes,omitempty"`
	Authz              []string          `json:"authz,omitempty"`
	CreatedDate        *string           `json:"created_date,omitempty"`
	UpdatedDate        *string           `json:"updated_date,omitempty"`
	CreatedTime        *string           `json:"created_time,omitempty"`
	UpdatedTime        *string           `json:"updated_time,omitempty"`
	ContentCreatedDate *string           `json:"content_created_date,omitempty"`
	ContentUpdatedDate *string           `json:"content_updated_date,omitempty"`
	Aliases            []string          `json:"aliases,omitempty"`
	Form               *string           `json:"form,omitempty"`

	Baseid       *string                `json:"baseid,omitempty"`
	Rev          *string                `json:"rev,omitempty"`
	Uploader     *string                `json:"uploader,omitempty"`
	ACL          []string               `json:"acl,omitempty"`
	Metadata     map[string]interface{} `json:"metadata,omitempty"`
	URLsMetadata map[string]interface{} `json:"urls_metadata,omitempty"`
}

IndexdRecord is the source record shape returned by Gen3 Indexd APIs. Deprecated Indexd fields are accepted so source payloads decode cleanly, but they are intentionally not loaded into Syfon.

type Loader

type Loader interface {
	LoadBatch(ctx context.Context, records []MigrationRecord) error
}

type MigrationRecord

type MigrationRecord struct {
	ID            string             `json:"id"`
	Name          *string            `json:"name,omitempty"`
	Size          int64              `json:"size"`
	Version       *string            `json:"version,omitempty"`
	Description   *string            `json:"description,omitempty"`
	CreatedTime   time.Time          `json:"created_time"`
	UpdatedTime   *time.Time         `json:"updated_time,omitempty"`
	Checksums     []drs.Checksum     `json:"checksums"`
	AccessMethods []drs.AccessMethod `json:"access_methods,omitempty"`
	Authz         []string           `json:"authz,omitempty"`
}

func Transform

func Transform(rec IndexdRecord, defaultAuthz []string, now time.Time) (MigrationRecord, error)

type MigrationRequest

type MigrationRequest struct {
	Records []MigrationRecord `json:"records"`
}

type SQLiteDump

type SQLiteDump struct {
	// contains filtered or unexported fields
}

func OpenExistingSQLiteDump

func OpenExistingSQLiteDump(path string) (*SQLiteDump, error)

func OpenSQLiteDump

func OpenSQLiteDump(path string) (*SQLiteDump, error)

func (*SQLiteDump) Close

func (d *SQLiteDump) Close() error

func (*SQLiteDump) Count

func (d *SQLiteDump) Count(ctx context.Context) (int, error)

func (*SQLiteDump) LoadBatch

func (d *SQLiteDump) LoadBatch(ctx context.Context, records []MigrationRecord) error

func (*SQLiteDump) ReadBatches

func (d *SQLiteDump) ReadBatches(ctx context.Context, batchSize int, fn func([]MigrationRecord) error) error

type SourceLister

type SourceLister interface {
	ListPage(ctx context.Context, limit int, start string) ([]IndexdRecord, error)
}

type Stats

type Stats struct {
	Fetched          int
	Transformed      int
	Loaded           int
	Skipped          int
	Errors           int
	CountOfUniqueIDs int
}

func Import

func Import(ctx context.Context, reader DumpReader, loader Loader, batchSize int) (Stats, error)

func Run

func Run(ctx context.Context, source SourceLister, loader Loader, cfg Config) (Stats, error)

type TransformError

type TransformError struct {
	DID string
	Err error
}

func (TransformError) Error

func (e TransformError) Error() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL