archive

type Blob struct {
	// Digest is the lower-case hexadecimal SHA-256 of the content, as the media object recorded
	// it - never recomputed by the archive writer, which would mean reading every attachment in
	// order to decide whether to read it.
	Digest string `json:"sha256"`
	Bytes  int64  `json:"bytes"`
}

type Checksums struct {
	// contains filtered or unexported fields
}

type Counter struct {
	// contains filtered or unexported fields
}

type Description struct {
	// Prefix is the archive's directory at the target, and what every other call here takes.
	Prefix   string
	Manifest Manifest
	// Complete is whether checksums.txt is there. An archive without it is a run that died or one
	// still going; it is not damaged, and the difference matters to whoever is deciding what to
	// restore from.
	Complete bool
	// Bytes is what the archive occupies at the target, members and media together.
	Bytes int64
}

type Encryption struct {
	// Mode is AES256_GCM or NONE, matching backup.EncryptionMode on the target that wrote it.
	Mode string `json:"mode"`
	// KeyID names the key. It is what makes a rotation cheap and BK-2's second half true: a new
	// archive is written under the new key, an old one keeps naming the old one, and neither is
	// rewritten. Empty on an unencrypted archive.
	KeyID string `json:"key_id,omitempty"`
	// KDF names the derivation, when the key came from a passphrase. Empty when the key was
	// supplied directly, and then the four numbers below are empty too.
	KDF         string `json:"kdf,omitempty"`
	Salt        string `json:"salt,omitempty"`
	Passes      uint32 `json:"passes,omitempty"`
	MemoryKiB   uint32 `json:"memory_kib,omitempty"`
	Parallelism uint8  `json:"parallelism,omitempty"`
	KeyLength   uint32 `json:"key_length,omitempty"`
	// ArchiveSalt is the salt this archive's key was derived with from the master key, base64:
	// drawn when the archive is written, so that every archive has a key of its own and the key
	// can be derived again from the manifest alone, after the database is lost
	// (backup-restore.md §4). Not secret. Empty on an archive written before archives carried
	// one, whose key is bound to the target that wrote it instead.
	ArchiveSalt string `json:"archive_salt,omitempty"`
}

type Entity struct {
	// Name is the file under data/, without the extension.
	Name string
	// Table is where the rows come from.
	Table string
	// Optional marks an entity a run may leave out. Only the audit trail is: `include_audit` is a
	// schedule setting, because including it gives better evidence and keeps personal metadata
	// longer (backup-restore.md §7).
	Optional bool
	// Later marks an entity format 1 gained after archives were already written (tenant-export.md
	// §11, additive). It is written into every archive like any other; an archive from before it
	// carries no such file, and reads as holding none of its rows rather than as an archive
	// somebody tampered with.
	Later bool
	// Whole marks an entity that cannot say when one of its rows changed, and is therefore
	// written complete in every archive - including an incremental one - with a restore replacing
	// its set rather than merging into it.
	//
	// It is a property of the schema rather than a choice. A join table has no timestamp at all,
	// and a configuration row that carries a creation stamp and no change stamp cannot tell an
	// edit from silence. Deriving "unchanged" from a column that does not move is how an
	// incremental chain quietly loses an edit, and there is no column to add here that would not
	// be a lie until every writer maintained it.
	//
	// All of them are small by nature - the join tables and the configuration - so writing them
	// whole costs little, and it is the reading that has to be right.
	Whole bool
	// Keys are the columns that make up a record's identity, in the order Record.ID joins them
	// with "/" - the primary key without `tenant_id`, because the tenant comes from the scope a
	// restore runs in rather than from the archive.
	//
	// Declared rather than derived. A restore has to be able to say what a line identifies
	// without a database to ask, which is the whole premise of §8.1, and a build that read the
	// key from its own schema would read a schema the archive was not written under.
	Keys []string
	// References are the fields that point at another entity's rows. Only the ones inside the
	// archive: a reference to something a restore does not import is not a reference it can do
	// anything about.
	//
	// They exist for one purpose - DUPLICATE, which has to give the copy new identities and keep
	// the copies pointing at each other rather than at the originals - and they are kept honest
	// by a test that compares them against the foreign keys the database actually has.
	References []Reference
	// Duplicable says whether a collision in this entity may be settled by making a copy.
	//
	// Not everything may. An account is who somebody is, a media object is the same bytes under a
	// content address, and a webhook subscription copied is a subscription that fires twice. For those a DUPLICATE restore falls back to SKIP and says so
	// in the report - the alternative is a merge that quietly doubles the tenant's identities and
	// its outbound integrations.
	Duplicable bool
	// Unique are the columns, other than the identity, that carry a uniqueness a copy would meet.
	//
	// A DUPLICATE gives the copy an identity of its own, and changing nothing else would make a
	// duplicated collection arrive under the living one's name and land nothing.
	// The identity is not the only thing the schema insists is unique, and what is left is not a
	// special case each: it is a property of the entity, declared here beside Keys and References
	// and kept honest by a test that compares it against the unique indexes the database has.
	//
	// Only DUPLICATE consults them. Every one of these indexes is per tenant, and a NEW_TENANT
	// copy lands in a tenant that did not exist a moment ago.
	Unique []UniqueField
}

type File struct {
	Path   string `json:"path"`
	Bytes  int64  `json:"bytes"`
	SHA256 string `json:"sha256"`
	// Records is how many JSON Lines the file holds before encryption. Zero for a member that is
	// not a data file.
	Records int64 `json:"records,omitempty"`
}

type Keys map[string]secret.Bytes

type Manifest struct {
	FormatVersion int `json:"format_version"`
	// ArchiveID identifies this archive across targets and is what a later incremental names as
	// its parent. It is supplied by the caller from the ID port, never derived from the clock:
	// two runs in the same second at two targets must not collide.
	ArchiveID string `json:"archive_id"`
	// SchemaVersion is the migration the source database stood at. A restore compares it against
	// its own and runs the upward migrations in between - which is the whole reason the data is
	// JSON Lines rather than a pg_dump (§3).
	SchemaVersion string `json:"schema_version"`
	// ProductVersion is the build that wrote the archive. Diagnostic rather than load-bearing:
	// the restore path reads SchemaVersion, and this is what an operator quotes in a bug report.
	ProductVersion string `json:"product_version"`
	Mode           Mode   `json:"mode"`
	Scope          Scope  `json:"scope"`
	Period         Period `json:"period"`
	// SnapshotAt is the instant the export's REPEATABLE READ snapshot was taken, and therefore
	// the instant the archive represents (§5). It equals Period.To and is repeated under its own
	// name because that is what `backup_run.snapshot_at` records and what an operator reads.
	SnapshotAt time.Time `json:"snapshot_at"`
	// ParentID is the archive this one continues, empty on a full archive.
	ParentID string `json:"parent_id,omitempty"`
	// ParentPrefix is where that parent lies at the same target, so that a restore can walk the
	// chain by reading manifests rather than by listing and matching identifiers.
	ParentPrefix string     `json:"parent_prefix,omitempty"`
	Encryption   Encryption `json:"encryption"`
	// Counts is how many records each entity contributed, by entity name. It is what a dry run
	// reports before anything is written (§8.3) and what BK-3 compares.
	Counts map[string]int64 `json:"counts"`
	// MediaCount and MediaBytes describe the media without listing them. A content-addressed file
	// is named after the SHA-256 of its content, so a list of media checksums would be a list of
	// the file names - and on a holding with a hundred thousand attachments it would also be the
	// largest thing in the archive.
	MediaCount int64 `json:"media_count"`
	MediaBytes int64 `json:"media_bytes"`
	// Whole names the entities this archive carried complete rather than as a delta, even when it
	// is an incremental one - the join tables and the configuration rows whose schema cannot say
	// when they changed (Entity.Whole).
	//
	// It is in the manifest rather than only in the registry because it is a restore instruction:
	// for these entities the newest archive of a chain holds the whole truth, and the copies in
	// older archives are superseded rather than merged. A reader that had to consult its own
	// build's registry would be a reader deciding from its own version what an older archive
	// meant.
	Whole []string `json:"whole,omitempty"`
	// Files are the data members and their checksums. Media are deliberately absent, for the
	// reason above.
	Files []File `json:"files"`
}

type Media interface {
	// Open answers the medium's content, or ErrNotFound. The caller closes the stream.
	Open(ctx context.Context, digest string) (io.ReadCloser, error)
}

type Mode string

type Op string

type Period struct {
	From time.Time `json:"from,omitempty"`
	To   time.Time `json:"to"`
}

type Reader struct {
	// contains filtered or unexported fields
}

type Record struct {
	// ID is the row's identity within its entity. For a table with a composite primary key it is
	// the key's parts joined by "/" in the order the schema declares them, which is what lets a
	// DELETE line in a later archive name the same row an UPSERT named in an earlier one.
	ID string `json:"id"`
	Op Op     `json:"op"`
	// UpdatedAt is what an incremental run selects on. On a tombstone it is the deletion's time.
	UpdatedAt time.Time `json:"updated_at"`
	// Data is the row. Nil on a tombstone - a deletion carries no content, for the reason
	// sync.Change gives and one more: the fields of a row that is being deleted are exactly the
	// personal data an erasure was meant to remove.
	//
	// Written even when it is empty, which is deliberate: `omitempty` would make a row with no
	// fields beyond its identity - an item_label is exactly that - indistinguishable from a line
	// that lost its payload. An UPSERT always carries an object and a DELETE always carries null,
	// so a reader never has to guess which of the two an absent field meant.
	Data map[string]any `json:"data"`
	// Blobs are the media this row references, if any. Absent on a tombstone.
	Blobs []Blob `json:"blobs,omitempty"`
}

type Reference struct {
	// Field is the column in this entity's data.
	Field string
	// Table is what it points at, in the schema's vocabulary - the same word the entity's own
	// Table is written in.
	Table string
	// Clearable marks a reference the row can lose and still be itself - a nullable column the
	// row merely uses, like an entry's cover. A restore that withholds the target clears such a
	// reference instead of withholding the row; a reference the row's identity rests on (a join
	// row, an attachment) keeps the row out with what it points at (backup-restore.md §7).
	Clearable bool
}

type Request struct {
	// ArchiveID identifies the archive. It comes from the ID port rather than from the clock, so
	// that two runs in the same second at two targets cannot collide.
	ArchiveID shared.ID
	// Prefix is the directory at the target, from Name. Everything this run writes lands beneath
	// it and nothing outside it.
	Prefix string
	Scope  Scope
	Mode   Mode
	// SnapshotAt is when the export's snapshot was taken. It is the archive's instant, and on an
	// incremental it is the next run's Since.
	SnapshotAt time.Time
	// Since is where an incremental starts, exclusive. Zero on a full archive.
	Since time.Time
	// ParentID and ParentPrefix name the archive an incremental continues.
	ParentID     string
	ParentPrefix string
	// Ancestors are the prefixes of the whole chain back to the full archive, newest first. They
	// are what makes "do not re-transfer a file that did not change" true: a medium already
	// stored anywhere in the chain is referenced rather than written again, and a restore
	// resolves a digest by searching the chain from newest to oldest.
	Ancestors []string

	SchemaVersion  string
	ProductVersion string

	// Encryption describes what protects the members, and is copied into the manifest verbatim.
	Encryption Encryption
	// Key is the archive key. Empty exactly when Encryption.Mode is NONE - a mismatch either way
	// is refused, because an archive that says it is encrypted and is not is worse than one that
	// says it is not.
	Key secret.Bytes

	// IncludeMedia and IncludeAudit are the two schedule settings that change what an archive
	// holds (backup-restore.md §5, §7).
	IncludeMedia bool
	IncludeAudit bool

	// Sections are files one archive carries beyond the format's entities, written after them
	// from the same source. Only a person's own copy of their data has one: what about them no
	// workspace archive may carry, because an administrator reads those (data-protection.md §4).
	// No restore reads a section - RestoredEntities never names one.
	Sections []Entity
}

type Scope struct {
	Kind ScopeKind `json:"kind"`
	ID   string    `json:"id"`
}

type ScopeKind string

type Source interface {
	// Records hands every record of one entity to yield, oldest first.
	//
	// since is exclusive and is the zero time on a full archive. An incremental run passes its
	// parent's snapshot instant, and the source answers what changed after it - including the
	// tombstones, which are what stop a chain resurrecting deleted objects.
	Records(ctx context.Context, entity Entity, since time.Time, yield func(Record) error) error
}

type UniqueField struct {
	// Field is the column in this entity's data.
	Field string
	// Rule is what a copy does with the value when Within has not already settled it.
	Rule UniqueRule
	// Within are the reference columns the uniqueness is scoped by - the rest of the index.
	//
	// It is what stops the rule firing where it is not needed, and that is most of the time. A
	// duplicated collection lands under the duplicated hub, because `parent_id` is a reference and
	// a live parent is remapped at the copy; under a hub of its own the collection's name is free,
	// and suffixing it would disfigure a copy for a collision that cannot happen. What is left is
	// the top of the duplicated tree - a hub, whose `parent_id` is null and has nothing to follow.
	Within []string
}

type UniqueRule string

type Writer struct {
	// contains filtered or unexported fields
}