Documentation
¶
Index ¶
- func ArrayExists(arrayPath string) bool
- func Consolidate(dir, region string, mode ConsolidationMode, ...) error
- func ConsolidateAll(dir, region string, fragConfig *FragmentConsolidationConfig) error
- func ConsolidateMetadata(dir string, region string) error
- func ConsolidateOverlapping(dir, region string, groups []OverlapGroup, vacuum bool, vacuumInterval int) error
- func ConvertBytesToUint16(bytes []byte) (uint16, error)
- func ConvertUint16ToBytes(num uint16) []byte
- func CreateArray(srcArrayPath, destArrayPath string, region string) (err error)
- func CreateTDCPArray(arr, region string) (err error)
- func DeleteOverlappingFragments(arrayURI, region string, plan *DeleteResult) error
- func GPSWeekSowToTime(gpsWeek uint16, tow int32) (dt time.Time)
- func GetTimeRange(arr string, region string) (start time.Time, end time.Time, err error)
- func NormalizeFragments(dir, region string, vacuum bool) error
- func PostWriteMaintenance(dir, region string) error
- func ReadMetArray(arr, region string, queryParams MetQueryParams) (epochs []met.Epoch, err error)
- func ReadMetV2Array(arr, region string, queryParams MetV2QueryParams) (epochs []met.Epoch, err error)
- func ReadNavArray(arr string, region string, queryParams NavQueryParams) (ephems []ephemeris.GenericEphemeris, err error)
- func ReadNavV2Array(arr string, region string, queryParams NavV2QueryParams) (ephems []ephemeris.GenericEphemeris, err error)
- func ReadObsV3Array(arr string, region string, queryParams ObsQueryParams) (epochs []observation.Epoch, err error)
- func ReadPosArray()
- func Vacuum(dir, region string, mode ConsolidationMode) error
- func VacuumAll(dir, region string) error
- func VacuumMetadata(dir string, region string) error
- func WriteBottleArray(arr string, region string, bSlice []bottle.Bottle3DFloat64, ...) (err error)
- func WriteMetArray(arr, edid, region string, epochs []met.Epoch, opts ...WriteOptions) (err error)
- func WriteMetV2Array(arr, region string, epochs []met.Epoch, opts ...WriteOptions) (err error)
- func WriteNavArray(arr string, edid string, region string, ephs []ephemeris.GenericEphemeris, ...) (err error)
- func WriteNavV2Array(arr string, region string, ephs []ephemeris.GenericEphemeris, ...) (err error)
- func WriteObsV3Array(arr string, region string, epochs []observation.Epoch, opts ...WriteOptions) (err error)
- func WritePosArray(arr string, region string, positionEpochs []position.GAGEPosition, ...) (err error)
- func WriteSP3Array(arr string, region string, tag string, epochs []precise.SP3Epoch) (err error)
- func WriteSP3Records(arrayURI string, region string, records []precise.SP3Record) error
- func WriteTDCPArray(arr, region, edid string, tdcpResults []tdcp.Velocity, opts ...WriteOptions) (err error)
- type ConsolidationMode
- type DeleteDecision
- type DeleteGroupResult
- type DeleteResult
- type FragmentConsolidationConfig
- type FragmentDetail
- type FragmentReport
- type GroupingStrategy
- type MetQueryParams
- type MetV2QueryParams
- type NavQueryParams
- type NavV2QueryParams
- type ObsQueryParams
- type ObsV2QueryParams
- type OverlapGroup
- type StringRange
- type TimeRange
- type Uint8Range
- type WriteOptions
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ArrayExists ¶ added in v0.17.2
Check if the TileDB array exists at the given path
func Consolidate ¶ added in v0.37.1
func Consolidate(dir, region string, mode ConsolidationMode, fragConfig *FragmentConsolidationConfig) error
Consolidate runs consolidation for the given mode on a TileDB array.
func ConsolidateAll ¶ added in v0.37.1
func ConsolidateAll(dir, region string, fragConfig *FragmentConsolidationConfig) error
ConsolidateAll runs all three consolidation modes in sequence: fragments -> fragment_meta -> commits.
func ConsolidateMetadata ¶
ConsolidateMetadata consolidates fragment metadata (backward-compatible wrapper).
func ConsolidateOverlapping ¶ added in v0.37.1
func ConsolidateOverlapping(dir, region string, groups []OverlapGroup, vacuum bool, vacuumInterval int) error
ConsolidateOverlapping consolidates each overlap group's fragments into a single fragment using targeted ConsolidateFragments. When vacuum is true, fragments are vacuumed every vacuumInterval groups to prevent unbounded growth of the __fragments directory, and a full cleanup (commits + fragment_meta) is performed at the end.
func ConvertBytesToUint16 ¶
ConvertBytesToUint16 converts a slice of two bytes to a uint16 using BigEndian byte order.
func ConvertUint16ToBytes ¶
ConvertUint16ToBytes converts a uint16 to a slice of two bytes assuming BigEndian byte order.
func CreateArray ¶
CreateArray creates a new TileDB array using the schema from the provided path to a reference array
func CreateTDCPArray ¶
func DeleteOverlappingFragments ¶ added in v0.37.1
func DeleteOverlappingFragments(arrayURI, region string, plan *DeleteResult) error
DeleteOverlappingFragments deletes all fragments marked for deletion in the plan using a single tiledb.DeleteFragmentsList call.
func GetTimeRange ¶ added in v0.16.2
For a given array, return the time range of the data
func NormalizeFragments ¶ added in v0.38.0
NormalizeFragments collapses all overlapping fragment groups into one fragment per group using ConsolidateOverlapping, then calls PostWriteMaintenance. When vacuum is true, old fragments are physically removed. If there are no overlapping groups, it is a no-op.
func PostWriteMaintenance ¶ added in v0.38.0
PostWriteMaintenance consolidates and vacuums commits and fragment_meta. It is cheap enough to run after every write: it reduces array-open time and query-planning overhead without touching fragment data. Errors are logged as warnings and do not abort — maintenance is best-effort. Returns the first error encountered, or nil if all steps succeeded.
func ReadMetArray ¶
func ReadMetArray(arr, region string, queryParams MetQueryParams) (epochs []met.Epoch, err error)
func ReadMetV2Array ¶
func ReadMetV2Array(arr, region string, queryParams MetV2QueryParams) (epochs []met.Epoch, err error)
func ReadNavArray ¶
func ReadNavArray(arr string, region string, queryParams NavQueryParams) (ephems []ephemeris.GenericEphemeris, err error)
func ReadNavV2Array ¶
func ReadNavV2Array(arr string, region string, queryParams NavV2QueryParams) (ephems []ephemeris.GenericEphemeris, err error)
func ReadObsV3Array ¶
func ReadObsV3Array(arr string, region string, queryParams ObsQueryParams) (epochs []observation.Epoch, err error)
ReadObsV3Array reads GNSS observations to a TileDB array
func ReadPosArray ¶
func ReadPosArray()
func Vacuum ¶ added in v0.37.1
func Vacuum(dir, region string, mode ConsolidationMode) error
Vacuum runs vacuum for the given mode on a TileDB array.
func VacuumAll ¶ added in v0.37.1
VacuumAll runs all three vacuum modes in sequence: fragments -> fragment_meta -> commits.
func VacuumMetadata ¶
VacuumMetadata vacuums fragment metadata (backward-compatible wrapper).
func WriteBottleArray ¶
func WriteBottleArray(arr string, region string, bSlice []bottle.Bottle3DFloat64, opts ...WriteOptions) (err error)
WriteBottleArray writes strain observations to a TileDB array
func WriteMetArray ¶
func WriteMetArray(arr, edid, region string, epochs []met.Epoch, opts ...WriteOptions) (err error)
WriteMetArray - writes meteorolgical data to a TileDB array
func WriteMetV2Array ¶
func WriteMetV2Array(arr, region string, epochs []met.Epoch, opts ...WriteOptions) (err error)
WriteMetArray - writes meteorolgical data to a TileDB array
func WriteNavArray ¶
func WriteNavArray(arr string, edid string, region string, ephs []ephemeris.GenericEphemeris, opts ...WriteOptions) (err error)
WriteNavArray -
func WriteNavV2Array ¶
func WriteNavV2Array(arr string, region string, ephs []ephemeris.GenericEphemeris, opts ...WriteOptions) (err error)
WriteNavV2Array -
func WriteObsV3Array ¶
func WriteObsV3Array(arr string, region string, epochs []observation.Epoch, opts ...WriteOptions) (err error)
WriteObsV3Array writes GNSS observations to a TileDB array
func WritePosArray ¶
func WritePosArray(arr string, region string, positionEpochs []position.GAGEPosition, opts ...WriteOptions) (err error)
func WriteSP3Array ¶
DEPRECATED: WriteSP3Array is the legacy function for backward compatibility Use WriteSP3Records instead
func WriteSP3Records ¶ added in v0.31.0
DEPRECATED: Use pkg/geodata/gnsstiledb.Client.WritePreciseOrbits() instead. WriteSP3Records writes SP3Record format to TileDB with support for: - Optional velocity fields (nullable) - Product metadata (AC, type, frame, reference point) - Event and prediction flags
func WriteTDCPArray ¶
func WriteTDCPArray(arr, region, edid string, tdcpResults []tdcp.Velocity, opts ...WriteOptions) (err error)
Types ¶
type ConsolidationMode ¶ added in v0.37.1
type ConsolidationMode string
ConsolidationMode specifies what to consolidate or vacuum.
const ( ModeFragments ConsolidationMode = "fragments" ModeFragmentMeta ConsolidationMode = "fragment_meta" ModeCommits ConsolidationMode = "commits" )
type DeleteDecision ¶ added in v0.37.1
type DeleteDecision struct {
Fragment FragmentDetail `json:"fragment"`
Action string `json:"action"` // "keep" or "delete"
Reason string `json:"reason"`
}
DeleteDecision records whether a single fragment should be kept or deleted.
type DeleteGroupResult ¶ added in v0.37.1
type DeleteGroupResult struct {
Group OverlapGroup `json:"group"`
Decisions []DeleteDecision `json:"decisions"`
}
DeleteGroupResult holds the deletion decisions for one overlap group.
type DeleteResult ¶ added in v0.37.1
type DeleteResult struct {
Groups []DeleteGroupResult `json:"groups"`
TotalKeep int `json:"total_keep"`
TotalDelete int `json:"total_delete"`
ReclaimableBytes uint64 `json:"reclaimable_bytes"`
}
DeleteResult is the complete deletion plan across all overlap groups.
func PlanOverlapDeletions ¶ added in v0.37.1
func PlanOverlapDeletions(groups []OverlapGroup) *DeleteResult
PlanOverlapDeletions determines which fragments to keep and which to delete in each overlap group. Strategy: keep the fragment with the highest WriteEnd (newest write). Tie-break by CellCount (most data), then Index (lowest).
type FragmentConsolidationConfig ¶ added in v0.37.1
type FragmentConsolidationConfig struct {
// Step parameters
Steps uint32
StepMinFrags uint32
StepMaxFrags uint32
// Memory budget (zero = TileDB defaults: total=10GB, reader=3, writer=2, buffers=1)
TotalBudgetBytes uint64 // sm.mem.total_budget
ReaderWeight uint32 // sm.mem.consolidation.reader_weight
WriterWeight uint32 // sm.mem.consolidation.writer_weight
BuffersWeight uint32 // sm.mem.consolidation.buffers_weight
// Fragment size control (zero = unlimited / no restriction)
MaxFragmentSizeBytes uint64 // sm.consolidation.max_fragment_size
StepSizeRatio float64 // sm.consolidation.step_size_ratio
// Delete-cell purging — set true so array.Delete() tombstones are physically
// removed during consolidation, not just suppressed in reads (experimental)
PurgeDeletedCells bool // sm.consolidation.purge_deleted_cells
}
FragmentConsolidationConfig controls step-based fragment consolidation.
func DefaultFragmentConfig ¶ added in v0.37.1
func DefaultFragmentConfig() FragmentConsolidationConfig
DefaultFragmentConfig returns sensible defaults for fragment consolidation.
type FragmentDetail ¶ added in v0.37.1
type FragmentDetail struct {
Index uint32 `json:"index"`
URI string `json:"uri"`
TimeStart int64 `json:"time_start"` // non-empty domain min on "time" dimension (ms)
TimeEnd int64 `json:"time_end"` // non-empty domain max on "time" dimension (ms)
CellCount uint64 `json:"cell_count"`
Size uint64 `json:"size"`
WriteStart uint64 `json:"write_start"` // fragment write timestamp
WriteEnd uint64 `json:"write_end"` // fragment write timestamp
}
FragmentDetail holds metadata about a single TileDB fragment.
type FragmentReport ¶ added in v0.37.1
type FragmentReport struct {
ArrayURI string `json:"array_uri"`
TotalFragments uint32 `json:"total_fragments"`
TotalCells uint64 `json:"total_cells"`
TotalSize uint64 `json:"total_size"`
Fragments []FragmentDetail `json:"fragments"`
OverlapGroups []OverlapGroup `json:"overlap_groups"` // only groups with 2+ fragments
}
FragmentReport contains the full analysis of fragments in an array.
func AnalyzeFragments ¶ added in v0.37.1
func AnalyzeFragments(dir, region string, strategies ...GroupingStrategy) (*FragmentReport, error)
AnalyzeFragments inspects all fragments in a TileDB array. If a GroupingStrategy is provided, fragments are grouped accordingly. If no strategy is provided, GroupExact is used, where fragments that share the same start and end times are grouped.
type GroupingStrategy ¶ added in v0.37.1
type GroupingStrategy int
GroupingStrategy defines how fragments are grouped together.
const ( // GroupExact matches only if Start AND End times are identical. // BEST FOR: Deduplication / Deletion. GroupExact GroupingStrategy = iota // GroupOverlaps matches if time ranges overlap or touch. // BEST FOR: Consolidation / Vacuuming. GroupOverlaps )
type MetQueryParams ¶
type MetQueryParams struct {
EDID []StringRange `json:"edid"`
Time []TimeRange `json:"time"`
Key []StringRange `json:"key"`
Ordered bool `json:"ordered"` // Ignored: met reads always use ROW_MAJOR (variable-length buffers require it)
}
type MetV2QueryParams ¶
type MetV2QueryParams struct {
Time []TimeRange `json:"time"`
Key []StringRange `json:"key"`
Ordered bool `json:"ordered"` // Ignored: met reads always use ROW_MAJOR (variable-length buffers require it)
}
type NavQueryParams ¶
type NavQueryParams struct {
}
type NavV2QueryParams ¶
type NavV2QueryParams struct {
}
type ObsQueryParams ¶
type ObsQueryParams struct {
Time []TimeRange `json:"time"`
Sys []Uint8Range `json:"system"`
SvID []Uint8Range `json:"svid"`
Obs []StringRange `json:"obs"`
Ordered bool `json:"ordered"` // Use ROW_MAJOR layout (enables C-level dedup, returns sorted by dims)
}
type ObsV2QueryParams ¶
type ObsV2QueryParams struct {
EDID []StringRange `json:"edid"`
Time []TimeRange `json:"time"`
Satellite []StringRange `json:"sat"`
Obs []StringRange `json:"obs"`
Ordered bool `json:"ordered"` // Use ROW_MAJOR layout (enables C-level dedup, returns sorted by dims)
}
type OverlapGroup ¶ added in v0.37.1
type OverlapGroup struct {
Fragments []FragmentDetail `json:"fragments"`
TimeStart int64 `json:"time_start"` // group union start
TimeEnd int64 `json:"time_end"` // group union end
}
OverlapGroup represents a set of fragments with overlapping time ranges.
func FindGroups ¶ added in v0.37.1
func FindGroups(fragments []FragmentDetail, strategy GroupingStrategy) []OverlapGroup
FindGroups organizes fragments into overlapping groups based on the strategy.
type StringRange ¶
type Uint8Range ¶
type WriteOptions ¶ added in v0.37.1
type WriteOptions struct {
Ordered bool // Use ROW_MAJOR layout instead of UNORDERED
}
WriteOptions controls write query behaviour. Pass as variadic parameter to Write* functions; zero value preserves current (UNORDERED) behaviour.