dmfr

package
v0.7.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 13, 2021 License: GPL-3.0 Imports: 36 Imported by: 3

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ActivateFeedVersion

func ActivateFeedVersion(atx tldb.Adapter, fvid int) error

ActivateFeedVersion .

func AfterFeedVersionImport

func AfterFeedVersionImport(atx tldb.Adapter, fvid int) error

AfterFeedVersionImport .

func AuthenticatedRequest

func AuthenticatedRequest(address string, secret Secret, auth tl.FeedAuthorization) (string, error)

AuthenticatedRequest fetches a url using a secret and auth description. Returns temp file path or error.

func FindImportableFeeds

func FindImportableFeeds(adapter tldb.Adapter) ([]int, error)

FindImportableFeeds .

func HideUnseedFeeds

func HideUnseedFeeds(atx tldb.Adapter, found []int) (int, error)

HideUnseedFeeds .

func ImportFeed

func ImportFeed(atx tldb.Adapter, rfeed Feed) (int, bool, error)

ImportFeed .

Types

type AuthenticatedURLAdapter

type AuthenticatedURLAdapter struct {
	tlcsv.ZipAdapter
	// contains filtered or unexported fields
}

AuthenticatedURLAdapter is similar to URLAdapter but takes auth and secrets.

func (*AuthenticatedURLAdapter) Close

func (adapter *AuthenticatedURLAdapter) Close() error

Close the adapter, and remove the temporary file. An error is returned if the file could not be deleted.

func (*AuthenticatedURLAdapter) Download

func (adapter *AuthenticatedURLAdapter) Download(address string, auth tl.FeedAuthorization, secret Secret) error

Download the URL to a temporary file and set the correct adapter

type Command

type Command struct {
	// contains filtered or unexported fields
}

Command is the main entry point to the DMFR command

func (*Command) Run

func (cmd *Command) Run(args []string) error

Run the DMFR command.

type EntityCounter

type EntityCounter map[string]int

EntityCounter .

func (*EntityCounter) Scan

func (a *EntityCounter) Scan(value interface{}) error

Scan .

func (EntityCounter) Value

func (a EntityCounter) Value() (driver.Value, error)

Value .

type Feed

type Feed = tl.Feed

Feed .

type FeedLanguages

type FeedLanguages = tl.FeedLanguages

FeedLanguages .

type FeedState

type FeedState struct {
	ID                    int
	FeedID                int
	FeedVersionID         tl.OptionalKey
	LastFetchError        string
	LastFetchedAt         tl.OptionalTime
	LastSuccessfulFetchAt tl.OptionalTime
	FeedPriority          sql.NullInt64
	FeedRealtimeEnabled   bool
	tl.Timestamps
}

FeedState .

func (*FeedState) EntityID

func (ent *FeedState) EntityID() string

EntityID .

func (*FeedState) GetID added in v0.7.1

func (ent *FeedState) GetID() int

GetID .

func (*FeedState) SetID

func (ent *FeedState) SetID(id int)

SetID .

func (*FeedState) TableName

func (ent *FeedState) TableName() string

TableName .

type FeedUrls

type FeedUrls = tl.FeedUrls

FeedUrls .

type FeedVersionFileInfo

type FeedVersionFileInfo struct {
	ID            int
	FeedVersionID int
	Name          string
	Size          int64
	Rows          int64
	Columns       int
	Header        string
	CSVLike       bool
	SHA1          string
	tl.Timestamps
}

FeedVersionFileInfo .

func NewFeedVersionFileInfosFromReader

func NewFeedVersionFileInfosFromReader(reader *tlcsv.Reader) ([]FeedVersionFileInfo, error)

NewFeedVersionFileInfosFromReader calculates statistics about the contents of a feed version

func (*FeedVersionFileInfo) EntityID

func (fvi *FeedVersionFileInfo) EntityID() string

EntityID .

func (FeedVersionFileInfo) TableName

func (FeedVersionFileInfo) TableName() string

TableName .

type FeedVersionImport

type FeedVersionImport struct {
	ID                        int
	FeedVersionID             int
	ImportLog                 string
	ExceptionLog              string
	ImportLevel               int  // deprecated
	Success                   bool // Finished, Success Yes/No
	InProgress                bool // In Progress
	InterpolatedStopTimeCount int
	EntityCount               EntityCounter
	WarningCount              EntityCounter
	GeneratedCount            EntityCounter
	SkipEntityErrorCount      EntityCounter
	SkipEntityReferenceCount  EntityCounter
	SkipEntityFilterCount     EntityCounter
	SkipEntityMarkedCount     EntityCounter
	tl.Timestamps
}

FeedVersionImport .

func ImportFeedVersion

func ImportFeedVersion(atx tldb.Adapter, fv tl.FeedVersion, opts ImportOptions) (FeedVersionImport, error)

ImportFeedVersion .

func (*FeedVersionImport) EntityID

func (fvi *FeedVersionImport) EntityID() string

EntityID .

func (*FeedVersionImport) GetID added in v0.7.1

func (fvi *FeedVersionImport) GetID() int

GetID returns the ID

func (*FeedVersionImport) SetID added in v0.7.1

func (fvi *FeedVersionImport) SetID(v int)

SetID sets the ID.

func (FeedVersionImport) TableName

func (FeedVersionImport) TableName() string

TableName .

type FeedVersionServiceLevel added in v0.7.1

type FeedVersionServiceLevel struct {
	ID            int
	FeedVersionID int
	RouteID       sql.NullString
	StartDate     time.Time
	EndDate       time.Time
	Monday        int
	Tuesday       int
	Wednesday     int
	Thursday      int
	Friday        int
	Saturday      int
	Sunday        int
	// Cached data
	AgencyName     string
	RouteShortName string
	RouteLongName  string
	RouteType      int
}

FeedVersionServiceLevel .

func NewFeedVersionServiceInfosFromReader added in v0.7.1

func NewFeedVersionServiceInfosFromReader(reader tl.Reader) ([]FeedVersionServiceLevel, error)

NewFeedVersionServiceInfosFromReader .

func (*FeedVersionServiceLevel) EntityID added in v0.7.1

func (fvi *FeedVersionServiceLevel) EntityID() string

EntityID .

func (FeedVersionServiceLevel) TableName added in v0.7.1

func (FeedVersionServiceLevel) TableName() string

TableName .

type FetchCommand

type FetchCommand struct {
	FetchOptions FetchOptions
	Workers      int
	Limit        int
	DBURL        string
	DryRun       bool
	FeedIDs      []string
	// contains filtered or unexported fields
}

FetchCommand fetches feeds defined a DMFR database.

func (*FetchCommand) Parse

func (cmd *FetchCommand) Parse(args []string) error

Parse sets options from command line flags.

func (*FetchCommand) Run

func (cmd *FetchCommand) Run() error

Run executes this command.

type FetchOptions

type FetchOptions struct {
	FeedURL                 string
	FeedID                  string
	FeedCreate              bool
	IgnoreDuplicateContents bool
	Directory               string
	S3                      string
	FetchedAt               time.Time
	Secrets                 Secrets
}

FetchOptions sets options for a fetch operation.

type FetchResult

type FetchResult struct {
	FeedVersion  tl.FeedVersion
	Path         string
	FoundSHA1    bool
	FoundDirSHA1 bool
	FetchError   error
}

FetchResult contains results of a fetch operation.

func DatabaseFetch

func DatabaseFetch(atx tldb.Adapter, opts FetchOptions) (FetchResult, error)

DatabaseFetch fetches and creates a new FeedVersion for a given Feed. An error return from this function is a serious failure. Saves FeedState.LastFetchError for regular failures.

type ImportCommand

type ImportCommand struct {
	Workers       int
	Limit         int
	DBURL         string
	CoverDate     string
	FetchedSince  string
	Latest        bool
	DryRun        bool
	FeedIDs       []string
	FVIDs         arrayFlags
	FVSHA1        arrayFlags
	Adapter       tldb.Adapter // allow for mocks
	ImportOptions ImportOptions
}

ImportCommand imports FeedVersions into a database.

func (*ImportCommand) Parse

func (cmd *ImportCommand) Parse(args []string) error

Parse command line flags

func (*ImportCommand) Run

func (cmd *ImportCommand) Run() error

Run this command

type ImportOptions

type ImportOptions struct {
	FeedVersionID              int
	Extensions                 []string
	Directory                  string
	S3                         string
	Activate                   bool
	CreateMissingShapes        bool
	InterpolateStopTimes       bool
	DeduplicateJourneyPatterns bool
}

ImportOptions sets various options for importing a feed.

type ImportResult

type ImportResult struct {
	FeedVersionImport FeedVersionImport
}

ImportResult contains the results of a feed import.

func MainImportFeedVersion

func MainImportFeedVersion(adapter tldb.Adapter, opts ImportOptions) (ImportResult, error)

MainImportFeedVersion create FVI and run Copier inside a Tx.

type MergeCommand

type MergeCommand struct{}

MergeCommand merges together multiple DMFR files. Not implemented.

func (MergeCommand) Parse

func (MergeCommand) Parse(args []string) error

Parse command line options

func (MergeCommand) Run

func (MergeCommand) Run() error

Run executes this command.

type RecalculateCommand added in v0.7.1

type RecalculateCommand struct {
	Workers            int
	DBURL              string
	FVIDs              arrayFlags
	FVSHA1             arrayFlags
	Adapter            tldb.Adapter // allow for mocks
	Limit              int
	DryRun             bool
	RecalculateOptions RecalculateOptions
}

RecalculateCommand updates statistics for a Feed Version

func (*RecalculateCommand) Parse added in v0.7.1

func (cmd *RecalculateCommand) Parse(args []string) error

Parse command line flags

func (*RecalculateCommand) Run added in v0.7.1

func (cmd *RecalculateCommand) Run() error

Run this command

type RecalculateOptions added in v0.7.1

type RecalculateOptions struct {
	FeedVersionID int
	Directory     string
	S3            string
}

RecalculateOptions .

type Registry

type Registry struct {
	Schema                string `json:"$schema"`
	Feeds                 []Feed
	LicenseSpdxIdentifier string `json:"license_spdx_identifier"`
}

Registry represents a parsed Distributed Mobility Feed Registry (DMFR) file

func AddCrosswalkIDs

func AddCrosswalkIDs(baseRegistry *Registry, comparisonRegistries map[string]*Registry) *Registry

AddCrosswalkIDs TODO

func LoadAndParseRegistry

func LoadAndParseRegistry(path string) (*Registry, error)

LoadAndParseRegistry loads and parses a Distributed Mobility Feed Registry (DMFR) file from either a file system path or a URL

func NewRegistry

func NewRegistry(reader io.Reader) (*Registry, error)

NewRegistry TODO

func ParseString

func ParseString(contents string) (*Registry, error)

ParseString TODO

type Secret

type Secret struct {
	Key                string `json:"key"`
	Username           string `json:"username"`
	Password           string `json:"password"`
	AWSAccessKeyID     string `json:"aws_access_key_id"`
	AWSSecretAccessKey string `json:"aws_secret_access_key"`
	FeedID             string `json:"feed_id"`
	Filename           string `json:"filename"`
}

Secret .

type Secrets

type Secrets []Secret

Secrets loads a JSON file of secrets

func (*Secrets) Load

func (s *Secrets) Load(filename string) error

Load secrets from a file.

func (Secrets) MatchFeed

func (s Secrets) MatchFeed(feedid string) (Secret, error)

MatchFeed finds secrets associated with a DMFR FeedID.

func (Secrets) MatchFilename

func (s Secrets) MatchFilename(filename string) (Secret, error)

MatchFilename finds secrets associated with a DMFR filename.

type SyncCommand

type SyncCommand struct {
	DBURL      string
	Filenames  []string
	HideUnseen bool
	// contains filtered or unexported fields
}

SyncCommand syncs a DMFR to a database.

func (*SyncCommand) Parse

func (cmd *SyncCommand) Parse(args []string) error

Parse command line options.

func (*SyncCommand) Run

func (cmd *SyncCommand) Run() error

Run this command.

type SyncOptions

type SyncOptions struct {
	Filenames  []string
	HideUnseen bool
}

SyncOptions sets options for a sync operation.

type SyncResult

type SyncResult struct {
	FeedIDs     []int
	Errors      []error
	HiddenCount int
}

SyncResult is the result of a sync operation.

func MainSync

func MainSync(atx tldb.Adapter, opts SyncOptions) (SyncResult, error)

MainSync .

type ValidateCommand

type ValidateCommand struct {
	Filenames []string
}

ValidateCommand validates a DMFR file.

func (*ValidateCommand) Parse

func (cmd *ValidateCommand) Parse(args []string) error

Parse command line options.

func (*ValidateCommand) Run

func (cmd *ValidateCommand) Run() error

Run this command.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL