Documentation
¶
Index ¶
- Constants
- Variables
- func GetHTTPCalls(s string) (n int64)
- func GetHTTPCallsMap() (m map[string]int64)
- func OperatorDomain(urlString string) string
- func RenderSQL(query string, args ...any) string
- func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
- func ScanDnsname(row Scanner, p *PgDnsname) error
- func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
- func ScanIdent(row Scanner, ident *PgIdent) error
- func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
- func ScanView(row Scanner, v *PgView) (err error)
- type CertStream
- func (cs *CertStream) CountStreams() (n int)
- func (cs *CertStream) DB() (db *PgDB)
- func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
- func (cs *CertStream) LogError(err error, msg string, args ...any) error
- func (cs *CertStream) LogInfo(msg string, args ...any)
- func (cs *CertStream) Operators() (operators []*LogOperator)
- type Certificate
- type Config
- type JsonCertificate
- type JsonIdentity
- type LogEntry
- type LogOperator
- func (lo *LogOperator) CallCount() (n int64)
- func (lo *LogOperator) Email() []string
- func (lo *LogOperator) ErrorCount() (n int)
- func (lo *LogOperator) Errors() (errs []*StreamError)
- func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
- func (lo *LogOperator) Name() string
- func (lo *LogOperator) StatusCounts() (m map[int]int)
- func (lo *LogOperator) StreamCount() (n int)
- func (lo *LogOperator) Streams() (sl []*LogStream)
- type LogStream
- type Logger
- type PgCertificate
- type PgDB
- func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)
- func (cdb *PgDB) CleanCertificates(ctx context.Context)
- func (cdb *PgDB) Close()
- func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) Estimate(table string) (f float64)
- func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)
- func (cdb *PgDB) GetHistoricalCertificates(ctx context.Context, expiresAfter time.Time, ...) (err error)
- func (cdb *PgDB) QueueUsage() (pct int)
- type PgDnsname
- type PgDnsnamesView
- type PgIdent
- type PgLogEntry
- type PgView
- type Scanner
- type StreamError
Constants ¶
const SelectBackfillIndex = `SELECT backfill_logindex FROM CERTDB_stream WHERE id = $1;`
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
const SelectFindGap = `SELECT start_gap, end_gap FROM CERTDB_findgap($1, $2, $3);`
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
const SelectMinIndexFrom = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1 AND logindex >= $2;`
const UpdateBackfillIndex = `UPDATE CERTDB_stream SET backfill_logindex = $1 WHERE id = $2 AND backfill_logindex < $1;`
Variables ¶
var CleanBatchSize = 10000
CleanBatchSize is the number of certificates the automatic cleanup deletes per statement.
var CleanEmptyCheckTimeout = 30 * time.Second
CleanEmptyCheckTimeout bounds each scan PgDB.DeleteStream makes over a stream's already deleted entries: restarting the cursor at the first entry, and confirming the stream holds none before deleting it.
A drained stream keeps its index entries until autovacuum removes them, and on a large table walking them to find that none are live takes minutes. When the check runs out of time the stream is left in place for a later call rather than deleted without proof.
var CleanResetInterval = time.Hour
CleanResetInterval is how often PgDB.DeleteCertificates restarts at the oldest certificate instead of resuming from its cursor.
var CreateSchema string
CreateSchema contains SQL to create the CertDB schema.
Existing tables remain unchanged. New databases receive B-tree domain indexes and no trigram index.
var DbIngestBatchSize = 1000 // number of entries to send to ingest at a time
var DefaultTransport = &http.Transport{ TLSHandshakeTimeout: 30 * time.Second, ResponseHeaderTimeout: 30 * time.Second, MaxIdleConnsPerHost: 2, DisableKeepAlives: false, ExpectContinueTimeout: 10 * time.Second, ForceAttemptHTTP2: true, }
var ErrHeadLogOpen = errors.New("head log open failed")
var ErrLogEntriesTooOld = errors.New("log entries are older than max age")
var ErrLogIdle errLogIdle
var ErrLogStreamRetryable = errors.New("logstream retryable")
var ErrSTHDiffTooLow = errors.New("STH diff too low")
var ErrSunlightClientMissing = errors.New("sunlight client missing")
var ErrTailLogOpen = errors.New("tail log open failed")
var FuncIngestBatch string
var FuncSetSince string
var FuncSubdomain string
var FunctionOperatorID string
var FunctionStreamID string
var HistoricalBatchSize = 1000 // number of rows to SELECT when getting historical certificates
var IdleCloseTime = time.Hour * 24 * 7
var LogBatchSize = int64(1024)
var MaxErrors = 100
Functions ¶
func GetHTTPCalls ¶ added in v0.32.7
func GetHTTPCallsMap ¶ added in v0.32.7
func OperatorDomain ¶
OperatorDomain returns the TLD+1 given an URL.
func ScanCertificate ¶ added in v0.12.0
func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
func ScanDnsname ¶ added in v0.12.0
func ScanDnsnamesView ¶ added in v0.12.0
func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
func ScanLogEntry ¶ added in v0.12.0
func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
Types ¶
type CertStream ¶
type CertStream struct {
Config // copy of config
C <-chan *LogEntry // log entry channel
HeadClient *http.Client // main HTTP client, uses Config.HeadDialer
TailClient *http.Client // may be nil if not backfilling
LogToggle atomic.Bool // if true, log stream activity
// contains filtered or unexported fields
}
func (*CertStream) CountStreams ¶ added in v0.0.3
func (cs *CertStream) CountStreams() (n int)
func (*CertStream) DB ¶ added in v0.12.0
func (cs *CertStream) DB() (db *PgDB)
func (*CertStream) GetLogStreamByID ¶ added in v0.27.0
func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
func (*CertStream) LogError ¶ added in v0.1.0
func (cs *CertStream) LogError(err error, msg string, args ...any) error
func (*CertStream) LogInfo ¶ added in v0.12.0
func (cs *CertStream) LogInfo(msg string, args ...any)
func (*CertStream) Operators ¶ added in v0.0.2
func (cs *CertStream) Operators() (operators []*LogOperator)
type Certificate ¶ added in v0.10.0
func (*Certificate) GetCommonName ¶ added in v0.24.29
func (c *Certificate) GetCommonName() (s string)
type Config ¶ added in v0.12.0
type Config struct {
Logger Logger // if not nil Logger to use, no default
HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
HeadLog string // log HTTP requests using the head dialer to this file
TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
PgUser string // PostgreSQL user, default "certstream"
PgPass string // PostgreSQL password, default "certstream"
PgName string // PostgreSQL db name, default "certstream"
PgAddr string // PostgreSQL address, no default
PgPrefix string // PostgreSQL naming prefix, default "certdb_"
PgConns int // max number of database connections, default 100
PgWorkerBits int // number of prefix bits that determine DB workers, default 5 (32 workers)
PgMaxAge int // maximum age in days to backfill
PgCertMaxAge time.Duration // delete certificates this long after they expire; zero disables cleanup
PgNoSSL bool // if true, do not use SSL
PgSyncCommit bool // if true, do not set synchronous_commit=off
Concurrency int // number of concurrent requests per stream, default is 4
DataDir string // log and cache directory; leave empty to disable
CacheMaxAge time.Duration // remove cached tile data older than this age; zero disables caching of tiles
TailLog string // log HTTP requests using the tail dialer to this file
}
type JsonCertificate ¶ added in v0.12.0
type JsonCertificate struct {
PreCert bool `json:",omitempty"`
Signature hexEncoded `json:",omitempty"` // SHA256 signature, searchable on crt.sh
Issuer JsonIdentity `json:",omitempty"`
Subject JsonIdentity `json:",omitempty"`
CommonName string `json:",omitempty"` // Subject common name
DNSNames []string `json:",omitempty"`
EmailAddresses []string `json:",omitempty"`
IPAddresses []string `json:",omitempty"`
URIs []string `json:",omitempty"`
NotBefore time.Time `json:",omitempty"`
NotAfter time.Time `json:",omitempty"`
Since time.Time `json:",omitzero"`
}
func NewJSONCertificate ¶ added in v0.12.0
func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)
func (*JsonCertificate) SetCommonName ¶ added in v0.24.29
func (js *JsonCertificate) SetCommonName()
type JsonIdentity ¶ added in v0.12.0
type LogEntry ¶
type LogEntry struct {
*LogStream
Err error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
LogIndex int64
PreCert bool
Certificate *x509.Certificate
Id int64 // database id, if available
Historical bool // true if the entry is from gap or backfilling
Signature []byte
Seen time.Time
}
func (*LogEntry) Cert ¶
func (le *LogEntry) Cert() (crt *Certificate)
Cert returns the Certificate given a LogEntry or nil.
type LogOperator ¶ added in v0.0.3
type LogOperator struct {
*CertStream
Domain string // e.g. "letsencrypt.org" or "googleapis.com"
Count atomic.Int64 // atomic; sum of the stream's Count
Status429 atomic.Int64 // atomic; number of 429 Too Many Requests
Id int32 // database ID, if available
// contains filtered or unexported fields
}
func (*LogOperator) CallCount ¶ added in v0.31.6
func (lo *LogOperator) CallCount() (n int64)
func (*LogOperator) Email ¶ added in v0.28.0
func (lo *LogOperator) Email() []string
func (*LogOperator) ErrorCount ¶ added in v0.22.0
func (lo *LogOperator) ErrorCount() (n int)
func (*LogOperator) Errors ¶ added in v0.22.0
func (lo *LogOperator) Errors() (errs []*StreamError)
func (*LogOperator) GetStreamByID ¶ added in v0.27.0
func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
func (*LogOperator) Name ¶ added in v0.28.0
func (lo *LogOperator) Name() string
func (*LogOperator) StatusCounts ¶ added in v0.30.8
func (lo *LogOperator) StatusCounts() (m map[int]int)
func (*LogOperator) StreamCount ¶ added in v0.18.0
func (lo *LogOperator) StreamCount() (n int)
func (*LogOperator) Streams ¶ added in v0.0.3
func (lo *LogOperator) Streams() (sl []*LogStream)
type LogStream ¶
type LogStream struct {
*LogOperator
Count atomic.Int64 // number of certificates sent to the channel
MinIndex atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
MaxIndex atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
LastIndex atomic.Int64 // atomic: highest index that is available from stream source
Backfill atomic.Int64 // atomic: number of remaining entries to backfill until we reach head
Logger *slog.Logger // toggled by LogToggle
Id int32 // database ID, if available
// contains filtered or unexported fields
}
type PgCertificate ¶ added in v0.12.0
type PgDB ¶ added in v0.12.0
type PgDB struct {
*CertStream
*pgxpool.Pool
Pfx func(string) string // prefix replacer
Workers atomic.Int32
// contains filtered or unexported fields
}
PgDB stores certificates and CT log entries in PostgreSQL.
func NewPgDB ¶ added in v0.12.0
func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)
NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.
func (*PgDB) AverageNewEntryTime ¶ added in v0.14.0
func (*PgDB) CleanCertificates ¶ added in v0.38.0
CleanCertificates deletes expired certificates until ctx is done.
A certificate is deleted once Config.PgCertMaxAge has passed since it expired. A PgCertMaxAge of zero or less returns immediately without deleting anything. Start runs this for the lifetime of the CertStream, so callers need it only when driving a PgDB themselves.
func (*PgDB) DeleteCertificates ¶ added in v0.28.4
func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)
DeleteCertificates deletes up to batchSize certificates that expired at or before cutoff, oldest first, cascading to the domain, IP address, email and URI rows that reference them. Rows in CERTDB_entry are left alone: they record log positions and have no foreign key to CERTDB_cert.
Deletion resumes where the previous call stopped, so repeated calls walk forward in notafter order instead of rescanning the index entries of rows already deleted. Finding nothing to delete moves the cursor up to cutoff, and at most once every CleanResetInterval restarts it at the oldest certificate to pick up any inserted below it.
A batchSize below one deletes nothing.
func (*PgDB) DeleteStream ¶ added in v0.28.4
func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)
DeleteStream deletes up to batchSize log entries belonging to the stream, oldest first, and deletes the stream itself once it holds none.
Deletion resumes from CERTDB_stream.clean_logindex, so repeated calls walk forward in logindex order instead of rescanning the index entries of rows already deleted, and a restart picks up where the last one stopped rather than from the first entry. Reaching the end of the stream restarts the scan at its first entry, so entries backfilled below the cursor are still found.
The stream itself is deleted only once it holds no entries at all, checked under a lock on the stream row. A batch deleting nothing is not on its own proof of that, since rows locked by another deleter are skipped.
A batchSize below one deletes nothing.
func (*PgDB) GetCertificateByHash ¶ added in v0.12.0
func (*PgDB) GetCertificateByID ¶ added in v0.12.0
func (*PgDB) GetCertificateByLogEntry ¶ added in v0.12.0
func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
func (*PgDB) GetCertificatesByCommonName ¶ added in v0.24.5
func (*PgDB) GetHistoricalCertificates ¶ added in v0.28.7
func (*PgDB) QueueUsage ¶ added in v0.15.0
type PgDnsnamesView ¶ added in v0.12.0
type PgLogEntry ¶ added in v0.12.0
type PgView ¶ added in v0.36.0
type PgView struct {
CertID int64
FQDN string
CommonName string
NotBefore time.Time
NotAfter time.Time
Since time.Time
PreCert bool
Sha256 []byte
IssuerOrganization string
IssuerProvince string
IssuerCountry string
SubjectOrganization string
SubjectProvince string
SubjectCountry string
Wild bool
WWW int
Domain string
Tld string
}
type StreamError ¶ added in v0.22.0
func (StreamError) Error ¶ added in v0.22.0
func (ewt StreamError) Error() string
func (StreamError) Unwrap ¶ added in v0.22.0
func (ewt StreamError) Unwrap() error
Source Files
¶
- atomic_minmax.go
- certificate.go
- certstream.go
- config.go
- errlogidle.go
- getloglist.go
- httpcallcounter.go
- jsoncertificate.go
- jsonidentity.go
- logentry.go
- logger.go
- logoperator.go
- logstream.go
- logstream_backoff.go
- logstream_rawentries.go
- logstream_tileentries.go
- operatordomain.go
- pgbackfill.go
- pgbackfill_log.go
- pgbatcher.go
- pgcertificate.go
- pgdb.go
- pgdnsname.go
- pgdnsnamesview.go
- pgident.go
- pglogentry.go
- pgschema.go
- pgview.go
- requestlog.go
- streamerror.go
- sunlight_cache.go
- sunlight_client.go
- toggled_logger.go
- updatestreams.go
- wraperr.go