Documentation
¶
Index ¶
- Constants
- Variables
- func GetHTTPCalls(s string) (n int64)
- func GetHTTPCallsMap() (m map[string]int64)
- func OperatorDomain(urlString string) string
- func RenderSQL(query string, args ...any) string
- func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
- func ScanDnsname(row Scanner, p *PgDnsname) error
- func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
- func ScanIdent(row Scanner, ident *PgIdent) error
- func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
- func ScanView(row Scanner, v *PgView) (err error)
- type CertStream
- func (cs *CertStream) CountStreams() (n int)
- func (cs *CertStream) DB() (db *PgDB)
- func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
- func (cs *CertStream) LogError(err error, msg string, args ...any) error
- func (cs *CertStream) LogInfo(msg string, args ...any)
- func (cs *CertStream) Operators() (operators []*LogOperator)
- type Certificate
- type Config
- type JsonCertificate
- type JsonIdentity
- type LogEntry
- type LogOperator
- func (lo *LogOperator) CallCount() (n int64)
- func (lo *LogOperator) Email() []string
- func (lo *LogOperator) ErrorCount() (n int)
- func (lo *LogOperator) Errors() (errs []*StreamError)
- func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
- func (lo *LogOperator) Name() string
- func (lo *LogOperator) StatusCounts() (m map[int]int)
- func (lo *LogOperator) StreamCount() (n int)
- func (lo *LogOperator) Streams() (sl []*LogStream)
- type LogStream
- type Logger
- type PgCertificate
- type PgDB
- func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)
- func (cdb *PgDB) CleanCertificates(ctx context.Context)
- func (cdb *PgDB) Close()
- func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) Estimate(table string) (f float64)
- func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)
- func (cdb *PgDB) GetHistoricalCertificates(ctx context.Context, expiresAfter time.Time, ...) (err error)
- func (cdb *PgDB) QueueUsage() (pct int)
- type PgDnsname
- type PgDnsnamesView
- type PgIdent
- type PgLogEntry
- type PgView
- type Scanner
- type StreamError
Constants ¶
const SelectBackfillIndex = `SELECT backfill_logindex FROM CERTDB_stream WHERE id = $1;`
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
const SelectFindGap = `SELECT start_gap, end_gap FROM CERTDB_findgap($1, $2, $3);`
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
const SelectMinIndexFrom = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1 AND logindex >= $2;`
const UpdateBackfillIndex = `UPDATE CERTDB_stream SET backfill_logindex = $1 WHERE id = $2 AND backfill_logindex < $1;`
Variables ¶
var ( DbIngestBatchSize = 1000 // number of entries to send to ingest at a time HistoricalBatchSize = 1000 // number of rows to SELECT when getting historical certificates LogBatchSize = int64(1024) MaxErrors = 100 IdleCloseTime = time.Hour * 24 * 7 )
var CleanBatchSize = 10000
CleanBatchSize limits the certificates automatic cleanup deletes per statement.
var CleanEmptyCheckTimeout = 30 * time.Second
CleanEmptyCheckTimeout bounds each scan PgDB.DeleteStream makes over a stream's already deleted entries: restarting the cursor at the first entry, and confirming the stream holds none before deleting it.
A drained stream keeps its index entries until autovacuum removes them, and on a large table walking them to find that none are live takes minutes. When the check runs out of time the stream is left in place for a later call rather than deleted without proof.
var CleanResetInterval = time.Hour
CleanResetInterval is the pause after a completed certificate cleanup sweep.
Each sweep revisits skipped locks and certificates inserted behind its position.
var CreateSchema string
CreateSchema contains SQL to create the CertDB schema.
Existing tables remain unchanged. New databases receive B-tree domain indexes and no trigram index.
var DefaultTransport = &http.Transport{ TLSHandshakeTimeout: 30 * time.Second, ResponseHeaderTimeout: 30 * time.Second, MaxIdleConnsPerHost: 2, DisableKeepAlives: false, ExpectContinueTimeout: 10 * time.Second, ForceAttemptHTTP2: true, }
var ErrHeadLogOpen = errors.New("head log open failed")
var ( // ErrLogEntriesTooOld reports that historical entries exceed the configured maximum age. ErrLogEntriesTooOld = errors.New("log entries are older than max age") )
var ErrLogIdle errLogIdle
var ErrLogStreamRetryable = errors.New("logstream retryable")
var ErrSTHDiffTooLow = errors.New("STH diff too low")
ErrSTHDiffTooLow indicates a checkpoint does not contain the requested entry.
var ErrSunlightClientMissing = errors.New("sunlight client missing")
var ErrTailLogOpen = errors.New("tail log open failed")
var FuncIngestBatch string
FuncIngestBatch contains SQL to install the certificate ingest functions.
NewPgDB installs these functions when opening the database.
var FuncSetSince string
var FuncSubdomain string
var FunctionOperatorID string
var FunctionStreamID string
Functions ¶
func GetHTTPCalls ¶ added in v0.32.7
func GetHTTPCallsMap ¶ added in v0.32.7
func OperatorDomain ¶
OperatorDomain returns the TLD+1 given an URL.
func ScanCertificate ¶ added in v0.12.0
func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
func ScanDnsname ¶ added in v0.12.0
func ScanDnsnamesView ¶ added in v0.12.0
func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
func ScanLogEntry ¶ added in v0.12.0
func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
Types ¶
type CertStream ¶
type CertStream struct {
Config // copy of config
C <-chan *LogEntry // log entry channel
HeadClient *http.Client // main HTTP client, uses Config.HeadDialer
TailClient *http.Client // may be nil if not backfilling
LogToggle atomic.Bool // if true, log stream activity
// contains filtered or unexported fields
}
func Start ¶ added in v0.12.0
Start starts CT log discovery and streaming until ctx is canceled.
It adds background goroutines to wg and returns after setup.
func (*CertStream) CountStreams ¶ added in v0.0.3
func (cs *CertStream) CountStreams() (n int)
func (*CertStream) DB ¶ added in v0.12.0
func (cs *CertStream) DB() (db *PgDB)
func (*CertStream) GetLogStreamByID ¶ added in v0.27.0
func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
func (*CertStream) LogError ¶ added in v0.1.0
func (cs *CertStream) LogError(err error, msg string, args ...any) error
func (*CertStream) LogInfo ¶ added in v0.12.0
func (cs *CertStream) LogInfo(msg string, args ...any)
func (*CertStream) Operators ¶ added in v0.0.2
func (cs *CertStream) Operators() (operators []*LogOperator)
type Certificate ¶ added in v0.10.0
func (*Certificate) GetCommonName ¶ added in v0.24.29
func (c *Certificate) GetCommonName() (s string)
type Config ¶ added in v0.12.0
type Config struct {
Logger Logger // if not nil Logger to use, no default
HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
HeadLog string // log HTTP requests using the head dialer to this file
TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
PgUser string // PostgreSQL user, default "certstream"
PgPass string // PostgreSQL password, default "certstream"
PgName string // PostgreSQL db name, default "certstream"
PgAddr string // PostgreSQL address, no default
PgPrefix string // PostgreSQL naming prefix, default "certdb_"
PgConns int // max number of database connections, default 100
PgWorkerBits int // number of prefix bits that determine DB workers, default 5 (32 workers)
PgMaxAge int // maximum age in days to backfill
PgCertMaxAge time.Duration // delete certificates this long after they expire; zero disables cleanup
PgNoSSL bool // if true, do not use SSL
PgSyncCommit bool // if true, do not set synchronous_commit=off
Concurrency int // number of concurrent requests per stream, default is 4
DataDir string // log and cache directory; leave empty to disable
CacheMaxAge time.Duration // remove cached tile data older than this age; zero disables caching of tiles
TailLog string // log HTTP requests using the tail dialer to this file
OnRetiredStreams func(context.Context, *PgDB, []int32) error // nil disables callback; invoked for retired database rows after readers stop and queued database writes finish
}
Config controls CT streaming and optional persistence.
type JsonCertificate ¶ added in v0.12.0
type JsonCertificate struct {
PreCert bool `json:",omitempty"`
Signature hexEncoded `json:",omitempty"` // SHA256 signature, searchable on crt.sh
Issuer JsonIdentity `json:",omitempty"`
Subject JsonIdentity `json:",omitempty"`
CommonName string `json:",omitempty"` // Subject common name
DNSNames []string `json:",omitempty"`
EmailAddresses []string `json:",omitempty"`
IPAddresses []string `json:",omitempty"`
URIs []string `json:",omitempty"`
NotBefore time.Time `json:",omitempty"`
NotAfter time.Time `json:",omitempty"`
Since time.Time `json:",omitzero"`
}
func NewJSONCertificate ¶ added in v0.12.0
func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)
func (*JsonCertificate) SetCommonName ¶ added in v0.24.29
func (js *JsonCertificate) SetCommonName()
type JsonIdentity ¶ added in v0.12.0
type LogEntry ¶
type LogEntry struct {
*LogStream
Err error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
LogIndex int64
PreCert bool
Certificate *x509.Certificate
Id int64 // database id, if available
Historical bool // true if the entry is from gap or backfilling
Signature []byte
Seen time.Time
// contains filtered or unexported fields
}
func (*LogEntry) Cert ¶
func (le *LogEntry) Cert() (crt *Certificate)
Cert returns the Certificate given a LogEntry or nil.
type LogOperator ¶ added in v0.0.3
type LogOperator struct {
*CertStream
Domain string // e.g. "letsencrypt.org" or "googleapis.com"
Count atomic.Int64 // atomic; sum of the stream's Count
Status429 atomic.Int64 // atomic; number of 429 Too Many Requests
Id int32 // database ID, if available
// contains filtered or unexported fields
}
func (*LogOperator) CallCount ¶ added in v0.31.6
func (lo *LogOperator) CallCount() (n int64)
func (*LogOperator) Email ¶ added in v0.28.0
func (lo *LogOperator) Email() []string
func (*LogOperator) ErrorCount ¶ added in v0.22.0
func (lo *LogOperator) ErrorCount() (n int)
func (*LogOperator) Errors ¶ added in v0.22.0
func (lo *LogOperator) Errors() (errs []*StreamError)
func (*LogOperator) GetStreamByID ¶ added in v0.27.0
func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
func (*LogOperator) Name ¶ added in v0.28.0
func (lo *LogOperator) Name() string
func (*LogOperator) StatusCounts ¶ added in v0.30.8
func (lo *LogOperator) StatusCounts() (m map[int]int)
func (*LogOperator) StreamCount ¶ added in v0.18.0
func (lo *LogOperator) StreamCount() (n int)
func (*LogOperator) Streams ¶ added in v0.0.3
func (lo *LogOperator) Streams() (sl []*LogStream)
type LogStream ¶
type LogStream struct {
*LogOperator
Count atomic.Int64 // number of certificates sent to the channel
MinIndex atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
MaxIndex atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
LastIndex atomic.Int64 // atomic: highest index that is available from stream source
Backfill atomic.Int64 // atomic: historical gap and reverse-backfill entries left to fetch
Logger *slog.Logger // toggled by LogToggle
Id int32 // database ID, if available
// contains filtered or unexported fields
}
LogStream tracks one CT log's head, entries, and historical fetches.
type PgCertificate ¶ added in v0.12.0
type PgDB ¶ added in v0.12.0
type PgDB struct {
*CertStream
*pgxpool.Pool
Pfx func(string) string // prefix replacer
Workers atomic.Int32
// contains filtered or unexported fields
}
PgDB stores certificates and CT log entries in PostgreSQL.
func NewPgDB ¶ added in v0.12.0
func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)
NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.
func (*PgDB) AverageNewEntryTime ¶ added in v0.14.0
func (*PgDB) CleanCertificates ¶ added in v0.38.0
CleanCertificates deletes expired certificates until ctx is done.
Certificates become eligible once Config.PgCertMaxAge has passed since expiry. A PgCertMaxAge of zero or less returns immediately without deleting anything. Start runs this for the lifetime of the CertStream, so callers need it only when driving a PgDB themselves.
func (*PgDB) DeleteCertificates ¶ added in v0.28.4
func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)
DeleteCertificates deletes up to batchSize expired certificates in storage order.
Certificates expiring at or before cutoff are eligible. Deletion cascades to their domain, IP address, email and URI rows; CT log entries are left alone. Each statement selects from a bounded range of adjacent heap pages. Empty ranges are crossed in separate transactions until a deletion or the end of the sweep. A sweep examines the whole heap, so sparse cleanup can still take a long time.
Calls on the same PgDB serialize and resume the current sweep. Progress is checkpointed after the first processed range, once per minute thereafter, and on sweep completion. A new PgDB resumes the checkpoint; uncheckpointed work can be repeated. A checkpoint from a different physical heap is discarded. Concurrent PgDBs may repeat work through their shared checkpoint.
Each sweep covers the heap size at its start. Later sweeps revisit certificates that expire, are inserted or move behind the cursor, as well as skipped locks and table rewrites. After a sweep completes, calls return zero until CleanResetInterval has passed. If saving a checkpoint fails, rowsDeleted can be positive with a non-nil error because deletion has already committed. A nil PgDB or a batchSize below one deletes nothing.
func (*PgDB) DeleteStream ¶ added in v0.28.4
func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)
DeleteStream deletes up to batchSize log entries belonging to the stream, oldest first, and deletes the stream itself once it holds none.
Deletion resumes from CERTDB_stream.clean_logindex, so repeated calls walk forward in logindex order instead of rescanning the index entries of rows already deleted, and a restart picks up where the last one stopped rather than from the first entry. Reaching the end of the stream restarts the scan at its first entry, so entries backfilled below the cursor are still found.
The stream itself is deleted only once it holds no entries at all, checked under a lock on the stream row. A batch deleting nothing is not on its own proof of that, since rows locked by another deleter are skipped.
A batchSize below one deletes nothing.
func (*PgDB) GetCertificateByHash ¶ added in v0.12.0
func (*PgDB) GetCertificateByID ¶ added in v0.12.0
func (*PgDB) GetCertificateByLogEntry ¶ added in v0.12.0
func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
func (*PgDB) GetCertificatesByCommonName ¶ added in v0.24.5
func (*PgDB) GetHistoricalCertificates ¶ added in v0.28.7
func (*PgDB) QueueUsage ¶ added in v0.15.0
type PgDnsnamesView ¶ added in v0.12.0
type PgLogEntry ¶ added in v0.12.0
type PgView ¶ added in v0.36.0
type PgView struct {
CertID int64
FQDN string
CommonName string
NotBefore time.Time
NotAfter time.Time
Since time.Time
PreCert bool
Sha256 []byte
IssuerOrganization string
IssuerProvince string
IssuerCountry string
SubjectOrganization string
SubjectProvince string
SubjectCountry string
Wild bool
WWW int
Domain string
Tld string
}
type StreamError ¶ added in v0.22.0
func (StreamError) Error ¶ added in v0.22.0
func (ewt StreamError) Error() string
func (StreamError) Unwrap ¶ added in v0.22.0
func (ewt StreamError) Unwrap() error
Source Files
¶
- atomic_minmax.go
- certificate.go
- certstream.go
- config.go
- errlogidle.go
- getloglist.go
- httpcallcounter.go
- jsoncertificate.go
- jsonidentity.go
- logentry.go
- logger.go
- logoperator.go
- logstream.go
- logstream_backoff.go
- logstream_rawentries.go
- logstream_tileentries.go
- operatordomain.go
- pgbackfill.go
- pgbackfill_log.go
- pgbatcher.go
- pgcertificate.go
- pgdb.go
- pgdb_clean.go
- pgdnsname.go
- pgdnsnamesview.go
- pgident.go
- pglogentry.go
- pgscan.go
- pgschema.go
- pgview.go
- requestlog.go
- streamerror.go
- sunlight_cache.go
- sunlight_client.go
- toggled_logger.go
- updatestreams.go
- wraperr.go