certstream

package module
v0.43.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 16, 2026 License: MIT Imports: 43 Imported by: 1

README

build coverage Docs

CertStream

Small library wrapping github.com/google/certificate-transparency-go and github.com/FiloSottile/sunlight.

PostgreSQL is optional. Leave Config.PgAddr empty for live streaming without persistence. Configure a database for certificate storage and backfill.

With Config.Logger set, gap discoveries are summarized per CT log every minute. Summaries include the number of gaps and missing entries found during that interval, including gaps larger than 1,000 entries, which are also logged immediately. Pending counts are reported on shutdown. These are discoveries, not remaining backfill counts.

See TUNING.md for PostgreSQL settings once the database grows past a few hundred GB.

func grabdata() {
	fulllimiter := bwlimit.NewLimiter()
	fulldialer := fulllimiter.Wrap(nil)
	var filllimiter *bwlimit.Limiter
	var filldialer proxy.ContextDialer

	cfg := certstream.NewConfig()
	cfg.Logger = slog.Default()
	cfg.PgUser = "username"
	cfg.PgPass = "password"
	cfg.PgName = "certstream"
	cfg.PgAddr = "127.0.0.1:5432"
	cfg.HeadDialer = fulldialer
	filllimiter = bwlimit.NewLimiter(int64(1 * 1024 * 1024))
	filldialer = filllimiter.Wrap(fulldialer)
	cfg.TailDialer = filldialer

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	var wg sync.WaitGroup
	cs, err := certstream.Start(ctx, &wg, cfg)
	defer wg.Wait()

	if err != nil {
        panic(err)
	} else {
		for le := range cs.C {
			fmt.Printf("%q %v %v\n", le.Domain, le.Historical, le.Cert().DNSNames)
		}
	}
}

Documentation

Index

Constants

View Source
const SelectBackfillIndex = `SELECT backfill_logindex FROM CERTDB_stream WHERE id = $1;`
View Source
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source
const SelectFindGap = `SELECT start_gap, end_gap FROM CERTDB_findgap($1, $2, $3);`
View Source
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndexFrom = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1 AND logindex >= $2;`
View Source
const UpdateBackfillIndex = `UPDATE CERTDB_stream SET backfill_logindex = $1 WHERE id = $2 AND backfill_logindex < $1;`

Variables

View Source
var (
	DbIngestBatchSize   = 1000 // number of entries to send to ingest at a time
	HistoricalBatchSize = 1000 // number of rows to SELECT when getting historical certificates
	LogBatchSize        = int64(1024)
	MaxErrors           = 100
	IdleCloseTime       = time.Hour * 24 * 7
)
View Source
var CleanBatchSize = 10000

CleanBatchSize is the number of certificates the automatic cleanup deletes per statement.

View Source
var CleanEmptyCheckTimeout = 30 * time.Second

CleanEmptyCheckTimeout bounds each scan PgDB.DeleteStream makes over a stream's already deleted entries: restarting the cursor at the first entry, and confirming the stream holds none before deleting it.

A drained stream keeps its index entries until autovacuum removes them, and on a large table walking them to find that none are live takes minutes. When the check runs out of time the stream is left in place for a later call rather than deleted without proof.

View Source
var CleanResetInterval = time.Hour

CleanResetInterval is how often PgDB.DeleteCertificates restarts at the oldest certificate instead of resuming from its cursor.

View Source
var CreateSchema string

CreateSchema contains SQL to create the CertDB schema.

Existing tables remain unchanged. New databases receive B-tree domain indexes and no trigram index.

View Source
var DefaultTransport = &http.Transport{
	TLSHandshakeTimeout:   30 * time.Second,
	ResponseHeaderTimeout: 30 * time.Second,
	MaxIdleConnsPerHost:   2,
	DisableKeepAlives:     false,
	ExpectContinueTimeout: 10 * time.Second,
	ForceAttemptHTTP2:     true,
}
View Source
var ErrHeadLogOpen = errors.New("head log open failed")
View Source
var ErrLogEntriesTooOld = errors.New("log entries are older than max age")
View Source
var ErrLogIdle errLogIdle
View Source
var ErrLogStreamRetryable = errors.New("logstream retryable")
View Source
var ErrSTHDiffTooLow = errors.New("STH diff too low")

ErrSTHDiffTooLow indicates a checkpoint does not contain the requested entry.

View Source
var ErrSunlightClientMissing = errors.New("sunlight client missing")
View Source
var ErrTailLogOpen = errors.New("tail log open failed")
View Source
var FuncIngestBatch string

FuncIngestBatch contains SQL to install the certificate ingest functions.

NewPgDB installs these functions when opening the database.

View Source
var FuncSetSince string
View Source
var FuncSubdomain string
View Source
var FunctionOperatorID string
View Source
var FunctionStreamID string

Functions

func GetHTTPCalls added in v0.32.7

func GetHTTPCalls(s string) (n int64)

func GetHTTPCallsMap added in v0.32.7

func GetHTTPCallsMap() (m map[string]int64)

func OperatorDomain

func OperatorDomain(urlString string) string

OperatorDomain returns the TLD+1 given an URL.

func RenderSQL added in v0.26.4

func RenderSQL(query string, args ...any) string

func ScanCertificate added in v0.12.0

func ScanCertificate(row Scanner, cert *PgCertificate) (err error)

func ScanDnsname added in v0.12.0

func ScanDnsname(row Scanner, p *PgDnsname) error

func ScanDnsnamesView added in v0.12.0

func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)

func ScanIdent added in v0.12.0

func ScanIdent(row Scanner, ident *PgIdent) error

func ScanLogEntry added in v0.12.0

func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)

func ScanView added in v0.36.0

func ScanView(row Scanner, v *PgView) (err error)

Types

type CertStream

type CertStream struct {
	Config                      // copy of config
	C          <-chan *LogEntry // log entry channel
	HeadClient *http.Client     // main HTTP client, uses Config.HeadDialer
	TailClient *http.Client     // may be nil if not backfilling
	LogToggle  atomic.Bool      // if true, log stream activity
	// contains filtered or unexported fields
}

func Start added in v0.12.0

func Start(ctx context.Context, wg *sync.WaitGroup, cfg *Config) (cs *CertStream, err error)

Start starts CT log discovery and streaming until ctx is canceled.

It adds background goroutines to wg and returns after setup.

func (*CertStream) CountStreams added in v0.0.3

func (cs *CertStream) CountStreams() (n int)

func (*CertStream) DB added in v0.12.0

func (cs *CertStream) DB() (db *PgDB)

func (*CertStream) GetLogStreamByID added in v0.27.0

func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)

func (*CertStream) LogError added in v0.1.0

func (cs *CertStream) LogError(err error, msg string, args ...any) error

func (*CertStream) LogInfo added in v0.12.0

func (cs *CertStream) LogInfo(msg string, args ...any)

func (*CertStream) Operators added in v0.0.2

func (cs *CertStream) Operators() (operators []*LogOperator)

type Certificate added in v0.10.0

type Certificate struct {
	PreCert   bool
	Seen      time.Time
	Signature []byte
	*x509.Certificate
}

func (*Certificate) GetCommonName added in v0.24.29

func (c *Certificate) GetCommonName() (s string)

type Config added in v0.12.0

type Config struct {
	Logger       Logger              // if not nil Logger to use, no default
	HeadDialer   proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
	HeadLog      string              // log HTTP requests using the head dialer to this file
	TailDialer   proxy.ContextDialer // if not nil, backfill db using this dialer, no default
	PgUser       string              // PostgreSQL user, default "certstream"
	PgPass       string              // PostgreSQL password, default "certstream"
	PgName       string              // PostgreSQL db name, default "certstream"
	PgAddr       string              // PostgreSQL address, no default
	PgPrefix     string              // PostgreSQL naming prefix, default "certdb_"
	PgConns      int                 // max number of database connections, default 100
	PgWorkerBits int                 // number of prefix bits that determine DB workers, default 5 (32 workers)
	PgMaxAge     int                 // maximum age in days to backfill
	PgCertMaxAge time.Duration       // delete certificates this long after they expire; zero disables cleanup
	PgNoSSL      bool                // if true, do not use SSL
	PgSyncCommit bool                // if true, do not set synchronous_commit=off
	Concurrency  int                 // number of concurrent requests per stream, default is 4
	DataDir      string              // log and cache directory; leave empty to disable
	CacheMaxAge  time.Duration       // remove cached tile data older than this age; zero disables caching of tiles
	TailLog      string              // log HTTP requests using the tail dialer to this file
}

func NewConfig added in v0.12.0

func NewConfig() *Config

NewConfig returns a new default Config

type JsonCertificate added in v0.12.0

type JsonCertificate struct {
	PreCert        bool         `json:",omitempty"`
	Signature      hexEncoded   `json:",omitempty"` // SHA256 signature, searchable on crt.sh
	Issuer         JsonIdentity `json:",omitempty"`
	Subject        JsonIdentity `json:",omitempty"`
	CommonName     string       `json:",omitempty"` // Subject common name
	DNSNames       []string     `json:",omitempty"`
	EmailAddresses []string     `json:",omitempty"`
	IPAddresses    []string     `json:",omitempty"`
	URIs           []string     `json:",omitempty"`
	NotBefore      time.Time    `json:",omitempty"`
	NotAfter       time.Time    `json:",omitempty"`
	Since          time.Time    `json:",omitzero"`
}

func NewJSONCertificate added in v0.12.0

func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)

func (*JsonCertificate) SetCommonName added in v0.24.29

func (js *JsonCertificate) SetCommonName()

type JsonIdentity added in v0.12.0

type JsonIdentity struct {
	ID           int    `json:",omitempty"`
	Country      string `json:",omitempty"`
	Organization string `json:",omitempty"`
	Province     string `json:",omitempty"`
	CommonName   string `json:",omitempty"`
}

type LogEntry

type LogEntry struct {
	*LogStream
	Err         error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
	LogIndex    int64
	PreCert     bool
	Certificate *x509.Certificate
	Id          int64 // database id, if available
	Historical  bool  // true if the entry is from gap or backfilling
	Signature   []byte
	Seen        time.Time
}

func (*LogEntry) Cert

func (le *LogEntry) Cert() (crt *Certificate)

Cert returns the Certificate given a LogEntry or nil.

func (*LogEntry) Index

func (le *LogEntry) Index() (index int64)

Index returns the log index or -1 if none is available.

func (*LogEntry) String

func (le *LogEntry) String() (s string)

type LogOperator added in v0.0.3

type LogOperator struct {
	*CertStream
	Domain    string       // e.g. "letsencrypt.org" or "googleapis.com"
	Count     atomic.Int64 // atomic; sum of the stream's Count
	Status429 atomic.Int64 // atomic; number of 429 Too Many Requests
	Id        int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogOperator) CallCount added in v0.31.6

func (lo *LogOperator) CallCount() (n int64)

func (*LogOperator) Email added in v0.28.0

func (lo *LogOperator) Email() []string

func (*LogOperator) ErrorCount added in v0.22.0

func (lo *LogOperator) ErrorCount() (n int)

func (*LogOperator) Errors added in v0.22.0

func (lo *LogOperator) Errors() (errs []*StreamError)

func (*LogOperator) GetStreamByID added in v0.27.0

func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)

func (*LogOperator) Name added in v0.28.0

func (lo *LogOperator) Name() string

func (*LogOperator) StatusCounts added in v0.30.8

func (lo *LogOperator) StatusCounts() (m map[int]int)

func (*LogOperator) StreamCount added in v0.18.0

func (lo *LogOperator) StreamCount() (n int)

func (*LogOperator) Streams added in v0.0.3

func (lo *LogOperator) Streams() (sl []*LogStream)

type LogStream

type LogStream struct {
	*LogOperator
	Count     atomic.Int64 // number of certificates sent to the channel
	MinIndex  atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
	MaxIndex  atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
	LastIndex atomic.Int64 // atomic: highest index that is available from stream source
	Backfill  atomic.Int64 // atomic: number of remaining entries to backfill until we reach head
	Logger    *slog.Logger // toggled by LogToggle
	Id        int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogStream) HTTPCalls added in v0.32.7

func (ls *LogStream) HTTPCalls() int64

func (*LogStream) String

func (ls *LogStream) String() string

func (*LogStream) URL added in v0.28.0

func (ls *LogStream) URL() string

type Logger added in v0.1.0

type Logger interface {
	Info(msg string, args ...any)
	Error(msg string, args ...any)
}

type PgCertificate added in v0.12.0

type PgCertificate struct {
	Id         int64
	NotBefore  time.Time
	NotAfter   time.Time
	CommonName string
	SubjectID  int
	IssuerID   int
	Sha256     []byte
	PreCert    bool
	Since      time.Time
}

type PgDB added in v0.12.0

type PgDB struct {
	*CertStream
	*pgxpool.Pool
	Pfx     func(string) string // prefix replacer
	Workers atomic.Int32
	// contains filtered or unexported fields
}

PgDB stores certificates and CT log entries in PostgreSQL.

func NewPgDB added in v0.12.0

func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)

NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.

func (*PgDB) AverageNewEntryTime added in v0.14.0

func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)

func (*PgDB) CleanCertificates added in v0.38.0

func (cdb *PgDB) CleanCertificates(ctx context.Context)

CleanCertificates deletes expired certificates until ctx is done.

A certificate is deleted once Config.PgCertMaxAge has passed since it expired. A PgCertMaxAge of zero or less returns immediately without deleting anything. Start runs this for the lifetime of the CertStream, so callers need it only when driving a PgDB themselves.

func (*PgDB) Close added in v0.13.0

func (cdb *PgDB) Close()

func (*PgDB) DeleteCertificates added in v0.28.4

func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)

DeleteCertificates deletes up to batchSize certificates that expired at or before cutoff, oldest first, cascading to the domain, IP address, email and URI rows that reference them. Rows in CERTDB_entry are left alone: they record log positions and have no foreign key to CERTDB_cert.

Deletion resumes where the previous call stopped, so repeated calls walk forward in notafter order instead of rescanning the index entries of rows already deleted. Finding nothing to delete moves the cursor up to cutoff, and at most once every CleanResetInterval restarts it at the oldest certificate to pick up any inserted below it.

A batchSize below one deletes nothing.

func (*PgDB) DeleteStream added in v0.28.4

func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)

DeleteStream deletes up to batchSize log entries belonging to the stream, oldest first, and deletes the stream itself once it holds none.

Deletion resumes from CERTDB_stream.clean_logindex, so repeated calls walk forward in logindex order instead of rescanning the index entries of rows already deleted, and a restart picks up where the last one stopped rather than from the first entry. Reaching the end of the stream restarts the scan at its first entry, so entries backfilled below the cursor are still found.

The stream itself is deleted only once it holds no entries at all, checked under a lock on the stream row. A batch deleting nothing is not on its own proof of that, since rows locked by another deleter are skipped.

A batchSize below one deletes nothing.

func (*PgDB) Estimate added in v0.12.0

func (cdb *PgDB) Estimate(table string) (f float64)

func (*PgDB) GetCertificateByHash added in v0.12.0

func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByID added in v0.12.0

func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByLogEntry added in v0.12.0

func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificatesByCommonName added in v0.24.5

func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)

func (*PgDB) GetHistoricalCertificates added in v0.28.7

func (cdb *PgDB) GetHistoricalCertificates(ctx context.Context, expiresAfter time.Time, callback func(ctx context.Context, cert *JsonCertificate) (err error)) (err error)

func (*PgDB) QueueUsage added in v0.15.0

func (cdb *PgDB) QueueUsage() (pct int)

type PgDnsname added in v0.12.0

type PgDnsname struct {
	Dnsname string
	CertID  int64
}

type PgDnsnamesView added in v0.12.0

type PgDnsnamesView struct {
	CertID    int64
	FQDN      string
	NotBefore time.Time
	Idna      bool
	Valid     bool
	PreCert   bool
	Issuer    string
	Subject   string
	Crtsh     string
	Domain    string
	Tld       string
}

type PgIdent added in v0.12.0

type PgIdent struct {
	Id           int
	Organization string
	Province     string
	Country      string
}

type PgLogEntry added in v0.12.0

type PgLogEntry struct {
	Seen     time.Time // CT log entry timestamp
	LogIndex int64     // CT log index for the stream
	CertID   int64     // database ID of cert
	StreamID int32     // database ID of stream
}

type PgView added in v0.36.0

type PgView struct {
	CertID              int64
	FQDN                string
	CommonName          string
	NotBefore           time.Time
	NotAfter            time.Time
	Since               time.Time
	PreCert             bool
	Sha256              []byte
	IssuerOrganization  string
	IssuerProvince      string
	IssuerCountry       string
	SubjectOrganization string
	SubjectProvince     string
	SubjectCountry      string
	Wild                bool
	WWW                 int
	Domain              string
	Tld                 string
}

type Scanner added in v0.12.0

type Scanner interface {
	Scan(dest ...any) error
}

type StreamError added in v0.22.0

type StreamError struct {
	*LogStream
	When time.Time
	Err  error
}

func (StreamError) Error added in v0.22.0

func (ewt StreamError) Error() string

func (StreamError) Unwrap added in v0.22.0

func (ewt StreamError) Unwrap() error

Directories

Path Synopsis
cmd
certstream command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL