Documentation
¶
Index ¶
- Constants
- Variables
- func GetLogList(ctx context.Context, httpClient *http.Client, listUrl string) (logList *loglist3.LogList, err error)
- func OperatorDomain(urlString string) string
- func RenderSQL(query string, args ...any) string
- func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
- func ScanDnsname(row Scanner, p *PgDnsname) error
- func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
- func ScanIdent(row Scanner, ident *PgIdent) error
- func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
- type CertStream
- type Certificate
- type Config
- type JsonCertificate
- type JsonIdentity
- type LogEntry
- type LogOperator
- type LogStream
- type Logger
- type PgCertificate
- type PgDB
- func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)
- func (cdb *PgDB) Close()
- func (cdb *PgDB) Estimate(table string) (f float64)
- func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateSince(ctx context.Context, jcert *JsonCertificate) (since time.Time, err error)
- func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)
- func (cdb *PgDB) QueueUsage() (pct int)
- type PgDnsname
- type PgDnsnamesView
- type PgIdent
- type PgLogEntry
- type Scanner
- type StreamError
Constants ¶
View Source
const SelectDnsnameLike = `SELECT * FROM CERTDB_domain WHERE domain LIKE $1;`
View Source
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
Variables ¶
View Source
var BulkRange = int64(4096)
View Source
var CreateSchema string
View Source
var DbBatchSize = 1000
View Source
var DefaultTransport = &http.Transport{ TLSHandshakeTimeout: 30 * time.Second, ResponseHeaderTimeout: 30 * time.Second, MaxIdleConnsPerHost: 2, DisableKeepAlives: false, ExpectContinueTimeout: 1 * time.Second, ForceAttemptHTTP2: true, }
View Source
var ErrLogIdle errLogIdle
View Source
var FuncAttachMetadata string
View Source
var FuncEnsureCert string
View Source
var FuncIngestBatch string
View Source
var FunctionFindSince string
View Source
var FunctionOperatorID string
View Source
var FunctionStreamID string
View Source
var IdleCloseTime = time.Hour * 24 * 7
View Source
var LogBatchSize = int64(1000)
View Source
var MaxErrors = 100
View Source
var SelectGaps string
View Source
var SelectIDSince string
Functions ¶
func GetLogList ¶
func GetLogList(ctx context.Context, httpClient *http.Client, listUrl string) (logList *loglist3.LogList, err error)
GetLogList fetches a CT log list from the given listUrl. Usually you would pass loglist3.AllLogListURL for the listUrl.
func OperatorDomain ¶
OperatorDomain returns the TLD+1 given an URL.
func ScanCertificate ¶ added in v0.12.0
func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
func ScanDnsname ¶ added in v0.12.0
func ScanDnsnamesView ¶ added in v0.12.0
func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
func ScanLogEntry ¶ added in v0.12.0
func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
Types ¶
type CertStream ¶
type CertStream struct {
Config // copy of config
C <-chan *LogEntry // log entry channel
HeadClient *http.Client // main HTTP client, uses Config.HeadDialer
TailClient *http.Client // may be nil if not backfilling
DB *PgDB
// contains filtered or unexported fields
}
func (*CertStream) Close ¶ added in v0.19.0
func (cs *CertStream) Close()
func (*CertStream) CountStreams ¶ added in v0.0.3
func (cs *CertStream) CountStreams() (n int)
func (*CertStream) LogError ¶ added in v0.1.0
func (cs *CertStream) LogError(err error, msg string, args ...any) error
func (*CertStream) LogInfo ¶ added in v0.12.0
func (cs *CertStream) LogInfo(msg string, args ...any)
func (*CertStream) Operators ¶ added in v0.0.2
func (cs *CertStream) Operators() (operators []*LogOperator)
type Certificate ¶ added in v0.10.0
func (*Certificate) GetCommonName ¶ added in v0.24.29
func (c *Certificate) GetCommonName() (s string)
type Config ¶ added in v0.12.0
type Config struct {
Logger Logger // if not nil Logger to use, no default
HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
PgUser string // PostgreSQL user, default "certstream"
PgPass string // PostgreSQL password, default "certstream"
PgName string // PostgreSQL db name, default "certstream"
PgAddr string // PostgreSQL address, no default
PgPrefix string // PostgreSQL naming prefix, default "certdb_"
PgConns int // max number of database connections, default 100
PgMaxAge int // maximum age in days to backfill
PgNoSSL bool // if true, do not use SSL
}
type JsonCertificate ¶ added in v0.12.0
type JsonCertificate struct {
PreCert bool `json:",omitempty"`
Signature hexEncoded `json:",omitempty"` // SHA256 signature, searchable on crt.sh
Issuer JsonIdentity `json:",omitempty"`
Subject JsonIdentity `json:",omitempty"`
CommonName string `json:",omitempty"` // Subject common name
DNSNames []string `json:",omitempty"`
EmailAddresses []string `json:",omitempty"`
IPAddresses []string `json:",omitempty"`
URIs []string `json:",omitempty"`
NotBefore time.Time `json:",omitempty"`
NotAfter time.Time `json:",omitempty"`
Since time.Time `json:",omitzero"`
}
func NewJSONCertificate ¶ added in v0.12.0
func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)
func (*JsonCertificate) SetCommonName ¶ added in v0.24.29
func (js *JsonCertificate) SetCommonName()
type JsonIdentity ¶ added in v0.12.0
type JsonIdentity struct {
ID int `json:",omitempty"`
Country string `json:",omitempty"`
Organization string `json:",omitempty"`
Province string `json:",omitempty"`
CommonName string `json:",omitempty"`
}
func (*JsonIdentity) Fill ¶ added in v0.12.0
func (id *JsonIdentity) Fill(name *pkix.Name)
type LogEntry ¶
type LogEntry struct {
*LogStream
Err error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
RawLogEntry *ct.RawLogEntry // may be nil in case of error
*ct.LogEntry // may be nil in case of error
Id int64 // database id, if available
Historical bool // true if the entry is from gap or backfilling
}
func (*LogEntry) Cert ¶
func (le *LogEntry) Cert() (crt *Certificate)
Cert returns the Certificate given a LogEntry or nil.
type LogOperator ¶ added in v0.0.3
type LogOperator struct {
*CertStream
*loglist3.Operator
Domain string // e.g. "letsencrypt.org" or "googleapis.com"
Count atomic.Int64 // atomic; sum of the stream's Count
Id int32 // database ID, if available
// contains filtered or unexported fields
}
func (*LogOperator) ErrorCount ¶ added in v0.22.0
func (lo *LogOperator) ErrorCount() (n int)
func (*LogOperator) Errors ¶ added in v0.22.0
func (lo *LogOperator) Errors() (errs []*StreamError)
func (*LogOperator) StreamCount ¶ added in v0.18.0
func (lo *LogOperator) StreamCount() (n int)
func (*LogOperator) Streams ¶ added in v0.0.3
func (lo *LogOperator) Streams() (sl []*LogStream)
type LogStream ¶
type LogStream struct {
*LogOperator
*loglist3.Log
HeadClient *client.LogClient
TailClient *client.LogClient
Count atomic.Int64 // number of certificates sent to the channel
MinIndex atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
MaxIndex atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
LastIndex atomic.Int64 // atomic: highest index that is available from stream source
InsideGaps atomic.Int64 // atomic: number of remaining entries inside gaps
Id int32 // database ID, if available
}
func (*LogStream) GetRawEntries ¶ added in v0.1.0
func (*LogStream) NewLastIndex ¶ added in v0.1.0
type PgCertificate ¶ added in v0.12.0
type PgDB ¶ added in v0.12.0
type PgDB struct {
*CertStream
*pgxpool.Pool
Pfx func(string) string // prefix replacer
Workers atomic.Int32
// contains filtered or unexported fields
}
PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database
func NewPgDB ¶ added in v0.12.0
func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)
NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.
func (*PgDB) AverageNewEntryTime ¶ added in v0.14.0
func (*PgDB) GetCertificateByHash ¶ added in v0.12.0
func (*PgDB) GetCertificateByID ¶ added in v0.12.0
func (*PgDB) GetCertificateByLogEntry ¶ added in v0.12.0
func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
func (*PgDB) GetCertificateSince ¶ added in v0.24.19
func (*PgDB) GetCertificatesByCommonName ¶ added in v0.24.5
func (*PgDB) QueueUsage ¶ added in v0.15.0
type PgDnsnamesView ¶ added in v0.12.0
type PgLogEntry ¶ added in v0.12.0
type StreamError ¶ added in v0.22.0
func (StreamError) Error ¶ added in v0.22.0
func (ewt StreamError) Error() string
func (StreamError) Unwrap ¶ added in v0.22.0
func (ewt StreamError) Unwrap() error
Source Files
¶
- certificate.go
- certstream.go
- config.go
- errlogidle.go
- getloglist.go
- jsoncertificate.go
- jsonidentity.go
- logentry.go
- logger.go
- logoperator.go
- logstream.go
- operatordomain.go
- pgbackfill.go
- pgbatcher.go
- pgcertificate.go
- pgdb.go
- pgdnsname.go
- pgdnsnamesview.go
- pgident.go
- pglogentry.go
- pgschema.go
- streamerror.go
- updatestreams.go
- wraperr.go
Click to show internal directories.
Click to hide internal directories.