certstream

package module
v0.28.6 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 15, 2026 License: MIT Imports: 36 Imported by: 1

README

build coverage goreport Docs

CertStream

Small library wrapping github.com/google/certificate-transparency-go and github.com/FiloSottile/sunlight.

Requires a Postgres database to use.

func grabdata() {
	fulllimiter := bwlimit.NewLimiter()
	fulldialer := fulllimiter.Wrap(nil)
	var filllimiter *bwlimit.Limiter
	var filldialer proxy.ContextDialer

	cfg := certstream.NewConfig()
	cfg.Logger = slog.Default()
	cfg.PgUser = "username"
	cfg.PgPass = "password"
	cfg.PgName = "certstream"
	cfg.PgAddr = "127.0.0.1:5432"
	cfg.HeadDialer = fulldialer
	filllimiter = bwlimit.NewLimiter(int64(1 * 1024 * 1024))
	filldialer = filllimiter.Wrap(fulldialer)
	cfg.TailDialer = filldialer

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	var wg sync.WaitGroup
	cs, err := certstream.Start(ctx, &wg, cfg)
	defer wg.Wait()

	if err != nil {
        panic(err)
	} else {
		for le := range cs.C {
			fmt.Printf("%q %v %v\n", le.Domain, le.Historical, le.Cert().DNSNames)
		}
	}
}

Documentation

Index

Constants

View Source
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`

Variables

View Source
var BulkRange = int64(4096)
View Source
var CreateSchema string
View Source
var DbBatchSize = 100
View Source
var DefaultTransport = &http.Transport{
	TLSHandshakeTimeout:   30 * time.Second,
	ResponseHeaderTimeout: 30 * time.Second,
	MaxIdleConnsPerHost:   2,
	DisableKeepAlives:     false,
	ExpectContinueTimeout: 1 * time.Second,
	ForceAttemptHTTP2:     true,
}
View Source
var ErrLogIdle errLogIdle
View Source
var ErrSunlightClientMissing = errors.New("sunlight client missing")
View Source
var FuncIngestBatch string
View Source
var FunctionOperatorID string
View Source
var FunctionStreamID string
View Source
var IdleCloseTime = time.Hour * 24 * 7
View Source
var LogBatchSize = int64(1000)
View Source
var MaxErrors = 100
View Source
var SelectAllGaps string

Functions

func OperatorDomain

func OperatorDomain(urlString string) string

OperatorDomain returns the TLD+1 given an URL.

func RenderSQL added in v0.26.4

func RenderSQL(query string, args ...any) string

func ScanCertificate added in v0.12.0

func ScanCertificate(row Scanner, cert *PgCertificate) (err error)

func ScanDnsname added in v0.12.0

func ScanDnsname(row Scanner, p *PgDnsname) error

func ScanDnsnamesView added in v0.12.0

func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)

func ScanIdent added in v0.12.0

func ScanIdent(row Scanner, ident *PgIdent) error

func ScanLogEntry added in v0.12.0

func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)

Types

type CertStream

type CertStream struct {
	Config                      // copy of config
	C          <-chan *LogEntry // log entry channel
	HeadClient *http.Client     // main HTTP client, uses Config.HeadDialer
	TailClient *http.Client     // may be nil if not backfilling
	// contains filtered or unexported fields
}

func Start added in v0.12.0

func Start(ctx context.Context, wg *sync.WaitGroup, cfg *Config) (cs *CertStream, err error)

func (*CertStream) Close added in v0.19.0

func (cs *CertStream) Close()

func (*CertStream) CountStreams added in v0.0.3

func (cs *CertStream) CountStreams() (n int)

func (*CertStream) DB added in v0.12.0

func (cs *CertStream) DB() (db *PgDB)

func (*CertStream) GetLogStreamByID added in v0.27.0

func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)

func (*CertStream) LogError added in v0.1.0

func (cs *CertStream) LogError(err error, msg string, args ...any) error

func (*CertStream) LogInfo added in v0.12.0

func (cs *CertStream) LogInfo(msg string, args ...any)

func (*CertStream) Operators added in v0.0.2

func (cs *CertStream) Operators() (operators []*LogOperator)

type Certificate added in v0.10.0

type Certificate struct {
	PreCert   bool
	Seen      time.Time
	Signature []byte
	*x509.Certificate
}

func (*Certificate) GetCommonName added in v0.24.29

func (c *Certificate) GetCommonName() (s string)

type Config added in v0.12.0

type Config struct {
	Logger                Logger              // if not nil Logger to use, no default
	HeadDialer            proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
	TailDialer            proxy.ContextDialer // if not nil, backfill db using this dialer, no default
	PgUser                string              // PostgreSQL user, default "certstream"
	PgPass                string              // PostgreSQL password, default "certstream"
	PgName                string              // PostgreSQL db name, default "certstream"
	PgAddr                string              // PostgreSQL address, no default
	PgPrefix              string              // PostgreSQL naming prefix, default "certdb_"
	PgConns               int                 // max number of database connections, default 100
	PgWorkerBits          int                 // number of prefix bits that determine DB workers, default 5 (32 workers)
	PgMaxAge              int                 // maximum age in days to backfill
	PgNoSSL               bool                // if true, do not use SSL
	GetEntriesParallelism int                 // number of concurrent GetRawEntries requests per range, default 8
}

func NewConfig added in v0.12.0

func NewConfig() *Config

NewConfig returns a new default Config

type JsonCertificate added in v0.12.0

type JsonCertificate struct {
	PreCert        bool         `json:",omitempty"`
	Signature      hexEncoded   `json:",omitempty"` // SHA256 signature, searchable on crt.sh
	Issuer         JsonIdentity `json:",omitempty"`
	Subject        JsonIdentity `json:",omitempty"`
	CommonName     string       `json:",omitempty"` // Subject common name
	DNSNames       []string     `json:",omitempty"`
	EmailAddresses []string     `json:",omitempty"`
	IPAddresses    []string     `json:",omitempty"`
	URIs           []string     `json:",omitempty"`
	NotBefore      time.Time    `json:",omitempty"`
	NotAfter       time.Time    `json:",omitempty"`
	Since          time.Time    `json:",omitzero"`
}

func NewJSONCertificate added in v0.12.0

func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)

func (*JsonCertificate) SetCommonName added in v0.24.29

func (js *JsonCertificate) SetCommonName()

type JsonIdentity added in v0.12.0

type JsonIdentity struct {
	ID           int    `json:",omitempty"`
	Country      string `json:",omitempty"`
	Organization string `json:",omitempty"`
	Province     string `json:",omitempty"`
	CommonName   string `json:",omitempty"`
}

type LogEntry

type LogEntry struct {
	*LogStream
	Err         error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
	LogIndex    int64
	PreCert     bool
	Certificate *x509.Certificate
	Id          int64 // database id, if available
	Historical  bool  // true if the entry is from gap or backfilling
	Signature   []byte
	Seen        time.Time
}

func (*LogEntry) Cert

func (le *LogEntry) Cert() (crt *Certificate)

Cert returns the Certificate given a LogEntry or nil.

func (*LogEntry) Index

func (le *LogEntry) Index() (index int64)

Index returns the log index or -1 if none is available.

func (*LogEntry) String

func (le *LogEntry) String() (s string)

type LogOperator added in v0.0.3

type LogOperator struct {
	*CertStream
	Domain string       // e.g. "letsencrypt.org" or "googleapis.com"
	Count  atomic.Int64 // atomic; sum of the stream's Count
	Id     int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogOperator) Email added in v0.28.0

func (lo *LogOperator) Email() []string

func (*LogOperator) ErrorCount added in v0.22.0

func (lo *LogOperator) ErrorCount() (n int)

func (*LogOperator) Errors added in v0.22.0

func (lo *LogOperator) Errors() (errs []*StreamError)

func (*LogOperator) GetStreamByID added in v0.27.0

func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)

func (*LogOperator) Name added in v0.28.0

func (lo *LogOperator) Name() string

func (*LogOperator) StreamCount added in v0.18.0

func (lo *LogOperator) StreamCount() (n int)

func (*LogOperator) Streams added in v0.0.3

func (lo *LogOperator) Streams() (sl []*LogStream)

type LogStream

type LogStream struct {
	*LogOperator
	Count      atomic.Int64 // number of certificates sent to the channel
	MinIndex   atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
	MaxIndex   atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
	LastIndex  atomic.Int64 // atomic: highest index that is available from stream source
	InsideGaps atomic.Int64 // atomic: number of remaining entries inside gaps
	Id         int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogStream) String

func (ls *LogStream) String() string

func (*LogStream) URL added in v0.28.0

func (ls *LogStream) URL() string

type Logger added in v0.1.0

type Logger interface {
	Info(msg string, args ...any)
	Error(msg string, args ...any)
}

type PgCertificate added in v0.12.0

type PgCertificate struct {
	Id         int64
	NotBefore  time.Time
	NotAfter   time.Time
	CommonName string
	SubjectID  int
	IssuerID   int
	Sha256     []byte
	PreCert    bool
	Since      time.Time
}

type PgDB added in v0.12.0

type PgDB struct {
	*CertStream
	*pgxpool.Pool
	Pfx     func(string) string // prefix replacer
	Workers atomic.Int32
	// contains filtered or unexported fields
}

PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database

func NewPgDB added in v0.12.0

func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)

NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.

func (*PgDB) AverageNewEntryTime added in v0.14.0

func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)

func (*PgDB) Close added in v0.13.0

func (cdb *PgDB) Close()

func (*PgDB) DeleteCertificates added in v0.28.4

func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)

func (*PgDB) DeleteStream added in v0.28.4

func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)

func (*PgDB) Estimate added in v0.12.0

func (cdb *PgDB) Estimate(table string) (f float64)

func (*PgDB) GetCertificateByHash added in v0.12.0

func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByID added in v0.12.0

func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByLogEntry added in v0.12.0

func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificatesByCommonName added in v0.24.5

func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)

func (*PgDB) QueueUsage added in v0.15.0

func (cdb *PgDB) QueueUsage() (pct int)

type PgDnsname added in v0.12.0

type PgDnsname struct {
	Dnsname string
	CertID  int64
}

type PgDnsnamesView added in v0.12.0

type PgDnsnamesView struct {
	CertID    int64
	FQDN      string
	NotBefore time.Time
	Idna      bool
	Valid     bool
	PreCert   bool
	Issuer    string
	Subject   string
	Crtsh     string
	Domain    string
	Tld       string
}

type PgIdent added in v0.12.0

type PgIdent struct {
	Id           int
	Organization string
	Province     string
	Country      string
}

type PgLogEntry added in v0.12.0

type PgLogEntry struct {
	Seen     time.Time // CT log entry timestamp
	LogIndex int64     // CT log index for the stream
	CertID   int64     // database ID of cert
	StreamID int32     // database ID of stream
}

type Scanner added in v0.12.0

type Scanner interface {
	Scan(dest ...any) error
}

type StreamError added in v0.22.0

type StreamError struct {
	*LogStream
	When time.Time
	Err  error
}

func (StreamError) Error added in v0.22.0

func (ewt StreamError) Error() string

func (StreamError) Unwrap added in v0.22.0

func (ewt StreamError) Unwrap() error

Directories

Path Synopsis
cmd
certstream command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL