CT log accuracy

From 6895534e0d59eae1b66a080dd390be7ee9c7db0b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 19 Nov 2016 18:21:50 -0500 Subject: separate diff logic from templating --- .gitignore | 1 + Makefile | 11 ++-- diff-pem2html.go | 144 +++++++++++++++++++++++++++++++++++++++++++++ diff.go | 173 ++++++++++++++----------------------------------------- 4 files changed, 192 insertions(+), 137 deletions(-) create mode 100644 diff-pem2html.go diff --git a/.gitignore b/.gitignore index bf916c5..a69fe3e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ NET-* crtsh-pem2html tls-getcerts tls-pem2html +diff-pem2html diff diff --git a/Makefile b/Makefile index d4ee17b..e3d6c36 100644 --- a/Makefile +++ b/Makefile @@ -9,18 +9,15 @@ all: diff.html crtsh.pem: crtsh-getcerts config-domains.txt NET-crtsh ./crtsh-getcerts $$(cat config-domains.txt) > $@ -crtsh.html: %.html: %.pem crtsh-pem2html - ./crtsh-pem2html < $< > $@ - tls.pem: tls-getcerts config-sockets.txt NET-tls ./tls-getcerts $$(cat config-sockets.txt) > $@ -tls.html: %.html: %.pem tls-pem2html - ./tls-pem2html < $< > $@ - -diff.html: diff tls.pem crtsh.pem +diff.pem: diff tls.pem crtsh.pem ./diff tls.pem crtsh.pem > $@ +tls.html crtsh.html diff.html: %.html: %.pem %-pem2html + ./$*-pem2html < $< > $@ + NET-%: date > $@ diff --git a/diff-pem2html.go b/diff-pem2html.go new file mode 100644 index 0000000..8f3b1e9 --- /dev/null +++ b/diff-pem2html.go @@ -0,0 +1,144 @@ +package main + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "html/template" + "io/ioutil" + "os" +) + +func handleErr(err error, str string, a ...interface{}) { + a = append([]interface{}{err}, a...) + if err != nil { + fmt.Fprintf(os.Stderr, str, a...) + os.Exit(1) + } +} + +func handleBool(ok bool, str string, a ...interface{}) { + if !ok { + fmt.Fprintf(os.Stderr, str, a...) + os.Exit(1) + } +} + +var tmpl = template.Must(template.New("2html").Parse(` + + + + CT log accuracy + + + + + + + + + +{{range $cert := .certs}} + + + + + + +{{end}} +

--- tls.pem
+++ crtsh.pem
@@ -1,{{.nTLS}} +1,{{.nCrtSh}} @@
{{if eq $cert.Pfix " "}} {{else}}{{$cert.Pfix}}{{end}}	{{$cert.X509.Subject.CommonName}}	{{$cert.X509.NotBefore.Local.Format "2006-01-02 15:04:05"}}	{{$cert.X509.NotAfter.Local.Format "2006-01-02 15:04:05"}}

+ + +`)) + +type Cert struct { + Url string + action string + X509 *x509.Certificate +} + +func (cert Cert) Pfix() string { + return map[string]string { + "add": "+", + "del": "-", + "ctx": " ", + }[cert.action] +} + +func (cert Cert) Class() string { + return "diff-" + cert.action +} + +func main() { + data, err := ioutil.ReadAll(os.Stdin) + handleErr(err, "Error reading stdin: %v\n") + + var certs []Cert + a := 0 + b := 0 + for len(data) > 0 { + var certPem *pem.Block + certPem, data = pem.Decode(data) + + var ok bool + var cert Cert + + cert.Url, ok = certPem.Headers["X-Crt-Sh-Url"] + handleBool(ok, "Did not get X-Crt-Sh-Url\n") + + cert.action, ok = certPem.Headers["X-Diff-Action"] + handleBool(ok, "Did not get X-Diff-Action\n") + switch cert.action { + case "add": + b++ + case "del": + a++ + case "ctx": + a++ + b++ + default: + handleBool(false, "Unknown X-Diff-Action: %q\n", cert.action) + } + + cert.X509, err = x509.ParseCertificate(certPem.Bytes) + handleErr(err, "Error parsing cert: %v\n") + + certs = append(certs, cert) + } + + handleErr(tmpl.Execute(os.Stdout, map[string]interface{}{ + "certs":certs, + "nTLS": a, + "nCrtSh": b, + }), "Could not execute template: %v\n") +} diff --git a/diff.go b/diff.go index 4a8a559..83b1d2e 100644 --- a/diff.go +++ b/diff.go @@ -4,8 +4,8 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "io" "io/ioutil" - "html/template" "os" "sort" ) @@ -18,7 +18,24 @@ func handleErr(err error, str string, a ...interface{}) { } } -func readTLS(filename string) (map[string]*x509.Certificate, error) { +type Cert struct { + Url string + X509 *x509.Certificate +} + +func (cert Cert) WriteTo(w io.Writer, action string) error { + block := pem.Block{ + Type: "CERTIFICATE", + Headers: map[string]string{ + "X-Crt-Sh-Url": cert.Url, + "X-Diff-Action": action, + }, + Bytes: cert.X509.Raw, + } + return pem.Encode(w, &block) +} + +func readTLS(filename string) (map[string]Cert, error) { file, err := os.Open(filename) if err != nil { return nil, err @@ -28,7 +45,7 @@ func readTLS(filename string) (map[string]*x509.Certificate, error) { return nil, err } - ret := make(map[string]*x509.Certificate) + ret := make(map[string]Cert) for len(data) > 0 { var certPem *pem.Block certPem, data = pem.Decode(data) @@ -36,12 +53,15 @@ func readTLS(filename string) (map[string]*x509.Certificate, error) { if err != nil { return nil, err } - ret[certX509.Subject.CommonName] = certX509 + ret[certX509.Subject.CommonName] = Cert{ + Url: fmt.Sprintf("https://crt.sh/?serial=%036x", certX509.SerialNumber), + X509: certX509, + } } return ret, nil } -func readCrtSh(filename string, hosts []string) (map[string]*x509.Certificate, error) { +func readCrtSh(filename string, hosts []string) (map[string]Cert, error) { file, err := os.Open(filename) if err != nil { return nil, err @@ -51,7 +71,7 @@ func readCrtSh(filename string, hosts []string) (map[string]*x509.Certificate, e return nil, err } - ret := make(map[string]*x509.Certificate) + ret := make(map[string]Cert) for len(data) > 0 { var certPem *pem.Block certPem, data = pem.Decode(data) @@ -61,8 +81,11 @@ func readCrtSh(filename string, hosts []string) (map[string]*x509.Certificate, e } for _, host := range hosts { if certX509.VerifyHostname(host) == nil { - if old, haveold := ret[host]; !haveold || certX509.NotBefore.After(old.NotBefore) { - ret[host] = certX509 + if old, haveold := ret[host]; !haveold || certX509.NotBefore.After(old.X509.NotBefore) { + ret[host] = Cert{ + Url: certPem.Headers["X-Crt-Sh-Url"], + X509: certX509, + } } } } @@ -70,7 +93,7 @@ func readCrtSh(filename string, hosts []string) (map[string]*x509.Certificate, e return ret, nil } -func keys(m map[string]*x509.Certificate) []string { +func keys(m map[string]Cert) []string { ret := make([]string, len(m)) i := 0 for k := range m { @@ -79,113 +102,6 @@ func keys(m map[string]*x509.Certificate) []string { } sort.Strings(ret) return ret -} - -func fmtCert(cert *x509.Certificate) string { - return fmt.Sprintf("%s\t%s\t%s", - cert.Subject.CommonName, - cert.NotBefore.Format("2006-01-02 15:04:05 MST(-07)"), - cert.NotAfter.Format("2006-01-02 15:04:05 MST(-07)")) -} - -var tmpl = template.Must(template.New("2html"). - Funcs(template.FuncMap{ - "class": class, - "link": link, - "join": join, - "isNil": isNil, - }).Parse(` - - - - CT log accuracy - - - - - - - - - -{{define "Row"}} - - - - - - -{{end}} -{{range $cert := .certs}} -{{if isNil $cert.CrtSh}} -{{template "Row" join "-" $cert.TLS}} -{{else if $cert.TLS.Equal $cert.CrtSh | not}} -{{template "Row" join "-" $cert.TLS}} -{{template "Row" join "+" $cert.CrtSh}} -{{else}} -{{template "Row" join " " $cert.TLS}} -{{end}} -{{end}} -

--- {{.fileTLS}}
+++ {{.fileCrtSh}}
@@ -1,{{len .certsTLS}} +1,{{len .certsCrtSh}} @@
{{if eq .pfix " "}} {{else}}{{.pfix}}{{end}}	{{.cert.Subject.CommonName}}	{{.cert.NotBefore.Local.Format "2006-01-02 15:04:05"}}	{{.cert.NotAfter.Local.Format "2006-01-02 15:04:05"}}

- - -`)) - -type Cert struct { - TLS, CrtSh *x509.Certificate -} - -func link(cert *x509.Certificate) string { - return fmt.Sprintf("https://crt.sh/?serial=%036x", cert.SerialNumber) -} - -func isNil(cert *x509.Certificate) bool { - return cert == nil -} - -func class(pfix string) string { - return map[string]string { - "+": "diff-add", - "-": "diff-del", - " ": "diff-ctx", - }[pfix] -} - -func join(pfix string, cert *x509.Certificate) map[string]interface{} { - return map[string]interface{} { - "pfix": pfix, - "cert": cert, - } } func main() { @@ -198,20 +114,17 @@ func main() { certsCrtSh, err := readCrtSh(os.Args[2], hostsTLS) handleErr(err, "Could load crt.sh file: %v\n") - certs := make([]Cert, len(certsTLS)) - i := 0 for _, host := range hostsTLS { - var cert Cert - cert.TLS = certsTLS[host] - cert.CrtSh = certsCrtSh[host] - certs[i] = cert - i++ + certTLS := certsTLS[host] + certCrtSh, haveCrtSh := certsCrtSh[host] + + if !haveCrtSh { + handleErr(certTLS.WriteTo(os.Stdout, "del"), "Could not encode PEM: %v\n") + } else if ! certTLS.X509.Equal(certCrtSh.X509) { + handleErr(certTLS.WriteTo(os.Stdout, "del"), "Could not encode PEM: %v\n") + handleErr(certCrtSh.WriteTo(os.Stdout, "add"), "Could not encode PEM: %v\n") + } else { + handleErr(certCrtSh.WriteTo(os.Stdout, "ctx"), "Could not encode PEM: %v\n") + } } - handleErr(tmpl.Execute(os.Stdout, map[string]interface{}{ - "certs": certs, - "certsTLS": certsTLS, - "certsCrtSh": certsCrtSh, - "fileTLS": os.Args[1], - "fileCrtSh": os.Args[2], - }), "Could not execute template: %v\n") } -- cgit v1.1-4-g5e80