From c4f6d3489ef91c539ba88e372635e4d0d87e6ad2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 18 Nov 2016 01:21:27 -0500 Subject: Check crt.sh against actual used certs --- .gitignore | 9 ++- Makefile | 21 ++++-- config-domains.txt | 4 ++ cron-daily | 3 +- crtsh-getcerts | 28 ++++++++ crtsh-pem2html.go | 207 +++++++++++++++++++++++++++++++++++++++++++++++++++++ diff.go | 108 ++++++++++++++++++++++++++++ domains.txt | 4 -- getcerts | 28 -------- pem2html.go | 207 ----------------------------------------------------- tls-getcerts.go | 35 +++++++++ 11 files changed, 406 insertions(+), 248 deletions(-) create mode 100644 config-domains.txt create mode 100755 crtsh-getcerts create mode 100644 crtsh-pem2html.go create mode 100644 diff.go delete mode 100644 domains.txt delete mode 100755 getcerts delete mode 100644 pem2html.go create mode 100644 tls-getcerts.go diff --git a/.gitignore b/.gitignore index 98b607b..40ffbfc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ -certs.* -pem2html +*.html +*.pem +*.txt NET-* + +crtsh-pem2html +tls-getcerts +diff diff --git a/Makefile b/Makefile index 15d69da..157fd34 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,22 @@ -all: certs.html +all: index.html -pem2html: %: %.go +index.html: crtsh.html + cp $< $@ + +%: %.go go build $< -certs.pem: getcerts domains.txt NET-crt.sh - ./getcerts $$(cat domains.txt) > $@ +crtsh.pem: crtsh-getcerts config-domains.txt NET-crtsh + ./crtsh-getcerts $$(cat config-domains.txt) > $@ + +crtsh.html: %.html: %.pem crtsh-pem2html + ./crtsh-pem2html < $< > $@ + +tls.pem: tls-getcerts config-servers.txt NET-tls + ./tls-getcerts $$(cat config-servers.txt) > $@ -certs.html: %.html: %.pem pem2html - ./pem2html < $< > $@ +diff.txt: diff tls.pem crtsh.pem + ./diff tls.pem crtsh.pem > $@ NET-%: date > $@ diff --git a/config-domains.txt b/config-domains.txt new file mode 100644 index 0000000..390a92e --- /dev/null +++ b/config-domains.txt @@ -0,0 +1,4 @@ +parabola.nu +parabolagnulinux.org +lukeshu.com +team4272.com diff --git a/cron-daily b/cron-daily index ecc5266..117eac4 100755 --- a/cron-daily +++ b/cron-daily @@ -1,4 +1,5 @@ #!/bin/sh cd "$(dirname -- "$0")" -date > NET-crt.sh +date > NET-crtsh +date > NET-tls make diff --git a/crtsh-getcerts b/crtsh-getcerts new file mode 100755 index 0000000..0191e2e --- /dev/null +++ b/crtsh-getcerts @@ -0,0 +1,28 @@ +#!/usr/bin/env ruby +require 'nokogiri' +require 'open-uri' + +certs = {} +ARGV.each do |domain| + [ domain, "%.#{domain}" ].each do |pattern| + Nokogiri::XML(open("https://crt.sh/atom?identity=#{pattern}&exclude=expired")).css('feed > entry').each do |entry| + url = entry.css('id').first.text.split("#").first + + updated = entry.css('updated').first.text + + html = Nokogiri::HTML(entry.css('summary').first.text) + html.css('br').each{|br| br.replace("\n")} + pem = html.css('div').first.text + + lines = pem.split("\n") + lines.insert(1, "X-Crt-Sh-Url: #{url}", "X-Crt-Sh-Updated: #{updated}") + pem = lines.join("\n")+"\n" + + certs[url] = pem + end + end +end + +certs.each do |url, pem| + print pem +end diff --git a/crtsh-pem2html.go b/crtsh-pem2html.go new file mode 100644 index 0000000..b6effa4 --- /dev/null +++ b/crtsh-pem2html.go @@ -0,0 +1,207 @@ +package main + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "html/template" + "io/ioutil" + "os" + "sort" + "time" +) + +func handleErr(err error, str string, a ...interface{}) { + a = append([]interface{}{err}, a...) + if err != nil { + fmt.Fprintf(os.Stderr, str, a...) + os.Exit(1) + } +} + +func handleBool(ok bool, str string, a ...interface{}) { + if !ok { + fmt.Fprintf(os.Stderr, str, a...) + } +} + +var tmpl = template.Must(template.New("pem2html"). + Funcs(template.FuncMap{ + "red": red, + "green": green, + }).Parse(` + + + + CT log + + + + + + + + + + + + + +{{range $cert := .certs}} + + + + + + + +{{end}} +
+

Updated {{.now.Local.Format "2006-01-02 15:04:05"}}

+
LoggedNotBeforeNotAfterSubject.CNIssuer.O
{{$cert.Updated.Local.Format "2006-01-02 15:04:05"}}{{$cert.X509.NotBefore.Local.Format "2006-01-02"}}{{$cert.X509.NotAfter.Local.Format "2006-01-02"}}{{$cert.X509.Subject.CommonName}}{{$cert.X509.Issuer.Organization}}
+ + +`)) + +var now = time.Now() + +type interpolation struct { + ta, tb time.Time + ba, bb byte +} + +func (i interpolation) interpolate(tc time.Time) byte { + db := i.tb.Sub(i.ta) + dc := tc.Sub(i.ta) + + pct := float64(dc) / float64(db) + if pct < 0 { + pct = 0 + } else if pct > 1 { + pct = 1 + } + + sb := int16(i.bb) - int16(i.ba) + sc := int16(pct * float64(sb)) + + return byte(int16(i.ba) + sc) +} + +var daysago = interpolation{ + ta: now.AddDate(0, 0, -30), + tb: now, + ba: 0xF3, + bb: 0x00, +} + +var daysuntil = interpolation{ + ta: now, + tb: now.AddDate(0, 0, 30), + ba: 0x00, + bb: 0xF3, +} + +func green(t time.Time) string { + b := daysago.interpolate(t) + return fmt.Sprintf("#%02X%02X%02X", b, 0xF3, b) +} + +func red(t time.Time) string { + b := daysuntil.interpolate(t) + return fmt.Sprintf("#%02X%02X%02X", 0xF3, b, b) +} + +type Cert struct { + Url string + Updated time.Time + X509 *x509.Certificate +} + +type Certs []Cert + +// Len is the number of elements in the collection. +func (l Certs) Len() int { + return len(l) +} + +// Less reports whether the element with +// index i should sort before the element with index j. +func (l Certs) Less(i, j int) bool { + return l[i].Updated.UTC().After(l[j].Updated.UTC()) +} + +// Swap swaps the elements with indexes i and j. +func (l Certs) Swap(i, j int) { + tmp := l[i] + l[i] = l[j] + l[j] = tmp +} + +func main() { + data, err := ioutil.ReadAll(os.Stdin) + handleErr(err, "Error reading stdin: %v\n") + + var certs Certs + for len(data) > 0 { + var certPem *pem.Block + certPem, data = pem.Decode(data) + + var ok bool + var cert Cert + + cert.Url, ok = certPem.Headers["X-Crt-Sh-Url"] + handleBool(ok, "Did not get X-Crt-Sh-Url\n") + + str, ok := certPem.Headers["X-Crt-Sh-Updated"] + handleBool(ok, "Did not get X-Crt-Sh-Updated\n") + cert.Updated, err = time.Parse("2006-01-02T15:04:05Z", str) + handleErr(err, "Could not parse updated time") + + cert.X509, err = x509.ParseCertificate(certPem.Bytes) + handleErr(err, "Error parsing cert: %v\n") + + certs = append(certs, cert) + } + + sort.Sort(certs) + handleErr(tmpl.Execute(os.Stdout, map[string]interface{}{"certs":certs, "now": now}), "Could not execute template: %v\n") +} diff --git a/diff.go b/diff.go new file mode 100644 index 0000000..24fa228 --- /dev/null +++ b/diff.go @@ -0,0 +1,108 @@ +package main + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "sort" +) + +func handleErr(err error, str string, a ...interface{}) { + a = append([]interface{}{err}, a...) + if err != nil { + fmt.Fprintf(os.Stderr, str, a...) + os.Exit(1) + } +} + +func readTLS(filename string) (map[string]*x509.Certificate, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + data, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + ret := make(map[string]*x509.Certificate) + for len(data) > 0 { + var certPem *pem.Block + certPem, data = pem.Decode(data) + certX509, err := x509.ParseCertificate(certPem.Bytes) + if err != nil { + return nil, err + } + ret[certX509.Subject.CommonName] = certX509 + } + return ret, nil +} + +func readCrtSh(filename string, hosts []string) (map[string]*x509.Certificate, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + data, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + ret := make(map[string]*x509.Certificate) + for len(data) > 0 { + var certPem *pem.Block + certPem, data = pem.Decode(data) + certX509, err := x509.ParseCertificate(certPem.Bytes) + if err != nil { + return nil, err + } + for _, host := range hosts { + if certX509.VerifyHostname(host) == nil { + if old, haveold := ret[host]; !haveold || certX509.NotBefore.After(old.NotBefore) { + ret[host] = certX509 + } + } + } + } + return ret, nil +} + +func keys(m map[string]*x509.Certificate) []string { + ret := make([]string, len(m)) + i := 0 + for k := range m { + ret[i] = k + i++ + } + sort.Strings(ret) + return ret +} + +func main() { + if len(os.Args) != 3 { + fmt.Fprintf(os.Stderr, "Usage: %s TLS-file crt.sh-file\n", os.Args[0]) + } + certsTLS, err := readTLS(os.Args[1]) + handleErr(err, "Could load TLS file: %v\n") + hostsTLS := keys(certsTLS) + certsCrtSh, err := readCrtSh(os.Args[2], hostsTLS) + handleErr(err, "Could load crt.sh file: %v\n") + + fmt.Printf("--- %s\n", os.Args[1]) + fmt.Printf("+++ %s\n", os.Args[2]) + fmt.Printf("@@ -1,%d +1,%d @@\n", len(hostsTLS), len(hostsTLS)) + for _, host := range hostsTLS { + certTLS := certsTLS[host] + certCrtSh, okCrtSh := certsCrtSh[host] + if !okCrtSh { + fmt.Printf("-%s %s\n", host, certTLS.NotBefore.Format("2006-01-02 15:04:05 MST(-07)")) + } else if !certTLS.Equal(certCrtSh) { + fmt.Printf("-%s %s\n", host, certTLS.NotBefore.Format("2006-01-02 15:04:05 MST(-07)")) + fmt.Printf("+%s %s\n", host, certCrtSh.NotBefore.Format("2006-01-02 15:04:05 MST(-07)")) + } else { + fmt.Printf(" %s %s\n", host, certTLS.NotBefore.Format("2006-01-02 15:04:05 MST(-07)")) + } + } +} diff --git a/domains.txt b/domains.txt deleted file mode 100644 index 390a92e..0000000 --- a/domains.txt +++ /dev/null @@ -1,4 +0,0 @@ -parabola.nu -parabolagnulinux.org -lukeshu.com -team4272.com diff --git a/getcerts b/getcerts deleted file mode 100755 index 0191e2e..0000000 --- a/getcerts +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env ruby -require 'nokogiri' -require 'open-uri' - -certs = {} -ARGV.each do |domain| - [ domain, "%.#{domain}" ].each do |pattern| - Nokogiri::XML(open("https://crt.sh/atom?identity=#{pattern}&exclude=expired")).css('feed > entry').each do |entry| - url = entry.css('id').first.text.split("#").first - - updated = entry.css('updated').first.text - - html = Nokogiri::HTML(entry.css('summary').first.text) - html.css('br').each{|br| br.replace("\n")} - pem = html.css('div').first.text - - lines = pem.split("\n") - lines.insert(1, "X-Crt-Sh-Url: #{url}", "X-Crt-Sh-Updated: #{updated}") - pem = lines.join("\n")+"\n" - - certs[url] = pem - end - end -end - -certs.each do |url, pem| - print pem -end diff --git a/pem2html.go b/pem2html.go deleted file mode 100644 index b6effa4..0000000 --- a/pem2html.go +++ /dev/null @@ -1,207 +0,0 @@ -package main - -import ( - "crypto/x509" - "encoding/pem" - "fmt" - "html/template" - "io/ioutil" - "os" - "sort" - "time" -) - -func handleErr(err error, str string, a ...interface{}) { - a = append([]interface{}{err}, a...) - if err != nil { - fmt.Fprintf(os.Stderr, str, a...) - os.Exit(1) - } -} - -func handleBool(ok bool, str string, a ...interface{}) { - if !ok { - fmt.Fprintf(os.Stderr, str, a...) - } -} - -var tmpl = template.Must(template.New("pem2html"). - Funcs(template.FuncMap{ - "red": red, - "green": green, - }).Parse(` - - - - CT log - - - - - - - - - - - - - -{{range $cert := .certs}} - - - - - - - -{{end}} -
-

Updated {{.now.Local.Format "2006-01-02 15:04:05"}}

-
LoggedNotBeforeNotAfterSubject.CNIssuer.O
{{$cert.Updated.Local.Format "2006-01-02 15:04:05"}}{{$cert.X509.NotBefore.Local.Format "2006-01-02"}}{{$cert.X509.NotAfter.Local.Format "2006-01-02"}}{{$cert.X509.Subject.CommonName}}{{$cert.X509.Issuer.Organization}}
- - -`)) - -var now = time.Now() - -type interpolation struct { - ta, tb time.Time - ba, bb byte -} - -func (i interpolation) interpolate(tc time.Time) byte { - db := i.tb.Sub(i.ta) - dc := tc.Sub(i.ta) - - pct := float64(dc) / float64(db) - if pct < 0 { - pct = 0 - } else if pct > 1 { - pct = 1 - } - - sb := int16(i.bb) - int16(i.ba) - sc := int16(pct * float64(sb)) - - return byte(int16(i.ba) + sc) -} - -var daysago = interpolation{ - ta: now.AddDate(0, 0, -30), - tb: now, - ba: 0xF3, - bb: 0x00, -} - -var daysuntil = interpolation{ - ta: now, - tb: now.AddDate(0, 0, 30), - ba: 0x00, - bb: 0xF3, -} - -func green(t time.Time) string { - b := daysago.interpolate(t) - return fmt.Sprintf("#%02X%02X%02X", b, 0xF3, b) -} - -func red(t time.Time) string { - b := daysuntil.interpolate(t) - return fmt.Sprintf("#%02X%02X%02X", 0xF3, b, b) -} - -type Cert struct { - Url string - Updated time.Time - X509 *x509.Certificate -} - -type Certs []Cert - -// Len is the number of elements in the collection. -func (l Certs) Len() int { - return len(l) -} - -// Less reports whether the element with -// index i should sort before the element with index j. -func (l Certs) Less(i, j int) bool { - return l[i].Updated.UTC().After(l[j].Updated.UTC()) -} - -// Swap swaps the elements with indexes i and j. -func (l Certs) Swap(i, j int) { - tmp := l[i] - l[i] = l[j] - l[j] = tmp -} - -func main() { - data, err := ioutil.ReadAll(os.Stdin) - handleErr(err, "Error reading stdin: %v\n") - - var certs Certs - for len(data) > 0 { - var certPem *pem.Block - certPem, data = pem.Decode(data) - - var ok bool - var cert Cert - - cert.Url, ok = certPem.Headers["X-Crt-Sh-Url"] - handleBool(ok, "Did not get X-Crt-Sh-Url\n") - - str, ok := certPem.Headers["X-Crt-Sh-Updated"] - handleBool(ok, "Did not get X-Crt-Sh-Updated\n") - cert.Updated, err = time.Parse("2006-01-02T15:04:05Z", str) - handleErr(err, "Could not parse updated time") - - cert.X509, err = x509.ParseCertificate(certPem.Bytes) - handleErr(err, "Error parsing cert: %v\n") - - certs = append(certs, cert) - } - - sort.Sort(certs) - handleErr(tmpl.Execute(os.Stdout, map[string]interface{}{"certs":certs, "now": now}), "Could not execute template: %v\n") -} diff --git a/tls-getcerts.go b/tls-getcerts.go new file mode 100644 index 0000000..b0d4533 --- /dev/null +++ b/tls-getcerts.go @@ -0,0 +1,35 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "encoding/pem" +) + +func getcert(server string) (*x509.Certificate, error){ + conn, err := tls.Dial("tcp", fmt.Sprintf("%s:443", server), &tls.Config{ServerName: server}) + if err != nil { + return nil, err + } + defer conn.Close() + chain := conn.ConnectionState().PeerCertificates + return chain[len(chain)-2], nil +} + +func main() { + for _, server := range os.Args[1:] { + cert, err := getcert(server) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not get certificate from server %q: %q\n", server, err) + os.Exit(1) + } + block := pem.Block{ + Type: "CERTIFICATE", + Headers: nil, + Bytes: cert.Raw, + } + pem.Encode(os.Stdout, &block) + } +} -- cgit v1.1-4-g5e80