From c9d51d08c9e9fe0e04c013809f4278036b05b50e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sun, 4 Dec 2011 17:51:36 +0100 Subject: download: add a link to the new netboot environment Signed-off-by: Dan McGee --- public/views.py | 1 + settings.py | 3 +++ templates/public/download.html | 2 ++ 3 files changed, 6 insertions(+) diff --git a/public/views.py b/public/views.py index 40512614..8fb60c78 100644 --- a/public/views.py +++ b/public/views.py @@ -63,6 +63,7 @@ def download(request): ) context = { 'releng_iso_url': settings.ISO_LIST_URL, + 'releng_pxeboot_url': settings.PXEBOOT_URL, } return list_detail.object_list(request, qset.order_by('mirror__country', 'mirror__name', 'protocol'), diff --git a/settings.py b/settings.py index 77f84ba2..b22b4293 100644 --- a/settings.py +++ b/settings.py @@ -118,6 +118,9 @@ PGP_SERVER = 'pgp.mit.edu:11371' # URL to fetch a current list of available ISOs ISO_LIST_URL = 'http://releng.archlinux.org/isos/' +# URL to the PXE netboot instructions +PXEBOOT_URL = 'http://releng.archlinux.org/pxeboot/' + # URL for SVN access for fetching commit messages (note absence of packages or # community bit on the end, repo.svn_root is appended) SVN_BASE_URL = 'svn+ssh://svn.archlinux.org/srv/svn-' diff --git a/templates/public/download.html b/templates/public/download.html index 36aca9ad..3e33ab32 100644 --- a/templates/public/download.html +++ b/templates/public/download.html @@ -122,6 +122,8 @@ -- cgit v1.1-4-g5e80 From fb61a6acdf113e2c2266b93fb690b0444c175264 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 5 Dec 2011 21:54:30 -0600 Subject: Add import signatures management command This allow importing signatures from a provided gpg keyring, such as that produced by the generate_keyring management command that already exists. These will eventually be used for producing stats involving developer signing keys and their certification by master keys. Signed-off-by: Dan McGee --- devel/management/commands/import_signatures.py | 104 +++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 devel/management/commands/import_signatures.py diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py new file mode 100644 index 00000000..8a4ce873 --- /dev/null +++ b/devel/management/commands/import_signatures.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +""" +import_signatures command + +Import signatures from a given GPG keyring. + +Usage: ./manage.py generate_keyring +""" + +from datetime import datetime +import logging +import subprocess +import sys + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from devel.models import PGPSignature + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(BaseCommand): + args = "" + help = "Import signatures from a given GPG keyring." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v == 2: + logger.level = logging.DEBUG + + if len(args) < 1: + raise CommandError("keyring_path must be provided") + + import_signatures(args[0]) + +def parse_sigdata(data): + nodes = {} + edges = [] + current_pubkey = None + + # parse all of the output from our successful GPG command + logger.info("parsing command output") + for line in data.split('\n'): + parts = line.split(':') + if parts[0] == 'pub': + current_pubkey = parts[4] + nodes[current_pubkey] = None + if parts[0] == 'uid': + uid = parts[9] + # only set uid if this is the first one encountered + if nodes[current_pubkey] is None: + nodes[current_pubkey] = uid + if parts[0] == 'sig': + created = datetime.utcfromtimestamp(int(parts[5])) + expires = None + if parts[6]: + expires = datetime.utcfromtimestamp(int(parts[6])) + valid = parts[1] != '-' + edge = (parts[4], current_pubkey, created, expires, valid) + edges.append(edge) + + return nodes, edges + + +def import_signatures(keyring): + gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, + "--list-sigs", "--with-colons", "--fixed-list-mode"] + logger.info("running command: %r", gpg_cmd) + proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE) + outdata, errdata = proc.communicate() + if proc.returncode != 0: + logger.error(errdata) + raise subprocess.CalledProcessError(proc.returncode, gpg_cmd) + + nodes, edges = parse_sigdata(outdata) + + # now prune the data down to what we actually want. + # prune edges not in nodes, remove duplicates, and self-sigs + pruned_edges = set(edge for edge in edges + if edge[0] in nodes and edge[0] != edge[1]) + + logger.info("creating or finding %d signatures", len(pruned_edges)) + created_ct = 0 + with transaction.commit_on_success(): + for edge in pruned_edges: + _, created = PGPSignature.objects.get_or_create( + signer=edge[0], signee=edge[1], + created=edge[2], expires=edge[3], + defaults={ 'valid': edge[4] }) + if created: + created_ct += 1 + + logger.info("created %d signatures", created_ct) + +# vim: set ts=4 sw=4 et: -- cgit v1.1-4-g5e80 From ab9162ac1413b5f59833f7f710863643766ee2f5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 5 Dec 2011 21:57:33 -0600 Subject: Add pgp_keys visualize JSON view This will be used for an SVG graph of all of the master and developer PGP keys. Signed-off-by: Dan McGee --- visualize/urls.py | 1 + visualize/views.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/visualize/urls.py b/visualize/urls.py index 57ee0626..02d83bec 100644 --- a/visualize/urls.py +++ b/visualize/urls.py @@ -4,6 +4,7 @@ urlpatterns = patterns('visualize.views', (r'^$', 'index', {}, 'visualize-index'), (r'^by_arch/$', 'by_arch', {}, 'visualize-byarch'), (r'^by_repo/$', 'by_repo', {}, 'visualize-byrepo'), + (r'^pgp_keys/$', 'pgp_keys', {}, 'visualize-pgp_keys'), ) # vim: set ts=4 sw=4 et: diff --git a/visualize/views.py b/visualize/views.py index f2b1d63b..be6057b2 100644 --- a/visualize/views.py +++ b/visualize/views.py @@ -1,10 +1,14 @@ -from django.db.models import Count, Sum +from datetime import datetime + +from django.contrib.auth.models import User +from django.db.models import Count, Sum, Q from django.http import HttpResponse from django.utils import simplejson from django.views.decorators.cache import cache_page from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo +from devel.models import MasterKey, PGPSignature def index(request): return direct_to_template(request, 'visualize/index.html', {}) @@ -66,4 +70,46 @@ def by_repo(request): to_json = simplejson.dumps(data['by_repo'], ensure_ascii=False) return HttpResponse(to_json, mimetype='application/json') + +@cache_page(1800) +def pgp_keys(request): + node_list = [] + + users = User.objects.filter(is_active=True).select_related('userprofile') + node_list.extend({ + 'name': dev.get_full_name(), + 'key': dev.userprofile.pgp_key, + 'group': 'dev' + } for dev in users.filter(groups__name='Developers')) + node_list.extend({ + 'name': tu.get_full_name(), + 'key': tu.userprofile.pgp_key, + 'group': 'tu' + } for tu in users.filter(groups__name='Trusted Users').exclude( + groups__name='Developers')) + + master_keys = MasterKey.objects.select_related('owner').filter( + revoked__isnull=True) + node_list.extend({ + 'name': 'Master Key (%s)' % key.owner.get_full_name(), + 'key': key.pgp_key, + 'group': 'master' + } for key in master_keys) + + node_list.append({ + 'name': 'CA Cert Signing Authority', + 'key': 'A31D4F81EF4EBD07B456FA04D2BB0D0165D0FD58', + 'group': 'cacert', + }) + + not_expired = Q(expires__gt=datetime.now) | Q(expires__isnull=True) + signatures = PGPSignature.objects.filter(not_expired, valid=True) + edge_list = [{ 'signee': sig.signee, 'signer': sig.signer } + for sig in signatures] + + data = { 'nodes': node_list, 'edges': edge_list } + + to_json = simplejson.dumps(data, ensure_ascii=False) + return HttpResponse(to_json, mimetype='application/json') + # vim: set ts=4 sw=4 et: -- cgit v1.1-4-g5e80 From 1c23308299f33e5b429899463eb207f07ad51403 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 5 Dec 2011 22:12:09 -0600 Subject: Add developer keys visualization Well, almost add it- it is currently commented out as I have a few more things I'd like to take care of, namely correcting static files versioning and caching, to ensure this doesn't break things. This is a force-directed graph drawn using D3 as the package treemap already does. We color the dots by "group", e.g. "dev", "tu", or "master", and then outline developer keys in green if they have at least 3 master key signatures, red if they have fewer. Hovering over a circle will show you who's key you are seeing in the visualization. Signed-off-by: Dan McGee --- media/archweb.css | 12 ++++ media/visualize.js | 130 ++++++++++++++++++++++++++++++++++++++++- templates/visualize/index.html | 13 +++-- 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/media/archweb.css b/media/archweb.css index f4bb92fa..a354cb96 100644 --- a/media/archweb.css +++ b/media/archweb.css @@ -986,3 +986,15 @@ ul.signoff-list { font-size: 0.85em; line-height: 1em; } + +#visualize-keys svg { + width: 100%; +} + + #visualize-keys circle { + stroke-width: 1.5px; + } + + #visualize-keys line { + stroke: #888; + } diff --git a/media/visualize.js b/media/visualize.js index d9196d4d..e73171ea 100644 --- a/media/visualize.js +++ b/media/visualize.js @@ -1,5 +1,5 @@ function packages_treemap(chart_id, orderings, default_order) { - var jq_div = $(chart_id), + var jq_div = jQuery(chart_id), color = d3.scale.category20(); var key_func = function(d) { return d.key; }; var value_package_count = function(d) { return d.count; }, @@ -109,7 +109,7 @@ function packages_treemap(chart_id, orderings, default_order) { }); }; - $.each(orderings, function(k, v) { + jQuery.each(orderings, function(k, v) { make_group_button(k, v); }); @@ -120,7 +120,131 @@ function packages_treemap(chart_id, orderings, default_order) { .data(treemap.size([jq_div.width(), jq_div.height()]), key_func) .call(cell); }; - $(window).resize(function() { + jQuery(window).resize(function() { + if (resize_timeout) { + clearTimeout(resize_timeout); + } + resize_timeout = setTimeout(real_resize, 200); + }); +} + +function developer_keys(chart_id, data_url) { + var jq_div = jQuery(chart_id), + r = 10; + + var force = d3.layout.force() + .gravity(0.1) + .charge(-200) + .linkStrength(0.2) + .size([jq_div.width(), jq_div.height()]); + + var svg = d3.select(chart_id) + .append("svg"); + + d3.json(data_url, function(json) { + var fill = d3.scale.category20(); + + var index_for_key = function(key) { + var i; + key = key.slice(-8); + for (i = 0; i < json.nodes.length; i++) { + var node_key = json.nodes[i].key; + if (node_key && node_key.slice(-8) === key) { + return i; + } + } + }; + + /* filter edges to only include those that we have two nodes for */ + var edges = jQuery.grep(json.edges, function(d, i) { + d.source = index_for_key(d.signer); + d.target = index_for_key(d.signee); + return d.source >= 0 && d.target >= 0; + }); + + jQuery.map(json.nodes, function(d, i) { d.master_sigs = 0; }); + jQuery.map(edges, function(d, i) { + if (json.nodes[d.source].group === "master") { + json.nodes[d.target].master_sigs += 1; + } + }); + jQuery.map(json.nodes, function(d, i) { + if (d.group === "dev" || d.group === "tu") { + d.approved = d.master_sigs >= 3; + } else { + d.approved = null; + } + }); + + var link = svg.selectAll("line") + .data(edges) + .enter() + .append("line"); + + var node = svg.selectAll("circle") + .data(json.nodes) + .enter().append("circle") + .attr("r", function(d) { + switch (d.group) { + case "master": + return r * 1.6 - 0.75; + case "cacert": + return r * 1.4 - 0.75; + case "dev": + case "tu": + default: + return r - 0.75; + } + }) + .style("fill", function(d) { return fill(d.group); }) + .style("stroke", function(d) { + if (d.approved === null) { + return d3.rgb(fill(d.group)).darker(); + } else if (d.approved) { + return "green"; + } else { + return "red"; + } + }) + .call(force.drag); + node.append("title").text(function(d) { return d.name; }); + + var distance = function(d, i) { + /* place a long line between all master keys and other keys. + * however, other connected clusters should be close together. */ + if (d.source.group === "master" || d.target.group === "master") { + return 200; + } else { + return 50; + } + }; + + var tick = function() { + var offset = r * 2, + w = jq_div.width(), + h = jq_div.height(); + node.attr("cx", function(d) { return (d.x = Math.max(offset, Math.min(w - offset, d.x))); }) + .attr("cy", function(d) { return (d.y = Math.max(offset, Math.min(h - offset, d.y))); }); + + link.attr("x1", function(d) { return d.source.x; }) + .attr("y1", function(d) { return d.source.y; }) + .attr("x2", function(d) { return d.target.x; }) + .attr("y2", function(d) { return d.target.y; }); + }; + + force.nodes(json.nodes) + .links(edges) + .linkDistance(distance) + .on("tick", tick) + .start(); + }); + + var resize_timeout = null; + var real_resize = function() { + resize_timeout = null; + force.size([jq_div.width(), jq_div.height()]); + }; + jQuery(window).resize(function() { if (resize_timeout) { clearTimeout(resize_timeout); } diff --git a/templates/visualize/index.html b/templates/visualize/index.html index 99525e69..b9459318 100644 --- a/templates/visualize/index.html +++ b/templates/visualize/index.html @@ -4,10 +4,7 @@ {% block content %}
- -

Visualizations of Packaging Data

- -

Package Treemap

+

Visualization of Package Data

@@ -25,9 +22,16 @@
+{% comment %} +
+

Visualization of PGP Master and Signing Keys

+
+
+{% endcomment %} {% load cdn %}{% jquery %} + @@ -38,6 +42,7 @@ $(document).ready(function() { "arch": { url: "{% url visualize-byarch %}", color_attr: "arch" }, }; packages_treemap("#visualize-archrepo", orderings, "repo"); + /*developer_keys("#visualize-keys", "{% url visualize-pgp_keys %}");*/ }); {% endblock %} -- cgit v1.1-4-g5e80