diff options
Diffstat (limited to 'devel')
| -rw-r--r-- | devel/management/commands/reporead.py | 176 | ||||
| -rw-r--r-- | devel/urls.py | 8 | ||||
| -rw-r--r-- | devel/views.py | 189 |
3 files changed, 290 insertions, 83 deletions
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 09e48559..a8875c7e 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -35,12 +35,11 @@ try: except ImportError: pass -from logging import ERROR, WARNING, INFO, DEBUG - from main.models import Arch, Package, PackageDepend, PackageFile, Repo +from packages.models import Conflict, Provision, Replacement logging.basicConfig( - level=WARNING, + level=logging.WARNING, format='%(asctime)s -> %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', stream=sys.stderr) @@ -69,11 +68,11 @@ class Command(BaseCommand): v = int(options.get('verbosity', 0)) if v == 0: - logger.level = ERROR + logger.level = logging.ERROR elif v == 1: - logger.level = INFO + logger.level = logging.INFO elif v == 2: - logger.level = DEBUG + logger.level = logging.DEBUG import signal, traceback handler = lambda sig, stack: traceback.print_stack(stack) @@ -88,13 +87,23 @@ class Pkg(object): bare = ( 'name', 'base', 'arch', 'desc', 'filename', 'md5sum', 'url', 'builddate', 'packager' ) number = ( 'csize', 'isize' ) + collections = ( 'depends', 'optdepends', 'conflicts', + 'provides', 'replaces', 'groups', 'license', 'files' ) + + version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') def __init__(self, repo): self.repo = repo self.ver = None self.rel = None + self.epoch = 0 for k in self.bare + self.number: setattr(self, k, None) + for k in self.collections: + setattr(self, k, ()) + # So we can tell the diffence between a package with no files, and a DB + # without files entries + self.has_files = False def populate(self, values): for k, v in values.iteritems(): @@ -103,16 +112,26 @@ class Pkg(object): setattr(self, k, v[0][:254]) elif k in self.number: setattr(self, k, long(v[0])) - elif k == 'force': - setattr(self, k, True) elif k == 'version': - ver, rel = v[0].rsplit('-') - setattr(self, 'ver', ver) - setattr(self, 'rel', rel) + match = self.version_re.match(v[0]) + self.ver = match.group(3) + self.rel = match.group(4) + if match.group(2): + self.epoch = int(match.group(2)) + elif k == 'files': + self.files = v + self.has_files = True else: - # files, depends, etc. + # anything left in collections setattr(self, k, v) + @property + def full_version(self): + '''Very similar to the main.models.Package method.''' + if self.epoch > 0: + return u'%d:%s-%s' % (self.epoch, self.ver, self.rel) + return u'%s-%s' % (self.ver, self.rel) + def find_user(userstring): ''' @@ -163,20 +182,58 @@ def find_user(userstring): # lookup more than strictly necessary. find_user.cache = {} +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") + def create_depend(package, dep_str, optional=False): depend = PackageDepend(pkg=package, optional=optional) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() - match = re.match(r"^(.+?)((>=|<=|=|>|<)(.*))?$", parts[0].strip()) + match = DEPEND_RE.match(parts[0].strip()) if match: depend.depname = match.group(1) if match.group(2): depend.depvcmp = match.group(2) + else: + logger.warning('Package %s had unparsable depend string %s', + package.pkgname, dep_str) + return None depend.save(force_insert=True) return depend +def create_related(model, package, rel_str, equals_only=False): + related = model(pkg=package) + match = DEPEND_RE.match(rel_str) + if match: + related.name = match.group(1) + if match.group(3): + comp = match.group(3) + if not equals_only: + related.comparison = comp + elif comp != '=': + logger.warning( + 'Package %s had unexpected comparison operator %s for %s in %s', + package.pkgname, comp, model.__name__, rel_str) + if match.group(4): + related.version = match.group(4) + else: + logger.warning('Package %s had unparsable %s string %s', + package.pkgname, model.___name__, rel_str) + return None + related.save(force_insert=True) + return related + +def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): + '''Populate the simplest of multivalued attributes. These are those that + only deal with a 'name' attribute, such as licenses, groups, etc. The input + and output objects and attribute names are specified, and everything is + done via getattr().''' + collection = getattr(dbpkg, db_attr) + collection.all().delete() + for name in getattr(repopkg, repo_attr): + collection.create(name=name) + def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): if repopkg.base: dbpkg.pkgbase = repopkg.base @@ -184,6 +241,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.pkgbase = repopkg.name dbpkg.pkgver = repopkg.ver dbpkg.pkgrel = repopkg.rel + dbpkg.epoch = repopkg.epoch dbpkg.pkgdesc = repopkg.desc dbpkg.url = repopkg.url dbpkg.filename = repopkg.filename @@ -210,38 +268,39 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): populate_files(dbpkg, repopkg, force=force) dbpkg.packagedepend_set.all().delete() - if hasattr(repopkg, 'depends'): - for y in repopkg.depends: - dep = create_depend(dbpkg, y) - if hasattr(repopkg, 'optdepends'): - for y in repopkg.optdepends: - dep = create_depend(dbpkg, y, True) + for y in repopkg.depends: + create_depend(dbpkg, y) + for y in repopkg.optdepends: + create_depend(dbpkg, y, True) - dbpkg.groups.all().delete() - if hasattr(repopkg, 'groups'): - for y in repopkg.groups: - dbpkg.groups.create(name=y) + dbpkg.conflicts.all().delete() + for y in repopkg.conflicts: + create_related(Conflict, dbpkg, y) + dbpkg.provides.all().delete() + for y in repopkg.provides: + create_related(Provision, dbpkg, y, equals_only=True) + dbpkg.replaces.all().delete() + for y in repopkg.replaces: + create_related(Replacement, dbpkg, y) - dbpkg.licenses.all().delete() - if hasattr(repopkg, 'license'): - for y in repopkg.license: - dbpkg.licenses.create(name=y) + create_multivalued(dbpkg, repopkg, 'groups', 'groups') + create_multivalued(dbpkg, repopkg, 'licenses', 'license') def populate_files(dbpkg, repopkg, force=False): if not force: - if dbpkg.pkgver != repopkg.ver or dbpkg.pkgrel != repopkg.rel: - logger.info("db version (%s-%s) didn't match repo version (%s-%s) " - "for package %s, skipping file list addition", - dbpkg.pkgver, dbpkg.pkgrel, repopkg.ver, repopkg.rel, - dbpkg.pkgname) + if dbpkg.pkgver != repopkg.ver or dbpkg.pkgrel != repopkg.rel \ + or dbpkg.epoch != repopkg.epoch: + logger.info("DB version (%s) didn't match repo version " + "(%s) for package %s, skipping file list addition", + dbpkg.full_version, repopkg.full_version, dbpkg.pkgname) return if not dbpkg.files_last_update or not dbpkg.last_update: pass elif dbpkg.files_last_update > dbpkg.last_update: return # only delete files if we are reading a DB that contains them - if hasattr(repopkg, 'files'): + if repopkg.has_files: dbpkg.packagefile_set.all().delete() logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) @@ -255,8 +314,8 @@ def populate_files(dbpkg, repopkg, force=False): is_directory=(filename is None), directory=dirname + '/', filename=filename) - pkgfile.save() - dbpkg.files_last_update = datetime.now() + pkgfile.save(force_insert=True) + dbpkg.files_last_update = datetime.utcnow() dbpkg.save() @transaction.commit_on_success @@ -273,25 +332,23 @@ def db_update(archname, reponame, pkgs, options): filesonly = options.get('filesonly', False) repository = Repo.objects.get(name__iexact=reponame) architecture = Arch.objects.get(name__iexact=archname) - dbpkgs = Package.objects.filter(arch=architecture, repo=repository) - # It makes sense to fully evaluate our DB query now because we will - # be using 99% of the objects in our "in both sets" loop. Force eval - # by calling list() on the QuerySet. - list(dbpkgs) + # no-arg order_by() removes even the default ordering; we don't need it + dbpkgs = Package.objects.filter( + arch=architecture, repo=repository).order_by() # This makes our inner loop where we find packages by name *way* more # efficient by not having to go to the database for each package to # SELECT them by name. dbdict = dict([(pkg.pkgname, pkg) for pkg in dbpkgs]) logger.debug("Creating sets") - dbset = set([pkg.pkgname for pkg in dbpkgs]) + dbset = set(dbdict.keys()) syncset = set([pkg.name for pkg in pkgs]) logger.info("%d packages in current web DB", len(dbset)) logger.info("%d packages in new updating db", len(syncset)) in_sync_not_db = syncset - dbset logger.info("%d packages in sync not db", len(in_sync_not_db)) - # Try to catch those random orphaning issues that make Eric so unhappy. + # Try to catch those random package deletions that make Eric so unhappy. if len(dbset): dbpercent = 100.0 * len(syncset) / len(dbset) else: @@ -302,12 +359,14 @@ def db_update(archname, reponame, pkgs, options): # means we expect the repo to fluctuate a lot. msg = "Package database has %.1f%% the number of packages in the " \ "web database" % dbpercent - if not filesonly and \ + if len(dbset) == 0 and len(syncset) == 0: + pass + elif not filesonly and \ len(dbset) > 20 and dbpercent < 50.0 and \ - not repository.testing: + not repository.testing and not repository.staging: logger.error(msg) raise Exception(msg) - if dbpercent < 75.0: + elif dbpercent < 75.0: logger.warning(msg) if not filesonly: @@ -315,14 +374,14 @@ def db_update(archname, reponame, pkgs, options): for p in [x for x in pkgs if x.name in in_sync_not_db]: logger.info("Adding package %s", p.name) pkg = Package(pkgname = p.name, arch = architecture, repo = repository) - populate_pkg(pkg, p, timestamp=datetime.now()) + populate_pkg(pkg, p, timestamp=datetime.utcnow()) # packages in database and not in syncdb (remove from database) in_db_not_sync = dbset - syncset for p in in_db_not_sync: logger.info("Removing package %s from database", p) - Package.objects.get( - pkgname=p, arch=architecture, repo=repository).delete() + dbp = dbdict[p] + dbp.delete() # packages in both database and in syncdb (update in database) pkg_in_both = syncset & dbset @@ -334,11 +393,12 @@ def db_update(archname, reponame, pkgs, options): # for a non-force, we don't want to do anything at all. if filesonly: pass - elif p.ver == dbp.pkgver and p.rel == dbp.pkgrel: + elif p.ver == dbp.pkgver and p.rel == dbp.pkgrel \ + and p.epoch == dbp.epoch: if not force: continue else: - timestamp = datetime.now() + timestamp = datetime.utcnow() if filesonly: logger.debug("Checking files for package %s in database", p.name) populate_files(dbp, p, force=force) @@ -421,10 +481,9 @@ def parse_repo(repopath): logger.info("Finished repo parsing, %d total packages", len(pkgs)) return (reponame, pkgs.values()) -def validate_arch(arch): +def validate_arch(archname): "Check if arch is valid." - available_arches = [x.name for x in Arch.objects.all()] - return arch in available_arches + return Arch.objects.filter(name__iexact=archname).exists() def read_repo(primary_arch, repo_file, options): """ @@ -432,21 +491,22 @@ def read_repo(primary_arch, repo_file, options): """ repo, packages = parse_repo(repo_file) - # sort packages by arch -- to handle noarch stuff + # group packages by arch -- to handle noarch stuff packages_arches = {} - packages_arches['any'] = [] + for arch in Arch.objects.filter(agnostic=True): + packages_arches[arch.name] = [] packages_arches[primary_arch] = [] for package in packages: - if package.arch in ('any', primary_arch): + if package.arch in packages_arches: packages_arches[package.arch].append(package) else: # we don't include mis-arched packages logger.warning("Package %s arch = %s", package.name,package.arch) logger.info('Starting database updates.') - for (arch, pkgs) in packages_arches.items(): - db_update(arch, repo, pkgs, options) + for arch in sorted(packages_arches.keys()): + db_update(arch, repo, packages_arches[arch], options) logger.info('Finished database updates.') return 0 diff --git a/devel/urls.py b/devel/urls.py index bcf9c071..8759562e 100644 --- a/devel/urls.py +++ b/devel/urls.py @@ -1,10 +1,14 @@ from django.conf.urls.defaults import patterns urlpatterns = patterns('devel.views', - (r'^$', 'index'), + (r'^admin_log/$','admin_log'), + (r'^admin_log/(?P<username>.*)/$','admin_log'), (r'^clock/$', 'clock'), - (r'^profile/$', 'change_profile'), + (r'^$', 'index'), (r'^newuser/$', 'new_user_form'), + (r'^profile/$', 'change_profile'), + (r'^reports/(?P<report>.*)/(?P<username>.*)/$', 'report'), + (r'^reports/(?P<report>.*)/$', 'report'), ) # vim: set ts=4 sw=4 et: diff --git a/devel/views.py b/devel/views.py index 311922ca..c0d10285 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,20 +1,28 @@ from django import forms from django.http import HttpResponseRedirect -from django.contrib.auth.decorators import login_required, permission_required -from django.contrib.auth.models import User +from django.contrib.auth.decorators import \ + login_required, permission_required, user_passes_test +from django.contrib.auth.models import User, Group from django.contrib.sites.models import Site from django.core.mail import send_mail +from django.db import transaction +from django.db.models import Q +from django.http import Http404 +from django.shortcuts import get_object_or_404 from django.template import loader, Context +from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template -from main.models import Package, Todolist, TodolistPkg +from main.models import Package, PackageDepend, PackageFile, TodolistPkg from main.models import Arch, Repo from main.models import UserProfile from packages.models import PackageRelation +from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers -import datetime +from datetime import datetime, timedelta +import operator import pytz import random from string import ascii_letters, digits @@ -22,7 +30,7 @@ from string import ascii_letters, digits @login_required @never_cache def index(request): - '''the Developer dashboard''' + '''the developer dashboard''' inner_q = PackageRelation.objects.filter(user=request.user).values('pkgbase') flagged = Package.objects.select_related('arch', 'repo').filter( flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') @@ -32,13 +40,27 @@ def index(request): todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by( 'list__name', 'pkg__pkgname') + todolists = get_annotated_todolists() + todolists = [todolist for todolist in todolists if todolist.incomplete_count > 0] + maintainers = get_annotated_maintainers() + maintained = PackageRelation.objects.filter( + type=PackageRelation.MAINTAINER).values('pkgbase') + total_orphans = Package.objects.exclude(pkgbase__in=maintained).count() + total_flagged_orphans = Package.objects.filter( + flag_date__isnull=False).exclude(pkgbase__in=maintained).count() + orphan = { + 'package_count': total_orphans, + 'flagged_count': total_flagged_orphans, + } + page_dict = { - 'todos': Todolist.objects.incomplete().order_by('-date_added'), + 'todos': todolists, 'repos': Repo.objects.all(), 'arches': Arch.objects.all(), 'maintainers': maintainers, + 'orphan': orphan, 'flagged' : flagged, 'todopkgs' : todopkgs, } @@ -52,10 +74,12 @@ def clock(request): 'username').select_related('userprofile') # now annotate each dev object with their current time - now = datetime.datetime.now() - utc_now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + now = datetime.now() + utc_now = datetime.utcnow().replace(tzinfo=pytz.utc) for dev in devs: - tz = pytz.timezone(dev.userprofile.time_zone) + # Work around https://bugs.launchpad.net/pytz/+bug/718673 + timezone = str(dev.userprofile.time_zone) + tz = pytz.timezone(timezone) dev.current_time = utc_now.astimezone(tz) page_dict = { @@ -103,14 +127,98 @@ def change_profile(request): return direct_to_template(request, 'devel/profile.html', {'form': form, 'profile_form': profile_form}) +@login_required +def report(request, report, username=None): + title = 'Developer Report' + packages = Package.objects.select_related('arch', 'repo') + names = attrs = user = None + + if report == 'old': + title = 'Packages last built more than two years ago' + cutoff = datetime.now() - timedelta(days=730) + packages = packages.filter( + build_date__lt=cutoff).order_by('build_date') + elif report == 'big': + title = 'Packages with compressed size > 50 MiB' + cutoff = 50 * 1024 * 1024 + packages = packages.filter( + compressed_size__gte=cutoff).order_by('-compressed_size') + names = [ 'Compressed Size', 'Installed Size' ] + attrs = [ 'compressed_size_pretty', 'installed_size_pretty' ] + # Format the compressed and installed sizes with MB/GB/etc suffixes + for package in packages: + package.compressed_size_pretty = filesizeformat( + package.compressed_size) + package.installed_size_pretty = filesizeformat( + package.installed_size) + elif report == 'uncompressed-man': + title = 'Packages with uncompressed manpages' + # magic going on here! Checking for all '.1'...'.9' extensions + invalid_endings = [Q(filename__endswith='.%d' % n) for n in range(1,10)] + invalid_endings.append(Q(filename__endswith='.n')) + bad_files = PackageFile.objects.filter(Q(directory__contains='man') & ( + reduce(operator.or_, invalid_endings)) + ).values_list('pkg_id', flat=True).distinct() + packages = packages.filter(id__in=set(bad_files)) + elif report == 'uncompressed-info': + title = 'Packages with uncompressed infopages' + # we don't worry abut looking for '*.info-1', etc., given that an + # uncompressed root page probably exists in the package anyway + bad_files = PackageFile.objects.filter(directory__endswith='/info/', + filename__endswith='.info').values_list( + 'pkg_id', flat=True).distinct() + packages = packages.filter(id__in=set(bad_files)) + elif report == 'unneeded-orphans': + title = 'Orphan packages required by no other packages' + owned = PackageRelation.objects.all().values('pkgbase') + required = PackageDepend.objects.all().values('depname') + # The two separate calls to exclude is required to do the right thing + packages = packages.exclude(pkgbase__in=owned).exclude( + pkgname__in=required) + else: + raise Http404 + + if username: + user = get_object_or_404(User, username=username, is_active=True) + maintained = PackageRelation.objects.filter(user=user, + type=PackageRelation.MAINTAINER).values('pkgbase') + packages = packages.filter(pkgbase__in=maintained) + + maints = User.objects.filter(id__in=PackageRelation.objects.filter( + type=PackageRelation.MAINTAINER).values('user')) + + context = { + 'all_maintainers': maints, + 'title': title, + 'maintainer': user, + 'packages': packages, + 'column_names': names, + 'column_attrs': attrs, + } + return direct_to_template(request, 'devel/packages.html', context) + + class NewUserForm(forms.ModelForm): - class Meta: - model = UserProfile - exclude = ('picture', 'user') username = forms.CharField(max_length=30) - email = forms.EmailField() + private_email = forms.EmailField() first_name = forms.CharField(required=False) last_name = forms.CharField(required=False) + groups = forms.ModelMultipleChoiceField(required=False, + queryset=Group.objects.all()) + + class Meta: + model = UserProfile + exclude = ('picture', 'user') + + def __init__(self, *args, **kwargs): + super(NewUserForm, self).__init__(*args, **kwargs) + # Hack ourself so certain fields appear first. self.fields is a + # SortedDict object where we can manipulate the keyOrder list. + order = self.fields.keyOrder + keys = ('username', 'private_email', 'first_name', 'last_name') + for key in reversed(keys): + order.remove(key) + order.insert(0, key) def clean_username(self): username = self.cleaned_data['username'] @@ -119,38 +227,61 @@ class NewUserForm(forms.ModelForm): "A user with that username already exists.") return username - def save(self): - profile = forms.ModelForm.save(self, False) + def save(self, commit=True): + profile = super(NewUserForm, self).save(False) pwletters = ascii_letters + digits password = ''.join([random.choice(pwletters) for i in xrange(8)]) user = User.objects.create_user(username=self.cleaned_data['username'], - email=self.cleaned_data['email'], password=password) + email=self.cleaned_data['private_email'], password=password) user.first_name = self.cleaned_data['first_name'] user.last_name = self.cleaned_data['last_name'] user.save() + # sucks that the MRM.add() method can't take a list directly... we have + # to resort to dirty * magic. + user.groups.add(*self.cleaned_data['groups']) profile.user = user - profile.save() + if commit: + profile.save() + self.save_m2m() - t = loader.get_template('devel/new_account.txt') - c = Context({ + template = loader.get_template('devel/new_account.txt') + ctx = Context({ 'site': Site.objects.get_current(), 'user': user, 'password': password, }) - send_mail("Your new archweb account", - t.render(c), - 'Arch Website Notification <nobody@archlinux.org>', + send_mail("Your new parabolaweb account", + template.render(ctx), + 'Parabola <dev@list.parabolagnulinux.org>', [user.email], fail_silently=False) +def log_addition(request, obj): + """Cribbed from ModelAdmin.log_addition.""" + from django.contrib.admin.models import LogEntry, ADDITION + from django.contrib.contenttypes.models import ContentType + from django.utils.encoding import force_unicode + LogEntry.objects.log_action( + user_id = request.user.pk, + content_type_id = ContentType.objects.get_for_model(obj).pk, + object_id = obj.pk, + object_repr = force_unicode(obj), + action_flag = ADDITION, + change_message = "Added via Create New User form." + ) + @permission_required('auth.add_user') @never_cache def new_user_form(request): if request.POST: form = NewUserForm(request.POST) if form.is_valid(): - form.save() + @transaction.commit_on_success + def inner_save(): + form.save() + log_addition(request, form.instance.user) + inner_save() return HttpResponseRedirect('/admin/auth/user/%d/' % \ form.instance.user.id) else: @@ -167,4 +298,16 @@ def new_user_form(request): } return direct_to_template(request, 'general_form.html', context) +@user_passes_test(lambda u: u.is_superuser) +@never_cache +def admin_log(request, username=None): + user = None + if username: + user = get_object_or_404(User, username=username) + context = { + 'title': "Admin Action Log", + 'log_user': user, + } + return direct_to_template(request, 'devel/admin_log.html', context) + # vim: set ts=4 sw=4 et: |