From 376ce4a69e016d13eff28589a5caa627bf7c451b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 7 Feb 2011 12:48:12 -0600 Subject: Clean up Package related objects code Main change is just to move groups from the default packagegroup_set location to a related_name of groups. Also refer to the Package class directly rather than by text string if we have it available. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index e31478c5..bdd0882c 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -203,10 +203,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.packagedepend_set.create(depname=dpname, depvcmp=dpvcmp) logger.debug('Added %s as dep for pkg %s', dpname, repopkg.name) - dbpkg.packagegroup_set.all().delete() + dbpkg.groups.all().delete() if 'groups' in repopkg.__dict__: for y in repopkg.groups: - dbpkg.packagegroup_set.create(name=y) + dbpkg.groups.create(name=y) def populate_files(dbpkg, repopkg, force=False): -- cgit v1.2.3-2-g168b From 4444f25d5cd9a9e1cb310a2d9c0b6b1ec1f55789 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 7 Feb 2011 13:45:05 -0600 Subject: Move license to a related model This allows us to store multiple licenses per package in a more elegant fashion, and will later allow us to search and filter on this information. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index bdd0882c..9b99e0f7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -79,14 +79,13 @@ class Pkg(object): """An interim 'container' object for holding Arch package data.""" bare = ( 'name', 'base', 'arch', 'desc', 'filename', 'md5sum', 'url', 'builddate', 'packager' ) - squash = ( 'license', ) number = ( 'csize', 'isize' ) def __init__(self, repo): self.repo = repo self.ver = None self.rel = None - for k in self.bare + self.squash + self.number: + for k in self.bare + self.number: setattr(self, k, None) def populate(self, values): @@ -94,8 +93,6 @@ class Pkg(object): # ensure we stay under our DB character limit if k in self.bare: setattr(self, k, v[0][:254]) - elif k in self.squash: - setattr(self, k, u', '.join(v)[:254]) elif k in self.number: setattr(self, k, long(v[0])) elif k == 'force': @@ -166,7 +163,6 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.pkgver = repopkg.ver dbpkg.pkgrel = repopkg.rel dbpkg.pkgdesc = repopkg.desc - dbpkg.license = repopkg.license dbpkg.url = repopkg.url dbpkg.filename = repopkg.filename dbpkg.compressed_size = repopkg.csize @@ -208,6 +204,11 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): for y in repopkg.groups: dbpkg.groups.create(name=y) + dbpkg.licenses.all().delete() + if 'license' in repopkg.__dict__: + for y in repopkg.license: + dbpkg.licenses.create(name=y) + def populate_files(dbpkg, repopkg, force=False): if not force: -- cgit v1.2.3-2-g168b From 7f1c7b08227e49172734f09552ceae8bc1f685ad Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Feb 2011 19:31:56 -0600 Subject: Read in optional deps and show in web interface Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 36 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 9b99e0f7..f3ec50e8 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -29,7 +29,7 @@ from optparse import make_option from logging import ERROR, WARNING, INFO, DEBUG -from main.models import Arch, Package, Repo +from main.models import Arch, Package, PackageDepend, Repo logging.basicConfig( level=WARNING, @@ -155,6 +155,20 @@ def find_user(userstring): # lookup more than strictly necessary. find_user.cache = {} +def create_depend(package, dep_str, optional=False): + depend = PackageDepend(pkg=package, optional=optional) + # lop off any description first + parts = dep_str.split(':', 1) + if len(parts) > 1: + depend.description = parts[1].strip() + match = re.match(r"^(.+?)((>=|<=|=|>|<)(.*))?$", parts[0].strip()) + if match: + depend.depname = match.group(1) + if match.group(2): + depend.depvcmp = match.group(2) + depend.save(force_insert=True) + return depend + def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): if repopkg.base: dbpkg.pkgbase = repopkg.base @@ -188,24 +202,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): populate_files(dbpkg, repopkg, force=force) dbpkg.packagedepend_set.all().delete() - if 'depends' in repopkg.__dict__: + if hasattr(repopkg, 'depends'): for y in repopkg.depends: - # make sure we aren't adding self depends.. - # yes *sigh* i have seen them in pkgbuilds - dpname, dpvcmp = re.match(r"([a-z0-9._+-]+)(.*)", y).groups() - if dpname == repopkg.name: - logger.warning('Package %s has a depend on itself', repopkg.name) - continue - dbpkg.packagedepend_set.create(depname=dpname, depvcmp=dpvcmp) - logger.debug('Added %s as dep for pkg %s', dpname, repopkg.name) + dep = create_depend(dbpkg, y) + if hasattr(repopkg, 'optdepends'): + for y in repopkg.optdepends: + dep = create_depend(dbpkg, y, True) dbpkg.groups.all().delete() - if 'groups' in repopkg.__dict__: + if hasattr(repopkg, 'groups'): for y in repopkg.groups: dbpkg.groups.create(name=y) dbpkg.licenses.all().delete() - if 'license' in repopkg.__dict__: + if hasattr(repopkg, 'license'): for y in repopkg.license: dbpkg.licenses.create(name=y) @@ -223,7 +233,7 @@ def populate_files(dbpkg, repopkg, force=False): elif dbpkg.files_last_update > dbpkg.last_update: return # only delete files if we are reading a DB that contains them - if 'files' in repopkg.__dict__: + if hasattr(repopkg, 'files'): dbpkg.packagefile_set.all().delete() logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) -- cgit v1.2.3-2-g168b From 4b12255d1cf52fcc1a98c230d940d0c1d3809ad2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Dec 2010 21:34:40 -0600 Subject: Use new split package file fields everywhere Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index f3ec50e8..72595c63 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -237,8 +237,14 @@ def populate_files(dbpkg, repopkg, force=False): dbpkg.packagefile_set.all().delete() logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) - for x in repopkg.files: - dbpkg.packagefile_set.create(path=x) + for f in repopkg.files: + dirname, filename = f.rsplit('/', 1) + if filename == '': + filename = None + dbpkg.packagefile_set.create( + is_directory=(filename is None), + directory=dirname + '/', + filename=filename) dbpkg.files_last_update = datetime.now() dbpkg.save() -- cgit v1.2.3-2-g168b From f6c41b273c8962718b303c6050c2fd8bcea533a8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Feb 2011 09:46:54 -0600 Subject: reporead performance improvements When importing over a million files, it makes sense to take the slightly faster route and call the PackageFile() constructor directly rather than going through the related manager's create method. We can also get huge performance improvements, especially with files databases, by using the 'io' rather than 'codecs' module. The former is now implemented in C in 2.7 and results in a no-work import (so measuring only the DB read speed) of extra.files.tar.gz from ~30 seconds to ~5 seconds. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 72595c63..bda3bd61 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -27,9 +27,17 @@ import logging from datetime import datetime from optparse import make_option +# New in 2.6, but fast (C implementation) in 2.7. We will use it over codecs if +# available. Eventually remove the codecs import completely. +io = None +try: + import io +except ImportError: + pass + from logging import ERROR, WARNING, INFO, DEBUG -from main.models import Arch, Package, PackageDepend, Repo +from main.models import Arch, Package, PackageDepend, PackageFile, Repo logging.basicConfig( level=WARNING, @@ -241,10 +249,13 @@ def populate_files(dbpkg, repopkg, force=False): dirname, filename = f.rsplit('/', 1) if filename == '': filename = None - dbpkg.packagefile_set.create( + # this is basically like calling dbpkg.packagefile_set.create(), + # but much faster as we can skip a lot of the repeated code paths + pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), directory=dirname + '/', filename=filename) + pkgfile.save() dbpkg.files_last_update = datetime.now() dbpkg.save() @@ -394,7 +405,11 @@ def parse_repo(repopath): if fname not in dbfiles: continue data_file = repodb.extractfile(tarinfo) - data_file = codecs.EncodedFile(data_file, 'utf-8') + if io is None: + data_file = codecs.EncodedFile(data_file, 'utf-8') + else: + data_file = io.TextIOWrapper(io.BytesIO(data_file.read()), + encoding='utf=8') try: data = parse_info(data_file) p = pkgs.setdefault(pkgid, Pkg(reponame)) -- cgit v1.2.3-2-g168b From 02e9c4b8f3f005af2a39a1927dd6d693fa4238f0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 27 Feb 2011 10:52:59 -0600 Subject: reporead: small cleanups --- devel/management/commands/reporead.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index bda3bd61..09e48559 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -391,16 +391,12 @@ def parse_repo(repopath): logger.error("File does not have the proper extension") raise Exception("File does not have the proper extension") - repodb = tarfile.open(repopath,"r") - ## assuming well formed tar, with dir first then files after - ## repo-add enforces this + repodb = tarfile.open(repopath, "r") logger.debug("Starting package parsing") dbfiles = ('desc', 'depends', 'files') pkgs = {} for tarinfo in repodb.getmembers(): - if tarinfo.isdir(): - continue - elif tarinfo.isreg(): + if tarinfo.isreg(): pkgid, fname = os.path.split(tarinfo.name) if fname not in dbfiles: continue -- cgit v1.2.3-2-g168b From 710ec0a9de9a2185621cd7f51cdd2a056e12f999 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Mar 2011 14:43:58 -0600 Subject: Remove devel notify view All of this can just be set on the main profile page. Signed-off-by: Dan McGee --- devel/tests.py | 7 ------- devel/urls.py | 1 - devel/views.py | 9 --------- 3 files changed, 17 deletions(-) (limited to 'devel') diff --git a/devel/tests.py b/devel/tests.py index 682f3d92..da5459d6 100644 --- a/devel/tests.py +++ b/devel/tests.py @@ -10,13 +10,6 @@ class DevelTest(TestCase): self.assertEqual(response['location'], 'http://testserver/login/?next=/devel/') - def test_notify(self): - response = self.client.get('/devel/notify/') - self.assertEqual(response.status_code, 302) - self.assertEqual(response.has_header('Location'), True) - self.assertEqual(response['location'], - 'http://testserver/login/?next=/devel/notify/') - def test_profile(self): response = self.client.get('/devel/profile/') self.assertEqual(response.status_code, 302) diff --git a/devel/urls.py b/devel/urls.py index 0a050a92..bcf9c071 100644 --- a/devel/urls.py +++ b/devel/urls.py @@ -3,7 +3,6 @@ from django.conf.urls.defaults import patterns urlpatterns = patterns('devel.views', (r'^$', 'index'), (r'^clock/$', 'clock'), - (r'^notify/$', 'change_notify'), (r'^profile/$', 'change_profile'), (r'^newuser/$', 'new_user_form'), ) diff --git a/devel/views.py b/devel/views.py index b26c7af0..f89d6870 100644 --- a/devel/views.py +++ b/devel/views.py @@ -65,15 +65,6 @@ def clock(request): return direct_to_template(request, 'devel/clock.html', page_dict) -@login_required -def change_notify(request): - maint = User.objects.get(username=request.user.username) - notify = request.POST.get('notify', 'no') - prof = maint.get_profile() - prof.notify = (notify == 'yes') - prof.save() - return HttpResponseRedirect('/devel/') - class ProfileForm(forms.Form): email = forms.EmailField(label='Private email (not shown publicly):', help_text="Used for out-of-date notifications, etc.") -- cgit v1.2.3-2-g168b From 9d12c0fac5c0580b30c6bf8f578a358dc22afdff Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 4 Mar 2011 12:13:29 -0600 Subject: Move new user email contents to template Signed-off-by: Dan McGee --- devel/views.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index f89d6870..311922ca 100644 --- a/devel/views.py +++ b/devel/views.py @@ -4,6 +4,7 @@ from django.contrib.auth.decorators import login_required, permission_required from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.core.mail import send_mail +from django.template import loader, Context from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template @@ -23,8 +24,8 @@ from string import ascii_letters, digits def index(request): '''the Developer dashboard''' inner_q = PackageRelation.objects.filter(user=request.user).values('pkgbase') - flagged = Package.objects.select_related('arch', 'repo').filter(flag_date__isnull=False) - flagged = flagged.filter(pkgbase__in=inner_q).order_by('pkgname') + flagged = Package.objects.select_related('arch', 'repo').filter( + flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') todopkgs = TodolistPkg.objects.select_related( 'pkg', 'pkg__arch', 'pkg__repo').filter(complete=False) @@ -111,25 +112,34 @@ class NewUserForm(forms.ModelForm): first_name = forms.CharField(required=False) last_name = forms.CharField(required=False) + def clean_username(self): + username = self.cleaned_data['username'] + if User.objects.filter(username=username).exists(): + raise forms.ValidationError( + "A user with that username already exists.") + return username + def save(self): profile = forms.ModelForm.save(self, False) pwletters = ascii_letters + digits - pw = ''.join([random.choice(pwletters) for i in xrange(8)]) + password = ''.join([random.choice(pwletters) for i in xrange(8)]) user = User.objects.create_user(username=self.cleaned_data['username'], - email=self.cleaned_data['email'], password=pw) + email=self.cleaned_data['email'], password=password) user.first_name = self.cleaned_data['first_name'] user.last_name = self.cleaned_data['last_name'] user.save() profile.user = user profile.save() - domain = Site.objects.get_current().domain + + t = loader.get_template('devel/new_account.txt') + c = Context({ + 'site': Site.objects.get_current(), + 'user': user, + 'password': password, + }) send_mail("Your new archweb account", - """You can now log into: -https://%s/login/ -with these login details: -Username: %s -Password: %s""" % (domain, user.username, pw), + t.render(c), 'Arch Website Notification ', [user.email], fail_silently=False) -- cgit v1.2.3-2-g168b