From 9c8b6615d537a9c7bb483457cd8a300d7f9c597e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:55:33 -0600 Subject: * Added filter.py * Added __init__.py for test module * Removed unused files --- filter.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 filter.py (limited to 'filter.py') diff --git a/filter.py b/filter.py new file mode 100644 index 0000000..5229e63 --- /dev/null +++ b/filter.py @@ -0,0 +1,90 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +import commands +import os +import re +from repm.config import * +from repm.pato2 import * + +rsync_list_command="rsync -av --no-motd --list-only " + +def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, + mirror_path=mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + mirror_name -> str + mirror_path -> str + dir_list -> list or tuple + destdir -> str Must be a dir + blacklist_file -> False or str File must exist + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + +def get_file_list_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. + + Parameters: + ---------- + rsync_output -> str containing output from rsync + + Returns: + ---------- + package_list -> tuple of Package objects. """ + a=list() + + def directory(line): + pass + + def package_or_link(line): + """ Take info out of filename """ + location_field = 4 + pkg = Package() + pkg["location"] = line.rsplit()[location_field] + fileattrs = pkg["location"].split("/")[-1].split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + options = { "d": directory, + "l": package_or_link, + "-": package_or_link} + + for line in rsync_output.split("\n"): + if ".pkg.tar.gz" or ".pkg.tar.xz" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) + + return tuple(a) + -- cgit v1.2.3-2-g168b From 2cae61ff561561e405f0cd0c150dbcd77ce25b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 13:26:54 -0600 Subject: config py: * Make Package class init, restrict keys. filter.py * Corrected function * Added generate_rsync_exclude --- filter.py | 53 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 9 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 5229e63..817a228 100644 --- a/filter.py +++ b/filter.py @@ -6,7 +6,7 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -av --no-motd --list-only " +rsync_list_command="rsync -a --no-motd --list-only " def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, mirror_path=mirrorpath, blacklist_file=False): @@ -18,8 +18,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name= mirror_name -> str mirror_path -> str dir_list -> list or tuple - destdir -> str Must be a dir - blacklist_file -> False or str File must exist + destdir -> str Path to dir, dir must exist. + blacklist_file -> False or str Path to file, file must exist. Return: ---------- @@ -54,11 +54,11 @@ def get_file_list_from_rsync_output(rsync_output): Parameters: ---------- - rsync_output -> str containing output from rsync + rsync_output -> str Contains output from rsync Returns: ---------- - package_list -> tuple of Package objects. """ + package_list -> tuple Contains Package objects. """ a=list() def directory(line): @@ -81,10 +81,45 @@ def get_file_list_from_rsync_output(rsync_output): "-": package_or_link} for line in rsync_output.split("\n"): - if ".pkg.tar.gz" or ".pkg.tar.xz" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + if ".pkg.tar" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) return tuple(a) +def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, + blacklist_file=rsync_blacklist, debug=verbose): + """ Generate an exclude list for rsync + + Parameters: + ---------- + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + blacklist_file -> str Path to file + debug -> bool + + Output: + ---------- + if debug == False -> None + if debug == True -> blacklist """ + a=list() + + for package in packages_iterable: + if not isinstance(package, Package): + raise ValueError(" %s is not a Package object " % package) + if package["name"] in blacklisted_names: + a.append(package["location"]) + + if debug: + printf(a) + + try: + fsock = open(blacklist_file,"w") + try: + fsock.write("\n".join(a)) + finally: + fsock.close() + except IOError: + printf("%s wasnt written" % blacklist_file) + -- cgit v1.2.3-2-g168b From 072787627a2339c3d5aca541086c2e4545bbad8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 14:06:58 -0600 Subject: * Changed variable name in filter.py * Added example in test1.py --- filter.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 817a228..ffc0965 100644 --- a/filter.py +++ b/filter.py @@ -89,20 +89,19 @@ def get_file_list_from_rsync_output(rsync_output): return tuple(a) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - blacklist_file=rsync_blacklist, debug=verbose): + exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync Parameters: ---------- - package_iterable -> list or tuple Contains Package objects - blacklisted_names-> list or tuple Contains blacklisted names - blacklist_file -> str Path to file - debug -> bool + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + exclude_file -> str Path to file + debug -> bool If True, file list gets logged Output: ---------- - if debug == False -> None - if debug == True -> blacklist """ + None """ a=list() for package in packages_iterable: @@ -115,7 +114,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, printf(a) try: - fsock = open(blacklist_file,"w") + fsock = open(exclude_file,"w") try: fsock.write("\n".join(a)) finally: -- cgit v1.2.3-2-g168b From 2321a53d35a9736b8c5c715ca40a9af69b1bf64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:14:47 -0600 Subject: Rearranged functions --- filter.py | 43 ------------------------------------------- 1 file changed, 43 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index ffc0965..b51dba8 100644 --- a/filter.py +++ b/filter.py @@ -6,49 +6,6 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -a --no-motd --list-only " - -def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, - mirror_path=mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. - - Parameters: - ---------- - base_command -> str - mirror_name -> str - mirror_path -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - - if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) - -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): - """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) - if debug: - printf("rsync_command" + cmd) - return commands.getoutput(cmd) - def get_file_list_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. -- cgit v1.2.3-2-g168b From 6203dc2fc926781db694ca2383b1e44e2c5469c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:29:33 -0600 Subject: - filter.py: * Makes a blacklist for rsync - pato2.py: * Has more functions --- filter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index b51dba8..14ae31e 100644 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ -#! /usr/bin/python + #! /usr/bin/python #-*- encoding: utf-8 -*- import commands import os @@ -79,3 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) +if name == "__main__": + a=run_rsync(rsync_list_command) + packages=get_file_list_from_rsync_output(a) + generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3-2-g168b From b27cbc08447fe5605bf877ee0991888d5ecf6382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:32:48 -0600 Subject: Corrected filter.py to run --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 14ae31e..c789cd5 100644 --- a/filter.py +++ b/filter.py @@ -79,7 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) -if name == "__main__": +if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=get_file_list_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3-2-g168b From cbe877f2ba25b398ad32b92d22dc6f5a108ef59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 02:28:01 -0600 Subject: * Changed get_file_list_ for pkginfo in function names * Added pkginfo_from_... funcions --- filter.py | 57 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 19 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index c789cd5..232319f 100644 --- a/filter.py +++ b/filter.py @@ -1,13 +1,34 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- import commands -import os import re from repm.config import * from repm.pato2 import * -def get_file_list_from_rsync_output(rsync_output): - """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. +def pkginfo_from_filename(filename): + """ Generates a Package object with info from a filename, + filename can be relative or absolute + + Parameters: + ---------- + filename -> str + + Returns: + ---------- + pkg -> Package object""" + pkg = Package() + pkg["location"] = filename + fileattrs = os.path.basename(filename).split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + +def pkginfo_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output + wich uses --list-only and --no-motd options. Parameters: ---------- @@ -18,33 +39,30 @@ def get_file_list_from_rsync_output(rsync_output): package_list -> tuple Contains Package objects. """ a=list() - def directory(line): - pass - def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkg = Package() - pkg["location"] = line.rsplit()[location_field] - fileattrs = pkg["location"].split("/")[-1].split("-") - pkg["arch"] = fileattrs.pop(-1).split(".")[0] - pkg["release"] = fileattrs.pop(-1) - pkg["version"] = fileattrs.pop(-1) - pkg["name"] = "-".join(fileattrs) - return pkg - + pkginfo_from_filename(line.rsplit()[location_field]) + + def directory(line): + pass + options = { "d": directory, "l": package_or_link, "-": package_or_link} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + pkginfo_=options[line[0]](line) + if pkginfo_: + a.append(pkginfo_) return tuple(a) +def pkginfo_from_files_in_dir(directory): + """ Returns pkginfo from filenames of packages in dir + wich has .pkg.tar.{xz,gz} on them """ + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync @@ -78,8 +96,9 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, fsock.close() except IOError: printf("%s wasnt written" % blacklist_file) + if __name__ == "__main__": a=run_rsync(rsync_list_command) - packages=get_file_list_from_rsync_output(a) + packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3-2-g168b From 4e7508d9a61c3653b46da8fa513513756bb3b6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:46:33 -0600 Subject: * pkginfo functions --- filter.py | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 232319f..ad3b33d 100644 --- a/filter.py +++ b/filter.py @@ -1,7 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import commands -import re +import glob from repm.config import * from repm.pato2 import * @@ -11,11 +10,13 @@ def pkginfo_from_filename(filename): Parameters: ---------- - filename -> str + filename -> str Must contain .pkg.tar. Returns: ---------- pkg -> Package object""" + if ".pkg.tar." not in filename: + raise NonValidFile pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -37,31 +38,49 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - a=list() + package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 pkginfo_from_filename(line.rsplit()[location_field]) - def directory(line): + def do_nothing(): pass - - options = { "d": directory, + + options = { "d": do_nothing, "l": package_or_link, - "-": package_or_link} + "-": package_or_link, + " ": do_nothing} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo_=options[line[0]](line) + pkginfo=options[line[0]](line) if pkginfo_: - a.append(pkginfo_) + package_list.append(pkginfo) - return tuple(a) + return tuple(package_list) def pkginfo_from_files_in_dir(directory): """ Returns pkginfo from filenames of packages in dir - wich has .pkg.tar.{xz,gz} on them """ + wich has .pkg.tar. on them + + Parameters: + ---------- + directory -> str Directory must exist + + Returns: + ---------- + package_list -> tuple Contains Package objects """ + package_list=list() + + if not os.path.isdir(directory): + raise NonValidDir + + for filename in glob(os.path.join(directory,"*")): + if ".pkg.tar." in filename: + package_list.append(pkginfo_from_filename(filename)) + return tuple(package_list) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): @@ -97,7 +116,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) - if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=pkginfo_from_rsync_output(a) -- cgit v1.2.3-2-g168b From f923ab304017a71136642ed7b9780441edcaf441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 19:36:26 -0600 Subject: * TestCase for pkginfo_from_rsync_output * Corrected __eq__ method in Package class * Corrected pkginfo_from_rsync_output --- filter.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index ad3b33d..55ab94a 100644 --- a/filter.py +++ b/filter.py @@ -38,26 +38,28 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkginfo_from_filename(line.rsplit()[location_field]) + return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - pass + """""" options = { "d": do_nothing, "l": package_or_link, "-": package_or_link, " ": do_nothing} + + package_list=list() - for line in rsync_output.split("\n"): - if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo_: - package_list.append(pkginfo) + lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] + + for line in lines: + pkginfo=options[line[0]](line) + if pkginfo: + package_list.append(pkginfo) return tuple(package_list) -- cgit v1.2.3-2-g168b From 3866f148dae1f6c90fc6d903784b65e452c048c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 22:45:11 -0600 Subject: * Added debug action to generate_exclude_list_from_blacklist * Added test data to test_pkginfo_from_rsync_output.py --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 55ab94a..f8182e3 100644 --- a/filter.py +++ b/filter.py @@ -108,7 +108,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, if debug: printf(a) - + return a try: fsock = open(exclude_file,"w") try: -- cgit v1.2.3-2-g168b From 9de298a7b88f7f36aec4c9344356a935c67cfeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 23:27:46 -0600 Subject: * Added test for generate_exclude_list_from_blacklist * Fixed listado to strip spaces --- filter.py | 1 - 1 file changed, 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index f8182e3..f91ec68 100644 --- a/filter.py +++ b/filter.py @@ -107,7 +107,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, a.append(package["location"]) if debug: - printf(a) return a try: fsock = open(exclude_file,"w") -- cgit v1.2.3-2-g168b From 5a4480f1f3e2e7d0aec965b21f72f4454618333e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 5 Mar 2011 13:13:41 -0600 Subject: =?UTF-8?q?*=20Added=20=C2=ABdepends=C2=BB=20field=20to=20Package?= =?UTF-8?q?=20class.=20*=20Merged=20tests=20for=20filter=20functions=20in?= =?UTF-8?q?=20test=5Ffilter.py=20*=20Added=20testfiles=20for=20test=5Ffilt?= =?UTF-8?q?er.py=20in=20test=20directory=20*=20Added=20pkginfo=5Ffrom=5Fde?= =?UTF-8?q?sc=20function=20to=20filter.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filter.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index f91ec68..2fe61f0 100644 --- a/filter.py +++ b/filter.py @@ -26,6 +26,37 @@ def pkginfo_from_filename(filename): pkg["name"] = "-".join(fileattrs) return pkg +def pkginfo_from_desc(filename): + """ Returns pkginfo from desc file. + + Parameters: + ---------- + filename -> str File must exist + + Returns: + ---------- + pkg -> Package object""" + if not os.path.isfile(filename): + raise NonValidFile + try: + f=open(filename) + info=f.read().rsplit() + finally: + f.close() + pkg = Package() + info_map={"name" :("%NAME%" , None), + "version" :("%VERSION%" , 0 ), + "release" :("%VERSION%" , 1 ), + "arch" :("%ARCH%" , None), + "license" :("%LICENSE%" , None), + "location":("%FILENAME%", None),} + + for key in info_map.keys(): + field,pos=info_map[key] + pkg[key]=info[info.index(field)+1] + if pos is not None: + pkg[key]=pkg[key].split("-")[pos] + return pkg def pkginfo_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output @@ -45,7 +76,7 @@ def pkginfo_from_rsync_output(rsync_output): return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - """""" + pass options = { "d": do_nothing, "l": package_or_link, @@ -84,6 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) + + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync -- cgit v1.2.3-2-g168b From 15ed2078e8743f87efcd63ac62cc0bfd5b39dc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:34:28 -0600 Subject: * Corrected import glob function --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 2fe61f0..b6e8105 100644 --- a/filter.py +++ b/filter.py @@ -1,6 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import glob +from glob import glob from repm.config import * from repm.pato2 import * -- cgit v1.2.3-2-g168b From 47045b1934932b0695dd301a9c76b9dab1b03023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 21 Mar 2011 17:30:40 -0600 Subject: * Changed sync_all_repo to adress bug83 --- filter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index b6e8105..668822b 100644 --- a/filter.py +++ b/filter.py @@ -115,7 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) - +def pkginfo_from_db(path_to_db): + """ """ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): -- cgit v1.2.3-2-g168b From 3fb5f8efef231dd7784be880934cd106603ab6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 01:09:25 -0500 Subject: bash-port ready for testing. --- filter.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 13 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 668822b..1a0fa6f 100644 --- a/filter.py +++ b/filter.py @@ -4,6 +4,13 @@ from glob import glob from repm.config import * from repm.pato2 import * +def listado(filename): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, filename can be relative or absolute @@ -116,10 +123,48 @@ def pkginfo_from_files_in_dir(directory): return tuple(package_list) def pkginfo_from_db(path_to_db): - """ """ + """ Get PKGINFO from db. + + Parameters: + ---------- + path_to_db -> str Path to file -def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=rsync_blacklist, debug=verbose): + Output: + ---------- + None """ + package_list=list() + + if not os.path.isfile(path_to_db): + raise NonValidFile(path_to_db + "is not a file") + + check_output("mkdir -p " + archdb) + + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s or not readable" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + package_list.append(pkginfo_from_desc( + os.path.join(dir_,"desc"))) + check_output("rm -r %s/*" % archdb) + if verbose_: + printf(package_list) + return package_list + +def generate_exclude_list_from_blacklist(packages_iterable, + blacklisted_names, + exclude_file=config["rsync_blacklist"], + debug=config["debug"]): """ Generate an exclude list for rsync Parameters: @@ -132,16 +177,12 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, Output: ---------- None """ - a=list() - - for package in packages_iterable: - if not isinstance(package, Package): - raise ValueError(" %s is not a Package object " % package) - if package["name"] in blacklisted_names: - a.append(package["location"]) + pkgs=[pkg["location"] for pkg in packages_iterable + if isinstance(pkg, Package) + and pkg["name"] in blacklisted_names] if debug: - return a + return pkgs try: fsock = open(exclude_file,"w") try: @@ -149,9 +190,10 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, finally: fsock.close() except IOError: - printf("%s wasnt written" % blacklist_file) + printf("%s wasnt written" % exclude_file) if __name__ == "__main__": - a=run_rsync(rsync_list_command) + cmd=generate_rsync_command(rsync_list_command) + a=run_rsync(cmd) packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3-2-g168b From f6184ee3c9422c40effb0265527ee5a31b027a1a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Sun, 10 Apr 2011 23:44:01 -0700 Subject: chmod +x --- filter.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 filter.py (limited to 'filter.py') diff --git a/filter.py b/filter.py old mode 100644 new mode 100755 -- cgit v1.2.3-2-g168b From 67038ba1840d0f57b0ce49fdabd3dfa8057e2451 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 11 Apr 2011 00:22:06 -0700 Subject: fixed some errors --- filter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 1a0fa6f..78ad410 100755 --- a/filter.py +++ b/filter.py @@ -137,15 +137,15 @@ def pkginfo_from_db(path_to_db): if not os.path.isfile(path_to_db): raise NonValidFile(path_to_db + "is not a file") - check_output("mkdir -p " + archdb) + check_output("mkdir -p " + config["archdb"]) try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % db_tar_file) + printf("No valid db_file %s or not readable" % path_to_db) return(tuple()) else: - printf("No db_file %s" % db_tar_file) + printf("No db_file %s" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): -- cgit v1.2.3-2-g168b From 3e27d11f68571bce92138f6cbfcaecac75fa1644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 12:39:40 -0500 Subject: Fixed some errors --- filter.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 78ad410..48e2d93 100755 --- a/filter.py +++ b/filter.py @@ -4,12 +4,17 @@ from glob import glob from repm.config import * from repm.pato2 import * -def listado(filename): +def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + fsock = open(filename,"r") + lista = fsock.read().split("\n") + fsock.close() + if end is not None: + return [pkg.split(":")[start:end].rstrip() + for pkg in lista if pkg] + else: + return [pkg.split(":")[start].rstrip() + for pkg in lista if pkg] def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, @@ -135,29 +140,26 @@ def pkginfo_from_db(path_to_db): package_list=list() if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + "is not a file") + raise NonValidFile(path_to_db + " is not a file") - check_output("mkdir -p " + config["archdb"]) + check_output(["mkdir", "-p", config["archdb"]]) try: db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % path_to_db) - return(tuple()) - else: - printf("No db_file %s" % path_to_db) + raise NonValidFile("No valid db_file %s or not readable" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) + db_open_tar.extract(file, config["archdb"]) db_open_tar.close() # Get info from file - for dir_ in glob(archdb + "/*"): + for dir_ in glob(config["archdb"] + "/*"): if isdir(dir_) and isfile(dir_ + "/desc"): package_list.append(pkginfo_from_desc( os.path.join(dir_,"desc"))) - check_output("rm -r %s/*" % archdb) - if verbose_: + check_output(["rm", "-r", config["archdb"]]) + if config["debug"]: printf(package_list) return package_list -- cgit v1.2.3-2-g168b From ddacb9a98b90fc2b51551fb73ab8513634f4f185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 01:01:57 -0500 Subject: filter.py to close correct file --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 1d70a63..c7f3f18 100755 --- a/filter.py +++ b/filter.py @@ -147,7 +147,7 @@ def pkginfo_from_db(path_to_db): % path_to_db) return(tuple()) finally: - db_open_tar.close() + dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, -- cgit v1.2.3-2-g168b From 062b7a2d9b0347e4e867e588826f052d26a3bb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 01:11:20 -0500 Subject: For testing --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index c7f3f18..4b9603d 100755 --- a/filter.py +++ b/filter.py @@ -137,7 +137,7 @@ def pkginfo_from_db(path_to_db): try: dbsock = tarfile.open(path_to_db, 'r:gz') - desc_files=[desc for desc in db_open_tar.getnames() + desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: desc=dbsock.extractfile(name) -- cgit v1.2.3-2-g168b From deab65fad4ced009fb31f7033b1db8ef0af78aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 15 Apr 2011 04:53:27 -0500 Subject: Python parts ready for bash usage in python 2 and 3 --- filter.py | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 4b9603d..add0235 100755 --- a/filter.py +++ b/filter.py @@ -2,7 +2,7 @@ #-*- encoding: utf-8 -*- from glob import glob from repm.config import * -from repm.pato2 import * +import tarfile def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" @@ -28,7 +28,7 @@ def pkginfo_from_filename(filename): ---------- pkg -> Package object""" if ".pkg.tar." not in filename: - raise NonValidFile + raise NonValidFile("File is not a pacman package") pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -140,19 +140,18 @@ def pkginfo_from_db(path_to_db): desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: - desc=dbsock.extractfile(name) - package_list.append(pkginfo_from_desc(desc.read())) + desc=dbsock.extractfile(name).read().decode("UTF-8") + package_list.append(pkginfo_from_desc(desc)) except tarfile.ReadError: raise NonValidFile("No valid db_file %s or not readable" % path_to_db) - return(tuple()) finally: dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=config["rsync_blacklist"]): + exclude_file): """ Generate an exclude list for rsync Parameters: @@ -168,20 +167,31 @@ def rsyncBlacklist_from_blacklist(packages_iterable, pkgs=[pkg["location"] for pkg in packages_iterable if isinstance(pkg, Package) and pkg["name"] in blacklisted_names] - - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() + if exclude_file: + try: + fsock = open(exclude_file,"w") + fsock.write("\n".join(pkgs) + "\n") + except IOError: + printf("%s wasnt written" % exclude_file) + exit(1) + finally: + fsock.close() return pkgs if __name__ == "__main__": - cmd=generate_rsync_command(rsync_list_command) - a=run_rsync(cmd) - packages=pkginfo_from_rsync_output(a) - rsyncBlaclist_from_blacklist(packages,listado(blacklist)) + import argparse + parser=argparse.ArgumentParser() + parser.add_argument("-r", "--rsync-exclude-file", type=str, + help="File in which to generate exclude list", + required=True,) + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + parser.add_argument("-c", "--rsync-command", type=str, + help="This command will be run to get a pkg list") + args=parser.parse_args() + rsout=check_output(args.rsync_command) + packages=pkginfo_from_rsync_output(rsout) + rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + args.rsync_exclude_file) -- cgit v1.2.3-2-g168b From 374c8e2a5183cdbaefe9f54184603ad8d09e30c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 03:15:01 -0500 Subject: * bash-repo for testing * merged dbscripts with repm --- filter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index add0235..4abdf38 100755 --- a/filter.py +++ b/filter.py @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) -- cgit v1.2.3-2-g168b From e0aad034cd3b7e91137f85584f9b53cdc871f44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:42:00 -0500 Subject: Modified local_config.example --- filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'filter.py') diff --git a/filter.py b/filter.py index 4abdf38..c71f54d 100755 --- a/filter.py +++ b/filter.py @@ -4,9 +4,9 @@ from glob import glob from repm.config import * import tarfile -def listado(filename,start=0,end=None): +def listado(filename, start=0, end=None): """Obtiene una lista de paquetes de un archivo.""" - fsock = open(filename,"r") + fsock = open(filename, "r") lista = fsock.read().split("\n") fsock.close() if end is not None: -- cgit v1.2.3-2-g168b