diff options
Diffstat (limited to 'pato2.py')
| -rw-r--r-- | pato2.py | 207 |
1 files changed, 11 insertions, 196 deletions
@@ -27,159 +27,15 @@ from repm.config import * from repm.filter import * import tarfile -from glob import glob from os.path import isdir, isfile -def printf(text,output_=output): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: print (str(text) + "\n") - -def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] - -def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) - -def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) - -def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) - a=run_rsync(cmd) - cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) - b=run_rsync(cmd) - if debug: - printf(a) - printf(b) - -def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] - -def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) - -def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") - -def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - -def cleanup_nonfree_in_dir(directory,blacklisted_names): - pkgs=pkginfo_from_files_in_dir(directory) - for package in pkgs: - if package["name"] in blacklisted_names: - os.remove(package["location"]) - -def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) - -def add_free_repo(verbose_=verbose): - cmd_=os.path.join(home,"/usr/bin/sync-free") - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - for dir_ in other: - for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - - printf(lista_) - - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) - -def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - -def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, - source=mirror+mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. +def generate_rsync_command(base_command, + dir_list=(config["repo_list"] + + config["dir_list"]), + destdir=config["repodir"], + source=config["mirror"] +config["mirrorpath"]): + """ Generates an rsync command for executing + it by combining all parameters. Parameters: ---------- @@ -192,57 +48,16 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi Return: ---------- rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - if not os.path.isdir(destdir): print(destdir + " is not a directory") raise NonValidDir dir_list="{" + ",".join(dir_list) + "}" + return " ".join((base_command, os.path.join(source, dir_list), + destdir)) - if blacklist_file: - return " ".join((base_command, "--exclude-from="+blacklist_file, - os.path.join(source, dir_list), destdir)) - return " ".join((base_command, os.path.join(source, dir_list), destdir)) - -def run_rsync(command,debug=verbose): +def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return commands.getoutput(command) - -if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + return check_output(command.split()) |