1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
|
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
parabola.py
Copyright 2009 Rafik Mas'ad
Copyright 2010 Joshua Ismael Haase Hernández
---------- GNU General Public License 3 ----------
This file is part of Parabola.
Parabola is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Parabola is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Parabola. If not, see <http://www.gnu.org/licenses/>.
"""
from repm.config import *
from repm.filter import *
import tarfile
from glob import glob
from os.path import isdir, isfile
def printf(text,output_=output):
"""Guarda el texto en la variable log y puede imprimir en pantalla."""
log_file = open(logname, 'a')
log_file.write("\n" + str(text) + "\n")
log_file.close()
if output_: print (str(text) + "\n")
def listado(filename_):
"""Obtiene una lista de paquetes de un archivo."""
archivo = open(filename_,"r")
lista = archivo.read().split("\n")
archivo.close()
return [pkg.split(":")[0].rstrip() for pkg in lista if pkg]
def db(repo_,arch_):
"""Construye un nombre para sincronizar una base de datos."""
return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_)
def packages(repo_, arch_, expr="*"):
""" Get packages on a repo, arch folder """
return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) )
def sync_all_repo(debug=verbose):
cmd=generate_rsync_command(rsync_list_command)
rsout=run_rsync(cmd)
pkgs=pkginfo_from_rsync_output(rsout)
generate_exclude_list_from_blacklist(pkgs,listado(blacklist))
cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist)
a=run_rsync(cmd)
if debug: printf(a)
def get_from_desc(desc, var,db_tar_file=False):
""" Get a var from desc file """
desc = desc.split("\n")
return desc[desc.index(var)+1]
def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose):
""" Makes a list of package name, file and license """
info=list()
# Extract DB tar.gz
commands.getoutput("mkdir -p " + archdb)
if not db_tar_file:
db_tar_file = repodir + db(repo_,arch_)
if isfile(db_tar_file):
try:
db_open_tar = tarfile.open(db_tar_file, 'r:gz')
except tarfile.ReadError:
printf("No valid db_file %s" % db_tar_file)
return(tuple())
else:
printf("No db_file %s" % db_tar_file)
return(tuple())
for file in db_open_tar.getmembers():
db_open_tar.extract(file, archdb)
db_open_tar.close()
# Get info from file
for dir_ in glob(archdb + "/*"):
if isdir(dir_) and isfile(dir_ + "/desc"):
pkg_desc_file = open(dir_ + "/desc", "r")
desc = pkg_desc_file.read()
pkg_desc_file.close()
info.append(( get_from_desc(desc,"%NAME%"),
dir_.split("/")[-1],
get_from_desc(desc,"%LICENSE%") ))
if verbose_: printf(info)
commands.getoutput("rm -r %s/*" % archdb)
return tuple(info)
def make_pending(repo_,arch_,info_):
""" Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending"""
search = tuple( listado(blacklist) + listado (whitelist) )
if verbose: printf("blaclist + whitelist= " + str(search) )
lista_=list()
for (name,pkg_,license_) in info_:
if "custom" in license_:
if name not in search:
lista_.append( (name, license_ ) )
elif not name:
printf( pkg_ + " package has no %NAME% attibute " )
if verbose: printf( lista_ )
a=open( pending + "-" + repo_ + ".txt", "w" ).write(
"\n".join([name + ":" + license_ for (name,license_) in lista_]) )
def remove_from_blacklist(repo_,arch_,info_,blacklist_):
""" Check the blacklist and remove packages on the db"""
lista_=list()
pack_=list()
for (name_, pkg_, license_) in info_:
if name_ in blacklist_:
lista_.append(name_)
for p in packages(repo_,arch_,pkg_ + "*"):
pack_.append(p)
if lista_:
lista_=" ".join(lista_)
com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_
printf(com_)
a = commands.getoutput(com_)
if verbose: printf(a)
def link(repo_,arch_,file_):
""" Makes a link in the repo for the package """
cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_
a=commands.getoutput(cmd_)
if verbose:
printf(cmd_ + a)
def add_free_repo(verbose_=verbose):
cmd_=home + "/usr/bin/sync-free"
printf(cmd_)
a=commands.getoutput(cmd_)
if verbose_: printf(a)
for repo_ in repo_list:
for arch_ in arch_list:
lista_=list()
for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"):
lista_.append(file_)
for dir_ in other:
for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"):
lista_.append(file_)
printf(lista_)
if lista_:
lista_=" ".join(lista_)
if verbose: printf(lista_)
cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_
printf(cmd_)
a=commands.getoutput(cmd_)
if verbose: printf(a)
def get_licenses(verbose_=verbose):
""" Extract the license from packages in repo_,arch_ and in pending_ file"""
cmd_=home + "/usr/bin/get_license.sh"
printf(cmd_)
a=commands.getoutput(cmd_)
if verbose_: printf(a)
def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir,
source=mirror+mirrorpath, blacklist_file=False):
""" Generates an rsync command for executing it by combining all parameters.
Parameters:
----------
base_command -> str
dir_list -> list or tuple
destdir -> str Path to dir, dir must exist.
source -> str The source for rsync
blacklist_file -> False or str Path to file, file must exist.
Return:
----------
rsync_command -> str """
from os.path import isfile, isdir
if blacklist_file and not isfile(blacklist_file):
print(blacklist_file + " is not a file")
raise NonValidFile
if not os.path.isdir(destdir):
print(destdir + " is not a directory")
raise NonValidDir
dir_list="{" + ",".join(dir_list) + "}"
if blacklist_file:
return " ".join((base_command, "--exclude-from="+blacklist_file,
os.path.join(source, dir_list), destdir))
return " ".join((base_command, os.path.join(source, dir_list), destdir))
def run_rsync(command,debug=verbose):
""" Runs rsync and gets returns it's output """
if debug:
printf("rsync_command: " + command)
return commands.getoutput(command)
if __name__ == "__main__":
from time import time
start_time = time()
def minute():
return str(round((time() - start_time)/60, 1))
printf(" Cleaning %s folder " % (tmp) )
commands.getoutput("rm -r %s/*" % tmp)
printf(" Syncing repo")
sync_all_repo(True)
printf(" Updating databases and pending files lists: minute %s \n" % minute() )
for repo in repo_list:
for arch in arch_list:
printf( "\n" + repo + "-" + arch + "\n" )
printf( "Get info: minute %s " % minute() )
info=get_info(repo,arch)
printf( "Make pending: minute %s" % minute() )
make_pending(repo,arch,info)
printf( "Update DB: minute %s" % minute() )
remove_from_blacklist(
repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) )
printf("Adding Parabola Packages: minute %s\n" % minute() )
add_free_repo(True)
printf("Extracting licenses in pending: minute %s" % minute() )
get_licenses()
printf("\n\nDelay: %s minutes \n" % minute())
|
