[Piuparts-commits] [SCM] piuparts git repository branch, develop, updated. 9bcbfe0fa48aba1da806fecfea79e3aa2d7bdb27
Holger Levsen
holger at layer-acht.org
Fri Dec 2 13:30:44 UTC 2011
The following commit has been merged in the develop branch:
commit 9bcbfe0fa48aba1da806fecfea79e3aa2d7bdb27
Merge: 241c05545defefcf0ad6e0a34684bc6f5dfc0897 c27ed5ec8dbed50bb1b50010fd5c6e9bac1e74f8
Author: Holger Levsen <holger at layer-acht.org>
Date: Fri Dec 2 14:29:58 2011 +0100
Merge branch 'feature/report' into develop
diff --combined piuparts-report.py
index 217c37e,bc89546..0cb3d6c
--- a/piuparts-report.py
+++ b/piuparts-report.py
@@@ -445,7 -445,6 +445,7 @@@ def emphasize_reason(reason)
"dependency-cannot-be-tested",
"dependency-does-not-exist",
"circular-dependency",
+ "does-not-exist",
"unknown",
"unknown-preferred-alternative",
"no-dependency-from-alternatives-exists",
@@@ -662,7 -661,7 +662,7 @@@ class Section
return link
def link_to_state_page(self, section, package_name, link_target):
- state = self._binary_db.state_by_name(package_name)
+ state = self._binary_db.get_package_state(package_name)
if state != "unknown":
link = "<a href=\"/%s/%s\">%s</a>" % (
section,
@@@ -726,30 -725,33 +726,30 @@@
return total
- def merge_maintainer_templates(self, templates):
- for maint_tpl in templates:
- tpl = os.path.join(self._output_directory,"maintainer",maintainer_subdir(maint_tpl),maint_tpl)
- lines = read_file(tpl)
+ def create_maintainer_summaries(self, maintainers, source_data):
+ logging.debug("Writing maintainer summaries in %s" % self._output_directory)
+ states = ["fail", "unknown", "pass"]
+ for maintainer in maintainers.keys():
+ sources = maintainers[maintainer]
rows = ""
- for line in lines:
- state, count, packages = line.split(",")
- if packages == "none\n":
- links = " "
- else:
- links = ""
- for package in packages.split(" "):
- links += "<a href=\"#%s\">%s</a> " % (package,package)
- rows += "<tr class=\"normalrow\"><td class=\"labelcell\">%s:</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"4\">%s</td></tr>" % \
- (state, count, links)
- os.unlink(tpl)
- template_path = tpl[:-len("_tpl")]
-
- for state in ("fail","unkn","pass"):
- filename = template_path+"_"+state
- if os.path.isfile(filename):
- f = file(filename, "r")
- rows += file.read(f)
- f.close()
- os.unlink(filename)
-
- maintainer = os.path.basename(maint_tpl[:-len("_tpl")])
+ package_rows = ""
+ packages = {}
+ for state in states:
+ packages[state] = []
+ for source in sorted(sources):
+ (state, sourcerows, binaryrows) = source_data[source]
+ packages[state].append(source)
+ package_rows += sourcerows + binaryrows
+
+ for state in states:
+ if len(packages[state]) > 0:
+ links = ""
+ for package in packages[state]:
+ links += "<a href=\"#%s\">%s</a> " % (package, package)
+ else:
+ links = " "
+ rows += "<tr class=\"normalrow\"><td class=\"labelcell\">%s:</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"4\">%s</td></tr>" % \
+ (state, len(packages[state]), links)
distrolinks = "<tr class=\"normalrow\"><td class=\"labelcell\">other distributions: </td><td class=\"contentcell2\" colspan=\"5\">"
for section in self._section_names:
@@@ -758,7 -760,7 +758,7 @@@
distrolinks += "</td></tr>"
htmlpage = string.Template(HTML_HEADER + MAINTAINER_BODY_TEMPLATE + HTML_FOOTER)
- filename = template_path+".html"
+ filename = os.path.join(self._output_directory, "maintainer", maintainer_subdir(maintainer), maintainer + ".html")
f = file(filename, "w")
f.write(htmlpage.safe_substitute( {
"page_title": html_protect("Status of "+maintainer+" packages in "+self._config.section),
@@@ -766,11 -768,10 +766,11 @@@
"distrolinks": distrolinks,
"section_navigation": create_section_navigation(self._section_names,self._config.section),
"time": time.strftime("%Y-%m-%d %H:%M %Z"),
- "rows": rows,
+ "rows": rows + package_rows,
}))
f.close()
+
def create_source_summary (self, source, logs_by_dir):
source_version = self._source_db.get_control_header(source, "Version")
binaries = self._source_db.get_control_header(source, "Binary")
@@@ -781,7 -782,7 +781,7 @@@
failed = False
binaryrows = ""
for binary in sorted(binaries.split(", ")):
- state = self._binary_db.state_by_name(binary)
+ state = self._binary_db.get_package_state(binary)
if state == "unknown":
# Don't track udebs and binary packages on other archs.
# The latter is a FIXME which needs parsing the Packages files from other archs too
@@@ -833,30 -834,61 +833,30 @@@
return sourcerows, binaryrows, source_state, maintainer, uploaders
- def create_maintainer_templates_for_source(self,source, source_state, sourcerows, binaryrows, maintainer, uploaders):
- maintainer_pages = []
- maintainer_pages.append(get_email_address(maintainer))
- for uploader in uploaders.split(", "):
- if uploader:
- maintainer_pages.append(get_email_address(uploader))
- for maintainer_page in maintainer_pages:
- maintainer_summary_page_path = os.path.join(self._output_directory, "maintainer", maintainer_subdir(maintainer_page))
-
- if not os.path.exists(maintainer_summary_page_path):
- os.makedirs(maintainer_summary_page_path)
- filename = os.path.join(maintainer_summary_page_path, (maintainer_page + "_tpl"))
- maintainer_package_count = {}
- maintainer_packages = {}
- if os.path.isfile(filename):
- lines = read_file(filename)
- for line in lines:
- state, count, packages = line.split(",")
- maintainer_package_count[state]=int(count)
- maintainer_packages[state]=packages[:-1]
- if maintainer_packages[source_state] == "none":
- maintainer_packages[source_state] = source
- else:
- maintainer_packages[source_state] = "%s %s" % (maintainer_packages[source_state],source)
- else:
- maintainer_package_count["fail"] = 0
- maintainer_package_count["unknown"] = 0
- maintainer_package_count["pass"] = 0
- for state in "fail", "unknown", "pass":
- maintainer_packages[state] = "none"
- maintainer_packages[source_state] = source
- if source_state == "fail":
- maintainer_package_count["fail"]+=1
- elif source_state == "unknown":
- maintainer_package_count["unknown"]+=1
- else:
- maintainer_package_count["pass"]+=1
- lines = ""
- for state in "fail", "unknown", "pass":
- lines += "%s,%s,%s\n" % (state,maintainer_package_count[state],maintainer_packages[state])
- write_file(filename,lines)
- append_file(filename[:-4]+"_"+source_state[:4],sourcerows+binaryrows)
def create_package_summaries(self, logs_by_dir):
logging.debug("Writing package templates in %s" % self._config.section)
+ maintainers = {}
+ source_binary_rows = {}
sources = ""
for source in self._source_db.get_all_packages():
(sourcerows, binaryrows, source_state, maintainer, uploaders) = self.create_source_summary(source, logs_by_dir)
if source_state != "udeb":
- sources += "%s: %s\n" % (source, source_state)
- self.create_maintainer_templates_for_source(source, source_state, sourcerows, binaryrows, maintainer, uploaders)
+ sources += "%s: %s\n" % (source, source_state)
+ source_binary_rows[source] = (source_state, sourcerows, binaryrows)
+ for maint in [maintainer] + uploaders.split(","):
+ if maint.strip():
+ email = get_email_address(maint.strip())
+ if not "INVALID" in email:
+ if not email in maintainers:
+ maintainers[email] = []
+ maintainers[email].append(source)
write_file(os.path.join(self._output_directory, "sources.txt"), sources)
+ self.create_maintainer_summaries(maintainers, source_binary_rows)
+
def make_stats_graph(self):
countsfile = os.path.join(self._output_directory, "counts.txt")
@@@ -969,12 -1001,21 +969,21 @@@
package["Package"],
self.link_to_source_summary(package["Package"]),
html_protect(package["Maintainer"]))
- if package.dependencies():
+ all_deps = package.all_dependencies()
+ if all_deps:
vlist += "\n<ul>\n"
- for dep in package.dependencies():
+ for alternatives in all_deps:
+ dep = alternatives[0]
vlist += "<li>dependency %s is %s</li>\n" % \
(self.link_to_state_page(self._config.section,dep,dep),
- emphasize_reason(html_protect(self._binary_db.state_by_name(dep))))
+ emphasize_reason(html_protect(self._binary_db.get_package_state(dep))))
+ if len(alternatives) > 1:
+ vlist += "\n<ul>\n"
+ for dep in alternatives[1:]:
+ vlist += "<li>alternative dependency %s is %s</li>\n" % \
+ (self.link_to_state_page(self._config.section,dep,dep),
+ emphasize_reason(html_protect(self._binary_db.state_by_name(dep))))
+ vlist += "</ul>\n"
vlist += "</ul>\n"
vlist += "</li>\n"
htmlpage = string.Template(HTML_HEADER + STATE_BODY_TEMPLATE + HTML_FOOTER)
@@@ -987,25 -1028,6 +996,25 @@@
"list": vlist
}))
+
+ def archive_logfile(self, vdir, log):
+ archivedir = os.path.join("archive", vdir)
+ if not os.path.exists(archivedir):
+ os.makedirs(archivedir)
+ os.rename(os.path.join(vdir, log), os.path.join("archive", vdir, log))
+
+
+ def cleanup_removed_packages(self, logs_by_dir):
+ for vdir in logs_by_dir.keys():
+ for log in sorted(logs_by_dir[vdir]):
+ if log.endswith(".log"):
+ package, version = log[:-len(".log")].split("_")
+ if not self._binary_db.has_package(package):
+ logging.debug("Package %s was removed, archiving %s/%s" % (package, vdir, log))
+ self.archive_logfile(vdir, log)
+ logs_by_dir[vdir].remove(log)
+
+
def generate_html(self):
logging.debug("Finding log files")
dirs = ["pass", "fail", "bugged", "reserved", "untestable"]
@@@ -1013,9 -1035,6 +1022,9 @@@
for vdir in dirs:
logs_by_dir[vdir] = find_files_with_suffix(vdir, ".log")
+ logging.debug("Archiving logs of obsolete packages")
+ self.cleanup_removed_packages(logs_by_dir)
+
logging.debug("Copying log files")
copy_logs(logs_by_dir, self._output_directory)
@@@ -1030,6 -1049,9 +1039,6 @@@
if self._config["sources-url"]:
self.create_package_summaries(logs_by_dir)
- logging.debug("Merging maintainer summaries in %s" % self._output_directory)
- self.merge_maintainer_templates(find_files_with_suffix(self._output_directory+"/maintainer/", "_tpl"))
-
logging.debug("Writing section index page")
self.write_section_index_page(dirs, total_packages)
@@@ -1053,7 -1075,6 +1062,7 @@@
self.generate_html()
os.chdir(oldcwd)
+
def main():
setup_logging(logging.DEBUG, None)
diff --combined piupartslib/packagesdb.py
index 175d547,dcd2b1a..3cfd351
--- a/piupartslib/packagesdb.py
+++ b/piupartslib/packagesdb.py
@@@ -54,6 -54,7 +54,7 @@@ class Package(UserDict.UserDict)
name, value = header.split(":", 1)
self[name.strip()] = value.strip()
self._parsed_deps = {}
+ self._parsed_alt_deps = {}
def _parse_dependencies(self, header_name):
if header_name in self._parsed_deps:
@@@ -65,6 -66,17 +66,17 @@@
self._parsed_deps[header_name] = depends
return depends
+ def _parse_alternative_dependencies(self, header_name):
+ if header_name in self._parsed_alt_deps:
+ depends = self._parsed_alt_deps[header_name]
+ else:
+ parser = DependencyParser(self[header_name])
+ depends = parser.get_dependencies()
+ depends = [[alt.name for alt in alternatives] for alternatives in depends]
+ self._parsed_alt_deps[header_name] = depends
+ return depends
+
+ # first alternative only - [package_name...]
def dependencies(self):
vlist = []
for header in ["Depends", "Pre-Depends"]:
@@@ -72,6 -84,14 +84,14 @@@
vlist += self._parse_dependencies(header)
return vlist
+ # all alternatives - [[package_name...]...]
+ def all_dependencies(self):
+ vlist = []
+ for header in ["Depends", "Pre-Depends"]:
+ if header in self:
+ vlist += self._parse_alternative_dependencies(header)
+ return vlist
+
def depends_with_alts(self, header_name):
vlist = []
if header_name in self:
@@@ -194,7 -214,6 +214,7 @@@ class LogDB
class PackagesDB:
+ # keep in sync with piuparts-report.py: emphasize_reason()
_states = [
"successfully-tested",
"failed-testing",
@@@ -206,7 -225,6 +226,7 @@@
"dependency-cannot-be-tested",
"dependency-does-not-exist",
"circular-dependency",
+ #"does-not-exist", # can only happen as query result for a dependency
"unknown",
"unknown-preferred-alternative",
"no-dependency-from-alternatives-exists",
@@@ -221,7 -239,6 +241,7 @@@
"dependency-cannot-be-tested": "dependency-cannot-be-tested",
"dependency-does-not-exist": "dependency-does-not-exist",
"circular-dependency": "circular-dependency",
+ "does-not-exist": "dependency-does-not-exist",
"unknown-preferred-alternative": "unknown-preferred-alternative",
"no-dependency-from-alternatives-exists": "dependency-cannot-be-tested",
}
@@@ -271,16 -288,13 +291,16 @@@
def _find_all_packages(self):
if self._packages is None:
self._packages = {}
+ self._virtual_packages = {}
for pf in self._packages_files:
for p in pf.values():
self._packages[p["Package"]] = p
for p in self._packages.values():
for provided in p.provides():
- if provided not in self._packages:
- self._packages[provided] = p
+ if provided != p["Package"]:
+ if provided not in self._virtual_packages:
+ self._virtual_packages[provided] = []
+ self._virtual_packages[provided].append(p["Package"])
def _get_recursive_dependencies(self, package, break_circles=True):
assert self._packages is not None
@@@ -293,8 -307,6 +313,8 @@@
deps.append(dep)
if dep in self._packages:
more += self._packages[dep].dependencies()
+ elif dep in self._virtual_packages:
+ more += self._packages[self._virtual_packages[dep][0]].dependencies()
# Break circular dependencies
if break_circles and package["Package"] in deps:
@@@ -339,9 -351,9 +359,9 @@@
prefer_alt = None
for alternative in alt_deps[d]:
dep = alternative.name
- if dep in self._package_state:
+ altdep_state = self.get_package_state(dep)
+ if altdep_state != "does-not-exist":
alt_found += 1
- altdep_state = self._package_state[dep]
if prefer_alt_score < 3 and altdep_state == "essential-required":
prefer_alt = alternative
prefer_alt_idx = d
@@@ -378,15 -390,18 +398,15 @@@
return state
for dep in package.dependencies():
- if dep not in self._package_state:
- return "dependency-does-not-exist"
- dep_state = self._package_state[dep]
- if dep_state is None:
- return "unknown"
- elif dep_state in self._dep_state_to_state:
+ dep_state = self.get_package_state(dep)
+ if dep_state in self._dep_state_to_state:
return self._dep_state_to_state[dep_state]
state = "waiting-to-be-tested"
for dep in package.dependencies():
- if self._package_state[dep] not in \
- ["successfully-tested", "essential-required"]:
+ dep_state = self.get_package_state(dep)
+ if dep_state not in \
+ ["successfully-tested", "essential-required"]:
state = "unknown"
break
if state == "waiting-to-be-tested":
@@@ -398,21 -413,20 +418,21 @@@
if pkg in deps:
deps.remove(pkg)
if package["Package"] in deps:
- return "circular-dependency" # actually, it's a unknown circular-dependency
+ return "circular-dependency" # actually, it's an unknown circular-dependency
# treat circular-dependencies as testable (for the part of the circle)
state = "unknown"
if package["Package"] in self._known_circular_depends:
- for dep in package.dependencies():
- if dep not in self._known_circular_depends and self._package_state[dep] not in \
- ["successfully-tested", "essential-required"]:
- state = "unknown"
- break
- if dep in self._known_circular_depends and self._package_state[dep] not in \
- ["failed-testing","dependency-failed-testing"]:
- state = "waiting-to-be-tested"
- continue
+ for dep in package.dependencies():
+ dep_state = self.get_package_state(dep)
+ if dep not in self._known_circular_depends and dep_state not in \
+ ["successfully-tested", "essential-required"]:
+ state = "unknown"
+ break
+ if dep in self._known_circular_depends and dep_state not in \
+ ["failed-testing", "dependency-failed-testing"]:
+ state = "waiting-to-be-tested"
+ continue
return state
def _compute_package_states(self):
@@@ -460,8 -474,6 +480,8 @@@
self._in_state["unknown"] = todo
self._in_state["unknown-preferred-alternative"] = unpreferred_alt
+ for package_name in unpreferred_alt:
+ self._package_state[package_name] = "unknown-preferred-alternative"
for state in self._states:
self._in_state[state].sort()
@@@ -474,17 -486,11 +494,17 @@@
return set(self._in_state[state])
def has_package(self, name):
+ self._find_all_packages()
return name in self._packages
def get_package(self, name):
return self._packages[name]
+ def get_providers(self, name):
+ if name in self._virtual_packages:
+ return self._virtual_packages[name]
+ return []
+
def get_all_packages(self):
self._find_all_packages()
return self._packages
@@@ -515,16 -521,14 +535,16 @@@
else:
return self._packages[package_name][header]
- def get_package_state(self, package_name):
- return self._package_state[package_name]
-
- def state_by_name(self, package_name):
+ def get_package_state(self, package_name, resolve_virtual=True):
if package_name in self._package_state:
return self._package_state[package_name]
- else:
- return "unknown"
+ if package_name in self._virtual_packages:
+ if resolve_virtual:
+ provider = self._virtual_packages[package_name][0]
+ return self._package_state[provider]
+ else:
+ return "virtual"
+ return "does-not-exist"
def _find_packages_ready_for_testing(self):
return self.get_pkg_names_in_state("waiting-to-be-tested")
--
piuparts git repository
More information about the Piuparts-commits
mailing list