[Piuparts-commits] [SCM] piuparts git repository branch, develop, updated. 9bcbfe0fa48aba1da806fecfea79e3aa2d7bdb27
Holger Levsen
holger at layer-acht.org
Fri Dec 2 13:30:42 UTC 2011
The following commit has been merged in the develop branch:
commit 6a680192b61b7a1acc6b7990cec66f7c16b5d4dd
Merge: 9432b1bc4337165b08a886a7bc4e61016bd41c89 35feee25bbd017d90a129b7702da20b92b801a5d
Author: Holger Levsen <holger at layer-acht.org>
Date: Fri Dec 2 14:22:16 2011 +0100
Merge branch 'feature/report-speedup-maintainer-summaries' into develop
diff --combined piuparts-report.py
index 81c8a02,1db9357..217c37e
--- a/piuparts-report.py
+++ b/piuparts-report.py
@@@ -445,7 -445,6 +445,7 @@@ def emphasize_reason(reason)
"dependency-cannot-be-tested",
"dependency-does-not-exist",
"circular-dependency",
+ "does-not-exist",
"unknown",
"unknown-preferred-alternative",
"no-dependency-from-alternatives-exists",
@@@ -662,7 -661,7 +662,7 @@@ class Section
return link
def link_to_state_page(self, section, package_name, link_target):
- state = self._binary_db.state_by_name(package_name)
+ state = self._binary_db.get_package_state(package_name)
if state != "unknown":
link = "<a href=\"/%s/%s\">%s</a>" % (
section,
@@@ -726,33 -725,30 +726,30 @@@
return total
- def merge_maintainer_templates(self, templates):
- for maint_tpl in templates:
- tpl = os.path.join(self._output_directory,"maintainer",maintainer_subdir(maint_tpl),maint_tpl)
- lines = read_file(tpl)
+ def create_maintainer_summaries(self, maintainers, source_data):
+ logging.debug("Writing maintainer summaries in %s" % self._output_directory)
+ states = ["fail", "unknown", "pass"]
+ for maintainer in maintainers.keys():
+ sources = maintainers[maintainer]
rows = ""
- for line in lines:
- state, count, packages = line.split(",")
- if packages == "none\n":
- links = " "
- else:
- links = ""
- for package in packages.split(" "):
- links += "<a href=\"#%s\">%s</a> " % (package,package)
- rows += "<tr class=\"normalrow\"><td class=\"labelcell\">%s:</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"4\">%s</td></tr>" % \
- (state, count, links)
- os.unlink(tpl)
- template_path = tpl[:-len("_tpl")]
-
- for state in ("fail","unkn","pass"):
- filename = template_path+"_"+state
- if os.path.isfile(filename):
- f = file(filename, "r")
- rows += file.read(f)
- f.close()
- os.unlink(filename)
-
- maintainer = os.path.basename(maint_tpl[:-len("_tpl")])
+ package_rows = ""
+ packages = {}
+ for state in states:
+ packages[state] = []
+ for source in sorted(sources):
+ (state, sourcerows, binaryrows) = source_data[source]
+ packages[state].append(source)
+ package_rows += sourcerows + binaryrows
+
+ for state in states:
+ if len(packages[state]) > 0:
+ links = ""
+ for package in packages[state]:
+ links += "<a href=\"#%s\">%s</a> " % (package, package)
+ else:
+ links = " "
+ rows += "<tr class=\"normalrow\"><td class=\"labelcell\">%s:</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"4\">%s</td></tr>" % \
+ (state, len(packages[state]), links)
distrolinks = "<tr class=\"normalrow\"><td class=\"labelcell\">other distributions: </td><td class=\"contentcell2\" colspan=\"5\">"
for section in self._section_names:
@@@ -761,7 -757,7 +758,7 @@@
distrolinks += "</td></tr>"
htmlpage = string.Template(HTML_HEADER + MAINTAINER_BODY_TEMPLATE + HTML_FOOTER)
- filename = template_path+".html"
+ filename = os.path.join(self._output_directory, "maintainer", maintainer_subdir(maintainer), maintainer + ".html")
f = file(filename, "w")
f.write(htmlpage.safe_substitute( {
"page_title": html_protect("Status of "+maintainer+" packages in "+self._config.section),
@@@ -769,10 -765,11 +766,11 @@@
"distrolinks": distrolinks,
"section_navigation": create_section_navigation(self._section_names,self._config.section),
"time": time.strftime("%Y-%m-%d %H:%M %Z"),
- "rows": rows,
+ "rows": rows + package_rows,
}))
f.close()
+
def create_source_summary (self, source, logs_by_dir):
source_version = self._source_db.get_control_header(source, "Version")
binaries = self._source_db.get_control_header(source, "Binary")
@@@ -783,7 -780,7 +781,7 @@@
failed = False
binaryrows = ""
for binary in sorted(binaries.split(", ")):
- state = self._binary_db.state_by_name(binary)
+ state = self._binary_db.get_package_state(binary)
if state == "unknown":
# Don't track udebs and binary packages on other archs.
# The latter is a FIXME which needs parsing the Packages files from other archs too
@@@ -835,61 -832,30 +833,30 @@@
return sourcerows, binaryrows, source_state, maintainer, uploaders
- def create_maintainer_templates_for_source(self,source, source_state, sourcerows, binaryrows, maintainer, uploaders):
- maintainer_pages = []
- maintainer_pages.append(get_email_address(maintainer))
- for uploader in uploaders.split(", "):
- if uploader:
- maintainer_pages.append(get_email_address(uploader))
- for maintainer_page in maintainer_pages:
- maintainer_summary_page_path = os.path.join(self._output_directory, "maintainer", maintainer_subdir(maintainer_page))
-
- if not os.path.exists(maintainer_summary_page_path):
- os.makedirs(maintainer_summary_page_path)
- filename = os.path.join(maintainer_summary_page_path, (maintainer_page + "_tpl"))
- maintainer_package_count = {}
- maintainer_packages = {}
- if os.path.isfile(filename):
- lines = read_file(filename)
- for line in lines:
- state, count, packages = line.split(",")
- maintainer_package_count[state]=int(count)
- maintainer_packages[state]=packages[:-1]
- if maintainer_packages[source_state] == "none":
- maintainer_packages[source_state] = source
- else:
- maintainer_packages[source_state] = "%s %s" % (maintainer_packages[source_state],source)
- else:
- maintainer_package_count["fail"] = 0
- maintainer_package_count["unknown"] = 0
- maintainer_package_count["pass"] = 0
- for state in "fail", "unknown", "pass":
- maintainer_packages[state] = "none"
- maintainer_packages[source_state] = source
- if source_state == "fail":
- maintainer_package_count["fail"]+=1
- elif source_state == "unknown":
- maintainer_package_count["unknown"]+=1
- else:
- maintainer_package_count["pass"]+=1
- lines = ""
- for state in "fail", "unknown", "pass":
- lines += "%s,%s,%s\n" % (state,maintainer_package_count[state],maintainer_packages[state])
- write_file(filename,lines)
- append_file(filename[:-4]+"_"+source_state[:4],sourcerows+binaryrows)
def create_package_summaries(self, logs_by_dir):
logging.debug("Writing package templates in %s" % self._config.section)
+ maintainers = {}
+ source_binary_rows = {}
sources = ""
for source in self._source_db.get_all_packages():
(sourcerows, binaryrows, source_state, maintainer, uploaders) = self.create_source_summary(source, logs_by_dir)
if source_state != "udeb":
- sources += "%s: %s\n" % (source, source_state)
- self.create_maintainer_templates_for_source(source, source_state, sourcerows, binaryrows, maintainer, uploaders)
+ sources += "%s: %s\n" % (source, source_state)
+ source_binary_rows[source] = (source_state, sourcerows, binaryrows)
+ for maint in [maintainer] + uploaders.split(","):
+ if maint.strip():
+ email = get_email_address(maint.strip())
+ if not "INVALID" in email:
+ if not email in maintainers:
+ maintainers[email] = []
+ maintainers[email].append(source)
write_file(os.path.join(self._output_directory, "sources.txt"), sources)
+ self.create_maintainer_summaries(maintainers, source_binary_rows)
+
def make_stats_graph(self):
countsfile = os.path.join(self._output_directory, "counts.txt")
@@@ -1007,7 -973,7 +974,7 @@@
for dep in package.dependencies():
vlist += "<li>dependency %s is %s</li>\n" % \
(self.link_to_state_page(self._config.section,dep,dep),
- emphasize_reason(html_protect(self._binary_db.state_by_name(dep))))
+ emphasize_reason(html_protect(self._binary_db.get_package_state(dep))))
vlist += "</ul>\n"
vlist += "</li>\n"
htmlpage = string.Template(HTML_HEADER + STATE_BODY_TEMPLATE + HTML_FOOTER)
@@@ -1020,25 -986,6 +987,25 @@@
"list": vlist
}))
+
+ def archive_logfile(self, vdir, log):
+ archivedir = os.path.join("archive", vdir)
+ if not os.path.exists(archivedir):
+ os.makedirs(archivedir)
+ os.rename(os.path.join(vdir, log), os.path.join("archive", vdir, log))
+
+
+ def cleanup_removed_packages(self, logs_by_dir):
+ for vdir in logs_by_dir.keys():
+ for log in sorted(logs_by_dir[vdir]):
+ if log.endswith(".log"):
+ package, version = log[:-len(".log")].split("_")
+ if not self._binary_db.has_package(package):
+ logging.debug("Package %s was removed, archiving %s/%s" % (package, vdir, log))
+ self.archive_logfile(vdir, log)
+ logs_by_dir[vdir].remove(log)
+
+
def generate_html(self):
logging.debug("Finding log files")
dirs = ["pass", "fail", "bugged", "reserved", "untestable"]
@@@ -1046,9 -993,6 +1013,9 @@@
for vdir in dirs:
logs_by_dir[vdir] = find_files_with_suffix(vdir, ".log")
+ logging.debug("Archiving logs of obsolete packages")
+ self.cleanup_removed_packages(logs_by_dir)
+
logging.debug("Copying log files")
copy_logs(logs_by_dir, self._output_directory)
@@@ -1063,9 -1007,6 +1030,6 @@@
if self._config["sources-url"]:
self.create_package_summaries(logs_by_dir)
- logging.debug("Merging maintainer summaries in %s" % self._output_directory)
- self.merge_maintainer_templates(find_files_with_suffix(self._output_directory+"/maintainer/", "_tpl"))
-
logging.debug("Writing section index page")
self.write_section_index_page(dirs, total_packages)
@@@ -1089,7 -1030,6 +1053,7 @@@
self.generate_html()
os.chdir(oldcwd)
+
def main():
setup_logging(logging.DEBUG, None)
--
piuparts git repository
More information about the Piuparts-commits
mailing list