[Piuparts-commits] [piuparts] 09/09: dwke: remove reporting part, integrated into piuparts-report.py
Holger Levsen
holger at moszumanska.debian.org
Mon Feb 10 13:03:04 UTC 2014
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch develop
in repository piuparts.
commit f484bd3a42b786cca2fb8970712fbcd4d52c8bac
Author: Andreas Beckmann <anbe at debian.org>
Date: Sun Feb 9 18:33:39 2014 +0100
dwke: remove reporting part, integrated into piuparts-report.py
Signed-off-by: Andreas Beckmann <anbe at debian.org>
---
debian/changelog | 3 +
master-bin/detect_well_known_errors.py | 190 +--------------------------------
2 files changed, 5 insertions(+), 188 deletions(-)
diff --git a/debian/changelog b/debian/changelog
index d0e1908..8eb3e5c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -53,6 +53,9 @@ piuparts (0.57) UNRELEASED; urgency=low
* piuparts-report.py:
- Mark essential-required as obsolete and hide it.
- Integrate the reporting part from detect_well_known_errors.py.
+ * master-bin/detect_well_known_errors.py:
+ - Only update .kpr files, the reporting part was integrated into
+ piuparts-report.
* scripts/post_{setup,distupgrade}_force-unsafe-io: New custom script to
automatically enable dpkg --force-unsafe-io once that is supported.
Also divert /bin/sync and replace it with a link to /bin/true. Syncing a
diff --git a/master-bin/detect_well_known_errors.py b/master-bin/detect_well_known_errors.py
index b4827f7..4cdc5ae 100755
--- a/master-bin/detect_well_known_errors.py
+++ b/master-bin/detect_well_known_errors.py
@@ -25,7 +25,6 @@ import os
import sys
import time
import logging
-import re
import argparse
import piupartslib
@@ -34,43 +33,8 @@ from piupartslib.dwke import *
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
-DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
KPR_DIRS = ('pass', 'bugged', 'affected', 'fail')
-TPL_EXT = '.tpl'
-
-PROB_TPL = \
-"""<tr class="titlerow"><td class="titlecell">
-$HEADER in $SECTION, sorted by reverse dependency count.
-</td></tr><tr class="normalrow"><td class="contentcell2">
-$HELPTEXT
-<p>The commandline to find these logs is: <pre>
-COMMAND='$COMMAND'
-</pre></p>
-</td></tr><tr class="titlerow"><td class="alerttitlecell">Please file bugs!</td></tr><tr class="normalrow"><td class="contentcell2" colspan="3">
-<ul>
-$PACKAGE_LIST</ul>
-<p>Affected packages in $SECTION: $COUNT</p></td></tr>
-"""
-
-UNKNOWN_TPL = \
-"""<tr class="titlerow"><td class="titlecell">
-Packages with unknown failures detected in $SECTION, sorted by reverse dependency count.
-</td></tr><tr class="normalrow"><td class="contentcell2">
-<p>Please investigate and improve detection of known error types!</p>
-</td></tr><tr class="titlerow"><td class="alerttitlecell">Please file bugs!</td></tr><tr class="normalrow"><td class="contentcell2" colspan="3">
-<ul>
-$PACKAGE_LIST
-</ul>
-<p>Affected packages in $SECTION: $COUNT</p></td></tr>
-"""
-
-PKG_ERROR_TPL = \
-"""<li>$RDEPS - <a href=\"$LOG\">$LOG</a>
- (<a href=\"http://packages.qa.debian.org/$SDIR/$SPKG.html\" target=\"_blank\">PTS</a>)
- (<a href=\"http://bugs.debian.org/$PACKAGE?dist=unstable\" target=\"_blank\">BTS</a>)
-$BUG</li>
-"""
class WKE_Config(piupartslib.conf.Config):
"""Configuration parameters for Well Known Errors"""
@@ -81,26 +45,10 @@ class WKE_Config(piupartslib.conf.Config):
piupartslib.conf.Config.__init__(self, self.section,
{
"sections": "report",
- "output-directory": "html",
"master-directory": ".",
"known-problem-directory": "@sharedir@/piuparts/known_problems",
- "proxy": None,
}, "")
-class WKE_Section_Config(piupartslib.conf.Config):
-
- def __init__(self, section):
- self.section = section
-
- piupartslib.conf.Config.__init__(self, self.section,
- {
- "mirror": None,
- "distro": None,
- "area": None,
- "arch": None,
- "upgrade-test-distros": None,
- }, "", defaults_section="global")
-
def setup_logging(log_level):
logger = logging.getLogger()
@@ -109,36 +57,6 @@ def setup_logging(log_level):
logger.addHandler(handler)
-def pts_subdir(source):
- if source[:3] == "lib":
- return source[:4]
- else:
- return source[:1]
-
-def source_pkg(pkgspec, db):
- source_name = db.get_control_header(get_pkg(pkgspec), "Source")
-
- return source_name
-
-def get_pkgspec(logpath):
- """For a log full file spec, return the pkgspec (<pkg>_<version)"""
- return logpath.split('/')[-1]
-
-def get_bug_text(logpath):
- bugpath = replace_ext(logpath, BUG_EXT)
-
- txt = ""
- if os.path.exists(bugpath):
- bf = open(bugpath, 'r')
- txt = bf.read()
- bf.close()
-
- return txt
-
-def section_path(logpath):
- """Convert a full log path name to one relative to the section directory"""
- return '/'.join([get_where(logpath), get_pkgspec(logpath)])
-
def mtime(path):
return os.path.getmtime(path)
@@ -192,96 +110,11 @@ def make_kprs(logdict, kprdict, problem_list):
return len(needs_kpr)
-def populate_tpl(tmpl, vals):
-
- for key in vals:
- tmpl = re.sub("\$%s" % key, str(vals[key]), tmpl)
-
- return tmpl
-
-def update_tpl(basedir, section, problem, failures, logdict, ftpl, ptpl, pkgsdb):
-
- pkg_text = ""
- bugged_section = False
- for failure in failures:
-
- pkgspec = failure.pkgspec
- bin_pkg = get_pkg(pkgspec)
- rdep_cnt = pkgsdb.rrdep_count(bin_pkg)
- pkg_obj = pkgsdb.get_package(bin_pkg)
-
- if not pkg_obj is None:
- src_pkg = source_pkg(pkgspec, pkgsdb)
- else:
- src_pkg = bin_pkg
-
- if bugged_section is False and get_where(logdict[pkgspec]) != 'fail':
- bugged_section = True
- pkg_text += "</ul><ul>\n"
-
- pkg_text += populate_tpl(ftpl, {
- 'LOG': section_path(logdict[pkgspec]),
- 'PACKAGE': bin_pkg,
- 'BUG': get_bug_text(logdict[pkgspec]),
- 'RDEPS': rdep_cnt,
- 'SDIR':pts_subdir(src_pkg),
- 'SPKG':src_pkg,
- })
-
- if len(pkg_text):
- pf = open(os.path.join(basedir, failures[0].problem[:-5] + TPL_EXT), 'w')
- tpl_text = populate_tpl(ptpl, {
- 'HEADER': problem.HEADER,
- 'SECTION': section,
- 'HELPTEXT': problem.HELPTEXT,
- 'COMMAND': problem.get_command(),
- 'PACKAGE_LIST': pkg_text,
- 'COUNT': len(failures),
- })
-
- pf.write(tpl_text)
- pf.close()
-
-def update_html(section, logdict, problem_list, failures, config, pkgsdb):
-
- html_dir = os.path.join(config['output-directory'], section)
- if not os.path.exists(html_dir):
- os.makedirs(html_dir)
-
- for problem in problem_list:
- update_tpl(html_dir, section, problem,
- failures.filtered(problem.name),
- logdict,
- PKG_ERROR_TPL, PROB_TPL, pkgsdb)
-
- # Make a failure list of all failed packages that don't show up as known
- failedpkgs = set([x for x in logdict.keys()
- if get_where(logdict[x]) != 'pass'])
- knownfailpkgs = set([failure.pkgspec for failure in failures.failures])
- unknownsasfailures = [make_failure("", "unknown_failures.conf", x)
- for x in failedpkgs.difference(knownfailpkgs)]
-
- def keyfunc(x, pkgsdb=pkgsdb, logdict=logdict):
- rdeps = pkgsdb.rrdep_count(get_pkg(x.pkgspec))
-
- is_failed = get_where(logdict[x.pkgspec]) == "fail"
-
- return (not is_failed, -rdeps, logdict[x.pkgspec])
-
- unknownsasfailures.sort(key=keyfunc)
-
- update_tpl(html_dir, section, problem_list[0], unknownsasfailures,
- logdict,
- PKG_ERROR_TPL, UNKNOWN_TPL, pkgsdb)
def process_section(section, config, problem_list,
recheck=False, recheck_failed=False, pkgsdb=None):
""" Update .bug and .kpr files for logs in this section """
- # raises MissingSection if the section does not exist in piuparts.conf
- section_config = WKE_Section_Config(section)
- section_config.read(CONFIG_FILE)
-
sectiondir = os.path.join(config['master-directory'], section)
workdirs = [os.path.join(sectiondir, x) for x in KPR_DIRS]
@@ -301,22 +134,7 @@ def process_section(section, config, problem_list,
add_cnt = make_kprs(logdict, kprdict, problem_list)
- if not pkgsdb:
- distro_config = piupartslib.conf.DistroConfig(
- DISTRO_CONFIG_FILE, section_config["mirror"])
-
- sectiondir = os.path.join(config['master-directory'], section)
- pkgsdb = piupartslib.packagesdb.PackagesDB(prefix=sectiondir)
- pkgsdb.load_packages_urls(
- distro_config.get_packages_urls(
- section_config.get_distro(),
- section_config.get_area(),
- section_config.get_arch()))
-
failures = FailureManager(logdict)
- failures.sort_by_bugged_and_rdeps(pkgsdb)
-
- update_html(section, logdict, problem_list, failures, config, pkgsdb)
return (del_cnt, add_cnt, failures)
@@ -347,12 +165,10 @@ if __name__ == '__main__':
setup_logging(logging.DEBUG)
parser = argparse.ArgumentParser(
- description="Detect/process well known errors to html",
+ description="Detect well known errors",
epilog="""
This script processes all log files against defined "known_problem" files,
-caching the problems found, by package, into ".kpr" files. The cached data
-is summarized into html ".tpl" files in <html_dir>/<section>, which are then
-incorporated by piuparts-report into the final web reports.
+caching the problems found, by package, into ".kpr" files.
""")
parser.add_argument('sections', nargs='*', metavar='SECTION',
@@ -369,8 +185,6 @@ incorporated by piuparts-report into the final web reports.
conf = WKE_Config()
conf.read(CONFIG_FILE)
- if conf["proxy"]:
- os.environ["http_proxy"] = conf["proxy"]
sections = args.sections
if not sections:
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git
More information about the Piuparts-commits
mailing list