[Collab-qa-commits] r1431 - in ddpo-by-mail: . dato-ood
Lucas Nussbaum
lucas at alioth.debian.org
Sun Apr 19 07:30:53 UTC 2009
Author: lucas
Date: 2009-04-19 07:30:53 +0000 (Sun, 19 Apr 2009)
New Revision: 1431
Added:
ddpo-by-mail/dato-ood/
ddpo-by-mail/dato-ood/buildd-stats/
ddpo-by-mail/dato-ood/common.py
ddpo-by-mail/dato-ood/fetch
ddpo-by-mail/dato-ood/ood_excuses.py
Log:
added dato-ood
Added: ddpo-by-mail/dato-ood/common.py
===================================================================
--- ddpo-by-mail/dato-ood/common.py (rev 0)
+++ ddpo-by-mail/dato-ood/common.py 2009-04-19 07:30:53 UTC (rev 1431)
@@ -0,0 +1,241 @@
+#! /usr/bin/env python2.4
+## vim: fileencoding=utf-8
+#
+# Copyright (c) 2006, 2007 Adeodato Simó (dato at net.com.org.es)
+# Licensed under the terms of the MIT license.
+
+"""Common functions, classes and variables for my testing-stuff code."""
+
+import re
+import os
+import sys
+
+from subprocess import Popen, PIPE
+
+##
+
+update_output = 'update.OUTPUT_py'
+update_excuses = 'update.EXCUSES_py'
+binary_source_map_file = 'sources'
+
+fuckedarches = [ ]
+all_arches = [ 'alpha', 'amd64', 'arm', 'armel', 'hppa', 'i386', 'ia64',
+ 'mips', 'mipsel', 'powerpc', 's390', 'sparc', ]
+
+buildd_stats_dir = os.path.expanduser('~/dato-ood/buildd-stats')
+
+##
+
+def binary_source_map():
+ sfile = file(binary_source_map_file, 'r')
+ binaries = {}
+
+ for line in sfile:
+ binary, section, source = line.split()
+ binaries[binary] = source
+
+ return binaries
+
+def parse_excuses(excuses_file=update_excuses, buildd_stats=None):
+ exfile = file(excuses_file, 'r')
+ ex = None
+ excuses = {}
+
+ re1 = re.compile(r'<li><a id="([^-][^"_]+?)"[^(]+\((\S+) to (\S+)\)', re.I)
+ re2 = re.compile(r'<li>([^<].*)', re.I)
+
+ for l in exfile:
+ m = re2.match(l)
+ if ex and m:
+ ex.update(m.group(1))
+ continue
+ m = re1.match(l)
+ if m:
+ p, ov, nv = m.groups()
+ p = p.split('/')[0] # handle binNMUs
+ ex = excuses[p] = Excuse(p, ov, nv, buildd_stats)
+ continue
+
+ return excuses
+
+def parse_buildd_stats(update=True):
+ if not os.path.isdir(buildd_stats_dir):
+ print "W: '%s' does not exist, creating it" % buildd_stats_dir
+ try:
+ os.makedirs(buildd_stats_dir)
+ except OSError, e:
+ print >>sys.stderr, 'E: could not mkdir %s: %s' % (
+ buildd_stats_dir, e)
+ sys.exit(1)
+
+ # update the files
+ if update:
+ try:
+ p = Popen("lftp -c 'open http://buildd.debian.org/stats/; "
+ "mirror -e -r -c -I *-all.txt'",
+ shell=True, cwd=buildd_stats_dir, stdout=PIPE)
+ status = p.wait()
+ except OSError, e:
+ print >>sys.stderr, 'E: lftp call failed: %s' % e
+ sys.exit(1)
+
+ if status != 0:
+ print >>sys.stderr, 'E: lftp call failed'
+ sys.exit(1)
+
+ ##
+
+ buildd_stats = {}
+ regex = re.compile(r'^(?P<section>\S+)/(?P<pkg_ver>\S+): (?P<status>\S+)')
+
+ for arch in all_arches:
+ buildd_stats.setdefault(arch, {})
+
+ for line in file(os.path.join(buildd_stats_dir, arch + '-all.txt')):
+ if line.startswith(' '):
+ continue
+ m = regex.match(line)
+ if not m:
+ continue
+ buildd_stats[arch][m.group('pkg_ver')] = m.group('status')
+
+ return buildd_stats
+
+##
+
+class Excuse(object):
+ regexes = {
+ '_maintainer': re.compile(r'^Maintainer: (.*)'),
+ '_age': re.compile(r'(?:^Too young, only (\d+) of (\d+))' '|'
+ r'(?:^(\d+) days old \(needed (\d+))'),
+
+ '_valid_candidate': re.compile(r'^Valid candidate'),
+ '_not_considered': re.compile(r'^Not considered'),
+ '_invalidated': re.compile(r'^Invalidated by dependency'),
+
+ '_depends': re.compile(r'^Depends: \S+ <a href="#([^"]+)'),
+ '_out_of_date': re.compile(r'^out of date on .+?>(\S+?)<'),
+ '_buggy': re.compile(r'^Updating \S+ introduces new bugs: (.+)'),
+
+ # TODO:
+ # unsatisfiable
+ # frozen
+ }
+
+ def __init__(self, package, old_version, new_version, buildd_stats=None):
+ if old_version == '-':
+ old_version = None
+
+ self.package = package
+ self.old_version = old_version
+ self.new_version = new_version
+
+ self.valid_candidate = False
+ self.not_considered = False
+ self.invalidated = False
+
+ self.days_needed = 0
+ self.days_waited = 0
+
+ self.rc_bugs = 0
+ self.missing_days = 0
+ self.ood_arches = set()
+
+ self.depends = set()
+ self._breaks = {}
+
+ self.html = '<li>(score: %%d) <a id="%(package)s" name="%(package)s">%(package)s</a>' \
+ " (%(old_version)s to %(new_version)s)\n<ul>\n" % locals()
+
+ self._buildd_stats = buildd_stats or {}
+
+ def score(self):
+ score = 0
+ score += 1 * len(self.ood_arches)
+ return score
+
+ def __cmp__(self, other):
+ return self.score() - other.score() or cmp(self.package, other.package)
+
+ def update(self, line):
+ self.html += "<li>" + line + "\n"
+
+ for f, r in Excuse.regexes.items():
+ m = r.search(line)
+ if m:
+ getattr(self, f)(*filter(None, m.groups()))
+ break
+ else:
+ pass
+ #if not re.match(r'Section|Ignoring', line):
+ # print "Don't now how to handle: ", line
+
+ def breaks(self, arch, broken):
+ self._breaks.setdefault(arch, {}).update(broken)
+
+ def __breaks_to_html(self, excuses, cluster):
+ html = ''
+ for arch, broken in self._breaks.items():
+ html += '<li> Breaks (%s): ' % (arch,)
+ for s, p in sorted(broken.items()):
+ if cluster and s in cluster:
+ pretty_s = '<a href="#%s">%s</a>' % (s, s)
+ elif excuses and s in excuses:
+ pretty_s = '<b>%s</b>' % (s,)
+ else:
+ pretty_s = s
+
+ if len(p) == 1 and p.copy().pop() == s:
+ html += "%s, " % (pretty_s, )
+ else:
+ html += '%s (<span style="color: gray;">%s</span>), ' % (pretty_s, ', '.join(sorted(p)))
+ html += "\n"
+
+ return html
+
+ def __str__(self, excuses=None, cluster=None):
+ return self.package + ' ' + self.new_version + ' ' + ','.join(self.ood_arches) + "\n"
+
+ def pretty_html(self, excuses, cluster):
+ return self.__str__(excuses, cluster)
+
+ def _maintainer(self, m):
+ self.maintainer = m
+
+ def _age(self, done, needed):
+ done, needed = map(int, (done, needed))
+ self.days_needed = needed
+ self.days_waited = done
+ if needed > done:
+ self.missing_days = (needed - done)
+
+ def _depends(self, d):
+ self.depends.add(d)
+
+ def _valid_candidate(self): self.valid_candidate = True
+ def _not_considered(self): self.not_considered = True
+ def _invalidated(self): self.invalidated = True
+
+ def _out_of_date(self, arch):
+ if arch not in fuckedarches:
+ self.ood_arches.add(arch)
+
+ # now add a link to buildd.d.o/~jeroen and maybe the buildd status
+
+ regex = re.compile(r'out of date (on <[aA].+?>(\S+?)</[aA]>)')
+ match = regex.search(self.html)
+
+ if match:
+ arch = match.group(2)
+ pkg_ver = '%s_%s' % (self.package, self.new_version)
+ status = self._buildd_stats.get(arch, {}).get(pkg_ver, '')
+
+ if status:
+ status = ' [%s]' % status
+
+ self.html = regex.sub(
+ '<a href="http://buildd.debian.org/~jeroen/status/package.php?p=%s">'
+ 'out of date</a> \\1%s' % (self.package, status), self.html)
+
+ def _buggy (self, bugs):
+ self.rc_bugs += len(bugs.split(','))
Added: ddpo-by-mail/dato-ood/fetch
===================================================================
--- ddpo-by-mail/dato-ood/fetch (rev 0)
+++ ddpo-by-mail/dato-ood/fetch 2009-04-19 07:30:53 UTC (rev 1431)
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+rsync /org/bugs.debian.org/etc/indices/sources .
+rm -f update.OUTPUT_py update.EXCUSES_py
+wget -q -O update.OUTPUT_py http://release.debian.org/britney/update_output.txt
+wget -q -O update.EXCUSES_py http://release.debian.org/britney/update_excuses.html
Property changes on: ddpo-by-mail/dato-ood/fetch
___________________________________________________________________
Added: svn:executable
+ *
Added: ddpo-by-mail/dato-ood/ood_excuses.py
===================================================================
--- ddpo-by-mail/dato-ood/ood_excuses.py (rev 0)
+++ ddpo-by-mail/dato-ood/ood_excuses.py 2009-04-19 07:30:53 UTC (rev 1431)
@@ -0,0 +1,70 @@
+#! /usr/bin/env python2.4
+## vim: fileencoding=utf-8
+#
+# Copyright (c) 2007 Adeodato Simó (dato at net.com.org.es)
+# Licensed under the terms of the MIT license.
+
+"""Produce a reduced excuses file that only includes otherwise ready OOD stuff."""
+
+import re
+import os
+import sys
+import time
+
+import common
+
+##
+
+OutFile = os.path.expanduser('~/public_html/ood_excuses.txt')
+
+##
+
+def main():
+ buildd_stats = common.parse_buildd_stats()
+ excuses = common.parse_excuses()
+
+ for ex in excuses.values():
+ # discard packages which are not OOD, or which are, but have other
+ # problems (like buggy or too young)
+ if not ex.ood_arches:
+ excuses.pop(ex.package)
+ continue
+
+ if ex.days_waited < 5: # FIXME remove that check?
+ excuses.pop(ex.package)
+ continue
+
+ pkg_ver = '%s_%s' % (ex.package, ex.new_version)
+
+ # discard packages which are in a state that does not need manual
+ # action
+ for arch in ex.ood_arches:
+ status = buildd_stats[arch].get(pkg_ver, None)
+ if status not in ['Dep-Wait', 'Needs-Build', 'Installed', 'Built', None]:
+ # TODO: discard 'Built' and 'Uploaded' as well?
+ break
+ else:
+ excuses.pop(ex.package)
+
+ tmp = OutFile + '.new'
+
+ try:
+ out = os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0644)
+ for p in sorted(excuses.values()):
+ os.write(out, str(p))
+ os.rename(tmp, OutFile)
+ os.close(out)
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ raise
+ else:
+ print "W: '%s' exists, skipping" % tmp
+ except:
+ if os.path.exists(tmp):
+ os.unlink(tmp)
+ raise
+
+###
+
+if __name__ == '__main__':
+ main()
Property changes on: ddpo-by-mail/dato-ood/ood_excuses.py
___________________________________________________________________
Added: svn:executable
+ *
More information about the Collab-qa-commits
mailing list