[Collab-qa-commits] r1705 - ddpo-by-mail
Lucas Nussbaum
lucas at alioth.debian.org
Sun Feb 28 19:57:37 UTC 2010
Author: lucas
Date: 2010-02-28 19:57:36 +0000 (Sun, 28 Feb 2010)
New Revision: 1705
Added:
ddpo-by-mail/udddata.rb
Removed:
ddpo-by-mail/pkgtestingstatus.rb
ddpo-by-mail/testingissues.rb
Modified:
ddpo-by-mail/Makefile
ddpo-by-mail/ddpo-config.rb
ddpo-by-mail/reminder-mail.rb
Log:
use UDD for almost everything
Modified: ddpo-by-mail/Makefile
===================================================================
--- ddpo-by-mail/Makefile 2010-02-28 16:46:25 UTC (rev 1704)
+++ ddpo-by-mail/Makefile 2010-02-28 19:57:36 UTC (rev 1705)
@@ -1,6 +1,6 @@
#!/usr/bin/make -f
-UPDATE_TARGETS:=testing-status.txt dehs.txt pts_subscriptions.txt wnpp_rm lintian-list.txt di-sources.txt bugs_stable.txt bugs_unstable.txt security-packages.txt maintainers.txt uploaders.txt wannabuild.txt
+UPDATE_TARGETS:=dehs.txt wnpp_rm bugs_stable.txt bugs_unstable.txt security-packages.txt
ARCHITECTURES:=alpha,amd64,armel,hppa,hurd-i386,i386,ia64,kfreebsd-amd64,kfreebsd-i386,mips,mipsel,powerpc,s390,sparc
#ARCHITECTURES:=$(shell dpkg-architecture -L | tr "\n" ",")
LOCAL_MIRROR:=/org/ftp.debian.org/ftp
@@ -21,19 +21,6 @@
update: $(DATA_DIR)
@cd $(DATA_DIR) && $(MAKE) -f ../Makefile FORCE=phony $(UPDATE_TARGETS)
-maintainers.txt:
- ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, maintainer_email, maintainer_name from sources_uniq where distribution = 'debian' and release = 'sid';\" 'service=udd'" > $@
-
-uploaders.txt:
- ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, email, name from uploaders where distribution = 'debian' and release = 'sid';\" 'service=udd'" > $@
-
-testing-status.txt: $(FORCE)
- ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, in_testing, current_date - in_testing as in_testing_age, sync, current_date - sync as sync_age, current_date - first_seen as debian_age from migrations where current_date - in_unstable < 2 and (sync is null or current_date - sync > 1);\" 'service=udd'" > $@
-
-# FIXME temporarily disabled mips and mipsel
-wannabuild.txt:
- ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source, architecture from wannabuild where distribution='unstable' and state not in ('Installed', 'Needs-Build', 'Dep-Wait', 'Not-For-Us') and (state not in ('Built', 'Uploaded') or now() - state_change > interval '3 days') and architecture not in ('hurd-i386', 'mips', 'mipsel') and notes <> 'uncompiled';;\" 'service=udd'" > $@
-
ood/update_output.txt ood/update_excuses.html: ood $(FORCE)
cd ood/ && $(WGET) http://release.debian.org/britney/$(subst ood/,,$@)
@@ -47,21 +34,12 @@
[ -L ddpomail.txt ] || ln -s $@ ddpomail.txt
$(WGET) http://dehs.alioth.debian.org/ddpomail.txt
-pts_subscriptions.txt: $(FORCE)
- -ssh master /org/packages.qa.debian.org/bin/get-summary-subscribers.pl > $@
-
wnpp_rm: $(FORCE)
$(WGET) http://qa.debian.org/data/bts/$@
-lintian-list.txt: $(FORCE)
- $(WGET) http://qa.debian.org/data/$@
-
security-packages.txt: $(FORCE)
$(WGET) -O$@ http://security-tracker.debian.org/tracker/data/pts/1
-di-sources.txt:
- ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -At -c \"select source from sources_uniq where distribution = 'debian' and release = 'sid' and source not in (select distinct source from packages where distribution = 'debian' and release = 'sid' and component not like '%/debian-installer');\" 'service=udd'" > $@
-
bugs_unstable.txt: $(FORCE)
ssh -oBatchMode=yes $(SSH_UDD_ACCESS_HOST) "psql -Atx -c \"SELECT b.id, b.source, b.title, b.arrival, b.severity, b.affects_unstable, b.affects_stable, bmw.merged_with, bt.tag, but.tag AS usertag, but.email FROM bugs_rt_affects_unstable, bugs AS b FULL JOIN bugs_merged_with AS bmw ON b.id = bmw.id FULL JOIN bugs_tags AS bt ON b.id = bt.id FULL JOIN bugs_usertags AS but ON b.id = but.id WHERE (bugs_rt_affects_unstable.id = b.id) AND b.status <> 'done' AND (b.severity > 'important' OR (bt.tag = '$(subst ;,' OR bt.tag = ',$(TAGS))') OR (but.email = '$(subst ;,' OR but.email = ',$(USERTAGS_EMAILS))'));\" 'service=udd'" > $@
Modified: ddpo-by-mail/ddpo-config.rb
===================================================================
--- ddpo-by-mail/ddpo-config.rb 2010-02-28 16:46:25 UTC (rev 1704)
+++ ddpo-by-mail/ddpo-config.rb 2010-02-28 19:57:36 UTC (rev 1705)
@@ -101,9 +101,9 @@
EXC_EMAILS_PKGS['mail.to.depa at gmail.com'] = ['icedove']
BUG_AGE = 2
-AGE_IN_DEBIAN = 30
-LAST_IN_SYNC = 40
-LAST_IN_SYNC_SERIOUS = 50
+AGE_IN_DEBIAN = 20
+LAST_IN_SYNC = 20
+LAST_IN_SYNC_SERIOUS = 30
# etch released on 2007-04-08. So
# (Date::today - Date::parse('2007-04-08')).to_i
Deleted: ddpo-by-mail/pkgtestingstatus.rb
===================================================================
--- ddpo-by-mail/pkgtestingstatus.rb 2010-02-28 16:46:25 UTC (rev 1704)
+++ ddpo-by-mail/pkgtestingstatus.rb 2010-02-28 19:57:36 UTC (rev 1705)
@@ -1,145 +0,0 @@
-require 'date'
-
-class PkgTestingStatus
- @@datezero = Date::parse('0000-01-01')
- @@curdate = Date::today
-
- attr_accessor :intesting, :testingversion, :inunstable, :unstableversion, :sync, :syncversion, :firstinunstable
- def initialize(t,tv,u,uv, s, sv, fu)
- @intesting = t
- @testingversion = tv
- @inunstable = u
- @unstableversion = uv
- @sync = s
- @syncversion = sv
- @firstinunstable = fu
- end
-
- def to_s
- "#{@intesting} #{@testingversion} #{@inunstable} #{@unstableversion} #{@sync} #{@syncversion} #{@firstinunstable}"
- end
-
- def testing_s
- if @intesting == @@datezero
- d = 'never'
- v = ''
- days = ''
- else
- d = @intesting
- v = @testingversion
- days = @@curdate - @intesting
- end
- return "<td>#{d}</td><td>#{days}</td><td>#{v}</td>"
- end
-
- def sync_s
- if @sync == @@datezero
- d = 'never'
- v = ''
- days = ''
- else
- d = @sync
- v = @syncversion
- days = @@curdate - @sync
- end
- return "<td>#{d}</td><td>#{days}</td><td>#{v}</td>"
- end
-
- def testingdays
- if @intesting != @@datezero
- return @@curdate - @intesting
- else
- return (@@curdate - @firstinunstable)
- end
- end
-
- def testing_ok?(days)
- return [true, nil] if @@curdate - @firstinunstable < days
- return [true, nil] if @@curdate - @intesting < 2
-
- if @intesting != @@datezero
- return [false, @@curdate - @intesting ]
- else
- return [false, @@curdate - @firstinunstable ]
- end
- end
-
- def sync_ok?(days)
- if @sync != @@datezero
- if @@curdate - @sync > days
- return [false, @@curdate - @sync]
- else
- return [true, nil]
- end
- else
- if @@curdate - @firstinunstable > days
- return [false, @@curdate - @firstinunstable ]
- else
- return [true, nil]
- end
- end
- end
-
- def syncdays
- if @sync != @@datezero
- return @@curdate - @sync
- else
- return (@@curdate - @firstinunstable)
- end
- end
-
- def to_row(pkg, orphaned = nil, comment = nil)
- s = "<tr><td><a href=\"http://packages.qa.debian.org/#{pkg}\">#{pkg}</a>"
- s += "<a href=\"http://bugs.debian.org/#{orphaned}\">(O)</a>" if orphaned
- s += "</td>"
- s += "#{testing_s}#{sync_s}"
- s += "<td>#{@unstableversion}</td><td>#{@firstinunstable}</td><td>"
- s += comment if comment
- s += "</td></tr>"
- return s
- end
-
- def update(curdate, testing, unstable)
-# STDERR.puts "#{curdate} #{testing} #{unstable}" if testing != unstable
- if testing
- if curdate >= @intesting
- @intesting = curdate
- @testingversion = testing
- else
- STDERR.puts "[testing] #{curdate} < #{@intesting}, skipping"
- end
- end
- if unstable
- if curdate >= @inunstable
- @inunstable = curdate
- @unstableversion = unstable
- else
- STDERR.puts "[unstable] #{curdate} < #{@inunstable}, skipping"
- end
- end
- if unstable and testing and unstable == testing
- if curdate >= @sync
- @sync = curdate
- @syncversion = unstable
- else
- STDERR.puts "[sync] #{curdate} < #{@sync}, skipping"
- end
- end
- end
-
- def PkgTestingStatus::read(io)
- pkgs = {}
- io.read.each_line do |l|
- pkg, t, tv, u, uv, s, sv, fu = l.split('|')
- t = '0000-01-01' if t.empty?
- u = '0000-01-01' if u.empty?
- s = '0000-01-01' if s.empty?
- fu = '0000-01-01' if fu.empty?
- pkgs[pkg] = PkgTestingStatus::new(Date::parse(t), tv,
- Date::parse(u), uv,
- Date::parse(s), sv,
- Date::parse(fu))
- end
- return pkgs
- end
-end
Modified: ddpo-by-mail/reminder-mail.rb
===================================================================
--- ddpo-by-mail/reminder-mail.rb 2010-02-28 16:46:25 UTC (rev 1704)
+++ ddpo-by-mail/reminder-mail.rb 2010-02-28 19:57:36 UTC (rev 1705)
@@ -1,12 +1,17 @@
-#!/usr/bin/ruby -w
+#!/usr/bin/ruby
puts "#{Time::now} Starting and reading the data we need"
require 'ddpo-config'
require 'bugs'
-require 'testingissues'
require 'dehs'
+require 'udddata'
+udd = UDDData::new
+udd.fetch
+
+puts "#{Time::now} Loaded data from UDD"
+
# Data.
# $dipkgs: list of packages that don't have binary packages in !d-i
# $email_maintains: hash of email => maintained pkgs
@@ -17,77 +22,32 @@
# parse debian-installer packages
-$dipkgs = IO::readlines('data/di-sources.txt').map { |l| l.chomp }
+$dipkgs = udd.debian_installer_sources
# Parse maintainers.txt and uploaders.txt
$email_maintains = {}
$email_uploads = {}
-$email_realname = {}
+$email_realname = udd.names
$pkg_maintlist = {}
-$pkg_maintainer = {}
-IO::readlines('data/maintainers.txt').each do |l|
- l.chomp!
- src, email, name = l.split('|')
- if $email_maintains[email].nil?
- $email_maintains[email] = [ src ]
- else
- $email_maintains[email] << src
- end
+$pkg_maintainer = udd.maintainers
+
+udd.maintainers.each_pair do |src, email|
+ $email_maintains[email] ||= []
+ $email_maintains[email] << src
$pkg_maintlist[src] = true if email =~ /@list.*\.debian\.org$/
- $pkg_maintainer[src] = email
- if name != nil
- name.gsub!(/í/, 'i')
- $email_realname[email] = name
- end
end
-IO::readlines('data/uploaders.txt').each do |l|
- l.chomp!
- src, email, name = l.split('|')
- if $email_uploads[email].nil?
- $email_uploads[email] = [ src ]
- else
+udd.uploaders.each_pair do |src, emails|
+ emails.each do |email|
+ $email_uploads[email] ||= []
$email_uploads[email] << src
+ $pkg_maintlist[src] = true if email =~ /@list.*\.debian\.org$/
end
- $pkg_maintlist[src] = true if email =~ /@list.*\.debian\.org$/
- if name != nil
- name.gsub!(/í/, 'i')
- $email_realname[email] = name
- end
end
-# read PTS subscribers
-subscribers = Hash::new { [] }
-f = IO::read('data/pts_subscriptions.txt')
-f.each_line do |l|
- l.chomp!
- pkg, subs = l.split(/\t/, 2)
- subs.split(/\, /).each do |s|
- if subscribers[s].length > 0
- subscribers[s] << pkg
- else
- subscribers[s] = [ pkg ]
- end
- end
-end
+subscribers = udd.pts
+builds = udd.missing_builds
+lintian = udd.lintian
-# Get missing builds
-=begin
-builds = {}
-IO::read('data/ood_excuses.txt').each_line do |l|
- pkg, ver, archs = l.chomp.split(' ')
- builds[pkg] = archs
-end
-=end
-builds = {}
-IO::readlines('data/wannabuild.txt').each do |l|
- src, arch = l.chomp.split('|')
- if builds[src].nil?
- builds[src] = [ arch ]
- else
- builds[src] << arch
- end
-end
-
# get removals
rms = []
IO::read('data/wnpp_rm').each_line do |l|
@@ -97,12 +57,6 @@
end
end
-# get lintian report
-lintian = {}
-IO::read('data/lintian-list.txt').each_line do |l|
- pkg, errors, warnings = l.chomp.split(' ')
- lintian[pkg] = [errors.to_i, warnings.to_i]
-end
# get security issues
security = {}
@@ -114,13 +68,34 @@
=end
bugs = parse_bugs
-testingissues = testing_issues($dipkgs)
dehs = get_dehs
+# compute testing issues
+testingissues = {}
+udd.testing_status.each_pair do |pkg, v|
+ next if $dipkgs.include?(pkg)
+ if v['in_testing_age'].nil?
+ if v['debian_age'] > AGE_IN_DEBIAN
+ testingissues[pkg] = [:not_testing, true, v['debian_age'] ]
+ end
+ elsif v['in_testing_age'] > 1 # in case there's some incoherency in udd
+ testingissues[pkg] = [:not_testing, true, v['in_testing_age'] ]
+ else
+ if v['sync_age'].nil?
+ # puts "Interesting buggy case with #{pkg}. Ignore."
+ elsif v['sync_age'] > LAST_IN_SYNC_SERIOUS
+ testingissues[pkg] = [:no_migration, true, v['sync_age'] ]
+ elsif v['sync_age'] > LAST_IN_SYNC
+ testingissues[pkg] = [:no_migration, false, v['sync_age'] ]
+ end
+ end
+end
+
puts "#{Time::now} Computing issues for each package ..."
# compute issues for each package
serious = {}
issues = {}
+
(bugs.keys + testingissues.keys + dehs.keys + security.keys).sort.uniq.each do |pkg|
next if $pkg_maintainer[pkg].nil? # means the package wasn't in unstable Sources!
@@ -167,7 +142,7 @@
FileUtils::mkdir MAIL_DIR
end
-# prevent typos by using constants, so that it breask :)
+# prevent typos by using constants, so that it breaks :)
T_INCLUDE = "include"
T_HEADER = "header"
T_BUGS = "bugs"
Deleted: ddpo-by-mail/testingissues.rb
===================================================================
--- ddpo-by-mail/testingissues.rb 2010-02-28 16:46:25 UTC (rev 1704)
+++ ddpo-by-mail/testingissues.rb 2010-02-28 19:57:36 UTC (rev 1705)
@@ -1,39 +0,0 @@
-# parse testing status
-require 'ddpo-config'
-
-def testing_issues(dipkgs)
-# file = Dir::glob('../testing-status/data.*').sort[-1]
- file = 'data/testing-status.txt'
- testingissues = {}
- IO::readlines('data/testing-status.txt').each do |l|
- l.chomp!
- pkg, in_testing, in_testing_age, sync, sync_age, debian_age = l.split('|')
- in_testing_age = (in_testing_age != "")?in_testing_age.to_i : nil
- sync_age = (sync_age != "")?sync_age.to_i : nil
- debian_age = (debian_age != "")?debian_age.to_i : nil
-# p [pkg, in_testing, in_testing_age, sync, sync_age, debian_age]
- next if dipkgs.include?(pkg)
- if in_testing_age.nil?
- if debian_age > AGE_IN_DEBIAN
- testingissues[pkg] = [:not_testing, true, debian_age ]
- end
- elsif in_testing_age > 1 # in case there's some incoherency in udd
- testingissues[pkg] = [:not_testing, true, in_testing_age ]
- else
- if sync_age.nil?
-# puts "Interesting buggy case with #{pkg}. Ignore."
- elsif sync_age > LAST_IN_SYNC_SERIOUS
- testingissues[pkg] = [:no_migration, true, sync_age ]
- elsif sync_age > LAST_IN_SYNC
- testingissues[pkg] = [:no_migration, false, sync_age ]
- end
- end
- end
- return testingissues
-end
-
-if $0 == __FILE__
- ti = testing_issues([])
- p ti
-end
-
Added: ddpo-by-mail/udddata.rb
===================================================================
--- ddpo-by-mail/udddata.rb (rev 0)
+++ ddpo-by-mail/udddata.rb 2010-02-28 19:57:36 UTC (rev 1705)
@@ -0,0 +1,75 @@
+#!/usr/bin/ruby
+require 'dbi'
+require 'pp'
+
+class UDDData
+ attr_reader :maintainers, :uploaders, :names, :testing_status, :missing_builds, :lintian, :debian_installer_sources, :pts
+
+ def initialize
+ @dbh = DBI::connect('DBI:Pg:dbname=udd;port=5441;host=localhost', 'guestdd', ENV['dbpw'])
+ @maintainers = {}
+ @uploaders = Hash::new { [] }
+ @names = {}
+ @testing_status = {}
+ @missing_builds = {}
+ @lintian = {}
+ @debian_installer_sources = []
+ @pts = Hash::new { [] }
+ end
+
+ def fetch
+ # maintainers
+ @dbh.select_all("select source, maintainer_email, maintainer_name from sources_uniq where distribution = 'debian' and release = 'sid'") do |r|
+ @maintainers[r['source']] = r['maintainer_email']
+ @names[r['maintainer_email']] = r['maintainer_name']
+ end
+ @dbh.select_all("select source, email, name from uploaders where distribution = 'debian' and release = 'sid'") do |r|
+ @uploaders[r['source']] = [] if @uploaders[r['source']].empty?
+ @uploaders[r['source']] << r['email']
+ if @names.has_key?(r['email'])
+ @names[r['email']]=r['name']
+ end
+ end
+ # testing status
+ @dbh.select_all("select source, in_testing, current_date - in_testing as in_testing_age, sync, current_date - sync as sync_age, current_date - first_seen as debian_age from migrations where current_date - in_unstable < 2 and (sync is null or current_date - sync > 1)") do |r|
+ @testing_status[r['source']] = {}
+ @testing_status[r['source']]['in_testing_age'] = r['in_testing_age']
+ @testing_status[r['source']]['in_testing'] = r['in_testing'] ? r['in_testing'].to_date : nil
+ @testing_status[r['source']]['debian_age'] = r['debian_age']
+ @testing_status[r['source']]['sync'] = r['sync'] ? r['sync'].to_date : nil
+ @testing_status[r['source']]['sync_age'] = r['sync_age']
+ end
+ # wannabuild
+ @dbh.select_all("select source, architecture from wannabuild where distribution='unstable' and state not in ('Installed', 'Needs-Build', 'Dep-Wait', 'Not-For-Us') and (state not in ('Built', 'Uploaded') or now() - state_change > interval '3 days') and architecture not in ('hurd-i386') and notes <> 'uncompiled'") do |r|
+ @missing_builds[r['source']] ||= []
+ @missing_builds[r['source']] << r['architecture']
+ end
+ # lintian
+ h = {}
+ @dbh.select_all("select COALESCE(p.source, lintian.package) as source, tag_type, count(*) as cnt from lintian left join (select distinct source, package from packages_summary where distribution='debian' and release='sid') p on (lintian.package = p.package) where tag_type in ('warning', 'error') group by COALESCE(p.source, lintian.package), tag_type") do |r|
+ h[r['source']] = {} if h[r['source']].nil?
+ h[r['source']][r['tag_type']] = r['cnt']
+ end
+ h.keys.each do |k|
+ @lintian[k] = [ h[k]['error'] || 0, h[k]['warning'] || 0 ]
+ end
+ # debian-installer sources
+ @dbh.select_all("select source from sources_uniq where distribution = 'debian' and release = 'sid' and source not in (select distinct source from packages where distribution = 'debian' and release = 'sid' and component not like '%/debian-installer')") do |r|
+ @debian_installer_sources << r['source']
+ end
+ # PTS
+ @dbh.select_all("select source, email from pts") do |r|
+ @pts[r['email']] = [] if @pts[r['email']].empty?
+ @pts[r['email']] << r['source']
+ end
+ end
+end
+
+if $0 == __FILE__
+ udd = UDDData::new
+ t = Time::now
+ udd.fetch
+ te = Time::now
+ pp udd.pts
+ puts te - t
+end
More information about the Collab-qa-commits
mailing list