[Collab-qa-commits] r1596 - in udd: . scripts sql udd web
Lucas Nussbaum
lucas at alioth.debian.org
Wed Oct 7 19:41:34 UTC 2009
Author: lucas
Date: 2009-10-07 19:41:33 +0000 (Wed, 07 Oct 2009)
New Revision: 1596
Added:
udd/README.kill-idle
Modified:
udd/crontabs
udd/scripts/check_timestamps
udd/sql/setup.sql
udd/udd.py
udd/udd/bugs_gatherer.pl
udd/web/index.html
Log:
tons of minor UDD stuff
Added: udd/README.kill-idle
===================================================================
--- udd/README.kill-idle (rev 0)
+++ udd/README.kill-idle 2009-10-07 19:41:33 UTC (rev 1596)
@@ -0,0 +1,2 @@
+Kill all idle transactions:
+udd=> select * from dsa_kill_all_idle_transactions();
Modified: udd/crontabs
===================================================================
--- udd/crontabs 2009-10-05 20:22:13 UTC (rev 1595)
+++ udd/crontabs 2009-10-07 19:41:33 UTC (rev 1596)
@@ -6,31 +6,36 @@
0 3 * * * $UAR debian-etch debian-backports-etch debian-volatile-etch debian-etch-proposed-updates debian-etch-security
0 15 * * * $UAR debian-lenny debian-backports-lenny debian-volatile-lenny debian-lenny-proposed-updates debian-lenny-security debian-squeeze-proposed-updates debian-squeeze-security
# Ubuntu Sources/Packages
-30 1 * * * $UAR ubuntu-karmic
-30 2 * * * $UAR ubuntu-hardy ubuntu-intrepid ubuntu-jaunty
+30 2 * * * $UAR ubuntu-karmic ubuntu-hardy ubuntu-intrepid ubuntu-jaunty
# Various simple things
-0 4 * * * $UAR dehs debian-popcon ubuntu-popcon lintian debtags carnivore orphaned-packages upload-history ldap | /org/udd.debian.org/udd/scripts/filter-output.rb
+0 4 * * * $UAR dehs debian-popcon ubuntu-popcon lintian debtags carnivore ldap | /org/udd.debian.org/udd/scripts/filter-output.rb
+49 */6 * * * $UAR upload-history | /org/udd.debian.org/udd/scripts/filter-output.rb
0 */12 * * * $UAR testing-migrations
# Debian bugs
-0 1,7,13,19 * * * $UAR bugs 2>&1 | /org/udd.debian.org/udd/scripts/filter-bugs-output.sh
+0 1,7,13,19 * * * $UAR bugs orphaned-packages 2>&1 | /org/udd.debian.org/udd/scripts/filter-bugs-output.sh | /org/udd.debian.org/udd/scripts/filter-output.rb
0 15 * * * $UAR bugs-archive 2>&1 | /org/udd.debian.org/udd/scripts/filter-bugs-output.sh
-# DDTP
-0 2 * * * $UAR ddtp | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD ftpnew cron job' tille at debian.org
-
-# FTP new queue
-0 3 * * * $UAR ftpnew | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD ftpnew cron job' tille at debian.org
-
-# screenshots.debian.net
-30 3 * * * $UAR screenshots | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD screenshots cron job' tille at debian.org
-
# Ubuntu bugs
-0 4 * * * $UAR ubuntu-bugs
+0 20 * * * $UAR ubuntu-bugs
# Dump DB
0 8 */2 * * /org/udd.debian.org/udd/scripts/dump-db.sh
+0 4 */2 * * /org/udd.debian.org/udd/scripts/generate-schema.sh
# checks
*/5 * * * * /org/udd.debian.org/udd/scripts/monitor_locks
0 * * * * /org/udd.debian.org/udd/scripts/check_timestamps
0 * * * * /org/udd.debian.org/udd/scripts/check_timestamps > /org/udd.debian.org/udd/web/timestamps.txt
+0 * * * * crontab -l > /org/udd.debian.org/udd/web/crontabs.txt
+0 * * * * (cd /org/udd.debian.org/timestamps && ls -l --time-style=full-iso) > /org/udd.debian.org/udd/web/timing.txt
+
+
+MAILTO=tille at debian.org
+# screenshots.debian.net
+30 3 * * * $UAR screenshots | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD screenshots cron job' tille at debian.org
+
+# DDTP
+0 2 * * * $UAR ddtp | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD ftpnew cron job' tille at debian.org
+
+# FTP new queue
+0 3 * * * $UAR ftpnew | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD ftpnew cron job' tille at debian.org
Modified: udd/scripts/check_timestamps
===================================================================
--- udd/scripts/check_timestamps 2009-10-05 20:22:13 UTC (rev 1595)
+++ udd/scripts/check_timestamps 2009-10-07 19:41:33 UTC (rev 1596)
@@ -1,3 +1,3 @@
-ls -l --time-style=full-iso /org/udd.debian.org/timestamps/ |sort -n -k 6 | grep -v $(date --iso) | grep -v $(date --iso -d '1 day ago') | grep -v "^total"
+ls -l --time-style=full-iso /org/udd.debian.org/timestamps/*.insert-end |sort -n -k 6 | grep -v $(date --iso) | grep -v $(date --iso -d '1 day ago') | grep -v "^total"
ls -l --time-style=full-iso /org/bugs.debian.org/www/project/trace/ |sort -n -k 6 | grep -v $(date --iso) | grep -v $(date --iso -d '1 day ago') | grep -v "^total"
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2009-10-05 20:22:13 UTC (rev 1595)
+++ udd/sql/setup.sql 2009-10-07 19:41:33 UTC (rev 1596)
@@ -557,3 +557,10 @@
left join popcon_src ps on s.source = ps.source
where s.distribution='debian' and s.release='sid';
GRANT SELECT ON bapase TO PUBLIC;
+
+-- really active dds
+CREATE VIEW really_active_dds AS
+ SELECT DISTINCT carnivore_login.id, carnivore_login.login FROM carnivore_login, carnivore_keys, ldap WHERE ((((carnivore_keys.id = carnivore_login.id) AND (carnivore_keys.key_type = 'keyring'::text)) AND (carnivore_login.login = ldap.login)) AND (ldap.activity_pgp > '2009-01-01 00:00:00+00'::timestamp with time zone));
+GRANT SELECT ON TABLE really_active_dds TO guestdd;
+
+
Modified: udd/udd/bugs_gatherer.pl
===================================================================
--- udd/udd/bugs_gatherer.pl 2009-10-05 20:22:13 UTC (rev 1595)
+++ udd/udd/bugs_gatherer.pl 2009-10-07 19:41:33 UTC (rev 1596)
@@ -213,11 +213,12 @@
}
} qw{date log_modified};
+ my $srcpkg;
if ($bug{package} =~ /^src:(.*)/)
{
- my $source = $1;
+ $srcpkg = $1;
} else {
- my $source = exists($pkgsrc{$bug{package}}) ? $pkgsrc{$bug{package}} : $bug{package};
+ $srcpkg = exists($pkgsrc{$bug{package}}) ? $pkgsrc{$bug{package}} : $bug{package};
}
# split emails
@@ -298,14 +299,14 @@
}
# Insert data into bugs table
- $insert_bugs_handle->execute($bug_nr, $bug{package}, $source, $bug{date}, $bug{pending},
+ $insert_bugs_handle->execute($bug_nr, $bug{package}, $srcpkg, $bug{date}, $bug{pending},
$bug{severity}, $bug{originator}, $submitter_name, $submitter_email, $bug{owner},
$owner_name, $owner_email, $bug{done}, $done_name, $done_email, $bug{subject},
$bug{forwarded}, $bug{log_modified},
$present_in_stable, $present_in_testing, $present_in_unstable, $present_in_experimental) or die $!;
my $src;
- foreach my $pkg (keys %{{ map { $_ => 1 } split(/\s*,\s*/, $bug{package})}}) {
+ foreach my $pkg (keys %{{ map { $_ => 1 } split(/\s*[, ]\s*/, $bug{package})}}) {
if ($pkg =~ /^src:(.*)/)
{
$src = $1;
Modified: udd/udd.py
===================================================================
--- udd/udd.py 2009-10-05 20:22:13 UTC (rev 1595)
+++ udd/udd.py 2009-10-07 19:41:33 UTC (rev 1596)
@@ -46,15 +46,25 @@
# can just use the gatherer's methods
if command == 'update':
if "update-command" in src_config:
+ if 'timestamp-dir' in config['general']:
+ f = open(os.path.join(config['general']['timestamp-dir'],
+ src+".update-start"), "w")
+ f.write(asctime())
+ f.close()
result = system(src_config['update-command'])
if result != 0:
sys.exit(result)
if 'timestamp-dir' in config['general']:
f = open(os.path.join(config['general']['timestamp-dir'],
- src+".update"), "w")
+ src+".update-end"), "w")
f.write(asctime())
f.close()
else:
+ if 'timestamp-dir' in config['general']:
+ f = open(os.path.join(config['general']['timestamp-dir'],
+ src+".insert-start"), "w")
+ f.write(asctime())
+ f.close()
(src_command,rest) = types[type].split(None, 1)
if src_command == "exec":
system(rest + " " + sys.argv[1] + " " + sys.argv[2] + " " + src)
@@ -75,7 +85,7 @@
connection.commit()
if 'timestamp-dir' in config['general']:
f = open(os.path.join(config['general']['timestamp-dir'],
- src+".dispatch"), "w")
+ src+".insert-end"), "w")
f.write(asctime())
f.close()
except:
Modified: udd/web/index.html
===================================================================
--- udd/web/index.html 2009-10-05 20:22:13 UTC (rev 1595)
+++ udd/web/index.html 2009-10-07 19:41:33 UTC (rev 1596)
@@ -16,55 +16,25 @@
<a href="http://wiki.debian.org/UltimateDebianDatabase"><tt>wiki.debian.org/UltimateDebianDatabase</tt></a>.
<h2>Examples</h2>
- <p>To give an idea what results UDD can deliver, here are some example
- queries: (<a href="http://svn.debian.org/viewsvn/collab-qa/udd/web/cgi-bin/">source code</a>)</p>
+
+ <p>Some example queries are provided as CGI scripts, to make it easy for
+ everyone to run them. You can <a href="cgi-bin/">browse them</a> or <a
+ href="http://svn.debian.org/viewsvn/collab-qa/udd/web/cgi-bin/">view
+ the source code</a>.</p>
+
+ <p>There are also some more complex "applications" using UDD:</p>
<ul>
- <li>
- <a href="/cgi-bin/packages_in_unstable_but_not_in_testing_by_popcon.cgi">
- Packages in unstable, but not in testing, sorted by popcon installs
- </a>
- </li>
- <li>
- <a href="/cgi-bin/sources_in_unstable_but_not_in_testing_by_popcon_max.cgi">
- Sources in unstable, but not in testing, sorted by maximum popcon installs
- </a>
- </li>
- <li>
- <a href="/cgi-bin/sources_in_ubuntu_but_not_in_debian_by_popcon.cgi">
- Sources in Ubuntu, but not in Debian, sorted by maximum popcon installs
- </a>
- </li>
- <li>
- <a href="/cgi-bin/package_bug_count.cgi">
- Number of bugs per package
- </a>
- </li>
- <li>
- <a href="/cgi-bin/package_rc_bug_count.cgi">
- Number of RC bugs per package
- </a>
- </li>
- <li>
- <a href="/cgi-bin/removal_candidates.cgi">
- Packages which are "good" candidates for removal from testing
- </a>
- </li>
- <li>
- <a href="/cgi-bin/sources_in_etch_but_not_in_lenny.cgi">
- Sources in etch but not in lenny
- </a>
- </li>
- <li>
- <a href="/cgi-bin/sponsorstats.cgi">
- Statistics about sponsors, for packages currently in the archive
- </a>
- </li>
- <li>
- <a href="/cgi-bin/ubuntubugs.cgi">
- Open bugs per Ubuntu package
- </a>
- </li>
- </ul>
+ <li><a href="cgi-bin/bapase.cgi">Bapase</a> allows to search for "interesting" packages using various criterias</li>
+ <li><a href="cgi-bin/sponsorstats.cgi">SponsorStats</a> gives some statistics about who is sponsoring uploads to Debian</li>
+ </ul>
+
+ <h2>Timing information</h2>
+
+ <p>If you want to know when UDD is being updated, you can check the <a
+ href="crontabs.txt">crontab file</a>, or <a href="timing.txt">the
+ timestamps written during updates</a>.</p>
+
+ <h2>Database dump</h2>
<p>A full dump of the database, generated every two days, is <a href="udd.sql.gz">also available</a>. Be careful, it's quite big (~800 MB gzipped).</p>
</body>
</html>
More information about the Collab-qa-commits
mailing list