[Collab-qa-commits] r1541 - in udd: . sql udd
Lucas Nussbaum
lucas at alioth.debian.org
Fri Jul 24 17:51:08 UTC 2009
Author: lucas
Date: 2009-07-24 17:51:07 +0000 (Fri, 24 Jul 2009)
New Revision: 1541
Added:
udd/udd/ldap_gatherer.py
Modified:
udd/config-org.yaml
udd/crontabs
udd/sql/setup.sql
udd/sql/upgrade.sql
Log:
LDAP importer. allows to get activity-from and activity-gpg, amongst others
Modified: udd/config-org.yaml
===================================================================
--- udd/config-org.yaml 2009-07-24 17:49:32 UTC (rev 1540)
+++ udd/config-org.yaml 2009-07-24 17:51:07 UTC (rev 1541)
@@ -10,7 +10,7 @@
popcon: module udd.popcon_gatherer
testing-migrations: module udd.testing_migrations_gatherer
upload-history: module udd.upload_history_gatherer
- orphaned_packages: module udd.orphaned_packages_gatherer
+ orphaned-packages: module udd.orphaned_packages_gatherer
bugs: exec DEBBUGS_CONFIG_FILE=/org/udd.debian.org/bugs/bugs.debian.org/etc/config perl /org/udd.debian.org/udd/udd/bugs_gatherer.pl
carnivore: module udd.carnivore_gatherer
lintian: module udd.lintian_gatherer
@@ -20,6 +20,7 @@
ftpnew: module udd.ftpnew_gatherer
screenshots: module udd.screenshot_gatherer
dehs: module udd.dehs_gatherer
+ ldap: module udd.ldap_gatherer
timestamp-dir: /org/udd.debian.org/timestamps
lock-dir: /org/udd.debian.org/locks
archs:
@@ -352,8 +353,8 @@
sources-table: sources
update-command: cd /org/udd.debian.org/bugs ; ./bugsync
-orphaned_packages:
- type: orphaned_packages
+orphaned-packages:
+ type: orphaned-packages
table: orphaned_packages
setup: orphaned_packages
unarchived-table: bugs
@@ -382,6 +383,9 @@
update-command: rm -f /org/udd.debian.org/mirrors/dehs.txt && wget -q http://dehs.alioth.debian.org/udd_dump.txt -O /org/udd.debian.org/mirrors/dehs.txt
path: /org/udd.debian.org/mirrors/dehs.txt
+ldap:
+ type: ldap
+
debtags:
type: debtags
update-command: rm -f /org/udd.debian.org/mirrors/debtags.txt && wget -q http://svn.debian.org/viewsvn/*checkout*/debtags/tagdb/tags -O /org/udd.debian.org/mirrors/debtags.txt
Modified: udd/crontabs
===================================================================
--- udd/crontabs 2009-07-24 17:49:32 UTC (rev 1540)
+++ udd/crontabs 2009-07-24 17:51:07 UTC (rev 1541)
@@ -9,20 +9,20 @@
30 1 * * * $UAR ubuntu-karmic
30 2 * * * $UAR ubuntu-hardy ubuntu-intrepid ubuntu-jaunty
# Various simple things
-0 4 * * * $UAR debian-popcon ubuntu-popcon upload-history lintian debtags carnivore orphaned_packages | /org/udd.debian.org/udd/scripts/filter-output.rb
+0 4 * * * $UAR dehs debian-popcon ubuntu-popcon lintian debtags carnivore orphaned-packages upload-history ldap | /org/udd.debian.org/udd/scripts/filter-output.rb
0 */12 * * * $UAR testing-migrations
# Debian bugs
0 1,7,13,19 * * * $UAR bugs 2>&1 | /org/udd.debian.org/udd/scripts/filter-bugs-output.sh
0 15 * * * $UAR bugs-archive 2>&1 | /org/udd.debian.org/udd/scripts/filter-bugs-output.sh
# DDTP
-0 2 * * * $UAR ddtp | /org/udd.debian.org/udd/scripts/filter-output.rb
+0 2 * * * $UAR ddtp | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD ftpnew cron job' tille at debian.org
# FTP new queue
-0 3 * * * $UAR ftpnew | /org/udd.debian.org/udd/scripts/filter-output.rb
+0 3 * * * $UAR ftpnew | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD ftpnew cron job' tille at debian.org
# screenshots.debian.net
-30 3 * * * $UAR screenshots | /org/udd.debian.org/udd/scripts/filter-output.rb
+30 3 * * * $UAR screenshots | /org/udd.debian.org/udd/scripts/filter-output.rb 2>&1 | mail -e -s 'UDD screenshots cron job' tille at debian.org
# Ubuntu bugs
0 4 * * * $UAR ubuntu-bugs
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2009-07-24 17:49:32 UTC (rev 1540)
+++ udd/sql/setup.sql 2009-07-24 17:51:07 UTC (rev 1541)
@@ -486,4 +486,22 @@
);
GRANT SELECT ON lintian TO PUBLIC;
-
+-- LDAP
+CREATE TABLE ldap (
+ uid numeric,
+ login text,
+ cn text,
+ sn text,
+ expire boolean,
+ location text,
+ country text,
+ activity_from timestamp with time zone,
+ activity_from_info text,
+ activity_gpg timestamp with time zone,
+ activity_gpg_info text,
+ gecos text,
+ birthdate date,
+ gender numeric,
+ PRIMARY KEY (uid)
+);
+GRANT SELECT ON ldap TO guestdd;
Modified: udd/sql/upgrade.sql
===================================================================
--- udd/sql/upgrade.sql 2009-07-24 17:49:32 UTC (rev 1540)
+++ udd/sql/upgrade.sql 2009-07-24 17:51:07 UTC (rev 1541)
@@ -216,3 +216,7 @@
ALTER TABLE archived_bugs add owner_email TEXT;
ALTER TABLE archived_bugs add done_name TEXT;
ALTER TABLE archived_bugs add done_email TEXT;
+
+-- LDAP
+CREATE TABLE ldap ( uid numeric, login text, cn text, sn text, expire boolean, location text, country text, activity_from timestamp with time zone, activity_from_info text, activity_gpg timestamp with time zone, activity_gpg_info text, gecos text, birthdate date, gender numeric, PRIMARY KEY (uid));
+GRANT SELECT ON ldap TO guestdd;
Added: udd/udd/ldap_gatherer.py
===================================================================
--- udd/udd/ldap_gatherer.py (rev 0)
+++ udd/udd/ldap_gatherer.py 2009-07-24 17:51:07 UTC (rev 1541)
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+"""
+This script imports some fields from the Debian LDAP into the database
+"""
+
+from aux import quote
+from gatherer import gatherer
+import ldap
+import re
+
+def get_gatherer(connection, config, source):
+ return ldap_gatherer(connection, config, source)
+
+class ldap_gatherer(gatherer):
+
+ def __init__(self, connection, config, source):
+ gatherer.__init__(self, connection, config, source)
+ self.assert_my_config()
+
+ def run(self):
+ #start harassing the DB, preparing the final inserts and making place
+ #for the new data:
+ cur = self.cursor()
+
+ cur.execute("DELETE FROM ldap")
+
+ cur.execute("""PREPARE ldap_insert
+ AS INSERT INTO ldap
+ (uid, login, cn, sn, expire, location, country, activity_from, activity_from_info, activity_gpg, activity_gpg_info, gecos, birthdate, gender)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)""")
+
+ entries = []
+ con = ldap.initialize('ldap://db.debian.org')
+ for e in con.search_s('ou=users,dc=debian,dc=org', ldap.SCOPE_SUBTREE, '(&(objectclass=debianAccount)(objectclass=debianDeveloper))'):
+ f = e[1]
+ if 'activity-from' in f:
+ af_date, af_info = f['activity-from'][0].split('] ',2)
+ af_date = af_date[1:]
+ else:
+ af_date = None
+ af_info = None
+ if 'activity-gpg' in f:
+ ag_date, ag_info = f['activity-gpg'][0].split('] ',2)
+ ag_date = ag_date[1:]
+ else:
+ ag_date = None
+ ag_info = None
+
+
+ birthdate = f['birthDate'][0] if 'birthDate' in f else None
+ gecos = f['gecos'][0] if 'gecos' in f else None
+ gender = int(f['gender'][0]) if 'gender' in f else None
+ loc = f['l'][0] if 'l' in f else None
+ country = f['c'][0] if 'c' in f else None
+ expired = ('shadowExpire' in f)
+
+ entries.append((int(f['uidNumber'][0]), f['uid'][0], f['cn'][0], f['sn'][0], expired, loc, country, af_date, af_info, ag_date, ag_info, gecos, birthdate, gender))
+
+ cur.executemany("EXECUTE ldap_insert (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", entries)
+ cur.execute("DEALLOCATE ldap_insert")
+ cur.execute("ANALYZE ldap")
+
+if __name__ == '__main__':
+ main()
+
+# vim:set et tabstop=2:
More information about the Collab-qa-commits
mailing list