[Collab-qa-commits] r1525 - udd/udd

[Lucas Nussbaum]

Author: lucas
Date: 2009-07-17 02:59:27 +0000 (Fri, 17 Jul 2009)
New Revision: 1525

Modified:
   udd/udd/packages_gatherer.py
   udd/udd/sources_gatherer.py
Log:
optimize import using executemany()

Modified: udd/udd/packages_gatherer.py
===================================================================
--- udd/udd/packages_gatherer.py	2009-07-17 02:58:42 UTC (rev 1524)
+++ udd/udd/packages_gatherer.py	2009-07-17 02:59:27 UTC (rev 1525)
@@ -78,6 +78,16 @@
     Sequence has to have an iterator interface, that yields a line every time
     it is called.The Format of the sequence is expected to be that of a
     debian packages file."""
+    pkgs = ()
+    query = """EXECUTE package_insert
+      (%(Package)s, %(Version)s, %(Architecture)s, %(Maintainer)s,
+      %(Description)s, %(Long_Description)s, %(Source)s, %(Source_Version)s, %(Essential)s,
+      %(Depends)s, %(Recommends)s, %(Suggests)s, %(Enhances)s,
+      %(Pre-Depends)s, %(Breaks)s, %(Installed-Size)s, %(Homepage)s, %(Size)s,
+      %(Build-Essential)s, %(Origin)s, %(SHA1)s,
+      %(Replaces)s, %(Section)s, %(MD5sum)s, %(Bugs)s, %(Priority)s,
+      %(Tag)s, %(Task)s, %(Python-Version)s, %(Provides)s,
+      %(Conflicts)s, %(SHA256)s, %(Original-Maintainer)s)"""
     # The fields that are to be read. Other fields are ignored
     for control in debian_bundle.deb822.Packages.iter_paragraphs(sequence):
       # Check whether packages with architectue 'all' have already been
@@ -114,21 +124,13 @@
 	else:
 	  d['Source'] = split[0]
 	  d['Source_Version'] = split[1].strip("()")
+        pkgs += (d,)
 
-      query = """EXECUTE package_insert
-	  (%(Package)s, %(Version)s, %(Architecture)s, %(Maintainer)s,
-	  %(Description)s, %(Long_Description)s, %(Source)s, %(Source_Version)s, %(Essential)s,
-	  %(Depends)s, %(Recommends)s, %(Suggests)s, %(Enhances)s,
-	  %(Pre-Depends)s, %(Breaks)s, %(Installed-Size)s, %(Homepage)s, %(Size)s,
-	  %(Build-Essential)s, %(Origin)s, %(SHA1)s,
-	  %(Replaces)s, %(Section)s, %(MD5sum)s, %(Bugs)s, %(Priority)s,
-	  %(Tag)s, %(Task)s, %(Python-Version)s, %(Provides)s,
-	  %(Conflicts)s, %(SHA256)s, %(Original-Maintainer)s)"""
-      try:
-	cur.execute(query, d)
-      except psycopg2.ProgrammingError:
-	print query
-	raise
+    try:
+      cur.executemany(query, pkgs)
+    except psycopg2.ProgrammingError:
+      print query
+      raise
 
   def setup(self):
     if 'schema-dir' in self.config['general']:
@@ -201,11 +203,11 @@
         package, version, source, source_version, maintainer, distribution,
         release, component
       FROM %s""" % (table + '_summary', table));
-    cur.execute("DELETE FROM all_packages_distrelcomparch");
-    cur.execute("""INSERT INTO all_packages_distrelcomparch
+    cur.execute("DELETE FROM %s" % (table + '_distrelcomparch'));
+    cur.execute("""INSERT INTO %s
       (distribution, release, component, architecture)
       SELECT DISTINCT distribution, release, component, architecture
-      FROM all_packages""")
+      FROM %s""" % (table + '_distrelcomparch', table))
 
     self.print_warnings()
 

Modified: udd/udd/sources_gatherer.py
===================================================================
--- udd/udd/sources_gatherer.py	2009-07-17 02:58:42 UTC (rev 1524)
+++ udd/udd/sources_gatherer.py	2009-07-17 02:59:27 UTC (rev 1525)
@@ -92,20 +92,23 @@
     is called.The Format of the file is expected to be that of a debian
     source file."""
     cur = self.cursor()
+    pkgs = ()
+    query = """EXECUTE source_insert
+      (%(Package)s, %(Version)s, %(Maintainer)s,
+      %(maintainer_name)s, %(maintainer_email)s, %(Format)s, %(Files)s,
+      %(Uploaders)s, %(Binary)s, %(Architecture)s, %(Standards-Version)s,
+      %(Homepage)s, %(Build-Depends)s, %(Build-Depends-Indep)s,
+      %(Build-Conflicts)s, %(Build-Conflicts-Indep)s, %(Priority)s,
+      %(Section)s, %(Vcs-Type)s, %(Vcs-Url)s, %(Vcs-Browser)s,
+      %(Python-Version)s, %(Checksums-Sha1)s, %(Checksums-Sha256)s,
+      %(Original-Maintainer)s, %(Dm-Upload-Allowed)s)"""
+    query_uploaders = """EXECUTE uploader_insert (%(Package)s, %(Version)s, %(Name)s, %(Email)s)"""
+    uploaders = ()
     for control in debian_bundle.deb822.Packages.iter_paragraphs(file):
       d = self.build_dict(control)
       d['maintainer_name'], d['maintainer_email'] = email.Utils.parseaddr(d['Maintainer'])
-      query = """EXECUTE source_insert
-         (%(Package)s, %(Version)s, %(Maintainer)s,
-	  %(maintainer_name)s, %(maintainer_email)s, %(Format)s, %(Files)s,
-	  %(Uploaders)s, %(Binary)s, %(Architecture)s, %(Standards-Version)s,
-	  %(Homepage)s, %(Build-Depends)s, %(Build-Depends-Indep)s,
-	  %(Build-Conflicts)s, %(Build-Conflicts-Indep)s, %(Priority)s,
-	  %(Section)s, %(Vcs-Type)s, %(Vcs-Url)s, %(Vcs-Browser)s,
-	  %(Python-Version)s, %(Checksums-Sha1)s, %(Checksums-Sha256)s,
-	  %(Original-Maintainer)s, %(Dm-Upload-Allowed)s)
-	  """ 
-      cur.execute(query, d)
+      pkgs += (d,)
+
       ud = {}
       ud['Package'] = d['Package']
       ud['Version'] = d['Version']
@@ -113,9 +116,9 @@
         for uploader in email.Utils.getaddresses([d['Uploaders']]):
           ud['Name'] = uploader[0]
           ud['Email'] = uploader[1]
-	  query = """EXECUTE uploader_insert (%(Package)s, %(Version)s, %(Name)s,
-	    %(Email)s)"""
-	  cur.execute(query, ud)
+          uploaders += (ud,)
+    cur.executemany(query, pkgs)
+    cur.executemany(query_uploaders, uploaders)
 
   def tables(self):
     return [self.my_config['sources-table']]