[Collab-qa-commits] r2150 - in udd: sql udd
Andreas Tille
tille at alioth.debian.org
Thu Feb 16 21:34:41 UTC 2012
Author: tille
Date: 2012-02-16 21:34:41 +0000 (Thu, 16 Feb 2012)
New Revision: 2150
Modified:
udd/sql/setup.sql
udd/sql/upgrade.sql
udd/udd/ddtp_gatherer.py
Log:
Finally we need to store also the component in descriptions table even if we can definitely identify the description via md5 sum, but handling the import of different files for different components makes keeping this date necessary
Modified: udd/sql/setup.sql
===================================================================
--- udd/sql/setup.sql 2012-02-15 12:15:39 UTC (rev 2149)
+++ udd/sql/setup.sql 2012-02-16 21:34:41 UTC (rev 2150)
@@ -604,6 +604,7 @@
CREATE TABLE descriptions (
package text,
release text,
+ component text,
language text,
description text,
long_description text,
@@ -615,6 +616,7 @@
CREATE TABLE ubuntu_descriptions (
package text,
release text,
+ component text,
language text,
description text,
long_description text,
@@ -626,6 +628,7 @@
CREATE TABLE derivatives_descriptions (
package text,
release text,
+ component text,
language text,
description text,
long_description text,
Modified: udd/sql/upgrade.sql
===================================================================
--- udd/sql/upgrade.sql 2012-02-15 12:15:39 UTC (rev 2149)
+++ udd/sql/upgrade.sql 2012-02-16 21:34:41 UTC (rev 2150)
@@ -414,3 +414,45 @@
import_date timestamp default now(),
PRIMARY KEY (release, component, language)
);
+
+-- 2012-02-16
+-- It is necessary to add component and we simply recreate the table to
+-- have a reasonable sequence of columns
+DROP TABLE descriptions;
+CREATE TABLE descriptions (
+ package text,
+ release text,
+ component text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON descriptions TO PUBLIC;
+
+DROP TABLE ubuntu_descriptions;
+CREATE TABLE ubuntu_descriptions (
+ package text,
+ release text,
+ component text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON ubuntu_descriptions TO PUBLIC;
+
+DROP TABLE derivatives_descriptions;
+CREATE TABLE derivatives_descriptions (
+ package text,
+ release text,
+ component text,
+ language text,
+ description text,
+ long_description text,
+ description_md5 text, -- md5 sum of the original English description
+ PRIMARY KEY (package, release, language, description, description_md5)
+);
+GRANT SELECT ON derivatives_descriptions TO PUBLIC;
Modified: udd/udd/ddtp_gatherer.py
===================================================================
--- udd/udd/ddtp_gatherer.py 2012-02-15 12:15:39 UTC (rev 2149)
+++ udd/udd/ddtp_gatherer.py 2012-02-16 21:34:41 UTC (rev 2150)
@@ -36,9 +36,10 @@
return ddtp_gatherer(connection, config, source)
class ddtp():
- def __init__(self, package, release, language):
+ def __init__(self, package, release, component, language):
self.package = package
self.release = release
+ self.component = component
self.language = language
self.description = ''
self.long_description = ''
@@ -68,18 +69,18 @@
self.log.addHandler(handler)
cur = self.cursor()
- query = "PREPARE ddtp_delete (text, text) AS DELETE FROM %s WHERE release = $1 AND language = $2" % my_config['table']
+ query = "PREPARE ddtp_delete (text, text) AS DELETE FROM %s WHERE release = $1 AND component = $2 AND language = $3" % my_config['table']
cur.execute(query)
query = """PREPARE ddtp_insert AS INSERT INTO %s
- (package, release, language, description, long_description, description_md5)
- VALUES ($1, $2, $3, $4, $5, $6)""" % (my_config['table'])
+ (package, release, component, language, description, long_description, description_md5)
+ VALUES ($1, $2, $3, $4, $5, $6, $7)""" % (my_config['table'])
cur.execute(query)
- query = """PREPARE ddtp_check_before_insert (text, text, text, text, text) AS
+ query = """PREPARE ddtp_check_before_insert (text, text, text, text, text, text) AS
SELECT COUNT(*) FROM %s
- WHERE package = $1 AND release = $2 AND language = $3 AND
- description = $4 AND description_md5 = $5""" % (my_config['table'])
+ WHERE package = $1 AND release = $2 AND component = $3 AND language = $4 AND
+ description = $5 AND description_md5 = $6""" % (my_config['table'])
cur.execute(query)
query = """PREPARE ddtp_check_previous_import (text, text, text) AS
@@ -148,7 +149,7 @@
if prev_import[0] == sha1:
self.log.info("File %s was imported at %s and has not changed since then" % (trfile, prev_import[1]))
continue
- self.import_translations(trfile, rel, lang)
+ self.import_translations(trfile, rel, comp, lang)
if has_previous_import == True:
cur.execute("EXECUTE ddtp_update_current_import (%s, %s, %s)", (rel, comp, lang))
else:
@@ -161,14 +162,14 @@
cur.execute("ANALYZE %s" % my_config['table'])
- def import_translations(self, trfile, rel, lang):
- self.log.info("Importing file %s for release %s in language %s" % (trfile, rel, lang))
+ def import_translations(self, trfile, rel, comp, lang):
+ self.log.info("Importing file %s for release %s, component %s in language %s" % (trfile, rel, comp, lang))
cur = self.cursor()
# Delete only records where we actually have Translation files. This
# prevents dump deletion of all data in case of broken downloads
- cur.execute('EXECUTE ddtp_delete (%s, %s)', (rel, lang))
- self.log.debug('EXECUTE ddtp_delete (%s, %s)', (rel, lang))
+ cur.execute('EXECUTE ddtp_delete (%s, %s, %s)', (rel, comp, lang))
+ self.log.debug('EXECUTE ddtp_delete (%s, %s, %s)', (rel, comp, lang))
i18n_error_flag=0
descstring = 'Description-'+lang
@@ -177,7 +178,7 @@
for stanza in deb822.Sources.iter_paragraphs(g, shared_storage=False):
if i18n_error_flag == 1:
continue
- self.pkg = ddtp(stanza['package'], rel, lang)
+ self.pkg = ddtp(stanza['package'], rel, comp, lang)
self.pkg.description_md5 = stanza['Description-md5']
try:
desc = stanza[descstring]
@@ -195,24 +196,24 @@
for line in lines[1:]:
self.pkg.long_description += line + "\n"
- paramtuple = (self.pkg.package, self.pkg.release, self.pkg.language, self.pkg.description, self.pkg.description_md5)
- cur.execute('EXECUTE ddtp_check_before_insert (%s, %s, %s, %s, %s)', paramtuple)
+ paramtuple = (self.pkg.package, self.pkg.release, self.pkg.component, self.pkg.language, self.pkg.description, self.pkg.description_md5)
+ cur.execute('EXECUTE ddtp_check_before_insert (%s, %s, %s, %s, %s, %s)', paramtuple)
if cur.fetchone()[0] > 0:
- self.log.error("Duplicated key for package %s in release %s in language %s: %s / %s" % paramtuple)
+ self.log.error("Duplicated key for package %s in release %s component %s in language %s: %s / %s" % paramtuple)
else:
- query = 'EXECUTE ddtp_insert (%s, %s, %s, %s, %s, %s)'
+ query = 'EXECUTE ddtp_insert (%s, %s, %s, %s, %s, %s, %s)'
try:
self.log.debug(query, tuple([quote(item) for item in paramtuple]))
- cur.execute(query, (self.pkg.package, self.pkg.release, self.pkg.language, self.pkg.description, self.pkg.long_description, self.pkg.description_md5))
+ cur.execute(query, (self.pkg.package, self.pkg.release, self.pkg.component, self.pkg.language, self.pkg.description, self.pkg.long_description, self.pkg.description_md5))
# self.connection.commit() # commit every single insert as long as translation files are featuring duplicated keys
except IntegrityError, err:
self.log.exception("Duplicated key in language %s: (%s)", self.pkg.language,
- ", ".join([to_unicode(item) for item in (self.pkg.package, self.pkg.release, self.pkg.description, self.pkg.description_md5)]))
+ ", ".join([to_unicode(item) for item in (self.pkg.package, self.pkg.release, self.pkg.component, self.pkg.description, self.pkg.description_md5)]))
self.connection.rollback()
continue
except ProgrammingError, err:
self.log.exception("Problem inserting translation %s: (%s)", self.pkg.language,
- ", ".join([to_unicode(item) for item in (self.pkg.package, self.pkg.release, self.pkg.description, self.pkg.description_md5)]))
+ ", ".join([to_unicode(item) for item in (self.pkg.package, self.pkg.release, self.pkg.component, self.pkg.description, self.pkg.description_md5)]))
self.connection.rollback()
continue
except IOError, err:
More information about the Collab-qa-commits
mailing list