[med-svn] r1399 - trunk/community/infrastructure/scripts
tille at alioth.debian.org
tille at alioth.debian.org
Sun Feb 17 07:36:40 UTC 2008
Author: tille
Date: 2008-02-17 07:36:40 +0000 (Sun, 17 Feb 2008)
New Revision: 1399
Added:
trunk/community/infrastructure/scripts/cddtasktools.py
trunk/community/infrastructure/scripts/update-tasks
trunk/community/infrastructure/scripts/update-tasks_using_Tools
Removed:
trunk/community/infrastructure/scripts/update-tasks
Log:
Putting the new version of update_tasks into operation.
Added: trunk/community/infrastructure/scripts/cddtasktools.py
===================================================================
--- trunk/community/infrastructure/scripts/cddtasktools.py (rev 0)
+++ trunk/community/infrastructure/scripts/cddtasktools.py 2008-02-17 07:36:40 UTC (rev 1399)
@@ -0,0 +1,453 @@
+#!/usr/bin/python
+# Copyright 2008: Andreas Tille <tille at debian.org>
+# License: GPL
+
+# CDD Meta packages are listing a set of Dependencies
+# These might be fullfilled by the Debian package
+# set or not.
+#
+# This interface provides some classes that contains
+# all available information about this Dependency ,
+# like whether it is an official package or not,
+# in which distribution it is contained
+# or if it is not contained it obtains information
+# from tasks file about home page, license, WNPP etc.
+
+import sys
+import os
+import urllib
+import StringIO
+import gzip
+import re
+
+from debian_bundle import deb822
+
+BASEURL = 'http://ftp.debian.org/'
+REPOS = { 'debian-med' : "svn://svn.debian.org/svn/cdd/projects/med/trunk/debian-med/tasks/",
+ 'debian-edu' : "svn://svn.debian.org/svn/debian-edu/trunk/src/debian-edu/tasks/",
+ 'debian-science' : "svn://svn.debian.org/svn/cdd/projects/science/trunk/debian-science/tasks/",
+ }
+HTMLBASE = "/var/lib/gforge/chroot/home/groups"
+
+def InitTasksFiles(cddname):
+ # Obtain tasks files from SVN of a CDD
+ # cddname can be: debian-med, debian-edu, debian-science
+ # In case you know another CDD that uses the meta package
+ # technology make sure to include the location in the
+ # REPOS dictionary
+ #
+ tasksdir = "%s/%s/data/tasks" % (HTMLBASE, cddname)
+ # Checkout/Update tasks from SVN
+ if os.path.isdir(tasksdir+'/.svn'):
+ os.system("svn up %s %s >> /dev/null" % (REPOS[cddname], tasksdir))
+ else:
+ os.system("mkdir -p %s" % (tasksdir))
+ os.system("svn co %s %s >> /dev/null" % (REPOS[cddname], tasksdir))
+ return tasksdir
+
+def SplitDescription(description):
+ # Split first line of Description value as short description
+
+ lines = description.splitlines()
+ ShortDesc = lines[0]
+ LongDesc = ''
+ for line in lines[1:]:
+ line = line.strip()
+ # Replace paragraph separators by <br />
+ if re.compile("^\s?\.\s*$").search(line):
+ LongDesc += "<br />\n"
+ else:
+ # This is to sanitize output for XHTML pages. --David
+ line = line.replace("&", "&").replace("<", "<").replace(">", ">") + ' \n'
+ LongDesc += re.sub('([fh]t?tp://[-./\w?=~]+)', '<a href="\\1">\\1</a>', line)
+ return (ShortDesc, LongDesc)
+
+
+class Task:
+ # This class just stores name and description of a task package
+ # FIXME: This is probably not used anymore and can be deleted
+
+ def __init__(self, cddname=None, taskname=None, shortDesc=None, longDesc=None):
+ self.cddname = cddname
+ self.taskname = taskname
+ self.shortDesc = shortDesc
+ self.longDesc = longDesc
+
+class DependantPackage:
+ # Hold information about a program that is in dependency list
+ # The
+
+ def __init__(self, cddname=None, taskname=None):
+ self.cddname = cddname # CDD that includes the package in dependency list
+ self.taskname = taskname # Task which includes the Dependency
+ self.pkg = None # Name of dependant package
+ self.dependencytype = None # Values: 'Depends', 'Recommends', 'Suggests'
+ self.dists = [] # Values: 'stable', 'testing', 'unstable', etc.
+ self.component = {} # Values: 'main', 'contrib', 'non-free', 'experimental'
+
+ self.why = None # basically used as comment
+
+ # The following keys will be mostly used for programs that
+ # are not yet existing in Debian and will go to our todo list
+ self.homepage = '#' # Homepage of program
+ self.version = None # Version of program
+ self.responsible = None # E-Mail address of issuer of ITP or some person
+ # who volunteered to care for this program
+ self.license = None # License of program
+ self.section = None # Section of package in the Debian hierarchy
+ self.filename = None # Filename of package in the Debian pool
+ self.wnpp = None # WNPP bug number
+ self.pkgShortDesc = None # Prospective packages should have a description ...
+ self.pkgLongDesc = None # ... which could be copied to (or from if exists)
+ # WNPP bug and finally can be used for packaging
+ self.pkgURL = None # URL of inofficial package
+
+ # sort these objects according to the package name
+ def __cmp__(self, other):
+ # Comparing with None object has to return something reasonable
+ if other == None:
+ return -2
+ # Sort according to package name
+ return cmp(self.pkg, other.pkg)
+
+class CddDependencies:
+ # Provide a list of depencencies defined in Metapackages
+ # This class concerns _all_ tasks of a CDD and is the most
+ # complete source of information. If only a single task
+ # should be handled by a tool that uses cddtasktools
+ # probably the class TaskDependencies (see below) is
+ # your friend
+
+ def __init__(self, cddname):
+
+ # This Instance of the Available class contains all
+ # information about packages that are avialable in Debian
+ # See below for more information about Available class
+ global available
+
+ if cddname not in REPOS.keys():
+ print >>sys.stderr, "Unknown CDD."
+ return None
+
+ self.cddname = cddname
+ self.tasksdir = InitTasksFiles(self.cddname)
+ self.tasks = {}
+ self.tasknames = []
+ self.available = available
+
+ def GetTasknames(self):
+ for task in os.listdir(self.tasksdir):
+ if os.path.isfile("%s/%s" % (self.tasksdir, task)):
+ self.tasknames.append(task)
+ self.tasknames.sort()
+
+ def GetAllDependencies(self):
+ if self.tasknames == []:
+ self.GetTasknames()
+ for task in self.tasknames:
+ td = TaskDependencies(self.cddname, task)
+ td.GetTaskDependencies()
+ self.tasks[task] = td
+
+ def GetNamesOnlyDict(self, dependencytypes=()):
+ # David Paleino needs for his web tools a dictionary
+ # { taskname : [list of dependencies]}
+ # This will be prepared here from the main
+ # datastructure
+ ret = {}
+ if dependencytypes == ():
+ # official: A package with this name is found in the Debian repository
+ # unofficial: The tasks file contains a tag Pkg-URL for the package which is not None
+ # prospective: At least a short and long description are attached to a package name which
+ # is not in the Debian pool and has no Pkg-URL for an unofficial package
+ # unknown: Any other package names that are listed as Dependencies but have
+ # incomplete information. This usually happens when packages are not
+ # available in Debian any more (for instance this might happen if a
+ # name has changed)
+ dependencytypes=('official', 'unofficial', 'prospective', 'unknown')
+ for task in self.tasknames:
+ tdeps = self.tasks[task]
+ list = []
+ for dep in dependencytypes:
+ for tdep in tdeps.dependencies[dep]:
+ list.append(tdep.pkg)
+ ret[task] = list
+ return ret
+
+ def GetListOfDepsForTask(self, task, dependencytypes=()):
+ # David Paleino needs for his web tools a dictionary
+ # [list of dependencies]
+ # This will be prepared here from the main
+ # datastructure
+ ret = []
+ if dependencytypes == ():
+ dependencytypes=('official', 'unofficial', 'prospective', 'unknown')
+ tdeps = self.tasks[task]
+ for dep in dependencytypes:
+ for tdep in tdeps.dependencies[dep]:
+ ret.append(tdep.pkg)
+ return ret
+
+ def GetTaskDescDict(self):
+ # David Paleino needs for his web tools a dictionary
+ # { taskname : { 'Task' : task
+ # 'ShortDesc' : shortdesc
+ # 'LongDesc' : longdesc }
+ # This will be prepared here from the main
+ # datastructure
+ ret = {}
+ for task in self.tasknames:
+ tdeps = self.tasks[task]
+ tdict = {}
+ tdict['Task'] = tdeps.taskPrintedName
+ tdict['ShortDesc'] = tdeps.taskShortDesc
+ tdict['LongDesc'] = tdeps.taskLongDesc
+ ret[task] = tdict
+ return ret
+
+class TaskDependencies:
+ # List of depencencies defined in one Metapackage
+ def __init__(self, cddname, task):
+ if cddname not in REPOS.keys():
+ print >>sys.stderr, "Unknown CDD."
+ return None
+
+ self.cddname = cddname
+ self.tasksdir = InitTasksFiles(self.cddname)
+ self.taskfile = self.tasksdir+'/'+task
+ if os.path.isfile(self.taskfile):
+ self.task = task
+ else:
+ print >>sys.stderr, "No such task file %s." % self.taskfile
+ return None
+
+ # Dictionary with dependencytype as key and list of DependantPackage
+ # instances
+ self.dependencies = { 'official' : [],
+ 'unofficial' : [],
+ 'prospective' : [],
+ 'unknown' : []
+ }
+ self.available = available
+
+ # Main information for a task
+ self.taskPrintedName = None
+ self.taskShortDesc = None
+ self.taskLongDesc = None
+
+ def GetTaskDependencies(self):
+ # First obtain information about Packages
+ # available in Debian
+ # This might take some time.
+ # Caching comes to mind, but the script is supposed
+ # to be run not more frequently than mirror pushes
+ # and thus only testing the script might profit from
+ # caching
+ self.available.GetPackageNames()
+
+ # These keys might contain more than one item that
+ # has to be separated to detect the dependency
+ dependency_keys = [ "Depends", "Recommends", "Suggests" ]
+
+ f = file(self.taskfile)
+ for stanza in deb822.Sources.iter_paragraphs(f):
+ # Why and Responsible can be valid for more than one dependency
+ # Store them in strings and use them for all Dependent Package objects
+ why = None
+ responsible = None
+ dep = None
+ for key in stanza:
+ if key == 'Task':
+ self.taskPrintedName = stanza['task']
+ continue
+ if key == 'Description':
+ (self.taskShortDesc, self.taskLongDesc) = SplitDescription(stanza['description'])
+ continue
+ if key == 'Why':
+ why = stanza['why']
+ continue
+ if key == 'Responsible':
+ responsible = stanza['responsible']
+ continue
+
+ if key in dependency_keys:
+ # turn alternatives ('|') into real depends for this purpose
+ # because we are finally interested in all alternatives
+ dependencies = stanza[key].replace('|',',').split(',')
+ # Collect all dependencies in one line first,
+ # create an object for each later
+ deps_in_one_line = []
+ for dependency in dependencies:
+ deps_in_one_line.append(dependency.strip())
+
+ for dep_in_line in deps_in_one_line:
+ # If there are more than one dependencies in one line
+ # just put the current one into the right list of dependencies
+ # before initiating the next instance
+ if dep != None:
+ self.dependencies[self._FindDependencyType(dep)].append(dep)
+ dep = DependantPackage(self.cddname, self.task)
+ # Store the comments in case they might be usefull for later applications
+ dep.why = why
+ dep.responsible = responsible
+ dep.dependencytype = key
+ dep.pkg = dep_in_line
+ dep.dists.append(self.available.dist)
+ # Find the component the Dependency might be in
+ for component in self.available.components:
+ if dep.pkg in self.available.packages[component].keys():
+ dep.component = component
+ if component == 'main':
+ dep.license = 'DFSG free'
+ elif component == 'contrib':
+ dep.license = 'DFSG free, but needs non-free components'
+ elif component == 'non-free':
+ dep.license = 'non-free'
+ else:
+ dep.license = 'unknown'
+ dep.pkgShortDesc = self.available.packages[component][dep.pkg].pkgShortDesc
+ dep.pkgLongDesc = self.available.packages[component][dep.pkg].pkgLongDesc
+ dep.homepage = self.available.packages[component][dep.pkg].homepage
+ dep.version = self.available.packages[component][dep.pkg].version
+ # TODO: Assuming 'unstable' is wrong --> handle list of dists
+ dep.pkgURL = 'http://packages.debian.org/' + 'unstable/' + \
+ self.available.packages[component][dep.pkg].section + \
+ '/' + dep.pkg
+ dep.filename = self.available.packages[component][dep.pkg].filename
+ break # The same package should be only in one component
+ # At least I currently see no reason for having a
+ # package with the same name in main and contrib
+ # at the same time for instance
+ continue
+ # The following keys will be mostly used for programs that
+ # are not yet existing in Debian and will go to our todo list
+ if key == 'homepage':
+ if dep != None:
+ dep.homepage = stanza['homepage']
+ else:
+ print >>stderr, "Dep not initiated before Homepage %s -> something is wrong." \
+ % stanza['homepage']
+ elif key == 'section':
+ if dep != None:
+ dep.section = stanza['section']
+ else:
+ print >>stderr, "Dep not initiated before Section %s -> something is wrong." \
+ % stanza['license']
+ elif key == 'License':
+ if dep != None:
+ dep.license = stanza['license']
+ else:
+ print >>stderr, "Dep not initiated before License %s -> something is wrong." \
+ % stanza['license']
+ elif key == 'WNPP':
+ if dep != None:
+ dep.wnpp = stanza['wnpp']
+ else:
+ print >>stderr, "Dep not initiated before WNPP %s -> something is wrong." \
+ % stanza['wnpp']
+ elif key == 'Pkg-URL':
+ if dep != None:
+ dep.pkgURL = stanza['pkg-url']
+ else:
+ print >>stderr, "Dep not initiated before Pkg-URL %s -> something is wrong." \
+ % stanza['pkg-url']
+ elif key == 'Pkg-Description':
+ if dep == None:
+ print >>stderr, "Dep not initiated before Pkg-Description %s -> something is wrong." \
+ % stanza['pkg-description'].splitlines()[0]
+ else:
+ (dep.pkgShortDesc, dep.pkgLongDesc) = SplitDescription(stanza['pkg-description'])
+ else:
+ print "Unknown key '%s': %s" % (key, stanza[key])
+ if dep != None:
+ self.dependencies[self._FindDependencyType(dep)].append(dep)
+
+ f.close()
+
+ for dependency in self.dependencies.keys():
+ self.dependencies[dependency].sort()
+
+
+ def _FindDependencyType(self, dep):
+ # Return the name of the Dependencytype to append the Dependency to the right list
+ if dep.component != {}:
+ return 'official'
+ if dep.pkgURL != None:
+ return 'unofficial'
+ if dep.pkgShortDesc != None and dep.pkgLongDesc != None:
+ return 'prospective'
+ return 'unknown'
+
+
+class Available:
+ # Information about available packages
+ #
+ # Usage example:
+ # available = Available( # Initialize instance
+ # dist='testing', # (default='unstable')
+ # components=('main'), # Regard only main, default: main, contrib, non-free
+ # source=1 # Use source package names, default: use binaries
+ # arch='sparc' # (default='i386')
+ # )
+ #
+ # available.GetPackageNames() # Actually parse the Packages files to obtain needed information
+ # # This has to be done at least once. It is verified that the effort
+ # # to obtain package information is not done twice per run
+
+
+ def __init__(self, dist=None, components=(), source=None, arch=None):
+ self.source = 'Packages.gz'
+ if source != None:
+ self.source = 'Sources.gz'
+ self.binary = 'source'
+ if source == None:
+ if arch == None:
+ # use arch=i386 as default because it contains most packages
+ self.binary = 'binary-i386'
+ else:
+ self.binary = 'binary-' + arch
+ self.dist = 'unstable'
+ if dist != None:
+ self.dist = dist
+ self.components = ('main', 'contrib', 'non-free')
+ if components != ():
+ self.components = components
+ # The dictionary packages contains the component as key
+ # The values are dictionaries holding package names as key
+ # and a DependantPackage object as values
+ self.packages = {}
+ for component in self.components:
+ self.packages[component] = {}
+
+ def GetPackageNames(self):
+ # Fetch Packages / Sources file and get list of package names out of it
+
+ # Check whether package names are just known
+ for component in self.packages:
+ if self.packages[component] != {}:
+ # Just got the Package names because at least one component
+ # has non zero elements
+ return
+
+ for component in self.components:
+ f = urllib.urlopen(BASEURL+'/dists/'+self.dist+'/'+component+'/'+self.binary+'/'+self.source)
+ compresseddata = f.read()
+ compressedstream = StringIO.StringIO(compresseddata)
+ g = gzip.GzipFile(fileobj=compressedstream)
+ for stanza in deb822.Sources.iter_paragraphs(g, shared_storage=False):
+ deppkg = DependantPackage()
+ (deppkg.pkgShortDesc, deppkg.pkgLongDesc) = SplitDescription(stanza['description'])
+ try:
+ deppkg.homepage = stanza['homepage']
+ except KeyError:
+ deppkg.homepage = '#' # Not every package has a homepage tag
+ except:
+ deppkg.homepage = '.' # Something else in case unexpected things happen
+ deppkg.version = stanza['version'].split('-')[0]
+ deppkg.section = stanza['section']
+ deppkg.filename = BASEURL+stanza['filename']
+ self.packages[component][stanza['package']] = deppkg
+ f.close()
+
+available = Available()
+
Deleted: trunk/community/infrastructure/scripts/update-tasks
===================================================================
--- trunk/community/infrastructure/scripts/update-tasks 2008-02-17 07:33:49 UTC (rev 1398)
+++ trunk/community/infrastructure/scripts/update-tasks 2008-02-17 07:36:40 UTC (rev 1399)
@@ -1,319 +0,0 @@
-#!/usr/bin/python -W ignore
-
-#
-# This Python script is:
-# (C) 2007, David Paleino <d.paleino at gmail.com>
-#
-# It is licensed under the terms of GNU General Public License (GPL)
-# v3, or any later revision.
-#
-
-import apt
-import apt_pkg
-import apt_inst
-import HTMLTemplate
-import re
-import sys
-import time
-from datetime import datetime
-from email.Utils import formatdate
-from Tools import *
-
-base = "/var/lib/gforge/chroot/home/groups/debian-med"
-tasks = "%s/scripts/tasks" % base
-
-official = {} # Official packages
-todo = {} # Packages not in repositories, nor unofficial,
- # nor prospected. They will eventually go into
- # "unavailable".
-det = {} # Official Packages details
-
-# let's get our nice dict in the form:
-# { 'task_foo': ['package1', 'package2', '...'],
-# 'task_bar': ['package3', 'package4', '...']}
-
-packages = parseTasks(None, True)
-unofficial = parseTasksNonOff()
-unavailable = parseTasksUnavail()
-task_details = parseTaskDetails()
-
-tasks = packages.keys()
-tasks.sort()
-
-apt_pkg.init()
-#~ apt_pkg.Config.Set("APT::Acquire::Translation", "it")
-
-cache = apt_pkg.GetCache()
-depcache = apt_pkg.GetDepCache(cache)
-aptcache = apt.Cache()
-
-###
-# Wrappers around apt_pkg
-###
-
-def __getRecords(package):
- ### TODO: convert to Python API
- (f, index) = depcache.GetCandidateVer(cache[package]).TranslatedDescription.FileList.pop(0)
- records = apt_pkg.GetPkgRecords(cache)
- records.Lookup ((f, index))
- return records
-
-def __getDesc(package):
- regex = r"""(?P<short>.*)
-(?P<long>(^ .*$\n)+\n?)"""
-
- p = re.compile(regex, re.MULTILINE)
- m = p.match(getSections(package)['Description'])
- if m:
- return {'ShortDesc': m.group("short"), 'LongDesc': m.group("long")}
- else:
- return False
-
-def getShort(package):
- ### TODO: convert to Python API
- desc = __getDesc(package)
- if desc:
- return desc['ShortDesc']
- else:
- # Fallback to the C++ wrapper
- return __getRecords(package).ShortDesc
-
-def getLong(package):
- ### TODO: convert to Python API
- desc = __getDesc(package)
- if desc:
- return desc['LongDesc']
- else:
- # Fallback to the C++ wrapper
- return __getRecords(package).LongDesc
-
-def getHomepage(package):
- sect = getSections(package)
- try:
- return sect['Homepage']
- # Fallback to the old " Homepage: ..." pseudo-field
- # TODO: also renders _wrong_ "URL" pseudo-fields! Fix the packages!
- except:
- p = re.compile(".*(?P<field>Homepage|URL): (?P<url>.*)", re.DOTALL)
- m = p.match(getLong(package))
- if m:
- tmp = det[package]['LongDesc']
- det[package]['LongDesc'] = tmp.replace(m.group("field") + ": " + m.group("url"), "")
- return m.group("url")
- else:
- # We don't have any valid field for homepage,
- # return a suitable "#" for <a href>s.
- return "#"
-
-def getDebUrl(package):
- try:
- return getSections(package)["Filename"]
- except:
- return "#"
-
-def getVersion(package):
- try:
- return getSections(package)["Version"]
- except:
- # Fallback to the C++ wrapper
- for pkg in cache.Packages:
- if pkg.Name in det:
- if not pkg.VersionList:
- return "N/A"
- else:
- return pkg.VersionList[0].VerStr
-
-def getLicense(package):
- ### FIX
- return "GPL-foo"
-
-def getSections(package):
- pkg = aptcache[package]
- pkg._lookupRecord(True)
- return apt_pkg.ParseSection(pkg._records.Record)
-
-###
-# Template handlers
-###
-
-def renderIndex(node, tasks):
- node.tasks.repeat(renderTaskList, tasks)
- t = datetime.now()
- node.date.content = formatdate(time.mktime(t.timetuple()))
-
-def renderTaskList(node, task):
- node.task.raw = """<a href="/tasks/%s.php" name="%s" id="%s">%s</a>""" % (task, task, task, task.capitalize())
-
-def renderTasks(node, task, packages, details):
- node.task.content = details['Task']
- node.shortdesc.content = details['ShortDesc']
- node.heading.content = details['ShortDesc']
- node.longdesc.content = details['LongDesc']
-
- t = datetime.now()
- node.date.content = formatdate(time.mktime(t.timetuple()))
-
- # Let's separate official packages from others
- for pkg in packages:
- # If the package has a short description in cache,
- # there's an high chance it is an official package.
- # Probably we can use a better algorithm? (I believe
- # Alioth's APT cache won't be contaminated by external
- # repositories)
- try:
- short = getShort(pkg)
- if not task in official:
- official[task] = []
- official[task].append(pkg)
- det[pkg] = {}
- det[pkg]['ShortDesc'] = short
- det[pkg]['LongDesc'] = getLong(pkg).replace("%s\n" % short, "")
- det[pkg]['LongDesc'] = det[pkg]['LongDesc'].replace("<", "<").replace(">", ">")
- # getHomepage() does some magic on ['LongDesc']
- det[pkg]['Homepage'] = getHomepage(pkg)
- det[pkg]['LongDesc'] = det[pkg]['LongDesc'].replace("\n .\n", "<br /><br />").replace("\n", "")
- det[pkg]['Version'] = getVersion(pkg)
- det[pkg]['License'] = getLicense(pkg)
- det[pkg]['Task'] = task
- det[pkg]['Pkg-URL'] = "http://packages.debian.org/unstable/%s/%s" % (getSections(pkg)['Section'], pkg)
- ### BUG: some packages don't get the right Filename
- ### see, for example, "treeviewx": it has a Filename
- ### field, but doesn't get parsed.
- ### FIX: installed packages with versions newer than the
- ### one in repositories don't have that field. :@!
- det[pkg]['Deb-URL'] = "http://ftp.debian.org/%s" % getDebUrl(pkg)
- except:
- pass
-
- if task in official:
- node.official_head.raw = """<h2>
-<a id="official-debs" name="official-debs"></a>
- Official Debian packages
-</h2>"""
- node.official.repeat(renderOfficial, official[task], task)
-
- if task in todo:
- error = True
- else:
- error = False
-
- if task in unofficial:
- node.unofficial_head.raw = """<h2>
-<a id="inofficial-debs" name="inofficial-debs"></a>
- Inofficial Debian packages
-</h2>"""
- node.unofficial.repeat(renderUnofficial, unofficial[task])
- error = False
-
- if task in unavailable:
- node.unavailable_head.raw = """<h2>
-<a id="debs-not-available" name="debs-not-available"></a>
- Debian packages not available
-</h2>"""
- node.unavailable.repeat(renderUnavailable, unavailable[task])
- error = False
-
- if error:
- # The package probably needs a proper prospective entry in the
- # task files. Write it to stdout.
- print "Error: problems with %s" % task
-
-def renderOfficial(node, package, task):
- # Here we parse just official packages
- node.shortdesc.content = det[package]['ShortDesc']
- node.project.raw = "<table class=\"project\" summary=\"%s\">" % package
- node.anchor.atts['name'] = package
- node.anchor.atts['id'] = package
- node.name.content = package.capitalize()
- node.url.atts['href'] = det[package]['Homepage']
- if det[package]['Homepage'] == "#":
- node.url.content = "Homepage not available"
- else:
- node.url.content = det[package]['Homepage']
-
- node.longdesc.raw = det[package]['LongDesc']
- node.version.content = "Version: %s" % det[package]['Version']
- node.license.content = "License: %s" % det[package]['License']
- node.pkgurl.atts['href'] = det[package]['Pkg-URL']
- node.pkgurl.content = "Official Debian package"
- node.deburl.atts['href'] = det[package]['Deb-URL']
- #~ node.deburl.content = "X" ### TODO: add a nice icon here to download the .deb package
- node.deburl.raw = "<img src=\"/img/deb-icon.png\" />"
-
-def renderUnofficial(node, package):
- # Here we parse just unofficial packages
- node.shortdesc.content = package['ShortDesc']
- node.longdesc.raw = package['LongDesc']
- node.project.raw = "<table class=\"project\" summary=\"%s\">" % package['Package']
- node.anchor.atts['name'] = package['Package']
- node.anchor.atts['id'] = package['Package']
- node.name.content = package['Package'].capitalize()
- node.url.atts['href'] = package['Homepage']
- node.url.content = package['Homepage']
- node.license.content = "License: %s" %package['License']
- node.pkgurl.atts['href'] = package['Pkg-URL']
- node.pkgurl.content = "Inofficial Debian package"
-
- # Let's try to get the version from the package name
- # (following Debian standards: <name>_<ver>_<arch>.deb)
- regex = ".*/%s_(?P<version>.*)_.*\.deb$" % package['Package']
- p = re.compile(regex)
- m = p.search(package['Pkg-URL'])
- if m:
- node.version.content = "Version: %s" % m.group("version")
- else:
- node.version.content = "Version: N/A"
-
-
-def renderUnavailable(node, package):
- # Parsing unavailable packages :(
- # PACKAGE THEM! :)
- name = package['Package']
- if package['ShortDesc']:
- node.shortdesc.content = package['ShortDesc']
- else:
- node.shortdesc.content = "N/A"
- node.longdesc.raw = package['LongDesc']
- node.project.raw = "<table class=\"project\" summary=\"%s\">" % name
- if package['Responsible']:
- node.responsible.content = package['Responsible']
- else:
- node.responsible.raw = "no one"
- if package['WNPP']:
- node.wnpp.raw = " — <a href=\"http://bugs.debian.org/%s\">wnpp</a>" % package['WNPP']
- node.anchor.atts['name'] = name
- node.anchor.atts['id'] = name
- node.name.content = name.capitalize()
- if package['Homepage']:
- node.url.atts['href'] = package['Homepage']
- node.url.content = package['Homepage']
- else:
- node.url.atts['href'] = "#"
- node.url.content = "N/A"
- if package['License']:
- node.license.raw = "<?=_('License')?>: %s" % package['License']
- else:
- node.license.raw = "<?=_('License')?>: N/A"
-
-# Let's render the Tasks Page index, first
-f = open("%s/htdocs/tasks_idx.tmpl" % base)
-tmpl = HTMLTemplate.Template(renderIndex, f.read())
-f.close()
-f = open("%s/static/tasks/index.php" % base, "w")
-f.write(tmpl.render(tasks))
-f.close()
-
-# Let's render single pages now.
-f = open("%s/htdocs/tasks.tmpl" % base)
-tmpl = HTMLTemplate.Template(renderTasks, f.read())
-f.close()
-
-for task in tasks:
- f = open("%s/static/tasks/%s.php" % (base, task), "w")
-
- # This is to avoid useless <br>eaks before closing the cell
- source = tmpl.render(task, packages[task], task_details[task])
- f.write(re.sub(r"<br /><br />[ ]*</td>", "</td>", source))
-
- f.close()
-
Added: trunk/community/infrastructure/scripts/update-tasks
===================================================================
--- trunk/community/infrastructure/scripts/update-tasks (rev 0)
+++ trunk/community/infrastructure/scripts/update-tasks 2008-02-17 07:36:40 UTC (rev 1399)
@@ -0,0 +1,208 @@
+#!/usr/bin/python -W ignore
+
+#
+# This Python script is:
+# (C) 2007, David Paleino <d.paleino at gmail.com>
+#
+# It is licensed under the terms of GNU General Public License (GPL)
+# v3, or any later revision.
+#
+
+import apt
+import apt_pkg
+import apt_inst
+import HTMLTemplate
+import re
+import sys
+import time
+from datetime import datetime
+from email.Utils import formatdate
+
+from cddtasktools import CddDependencies, HTMLBASE
+
+CDD='debian-med'
+
+###
+# Template handlers
+###
+
+def renderIndex(node, tasks):
+ node.tasks.repeat(renderTaskList, tasks)
+ t = datetime.now()
+ node.date.content = formatdate(time.mktime(t.timetuple()))
+
+def renderTaskList(node, task):
+ node.task.raw = """<a href="/tasks/%s.php" name="%s" id="%s">%s</a>""" % (task, task, task, task.capitalize())
+
+def renderTasks(node, task, packages, details):
+ global cdeps
+
+ node.task.content = details['Task']
+ node.shortdesc.content = details['ShortDesc']
+ node.heading.content = details['ShortDesc']
+ node.longdesc.content = details['LongDesc']
+
+ t = datetime.now()
+ node.date.content = formatdate(time.mktime(t.timetuple()))
+
+ official = cdeps.GetNamesOnlyDict(('official',))
+ # for deppkg in cdeps.tasks[task].dependencies['official']:
+ node.official_head.raw = """<h2>
+<a id="official-debs" name="official-debs"></a>
+ Official Debian packages
+</h2>"""
+ # HTML_Template wants a list as argument so we provide a list of indizes to address
+ # the list of DependantPackage instances inside renderOfficial
+ list_of_dependencies = cdeps.GetListOfDepsForTask(task, dependencytypes=('official',))
+ node.official.repeat(renderOfficial,
+ range(len(list_of_dependencies)))
+
+# if task in todo:
+# error = True
+# else:
+# error = False
+
+ unofficial = cdeps.GetNamesOnlyDict(('unofficial',))
+ list_of_dependencies = cdeps.GetListOfDepsForTask(task, dependencytypes=('unofficial',))
+ if len(list_of_dependencies) > 0:
+ node.unofficial_head.raw = """<h2>
+<a id="inofficial-debs" name="inofficial-debs"></a>
+ Unofficial Debian packages
+</h2>"""
+ node.unofficial.repeat(renderUnofficial, range(len(list_of_dependencies)))
+ # error = False
+
+ prospective = cdeps.GetNamesOnlyDict(('prospective',))
+ list_of_dependencies = cdeps.GetListOfDepsForTask(task, dependencytypes=('prospective',))
+ if len(list_of_dependencies) > 0:
+ node.unavailable_head.raw = """<h2>
+<a id="debs-not-available" name="debs-not-available"></a>
+ Debian packages not available
+</h2>"""
+ node.unavailable.repeat(renderProspective, range(len(list_of_dependencies)))
+ # error = False
+
+# if error:
+ # The package probably needs a proper prospective entry in the
+ # task files. Write it to stdout.
+# print "Error: problems with %s" % task
+
+def renderOfficial(node, package_no):
+ # Here we parse just official packages
+ deppkg = cdeps.tasks[task].dependencies['official'][package_no]
+
+ node.shortdesc.content = deppkg.pkgShortDesc
+ node.project.raw = "<table class=\"project\" summary=\"%s\">" % deppkg.pkg
+ node.anchor.atts['name'] = deppkg.pkg
+ node.anchor.atts['id'] = deppkg.pkg
+ node.name.content = deppkg.pkg.capitalize()
+ node.url.atts['href'] = deppkg.homepage
+ if deppkg.homepage == "#":
+ node.url.content = "Homepage not available"
+ else:
+ node.url.content = deppkg.homepage
+
+ node.longdesc.raw = deppkg.pkgLongDesc
+ node.version.content = "Version: %s" % deppkg.version
+ if deppkg.license != None:
+ node.license.content = "License: %s" % deppkg.license
+ node.pkgurl.atts['href'] = deppkg.pkgURL
+ node.pkgurl.content = "Official Debian package"
+ node.deburl.atts['href'] = deppkg.filename
+ #~ node.deburl.content = "X" ### TODO: add a nice icon here to download the .deb package
+ node.deburl.raw = "<img src=\"/img/deb-icon.png\" />"
+
+
+def renderUnofficial(node, package_no):
+ # Here we parse just unofficial packages
+ deppkg = cdeps.tasks[task].dependencies['unofficial'][package_no]
+
+ node.shortdesc.content = deppkg.pkgShortDesc
+ node.longdesc.raw = deppkg.pkgLongDesc
+ node.project.raw = "<table class=\"project\" summary=\"%s\">" % deppkg.pkg
+ node.anchor.atts['name'] = deppkg.pkg
+ node.anchor.atts['id'] = deppkg.pkg
+ node.name.content = deppkg.pkg.capitalize()
+ node.url.atts['href'] = deppkg.homepage
+ node.url.content = deppkg.homepage
+ node.license.content = "License: %s" % deppkg.license
+ node.pkgurl.atts['href'] = deppkg.pkgURL
+ node.pkgurl.content = "Unofficial Debian package"
+
+ # Let's try to get the version from the package name
+ # (following Debian standards: <name>_<ver>_<arch>.deb)
+ regex = ".*/%s_(?P<version>.*)_.*\.deb$" % deppkg.pkg
+ p = re.compile(regex)
+ m = p.search(deppkg.pkgURL)
+ if m:
+ node.version.content = "Version: %s" % m.group("version")
+ else:
+ node.version.content = "Version: N/A"
+
+
+def renderProspective(node, package_no):
+ # Parsing unavailable packages :(
+ # PACKAGE THEM! :)
+ deppkg = cdeps.tasks[task].dependencies['prospective'][package_no]
+
+ if deppkg.pkgShortDesc:
+ node.shortdesc.content = deppkg.pkgShortDesc
+ else:
+ node.shortdesc.content = "N/A"
+ if deppkg.pkgLongDesc:
+ node.longdesc.content = deppkg.pkgLongDesc
+ else:
+ node.longdesc.content = "N/A"
+ node.longdesc.raw = deppkg.pkgLongDesc
+ node.project.raw = "<table class=\"project\" summary=\"%s\">" % deppkg.pkg
+ if deppkg.responsible:
+ node.responsible.content = deppkg.responsible
+ else:
+ node.responsible.raw = "no one"
+ if deppkg.wnpp:
+ node.wnpp.raw = " — <a href=\"http://bugs.debian.org/%s\">wnpp</a>" % deppkg.wnpp
+ node.anchor.atts['name'] = deppkg.pkg
+ node.anchor.atts['id'] = deppkg.pkg
+ node.name.content = deppkg.pkg.capitalize()
+ if deppkg.homepage:
+ node.url.atts['href'] = deppkg.homepage
+ node.url.content = deppkg.homepage
+ else:
+ node.url.atts['href'] = "#"
+ node.url.content = "N/A"
+ if deppkg.license:
+ node.license.raw = "<?=_('License')?>: %s" % deppkg.license
+ else:
+ node.license.raw = "<?=_('License')?>: N/A"
+
+cdeps=CddDependencies('debian-med')
+cdeps.GetAllDependencies()
+
+base=HTMLBASE + '/' + CDD
+# Let's render the Tasks Page index, first
+f = open("%s/htdocs/tasks_idx.tmpl" % base)
+tmpl = HTMLTemplate.Template(renderIndex, f.read())
+f.close()
+f = open("%s/static/tasks/index.php" % base, "w")
+
+tasks = cdeps.tasknames
+packages = cdeps.GetNamesOnlyDict()
+task_details = cdeps.GetTaskDescDict()
+
+f.write(tmpl.render(tasks))
+f.close()
+
+# Let's render single pages now.
+f = open("%s/htdocs/tasks.tmpl" % base)
+tmpl = HTMLTemplate.Template(renderTasks, f.read())
+f.close()
+
+for task in tasks:
+ f = open("%s/static/tasks/%s.php" % (base, task), "w")
+
+ # This is to avoid useless <br>eaks before closing the cell
+ source = tmpl.render(task, packages[task], task_details[task])
+ f.write(re.sub(r"<br /><br />[ ]*</td>", "</td>", source))
+
+ f.close()
+
Copied: trunk/community/infrastructure/scripts/update-tasks_using_Tools (from rev 1397, trunk/community/infrastructure/scripts/update-tasks)
===================================================================
--- trunk/community/infrastructure/scripts/update-tasks_using_Tools (rev 0)
+++ trunk/community/infrastructure/scripts/update-tasks_using_Tools 2008-02-17 07:36:40 UTC (rev 1399)
@@ -0,0 +1,319 @@
+#!/usr/bin/python -W ignore
+
+#
+# This Python script is:
+# (C) 2007, David Paleino <d.paleino at gmail.com>
+#
+# It is licensed under the terms of GNU General Public License (GPL)
+# v3, or any later revision.
+#
+
+import apt
+import apt_pkg
+import apt_inst
+import HTMLTemplate
+import re
+import sys
+import time
+from datetime import datetime
+from email.Utils import formatdate
+from Tools import *
+
+base = "/var/lib/gforge/chroot/home/groups/debian-med"
+tasks = "%s/scripts/tasks" % base
+
+official = {} # Official packages
+todo = {} # Packages not in repositories, nor unofficial,
+ # nor prospected. They will eventually go into
+ # "unavailable".
+det = {} # Official Packages details
+
+# let's get our nice dict in the form:
+# { 'task_foo': ['package1', 'package2', '...'],
+# 'task_bar': ['package3', 'package4', '...']}
+
+packages = parseTasks(None, True)
+unofficial = parseTasksNonOff()
+unavailable = parseTasksUnavail()
+task_details = parseTaskDetails()
+
+tasks = packages.keys()
+tasks.sort()
+
+apt_pkg.init()
+#~ apt_pkg.Config.Set("APT::Acquire::Translation", "it")
+
+cache = apt_pkg.GetCache()
+depcache = apt_pkg.GetDepCache(cache)
+aptcache = apt.Cache()
+
+###
+# Wrappers around apt_pkg
+###
+
+def __getRecords(package):
+ ### TODO: convert to Python API
+ (f, index) = depcache.GetCandidateVer(cache[package]).TranslatedDescription.FileList.pop(0)
+ records = apt_pkg.GetPkgRecords(cache)
+ records.Lookup ((f, index))
+ return records
+
+def __getDesc(package):
+ regex = r"""(?P<short>.*)
+(?P<long>(^ .*$\n)+\n?)"""
+
+ p = re.compile(regex, re.MULTILINE)
+ m = p.match(getSections(package)['Description'])
+ if m:
+ return {'ShortDesc': m.group("short"), 'LongDesc': m.group("long")}
+ else:
+ return False
+
+def getShort(package):
+ ### TODO: convert to Python API
+ desc = __getDesc(package)
+ if desc:
+ return desc['ShortDesc']
+ else:
+ # Fallback to the C++ wrapper
+ return __getRecords(package).ShortDesc
+
+def getLong(package):
+ ### TODO: convert to Python API
+ desc = __getDesc(package)
+ if desc:
+ return desc['LongDesc']
+ else:
+ # Fallback to the C++ wrapper
+ return __getRecords(package).LongDesc
+
+def getHomepage(package):
+ sect = getSections(package)
+ try:
+ return sect['Homepage']
+ # Fallback to the old " Homepage: ..." pseudo-field
+ # TODO: also renders _wrong_ "URL" pseudo-fields! Fix the packages!
+ except:
+ p = re.compile(".*(?P<field>Homepage|URL): (?P<url>.*)", re.DOTALL)
+ m = p.match(getLong(package))
+ if m:
+ tmp = det[package]['LongDesc']
+ det[package]['LongDesc'] = tmp.replace(m.group("field") + ": " + m.group("url"), "")
+ return m.group("url")
+ else:
+ # We don't have any valid field for homepage,
+ # return a suitable "#" for <a href>s.
+ return "#"
+
+def getDebUrl(package):
+ try:
+ return getSections(package)["Filename"]
+ except:
+ return "#"
+
+def getVersion(package):
+ try:
+ return getSections(package)["Version"]
+ except:
+ # Fallback to the C++ wrapper
+ for pkg in cache.Packages:
+ if pkg.Name in det:
+ if not pkg.VersionList:
+ return "N/A"
+ else:
+ return pkg.VersionList[0].VerStr
+
+def getLicense(package):
+ ### FIX
+ return "GPL-foo"
+
+def getSections(package):
+ pkg = aptcache[package]
+ pkg._lookupRecord(True)
+ return apt_pkg.ParseSection(pkg._records.Record)
+
+###
+# Template handlers
+###
+
+def renderIndex(node, tasks):
+ node.tasks.repeat(renderTaskList, tasks)
+ t = datetime.now()
+ node.date.content = formatdate(time.mktime(t.timetuple()))
+
+def renderTaskList(node, task):
+ node.task.raw = """<a href="/tasks/%s.php" name="%s" id="%s">%s</a>""" % (task, task, task, task.capitalize())
+
+def renderTasks(node, task, packages, details):
+ node.task.content = details['Task']
+ node.shortdesc.content = details['ShortDesc']
+ node.heading.content = details['ShortDesc']
+ node.longdesc.content = details['LongDesc']
+
+ t = datetime.now()
+ node.date.content = formatdate(time.mktime(t.timetuple()))
+
+ # Let's separate official packages from others
+ for pkg in packages:
+ # If the package has a short description in cache,
+ # there's an high chance it is an official package.
+ # Probably we can use a better algorithm? (I believe
+ # Alioth's APT cache won't be contaminated by external
+ # repositories)
+ try:
+ short = getShort(pkg)
+ if not task in official:
+ official[task] = []
+ official[task].append(pkg)
+ det[pkg] = {}
+ det[pkg]['ShortDesc'] = short
+ det[pkg]['LongDesc'] = getLong(pkg).replace("%s\n" % short, "")
+ det[pkg]['LongDesc'] = det[pkg]['LongDesc'].replace("<", "<").replace(">", ">")
+ # getHomepage() does some magic on ['LongDesc']
+ det[pkg]['Homepage'] = getHomepage(pkg)
+ det[pkg]['LongDesc'] = det[pkg]['LongDesc'].replace("\n .\n", "<br /><br />").replace("\n", "")
+ det[pkg]['Version'] = getVersion(pkg)
+ det[pkg]['License'] = getLicense(pkg)
+ det[pkg]['Task'] = task
+ det[pkg]['Pkg-URL'] = "http://packages.debian.org/unstable/%s/%s" % (getSections(pkg)['Section'], pkg)
+ ### BUG: some packages don't get the right Filename
+ ### see, for example, "treeviewx": it has a Filename
+ ### field, but doesn't get parsed.
+ ### FIX: installed packages with versions newer than the
+ ### one in repositories don't have that field. :@!
+ det[pkg]['Deb-URL'] = "http://ftp.debian.org/%s" % getDebUrl(pkg)
+ except:
+ pass
+
+ if task in official:
+ node.official_head.raw = """<h2>
+<a id="official-debs" name="official-debs"></a>
+ Official Debian packages
+</h2>"""
+ node.official.repeat(renderOfficial, official[task], task)
+
+ if task in todo:
+ error = True
+ else:
+ error = False
+
+ if task in unofficial:
+ node.unofficial_head.raw = """<h2>
+<a id="inofficial-debs" name="inofficial-debs"></a>
+ Inofficial Debian packages
+</h2>"""
+ node.unofficial.repeat(renderUnofficial, unofficial[task])
+ error = False
+
+ if task in unavailable:
+ node.unavailable_head.raw = """<h2>
+<a id="debs-not-available" name="debs-not-available"></a>
+ Debian packages not available
+</h2>"""
+ node.unavailable.repeat(renderUnavailable, unavailable[task])
+ error = False
+
+ if error:
+ # The package probably needs a proper prospective entry in the
+ # task files. Write it to stdout.
+ print "Error: problems with %s" % task
+
+def renderOfficial(node, package, task):
+ # Here we parse just official packages
+ node.shortdesc.content = det[package]['ShortDesc']
+ node.project.raw = "<table class=\"project\" summary=\"%s\">" % package
+ node.anchor.atts['name'] = package
+ node.anchor.atts['id'] = package
+ node.name.content = package.capitalize()
+ node.url.atts['href'] = det[package]['Homepage']
+ if det[package]['Homepage'] == "#":
+ node.url.content = "Homepage not available"
+ else:
+ node.url.content = det[package]['Homepage']
+
+ node.longdesc.raw = det[package]['LongDesc']
+ node.version.content = "Version: %s" % det[package]['Version']
+ node.license.content = "License: %s" % det[package]['License']
+ node.pkgurl.atts['href'] = det[package]['Pkg-URL']
+ node.pkgurl.content = "Official Debian package"
+ node.deburl.atts['href'] = det[package]['Deb-URL']
+ #~ node.deburl.content = "X" ### TODO: add a nice icon here to download the .deb package
+ node.deburl.raw = "<img src=\"/img/deb-icon.png\" />"
+
+def renderUnofficial(node, package):
+ # Here we parse just unofficial packages
+ node.shortdesc.content = package['ShortDesc']
+ node.longdesc.raw = package['LongDesc']
+ node.project.raw = "<table class=\"project\" summary=\"%s\">" % package['Package']
+ node.anchor.atts['name'] = package['Package']
+ node.anchor.atts['id'] = package['Package']
+ node.name.content = package['Package'].capitalize()
+ node.url.atts['href'] = package['Homepage']
+ node.url.content = package['Homepage']
+ node.license.content = "License: %s" %package['License']
+ node.pkgurl.atts['href'] = package['Pkg-URL']
+ node.pkgurl.content = "Inofficial Debian package"
+
+ # Let's try to get the version from the package name
+ # (following Debian standards: <name>_<ver>_<arch>.deb)
+ regex = ".*/%s_(?P<version>.*)_.*\.deb$" % package['Package']
+ p = re.compile(regex)
+ m = p.search(package['Pkg-URL'])
+ if m:
+ node.version.content = "Version: %s" % m.group("version")
+ else:
+ node.version.content = "Version: N/A"
+
+
+def renderUnavailable(node, package):
+ # Parsing unavailable packages :(
+ # PACKAGE THEM! :)
+ name = package['Package']
+ if package['ShortDesc']:
+ node.shortdesc.content = package['ShortDesc']
+ else:
+ node.shortdesc.content = "N/A"
+ node.longdesc.raw = package['LongDesc']
+ node.project.raw = "<table class=\"project\" summary=\"%s\">" % name
+ if package['Responsible']:
+ node.responsible.content = package['Responsible']
+ else:
+ node.responsible.raw = "no one"
+ if package['WNPP']:
+ node.wnpp.raw = " — <a href=\"http://bugs.debian.org/%s\">wnpp</a>" % package['WNPP']
+ node.anchor.atts['name'] = name
+ node.anchor.atts['id'] = name
+ node.name.content = name.capitalize()
+ if package['Homepage']:
+ node.url.atts['href'] = package['Homepage']
+ node.url.content = package['Homepage']
+ else:
+ node.url.atts['href'] = "#"
+ node.url.content = "N/A"
+ if package['License']:
+ node.license.raw = "<?=_('License')?>: %s" % package['License']
+ else:
+ node.license.raw = "<?=_('License')?>: N/A"
+
+# Let's render the Tasks Page index, first
+f = open("%s/htdocs/tasks_idx.tmpl" % base)
+tmpl = HTMLTemplate.Template(renderIndex, f.read())
+f.close()
+f = open("%s/static/tasks/index.php" % base, "w")
+f.write(tmpl.render(tasks))
+f.close()
+
+# Let's render single pages now.
+f = open("%s/htdocs/tasks.tmpl" % base)
+tmpl = HTMLTemplate.Template(renderTasks, f.read())
+f.close()
+
+for task in tasks:
+ f = open("%s/static/tasks/%s.php" % (base, task), "w")
+
+ # This is to avoid useless <br>eaks before closing the cell
+ source = tmpl.render(task, packages[task], task_details[task])
+ f.write(re.sub(r"<br /><br />[ ]*</td>", "</td>", source))
+
+ f.close()
+
More information about the debian-med-commit
mailing list