[apt-proxy-devel] r610 - in tags/1.9.33-0.1: . apt_proxy
apt_proxy/test apt_proxy/twisted_compat bin debian doc doc/po
Chris Halls
halls at costa.debian.org
Thu Aug 3 10:12:50 UTC 2006
Author: halls
Date: Thu Aug 3 10:12:44 2006
New Revision: 610
Added:
tags/1.9.33-0.1/
tags/1.9.33-0.1/README
tags/1.9.33-0.1/apbuilddoc (contents, props changed)
tags/1.9.33-0.1/apt-proxy.conf.test
tags/1.9.33-0.1/apt_proxy/
tags/1.9.33-0.1/apt_proxy/.cvsignore
tags/1.9.33-0.1/apt_proxy/__init__.py
tags/1.9.33-0.1/apt_proxy/apt_proxy.py
tags/1.9.33-0.1/apt_proxy/apt_proxy_conf.py
tags/1.9.33-0.1/apt_proxy/apt_proxytap.py
tags/1.9.33-0.1/apt_proxy/memleak.py
tags/1.9.33-0.1/apt_proxy/misc.py
tags/1.9.33-0.1/apt_proxy/packages.py
tags/1.9.33-0.1/apt_proxy/plugins.tml
tags/1.9.33-0.1/apt_proxy/test/
tags/1.9.33-0.1/apt_proxy/test/__init__.py
tags/1.9.33-0.1/apt_proxy/test/test_apt_proxy.py
tags/1.9.33-0.1/apt_proxy/test/test_config.py
tags/1.9.33-0.1/apt_proxy/test/test_packages.py
tags/1.9.33-0.1/apt_proxy/twisted_compat/
tags/1.9.33-0.1/apt_proxy/twisted_compat/.cvsignore
tags/1.9.33-0.1/apt_proxy/twisted_compat/__init__.py
tags/1.9.33-0.1/apt_proxy/twisted_compat/compat.py
tags/1.9.33-0.1/apt_proxy/twisted_compat/http.py
tags/1.9.33-0.1/aptest (contents, props changed)
tags/1.9.33-0.1/aptest.testdb (contents, props changed)
tags/1.9.33-0.1/aptproxy.kdevelop
tags/1.9.33-0.1/bin/
tags/1.9.33-0.1/bin/apt-proxy (contents, props changed)
tags/1.9.33-0.1/bin/apt-proxy-import (contents, props changed)
tags/1.9.33-0.1/bin/apt-proxy-v1tov2 (contents, props changed)
tags/1.9.33-0.1/debian/
tags/1.9.33-0.1/debian/TODO
tags/1.9.33-0.1/debian/changelog
tags/1.9.33-0.1/debian/compat
tags/1.9.33-0.1/debian/control
tags/1.9.33-0.1/debian/copyright
tags/1.9.33-0.1/debian/default
tags/1.9.33-0.1/debian/dirs
tags/1.9.33-0.1/debian/docs
tags/1.9.33-0.1/debian/init.d (contents, props changed)
tags/1.9.33-0.1/debian/install
tags/1.9.33-0.1/debian/lintian-overrides
tags/1.9.33-0.1/debian/logrotate
tags/1.9.33-0.1/debian/manpages
tags/1.9.33-0.1/debian/po/
tags/1.9.33-0.1/debian/po/POTFILES.in
tags/1.9.33-0.1/debian/po/ca.po
tags/1.9.33-0.1/debian/po/cs.po
tags/1.9.33-0.1/debian/po/da.po
tags/1.9.33-0.1/debian/po/de.po
tags/1.9.33-0.1/debian/po/es.po
tags/1.9.33-0.1/debian/po/fr.po
tags/1.9.33-0.1/debian/po/ja.po
tags/1.9.33-0.1/debian/po/nl.po
tags/1.9.33-0.1/debian/po/pt.po
tags/1.9.33-0.1/debian/po/pt_BR.po
tags/1.9.33-0.1/debian/po/ru.po
tags/1.9.33-0.1/debian/po/sv.po
tags/1.9.33-0.1/debian/po/templates.pot
tags/1.9.33-0.1/debian/po/vi.po
tags/1.9.33-0.1/debian/postinst
tags/1.9.33-0.1/debian/postrm
tags/1.9.33-0.1/debian/preinst (contents, props changed)
tags/1.9.33-0.1/debian/prerm (contents, props changed)
tags/1.9.33-0.1/debian/rules (contents, props changed)
tags/1.9.33-0.1/debian/templates
tags/1.9.33-0.1/doc/
tags/1.9.33-0.1/doc/.cvsignore
tags/1.9.33-0.1/doc/HISTORY
tags/1.9.33-0.1/doc/Makefile
tags/1.9.33-0.1/doc/TODO
tags/1.9.33-0.1/doc/UPGRADING
tags/1.9.33-0.1/doc/apt-proxy-import.8.inc
tags/1.9.33-0.1/doc/apt-proxy-v1tov2.8
tags/1.9.33-0.1/doc/apt-proxy.8
tags/1.9.33-0.1/doc/apt-proxy.add.fr
tags/1.9.33-0.1/doc/apt-proxy.conf
tags/1.9.33-0.1/doc/apt-proxy.conf.5
tags/1.9.33-0.1/doc/design.txt
tags/1.9.33-0.1/doc/offline-tips
tags/1.9.33-0.1/doc/po/
tags/1.9.33-0.1/doc/po/apt-proxy.pot
tags/1.9.33-0.1/doc/po/fr.po
tags/1.9.33-0.1/doc/po4a.cfg
tags/1.9.33-0.1/pychecker (contents, props changed)
tags/1.9.33-0.1/runtests (contents, props changed)
Log:
Import NMU info svn
Added: tags/1.9.33-0.1/README
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/README Thu Aug 3 10:12:44 2006
@@ -0,0 +1,109 @@
+ apt-proxy README
+
+ Chris Halls <halls at debian.org>
+ Revised by Manuel Estrada Sainz <ranty at debian.org>
+
+This is part of the Debian GNU/Linux package for apt-proxy v2.
+
+apt-proxy v2 is dedicated to Manuel Estrada 'ranty' Sainz, who rewrote
+apt-proxy version 1 in Python. He died in a tragic car accident while
+returning from the Free Software conference held at Valencia, Spain in May
+2004. http://www.debian.org/News/2004/20040515
+
+If you have installed apt-proxy using the Debian package, you now need to
+do the following:
+
+1. Edit apt-proxy.conf to select the nearest backend servers for you. See
+ the apt-proxy.conf(5) manpage for details.
+2. Point your apt clients to the proxy. See the apt-proxy(8) manpage for
+ details.
+3. run apt-get update on a client, to initialize the archive directories and
+ file lists.
+4. If you have an apt cache directory full of .debs, you can use
+ apt-proxy-import(8) to copy them into the apt-proxy archive.
+
+If you are installing from the source tar.gz, for example on a non-Debian
+machine, have a look at INSTALL for manual installation instructions.
+
+NOTE: Forget that for now, there is no support for installing on non-Debian
+machines currently, and that may take a while. It also depends on apt being
+installed, which may not be the case on a non-Debian machines.
+
+Using apt-proxy with Debian Potato/Woody?
+-----------------------------------------
+
+apt-proxy versoin 2 will not easily work on this version of Debian, it requires
+versions of twisted, python and python-apt not present in them. Upgrade to
+Sarge or stick with version 1.
+
+Upgrading from apt-proxy v1
+---------------------------
+
+The maintainer scripts should have converted apt-proxy.conf to the new format
+as apt-proxy-v2.conf, and that will be used instead so you can upgrade and
+downgrade freely.
+
+FIXME: This is too short a description, should be extended when we actually
+implement this behavior.
+
+Frequently Asked Questions
+--------------------------
+Here are some of the issues that have been raised about apt-proxy:
+
+- Client setup -
+
+Q: Is apt-proxy really a proxy or is it an HTTP server?
+
+A: Strictly speaking, apt-proxy behaves like an HTTP server, not a proxy. It
+ is very similar in concept to a proxy, because it sits between the client
+ and the backend server, forwarding requests to remote servers. With a true
+ proxy, such as Squid, apt sends a request to the proxy for a file from
+ another site, such as ftp.debian.org. Yet this doesn't make sense for
+ apt-proxy, because it decides itself which backend to use depending on
+ availability and type of file requested. It also has the advantage that all
+ the apt clients do not have to be reconfigured whenever there is a change to
+ the backend server that is to be used.
+
+Q: My web proxy does not know about the apt-proxy machine (for example, you are
+ using a proxy at your ISP), and I can't get apt to contact apt-proxy, even
+ though I have added this to apt.conf:
+
+ Acquire::http::Proxy::<host> "DIRECT";
+
+A: Have you got http_proxy set? From the apt.conf manpage:
+ "The http_proxy environment variable will override all settings."
+
+ So you must unset http_proxy before running apt.
+
+- Using apt-proxy with other clients -
+
+Q: What else is apt-proxy known to work with? How do I configure it?
+
+A1: wget. For example, to get the Woody Release file:
+ wget http://localhost:9999/main/dists/woody/Release
+
+ If you normally use a proxy, and that proxy is not aware of the machine that
+ apt-proxy is running, on you may need to specify --proxy=off.
+
+A2: debootstrap, which uses wget. This means you can easily install new
+ machine using the packages out of your apt-proxy cache. In boot floppies,
+ specify http://APTPROXY:9999/main as your debian mirror (replacing APTPROXY
+ with the name or IP address of the machine where apt-proxy is running).
+
+A3: rootstrap, a tool for making root images for user-mode-linux. Assuming
+ that you are running rootstrap on the same machine as apt-proxy and have
+ used the default network addresses 192.168.10.x, put this in
+ rootstrap.conf:
+
+ mirror=http://192.168.10.1:9999/main
+
+ [Note: during testing, we encountered a strange problem where rootstrap
+ thought the architecture was i386-none, so we had to add --arch=i386 to the
+ deboostrap call in /usr/lib/rootstrap/modules/debian.]
+
+A4: pbuilder, which also uses debootstrap. Add this to /etc/pbuilderrc:
+
+ MIRRORSITE=http://APTPROXT:9999/main
+ NONUSMIRRORSITE=http://APTPROXT:9999/non-US
+
+April 2004
Added: tags/1.9.33-0.1/apbuilddoc
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apbuilddoc Thu Aug 3 10:12:44 2006
@@ -0,0 +1,6 @@
+#!/bin/sh
+[ ! -d html ] && mkdir html
+cd html
+PYTHONPATH="`pwd`:../apt_proxy" pydoc -w ../apt_proxy/*
+
+
Added: tags/1.9.33-0.1/apt-proxy.conf.test
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt-proxy.conf.test Thu Aug 3 10:12:44 2006
@@ -0,0 +1,147 @@
+[DEFAULT]
+;; All times are in seconds, but you can add a suffix
+;; for minutes(m), hours(h) or days(d)
+
+;; Server IP to listen on
+;address = 192.168.0.254
+
+;; Server port to listen on
+port = 9998
+
+;; Control files (Packages/Sources/Contents) refresh rate
+;;
+;; Minimum time between attempts to refresh a file
+min_refresh_delay = 1h
+
+;; Minimum age of a file before attempting an update (NOT YET IMPLEMENTED)
+;min_age = 23h
+
+;; Uncomment to make apt-proxy continue downloading even if all
+;; clients disconnect. This is probably not a good idea on a
+;; dial up line.
+;; complete_clientless_downloads = 1
+
+;; Debugging settings.
+;; for all debug information use this:
+;; debug = all:9
+debug = all:9
+
+;; Debugging remote python console
+;; Do not enable in an untrusted environment
+;telnet_port = 9998
+;telnet_user = apt-proxy
+;telnet_password = secret
+
+;; Network timeout when retrieving from backend servers
+timeout = 15
+
+;; Cache directory for apt-proxy
+cache_dir = testcache
+
+;; Use passive FTP? (default=on)
+;passive_ftp = on
+
+;; Use HTTP proxy?
+;http_proxy = host:port
+
+;; Enable HTTP pipelining within apt-proxy (for test purposes)
+;disable_pipelining=0
+
+;;--------------------------------------------------------------
+;; Cache housekeeping
+
+;; Time to perform periodic housekeeping:
+;; - delete files that have not been accessed in max_age
+;; - scan cache directories and update internal tables
+cleanup_freq = 1d
+
+;; Maximum age of files before deletion from the cache (seconds)
+max_age = 120d
+
+;; Maximum number of versions of a .deb to keep per distribution
+max_versions = 3
+
+;; Add HTTP backends dynamicaly if not already defined? (default=on)
+;dynamic_backends = on
+
+username=chris
+
+;;---------------------------------------------------------------
+;;---------------------------------------------------------------
+;; Backend servers
+;;
+;; Place each server in its own [section]
+
+[debian]
+;; The main Debian archive
+;; You can override the default timeout like this:
+;timeout = 30
+
+;; Rsync server used to rsync the Packages file (NOT YET IMPLEMENTED)
+;;rsyncpackages = rsync://ftp.de.debian.org/debian
+
+;; Backend servers, in order of preference
+backends =
+ http://ftp.us.debian.org/debian
+ http://ftp.de.debian.org/debian
+ http://ftp2.de.debian.org/debian
+ ftp://ftp.uk.debian.org/debian
+
+
+[debian-non-US]
+;; Debian debian-non-US archive
+;timeout will be the global value
+backends =
+ http://ftp.uk.debian.org/debian-non-US
+ http://ftp.de.debian.org/debian-non-US
+ ftp://ftp.uk.debian.org/debian
+
+[security]
+;; Debian security archive
+backends =
+ http://security.debian.org/debian-security
+ http://ftp2.de.debian.org/debian-security
+
+[ubuntu]
+;; Ubuntu archive
+backends = http://archive.ubuntu.com/ubuntu
+
+[ubuntu-security]
+;; Ubuntu security updates
+backends = http://security.ubuntu.com/ubuntu
+
+;[openoffice]
+;; OpenOffice.org packages
+;backends =
+; http://ftp.freenet.de/pub/debian-openoffice
+; http://ftp.sh.cvut.cz/MIRRORS/OpenOffice.deb
+; http://borft.student.utwente.nl/debian
+
+;[apt-proxy]
+;; Apt-proxy new versions
+;backends = http://apt-proxy.sourceforge.net/apt-proxy
+
+;[backports.org]
+;; backports.org
+;backends = http://backports.org/debian
+
+;[blackdown]
+;; Blackdown Java
+;backends = http://ftp.gwdg.de/pub/languages/java/linux/debian
+
+
+;[debian-people]
+;; people.debian.org
+;backends = http://people.debian.org
+
+;[emdebian]
+;; The Emdebian project
+;backends = http://emdebian.sourceforge.net/emdebian
+
+;[rsync]
+;; An example using an rsync server. This is not recommended
+;; unless http is not available, becuause rsync is only more
+;; efficient for transferring uncompressed files and puts much
+;; more overhead on the server. See the rsyncpacakges parameter
+;; for a way of rsyncing just the Packages files.
+;backends = rsync://ftp.uk.debian.org/debian
Added: tags/1.9.33-0.1/apt_proxy/.cvsignore
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/.cvsignore Thu Aug 3 10:12:44 2006
@@ -0,0 +1,2 @@
+*.pyo
+*.pyc
Added: tags/1.9.33-0.1/apt_proxy/__init__.py
==============================================================================
Added: tags/1.9.33-0.1/apt_proxy/apt_proxy.py
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/apt_proxy.py Thu Aug 3 10:12:44 2006
@@ -0,0 +1,1962 @@
+#
+# Copyright (C) 2002 Manuel Estrada Sainz <ranty at debian.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from twisted.internet import reactor, defer, abstract, protocol
+from twisted.protocols import ftp, basic
+
+import os, stat, signal, fcntl, exceptions
+from os.path import dirname, basename
+import tempfile
+import glob
+import re
+import urlparse
+import time
+import string
+import packages
+from twisted.python.failure import Failure
+import memleak
+from twisted.internet import error
+#from posixfile import SEEK_SET, SEEK_CUR, SEEK_END
+#since posixfile is considered obsolete I'll define the SEEK_* constants
+#myself.
+SEEK_SET = 0
+SEEK_CUR = 1
+SEEK_END = 2
+
+from types import *
+
+#sibling imports
+import misc
+log = misc.log
+
+from twisted_compat import compat
+from twisted_compat import http
+
+status_dir = '.apt-proxy'
+
+class FileType:
+ """
+ This is just a way to distinguish between different filetypes.
+
+ self.regex: regular expression that files of this type should
+ match. It could probably be replaced with something simpler,
+ but... o well, it works.
+
+ self.contype: mime string for the content-type http header.
+
+ mutable: do the contents of this file ever change? Files such as
+ .deb and .dsc are never changed once they are created.
+
+ """
+ def __init__ (self, regex, contype, mutable):
+ self.regex = regex
+ self.contype = contype
+ self.mutable = mutable
+
+ def check (self, name):
+ "Returns true if name is of this filetype"
+ if self.regex.search(name):
+ return 1
+ else:
+ return 0
+
+# Set up the list of filetypes that we are prepared to deal with.
+# If it is not in this list, then we will ignore the file and
+# return an error.
+filetypes = (
+ FileType(re.compile(r"\.deb$"), "application/dpkg", 0),
+ FileType(re.compile(r"\.udeb$"), "application/dpkg", 0),
+ FileType(re.compile(r"\.tar\.gz$"), "application/x-gtar", 0),
+ FileType(re.compile(r"\.dsc$"),"text/plain", 0),
+ FileType(re.compile(r"\.diff\.gz$"), "application/x-gzip", 0),
+ FileType(re.compile(r"\.gz$"), "application/x-gzip", 1),
+ FileType(re.compile(r"\.bin$"), "application/octet-stream", 0),
+ FileType(re.compile(r"\.tgz$"), "application/x-gtar", 0),
+ FileType(re.compile(r"\.txt$"), "application/plain-text", 1),
+ FileType(re.compile(r"\.html$"), "application/text-html", 1),
+
+ FileType(re.compile(r"/(Packages|Release(\.gpg)?|Sources|Contents-.*)"
+ r"(\.(gz|bz2))?$"),
+ "text/plain", 1),
+
+ FileType(re.compile(r"\.rpm$"), "application/rpm", 0),
+
+ FileType(re.compile(r"/(pkglist|release|srclist)(\.(\w|-)+)?"
+ r"(\.(gz|bz2))?$"),
+ "text/plain", 1),
+ )
+
+class FileVerifier(protocol.ProcessProtocol):
+ """
+ Verifies the integrity of a file by running an external
+ command.
+
+ self.deferred: a deferred that will be triggered when the command
+ completes, or if a timeout occurs.
+
+ Sample:
+
+ verifier = FileVerifier(self)
+ verifier.deferred.addCallbacks(callback_if_ok, callback_if_fail)
+
+ then either callback_if_ok or callback_if_fail will be called
+ when the subprocess finishes execution.
+
+ Checkout twisted.internet.defer.Deferred on how to use self.deferred
+
+ """
+ def __init__(self, request):
+ self.factory = request.factory
+ self.deferred = defer.Deferred() # Deferred that passes status back
+ self.path = request.local_file
+
+ if re.search(r"\.deb$", self.path):
+ exe = '/usr/bin/dpkg'
+ args = (exe, '--fsys-tarfile', self.path)
+ elif re.search(r"\.gz$", self.path):
+ exe = '/bin/gunzip'
+ args = (exe, '-t', '-v', self.path)
+ elif re.search(r"\.bz2$", self.path):
+ exe = '/usr/bin/bunzip2'
+ args = (exe, '--test', self.path)
+ else:
+ # Unknown file, just check it is not 0 size
+ try:
+ filesize = os.stat(self.path)[stat.ST_SIZE]
+ except:
+ filesize = 0
+
+ if(os.stat(self.path)[stat.ST_SIZE]) < 1:
+ log.debug('Verification failed for ' + self.path)
+ self.failed()
+ else:
+ log.debug('Verification skipped for ' + self.path)
+ self.deferred.callback(None)
+ return
+
+ log.debug("starting verification: " + exe + " " + str(args))
+ self.nullhandle = open("/dev/null", "w")
+ self.process = reactor.spawnProcess(self, exe, args, childFDs = { 0:"w", 1:self.nullhandle.fileno(), 2:"r" })
+ self.laterID = reactor.callLater(self.factory.config.timeout, self.timedout)
+
+ def connectionMade(self):
+ self.data = ''
+
+ def outReceived(self, data):
+ #we only care about errors
+ pass
+
+ def errReceived(self, data):
+ self.data = self.data + data
+
+ def failed(self):
+ log.debug("verification failed: %s"%(self.path), 'verify', 1)
+ os.unlink(self.path)
+ self.deferred.errback(None)
+
+ def timedout(self):
+ """
+ this should not happen, but if we timeout, we pretend that the
+ operation failed.
+ """
+ self.laterID=None
+ log.debug("Process Timedout:",'verify')
+ self.failed()
+
+ def processEnded(self, reason=None):
+ """
+ This get's automatically called when the process finishes, we check
+ the status and report through the Deferred.
+ """
+ __pychecker__ = 'unusednames=reason'
+ #log.debug("Process Status: %d" %(self.process.status),'verify')
+ #log.debug(self.data, 'verify')
+ if self.laterID:
+ self.laterID.cancel()
+ if self.process.status == 0:
+ self.deferred.callback(None)
+ else:
+ self.failed()
+
+def findFileType(name):
+ "Look for the FileType of 'name'"
+ for type in filetypes:
+ if type.check(name):
+ return type
+ return None
+
+class TempFile (file):
+ def __init__(self, mode='w+b', bufsize=-1):
+ (fd, name) = tempfile.mkstemp('.apt-proxy')
+ os.close(fd)
+ file.__init__(self, name, mode, bufsize)
+ os.unlink(name)
+ def append(self, data):
+ self.seek(0, SEEK_END)
+ self.write(data)
+ def size(self):
+ return self.tell()
+ def read_from(self, size=-1, start=None):
+ if start != None:
+ self.seek(start, SEEK_SET)
+ data = file.read(self, size)
+ return data
+
+
+class Fetcher:
+ """
+ This is the base class for all Fetcher*, it tries to hold as much
+ common code as posible.
+
+ Subclasses of this class are the ones responsible for contacting
+ the backend servers and fetching the actual data.
+ """
+ gzip_convert = re.compile(r"/Packages$")
+ post_convert = re.compile(r"/Packages.gz$")
+ status_code = http.OK
+ status_message = None
+ requests = None
+ request = None
+ length = None
+ transport = None
+
+ def insert_request(self, request):
+ """
+ Request should be served through this Fetcher because it asked for
+ the same uri that we are waiting for.
+
+ We also have to get it up to date, give it all received data, send it
+ the appropriate headers and set the response code.
+ """
+ if request in self.requests:
+ raise RuntimeError, \
+ 'this request is already assigned to this Fetcher'
+ self.requests.append(request)
+ request.apFetcher = self
+ if (self.request):
+ self.update_request(request)
+
+ def update_request(self, request):
+ """
+ get a new request up to date
+ """
+ request.local_mtime = self.request.local_mtime
+ request.local_size = self.request.local_size
+ if(self.status_code != None):
+ request.setResponseCode(self.status_code, self.status_message)
+ for name, value in self.request.headers.items():
+ request.setHeader(name, value)
+ if self.transfered.size() != 0:
+ request.write(self.transfered.read_from(start=0))
+
+ def remove_request(self, request):
+ """
+ Request should NOT be served through this Fetcher, the client
+ probably closed the connection.
+
+ If this is our last request, we may also close the connection with the
+ server depending on the configuration.
+
+ We keep the last request for reference even if the client closed the
+ connection.
+ """
+ self.requests.remove(request)
+ if len(self.requests) == 0:
+ log.debug("Last request removed",'Fetcher')
+ if not self.factory.config.complete_clientless_downloads:
+ if self.transport:
+ log.debug(
+ "telling the transport to loseConnection",'Fetcher')
+ try:
+ self.transport.loseConnection()
+ except KeyError:
+ # Rsync fetcher already loses conneciton for us
+ pass
+ if hasattr(self, 'loseConnection'):
+ self.loseConnection()
+ else:
+ self.request = self.requests[0]
+ request.apFetcher = None
+
+ def transfer_requests(self, fetcher):
+ "Transfer all requests from self to fetcher"
+ for req in self.requests:
+ self.remove_request(req)
+ fetcher.insert_request(req)
+
+ def setResponseCode(self, code, message=None):
+ "Set response code for all requests"
+ #log.debug('Response code: %d - %s' % (code, message),'Fetcher')
+ self.status_code = code
+ self.status_message = message
+ for req in self.requests:
+ req.setResponseCode(code, message)
+
+ def setResponseHeader(self, name, value):
+ "set 'value' for header 'name' on all requests"
+ for req in self.requests:
+ req.setHeader(name, value)
+
+ def __init__(self, request=None):
+ self.requests = []
+ self.transfered = TempFile()
+ if(request):
+ self.activate(request)
+
+ def activate(self, request):
+ log.debug(str(request.backend) + request.uri, 'Fetcher.activate')
+ self.local_file = request.local_file
+ self.local_mtime = request.local_mtime
+ self.factory = request.factory
+ self.request = request
+ request.content.read()
+
+ for req in self.requests:
+ self.update_request(req)
+ self.requests.append(request)
+
+ request.apFetcher = self
+ if self.factory.runningFetchers.has_key(request.uri):
+ raise RuntimeError, 'There already is a running fetcher'
+ self.factory.runningFetchers[request.uri]=self
+
+ def apDataReceived(self, data):
+ """
+ Should be called from the subclasses when data is available for
+ streaming.
+
+ Keeps all transfered data in 'self.transfered' for requests which arrive
+ later and to write it in the cache at the end.
+
+ Note: self.length if != None is the amount of data pending to be
+ received.
+ """
+ if self.length != None:
+ self.transfered.append(data[:self.length])
+ for req in self.requests:
+ req.write(data[:self.length])
+ else:
+ self.transfered.append(data)
+ for req in self.requests:
+ req.write(data)
+
+ def apDataEnd(self, data, saveData=True):
+ """
+ Called by subclasses when the data transfer is over.
+
+ -caches the received data if everyting went well (if saveData=True)
+ -takes care of mtime and atime
+ -finishes connection with server and the requests
+
+ """
+ import shutil
+ log.debug("Finished receiving data, status:%d saveData:%d" %(self.status_code, saveData), 'Fetcher');
+ if (self.status_code == http.OK):
+ if saveData:
+ dir = dirname(self.local_file)
+ if(not os.path.exists(dir)):
+ os.makedirs(dir)
+ f = open(self.local_file, "w")
+ fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
+ f.truncate(0)
+ if type(data) is StringType:
+ f.write(data)
+ else:
+ data.seek(0, SEEK_SET)
+ shutil.copyfileobj(data, f)
+ f.close()
+ if self.local_mtime != None:
+ os.utime(self.local_file, (time.time(), self.local_mtime))
+ else:
+ log.debug("no local time: "+self.local_file,'Fetcher')
+ os.utime(self.local_file, (time.time(), 0))
+
+ self.factory.file_served(self.request.uri)
+
+ #self.request.backend.get_packages_db().packages_file(self.request.uri)
+
+ if self.transport:
+ try:
+ self.transport.loseConnection()
+ except exceptions.KeyError:
+ # Couldn't close connection - already closed?
+ log.debug("transport.loseConnection() - "
+ "connection already closed", 'Fetcher')
+ pass
+
+ for req in self.requests:
+ req.finish()
+
+ self.transfered.close()
+ self.apEnd()
+
+ def apEnd(self):
+ """
+ Called by subclasses when apDataEnd does too many things.
+
+ Let's everyone know that we are not the active Fetcher for our uri.
+ """
+ try:
+ del self.factory.runningFetchers[self.request.uri]
+ except exceptions.KeyError:
+ log.debug("We are not on runningFetchers!!!",'Fetcher')
+ log.debug("Class is not in runningFetchers: "+str(self.__class__),
+ 'Fetcher')
+ if self.request:
+ log.debug(' URI:' + self.request.uri, 'Fetcher')
+ log.debug('Running fetchers: '
+ +str(self.factory.runningFetchers),'Fetcher')
+ #raise exceptions.KeyError
+ for req in self.requests[:]:
+ self.remove_request(req)
+
+ import gc
+ #Cleanup circular references
+ reactor.callLater(5, gc.collect)
+
+ def apEndCached(self):
+ """
+ A backend has indicated that this file has not changed,
+ so serve the file from the disk cache
+ """
+ self.setResponseCode(http.OK)
+ self.apEndTransfer(FetcherCachedFile)
+
+ def apEndTransfer(self, fetcher_class):
+ """
+ Remove this Fetcher and transfer all it's requests to a new instance of
+ 'fetcher_class'.
+ """
+ #Consider something like this:
+ #req = dummyFetcher.fix_ref_request()
+ #fetcher = fetcher_class()
+ #dummyFetcher.transfer_requests(fetcher)
+ #dummyFetcher.apEnd()
+ #fetcher.activate(req)
+
+ #self.setResponseCode(http.OK)
+ requests = self.requests[:]
+ self.apEnd() # Remove requests from this fetcher
+ fetcher = None
+ for req in requests:
+ if (fetcher_class != FetcherCachedFile or req.serve_if_cached):
+ running = req.factory.runningFetchers
+ if (running.has_key(req.uri)):
+ #If we have an active Fetcher just use that
+ log.debug("have active Fetcher",'Fetcher')
+ running[req.uri].insert_request(req)
+ fetcher = running[req.uri]
+ else:
+ fetcher = fetcher_class(req)
+ else:
+ req.finish()
+ return fetcher
+
+ def connectionFailed(self, reason=None):
+ """
+ Tell our requests that the connection with the server failed.
+ """
+ msg = '[%s] Connection Failed: %s/%s'%(
+ self.request.backend.base,
+ self.request.backendServer.path, self.request.backend_uri)
+
+ if reason:
+ msg = '%s (%s)'%(msg, reason.getErrorMessage())
+ log.debug("Connection Failed: "+str(reason), 'Fetcher')
+ log.err(msg)
+
+ # Look for alternative fetchers
+ if not self.request.activateNextBackendServer(self):
+ # No more backends, send error response back to client
+ if reason.check(error.ConnectError):
+ self.setResponseCode(http.SERVICE_UNAVAILABLE, "Connect Error")
+ else:
+ self.setResponseCode(http.SERVICE_UNAVAILABLE)
+ self.apDataReceived("")
+ self.apDataEnd(self.transfered, False)
+ #Because of a bug in tcp.Client we may be called twice,
+ #Make sure that next time nothing will happen
+ #FIXME: This hack is probably not anymore pertinent.
+ self.connectionFailed = lambda : log.debug('connectionFailed(2)',
+ 'Fetcher','9')
+
+
+class FetcherDummy(Fetcher):
+ """
+ """
+ gzip_convert = re.compile(r"^Nothing should match this$")
+ post_convert = re.compile(r"^Nothing should match this$")
+ status_code = http.INTERNAL_SERVER_ERROR
+ status_message = None
+
+ def insert_request(self, request):
+ """
+ """
+ if request in self.requests:
+ raise RuntimeError, \
+ 'this request is already assigned to this Fetcher'
+ self.requests.append(request)
+ request.apFetcher = self
+
+ def remove_request(self, request):
+ """
+ """
+ #make sure that it has updated values, since the requests
+ #may be cached and we need them to serve it.
+ request.local_mtime = self.request.local_mtime
+ request.local_size = self.request.local_size
+
+ self.requests.remove(request)
+ request.apFetcher = None
+
+ def fix_ref_request(self):
+ if self.requests != []:
+ if self.request not in self.requests:
+ request = self.requests[0]
+ request.local_mtime = self.request.local_mtime
+ request.local_size = self.request.local_size
+ self.request = request
+ self.remove_request(self.request)
+ else:
+ self.request = None
+
+ return self.request
+
+class FetcherFile(Fetcher):
+
+ def activate(self, request):
+ Fetcher.activate(self, request)
+ log.debug("FetcherFile.activate(): uri='%s' server='%s'" % (request.uri, request.backendServer.uri))
+ if not request.apFetcher:
+ log.debug("no request.apFetcher")
+ return
+
+ self.factory.file_served(request.uri)
+
+ # start the transfer
+ self.local_file = request.backendServer.uri[len("file:"):]+ request.uri
+ if not os.path.exists(self.local_file):
+ log.debug("not found: %s" % self.local_file)
+ request.setResponseCode(http.NOT_FOUND)
+ request.write("")
+ request.finish()
+ self.remove_request(request)
+ Fetcher.apEnd(self)
+ return
+ self.local_size = os.stat(self.local_file)[stat.ST_SIZE]
+
+ log.debug("Serving local file: " + self.local_file + " size:" + str(self.local_size), 'FetcherCachedFile')
+ file = open(self.local_file,'rb')
+ fcntl.lockf(file.fileno(), fcntl.LOCK_SH)
+
+ request.setHeader("Content-Length", self.local_size)
+ #request.setHeader("Last-modified",
+ # http.datetimeToString(request.local_mtime))
+ basic.FileSender().beginFileTransfer(file, request) \
+ .addBoth(self.file_transfer_complete, request) \
+ .addBoth(lambda r: file.close())
+
+ # A file transfer has completed
+ def file_transfer_complete(self, result, request):
+ log.debug("transfer complete", 'FetcherCachedFile')
+ request.finish()
+ # Remove this client from request list
+ self.remove_request(request)
+ if len(self.requests) == 0:
+ Fetcher.apEnd(self)
+
+class FetcherHttp(Fetcher, http.HTTPClient):
+
+ forward_headers = [
+ 'last-modified',
+ 'content-length'
+ ]
+ log_headers = None
+
+ proxy_host = None
+ proxy_port = None
+
+ def activate(self, request):
+ Fetcher.activate(self, request)
+
+ if not self.factory.config.http_proxy is '':
+ (self.proxy_host, self.proxy_port) = request.factory.config.http_proxy.split(':')
+
+ if not request.apFetcher:
+ return
+
+ class ClientFactory(protocol.ClientFactory):
+ "Dummy ClientFactory to comply with current twisted API"
+ #FIXME: Double check this, haggai thinks it is to blame for the
+ #hangs.
+ def __init__(self, instance):
+ self.instance = instance
+ def buildProtocol(self, addr):
+ return self.instance
+ def clientConnectionFailed(self, connector, reason):
+ self.instance.connectionFailed(reason)
+ def clientConnectionLost(self, connector, reason):
+ log.debug("XXX clientConnectionLost", "http-client")
+
+ if not self.proxy_host:
+ reactor.connectTCP(request.backendServer.host, request.backendServer.port,
+ ClientFactory(self), request.backend.config.timeout)
+ else:
+ reactor.connectTCP(self.proxy_host, int(self.proxy_port),
+ ClientFactory(self), request.backend.config.timeout)
+ def connectionMade(self):
+ if not self.proxy_host:
+ self.sendCommand(self.request.method, self.request.backendServer.path
+ + "/" + self.request.backend_uri)
+ else:
+ self.sendCommand(self.request.method, "http://"
+ + self.request.backendServer.host + ":" + str(self.request.backendServer.port)
+ + "/" + self.request.backendServer.path
+ + "/" + self.request.backend_uri)
+
+ self.sendHeader('host', self.request.backendServer.host)
+
+ if self.local_mtime != None:
+ datetime = http.datetimeToString(self.local_mtime)
+ self.sendHeader('if-modified-since', datetime)
+
+ self.endHeaders()
+
+ def handleStatus(self, version, code, message):
+ __pychecker__ = 'unusednames=version,message'
+ log.debug('handleStatus %s - %s' % (code, message), 'http_client')
+ self.status_code = int(code)
+
+ # Keep a record of server response even if overriden later by setReponseCode
+ self.http_status = self.status_code
+
+ self.setResponseCode(self.status_code)
+
+ def handleHeader(self, key, value):
+
+ log.debug("Received: " + key + " " + str(value))
+ key = string.lower(key)
+
+ if key == 'last-modified':
+ self.local_mtime = http.stringToDatetime(value)
+
+ if key in self.forward_headers:
+ self.setResponseHeader(key, value)
+
+ def handleEndHeaders(self):
+ if self.http_status == http.NOT_MODIFIED:
+ log.debug("NOT_MODIFIED " + str(self.status_code),'http_client')
+ self.apEndCached()
+
+ def rawDataReceived(self, data):
+ self.apDataReceived(data)
+
+ def handleResponse(self, buffer):
+ if self.length == 0:
+ self.setResponseCode(http.NOT_FOUND)
+ # print "length: " + str(self.length), "response:", self.status_code
+ if self.http_status == http.NOT_MODIFIED:
+ self.apDataEnd(self.transfered, False)
+ else:
+ self.apDataEnd(self.transfered, True)
+
+ def lineReceived(self, line):
+ """
+ log the line and handle it to the appropriate the base classe.
+
+ The location header gave me trouble at some point, so I filter it just
+ in case.
+
+ Note: when running a class method directly and not from an object you
+ have to give the 'self' parameter manualy.
+ """
+ #log.debug(line,'http_client')
+ if self.log_headers == None:
+ self.log_headers = line
+ else:
+ self.log_headers += ", " + line;
+ if not re.search('^Location:', line):
+ http.HTTPClient.lineReceived(self, line)
+
+ def sendCommand(self, command, path):
+ "log the line and handle it to the base class."
+ log.debug(command + ":" + path,'http_client')
+ http.HTTPClient.sendCommand(self, command, path)
+
+ def endHeaders(self):
+ "log and handle to the base class."
+ if self.log_headers != None:
+ log.debug(" Headers: " + self.log_headers, 'http_client')
+ self.log_headers = None;
+ http.HTTPClient.endHeaders(self)
+
+ def sendHeader(self, name, value):
+ "log and handle to the base class."
+ log.debug(name + ":" + value,'http_client')
+ http.HTTPClient.sendHeader(self, name, value)
+
+class FetcherFtp(Fetcher, protocol.Protocol):
+ """
+ This is the secuence here:
+
+ -Start and connect the FTPClient
+ -Ask for mtime
+ -Ask for size
+ -if couldn't get the size
+ -try to get it by listing
+ -get all that juicy data
+
+ NOTE: Twisted's FTPClient code uses it's own timeouts here and there,
+ so the timeout specified for the backend may not always be used
+ """
+ def activate (self, request):
+ Fetcher.activate(self, request)
+ if not request.apFetcher:
+ return
+
+ self.passive_ftp = self.request.backend.config.passive_ftp
+
+ self.remote_file = (self.request.backendServer.path + "/"
+ + self.request.backend_uri)
+
+ from twisted.internet.protocol import ClientCreator
+
+ if not request.backendServer.username:
+ creator = ClientCreator(reactor, ftp.FTPClient, passive=0)
+ else:
+ creator = ClientCreator(reactor, ftp.FTPClient, request.backendServer.username,
+ request.backendServer.password, passive=0)
+ d = creator.connectTCP(request.backendServer.host, request.backendServer.port,
+ request.backend.config.timeout)
+ d.addCallback(self.controlConnectionMade)
+ d.addErrback(self.connectionFailed)
+
+ def controlConnectionMade(self, ftpclient):
+ self.ftpclient = ftpclient
+
+ if(self.passive_ftp):
+ log.debug('Got control connection, using passive ftp', 'ftp_client')
+ self.ftpclient.passive = 1
+ else:
+ log.debug('Got control connection, using active ftp', 'ftp_client')
+ self.ftpclient.passive = 0
+
+ if log.isEnabled('ftp_client'):
+ self.ftpclient.debug = 1
+
+ self.ftpFetchMtime()
+
+ def ftpFinish(self, code, message=None):
+ "Finish the transfer with code 'code'"
+ self.ftpclient.quit()
+ self.setResponseCode(code, message)
+ self.apDataReceived("")
+ self.apDataEnd(self.transfered)
+
+ def ftpFinishCached(self):
+ "Finish the transfer giving the requests the cached file."
+ self.ftpclient.quit()
+ self.apEndCached()
+
+ def ftpFetchMtime(self):
+ "Get the modification time from the server."
+ def apFtpMtimeFinish(msgs, fetcher, fail):
+ """
+ Got an answer to the mtime request.
+
+ Someone should check that this is timezone independent.
+ """
+ code = None
+ if not fail:
+ code, msg = msgs[0].split()
+ mtime = None
+ if code == '213':
+ time_tuple=time.strptime(msg[:14], "%Y%m%d%H%M%S")
+ #replace day light savings with -1 (current)
+ time_tuple = time_tuple[:8] + (-1,)
+ #correct the result to GMT
+ mtime = time.mktime(time_tuple) - time.altzone
+ if (fetcher.local_mtime and mtime
+ and fetcher.local_mtime >= mtime):
+ fetcher.ftpFinishCached()
+ else:
+ fetcher.local_mtime = mtime
+ fetcher.ftpFetchSize()
+
+ d = self.ftpclient.queueStringCommand('MDTM ' + self.remote_file)
+ d.addCallbacks(apFtpMtimeFinish, apFtpMtimeFinish,
+ (self, 0), None, (self, 1), None)
+
+ def ftpFetchSize(self):
+ "Get the size of the file from the server"
+ def apFtpSizeFinish(msgs, fetcher, fail):
+ code = None
+ if not fail:
+ code, msg = msgs[0].split()
+ if code != '213':
+ log.debug("SIZE FAILED",'ftp_client')
+ fetcher.ftpFetchList()
+ else:
+ fetcher.setResponseHeader('content-length', msg)
+ fetcher.ftpFetchFile()
+
+ d = self.ftpclient.queueStringCommand('SIZE ' + self.remote_file)
+ d.addCallbacks(apFtpSizeFinish, apFtpSizeFinish,
+ (self, 0), None, (self, 1), None)
+
+ def ftpFetchList(self):
+ "If ftpFetchSize didn't work try to get the size with a list command."
+ def apFtpListFinish(msg, filelist, fetcher, fail):
+ __pychecker__ = 'unusednames=msg'
+ if fail:
+ fetcher.ftpFinish(http.INTERNAL_SERVER_ERROR)
+ return
+ if len(filelist.files)== 0:
+ fetcher.ftpFinish(http.NOT_FOUND)
+ return
+ file = filelist.files[0]
+ fetcher.setResponseHeader('content-length', file['size'])
+ fetcher.ftpFetchFile()
+ filelist = ftp.FTPFileListProtocol()
+ d = self.ftpclient.list(self.remote_file, filelist)
+ d.addCallbacks(apFtpListFinish, apFtpListFinish,
+ (filelist, self, 0), None,
+ (filelist, self, 1), None)
+
+ def ftpFetchFile(self):
+ "And finally, we ask for the file."
+ def apFtpFetchFinish(msg, code, status, fetcher):
+ __pychecker__ = 'unusednames=msg,status'
+ fetcher.ftpFinish(code)
+ log.debug('ftpFetchFile: ' + self.remote_file, 'ftp_client')
+ d = self.ftpclient.retrieveFile(self.remote_file, self)
+ d.addCallbacks(apFtpFetchFinish, apFtpFetchFinish,
+ (http.OK, "good", self), None,
+ (http.NOT_FOUND, "fail", self), None)
+
+ def dataReceived(self, data):
+ self.setResponseCode(http.OK)
+ self.apDataReceived(data)
+
+ def connectionLost(self, reason=None):
+ """
+ Maybe we should do some recovery here, I don't know, but the Deferred
+ should be enough.
+ """
+ log.debug("lost connection: %s"%(reason),'ftp_client')
+
+class FetcherGzip(Fetcher, protocol.ProcessProtocol):
+ """
+ This is a fake Fetcher, it uses the real Fetcher from the request's
+ backend via LoopbackRequest to get the data and gzip's or gunzip's as
+ needed.
+
+ NOTE: We use the serve_cached=0 parameter to Request.fetch so if
+ it is cached it doesn't get uselessly read, we just get it from the cache.
+ """
+ post_convert = re.compile(r"^Should not match anything$")
+ gzip_convert = post_convert
+
+ exe = '/bin/gzip'
+ def activate(self, request, postconverting=0):
+ log.debug("FetcherGzip request:" + str(request.uri) + " postconvert:" + str(postconverting), 'gzip')
+ Fetcher.activate(self, request)
+ if not request.apFetcher:
+ return
+
+ self.args = (self.exe, '-c', '-9', '-n')
+ if(log.isEnabled('gzip',9)):
+ self.args += ('-v',)
+
+ if request.uri[-3:] == '.gz':
+ host_uri = request.uri[:-3]
+ else:
+ host_uri = request.uri+'.gz'
+ self.args += ('-d',)
+ self.host_file = self.factory.config.cache_dir + host_uri
+ self.args += (self.host_file,)
+
+ running = self.factory.runningFetchers
+ if not postconverting or running.has_key(host_uri):
+ #Make sure that the file is there
+ loop = LoopbackRequest(request, self.host_transfer_done)
+ loop.uri = host_uri
+ loop.local_file = self.host_file
+ loop.process()
+ self.loop_req = loop
+ loop.serve_if_cached=0
+ if running.has_key(host_uri):
+ #the file is on it's way, wait for it.
+ running[host_uri].insert_request(loop)
+ else:
+ #we are not postconverting, so we need to fetch the host file.
+ loop.fetch(serve_cached=0)
+ else:
+ #The file should be there already.
+ self.loop_req = None
+ self.host_transfer_done()
+
+ def host_transfer_done(self):
+ """
+ Called by our LoopbackRequest when the real Fetcher calls
+ finish() on it.
+
+ If everything went well, check mtimes and only do the work if needed.
+
+ If posible arrange things so the target file gets the same mtime as
+ the host file.
+ """
+ log.debug('transfer done', 'gzip')
+ if self.loop_req and self.loop_req.code != http.OK:
+ self.setResponseCode(self.loop_req.code,
+ self.loop_req.code_message)
+ self.apDataReceived("")
+ self.apDataEnd("")
+ return
+
+ if os.path.exists(self.host_file):
+ self.local_mtime = os.stat(self.host_file)[stat.ST_MTIME]
+ old_mtime = None
+ if os.path.exists(self.local_file):
+ old_mtime = os.stat(self.local_file)[stat.ST_MTIME]
+ if self.local_mtime == old_mtime:
+ self.apEndCached()
+ else:
+ log.debug("Starting process: " + self.exe + " " + str(self.args), 'gzip')
+ self.process = reactor.spawnProcess(self, self.exe, self.args)
+
+ def outReceived(self, data):
+ self.setResponseCode(http.OK)
+ self.apDataReceived(data)
+
+ def errReceived(self, data):
+ log.debug('gzip: ' + data,'gzip')
+
+ def loseConnection(self):
+ """
+ This is a bad workaround Process.loseConnection not doing it's
+ job right.
+ The problem only happends when we try to finish the process
+ while decompresing.
+ """
+ if hasattr(self, 'process') and self.process.pid:
+ try:
+ os.kill(self.process.pid, signal.SIGTERM)
+ self.process.connectionLost()
+ except exceptions.OSError, Error:
+ import errno
+ (Errno, Errstr) = Error
+ if Errno != errno.ESRCH:
+ log.debug('Passing OSError exception '+Errstr)
+ raise
+ else:
+ log.debug('Threw away exception OSError no such process')
+
+ def processEnded(self, reason=None):
+ __pychecker__ = 'unusednames=reason'
+ log.debug("Status: %d" %(self.process.status),'gzip')
+ if self.process.status != 0:
+ self.setResponseCode(http.NOT_FOUND)
+
+ self.apDataReceived("")
+ self.apDataEnd(self.transfered)
+
+class FetcherRsync(Fetcher, protocol.ProcessProtocol):
+ """
+ I frequently am not called directly, Request.fetch makes the
+ arrangement for FetcherGzip to use us and gzip the result if needed.
+ """
+ post_convert = re.compile(r"^Should not match anything$")
+ gzip_convert = re.compile(r"/Packages.gz$")
+
+ "Temporary filename that rsync streams to"
+ rsyncTempFile = None
+
+ "Number of bytes sent to client already"
+ bytes_sent = 0
+
+ def activate (self, request):
+ Fetcher.activate(self, request)
+ if not request.apFetcher:
+ return
+
+ # Change /path/to/FILE -> /path/to/.FILE.* to match rsync tempfile
+ self.globpattern = re.sub(r'/([^/]*)$', r'/.\1.*', self.local_file)
+
+ for file in glob.glob(self.globpattern):
+ log.msg('Deleting stale tempfile:' + file)
+ unlink(file)
+
+ uri = 'rsync://'+request.backendServer.host\
+ +request.backendServer.path+'/'+request.backend_uri
+ self.local_dir=re.sub(r"/[^/]*$", "", self.local_file)+'/'
+
+ exe = '/usr/bin/rsync'
+ if(log.isEnabled('rsync',9)):
+ args = (exe, '--partial', '--progress', '--verbose', '--times',
+ '--timeout', "%d"%(request.backend.config.timeout),
+ uri, '.',)
+ else:
+ args = (exe, '--quiet', '--times', uri, '.',
+ '--timeout', "%d"%(request.backend.config.timeout),
+ )
+ if(not os.path.exists(self.local_dir)):
+ os.makedirs(self.local_dir)
+ self.process = reactor.spawnProcess(self, exe, args, None,
+ self.local_dir)
+
+ def findRsyncTempFile(self):
+ """
+ Look for temporary file created by rsync during streaming
+ """
+ files = glob.glob(self.globpattern)
+
+ if len(files)==1:
+ self.rsyncTempFile = files[0]
+ log.debug('tempfile: ' + self.rsyncTempFile, 'rsync_client')
+ elif not files:
+ # No file created yet
+ pass
+ else:
+ log.err('found more than one tempfile, abort rsync')
+ self.transport.loseConnection()
+
+ def connectionMade(self):
+ pass
+
+ "Data received from rsync process to stdout"
+ def outReceived(self, data):
+ for s in string.split(data, '\n'):
+ if len(s):
+ log.debug('rsync: ' + s, 'rsync_client')
+ #self.apDataReceived(data)
+ if not self.rsyncTempFile:
+ self.findRsyncTempFile()
+ # Got tempfile?
+ if self.rsyncTempFile:
+ self.setResponseCode(http.OK)
+ if self.rsyncTempFile:
+ self.sendData()
+
+
+ "Data received from rsync process to stderr"
+ def errReceived(self, data):
+ for s in string.split(data, '\n'):
+ if len(s):
+ log.err('rsync error: ' + s, 'rsync_client')
+
+ def sendData(self):
+ f = None
+ if self.rsyncTempFile:
+ try:
+ f = open(self.rsyncTempFile, 'rb')
+ except IOError:
+ return
+ else:
+ # Tempfile has gone, stream main file
+ #log.debug("sendData open dest " + str(self.bytes_sent))
+ f = open(self.local_file, 'rb')
+
+ if f:
+ f.seek(self.bytes_sent)
+ data = f.read(abstract.FileDescriptor.bufferSize)
+ #log.debug("sendData got " + str(len(data)))
+ f.close()
+ if data:
+ self.apDataReceived(data)
+ self.bytes_sent = self.bytes_sent + len(data)
+ reactor.callLater(0, self.sendData)
+ elif not self.rsyncTempFile:
+ # Finished reading final file
+ #self.transport = None
+ log.debug("sendData complete")
+ # Tell clients, but data is already saved by rsync so don't
+ # write file again
+ self.apDataEnd(self.transfered, False)
+
+
+ def processEnded(self, status_object):
+ __pychecker__ = 'unusednames=reason'
+ log.debug("Status: %d" %(status_object.value.exitCode)
+ ,'rsync_client')
+ self.rsyncTempFile = None
+
+ # Success?
+ exitcode = status_object.value.exitCode
+
+ if exitcode == 0:
+ # File received. Send to clients.
+ self.local_mtime = os.stat(self.local_file)[stat.ST_MTIME]
+ reactor.callLater(0, self.sendData)
+ else:
+ if exitcode == 10:
+ # Host not found
+ self.setResponseCode(http.INTERNAL_SERVER_ERROR)
+ else:
+ self.setResponseCode(http.NOT_FOUND)
+
+ if not os.path.exists(self.local_file):
+ try:
+ os.removedirs(self.local_dir)
+ except:
+ pass
+ self.apDataReceived("")
+ self.apDataEnd(self.transfered)
+
+ def loseConnection(self):
+ "Kill rsync process"
+ if self.transport:
+ if self.transport.pid:
+ log.debug("killing rsync child" +
+ str(self.transport.pid), 'rsync_client')
+ os.kill(self.transport.pid, signal.SIGTERM)
+ #self.transport.loseConnection()
+
+
+
+class FetcherCachedFile(Fetcher):
+ """
+ Sends the cached file or tells the client that the file was not
+ 'modified-since' if appropriate.
+ """
+ post_convert = re.compile(r"/Packages.gz$")
+ gzip_convert = re.compile(r"^Should not match anything$")
+
+ request = None
+ def if_modified(self, request):
+ """
+ Check if the file was 'modified-since' and tell the client if it
+ wasn't.
+ """
+ if_modified_since = request.getHeader('if-modified-since')
+ if if_modified_since != None:
+ if_modified_since = http.stringToDatetime(
+ if_modified_since)
+
+ if request.local_mtime <= if_modified_since:
+ request.setResponseCode(http.NOT_MODIFIED)
+ request.setHeader("Content-Length", 0)
+ request.write("")
+ request.finish()
+ self.remove_request(request)
+
+ def insert_request(self, request):
+ if not request.serve_if_cached:
+ request.finish()
+ return
+ Fetcher.insert_request(self, request)
+
+ log.debug("Serving from cache for additional client: " + self.local_file + " size:" + str(self.size))
+ self.start_transfer(request)
+
+ def activate(self, request):
+ Fetcher.activate(self, request)
+ if not request.apFetcher:
+ return
+ self.factory.file_served(request.uri)
+ self.size = request.local_size
+
+ self.start_transfer(request)
+
+ def start_transfer(self, request):
+ self.if_modified(request)
+
+ if len(self.requests) == 0:
+ #we had a single request and didn't have to send it
+ self.apEnd()
+ return
+
+ if self.size:
+ log.debug("Serving from cache: " + self.local_file + " size:" + str(self.size), 'FetcherCachedFile')
+ file = open(self.local_file,'rb')
+ fcntl.lockf(file.fileno(), fcntl.LOCK_SH)
+
+ request.setHeader("Content-Length", request.local_size)
+ request.setHeader("Last-modified",
+ http.datetimeToString(request.local_mtime))
+ basic.FileSender().beginFileTransfer(file, request) \
+ .addBoth(self.file_transfer_complete, request) \
+ .addBoth(lambda r: file.close())
+# .addBoth(lambda r: request.transport.loseConnection())
+ else:
+ log.debug("Zero length file! " + self.local_file, 'FetcherCachedFile')
+ self.file_transfer_complete(None, request)
+ request.finish()
+
+ # A file transfer has completed
+ def file_transfer_complete(self, result, request):
+ log.debug("transfer complete", 'FetcherCachedFile')
+ request.finish()
+ # Remove this client from request list
+ self.remove_request(request)
+ if len(self.requests) == 0:
+ Fetcher.apEnd(self)
+
+class Backend:
+ """
+ A backend repository. There is one Backend for each [...] section
+ in apt-proxy.conf
+ """
+
+ "Sequence of BackendServers, in order of preference"
+ uris = []
+
+ "Packages database for this backend"
+ packages = None
+ base = None
+
+ def __init__(self, factory, config):
+ self.factory = factory
+ self.config = config # apBackendConfig configuration information
+ self.base = config.name # Name of backend
+ self.uris=[]
+
+ for uri in config.backends:
+ self.addURI(uri)
+ #self.get_packages_db().load()
+
+ def addURI(self, uri):
+ newBackend = BackendServer(self, uri)
+ self.uris.append(newBackend)
+
+ def get_first_server(self):
+ "Provide first BackendServer for this Backend"
+ return self.uris[0]
+
+ def get_next_server(self, previous_server):
+ "Return next server, or None if this is the last server"
+ oldServerIdx = self.uris.index(previous_server)
+ if(oldServerIdx+1 >= len(self.uris)):
+ return None
+ return self.uris[oldServerIdx+1]
+
+ def __str__(self):
+ return '('+self.base+')'+' servers:'+str(len(self.uris))
+
+ def get_packages_db(self):
+ "Return packages parser object for the backend, creating one if necessary"
+ if self.packages == None:
+ self.packages = packages.AptPackages(self.base, self.factory.config.cache_dir)
+ return self.packages
+
+ def get_path(self, path):
+ """
+ 'path' is the original uri of the request.
+
+ We return the path to be appended to the backend path to
+ request the file from the backend server
+ """
+ return path[len(self.base)+2:]
+
+class BackendServer:
+ """
+ A repository server. A BackendServer is created for each URI defined in 'backends'
+ for a Backend
+ """
+
+ backend = None # Backend for this URI
+ uri = None # URI of server
+
+ fetchers = {
+ 'http' : FetcherHttp,
+ 'ftp' : FetcherFtp,
+ 'rsync': FetcherRsync,
+ 'file' : FetcherFile,
+ }
+ ports = {
+ 'http' : 80,
+ 'ftp' : 21,
+ 'rsync': 873,
+ 'file' : 0,
+ }
+
+ def __init__(self, backend, uri):
+ self.backend = backend
+ self.uri = uri
+ log.debug("Created new BackendServer: " + uri)
+
+ # hack because urlparse doesn't support rsync
+ if uri[0:5] == 'rsync':
+ uri = 'http'+uri[5:]
+ is_rsync=1
+ else:
+ is_rsync=0
+
+ self.scheme, netloc, self.path, parameters, \
+ query, fragment = urlparse.urlparse(uri)
+
+ if '@' in netloc:
+ auth = netloc[:netloc.rindex('@')]
+ netloc = netloc[netloc.rindex('@'):]
+ self.username, self.password = auth.split(':')
+ else:
+ self.username = None
+ if ':' in netloc:
+ self.host, self.port = netloc.split(':')
+ else:
+ self.host = netloc
+ self.port = self.ports[self.scheme]
+ if is_rsync:
+ self.scheme = 'rsync'
+ self.fetcher = self.fetchers[self.scheme]
+ try:
+ self.port = int(self.port)
+ except ValueError:
+ pass
+
+ def __str__(self):
+ return ('(' + self.backend.base + ') ' + self.scheme + '://' +
+ self.host + ':' + str(self.port))
+
+class Request(http.Request):
+ """
+ Each new request from connected clients generates a new instance of this
+ class, and process() is called.
+ """
+ local_mtime = None
+ local_size = None
+ serve_if_cached = 1
+ apFetcher = None
+ uriIndex = 0 # Index of backend URI
+ backend = None # Backend for this request
+ backendServer = None # Current server to be tried
+
+ def __init__(self, channel, queued):
+ self.factory=channel.factory
+ http.Request.__init__(self, channel, queued)
+
+ def process(self):
+ """
+ Each new request begins processing here
+ """
+ log.debug("Request: " + self.method + " " + self.uri);
+ # Clean up URL
+ self.uri = self.simplify_path(self.uri)
+
+ self.local_file = self.factory.config.cache_dir + self.uri
+ backendName = self.uri[1:].split('/')[0]
+ log.debug("Request: %s %s backend=%s local_file=%s"%(self.method, self.uri, backendName, self.local_file))
+
+ if self.factory.config.disable_pipelining:
+ self.setHeader('Connection','close')
+ self.channel.persistent = 0
+
+ if self.method != 'GET':
+ #we currently only support GET
+ log.debug("abort - method not implemented")
+ self.finishCode(http.NOT_IMPLEMENTED)
+ return
+
+ if re.search('/\.\./', self.uri):
+ log.debug("/../ in simplified uri ("+self.uri+")")
+ self.finishCode(http.FORBIDDEN)
+ return
+
+ self.backend = self.factory.getBackend(backendName)
+ if self.backend is None:
+ if not self.factory.config.dynamic_backends:
+ log.debug("abort - non existent Backend")
+ self.finishCode(http.NOT_FOUND, "NON-EXISTENT BACKEND")
+ return
+
+ # We are using dynamic backends so we will use the name as
+ # the hostname to get the files.
+ backendName = self.uri[1:].split('/')[0]
+ backendServer = "http://" + backendName
+ log.debug("Adding " + backendName + " backend dynamicaly")
+ backendConfig = self.factory.config.addBackend(None, backendName, (backendServer,))
+ self.backend = Backend(self.factory, backendConfig)
+ self.backend_uri = self.backend.get_path(self.uri)
+
+ log.debug("backend: %s %s" % (self.backend.base, self.backend.uris))
+ self.backendServer = self.backend.get_first_server()
+ self.filetype = findFileType(self.uri)
+
+ if not self.filetype:
+ log.debug("abort - unknown extension")
+ self.finishCode(http.NOT_FOUND)
+ return
+
+ self.setHeader('content-type', self.filetype.contype)
+
+ if os.path.isdir(self.local_file):
+ log.debug("abort - Directory listing not allowed")
+ self.finishCode(http.FORBIDDEN)
+ return
+
+ self.fetch()
+
+ def fetch(self, serve_cached=1):
+ """
+ Serve 'self' from cache or through the appropriate Fetcher
+ depending on the asociated backend.
+
+ Use post_convert and gzip_convert regular expresions of the Fetcher
+ to gzip/gunzip file before and after download.
+
+ 'serve_cached': this is somewhat of a hack only useful for
+ LoopbackRequests (See LoopbackRequest class for more information).
+ """
+ def fetch_real(result, dummyFetcher, cached, running):
+ """
+ This is called after verifying if the file is properly cached.
+
+ If 'cached' the requested file is properly cached.
+ If not 'cached' the requested file was not there, didn't pass the
+ integrity check or may be outdated.
+ """
+ __pychecker__ = 'unusednames=result'
+
+ if len(dummyFetcher.requests)==0:
+ #The request's are gone, the clients probably closed the
+ #conection
+ log.debug("THE REQUESTS ARE GONE (Clients closed conection)",
+ 'fetch')
+ dummyFetcher.apEnd()
+ return
+
+
+ req = dummyFetcher.request
+
+ log.debug("cached: %s" % cached)
+
+ if cached:
+ msg = ("Using cached copy of %s"
+ %(dummyFetcher.request.local_file))
+ fetcher_class = FetcherCachedFile
+ else:
+ msg = ("Consulting server about %s"
+ %(dummyFetcher.request.local_file))
+ fetcher_class = req.backendServer.fetcher
+
+ if fetcher_class.gzip_convert.search(req.uri):
+ msg = ("Using gzip/gunzip to get %s"
+ %(dummyFetcher.request.local_file))
+ fetcher_class = FetcherGzip
+
+ log.debug(msg, 'fetch_real')
+ fetcher = dummyFetcher.apEndTransfer(fetcher_class)
+ if (fetcher and fetcher.post_convert.search(req.uri)
+ and not running.has_key(req.uri[:-3])):
+ log.debug("post converting: "+req.uri,'convert')
+ loop = LoopbackRequest(req)
+ loop.uri = req.uri[:-3]
+ loop.local_file = req.local_file[:-3]
+ loop.process()
+ loop.serve_if_cached=0
+ #FetcherGzip will attach as a request of the
+ #original Fetcher, efectively waiting for the
+ #original file if needed
+ gzip = FetcherGzip()
+ gzip.activate(loop, postconverting=1)
+
+ self.serve_if_cached = serve_cached
+ running = self.factory.runningFetchers
+ if (running.has_key(self.uri)):
+ #If we have an active fetcher just use that
+ log.debug("have active fetcher: "+self.uri,'client')
+ running[self.uri].insert_request(self)
+ return running[self.uri]
+ else:
+ #we make a FetcherDummy instance to hold other requests for the
+ #same file while the check is in process. We will transfer all
+ #the requests to a real fetcher when the check is done.
+ dummyFetcher = FetcherDummy(self)
+ #Standard Deferred practice
+ d = self.check_cached()
+ d.addCallbacks(fetch_real, fetch_real,
+ (dummyFetcher, 1, running,), None,
+ (dummyFetcher, 0, running,), None)
+ return None
+
+ def simplify_path(self, old_path):
+ """
+ change //+ with /
+ change /directory/../ with /
+ More than three ocurrences of /../ together will not be
+ properly handled
+
+ NOTE: os.path.normpath could probably be used here.
+ """
+ path = re.sub(r"//+", "/", old_path)
+ path = re.sub(r"/\./+", "/", path)
+ new_path = re.sub(r"/[^/]+/\.\./", "/", path)
+ while (new_path != path):
+ path = new_path
+ new_path = re.sub(r"/[^/]+/\.\./", "/", path)
+ if (new_path != old_path):
+ log.debug("simplified path from " + old_path +
+ " to " + new_path,'simplify_path')
+ return path
+
+ def finishCode(self, responseCode, message=None):
+ "Finish the request with an status code"
+ self.setResponseCode(responseCode, message)
+ self.write("")
+ self.finish()
+
+ def finish(self):
+ http.Request.finish(self)
+ if self.factory.config.disable_pipelining:
+ if hasattr(self.transport, 'loseConnection'):
+ self.transport.loseConnection()
+
+ def check_cached(self):
+ """
+ check the existence and ask for the integrity of the requested file and
+ return a Deferred to be trigered when we find out.
+ """
+ def file_ok(result, deferred, self):
+ """
+ called if FileVerifier has determined that the file is cached and
+ in good shape.
+
+ Now we check NOTE: The file may still be too old or not fresh
+ enough.
+ """
+ __pychecker__ = 'unusednames=result'
+ stat_tuple = os.stat(self.local_file)
+
+ self.local_mtime = stat_tuple[stat.ST_MTIME]
+ self.local_size = stat_tuple[stat.ST_SIZE]
+ log.debug("Modification time:" +
+ time.asctime(time.localtime(self.local_mtime)),
+ "file_ok")
+ update_times = self.factory.update_times
+
+ if update_times.has_key(self.uri):
+ last_access = update_times[self.uri]
+ log.debug("last_access from db: " +
+ time.asctime(time.localtime(last_access)),
+ "file_ok")
+ else:
+ last_access = self.local_mtime
+
+
+ cur_time = time.time()
+ min_time = cur_time - self.factory.config.min_refresh_delay
+
+ if not self.filetype.mutable:
+ log.debug("file is immutable: "+self.local_file, 'file_ok')
+ deferred.callback(None)
+ elif last_access < min_time:
+ log.debug("file is too old: "+self.local_file, 'file_ok')
+ update_times[self.uri] = cur_time
+ deferred.errback()
+ else:
+ log.debug("file is ok: "+self.local_file, 'file_ok')
+ deferred.callback(None)
+
+ log.debug("check_cached: "+self.local_file, 'file_ok')
+ deferred = defer.Deferred()
+ if os.path.exists(self.local_file):
+ verifier = FileVerifier(self)
+ verifier.deferred.addCallbacks(file_ok, deferred.errback,
+ (deferred, self), None,
+ None, None)
+ else:
+ deferred.errback()
+ return deferred
+
+ def connectionLost(self, reason=None):
+ """
+ The connection with the client was lost, remove this request from its
+ Fetcher.
+ """
+ __pychecker__ = 'unusednames=reason'
+ #If it is waiting for a file verification it may not have an
+ #apFetcher assigned
+ if self.apFetcher:
+ self.apFetcher.remove_request(self)
+ self.finish()
+
+ def activateNextBackendServer(self, fetcher):
+ """
+ The attempt to retrieve a file from the BackendServer failed.
+ Look for the next possible BackendServer and transfer requests to that
+ Returns true if another BackendServer was found
+ """
+ self.backendServer = self.backend.get_next_server(self.backendServer)
+ if(self.backendServer == None):
+ log.debug("no more Backends", "fetcher")
+ return False
+
+ fetcher_class = self.backendServer.fetcher
+ log.debug('Trying next backendServer', 'fetcher')
+ fetcher.apEndTransfer(fetcher_class)
+
+ return True
+
+
+class LoopbackRequest(Request):
+ """
+ This is just a fake Request so a Fetcher can attach to another
+ Fetcher and be notified when then transaction is completed.
+
+ Look at FetcherGzip for a sample.
+ """
+ __pychecker__ = 'no-callinit'
+ import cStringIO
+ local_mtime = None
+ headers = {}
+ content = cStringIO.StringIO()
+
+ def __init__(self, other_req, finish=None):
+
+ self.finish_cb = finish
+ http.Request.__init__(self, None, 1)
+ self.backend = other_req.backend
+ self.factory = other_req.factory
+ self.filetype = other_req.filetype
+ self.method = other_req.method
+ self.clientproto = other_req.clientproto
+ def process(self):
+ self.backend_uri = self.backend.get_path(self.uri)
+ def write(self, data):
+ "We don't care for the data, just want to know then it is served."
+ pass
+ def finish(self):
+ "If he wanted to know, tell daddy that we are served."
+ if self.finish_cb:
+ self.finish_cb()
+ self.transport = None
+ pass
+
+class Channel(http.HTTPChannel):
+ """
+ This class encapsulates a channel (an HTTP socket connection with a single
+ client).
+
+ Each incoming request is passed to a new Request instance.
+ """
+ requestFactory = Request
+ log_headers = None
+
+ def headerReceived(self, line):
+ "log and pass over to the base class"
+ #log.debug("Header: " + line)
+ if self.log_headers == None:
+ self.log_headers = line
+ else:
+ self.log_headers += ", " + line
+ http.HTTPChannel.headerReceived(self, line)
+
+ def allContentReceived(self):
+ if self.log_headers != None:
+ log.debug("Headers: " + self.log_headers)
+ self.log_headers = None
+ http.HTTPChannel.allContentReceived(self)
+
+ def connectionLost(self, reason=None):
+ "If the connection is lost, notify all my requests"
+ __pychecker__ = 'unusednames=reason'
+ for req in self.requests:
+ req.connectionLost()
+ log.debug("Client connection closed")
+ if log.isEnabled('memleak'):
+ memleak.print_top_10()
+ #reactor.stop() # use for shutting down apt-proxy when a client disconnects
+
+class Factory(protocol.ServerFactory):
+ """
+ This is the center of apt-proxy, it holds all configuration and global data
+ and gets attached everywhere.
+
+ Factory receives incoming client connections and creates a Channel for
+ each client request.
+
+ interesting attributes:
+
+ self.runningFetchers: a dictionary, uri/Fetcher pairs, that holds the
+ active Fetcher for that uri if any. If there is an active Fetcher for
+ a certain uri at a certain time the request is inserted into the Fetcher
+ found here instead of instanciating a new one.
+
+ Persisten dictionaries:
+ self.update_times: last time we checked the freashness of a certain file.
+ self.access_times: last time that a certain file was requested.
+ self.packages: all versions of a certain package name.
+
+ """
+ databases=('update_times', 'access_times', 'packages')
+
+ def periodic(self):
+ "Called periodically as configured mainly to do mirror maintanace."
+ log.debug("Doing periodic cleaning up")
+ self.clean_old_files()
+ self.recycler.start()
+ log.debug("Periodic cleaning done")
+ if (self.config.cleanup_freq != None):
+ reactor.callLater(self.config.cleanup_freq, self.periodic)
+ def __del__(self):
+ for f in self.databases:
+ try:
+ if hasattr(self, f):
+ getattr(self, f).close()
+ except Exception:
+ pass
+ def __init__ (self, config):
+ self.runningFetchers = {}
+ self.backends = []
+ self.config = config
+
+ def __getattr__ (self, name):
+ def open_shelve(dbname):
+ from bsddb import db,dbshelve
+
+ shelve = dbshelve.DBShelf()
+ db_dir = self.config.cache_dir+'/'+status_dir+'/db'
+ if not os.path.exists(db_dir):
+ os.makedirs(db_dir)
+
+ filename = db_dir + '/' + dbname + '.db'
+ if os.path.exists(filename):
+ try:
+ log.debug('Verifying database: ' + filename)
+ shelve.verify(filename)
+ except:
+ os.rename(filename, filename+'.error')
+ log.msg(filename+' could not be opened, moved to '+filename+'.error','db', 1)
+ log.msg('Recreating '+ filename,'db', 1)
+ try:
+ log.debug('Opening database ' + filename)
+ shelve = dbshelve.open(filename)
+
+ # Handle upgrade to new format included on 1.9.20.
+ except db.DBInvalidArgError:
+ log.msg('Upgrading from previous database format: %s' % filename + '.previous')
+ import bsddb.dbshelve
+ os.rename(filename, filename + '.previous')
+ previous_shelve = bsddb.dbshelve.open(filename + '.previous')
+ shelve = dbshelve.open(filename)
+
+ for k in previous_shelve.keys():
+ shelve[k] = previous_shelve[k]
+ log.msg('Upgrade complete')
+
+ return shelve
+
+ if name == 'update_times':
+ self.update_times = open_shelve('update')
+ return self.update_times
+ elif name == 'access_times':
+ self.access_times = open_shelve('access')
+ return self.access_times
+ elif name == 'packages':
+ self.packages = open_shelve('packages')
+ return self.packages
+ else:
+ raise AttributeError(name)
+
+ def startFactory(self):
+ #start periodic updates
+ self.configurationChanged()
+ self.recycler = misc.MirrorRecycler(self, 1)
+ self.recycler.start()
+
+ def configurationChanged(self, oldconfig = None):
+ """
+ Configuration has changed - update backends and background processes
+ """
+ if oldconfig is not None:
+ for param in 'address', 'port', 'telnet_port', 'telnet_user', 'telnet_pass', 'cache_dir':
+ if getattr(self.config, param) != getattr(oldconfig, param):
+ log.err('Configuration value %s has changed, ignored'%(param))
+ log.err('You must restart apt-proxy for the change to take effect')
+ setattr(self.config, param, getattr(oldconfig, param))
+
+ if self.config.cleanup_freq != None and (oldconfig is None or oldconfig.cleanup_freq == None):
+ reactor.callLater(self.config.cleanup_freq, self.periodic)
+ self.createBackends()
+
+ def createBackends(self):
+ self.backends = {}
+ for name, bconf in self.config.backends.items():
+ #print "[",name,"]"
+ self.backends[name] = Backend(self, bconf)
+
+ def getBackend(self, name):
+ """
+ Return backend with given name
+ @param name Name of backend as specified in [backendName] section in config file
+ @return Backend class, or None if not found
+ """
+ if self.backends.has_key(name):
+ return self.backends[name]
+ return None
+
+ def clean_versions(self, packages):
+ """
+ Remove entries for package versions which are not in cache, and delete
+ some files if needed to respect the max_versions configuration.
+
+ TODO: This must be properly done per distribution.
+ """
+ if self.config.max_versions == None:
+ #max_versions is disabled
+ return
+ package_name = None
+ cache_dir = self.config.cache_dir
+
+ cached_packages = [] # all packages in cache directory
+ current_packages = [] # packages referenced by Packages files
+
+ import apt_pkg
+ def reverse_compare(a, b):
+ """ Compare package versions in reverse order """
+ return apt_pkg.VersionCompare(b[0], a[0])
+
+ if len(packages) <= self.config.max_versions:
+ return
+
+ from packages import AptDpkgInfo, get_mirror_versions
+ for uri in packages[:]:
+ if not os.path.exists(cache_dir +'/'+ uri):
+ packages.remove(uri)
+ else:
+ try:
+ info = AptDpkgInfo(cache_dir +'/'+ uri)
+ cached_packages.append([info['Version'], uri])
+ package_name = info['Package']
+ except SystemError:
+ log.msg("Found problems with %s, aborted cleaning"%(uri),
+ 'max_versions')
+ return
+
+ if len(info):
+ import apt_pkg
+ cached_packages.sort(reverse_compare)
+ log.debug(str(cached_packages), 'max_versions')
+
+ current_packages = get_mirror_versions(self, package_name)
+ current_packages.sort(reverse_compare)
+ log.debug("Current Versions: " + str(current_packages), 'max_versions')
+
+ version_count = 0
+
+ while len(cached_packages):
+ #print 'current:',len(current_packages),'cached:',len(cached_packages), 'count:', version_count
+ if len(current_packages):
+ compare_result = apt_pkg.VersionCompare(current_packages[0][0], cached_packages[0][0])
+ #print 'compare_result %s , %s = %s ' % (
+ # current_packages[0][0], cached_packages[0][0], compare_result)
+ else:
+ compare_result = -1
+
+ if compare_result >= 0:
+ log.debug("reset at "+ current_packages[0][1], 'max_versions')
+ del current_packages[0]
+ version_count = 0
+ else:
+ version_count += 1
+ if version_count > self.config.max_versions:
+ log.msg("Deleting " + cache_dir +'/'+ cached_packages[0][1], 'max_versions')
+ os.unlink(cache_dir +'/'+ cached_packages[0][1])
+ del cached_packages[0]
+
+ def clean_old_files(self):
+ """
+ Remove files which haven't been accessed for more than 'max_age' and
+ all entries for files which are no longer there.
+ """
+ if self.config.max_age == None:
+ #old file cleaning is disabled
+ return
+ cache_dir = self.config.cache_dir
+ files = self.access_times.keys()
+ min_time = time.time() - self.config.max_age
+
+ for file in files:
+ local_file = cache_dir + '/' + file
+ if not os.path.exists(local_file):
+ log.debug("old_file: non-existent "+file)
+ del self.access_times[file]
+ elif self.access_times[file] < min_time:
+ log.debug("old_file: removing "+file)
+ os.unlink(local_file)
+ del self.access_times[file]
+
+ #since we are at it, clear update times for non existent files
+ files = self.update_times.keys()
+ for file in files:
+ if not os.path.exists(cache_dir+'/'+file):
+ log.debug("old_file: non-existent "+file)
+ del self.update_times[file]
+
+ def file_served(self, uri):
+ "Update the databases, this file has just been served."
+ self.access_times[uri]=time.time()
+ if re.search("\.deb$", uri):
+ package = re.sub("^.*/", "", uri)
+ package = re.sub("_.*$", "", package)
+ if not self.packages.has_key(package):
+ packages = [uri]
+ self.packages[package] = packages
+ else:
+ packages = self.packages[package]
+ if not uri in packages:
+ packages.append(uri)
+ self.clean_versions(packages)
+ self.packages[package] = packages
+ self.dumpdbs()
+
+ def stopFactory(self):
+ import packages
+ self.dumpdbs()
+ self.update_times.close()
+ self.access_times.close()
+ self.packages.close()
+ packages.cleanup(self)
+
+ def dumpdbs (self):
+ def dump_update(key, value):
+ log.msg("%s: %s"%(key, time.ctime(value)),'db')
+ def dump_access(key, value):
+ log.msg("%s: %s"%(key, time.ctime(value)),'db')
+ def dump_packages(key, list):
+ log.msg("%s: "%(key),'db')
+ for file in list:
+ log.msg("\t%s"%(file),'db')
+ def dump(db, func):
+ keys = db.keys()
+ for key in keys:
+ func(key,db[key])
+ if log.isEnabled('db'):
+ log.msg("=========================",'db')
+ log.msg("Dumping update times",'db')
+ log.msg("=========================",'db')
+ dump(self.update_times, dump_update)
+ log.msg("=========================",'db')
+ log.msg("Dumping access times",'db')
+ log.msg("=========================",'db')
+ dump(self.access_times, dump_access)
+ log.msg("=========================",'db')
+ log.msg("Dumping packages",'db')
+ log.msg("=========================",'db')
+ dump(self.packages, dump_packages)
+
+
+ def buildProtocol(self, addr):
+ __pychecker__ = 'unusednames=addr'
+ proto = Channel()
+ proto.factory = self;
+ return proto
+
+ def log(self, request):
+ return
+
+ def debug(self, message):
+ log.debug(message)
Added: tags/1.9.33-0.1/apt_proxy/apt_proxy_conf.py
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/apt_proxy_conf.py Thu Aug 3 10:12:44 2006
@@ -0,0 +1,264 @@
+#
+# Copyright (C) 2002 Manuel Estrada Sainz <ranty at debian.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from apt_proxy import Backend
+from misc import log
+import os, sys
+from types import StringType, NoneType
+import urlparse
+from ConfigParser import RawConfigParser,DEFAULTSECT
+
+class ConfigError(Exception):
+ def __init__(self, message):
+ self.message = message
+ def __str__(self):
+ return repr(self.message)
+
+class apConfigParser(RawConfigParser):
+ """
+ Adds 'gettime' to ConfigParser to interpret the suffixes.
+ Interprets 'disabled_keyword' as disabled (None).
+ """
+ time_multipliers={
+ 's': 1, #seconds
+ 'm': 60, #minutes
+ 'h': 3600, #hours
+ 'd': 86400,#days
+ }
+ DISABLED_KEYWORD = 'off'
+ def isOff(self, section, option):
+ value = self.get(section, option)
+ return value == self.DISABLED_KEYWORD
+
+ def getint(self, section, option):
+ value = self.get(section, option)
+ return int(value)
+ def gettime(self, section, option):
+ mult = 1
+ value = self.get(section, option)
+ suffix = value[-1].lower()
+ if suffix in self.time_multipliers.keys():
+ mult = self.time_multipliers[suffix]
+ value = value[:-1]
+ return int(value)*mult
+ def getstring(self, section, option):
+ return self.get(section,option)
+ def getstringlist(self, section, option):
+ return self.get(section,option).split()
+
+class apConfig:
+ """
+ Configuration module for apt-proxy
+ holds main configuration values in class and backend
+ configs in backends[backend-name]
+ """
+
+ """
+ Configuration items for default section
+ [0] - name of config parameter and resulting class variable name
+ [1] - default value
+ [2] - getXXX method to use to read config.
+ A method prefixed with '*' will return None if disabled
+ """
+ CONFIG_ITEMS = [
+ ['address', '', 'string'],
+ ['port', 9999, 'int'],
+ ['min_refresh_delay', 30, 'time'],
+ ['complete_clientless_downloads', '0', 'boolean'],
+ ['telnet_port', 0, 'int'],
+ ['telnet_user', '', 'string'],
+ ['telnet_pass', '', 'string'],
+ ['timeout', 30, 'time'],
+ ['cleanup_freq', 600, '*time'],
+ ['cache_dir', '/var/cache/apt-proxy', 'string'],
+ ['max_versions', 3, '*int'],
+ ['max_age', 10, '*time'],
+ ['import_dir', '/var/cache/apt-proxy/import', 'string'],
+ ['disable_pipelining', '1', 'boolean'],
+ ['passive_ftp', 'on', 'boolean'],
+ ['dynamic_backends', 'on', 'boolean'],
+ ['http_proxy', '' , 'string'],
+ ['username', 'aptproxy', 'string']
+ ]
+
+ """
+ Configuration items for backends
+ [0] - name of config parameter and resulting class variable name
+ [1] - default value, None to use factory default
+ [2] - getXXX method to use to read config.
+ A method prefixed with '*' will return None if disabled
+ """
+ BACKEND_CONFIG_ITEMS = [
+ ['timeout', None, 'time'],
+ ['passive_ftp', None, 'boolean'],
+ ['backends', '', 'stringlist']
+ ]
+
+ DEFAULT_CONFIG_FILE = ['/etc/apt-proxy/apt-proxy-v2.conf',
+ '/etc/apt-proxy/apt-proxy-2.conf',
+ '/etc/apt-proxy/apt-proxy.conf']
+
+ "Backend configurations are held here"
+ backends = {}
+ parser = None
+ debugDomains = {}
+ debug = '0'
+
+ def __init__(self, config_file = None):
+ """
+ Read configuration from specified source, or default config
+ file location if not specified
+
+ @param config_file Filename to read, or descriptor of already-open file
+ """
+ self.backends = {}
+ if type(config_file) is StringType or type(config_file) is NoneType:
+ c = self.readFromFile(config_file)
+ else:
+ c = self.readFromStream(config_file)
+
+ self.parseConfig(c)
+
+ def readFromFile(self, config_file):
+ """
+ Read configuration from filename given
+ @param config_file filename to read from, or None for default
+ """
+ conf = apConfigParser()
+
+ if config_file is not None:
+ config_files = config_file,
+ else:
+ config_files = self.DEFAULT_CONFIG_FILE
+
+ for f in config_files:
+ if os.path.exists(f):
+ conf.read(f)
+ break
+ else:
+ raise ConfigError("%s: Configuration file does not exist" % config_files[0])
+
+ return conf
+
+ def readFromStream(self, filehandle):
+ "Read from open file handle, for test suite"
+ conf = apConfigParser()
+ conf.readfp(filehandle)
+ filehandle.close()
+ return conf
+
+ def setDebug(self, levels):
+ "Set logger debug level"
+ self.debug = levels
+ for domain in self.debug.split():
+ #print "domain:",domain
+ if domain.find(':') != -1:
+ name, level = domain.split(':')
+ else:
+ name, level = domain, 9
+ self.debugDomains[name] = int(level)
+ log.setDomains(self.debugDomains)
+
+ def parseConfig(self, config):
+ "Read values from apConfigParser config"
+
+ # debug setting
+ if config.has_option(DEFAULTSECT, 'debug'):
+ self.debug=config.get(DEFAULTSECT, 'debug')
+ else:
+ self.debug='all:3'
+ self.setDebug(self.debug)
+
+ # read default values
+ for name,default,getmethod in self.CONFIG_ITEMS:
+ value = self.parseConfigValue(config, DEFAULTSECT, name, default, getmethod)
+ setattr(self, name, value)
+ if value != default and name != "telnet_pass":
+ log.debug("config value %s=%s"%(name, value), 'config')
+
+ self.address = self.address.split(" ")
+
+ if not self.telnet_user or not self.telnet_pass:
+ self.telnet_port = 0 # No server if empty username or password
+
+ # Read backend configurations
+ for backendName in config.sections():
+ self.addBackend(config, backendName)
+
+ def addBackend(self, config, backendName, backendServers=None):
+ """
+ Add a new backend configuration
+ @param config Configuration file parser to get backend values from. If None, backend is dynamic
+ @param backendName Name of backend to create
+ @param backendServers List of backend servers to use (if backend is dynamic)
+ @ret newly created apBackendConfig
+ """
+ if backendName.find('/') != -1:
+ log.msg("WARNING: backend %s contains '/' (ignored)"%(name))
+ return None
+
+ backend = apBackendConfig(backendName)
+ for paramName,default,getmethod in self.BACKEND_CONFIG_ITEMS:
+ if default is None:
+ default = getattr(self, paramName) # Use default section's default value
+ value = self.parseConfigValue(config, backendName, paramName, default, getmethod)
+ setattr(backend,paramName, value)
+ if value != default:
+ log.debug("[backend %s] %s=%s"%(backendName, paramName, value), 'config')
+
+ if backendServers is None:
+ backend.dynamic = False
+ backendServers = backend.backends
+ else:
+ # Dynamic backend
+ backend.dynamic = True
+
+ backend.backends = []
+ for server in backendServers:
+ if server[-1] == '/':
+ log.msg ("Removing unnecessary '/' at the end of %s"%(server))
+ server = server[0:-1]
+ if urlparse.urlparse(server)[0] in ['http', 'ftp', 'rsync', 'file']:
+ backend.backends.append(server)
+ else:
+ log.msg ("WARNING: Wrong server '%s' found in backend '%s'. It was skipped." % (server, backendName))
+ return None
+ if len(backend.backends) == 0:
+ log.msg("WARNING: [%s] has no backend servers (ignored)"%backendName)
+ return None
+
+ self.backends[backendName] = backend
+ return backend
+
+ def parseConfigValue(self, config, section, name, default, getmethod):
+ "Determine value of given config item"
+ if config is None or not config.has_option(section, name):
+ return default
+ if getmethod[0]=='*':
+ if config.isOff(section, name):
+ return None
+ else:
+ return getattr(config, 'get'+getmethod[1:])(section, name)
+ else:
+ return getattr(config, 'get'+getmethod)(section, name)
+
+class apBackendConfig:
+ """
+ Configuration information for an apt-proxy backend
+ """
+ name = "UNKNOWN"
+ def __init__(self, name):
+ self.name = name
Added: tags/1.9.33-0.1/apt_proxy/apt_proxytap.py
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/apt_proxytap.py Thu Aug 3 10:12:44 2006
@@ -0,0 +1,16 @@
+import apt_proxy
+from apt_proxy_conf import aptProxyFactoryConfig
+from apt_proxy import AptProxyFactory
+from twisted.internet.app import Application
+
+from twisted.python import usage # twisted command-line processing
+
+class Options(usage.Options):
+ optParameters = [];
+
+def updateApplication(app, config):
+ factory = AptProxyFactory()
+ aptProxyFactoryConfig(factory)
+ app = Application("AptProxy")
+ app.listenTCP(factory.proxy_port, factory)
+
Added: tags/1.9.33-0.1/apt_proxy/memleak.py
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/memleak.py Thu Aug 3 10:12:44 2006
@@ -0,0 +1,28 @@
+import sys
+import types
+
+def get_refcounts():
+ d = {}
+ sys.modules
+ # collect all classes
+ for m in sys.modules.values():
+ for sym in dir(m):
+ o = getattr (m, sym)
+ if type(o) is types.ClassType:
+ d[o] = sys.getrefcount (o)
+ # sort by refcount
+ pairs = map (lambda x: (x[1],x[0]), d.items())
+ pairs.sort()
+ pairs.reverse()
+ return pairs
+
+def print_top_n(n):
+ print "Top %i:"%n
+ for n, c in get_refcounts()[:n]:
+ print '%10d %s' % (n, c.__name__)
+
+def print_top_100():
+ print_top_n(100)
+
+def print_top_10():
+ print_top_n(10)
Added: tags/1.9.33-0.1/apt_proxy/misc.py
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/misc.py Thu Aug 3 10:12:44 2006
@@ -0,0 +1,190 @@
+#
+# Copyright (C) 2002 Manuel Estrada Sainz <ranty at debian.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+from twisted.internet import reactor
+from twisted import python
+
+class DomainLogger:
+ """
+ This class should help us classify messages into domains and levels.
+
+ This way we can select messages by kind and level.
+
+ You just have to set in the configuration file something like:
+
+ debug = db:3 import:8
+
+ Which means that we only want to see messages of domain 'db' and
+ level <= 3 and domain 'import' and level <= 8
+
+ There are three special domains:
+
+ all: if enabled all messages will be shown.
+ log: is on by default and only the level can be changed
+ it is meant for production logging.
+ debug: aptProxyConfig will define it if you select any loging
+ domains.
+
+ Pretended meaning of levels:
+ 0: nothing or maybe critical information
+ 1: important information
+ ...
+ 9: useless information
+ """
+ def __init__(self, enabled={'all':9}):
+ self.enabled = enabled
+
+ def setDomains(self, domains):
+ self.enabled = domains
+
+ def addDomains(self, domains):
+ self.enabled.update(domains)
+ #print "enabled: ", self.enabled
+ def isEnabled(self, domain, level=9):
+ domains = self.enabled.keys()
+ if domain in domains and level > self.enabled[domain]:
+ return 0
+ if(('all' in domains and level <= self.enabled['all'])
+ or (domain in domains and level <= self.enabled[domain])):
+ return 1
+ else:
+ return 0
+
+ def msg(self, msg, domain='log', level=4):
+ "Logs 'msg' if domain and level are appropriate"
+ #print 'domain:', domain, 'level:', level
+ if self.isEnabled(domain, level):
+ try:
+ python.log.msg("[%s] %s"%(domain, msg))
+ except IOError:
+ pass
+ def debug(self, msg, domain='debug', level=9):
+ "Useful to save typing on new debuging messages"
+ if self.isEnabled(domain, level):
+ try:
+ python.log.msg("[%s] %s"%(domain, msg), debug=True)
+ except IOError:
+ pass
+ def err(self, msg, domain='error', level=9):
+ "Log an error message"
+ try:
+ python.log.err("[%s] %s"%(domain, msg))
+ except IOError:
+ pass
+
+
+# Prevent log being replace on reload. This only works in cpython.
+try:
+ log
+except NameError:
+ log = DomainLogger()
+
+
+
+class MirrorRecycler:
+ """
+ Reads the mirror tree looking for 'forgotten' files and adds them to
+ factory.access_times so they can age and be removed like the others.
+
+ It processes one directory entry per 'timer' seconds, which unless
+ set to 0 is very slow, but it is also very light weight. And files
+ which get recuested are recycled automatically anyway, so it is
+ not urgent to find forgotten files. If also uses the files oun
+ atime, so if the files has been there for a long time it will soon
+ be removed anyway.
+
+ """
+ working = 0
+
+ def __init__(self, factory, timer):
+ self.timer = timer
+ self.factory = factory
+ def start(self):
+ """
+ Starts the Recycler if it is not working, it will use
+ callLater to keep working until it finishes with the whole
+ tree.
+ """
+ if not self.working:
+ if self.factory.backends == []:
+ log.msg("NO BACKENDS FOUND",'recycle')
+ return
+ self.cur_uri = '/'
+ self.cur_dir = self.factory.config.cache_dir
+ self.pending = []
+ for backend in self.factory.backends.values():
+ self.pending.append(backend.base)
+ self.stack = []
+ reactor.callLater(self.timer, self.process)
+ self.working = 1
+ def pop(self):
+ if self.stack:
+ (self.cur_dir, self.cur_uri, self.pending) = self.stack.pop()
+ else:
+ self.working = 0
+ def push(self):
+ if self.pending:
+ self.stack.append((self.cur_dir, self.cur_uri, self.pending))
+
+ def process(self):
+ """
+ Process the next entry, is called automatically via callLater.
+ """
+ entry = self.pending.pop()
+ uri = os.path.join(self.cur_uri, entry)
+ path = os.path.join(self.cur_dir, entry)
+ if not os.path.exists(path):
+ pass
+ elif os.path.isdir(path):
+ self.push()
+ self.cur_dir = path
+ self.cur_uri = uri
+ self.pending = os.listdir(self.cur_dir)
+ if not self.pending:
+ log.msg("Pruning empty directory: "+path,'recycle')
+ os.removedirs(path)
+ else:
+ if os.path.isfile(path):
+ #print "PATH:", path
+ #print "URI: ", uri
+ if not self.factory.access_times.has_key(uri):
+ log.msg("Adopting new file: "+ uri,'recycle')
+ self.factory.access_times[uri] = os.path.getatime(path)
+ else:
+ log.msg("UNKNOWN:"+path,'recycle')
+
+ if not self.pending:
+ self.pop()
+ if self.working:
+ reactor.callLater(self.timer, self.process)
+
+if __name__ == '__main__':
+ #Just for testing purposes.
+ from apt_proxy_conf import aptProxyFactoryConfig
+ import shelve
+
+ class DummyFactory:
+ pass
+ factory = DummyFactory()
+ aptProxyFactoryConfig(factory)
+ factory.access_times=shelve.open("tmp.db")
+ recycle = MirrorRecycler(factory, 10)
+ recycle.start()
+ while recycle.working:
+ recycle.process()
+
+ factory.access_times.close()
Added: tags/1.9.33-0.1/apt_proxy/packages.py
==============================================================================
--- (empty file)
+++ tags/1.9.33-0.1/apt_proxy/packages.py Thu Aug 3 10:12:44 2006
@@ -0,0 +1,455 @@
+#
+# Copyright (C) 2002 Manuel Estrada Sainz <ranty at debian.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import apt_pkg, apt_inst, sys, os, stat
+from os.path import dirname, basename
+import re, shelve, shutil, fcntl
+from twisted.internet import process
+import apt_proxy, copy, UserDict
+from misc import log
+
+aptpkg_dir='.apt-proxy'
+apt_pkg.InitSystem()
+
+class AptDpkgInfo(UserDict.UserDict):
+ """
+ Gets control fields from a .deb file.
+
+ And then behaves like a regular python dictionary.
+
+ See AptPackages.get_mirror_path
+ """
+
+ def __init__(self, filename):
+ UserDict.UserDict.__init__(self)
+ try:
+ filehandle = open(filename);
+ try:
+ self.control = apt_inst.debExtractControl(filehandle)
+ finally:
+ # Make sure that file is always closed.
+ filehandle.close()
+ except SystemError:
+ log.debug("Had problems reading: %s"%(filename), 'AptDpkgInfo')
+ raise
+ for line in self.control.split('\n'):
+ if line.find(': ') != -1:
+ key, value = line.split(': ', 1)
+ self.data[key] = value
+
+class PackageFileList:
+ """
+ Manages a list of package files belonging to a backend
+ """
+ def __init__(self, backendName, cache_dir):
+ self.cache_dir = cache_dir
+ packagedb_dir = cache_dir+'/'+ apt_proxy.status_dir + \
+ '/backends/' + backendName
+ if not os.path.exists(packagedb_dir):
+ os.makedirs(packagedb_dir)
+ self.packages = shelve.open(packagedb_dir+'/packages.db')
+ def __del__(self):
+ try:
+ self.packages.close()
+ except:
+ pass
+
+ def update_file(self, uri):
+ """
+ Called from apt_proxy.py when files get updated so we can update our
+ fake lists/ directory and sources.list.
+
+ @param uri Filename of cached file (without cache_dir prefix)
+ """
+ if basename(uri)=="Packages" or basename(uri)=="Release":
+ log.msg("REGISTERING PACKAGE:"+uri,'apt_pkg',4)
+ stat_result = os.stat(self.cache_dir+'/'+uri)
+ self.packages[uri] = stat_result
+ def get_files(self):
+ """
+ Get list of files in database. Each file will be checked that it exists
+ """
+ files = self.packages.keys()
+ #print self.packages.keys()
+ for f in files:
+ if not os.path.exists(self.cache_dir + '/' + f):
+ log.debug("File in packages database has been deleted: "+f, 'apt_pkg')
+ del files[files.index(f)]
+ del self.packages[f]
+ return files
+
+class AptPackages:
+ """
+ Uses AptPackagesServer to answer queries about packages.
+
+ Makes a fake configuration for python-apt for each backend.
+ """
+ DEFAULT_APT_CONFIG = {
+ #'APT' : '',
+ 'APT::Architecture' : apt_pkg.CPU,
+ #'APT::Default-Release' : 'unstable',
+
+ 'Dir':'.', # /
+ 'Dir::State' : 'apt/', # var/lib/apt/
+ 'Dir::State::Lists': 'lists/', # lists/
+ #'Dir::State::cdroms' : 'cdroms.list',
+ 'Dir::State::userstatus' : 'status.user',
+ 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
+ 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
+ #'Dir::Cache::archives' : 'archives/',
+ 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
+ 'Dir::Cache::pkgcache' : 'pkgcache.bin',
+ 'Dir::Etc' : 'apt/etc/', # etc/apt/
+ 'Dir::Etc::sourcelist' : 'sources.list',
+ 'Dir::Etc::vendorlist' : 'vendors.list',
+ 'Dir::Etc::vendorparts' : 'vendors.list.d',
+ #'Dir::Etc::main' : 'apt.conf',
+ #'Dir::Etc::parts' : 'apt.conf.d',
+ #'Dir::Etc::preferences' : 'preferences',
+ 'Dir::Bin' : '',
+ #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
+ 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
+ #'DPkg' : '',
+ #'DPkg::Pre-Install-Pkgs' : '',
+ #'DPkg::Tools' : '',
+ #'DPkg::Tools::Options' : '',
+ #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
+ #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
+ #'DPkg::Post-Invoke' : '',
+ }
+ essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
+ 'apt/lists/partial')
+ essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
+
+ def __init__(self, backendName, cache_dir):
+ """
+ Construct new packages manager
+ backend: Name of backend associated with this packages file
+ cache_dir: cache directory from config file
+ """
+ self.backendName = backendName
+ self.cache_dir = cache_dir
+ self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
+
+ self.status_dir = (cache_dir+'/'+ aptpkg_dir
+ +'/backends/'+backendName)
+ for dir in self.essential_dirs:
+ path = self.status_dir+'/'+dir
+ if not os.path.exists(path):
+ os.makedirs(path)
+ for file in self.essential_files:
+ path = self.status_dir+'/'+file
+ if not os.path.exists(path):
+ f = open(path,'w')
+ f.close()
+ del f
+
+ self.apt_config['Dir'] = self.status_dir
+ self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
+ #os.system('find '+self.status_dir+' -ls ')
+ #print "status:"+self.apt_config['Dir::State::status']
+ self.packages = PackageFileList(backendName, cache_dir)
+ self.loaded = 0
+ #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
+
+ def __del__(self):
+ self.cleanup()
+ #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
+ del self.packages
+ #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
+ def file_updated(self, uri):
+ """
+ A file in the backend has changed. If this affects us, unload our apt database
+ """
+ if self.packages.update_file(uri):
+ self.unload()
+
+ def __save_stdout(self):
+ self.real_stdout_fd = os.dup(1)
+ os.close(1)
+
+ def __restore_stdout(self):
+ os.dup2(self.real_stdout_fd, 1)
+ os.close(self.real_stdout_fd)
+ del self.real_stdout_fd
+
+ def load(self):
+ """
+ Regenerates the fake configuration and load the packages server.
+ """
+ if self.loaded: return True
+ apt_pkg.InitSystem()
+ #print "Load:", self.status_dir
+ shutil.rmtree(self.status_dir+'/apt/lists/')
+ os.makedirs(self.status_dir+'/apt/lists/partial')
+ sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
+ sources = open(sources_filename, 'w')
+ sources_count = 0
+ for file in self.packages.get_files():
+ # we should probably clear old entries from self.packages and
+ # take into account the recorded mtime as optimization
+ filepath = self.cache_dir + file
+ fake_uri='http://apt-proxy/'+file
+ source_line='deb '+dirname(fake_uri)+'/ /'
+ listpath=(self.status_dir+'/apt/lists/'
+ +apt_pkg.URItoFileName(fake_uri))
+ sources.write(source_line+'\n')
+ log.debug("Sources line: " + source_line, 'apt_pkg')
+ sources_count = sources_count + 1
+
+ try:
+ #we should empty the directory instead
+ os.unlink(listpath)
+ except:
+ pass
+ os.symlink('../../../../../'+file, listpath)
+ sources.close()
+
+ if sources_count == 0:
+ log.msg("No Packages files available for %s backend"%(self.backendName), 'apt_pkg')
+ return False
+
+ log.msg("Loading Packages database for "+self.status_dir,'apt_pkg')
+ #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
+ for key, value in self.apt_config.items():
+ apt_pkg.Config[key] = value
+# print "apt_pkg config:"
+# for I in apt_pkg.Config.keys():
+# print "%s \"%s\";"%(I,apt_pkg.Config[I]);
+
+ if log.isEnabled('apt'):
+ self.cache = apt_pkg.GetCache()
+ else:
+ # apt_pkg prints progress messages to stdout, disable
+ self.__save_stdout()
+ try:
+ self.cache = apt_pkg.GetCache()
+ finally:
+ self.__restore_stdout()
+
+ self.records = apt_pkg.GetPkgRecords(self.cache)
+ #for p in self.cache.Packages:
+ # print p
+ #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
+ self.loaded = 1
+ return True
+
+ def unload(self):
+ "Tries to make the packages server quit."
+ if self.loaded:
+ del self.cache
+ del self.records
+ self.loaded = 0
+
+ def cleanup(self):
+ self.unload()
+
+ def get_mirror_path(self, name, version):
+ "Find the path for version 'version' of package 'name'"
+ if not self.load(): return None
+ try:
+ for pack_vers in self.cache[name].VersionList:
+ if(pack_vers.VerStr == version):
+ file, index = pack_vers.FileList[0]
+ self.records.Lookup((file,index))
+ path = self.records.FileName
+ if len(path)>2 and path[0:2] == './':
+ path = path[2:] # Remove any leading './'
+ return path
+
+ except KeyError:
+ pass
+ return None
+
+
+ def get_mirror_versions(self, package_name):
+ """
+ Find the available versions of the package name given
+ @type package_name: string
+ @param package_name: package name to search for e.g. ;apt'
+ @return: A list of mirror versions available
+
+ """
+ vers = []
+ if not self.load(): return vers
+ try:
+ for pack_vers in self.cache[package_name].VersionList:
+ vers.append(pack_vers.VerStr)
+ except KeyError:
+ pass
+ return vers
+
+
+def cleanup(factory):
+ for backend in factory.backends:
+ backend.get_packages_db().cleanup()
+
+def get_mirror_path(factory, file):
+ """
+ Look for the path of 'file' in all backends.
+ """
+ info = AptDpkgInfo(file)
+ paths = []
+ for backend in factory.backends.values():
+ path = backend.get_packages_db().get_mirror_path(info['Package'],
+ info['Version'])
+ if path:
+ paths.append('/'+backend.base+'/'+path)
+ return paths
+
+def get_mirror_versions(factory, package):
+ """
+ Look for the available version of a package in all backends, given
+ an existing package name
+ """
+ all_vers = []
+ for backend in factory.backends.values():
+ vers = backend.get_packages_db().get_mirror_versions(package)
+ for ver in vers:
+ path = backend.get_packages_db().get_mirror_path(package, ver)
+ all_vers.append((ver, "%s/%s"%(backend.base,path)))
+ return all_vers
+
+def closest_match(info, others):
+ def compare(a, b):
+ return apt_pkg.VersionCompare(a[0], b[0])
+
+ others.sort(compare)
+ version = info['Version']
+ match = None
+ for ver,path in others:
+ if version <= ver:
+ match = path
+ break
+ if not match:
+ if not others:
+ return None
+ match = others[-1][1]
+
+ dirname=re.sub(r'/[^/]*$', '', match)
+ version=re.sub(r'^[^:]*:', '', info['Version'])
+ if dirname.find('/pool/') != -1:
+ return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
+ version, info['Architecture'])
+ else:
+ return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
+
+def import_directory(factory, dir, recursive=0):
+ """
+ Import all files in a given directory into the cache
+ This is used by apt-proxy-import to import new files
+ into the cache
+ """
+ if not os.path.exists(dir):
+ log.err('Directory ' + dir + ' does not exist', 'import')
+ return
+
+ if recursive:
+ log.msg("Importing packages from directory tree: " + dir, 'import',3)
+ for root, dirs, files in os.walk(dir):
+ for file in files:
+ import_file(factory, root, file)
+ else:
+ log.debug("Importing packages from directory: " + dir, 'import',3)
+ for file in os.listdir(dir):
+ mode = os.stat(dir + '/' + file)[stat.ST_MODE]
+ if not stat.S_ISDIR(mode):
+ import_file(factory, dir, file)
+
+ for backend in factory.backends.values():
+ backend.get_packages_db().unload()
+
+
+def import_file(factory, dir, file):
+ """
+ Import a .deb or .udeb into cache from given filename
+ """
+ if file[-4:]!='.deb' and file[-5:]!='.udeb':
+ log.msg("Ignoring (unknown file type):"+ file, 'import')
+ return
+
+ log.debug("considering: " + dir + '/' + file, 'import')
+ try:
+ paths = get_mirror_path(factory, dir+'/'+file)
+ except SystemError:
+ log.msg(file + ' skipped - wrong format or corrupted', 'import')
+ return
+ if paths:
+ if len(paths) != 1:
+ log.debug("WARNING: multiple ocurrences", 'import')
+ log.debug(str(paths), 'import')
+ cache_path = paths[0]
+ else:
+ log.debug("Not found, trying to guess", 'import')
+ info = AptDpkgInfo(dir+'/'+file)
+ cache_path = closest_match(info,
+ get_mirror_versions(factory, info['Package']))
+ if cache_path:
+ log.debug("MIRROR_PATH:"+ cache_path, 'import')
+ src_path = dir+'/'+file
+ dest_path = factory.config.cache_dir+cache_path
+
+ if not os.path.exists(dest_path):
+ log.debug("IMPORTING:" + src_path, 'import')
+ dest_path = re.sub(r'/\./', '/', dest_path)
+ if not os.path.exists(dirname(dest_path)):
+ os.makedirs(dirname(dest_path))
+ f = open(dest_path, 'w')
+ fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
+ f.truncate(0)
+ shutil.copy2(src_path, dest_path)
+ f.close()
+ if hasattr(factory, 'acc