r383 - in /debtorrent/trunk: DebTorrent/BT1/AptListener.py DebTorrent/BT1/HTTPDownloader.py DebTorrent/BT1/makemetafile.py debian/changelog
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Fri Jun 20 03:30:24 UTC 2008
Author: camrdale-guest
Date: Fri Jun 20 03:30:24 2008
New Revision: 383
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=383
Log:
Decompress the files needed to create torrents so they are created
and started faster (this and the previous Closes: #463676)
Modified:
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/HTTPDownloader.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/debian/changelog
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=383&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Fri Jun 20 03:30:24 2008
@@ -368,9 +368,13 @@
if path[-1][-4:] == '.deb':
return self.get_package(connection, path, httpreq)
else:
+ decompress = False
+ if path[-1] in ('Packages.gz', 'Packages.bz2'):
+ decompress = True
+
# Save the connection info and start downloading the file
self.cache_waiting.setdefault('/'.join(path), []).append((connection, httpreq))
- self.Cache.download_get(path, self.get_cached_callback)
+ self.Cache.download_get(path, self.get_cached_callback, decompress)
return None
# Save the identifiers from Release files
Modified: debtorrent/trunk/DebTorrent/BT1/HTTPDownloader.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/HTTPDownloader.py?rev=383&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/HTTPDownloader.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/HTTPDownloader.py Fri Jun 20 03:30:24 2008
@@ -147,7 +147,7 @@
self.request_size = 0
self.endflag = False
self.error = None
- self.retry_period = 10
+ self.retry_period = 2
self._retry_period = None
self.errorcount = 0
self.goodseed = False
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=383&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Fri Jun 20 03:30:24 2008
@@ -18,7 +18,7 @@
"""
-from os.path import getsize, split, join, abspath, isdir
+from os.path import getsize, split, join, abspath, isdir, exists
from os import listdir
from sha import sha
from copy import copy
@@ -155,6 +155,34 @@
# Otherwise add '-all' to the end
f_all = f + '-all'
return f_all
+
+def universal_open(filename):
+ """Open the file for reading, works for compressed files.
+
+ Will check for a decompressed version of compressed files, otherwise uses
+ an on-the-fly decompression file-like object.
+
+ @type filename: C{string}
+ @param filename: the path of the file to open
+ @rtype: C{file}
+ @return: the read-only file-like object
+
+ """
+
+ # Open and possibly decompress the file
+ if filename.endswith('.gz'):
+ if exists(filename[:-3]):
+ f = open(filename[:-3])
+ else:
+ f = gzip.open(filename, 'r')
+ elif filename.endswith('.bz2'):
+ if exists(filename[:-4]):
+ f = open(filename[:-4])
+ else:
+ f = BZ2File(filename)
+ else:
+ f = open(filename)
+ return f
def make_meta_file(file, params = {}, progress = lambda x: None):
"""Create the torrent files from a Packages file.
@@ -312,10 +340,7 @@
piece_url = ''
try:
- if pieces_file.endswith('.gz'):
- f = gzip.open(pieces_file)
- else:
- f = open(pieces_file)
+ f = universal_open(pieces_file)
except:
logger.exception('sub-pieces file not found: '+pieces_file)
return {}
@@ -359,10 +384,7 @@
piece_url = ''
try:
- if torrent_file.endswith('.gz'):
- f = gzip.open(torrent_file)
- else:
- f = open(torrent_file)
+ f = universal_open(torrent_file)
except:
logger.exception('torrent ordering file not found: '+torrent_file)
return (pieces, headers)
@@ -587,10 +609,7 @@
(piece_ordering_all, ordering_all_headers) = getordering(torrent_all_file)
file = abspath(file)
- if file.endswith('.gz'):
- f = gzip.open(file)
- else:
- f = open(file)
+ f = universal_open(file)
(info, info_all) = getpieces(f, encoding, progress, separate_all, sub_pieces,
piece_ordering, piece_ordering_all,
int(ordering_headers.get('NextPiece', 0)),
@@ -748,7 +767,7 @@
setattr(self, down + '_file', filename)
self.download.remove(down)
else:
- self.cache.download_get(down_path, self._download_callback)
+ self.cache.download_get(down_path, self._download_callback, True)
if not self.download:
self._start()
@@ -794,13 +813,7 @@
"""Process a downloaded Packages file and start a torrent."""
try:
- # Open and possibly decompress the file
- if self.filename.endswith('.gz'):
- f = gzip.open(self.filename, 'r')
- elif self.filename.endswith('.bz2'):
- f = BZ2File(self.filename)
- else:
- f = open(self.filename)
+ f = universal_open(self.filename)
except:
logger.warning('Packages file could not be opened')
self.sched(self._finished)
Modified: debtorrent/trunk/debian/changelog
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/debian/changelog?rev=383&op=diff
==============================================================================
--- debtorrent/trunk/debian/changelog (original)
+++ debtorrent/trunk/debian/changelog Fri Jun 20 03:30:24 2008
@@ -6,8 +6,10 @@
so that apt treats debtorrent sources as local
* Make the pieces and unique piece number locations config options,
and use the new ones at merkel.debian.org/~camrdale/.
+ * Decompress the files needed to create torrents so they are created
+ and started faster (this and the previous Closes: #463676)
- -- Cameron Dale <camrdale at gmail.com> Thu, 19 Jun 2008 15:25:05 -0700
+ -- Cameron Dale <camrdale at gmail.com> Thu, 19 Jun 2008 20:29:13 -0700
debtorrent (0.1.6) experimental; urgency=low
More information about the Debtorrent-commits
mailing list