r334 - in /debtorrent/trunk: ./ DebTorrent/BT1/FileSelector.py DebTorrent/BT1/PiecePicker.py DebTorrent/BT1/Storage.py DebTorrent/download_bt1.py test.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Sat Jan 19 21:41:09 UTC 2008
Author: camrdale-guest
Date: Sat Jan 19 21:41:09 2008
New Revision: 334
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=334
Log:
Merged revisions 227 via svnmerge from
svn+ssh://camrdale-guest@svn.debian.org/svn/debtorrent/debtorrent/branches/unique
........
r227 | camrdale-guest | 2007-08-12 20:04:18 -0700 (Sun, 12 Aug 2007) | 1 line
Modify the pickling of torrent data to work with unique piece numbers.
........
Modified:
debtorrent/trunk/ (props changed)
debtorrent/trunk/DebTorrent/BT1/FileSelector.py
debtorrent/trunk/DebTorrent/BT1/PiecePicker.py
debtorrent/trunk/DebTorrent/BT1/Storage.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/test.py
Propchange: debtorrent/trunk/
------------------------------------------------------------------------------
--- svnmerge-integrated (original)
+++ svnmerge-integrated Sat Jan 19 21:41:09 2008
@@ -1,1 +1,1 @@
-/debtorrent/branches/http1.1:1-257 /debtorrent/branches/unique:1-204,209-213,216-217,219,222-225
+/debtorrent/branches/http1.1:1-257 /debtorrent/branches/unique:1-204,209-213,216-217,219,222-225,227
Modified: debtorrent/trunk/DebTorrent/BT1/FileSelector.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/FileSelector.py?rev=334&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/FileSelector.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/FileSelector.py Sat Jan 19 21:41:09 2008
@@ -167,14 +167,18 @@
"""Unpickle the previously saved state.
Data is in the format::
- d['priority'] = [file #1 priority, file #2 priority, ...]
-
- It is a list of download priorities for each file. The priority may be::
+ d['priority'] = [file name, priority, {file name, priority, ...}]
+
+ It is a list of file names and their corrseponding download priorities
+ for each file. The priority must be::
-1 -- download disabled
0 -- highest
1 -- normal
2 -- lowest
+ Any files not listed are assumed to have a priority of -1 (download
+ disabled).
+
Also see Storage.pickle and StorageWrapper.pickle for additional keys.
@type d: C{dictionary}
@@ -183,8 +187,30 @@
"""
if d.has_key('priority'):
- if not self.init_priorities(d['priority']):
+ try:
+ # Create a dictionary of the saved file priorities
+ saved_files = {}
+ l = d['priority']
+ assert len(l) % 2 == 0
+ l = [l[x:x+2] for x in xrange(0,len(l),2)]
+ for file, priority in l:
+ saved_files[file] = priority
+
+ # Create the new file priorities list from the saved ones
+ new_priority = []
+ for file, length in self.files:
+ if not length or not file:
+ # Empty files are not downloaded
+ new_priority.append(-1)
+ else:
+ new_priority.append(saved_files.get(file, -1))
+
+ if not self.init_priorities(new_priority):
+ return
+ except:
+ logger.exception('Error unpickling file priority cache')
return
+
pieces = self.storage.unpickle(d)
init_piece_priority = self._get_piece_priority_list(self.init_priority)
self.storagewrapper.reblock([i == -1 for i in init_piece_priority])
@@ -467,7 +493,15 @@
"""
- d = {'priority': self.priority}
+ files = []
+ for i in xrange(len(self.priority)):
+ if not self.files[i][1]: # length == 0
+ continue
+ if self.priority[i] == -1:
+ continue
+ files.extend([self.files[i][0], self.priority[i]])
+
+ d = {'priority': files}
try:
s = self.storage.pickle()
sw = self.storagewrapper.pickle()
Modified: debtorrent/trunk/DebTorrent/BT1/PiecePicker.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/PiecePicker.py?rev=334&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/PiecePicker.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/PiecePicker.py Sat Jan 19 21:41:09 2008
@@ -284,6 +284,7 @@
l = self.interests[self.level_in_interests[piece]]
p = self.pos_in_interests[piece]
+ logger.info('removing from position ' + str(p) + ' of interests level ' + str(self.level_in_interests[piece]) + ' of length ' + str(len(l)) + ' piece ' + str(piece))
assert l[p] == piece
q = l[-1]
l[p] = q
Modified: debtorrent/trunk/DebTorrent/BT1/Storage.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Storage.py?rev=334&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Storage.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Storage.py Sat Jan 19 21:41:09 2008
@@ -807,20 +807,10 @@
Pickled data format::
- d['files'] = [ file #, size, mtime {, file #, size, mtime...} ]
- file # in torrent, and the size and last modification
+ d['files'] = [ file name, size, mtime {, file name, size, mtime...} ]
+ file name in torrent, and the size and last modification
time for those files. Missing files are either empty
or disabled.
- d['partial files'] = [ name, size, mtime... ]
- Names, sizes and last modification times of files containing
- partial piece data. Filenames go by the following convention:
- {file #, 0-based}{nothing, "b" or "e"}
- eg: "0e" "3" "4b" "4e"
- Where "b" specifies the partial data for the first piece in
- the file, "e" the last piece, and no letter signifying that
- the file is disabled but is smaller than one piece, and that
- all the data is cached inside so adjacent files may be
- verified.
@rtype: C{dictionary}
@return: the pickled current status of the download
@@ -828,18 +818,14 @@
"""
files = []
- pfiles = []
for i in xrange(len(self.files)):
if not self.files[i][1]: # length == 0
continue
if self.disabled[i]:
- # Removed due to files always ending on pieces
- #for start, end, offset, file in self._get_disabled_ranges(i)[2]:
- # pfiles.extend([basename(file),getsize(file),int(getmtime(file))])
continue
file = self.files[i][0]
- files.extend([i,getsize(file),int(getmtime(file))])
- return {'files': files, 'partial files': pfiles}
+ files.extend([file,getsize(file),int(getmtime(file))])
+ return {'files': files}
def unpickle(self, data):
@@ -856,18 +842,13 @@
try:
files = {}
- pfiles = {}
l = data['files']
assert len(l) % 3 == 0
l = [l[x:x+3] for x in xrange(0,len(l),3)]
- for f, size, mtime in l:
- files[f] = (size, mtime)
- l = data.get('partial files',[])
- assert len(l) % 3 == 0
- l = [l[x:x+3] for x in xrange(0,len(l),3)]
for file, size, mtime in l:
- pfiles[file] = (size, mtime)
-
+ files[file] = (size, mtime)
+ logger.debug('saved file characteristics: %r', files)
+
valid_pieces = {}
for i in xrange(len(self.files)):
if self.disabled[i]:
@@ -899,22 +880,24 @@
oldsize, oldmtime = old
if size != oldsize:
+ logger.debug('Size of file has changed: %d to %d', oldsize, size)
return False
if mtime > oldmtime+1:
+ logger.debug('Time of file has changed: %d to %d', oldmtime, mtime)
return False
if mtime < oldmtime-1:
+ logger.debug('Time of file has changed: %d to %d', oldmtime, mtime)
return False
return True
for i in xrange(len(self.files)):
if self.disabled[i]:
- # Simplified due to file boundaries being piece boundaries
continue
file, size = self.files[i]
if not size:
continue
- if ( not files.has_key(i)
- or not test(files[i],getsize(file),getmtime(file)) ):
+ if ( not files.has_key(file)
+ or not test(files[file],getsize(file),getmtime(file)) ):
start, end, offset, file = self.file_ranges[i]
start_piece, end_piece = self.file_pieces[i]
logger.debug('removing '+file)
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=334&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Sat Jan 19 21:41:09 2008
@@ -938,17 +938,6 @@
return self.storagewrapper.initialize
- def getCachedTorrentData(self):
- """Get the cached torrent data from the cache directory.
-
- @rtype: C{dictionary}
- @return: the bdecoded cached data
-
- """
-
- return self.appdataobj.getTorrentData(self.infohash)
-
-
def _make_upload(self, connection, ratelimiter, totalup):
"""Create a new Upload instance
@@ -1085,6 +1074,7 @@
for i in xrange(self.len_pieces):
if self.storagewrapper.do_I_have(i):
+ logger.info('informing picker I have piece: ' + str(i))
self.picker.complete(i)
self.upmeasure = Measure(self.config['max_rate_period'],
self.config['upload_rate_fudge'])
Modified: debtorrent/trunk/test.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/test.py?rev=334&op=diff
==============================================================================
--- debtorrent/trunk/test.py (original)
+++ debtorrent/trunk/test.py Sat Jan 19 21:41:09 2008
@@ -251,6 +251,22 @@
]),
]),
+ '9': ('Run this test multiple times to test restarting the downloader.',
+ {1: []},
+ {1: (1, [], {'clean': False})},
+ [(1, ['update']),
+ (1, ['install', 'aboot-base']),
+ (1, ['install', 'aap-doc']),
+ (1, ['install', 'ada-reference-manual']),
+ (1, ['install', 'aspectj-doc']),
+ (1, ['install', 'fop-doc']),
+ (1, ['install', 'jswat-doc']),
+ (1, ['install', 'asis-doc']),
+ (1, ['install', 'bison-doc']),
+ (1, ['install', 'crash-whitepaper']),
+ (1, ['install', 'doc-iana']),
+ ]),
+
}
assert 'all' not in tests
@@ -330,11 +346,12 @@
"""
- for root, dirs, files in os.walk(top, topdown=False):
- for name in files:
- os.remove(os.path.join(root, name))
- for name in dirs:
- os.rmdir(os.path.join(root, name))
+ if exists(top):
+ for root, dirs, files in os.walk(top, topdown=False):
+ for name in files:
+ os.remove(os.path.join(root, name))
+ for name in dirs:
+ os.rmdir(os.path.join(root, name))
def join(dir):
"""Join together a list of directories into a path string.
@@ -359,7 +376,8 @@
"""
- os.makedirs(join(dir))
+ if not exists(join(dir)):
+ os.makedirs(join(dir))
def touch(path):
"""Create an empty file.
@@ -368,9 +386,10 @@
@param path: the path to create
"""
-
- f = open(join(path), 'w')
- f.close()
+
+ if not exists(join(path)):
+ f = open(join(path), 'w')
+ f.close()
def start(func, args, work_dir = None):
"""Fork and start a background process running.
@@ -536,16 +555,18 @@
except:
pass
- # Create the directory structure needed by apt
- makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d'])
- makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial'])
- makedirs([downloader_dir, 'var', 'lib', 'dpkg'])
- makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial'])
- touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock'])
- touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock'])
- touch([downloader_dir, 'var', 'lib', 'dpkg', 'status'])
- touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock'])
-
+ # Create the directory structure needed by apt
+ makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d'])
+ makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial'])
+ makedirs([downloader_dir, 'var', 'lib', 'dpkg'])
+ rmrf(join([downloader_dir, 'var', 'cache', 'apt', 'archives']))
+ makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial'])
+ touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock'])
+ touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock'])
+ touch([downloader_dir, 'var', 'lib', 'dpkg', 'status'])
+ touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock'])
+
+ if not exists(join([downloader_dir, 'etc', 'apt', 'sources.list'])):
# Create apt's config files
f = open(join([downloader_dir, 'etc', 'apt', 'sources.list']), 'w')
if exists('/usr/lib/apt/methods/newdebtorrent'):
@@ -556,6 +577,7 @@
f.write('deb http://localhost:' + str(num_down) + '988/' + mirror + '/ stable ' + suites + '\n')
f.close()
+ if not exists(join([downloader_dir, 'etc', 'apt', 'apt.conf'])):
f = open(join([downloader_dir, 'etc', 'apt', 'apt.conf']), 'w')
f.write('Dir "' + downloader_dir + '"')
f.write(apt_conf_template)
More information about the Debtorrent-commits
mailing list