[med-svn] [python-mne] 36/353: ENH: preload keyword in init, support memmapped-file, moved test
Yaroslav Halchenko
debian at onerussian.com
Fri Nov 27 17:24:28 UTC 2015
This is an automated email from the git hooks/post-receive script.
yoh pushed a commit to tag 0.4
in repository python-mne.
commit 58e8c0dc27bb6e63d5b3d343951d31ee0ee0b1dd
Author: Martin Luessi <mluessi at nmr.mgh.harvard.edu>
Date: Tue Jan 3 12:37:51 2012 -0500
ENH: preload keyword in init, support memmapped-file, moved test
---
mne/fiff/raw.py | 67 +++++++++++++++++++++++++++++++---------------
mne/fiff/tests/test_raw.py | 30 +++++++++++++++++++++
mne/tests/test_raw.py | 46 -------------------------------
3 files changed, 75 insertions(+), 68 deletions(-)
diff --git a/mne/fiff/raw.py b/mne/fiff/raw.py
index 47a8b8b..2b99e01 100644
--- a/mne/fiff/raw.py
+++ b/mne/fiff/raw.py
@@ -31,7 +31,7 @@ class Raw(dict):
"""
- def __init__(self, fname, allow_maxshield=False):
+ def __init__(self, fname, allow_maxshield=False, preload=False):
"""
Parameters
----------
@@ -41,6 +41,12 @@ class Raw(dict):
allow_maxshield: bool, (default False)
allow_maxshield if True XXX ???
+ preload: bool or str (default False)
+ Preload data into memory for data manipulation and faster indexing.
+ If True, the data will be preloaded into memory (fast, requires
+ large amount of memory). If preload is a string, preload is the
+ file name of a memory-mapped file which is used to store the data
+ (slower, requires less memory).
"""
# Open the file
@@ -155,8 +161,22 @@ class Raw(dict):
self.fid = fid
self.info = info
- self._data = None
- self._times = None
+
+ if preload:
+ nchan = self.info['nchan']
+ nsamp = self.last_samp - self.first_samp + 1
+ if isinstance(preload, str):
+ # preload data using a memmap file
+ self._data = np.memmap(preload, mode='w+', dtype='float32',
+ shape=(nchan, nsamp))
+ else:
+ self._data = np.empty((nchan, nsamp))
+
+ self._data, self._times = read_raw_segment(self,
+ data_buffer=self._data)
+ self._preloaded = True
+ else:
+ self._preloaded = False
def __getitem__(self, item):
"""getting raw data content with python slicing"""
@@ -187,7 +207,7 @@ class Raw(dict):
if sel is not None and len(sel) == 0:
raise Exception("Empty channel list")
- if self._data is not None:
+ if self._preloaded:
return (self._data[sel, start:stop], self._times[start:stop])
else:
return read_raw_segment(self, start=start, stop=stop, sel=sel)
@@ -197,6 +217,10 @@ class Raw(dict):
def __setitem__(self, item, value):
"""setting raw data content with python slicing"""
if isinstance(item, tuple): # slicing required
+ if not self._preloaded:
+ raise RuntimeError('Modifying data of Raw is only supported '
+ 'when preloading is used. Use preload=True '
+ '(or string) in the constructor.')
if len(item) == 2: # channels and time instants
time_slice = item[1]
if isinstance(item[0], slice):
@@ -223,25 +247,12 @@ class Raw(dict):
if sel is not None and len(sel) == 0:
raise Exception("Empty channel list")
- if self._data is None:
- # data needs to be loaded into memory
- self.preload()
-
# set the data
self._data[sel, start:stop] = value
else:
super(Raw, self).__setitem__(item, value)
- def preload(self):
- """preload the raw data into memory in order to modify the data and
- for faster indexing
- """
- if self._data is None:
- data, times = self[:, :]
- self._data = data
- self._times = times
-
def save(self, fname, picks=None, tmin=None, tmax=None, buffer_size_sec=10,
drop_small_buffer=False):
"""Save raw data to file
@@ -332,7 +343,7 @@ class Raw(dict):
return self.info['ch_names']
-def read_raw_segment(raw, start=0, stop=None, sel=None):
+def read_raw_segment(raw, start=0, stop=None, sel=None, data_buffer=None):
"""Read a chunck of raw data
Parameters
@@ -351,8 +362,8 @@ def read_raw_segment(raw, start=0, stop=None, sel=None):
sel: array, optional
Indices of channels to select
- node: tree node
- The node of the tree where to look
+ data_buffer: array, optional
+ numpy array to fill with data read, must have the correct shape
Returns
-------
@@ -387,7 +398,13 @@ def read_raw_segment(raw, start=0, stop=None, sel=None):
cal = np.diag(raw.cals.ravel())
if sel is None:
- data = np.empty((nchan, stop - start))
+ data_shape = (nchan, stop - start)
+ if data_buffer is not None:
+ if data_buffer.shape != data_shape:
+ raise ValueError('data_buffer has incorrect shape')
+ data = data_buffer
+ else:
+ data = np.empty(data_shape)
if raw.proj is None and raw.comp is None:
mult = None
else:
@@ -399,7 +416,13 @@ def read_raw_segment(raw, start=0, stop=None, sel=None):
mult = raw.proj * raw.comp * cal
else:
- data = np.empty((len(sel), stop - start))
+ data_shape = (len(sel), stop - start)
+ if data_buffer is not None:
+ if data_buffer.shape != data_shape:
+ raise ValueError('data_buffer has incorrect shape')
+ data = data_buffer
+ else:
+ data = np.empty(data_shape)
if raw.proj is None and raw.comp is None:
mult = None
cal = np.diag(raw.cals[sel].ravel())
diff --git a/mne/fiff/tests/test_raw.py b/mne/fiff/tests/test_raw.py
index 8952f70..abe4eb1 100644
--- a/mne/fiff/tests/test_raw.py
+++ b/mne/fiff/tests/test_raw.py
@@ -1,5 +1,6 @@
import os.path as op
+import numpy as np
from nose.tools import assert_true
from numpy.testing import assert_array_almost_equal
@@ -63,3 +64,32 @@ def test_io_raw():
assert_array_almost_equal(raw.info['dig'][0]['r'], raw2.info['dig'][0]['r'])
fname = op.join(op.dirname(__file__), 'data', 'test_raw.fif')
+
+
+def test_preload_modify():
+ """ Test preloading and modifying data
+ """
+ for preload in [False, True, 'memmap.dat']:
+ raw = Raw(fif_fname, preload=preload)
+
+ nsamp = raw.last_samp - raw.first_samp + 1
+ picks = pick_types(raw.info, meg='grad')
+
+ data = np.random.randn(len(picks), nsamp / 2)
+
+ try:
+ raw[picks, :nsamp / 2] = data
+ except RuntimeError as err:
+ if not preload:
+ continue
+ else:
+ raise err
+
+ tmp_fname = 'raw.fif'
+ raw.save(tmp_fname)
+
+ raw_new = Raw(tmp_fname)
+ data_new, _ = raw_new[picks, :nsamp / 2]
+
+ assert_array_almost_equal(data, data_new)
+
diff --git a/mne/tests/test_raw.py b/mne/tests/test_raw.py
deleted file mode 100644
index b43aaec..0000000
--- a/mne/tests/test_raw.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os.path as op
-
-import numpy as np
-from numpy.testing import assert_array_almost_equal
-
-from ..datasets import sample
-from ..fiff import pick_types
-
-from ..fiff.raw import Raw
-
-examples_folder = op.join(op.dirname(__file__), '..', '..', 'examples')
-data_path = sample.data_path(examples_folder)
-fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif')
-
-
-def test_read_write():
- raw_orig = Raw(fname)
- data_orig, times_orig = raw_orig[:, :]
-
- tmp_fname = 'tmp.fif'
- raw_orig.save(tmp_fname)
-
- raw = Raw(tmp_fname)
- data, times = raw[:, :]
-
- assert_array_almost_equal(data_orig, data)
- assert_array_almost_equal(times_orig, times)
-
-
-def test_modify_data():
- raw = Raw(fname)
-
- n_samp = raw.last_samp - raw.first_samp
- picks = pick_types(raw.info, meg='grad')
-
- data = np.random.randn(len(picks), n_samp / 2)
-
- raw[picks, :n_samp / 2] = data
-
- tmp_fname = 'tmp.fif'
- raw.save(tmp_fname)
-
- raw_new = Raw(tmp_fname)
- data_new, _ = raw_new[picks, :n_samp / 2]
-
- assert_array_almost_equal(data, data_new)
\ No newline at end of file
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/python-mne.git
More information about the debian-med-commit
mailing list