[python-hdf5storage] 70/84: Added HDF5 filter tests for writing compressed and uncompressed data. (cherry picked from commit bdd216c3b71e9ff22ccd87bf196053ba09f7f166)

[Ghislain Vaillant]

This is an automated email from the git hooks/post-receive script.

ghisvail-guest pushed a commit to annotated tag 0.1.10
in repository python-hdf5storage.

commit b8e4ac6b2ad756ca2bc594a3e1d840a5ecf9f954
Author: Freja Nordsiek <fnordsie at gmail.com>
Date:   Mon Aug 17 23:10:06 2015 -0400

    Added HDF5 filter tests for writing compressed and uncompressed data.
    (cherry picked from commit bdd216c3b71e9ff22ccd87bf196053ba09f7f166)
---
 tests/test_hdf5_filters.py | 169 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 169 insertions(+)

diff --git a/tests/test_hdf5_filters.py b/tests/test_hdf5_filters.py
index 3e2a30e..e6429a2 100644
--- a/tests/test_hdf5_filters.py
+++ b/tests/test_hdf5_filters.py
@@ -84,6 +84,135 @@ def check_read_filters(filters):
     assert_equal(out, data)
 
 
+def check_write_filters(filters):
+    # Read out the filter arguments.
+    filts = {'compression': 'gzip',
+             'shuffle': True,
+             'fletcher32': True,
+             'gzip_level': 7}
+    for k, v in filters.items():
+        filts[k] = v
+
+    # Make some random data. The dtype must be restricted so that it can
+    # be read back reliably.
+    dims = random.randint(1, 4)
+    dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
+        'complex128']))
+
+    data = random_numpy(shape=random_numpy_shape(dims,
+                        max_array_axis_length),
+                        dtype=random.choice(dts))
+    # Make a random name.
+    name = random_name()
+
+    # Write the data to the proper file with the given name with the
+    # provided filters and read it backt. The file needs to be deleted
+    # before and after to keep junk from building up.
+    if os.path.exists(filename):
+        os.remove(filename)
+    try:
+        hdf5storage.write(data, path=name, filename=filename, \
+            store_python_metadata=False, matlab_compatible=False, \
+            compress=True, compress_size_threshold=0, \
+            compression_algorithm=filts['compression'], \
+            gzip_compression_level=filts['gzip_level'], \
+            shuffle_filter=filts['shuffle'], \
+            compressed_fletcher32_filter=filts['fletcher32'])
+
+        with h5py.File(filename) as f:
+            d = f[name]
+            fletcher32 = d.fletcher32
+            shuffle = d.shuffle
+            compression = d.compression
+            gzip_level = d.compression_opts
+            out = d[...]
+    except:
+        raise
+    finally:
+        if os.path.exists(filename):
+            os.remove(filename)
+
+    # Check the filters
+    assert fletcher32 == filts['fletcher32']
+    assert shuffle == filts['shuffle']
+    assert compression == filts['compression']
+    if filts['compression'] == 'gzip':
+        assert gzip_level == filts['gzip_level']
+
+    # Compare
+    assert_equal(out, data)
+
+
+def check_uncompressed_write_filters(method,
+                                     uncompressed_fletcher32_filter,
+                                     filters):
+    # Read out the filter arguments.
+    filts = {'compression': 'gzip',
+             'shuffle': True,
+             'fletcher32': True,
+             'gzip_level': 7}
+    for k, v in filters.items():
+        filts[k] = v
+
+    # Make some random data. The dtype must be restricted so that it can
+    # be read back reliably.
+    dims = random.randint(1, 4)
+    dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
+        'complex128']))
+
+    data = random_numpy(shape=random_numpy_shape(dims,
+                        max_array_axis_length),
+                        dtype=random.choice(dts))
+    # Make a random name.
+    name = random_name()
+
+    # Make the options to disable compression by the method specified,
+    # which is either that it is outright disabled or that the data is
+    # smaller than the compression threshold.
+    if method == 'compression_disabled':
+        opts = {'compress': False, 'compress_size_threshold': 0}
+    else:
+        opts = {'compress': True,
+                'compress_size_threshold': data.nbytes + 1}
+
+    # Write the data to the proper file with the given name with the
+    # provided filters and read it backt. The file needs to be deleted
+    # before and after to keep junk from building up.
+    if os.path.exists(filename):
+        os.remove(filename)
+    try:
+        hdf5storage.write(data, path=name, filename=filename, \
+            store_python_metadata=False, matlab_compatible=False, \
+            compression_algorithm=filts['compression'], \
+            gzip_compression_level=filts['gzip_level'], \
+            shuffle_filter=filts['shuffle'], \
+            compressed_fletcher32_filter=filts['fletcher32'], \
+            uncompressed_fletcher32_filter= \
+            uncompressed_fletcher32_filter, \
+            **opts)
+
+        with h5py.File(filename) as f:
+            d = f[name]
+            fletcher32 = d.fletcher32
+            shuffle = d.shuffle
+            compression = d.compression
+            gzip_level = d.compression_opts
+            out = d[...]
+    except:
+        raise
+    finally:
+        if os.path.exists(filename):
+            os.remove(filename)
+
+    # Check the filters
+    assert compression == None
+    assert shuffle == False
+    assert fletcher32 == uncompressed_fletcher32_filter
+
+    # Compare
+    assert_equal(out, data)
+
+
 def test_read_filtered_data():
     for compression in ('gzip', 'lzf'):
         for shuffle in (True, False):
@@ -100,3 +229,43 @@ def test_read_filtered_data():
                                    'fletcher32': fletcher32,
                                    'gzip_level': level}
                         yield check_read_filters, filters
+
+
+def test_write_filtered_data():
+    for compression in ('gzip', 'lzf'):
+        for shuffle in (True, False):
+            for fletcher32 in (True, False):
+                if compression != 'gzip':
+                    filters = {'compression': compression,
+                               'shuffle': shuffle,
+                               'fletcher32': fletcher32}
+                    yield check_read_filters, filters
+                else:
+                    for level in range(10):
+                        filters = {'compression': compression,
+                                   'shuffle': shuffle,
+                                   'fletcher32': fletcher32,
+                                   'gzip_level': level}
+                        yield check_write_filters, filters
+
+
+def test_uncompressed_write_filtered_data():
+    for method in ('compression_disabled', 'data_too_small'):
+        for uncompressed_fletcher32_filter in (True, False):
+            for compression in ('gzip', 'lzf'):
+                for shuffle in (True, False):
+                    for fletcher32 in (True, False):
+                        if compression != 'gzip':
+                            filters = {'compression': compression,
+                                       'shuffle': shuffle,
+                                       'fletcher32': fletcher32}
+                            yield check_read_filters, filters
+                        else:
+                            for level in range(10):
+                                filters = {'compression': compression,
+                                           'shuffle': shuffle,
+                                           'fletcher32': fletcher32,
+                                           'gzip_level': level}
+                                yield check_uncompressed_write_filters,\
+                                    method, uncompressed_fletcher32_filter,\
+                                    filters

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/python-hdf5storage.git