[pytables] 01/15: New upstream version 3.4.2
Antonio Valentino
a_valentino-guest at moszumanska.debian.org
Fri Jun 30 18:07:42 UTC 2017
This is an automated email from the git hooks/post-receive script.
a_valentino-guest pushed a commit to branch master
in repository pytables.
commit b5838e8771bbac681445cb850812d87e24c9aac7
Author: Antonio Valentino <antonio.valentino at tiscali.it>
Date: Wed Jun 28 20:25:29 2017 +0000
New upstream version 3.4.2
---
.readthedocs.yml | 7 +
.travis.yml | 6 +-
ANNOUNCE.txt.in | 2 +-
.../zstd-1.0.0/LICENSE => LICENSES/ZSTD.TXT | 0
Makefile | 2 +-
README.rst | 5 +-
RELEASE_NOTES.txt | 36 +-
VERSION | 2 +-
appveyor.yml | 72 +-
c-blosc/.gitignore | 2 +
c-blosc/ANNOUNCE.rst | 21 +-
c-blosc/CMakeLists.txt | 87 +-
c-blosc/README.rst | 2 +-
c-blosc/RELEASE_NOTES.rst | 23 +
c-blosc/bench/bench.c | 8 +-
c-blosc/blosc.pc.in | 14 +
c-blosc/blosc/CMakeLists.txt | 67 +-
c-blosc/blosc/bitshuffle-generic.c | 74 +-
c-blosc/blosc/bitshuffle-generic.h | 26 +-
c-blosc/blosc/blosc.c | 14 +-
c-blosc/blosc/blosc.h | 6 +-
c-blosc/internal-complibs/lz4-1.7.2/lz4.h | 360 -------
c-blosc/internal-complibs/lz4-1.7.2/lz4hc.h | 189 ----
.../{lz4-1.7.2 => lz4-1.7.5}/lz4.c | 701 ++++++-------
c-blosc/internal-complibs/lz4-1.7.5/lz4.h | 463 +++++++++
.../{lz4-1.7.2 => lz4-1.7.5}/lz4hc.c | 352 +++----
c-blosc/internal-complibs/lz4-1.7.5/lz4hc.h | 228 +++++
c-blosc/internal-complibs/lz4-1.7.5/lz4opt.h | 363 +++++++
c-blosc/internal-complibs/zstd-1.0.0/Makefile | 115 ---
.../zstd-1.0.0/compress/zbuff_compress.c | 319 ------
.../zstd-1.0.0/decompress/zbuff_decompress.c | 252 -----
.../zstd-1.0.0/dictBuilder/zdict.h | 111 --
c-blosc/internal-complibs/zstd-1.1.3/BUCK | 186 ++++
c-blosc/internal-complibs/zstd-1.1.3/Makefile | 160 +++
.../{zstd-1.0.0 => zstd-1.1.3}/README.md | 22 +-
.../{zstd-1.0.0 => zstd-1.1.3}/common/bitstream.h | 8 +-
.../common/entropy_common.c | 15 +-
.../common/error_private.c} | 71 +-
.../common/error_private.h | 32 +-
.../{zstd-1.0.0 => zstd-1.1.3}/common/fse.h | 72 +-
.../common/fse_decompress.c | 49 +-
.../{zstd-1.0.0 => zstd-1.1.3}/common/huf.h | 46 +-
.../{zstd-1.0.0 => zstd-1.1.3}/common/mem.h | 6 +-
c-blosc/internal-complibs/zstd-1.1.3/common/pool.c | 194 ++++
c-blosc/internal-complibs/zstd-1.1.3/common/pool.h | 56 ++
.../zstd-1.1.3/common/threading.c | 79 ++
.../zstd-1.1.3/common/threading.h | 104 ++
.../{zstd-1.0.0 => zstd-1.1.3}/common/xxhash.c | 2 +-
.../{zstd-1.0.0 => zstd-1.1.3}/common/xxhash.h | 0
.../common/zstd_common.c | 10 -
.../common/zstd_errors.h} | 25 +-
.../common/zstd_internal.h | 59 +-
.../compress/fse_compress.c | 217 ++--
.../compress/huf_compress.c | 209 ++--
.../compress/zstd_compress.c | 691 ++++++++-----
.../{zstd-1.0.0 => zstd-1.1.3}/compress/zstd_opt.h | 159 +--
.../zstd-1.1.3/compress/zstdmt_compress.c | 740 ++++++++++++++
.../zstd-1.1.3/compress/zstdmt_compress.h | 78 ++
.../decompress/huf_decompress.c | 22 +-
.../decompress/zstd_decompress.c | 1056 +++++++++++++++-----
.../common => zstd-1.1.3/deprecated}/zbuff.h | 97 +-
.../zstd-1.1.3/deprecated/zbuff_common.c | 26 +
.../zstd-1.1.3/deprecated/zbuff_compress.c | 145 +++
.../zstd-1.1.3/deprecated/zbuff_decompress.c | 74 ++
.../zstd-1.1.3/dictBuilder/cover.c | 1021 +++++++++++++++++++
.../dictBuilder/divsufsort.c | 0
.../dictBuilder/divsufsort.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/dictBuilder/zdict.c | 189 ++--
.../zstd-1.1.3/dictBuilder/zdict.h | 201 ++++
.../zstd-1.1.3/dll/example/Makefile | 47 +
.../zstd-1.1.3/dll/example/README.md | 69 ++
.../zstd-1.1.3/dll/example/build_package.bat | 17 +
.../zstd-1.1.3/dll/example/fullbench-dll.sln | 25 +
.../zstd-1.1.3/dll/example/fullbench-dll.vcxproj | 181 ++++
.../internal-complibs/zstd-1.1.3/dll/libzstd.def | 86 ++
.../legacy/zstd_legacy.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v01.c | 198 ++--
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v01.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v02.c | 102 +-
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v02.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v03.c | 105 +-
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v03.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v04.c | 278 +-----
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v04.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v05.c | 292 ++----
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v05.h | 0
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v06.c | 287 +-----
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v06.h | 68 +-
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v07.c | 319 ++----
.../{zstd-1.0.0 => zstd-1.1.3}/legacy/zstd_v07.h | 59 +-
.../{zstd-1.0.0 => zstd-1.1.3}/libzstd.pc.in | 10 +-
.../{zstd-1.0.0 => zstd-1.1.3}/zstd.h | 456 +++++----
c-blosc/tests/Makefile | 4 +-
c-blosc/tests/test_common.h | 2 +-
cpuinfo.py | 737 ++++++++------
doc/source/_templates/travis-ci.html | 3 -
.../altered_nature/static/altered_nature.css_t | 299 ------
.../_theme/altered_nature/static/old_pygments.css | 54 -
doc/source/_theme/altered_nature/theme.conf | 4 -
doc/source/_theme/cloud/layout.html | 98 --
doc/source/_theme/cloud/static/cloud.css_t | 969 ------------------
doc/source/_theme/cloud/static/icon-note.png | Bin 1204 -> 0 bytes
doc/source/_theme/cloud/static/icon-seealso.png | Bin 951 -> 0 bytes
doc/source/_theme/cloud/static/icon-todo.png | Bin 1324 -> 0 bytes
doc/source/_theme/cloud/static/icon-warning.png | Bin 887 -> 0 bytes
doc/source/_theme/cloud/static/jquery.cookie.js | 41 -
doc/source/_theme/cloud/static/toggle_sections.js | 67 --
doc/source/_theme/cloud/static/toggle_sidebar.js_t | 49 -
doc/source/_theme/cloud/theme.conf | 74 --
doc/source/conf.py | 155 +--
doc/source/cookbook/py2exe_howto/pytables_test.py | 5 +-
doc/source/cookbook/threading.rst | 4 +-
doc/source/dev_team.rst | 3 +-
doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst | 34 +-
...E_NOTES_v3.3.x.rst => RELEASE_NOTES_v3.4.x.rst} | 0
doc/source/release_notes.rst | 7 +-
doc/source/usersguide/index.rst | 2 +-
doc/source/usersguide/installation.rst | 35 +-
.../usersguide/libref/homogenous_storage.rst | 4 +-
doc/source/usersguide/usersguide.rst | 3 +-
doc/sphinxext/LICENSE.txt | 141 ---
doc/sphinxext/__init__.py | 1 -
doc/sphinxext/apigen.py | 427 --------
doc/sphinxext/comment_eater.py | 158 ---
doc/sphinxext/compiler_unparse.py | 860 ----------------
doc/sphinxext/docscrape.py | 500 ---------
doc/sphinxext/docscrape_sphinx.py | 227 -----
doc/sphinxext/inheritance_diagram.py | 407 --------
doc/sphinxext/ipython_console_highlighting.py | 114 ---
doc/sphinxext/ipython_directive.py | 635 ------------
doc/sphinxext/numpydoc.py | 169 ----
doc/sphinxext/phantom_import.py | 162 ---
doc/sphinxext/plot_directive.py | 636 ------------
doc/sphinxext/setup.py | 31 -
doc/sphinxext/traitsdoc.py | 140 ---
environment.yml | 3 +
requirements.txt | 2 +-
setup.py | 106 +-
src/H5ARRAY.c | 4 +-
src/H5ARRAY.h | 2 +-
src/H5TB-opt.c | 2 +-
src/H5TB-opt.h | 2 +-
src/H5VLARRAY.c | 2 +-
src/H5VLARRAY.h | 2 +-
tables/atom.py | 2 +-
tables/attributeset.py | 1 +
tables/definitions.pxd | 117 ++-
tables/description.py | 1 +
tables/exceptions.py | 1 +
tables/file.py | 1 +
tables/filters.py | 1 +
tables/group.py | 2 +-
tables/hdf5extension.pyx | 10 +-
tables/idxutils.py | 3 +-
tables/index.py | 3 +-
tables/indexes.py | 12 +-
tables/leaf.py | 2 +
tables/link.py | 5 +-
tables/linkextension.pyx | 8 +-
tables/scripts/ptdump.py | 2 +-
tables/scripts/ptrepack.py | 13 +-
tables/scripts/pttree.py | 2 +-
tables/table.py | 2 +
tables/tests/test_all.py | 1 +
tables/tests/test_create.py | 46 +-
tables/tests/test_utils.py | 95 ++
tables/unimplemented.py | 2 +
tables/utilsextension.pxd | 4 +-
tables/utilsextension.pyx | 10 +-
169 files changed, 8862 insertions(+), 11519 deletions(-)
diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 0000000..3f256ce
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,7 @@
+conda:
+ file: environment.yml
+python:
+ version: 3
+ setup_py_install: true
+ extra_requirements:
+ - docs
diff --git a/.travis.yml b/.travis.yml
index 4452c6f..da7c6c3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,7 @@ python:
- 2.7
- 3.4
- 3.5
+ - 3.6
sudo: false
@@ -23,10 +24,13 @@ install:
script:
- cd .. && python -m tables.tests.test_all
+ - pt2to3 -h
+ - ptrepack -h
+ - ptdump -h
+ - pttree -h
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/58f8a9014ef02f6217ec
on_success: change
-
diff --git a/ANNOUNCE.txt.in b/ANNOUNCE.txt.in
index dbbc856..fc56b7e 100644
--- a/ANNOUNCE.txt.in
+++ b/ANNOUNCE.txt.in
@@ -15,7 +15,7 @@ version, please refer to: http://www.pytables.org/release_notes.html
You can install it via pip or download a source package with generated
PDF and HTML docs from:
-http://sourceforge.net/projects/pytables/files/pytables/@VERSION@
+https://github.com/PyTables/PyTables/releases/v@VERSION@
For an online version of the manual, visit:
http://www.pytables.org/usersguide/index.html
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/LICENSE b/LICENSES/ZSTD.TXT
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/LICENSE
rename to LICENSES/ZSTD.TXT
diff --git a/Makefile b/Makefile
index 6ebf72d..bc2b685 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@
VERSION = $(shell cat VERSION)
SRCDIRS = src doc
GENERATED = ANNOUNCE.txt
-PYTHON = python
+PYTHON = python3
PYPLATFORM = $(shell $(PYTHON) -c "from distutils.util import get_platform; print(get_platform())")
PYVER = $(shell $(PYTHON) -V 2>&1 | cut -c 8-10)
PYBUILDDIR = $(PWD)/build/lib.$(PYPLATFORM)-$(PYVER)
diff --git a/README.rst b/README.rst
index 3e73474..c225f6c 100644
--- a/README.rst
+++ b/README.rst
@@ -97,7 +97,7 @@ To compile PyTables you will need, at least, a recent version of HDF5
(C flavor) library, the Zlib compression library and the NumPy and
Numexpr packages. Besides, it comes with support for the Blosc, LZO
and bzip2 compressor libraries. Blosc is mandatory, but PyTables comes
-with Blosc sources so, although it si recommended to have Blosc
+with Blosc sources so, although it is recommended to have Blosc
installed in your system, you don't absolutely need to install it
separately. LZO and bzip2 compression libraries are, however,
optional.
@@ -105,8 +105,7 @@ optional.
Installation
------------
-1. Make sure you have HDF5 version 1.8.4 or above. HDF5 1.10.x is not
-supported.
+1. Make sure you have HDF5 version 1.8.4 or above.
On OSX you can install HDF5 using `Homebrew <http://brew.sh>`_::
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 2268dca..ce04ae4 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,5 +1,5 @@
=======================================
- Release notes for PyTables 3.3 series
+ Release notes for PyTables 3.4 series
=======================================
:Author: PyTables Developers
@@ -7,33 +7,25 @@
.. py:currentmodule:: tables
-Changes from 3.2.3.1 to 3.3
+
+Changes from 3.4.1 to 3.4.2
===========================
Improvements
------------
-
-- Single codebase Python 2 and 3 support (PR #493).
-- Internal Blosc version updated to 1.11.1 (closes :issue:`541`)
-- Full BitShuffle support for new Blosc versions (>= 1.8).
-- It is now possible to remove all rows from a table.
-- It is now possible to read reference types by dereferencing them as
- numpy array of objects (closes :issue:`518` and :issue:`519`).
- Thanks to Ehsan Azar
-- Get rid of the `-native` compile flag (closes :issue:`503`)
-- The default number of threads to run numexpr (MAX_NUMEXPR_THREADS)
- internally has been raised from 2 to 4. This is because we are in
- 2016 and 4 core configurations are becoming common.
-- In order to avoid locking issues when using PyTables concurrently in
- several process, MAX_BLOSC_THREADS has been set to 1 by default. If
- you are running PyTables in one single process, you may want to
- experiment if higher values (like 2 or 4) bring better performance for
- you.
+ - setup.py detects conda env and uses installed conda (hdf5, bzip2, lzo
+ and/or blosc) packages when building from source.
Bugs fixed
----------
+ - Linux wheels now built against built-in blosc.
+ - Fixed windows absolute paths in ptrepack, ptdump, ptree.
+ :issue:`616`. Thanks to oscar6echo.
+
-- On python 3 try 'latin1' encoding before 'bytes' encoding during unpickling
- of node attributes pickled on python 2. Better fix for :issue:`560`.
-- Fixed Windows 32 and 64-bit builds.
+Changes from 3.4.0 to 3.4.1
+===========================
+Bugs fixed
+----------
+ - Fixed bug in ptrepack
diff --git a/VERSION b/VERSION
index 15a2799..4d9d11c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.3.0
+3.4.2
diff --git a/appveyor.yml b/appveyor.yml
index 8c2deb6..72621cb 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,6 +1,8 @@
environment:
-
- # Make setup.py include conda specific dll's in wheels
+ PACKAGE_NAME: tables
+ BUILD_DEPENDS: "numpy>=1.8.0 numexpr>=2.5.2 six cython bzip2"
+ TEST_DEPENDS: "numpy>=1.8.0 numexpr>=2.5.2 six mock"
+ # Make setup.py include conda specific dll's in wheel
BUILDWHEEL: True
global:
@@ -18,40 +20,58 @@ environment:
- PYTHON: "C:\\Miniconda-x64"
PYTHON_VERSION: "2.7"
PYTHON_ARCH: "64"
+ HDF5_VERSION: "1.8"
-# - PYTHON: "C:\\Miniconda35"
-# PYTHON_VERSION: "3.5"
-# PYTHON_ARCH: "32"
+# - PYTHON: "C:\\Miniconda-x64"
+# PYTHON_VERSION: "2.7"
+# PYTHON_ARCH: "64"
+# HDF5_VERSION: "1.10"
+
+ - PYTHON: "C:\\Miniconda36-x64"
+ PYTHON_VERSION: "3.6"
+ PYTHON_ARCH: "64"
+ HDF5_VERSION: "1.8"
- - PYTHON: "C:\\Miniconda35-x64"
- PYTHON_VERSION: "3.5"
+ - PYTHON: "C:\\Miniconda36-x64"
+ PYTHON_VERSION: "3.6"
PYTHON_ARCH: "64"
+ HDF5_VERSION: "1.10"
install:
# this installs the appropriate Miniconda (Py2/Py3, 32/64 bit),
# as well as pip, conda-build, and the binstar CLI
- - powershell .\\ci\\appveyor\\install.ps1
+ #- powershell .\\ci\\appveyor\\install.ps1
- powershell .\\ci\\appveyor\\missing-headers.ps1
- - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\Library\\bin;%PATH%"
- - cmd: conda install --yes numexpr cython bzip2
- # Install hdf5=1.8.17 from conda-forge. Anaconda default is 1.8.16.
- - cmd: conda install --yes -c conda-forge hdf5=1.8.17
- - cmd: set HDF5_DIR="%PYTHON%\\Library\\"
- - cmd: set BZIP2_DIR="%PYTHON%\\Library\\"
- - cmd: SET LIBRARY_PATH="%HDF5_DIR%\\lib"
- # Make sure the wheel works:
- # Build wheel and install using pip.
- - "%CMD_IN_ENV% python setup.py bdist_wheel"
- - cmd: dir /B dist\*whl >WHEEL
- - cmd: set /p WHEEL=<WHEEL
- - cmd: pip install dist\%WHEEL%
+ - "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+
+build_script:
+ - conda config --add channels conda-forge
+ - conda create --yes -n build_env python=%PYTHON_VERSION% %BUILD_DEPENDS%
+ - activate build_env
-build: false
+ # Install HDF5 Library
+ # Add tomkooij channel for HDF5 v1.10 conda packages
+ - conda config --add channels http://conda.anaconda.org/tomkooij
+ - conda install --yes hdf5=%HDF5_VERSION%
+
+ # Build wheel
+ - "%CMD_IN_ENV% python setup.py bdist_wheel"
test_script:
- - cmd: cd ..
- - cmd: python -m tables.tests.test_all
+ # create test env
+ - conda create --yes -n test_env python=%PYTHON_VERSION% %TEST_DEPENDS%
+ - activate test_env
+
+ # install from wheel
+ - pip install --no-index --find-links dist/ %PACKAGE_NAME%
+
+ - cd ..
+ - python -m tables.tests.test_all
+
+ - pt2to3 -h
+ - ptrepack -h
+ - ptdump -h
+ - pttree -h
artifacts:
- # download wheels
- - path: dist\*
+ - path: "dist\\*"
diff --git a/c-blosc/.gitignore b/c-blosc/.gitignore
index 1596d01..aa0fc24 100644
--- a/c-blosc/.gitignore
+++ b/c-blosc/.gitignore
@@ -1,2 +1,4 @@
bench/bench
build/
+
+*.sw?
diff --git a/c-blosc/ANNOUNCE.rst b/c-blosc/ANNOUNCE.rst
index 165ae9f..d363afa 100644
--- a/c-blosc/ANNOUNCE.rst
+++ b/c-blosc/ANNOUNCE.rst
@@ -1,26 +1,19 @@
===============================================================
- Announcing C-Blosc 1.11.1
+ Announcing C-Blosc 1.11.3
A blocking, shuffling and lossless compression library for C
===============================================================
What is new?
============
-Fixed a bug introduced in 1.11.0 and discovered by pandas test suite.
-This basically prevented to decompress buffers compressed with previous
-versions of C-Blosc.See:
-https://github.com/Blosc/python-blosc/issues/115
+Fixed an important bug in bitshuffle filter for big endian machines.
+This prevented files written in bigendian machines to be read from
+little endian ones. See issue https://github.com/Blosc/c-blosc/issues/181.
-Also, since support for Zstd has been introduced the experience with it
-has been really pleasant. As an example, see how Blosc + Zstd can
-collaborate compressing images delivering pretty impressive compression
-ratios and extremely fast decompression:
+Also, the internal Zstd codec has been updated to 1.1.3.
-https://github.com/Cyan4973/zstd/issues/256
-
-There is also a blog about what you can expect of it in:
-
-http://blosc.org/blog/zstd-has-just-landed-in-blosc.html
+Finally, the blocksize for compression level 8 has been made 2x larger.
+This should help specially Zstd codec to achieve better compression ratios.
For more info, please see the release notes in:
diff --git a/c-blosc/CMakeLists.txt b/c-blosc/CMakeLists.txt
index d44e0aa..93dbcb8 100644
--- a/c-blosc/CMakeLists.txt
+++ b/c-blosc/CMakeLists.txt
@@ -5,6 +5,8 @@
#
# BUILD_STATIC: default ON
# build the static version of the Blosc library
+# BUILD_SHARED: default ON
+# build the shared library version of the Blosc library
# BUILD_TESTS: default ON
# build test programs and generates the "test" target
# BUILD_BENCHMARKS: default ON
@@ -66,6 +68,9 @@
cmake_minimum_required(VERSION 2.8.10)
+if (NOT CMAKE_VERSION VERSION_LESS 3.3)
+ cmake_policy(SET CMP0063 NEW)
+endif()
project(blosc)
# parse the full version numbers from blosc.h
@@ -84,6 +89,8 @@ message("Configuring for Blosc version: " ${BLOSC_VERSION_STRING})
# options
option(BUILD_STATIC
"Build a static version of the blosc library." ON)
+option(BUILD_SHARED
+ "Build a shared library version of the blosc library." ON)
option(BUILD_TESTS
"Build test programs form the blosc compression library" ON)
option(BUILD_BENCHMARKS
@@ -107,7 +114,7 @@ option(PREFER_EXTERNAL_ZLIB
option(PREFER_EXTERNAL_ZSTD
"Find and use external Zstd library instead of included sources." ON)
-set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
if(NOT DEACTIVATE_LZ4)
@@ -195,7 +202,7 @@ if(CMAKE_SYSTEM_PROCESSOR STREQUAL i386 OR
CMAKE_SYSTEM_PROCESSOR STREQUAL AMD64)
if(CMAKE_C_COMPILER_ID STREQUAL GNU)
# We need C99 (GNU99 more exactly)
- SET(CMAKE_C_FLAGS "-std=gnu99")
+ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
set(COMPILER_SUPPORT_SSE2 TRUE)
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 4.7 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.7)
set(COMPILER_SUPPORT_AVX2 TRUE)
@@ -272,6 +279,23 @@ if(WIN32)
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/blosc")
endif(WIN32)
+if (NOT DEFINED BLOSC_IS_SUBPROJECT)
+ if ("^${CMAKE_SOURCE_DIR}$" STREQUAL "^${PROJECT_SOURCE_DIR}$")
+ set (BLOSC_IS_SUBPROJECT FALSE)
+ else ()
+ set (BLOSC_IS_SUBPROJECT TRUE)
+ message(STATUS "Detected that BLOSC is used a subproject.")
+ endif ()
+endif ()
+
+if (NOT DEFINED BLOSC_INSTALL)
+ if (BLOSC_IS_SUBPROJECT)
+ set(BLOSC_INSTALL FALSE)
+ else()
+ set(BLOSC_INSTALL TRUE)
+ endif()
+endif()
+
# subdirectories
add_subdirectory(blosc)
@@ -287,30 +311,39 @@ endif(BUILD_BENCHMARKS)
# uninstall target
-configure_file(
- "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
- "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
- IMMEDIATE @ONLY)
-
-add_custom_target(uninstall
- COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
-
+if (BLOSC_INSTALL)
+ configure_file(
+ "${CMAKE_CURRENT_SOURCE_DIR}/blosc.pc.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/blosc.pc"
+ @ONLY)
+ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/blosc.pc"
+ DESTINATION lib/pkgconfig COMPONENT DEV)
+
+ configure_file(
+ "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
+ "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
+ IMMEDIATE @ONLY)
+ add_custom_target(uninstall
+ COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
+endif()
# packaging
-include(InstallRequiredSystemLibraries)
-
-set(CPACK_GENERATOR TGZ ZIP)
-set(CPACK_SOURCE_GENERATOR TGZ ZIP)
-set(CPACK_PACKAGE_VERSION_MAJOR ${BLOSC_VERSION_MAJOR})
-set(CPACK_PACKAGE_VERSION_MINOR ${BLOSC_VERSION_MINOR})
-set(CPACK_PACKAGE_VERSION_PATCH ${BLOSC_VERSION_PATCH})
-set(CPACK_PACKAGE_VERSION ${BLOSC_STRING_VERSION})
-set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.rst")
-set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
- "A blocking, shuffling and lossless compression library")
-set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSES/BLOSC.txt")
-set(CPACK_SOURCE_IGNORE_FILES "/build.*;.*~;\\\\.git.*;\\\\.DS_Store")
-set(CPACK_STRIP_FILES TRUE)
-set(CPACK_SOURCE_STRIP_FILES TRUE)
-
-include(CPack)
+if (NOT BLOSC_IS_SUBPROJECT)
+ include(InstallRequiredSystemLibraries)
+
+ set(CPACK_GENERATOR TGZ ZIP)
+ set(CPACK_SOURCE_GENERATOR TGZ ZIP)
+ set(CPACK_PACKAGE_VERSION_MAJOR ${BLOSC_VERSION_MAJOR})
+ set(CPACK_PACKAGE_VERSION_MINOR ${BLOSC_VERSION_MINOR})
+ set(CPACK_PACKAGE_VERSION_PATCH ${BLOSC_VERSION_PATCH})
+ set(CPACK_PACKAGE_VERSION ${BLOSC_STRING_VERSION})
+ set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.rst")
+ set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
+ "A blocking, shuffling and lossless compression library")
+ set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSES/BLOSC.txt")
+ set(CPACK_SOURCE_IGNORE_FILES "/build.*;.*~;\\\\.git.*;\\\\.DS_Store")
+ set(CPACK_STRIP_FILES TRUE)
+ set(CPACK_SOURCE_STRIP_FILES TRUE)
+
+ include(CPack)
+endif()
diff --git a/c-blosc/README.rst b/c-blosc/README.rst
index 163e07a..87eb71a 100644
--- a/c-blosc/README.rst
+++ b/c-blosc/README.rst
@@ -51,7 +51,7 @@ compressors integrated in Blosc in all supported platforms.
You can see some benchmarks about Blosc performance in [3]_
-Blosc is distributed using the MIT license, see LICENSES/BLOSC.txt for
+Blosc is distributed using the BSD license, see LICENSES/BLOSC.txt for
details.
.. [1] http://www.blosc.org
diff --git a/c-blosc/RELEASE_NOTES.rst b/c-blosc/RELEASE_NOTES.rst
index e21bfa2..a685ec8 100644
--- a/c-blosc/RELEASE_NOTES.rst
+++ b/c-blosc/RELEASE_NOTES.rst
@@ -7,6 +7,29 @@
:URL: http://www.blosc.org
+Changes from 1.11.2 to 1.11.3
+=============================
+
+- Fixed #181: bitshuffle filter for big endian machines.
+
+- Internal Zstd codec updated to 1.1.3.
+
+- New blocksize for complevel 8 in automatic mode. This should help specially
+ the Zstd codec to achieve better compression ratios.
+
+
+Changes from 1.11.1 to 1.11.2
+=============================
+
+- Enabled use as a CMake subproject, exporting shared & static library targets
+ for super-projects to use. See PRs #178, #179 and #180. Thanks to Kevin
+ Murray.
+
+- Internal LZ4 codec updated to 1.7.5.
+
+- Internal Zstd codec updated to 1.1.2.
+
+
Changes from 1.11.0 to 1.11.1
=============================
diff --git a/c-blosc/bench/bench.c b/c-blosc/bench/bench.c
index 8932c2e..9852313 100644
--- a/c-blosc/bench/bench.c
+++ b/c-blosc/bench/bench.c
@@ -17,7 +17,7 @@
Note: Compiling this with VS2008 does not work well with cmake. Here
it is a way to compile the benchmark (with added support for LZ4):
- > cl /DHAVE_LZ4 /arch:SSE2 /Ox /Febench.exe /Iblosc /Iinternal-complibs\lz4-1.7.2 bench\bench.c blosc\blosc.c blosc\blosclz.c blosc\shuffle.c blosc\shuffle-sse2.c blosc\shuffle-generic.c blosc\bitshuffle-generic.c blosc\bitshuffle-sse2.c internal-complibs\lz4-1.7.2\*.c
+ > cl /DHAVE_LZ4 /arch:SSE2 /Ox /Febench.exe /Iblosc /Iinternal-complibs\lz4-1.7.5 bench\bench.c blosc\blosc.c blosc\blosclz.c blosc\shuffle.c blosc\shuffle-sse2.c blosc\shuffle-generic.c blosc\bitshuffle-generic.c blosc\bitshuffle-sse2.c internal-complibs\lz4-1.7.5\*.c
See LICENSES/BLOSC.txt for details about copyright and rights to use.
**********************************************************************/
@@ -31,14 +31,14 @@
#if defined(_WIN32)
/* For QueryPerformanceCounter(), etc. */
#include <windows.h>
-#elif defined(__MACH__)
+#elif defined(__MACH__) && defined(__APPLE__)
#include <mach/clock.h>
#include <mach/mach.h>
#include <time.h>
#include <sys/time.h>
#elif defined(__unix__)
#include <unistd.h>
- #if defined(__linux__)
+ #if defined(__GLIBC__)
#include <time.h>
#else
#include <sys/time.h>
@@ -89,7 +89,7 @@ double blosc_elapsed_usecs(blosc_timestamp_t start_time, blosc_timestamp_t end_t
/* Set a timestamp value to the current time. */
void blosc_set_timestamp(blosc_timestamp_t* timestamp) {
-#ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time
+#if defined(__MACH__) && defined(__APPLE__) // OS X does not have clock_gettime, use clock_get_time
clock_serv_t cclock;
mach_timespec_t mts;
host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
diff --git a/c-blosc/blosc.pc.in b/c-blosc/blosc.pc.in
new file mode 100644
index 0000000..c01273b
--- /dev/null
+++ b/c-blosc/blosc.pc.in
@@ -0,0 +1,14 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=${exec_prefix}/lib
+sharedlibdir=${libdir}
+includedir=${prefix}/include
+
+Name: blosc
+Description: A blocking, shuffling and lossless compression library
+URL: http://blosc.org/
+Version: @BLOSC_VERSION_STRING@
+
+Requires:
+Libs: -L${libdir} -L${sharedlibdir} -lblosc
+Cflags: -I${includedir}
diff --git a/c-blosc/blosc/CMakeLists.txt b/c-blosc/blosc/CMakeLists.txt
index 2691b99..5979bc2 100644
--- a/c-blosc/blosc/CMakeLists.txt
+++ b/c-blosc/blosc/CMakeLists.txt
@@ -1,7 +1,7 @@
# a simple way to detect that we are using CMAKE
add_definitions(-DUSING_CMAKE)
-set(INTERNAL_LIBS ${CMAKE_SOURCE_DIR}/internal-complibs)
+set(INTERNAL_LIBS ${PROJECT_SOURCE_DIR}/internal-complibs)
# Hide symbols by default unless they're specifically exported.
# This makes it easier to keep the set of exported symbols the
@@ -9,42 +9,45 @@ set(INTERNAL_LIBS ${CMAKE_SOURCE_DIR}/internal-complibs)
set(CMAKE_C_VISIBILITY_PRESET hidden)
# includes
+set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR})
if(NOT DEACTIVATE_LZ4)
if (LZ4_FOUND)
- include_directories( ${LZ4_INCLUDE_DIR} )
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR})
else(LZ4_FOUND)
- set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.7.2)
- include_directories( ${LZ4_LOCAL_DIR} )
+ set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.7.5)
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_LOCAL_DIR})
endif(LZ4_FOUND)
endif(NOT DEACTIVATE_LZ4)
if(NOT DEACTIVATE_SNAPPY)
if (SNAPPY_FOUND)
- include_directories( ${SNAPPY_INCLUDE_DIR} )
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIR})
else(SNAPPY_FOUND)
set(SNAPPY_LOCAL_DIR ${INTERNAL_LIBS}/snappy-1.1.1)
- include_directories( ${SNAPPY_LOCAL_DIR} )
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${SNAPPY_LOCAL_DIR})
endif(SNAPPY_FOUND)
endif(NOT DEACTIVATE_SNAPPY)
if(NOT DEACTIVATE_ZLIB)
if (ZLIB_FOUND)
- include_directories( ${ZLIB_INCLUDE_DIR} )
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR})
else(ZLIB_FOUND)
set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.2.8)
- include_directories( ${ZLIB_LOCAL_DIR} )
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_LOCAL_DIR})
endif(ZLIB_FOUND)
endif(NOT DEACTIVATE_ZLIB)
if (NOT DEACTIVATE_ZSTD)
if (ZSTD_FOUND)
- include_directories(${ZSTD_INCLUDE_DIR})
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR})
else (ZSTD_FOUND)
- set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.0.0)
- include_directories(${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common)
+ set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.1.3)
+ set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common)
endif (ZSTD_FOUND)
endif (NOT DEACTIVATE_ZSTD)
+include_directories(${BLOSC_INCLUDE_DIRS})
+
# library sources
set(SOURCES blosc.c blosclz.c shuffle-generic.c bitshuffle-generic.c)
if(COMPILER_SUPPORT_SSE2)
@@ -115,16 +118,19 @@ if (NOT DEACTIVATE_ZSTD)
endif (ZSTD_FOUND)
endif (NOT DEACTIVATE_ZSTD)
+
# targets
-add_library(blosc_shared SHARED ${SOURCES})
-set_target_properties(blosc_shared PROPERTIES OUTPUT_NAME blosc)
-set_target_properties(blosc_shared PROPERTIES
- VERSION ${version_string}
- SOVERSION 1 # Change this when an ABI change happens
- )
-set_property(
- TARGET blosc_shared
- APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY)
+if (BUILD_SHARED)
+ add_library(blosc_shared SHARED ${SOURCES})
+ set_target_properties(blosc_shared PROPERTIES OUTPUT_NAME blosc)
+ set_target_properties(blosc_shared PROPERTIES
+ VERSION ${version_string}
+ SOVERSION 1 # Change this when an ABI change happens
+ )
+ set_property(
+ TARGET blosc_shared
+ APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY)
+endif()
# Based on the target architecture and hardware features supported
# by the C compiler, set hardware architecture optimization flags
@@ -182,9 +188,14 @@ if (BUILD_TESTS)
endif()
endif()
-target_link_libraries(blosc_shared ${LIBS})
+if (BUILD_SHARED)
+ target_link_libraries(blosc_shared ${LIBS})
+ target_include_directories(blosc_shared PUBLIC ${BLOSC_INCLUDE_DIRS})
+endif()
+
if (BUILD_TESTS)
target_link_libraries(blosc_shared_testing ${LIBS})
+ target_include_directories(blosc_shared_testing PUBLIC ${BLOSC_INCLUDE_DIRS})
endif()
if(BUILD_STATIC)
@@ -194,12 +205,14 @@ if(BUILD_STATIC)
set_target_properties(blosc_static PROPERTIES PREFIX lib)
endif()
target_link_libraries(blosc_static ${LIBS})
+ target_include_directories(blosc_static PUBLIC ${BLOSC_INCLUDE_DIRS})
endif(BUILD_STATIC)
-
# install
-install(FILES blosc.h blosc-export.h DESTINATION include COMPONENT DEV)
-install(TARGETS blosc_shared DESTINATION ${lib_dir} COMPONENT LIB)
-if(BUILD_STATIC)
- install(TARGETS blosc_static DESTINATION ${lib_dir} COMPONENT DEV)
-endif(BUILD_STATIC)
+if(BLOSC_INSTALL)
+ install(FILES blosc.h blosc-export.h DESTINATION include COMPONENT DEV)
+ install(TARGETS blosc_shared DESTINATION ${lib_dir} COMPONENT LIB)
+ if(BUILD_STATIC)
+ install(TARGETS blosc_static DESTINATION ${lib_dir} COMPONENT DEV)
+ endif(BUILD_STATIC)
+endif(BLOSC_INSTALL)
diff --git a/c-blosc/blosc/bitshuffle-generic.c b/c-blosc/blosc/bitshuffle-generic.c
index 589803f..c7b3b47 100644
--- a/c-blosc/blosc/bitshuffle-generic.c
+++ b/c-blosc/blosc/bitshuffle-generic.c
@@ -10,7 +10,7 @@
/* Transpose bytes within elements, starting partway through input. */
-int64_t bshuf_trans_byte_elem_remainder(void* in, void* out, const size_t size,
+int64_t bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size,
const size_t elem_size, const size_t start) {
char* in_b = (char*) in;
@@ -41,7 +41,7 @@ int64_t bshuf_trans_byte_elem_remainder(void* in, void* out, const size_t size,
/* Transpose bytes within elements. */
-int64_t bshuf_trans_byte_elem_scal(void* in, void* out, const size_t size,
+int64_t bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size,
const size_t elem_size) {
return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0);
@@ -49,27 +49,35 @@ int64_t bshuf_trans_byte_elem_scal(void* in, void* out, const size_t size,
/* Transpose bits within bytes. */
-int64_t bshuf_trans_bit_byte_remainder(void* in, void* out, const size_t size,
+int64_t bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size,
const size_t elem_size, const size_t start_byte) {
- int64_t* in_b = in;
- int8_t* out_b = out;
+ const uint64_t* in_b = (const uint64_t*) in;
+ uint8_t* out_b = (uint8_t*) out;
- int64_t x, t;
+ uint64_t x, t;
+ size_t ii, kk;
size_t nbyte = elem_size * size;
size_t nbyte_bitrow = nbyte / 8;
- size_t ii;
- int kk;
+
+ uint64_t e=1;
+ const int little_endian = *(uint8_t *) &e == 1;
+ const size_t bit_row_skip = little_endian ? nbyte_bitrow : -nbyte_bitrow;
+ const int64_t bit_row_offset = little_endian ? 0 : 7 * nbyte_bitrow;
CHECK_MULT_EIGHT(nbyte);
CHECK_MULT_EIGHT(start_byte);
for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) {
x = in_b[ii];
- TRANS_BIT_8X8(x, t);
+ if (little_endian) {
+ TRANS_BIT_8X8(x, t);
+ } else {
+ TRANS_BIT_8X8_BE(x, t);
+ }
for (kk = 0; kk < 8; kk ++) {
- out_b[kk * nbyte_bitrow + ii] = x;
+ out_b[bit_row_offset + kk * bit_row_skip + ii] = x;
x = x >> 8;
}
}
@@ -78,7 +86,7 @@ int64_t bshuf_trans_bit_byte_remainder(void* in, void* out, const size_t size,
/* Transpose bits within bytes. */
-int64_t bshuf_trans_bit_byte_scal(void* in, void* out, const size_t size,
+int64_t bshuf_trans_bit_byte_scal(const void* in, void* out, const size_t size,
const size_t elem_size) {
return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0);
@@ -86,7 +94,7 @@ int64_t bshuf_trans_bit_byte_scal(void* in, void* out, const size_t size,
/* General transpose of an array, optimized for large element sizes. */
-int64_t bshuf_trans_elem(void* in, void* out, const size_t lda,
+int64_t bshuf_trans_elem(const void* in, void* out, const size_t lda,
const size_t ldb, const size_t elem_size) {
char* in_b = (char*) in;
@@ -103,7 +111,7 @@ int64_t bshuf_trans_elem(void* in, void* out, const size_t lda,
/* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */
-int64_t bshuf_trans_bitrow_eight(void* in, void* out, const size_t size,
+int64_t bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size,
const size_t elem_size) {
size_t nbyte_bitrow = size / 8;
@@ -115,7 +123,7 @@ int64_t bshuf_trans_bitrow_eight(void* in, void* out, const size_t size,
/* Transpose bits within elements. */
-int64_t bshuf_trans_bit_elem_scal(void* in, void* out, const size_t size,
+int64_t bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size,
const size_t elem_size, void* tmp_buf) {
int64_t count;
@@ -134,7 +142,7 @@ int64_t bshuf_trans_bit_elem_scal(void* in, void* out, const size_t size,
/* For data organized into a row for each bit (8 * elem_size rows), transpose
* the bytes. */
-int64_t bshuf_trans_byte_bitrow_scal(void* in, void* out, const size_t size,
+int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size,
const size_t elem_size) {
char* in_b = (char*) in;
char* out_b = (char*) out;
@@ -157,22 +165,38 @@ int64_t bshuf_trans_byte_bitrow_scal(void* in, void* out, const size_t size,
/* Shuffle bits within the bytes of eight element blocks. */
-int64_t bshuf_shuffle_bit_eightelem_scal(void* in, void* out,
+int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \
const size_t size, const size_t elem_size) {
- char* in_b = (char*) in;
- char* out_b = (char*) out;
- size_t nbyte = elem_size * size;
- int64_t x, t;
- size_t jj, ii, kk;
+
+ const char *in_b;
+ char *out_b;
+ uint64_t x, t;
+ size_t ii, jj, kk;
+ size_t nbyte, out_index;
+
+ uint64_t e=1;
+ const int little_endian = *(uint8_t *) &e == 1;
+ const size_t elem_skip = little_endian ? elem_size : -elem_size;
+ const uint64_t elem_offset = little_endian ? 0 : 7 * elem_size;
CHECK_MULT_EIGHT(size);
+ in_b = (const char*) in;
+ out_b = (char*) out;
+
+ nbyte = elem_size * size;
+
for (jj = 0; jj < 8 * elem_size; jj += 8) {
for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) {
- x = *((int64_t*) &in_b[ii + jj]);
- TRANS_BIT_8X8(x, t);
+ x = *((uint64_t*) &in_b[ii + jj]);
+ if (little_endian) {
+ TRANS_BIT_8X8(x, t);
+ } else {
+ TRANS_BIT_8X8_BE(x, t);
+ }
for (kk = 0; kk < 8; kk++) {
- *((uint8_t*) &out_b[ii + jj / 8 + kk * elem_size]) = x;
+ out_index = ii + jj / 8 + elem_offset + kk * elem_skip;
+ *((uint8_t*) &out_b[out_index]) = x;
x = x >> 8;
}
}
@@ -182,7 +206,7 @@ int64_t bshuf_shuffle_bit_eightelem_scal(void* in, void* out,
/* Untranspose bits within elements. */
-int64_t bshuf_untrans_bit_elem_scal(void* in, void* out, const size_t size,
+int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size,
const size_t elem_size, void* tmp_buf) {
int64_t count;
diff --git a/c-blosc/blosc/bitshuffle-generic.h b/c-blosc/blosc/bitshuffle-generic.h
index 03b3f55..bc76605 100644
--- a/c-blosc/blosc/bitshuffle-generic.h
+++ b/c-blosc/blosc/bitshuffle-generic.h
@@ -48,6 +48,16 @@ extern "C" {
x = x ^ t ^ (t << 28); \
}
+/* Transpose 8x8 bit array along the diagonal from upper right
+ to lower left */
+#define TRANS_BIT_8X8_BE(x, t) { \
+ t = (x ^ (x >> 9)) & 0x0055005500550055LL; \
+ x = x ^ t ^ (t << 9); \
+ t = (x ^ (x >> 18)) & 0x0000333300003333LL; \
+ x = x ^ t ^ (t << 18); \
+ t = (x ^ (x >> 36)) & 0x000000000F0F0F0FLL; \
+ x = x ^ t ^ (t << 36); \
+ }
/* Transpose of an array of arbitrarily typed elements. */
#define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \
@@ -72,27 +82,27 @@ extern "C" {
/* Private functions */
BLOSC_NO_EXPORT int64_t
-bshuf_trans_byte_elem_remainder(void* in, void* out, const size_t size,
+bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size,
const size_t elem_size, const size_t start);
BLOSC_NO_EXPORT int64_t
-bshuf_trans_byte_elem_scal(void* in, void* out, const size_t size,
+bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size,
const size_t elem_size);
BLOSC_NO_EXPORT int64_t
-bshuf_trans_bit_byte_remainder(void* in, void* out, const size_t size,
+bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size,
const size_t elem_size, const size_t start_byte);
BLOSC_NO_EXPORT int64_t
-bshuf_trans_elem(void* in, void* out, const size_t lda,
+bshuf_trans_elem(const void* in, void* out, const size_t lda,
const size_t ldb, const size_t elem_size);
BLOSC_NO_EXPORT int64_t
-bshuf_trans_bitrow_eight(void* in, void* out, const size_t size,
+bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size,
const size_t elem_size);
BLOSC_NO_EXPORT int64_t
-bshuf_shuffle_bit_eightelem_scal(void* in, void* out,
+bshuf_shuffle_bit_eightelem_scal(const void* in, void* out,
const size_t size, const size_t elem_size);
@@ -115,7 +125,7 @@ bshuf_shuffle_bit_eightelem_scal(void* in, void* out,
*/
BLOSC_NO_EXPORT int64_t
-bshuf_trans_bit_elem_scal(void* in, void* out, const size_t size,
+bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size,
const size_t elem_size, void* tmp_buf);
/* Unshuffle bitshuffled data.
@@ -140,7 +150,7 @@ bshuf_trans_bit_elem_scal(void* in, void* out, const size_t size,
*/
BLOSC_NO_EXPORT int64_t
-bshuf_untrans_bit_elem_scal(void* in, void* out, const size_t size,
+bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size,
const size_t elem_size, void* tmp_buf);
diff --git a/c-blosc/blosc/blosc.c b/c-blosc/blosc/blosc.c
index 001a160..d858c51 100644
--- a/c-blosc/blosc/blosc.c
+++ b/c-blosc/blosc/blosc.c
@@ -419,9 +419,10 @@ static int lz4hc_wrap_compress(const char* input, size_t input_length,
int cbytes;
if (input_length > (size_t)(2<<30))
return -1; /* input larger than 1 GB is not supported */
- /* clevel for lz4hc goes up to 16, at least in LZ4 1.1.3 */
- cbytes = LZ4_compressHC2_limitedOutput(input, output, (int)input_length,
- (int)maxout, clevel*2-1);
+ /* clevel for lz4hc goes up to 12, at least in LZ4 1.7.5
+ * but levels larger than 9 does not buy much compression. */
+ cbytes = LZ4_compress_HC(input, output, (int)input_length, (int)maxout,
+ clevel);
return cbytes;
}
@@ -499,6 +500,8 @@ static int zstd_wrap_compress(const char* input, size_t input_length,
char* output, size_t maxout, int clevel) {
size_t code;
clevel = (clevel < 9) ? clevel * 2 - 1 : ZSTD_maxCLevel();
+ /* Make the level 8 close enough to maxCLevel */
+ if (clevel == 8) clevel = ZSTD_maxCLevel() - 2;
code = ZSTD_compress(
(void*)output, maxout, (void*)input, input_length, clevel);
if (ZSTD_isError(code)) {
@@ -945,9 +948,12 @@ static int32_t compute_blocksize(struct blosc_context* context, int32_t clevel,
else if (clevel <= 6) {
blocksize *= 2;
}
- else if (clevel < 9) {
+ else if (clevel == 7) {
blocksize *= 4;
}
+ else if (clevel == 8) {
+ blocksize *= 8;
+ }
else {
blocksize *= 16;
}
diff --git a/c-blosc/blosc/blosc.h b/c-blosc/blosc/blosc.h
index d27cd87..e855969 100644
--- a/c-blosc/blosc/blosc.h
+++ b/c-blosc/blosc/blosc.h
@@ -19,11 +19,11 @@ extern "C" {
/* Version numbers */
#define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */
#define BLOSC_VERSION_MINOR 11 /* for minor interface/format changes */
-#define BLOSC_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
+#define BLOSC_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
-#define BLOSC_VERSION_STRING "1.11.1" /* string version. Sync with above! */
+#define BLOSC_VERSION_STRING "1.11.3" /* string version. Sync with above! */
#define BLOSC_VERSION_REVISION "$Rev$" /* revision version */
-#define BLOSC_VERSION_DATE "$Date:: 2016-09-03 #$" /* date version */
+#define BLOSC_VERSION_DATE "$Date:: 2017-03-09 #$" /* date version */
#define BLOSCLZ_VERSION_STRING "1.0.5" /* the internal compressor version */
diff --git a/c-blosc/internal-complibs/lz4-1.7.2/lz4.h b/c-blosc/internal-complibs/lz4-1.7.2/lz4.h
deleted file mode 100644
index 96e25a6..0000000
--- a/c-blosc/internal-complibs/lz4-1.7.2/lz4.h
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- LZ4 - Fast LZ compression algorithm
- Header File
- Copyright (C) 2011-2015, Yann Collet.
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 source repository : https://github.com/Cyan4973/lz4
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/*
- * lz4.h provides block compression functions, and gives full buffer control to programmer.
- * If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
- * and can let the library handle its own memory, please use lz4frame.h instead.
-*/
-
-/**************************************
-* Version
-**************************************/
-#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
-#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
-#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
-int LZ4_versionNumber (void);
-
-/**************************************
-* Tuning parameter
-**************************************/
-/*
- * LZ4_MEMORY_USAGE :
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio
- * Reduced memory usage can improve speed, due to cache effect
- * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
- */
-#define LZ4_MEMORY_USAGE 14
-
-
-/**************************************
-* Simple Functions
-**************************************/
-
-int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
-int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-
-/*
-LZ4_compress_default() :
- Compresses 'sourceSize' bytes from buffer 'source'
- into already allocated 'dest' buffer of size 'maxDestSize'.
- Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
- It also runs faster, so it's a recommended setting.
- If the function cannot compress 'source' into a more limited 'dest' budget,
- compression stops *immediately*, and the function result is zero.
- As a consequence, 'dest' content is not valid.
- This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
- sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
- maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
- return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
- or 0 if compression fails
-
-LZ4_decompress_safe() :
- compressedSize : is the precise full size of the compressed block.
- maxDecompressedSize : is the size of destination buffer, which must be already allocated.
- return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
- If destination buffer is not large enough, decoding will stop and output an error code (<0).
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function is protected against buffer overflow exploits, including malicious data packets.
- It never writes outside output buffer, nor reads outside input buffer.
-*/
-
-
-/**************************************
-* Advanced Functions
-**************************************/
-#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
-#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
-
-/*
-LZ4_compressBound() :
- Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
- This function is primarily useful for memory allocation purposes (destination buffer size).
- Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
- Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
- inputSize : max supported value is LZ4_MAX_INPUT_SIZE
- return : maximum output size in a "worst case" scenario
- or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
-*/
-int LZ4_compressBound(int inputSize);
-
-/*
-LZ4_compress_fast() :
- Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
- The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
- It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
- An acceleration value of "1" is the same as regular LZ4_compress_default()
- Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
-*/
-int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
-
-
-/*
-LZ4_compress_fast_extState() :
- Same compression function, just using an externally allocated memory space to store compression state.
- Use LZ4_sizeofState() to know how much memory must be allocated,
- and allocate it on 8-bytes boundaries (using malloc() typically).
- Then, provide it as 'void* state' to compression function.
-*/
-int LZ4_sizeofState(void);
-int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
-
-
-/*
-LZ4_compress_destSize() :
- Reverse the logic, by compressing as much data as possible from 'source' buffer
- into already allocated buffer 'dest' of size 'targetDestSize'.
- This function either compresses the entire 'source' content into 'dest' if it's large enough,
- or fill 'dest' buffer completely with as much data as possible from 'source'.
- *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
- New value is necessarily <= old value.
- return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
- or 0 if compression fails
-*/
-int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
-
-
-/*
-LZ4_decompress_fast() :
- originalSize : is the original and therefore uncompressed size
- return : the number of bytes read from the source buffer (in other words, the compressed size)
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
- note : This function fully respect memory boundaries for properly formed compressed data.
- It is a bit faster than LZ4_decompress_safe().
- However, it does not provide any protection against intentionally modified data stream (malicious input).
- Use this function in trusted environment only (data to decode comes from a trusted source).
-*/
-int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
-
-/*
-LZ4_decompress_safe_partial() :
- This function decompress a compressed block of size 'compressedSize' at position 'source'
- into destination buffer 'dest' of size 'maxDecompressedSize'.
- The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
- reducing decompression time.
- return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
- Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
- Always control how many bytes were decoded.
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
-*/
-int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
-
-
-/***********************************************
-* Streaming Compression Functions
-***********************************************/
-#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
-#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long))
-/*
- * LZ4_stream_t
- * information structure to track an LZ4 stream.
- * important : init this structure content before first use !
- * note : only allocated directly the structure if you are statically linking LZ4
- * If you are using liblz4 as a DLL, please use below construction methods instead.
- */
-typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
-
-/*
- * LZ4_resetStream
- * Use this function to init an allocated LZ4_stream_t structure
- */
-void LZ4_resetStream (LZ4_stream_t* streamPtr);
-
-/*
- * LZ4_createStream will allocate and initialize an LZ4_stream_t structure
- * LZ4_freeStream releases its memory.
- * In the context of a DLL (liblz4), please use these methods rather than the static struct.
- * They are more future proof, in case of a change of LZ4_stream_t size.
- */
-LZ4_stream_t* LZ4_createStream(void);
-int LZ4_freeStream (LZ4_stream_t* streamPtr);
-
-/*
- * LZ4_loadDict
- * Use this function to load a static dictionary into LZ4_stream.
- * Any previous data will be forgotten, only 'dictionary' will remain in memory.
- * Loading a size of 0 is allowed.
- * Return : dictionary size, in bytes (necessarily <= 64 KB)
- */
-int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
-
-/*
- * LZ4_compress_fast_continue
- * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
- * Important : Previous data blocks are assumed to still be present and unmodified !
- * 'dst' buffer must be already allocated.
- * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
- */
-int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
-
-/*
- * LZ4_saveDict
- * If previously compressed data block is not guaranteed to remain available at its memory location
- * save it into a safer place (char* safeBuffer)
- * Note : you don't need to call LZ4_loadDict() afterwards,
- * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
- * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
- */
-int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
-
-
-/************************************************
-* Streaming Decompression Functions
-************************************************/
-
-#define LZ4_STREAMDECODESIZE_U64 4
-#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
-typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t;
-/*
- * LZ4_streamDecode_t
- * information structure to track an LZ4 stream.
- * init this structure content using LZ4_setStreamDecode or memset() before first use !
- *
- * In the context of a DLL (liblz4) please prefer usage of construction methods below.
- * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
- * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
- * LZ4_freeStreamDecode releases its memory.
- */
-LZ4_streamDecode_t* LZ4_createStreamDecode(void);
-int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-
-/*
- * LZ4_setStreamDecode
- * Use this function to instruct where to find the dictionary.
- * Setting a size of 0 is allowed (same effect as reset).
- * Return : 1 if OK, 0 if error
- */
-int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-
-/*
-*_continue() :
- These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- In the case of a ring buffers, decoding buffer must be either :
- - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including small ones ( < 64 KB).
- - _At least_ 64 KB + 8 bytes + maxBlockSize.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including larger than decoding buffer.
- Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- and indicate where it is saved using LZ4_setStreamDecode()
-*/
-int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-
-
-/*
-Advanced decoding functions :
-*_usingDict() :
- These decoding functions work the same as
- a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
- They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
-*/
-int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
-int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
-
-
-/**************************************
-* Obsolete Functions
-**************************************/
-/* Deprecate Warnings */
-/* Should these warnings messages be a problem,
- it is generally possible to disable them,
- with -Wno-deprecated-declarations for gcc
- or _CRT_SECURE_NO_WARNINGS in Visual for example.
- Otherwise, you can also define LZ4_DISABLE_DEPRECATE_WARNINGS */
-#define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
-# define LZ4_DEPRECATED() /* disable deprecation warnings */
-#else
-# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
-# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-# elif (LZ4_GCC_VERSION >= 301)
-# define LZ4_DEPRECATED(message) __attribute__((deprecated))
-# elif defined(_MSC_VER)
-# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
-# else
-# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
-# define LZ4_DEPRECATED(message)
-# endif
-#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
-
-/* Obsolete compression functions */
-/* These functions will generate warnings in a future release */
-int LZ4_compress (const char* source, char* dest, int sourceSize);
-int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
-int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/* Obsolete decompression functions */
-/* These function names are completely deprecated and must no longer be used.
- They are only provided in lz4.c for compatibility with older programs.
- - LZ4_uncompress is the same as LZ4_decompress_fast
- - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
- These function prototypes are now disabled; uncomment them only if you really need them.
- It is highly recommended to stop using these prototypes and migrate to maintained ones */
-/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
-/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
-
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
-
-/* Obsolete streaming decoding functions */
-LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
-
-
-#if defined (__cplusplus)
-}
-#endif
diff --git a/c-blosc/internal-complibs/lz4-1.7.2/lz4hc.h b/c-blosc/internal-complibs/lz4-1.7.2/lz4hc.h
deleted file mode 100644
index 431f7c8..0000000
--- a/c-blosc/internal-complibs/lz4-1.7.2/lz4hc.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- LZ4 HC - High Compression Mode of LZ4
- Header File
- Copyright (C) 2011-2015, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 source repository : https://github.com/Cyan4973/lz4
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
-
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/*****************************
-* Includes
-*****************************/
-#include <stddef.h> /* size_t */
-
-
-/**************************************
-* Block Compression
-**************************************/
-int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
-/*
-LZ4_compress_HC :
- Destination buffer 'dst' must be already allocated.
- Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible)
- Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
- srcSize : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
- compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work.
- 0 means "use default value" (see lz4hc.c).
- Values >16 behave the same as 16.
- return : the number of bytes written into buffer 'dst'
- or 0 if compression fails.
-*/
-
-
-/* Note :
- Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
-*/
-
-
-int LZ4_sizeofStateHC(void);
-int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
-/*
-LZ4_compress_HC_extStateHC() :
- Use this function if you prefer to manually allocate memory for compression tables.
- To know how much memory must be allocated for the compression tables, use :
- int LZ4_sizeofStateHC();
-
- Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
-
- The allocated memory can then be provided to the compression functions using 'void* state' parameter.
- LZ4_compress_HC_extStateHC() is equivalent to previously described function.
- It just uses externally allocated memory for stateHC.
-*/
-
-
-/**************************************
-* Streaming Compression
-**************************************/
-#define LZ4_STREAMHCSIZE 262192
-#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
-typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t;
-/*
- LZ4_streamHC_t
- This structure allows static allocation of LZ4 HC streaming state.
- State must then be initialized using LZ4_resetStreamHC() before first use.
-
- Static allocation should only be used in combination with static linking.
- If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof.
-*/
-
-
-LZ4_streamHC_t* LZ4_createStreamHC(void);
-int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
-/*
- These functions create and release memory for LZ4 HC streaming state.
- Newly created states are already initialized.
- Existing state space can be re-used anytime using LZ4_resetStreamHC().
- If you use LZ4 as a DLL, use these functions instead of static structure allocation,
- to avoid size mismatch between different versions.
-*/
-
-void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
-int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
-
-int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
-
-int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
-
-/*
- These functions compress data in successive blocks of any size, using previous blocks as dictionary.
- One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
- There is an exception for ring buffers, which can be smaller 64 KB.
- Such case is automatically detected and correctly handled by LZ4_compress_HC_continue().
-
- Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
- A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
-
- Then, use LZ4_compress_HC_continue() to compress each successive block.
- It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression.
- Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
- As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation.
-
- If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
- you must save it to a safer memory space, using LZ4_saveDictHC().
- Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
-*/
-
-
-
-/**************************************
-* Deprecated Functions
-**************************************/
-/* Deprecate Warnings */
-/* Should these warnings messages be a problem,
- it is generally possible to disable them,
- with -Wno-deprecated-declarations for gcc
- or _CRT_SECURE_NO_WARNINGS in Visual for example.
- You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
-#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
-# define LZ4_DEPRECATE_WARNING_DEFBLOCK
-# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
-# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-# elif (LZ4_GCC_VERSION >= 301)
-# define LZ4_DEPRECATED(message) __attribute__((deprecated))
-# elif defined(_MSC_VER)
-# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
-# else
-# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
-# define LZ4_DEPRECATED(message)
-# endif
-#endif // LZ4_DEPRECATE_WARNING_DEFBLOCK
-
-/* compression functions */
-/* these functions are planned to trigger warning messages by r131 approximately */
-int LZ4_compressHC (const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
-int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
-int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/* Streaming functions following the older model; should no longer be used */
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
-LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
-
-
-#if defined (__cplusplus)
-}
-#endif
diff --git a/c-blosc/internal-complibs/lz4-1.7.2/lz4.c b/c-blosc/internal-complibs/lz4-1.7.5/lz4.c
similarity index 70%
rename from c-blosc/internal-complibs/lz4-1.7.2/lz4.c
rename to c-blosc/internal-complibs/lz4-1.7.5/lz4.c
index ff48644..143c36e 100644
--- a/c-blosc/internal-complibs/lz4-1.7.2/lz4.c
+++ b/c-blosc/internal-complibs/lz4-1.7.5/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2015, Yann Collet.
+ Copyright (C) 2011-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -28,12 +28,12 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 source repository : https://github.com/Cyan4973/lz4
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+ - LZ4 homepage : http://www.lz4.org
+ - LZ4 source repository : https://github.com/lz4/lz4
*/
-/**************************************
+/*-************************************
* Tuning parameters
**************************************/
/*
@@ -41,7 +41,9 @@
* Select how default compression functions will allocate memory for their hash table,
* in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
*/
-#define HEAPMODE 0
+#ifndef HEAPMODE
+# define HEAPMODE 0
+#endif
/*
* ACCELERATION_DEFAULT :
@@ -50,7 +52,7 @@
#define ACCELERATION_DEFAULT 1
-/**************************************
+/*-************************************
* CPU Feature Detection
**************************************/
/* LZ4_FORCE_MEMORY_ACCESS
@@ -63,7 +65,7 @@
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets which generate assembly depending on alignment.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
- * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
@@ -84,13 +86,14 @@
#endif
-/**************************************
-* Includes
+/*-************************************
+* Dependency
**************************************/
#include "lz4.h"
+/* see also "memory routines" below */
-/**************************************
+/*-************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
@@ -99,19 +102,16 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
#else
-# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
-# if defined(__GNUC__) || defined(__clang__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
+# if defined(__GNUC__) || defined(__clang__)
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define FORCE_INLINE static inline
# else
# define FORCE_INLINE static
-# endif /* __STDC_VERSION__ */
+# endif
#endif /* _MSC_VER */
-/* LZ4_GCC_VERSION is defined into lz4.h */
-#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
+#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
# define expect(expr,value) (expr)
@@ -121,7 +121,7 @@
#define unlikely(expr) expect((expr) != 0, 0)
-/**************************************
+/*-************************************
* Memory routines
**************************************/
#include <stdlib.h> /* malloc, calloc, free */
@@ -131,60 +131,66 @@
#define MEM_INIT memset
-/**************************************
+/*-************************************
* Basic Types
**************************************/
-#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
+#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
+ typedef uintptr_t uptrval;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
+ typedef size_t uptrval; /* generally true, except OpenVMS-64 */
#endif
+#if defined(__x86_64__)
+ typedef U64 reg_t; /* 64-bits in x32 mode */
+#else
+ typedef size_t reg_t; /* 32-bits in x32 mode */
+#endif
-/**************************************
+/*-************************************
* Reading and writing into memory
**************************************/
-#define STEPSIZE sizeof(size_t)
-
-static unsigned LZ4_64bits(void) { return sizeof(void*)==8; }
-
static unsigned LZ4_isLittleEndian(void)
{
- const union { U32 i; BYTE c[4]; } one = { 1 }; // don't use static : performance detrimental
+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
}
#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2)
+/* lie to the compiler about data alignment; use with caution */
static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; }
static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; }
-static size_t LZ4_read_ARCH(const void* memPtr) { return *(const size_t*) memPtr; }
+static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; }
static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1)
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
-typedef union { U16 u16; U32 u32; size_t uArch; } __attribute__((packed)) unalign;
+typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;
static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-static size_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
+static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; }
static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-#else
+#else /* safe and portable access through memcpy() */
static U16 LZ4_read16(const void* memPtr)
{
@@ -196,9 +202,9 @@ static U32 LZ4_read32(const void* memPtr)
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
-static size_t LZ4_read_ARCH(const void* memPtr)
+static reg_t LZ4_read_ARCH(const void* memPtr)
{
- size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
+ reg_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
static void LZ4_write16(void* memPtr, U16 value)
@@ -206,17 +212,19 @@ static void LZ4_write16(void* memPtr, U16 value)
memcpy(memPtr, &value, sizeof(value));
}
-#endif // LZ4_FORCE_MEMORY_ACCESS
+static void LZ4_write32(void* memPtr, U32 value)
+{
+ memcpy(memPtr, &value, sizeof(value));
+}
+
+#endif /* LZ4_FORCE_MEMORY_ACCESS */
static U16 LZ4_readLE16(const void* memPtr)
{
- if (LZ4_isLittleEndian())
- {
+ if (LZ4_isLittleEndian()) {
return LZ4_read16(memPtr);
- }
- else
- {
+ } else {
const BYTE* p = (const BYTE*)memPtr;
return (U16)((U16)p[0] + (p[1]<<8));
}
@@ -224,12 +232,9 @@ static U16 LZ4_readLE16(const void* memPtr)
static void LZ4_writeLE16(void* memPtr, U16 value)
{
- if (LZ4_isLittleEndian())
- {
+ if (LZ4_isLittleEndian()) {
LZ4_write16(memPtr, value);
- }
- else
- {
+ } else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE) value;
p[1] = (BYTE)(value>>8);
@@ -241,24 +246,18 @@ static void LZ4_copy8(void* dst, const void* src)
memcpy(dst,src,8);
}
-/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */
+/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
{
BYTE* d = (BYTE*)dstPtr;
const BYTE* s = (const BYTE*)srcPtr;
BYTE* const e = (BYTE*)dstEnd;
-#if 0
- const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1));
- LZ4_copy8(d,s); if (d>e-9) return;
- d+=l2; s+=l2;
-#endif /* join to align */
-
do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e);
}
-/**************************************
+/*-************************************
* Common Constants
**************************************/
#define MINMATCH 4
@@ -281,55 +280,48 @@ static const int LZ4_minLength = (MFLIMIT+1);
#define RUN_MASK ((1U<<RUN_BITS)-1)
-/**************************************
+/*-************************************
* Common Utils
**************************************/
#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
-/**************************************
+/*-************************************
* Common functions
**************************************/
-static unsigned LZ4_NbCommonBytes (register size_t val)
+static unsigned LZ4_NbCommonBytes (register reg_t val)
{
- if (LZ4_isLittleEndian())
- {
- if (LZ4_64bits())
- {
+ if (LZ4_isLittleEndian()) {
+ if (sizeof(val)==8) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
- }
- else /* 32 bits */
- {
+ } else /* 32 bits */ {
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
- }
- else /* Big Endian CPU */
- {
- if (LZ4_64bits())
- {
+ } else /* Big Endian CPU */ {
+ if (sizeof(val)==8) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll((U64)val) >> 3);
# else
unsigned r;
@@ -338,14 +330,12 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
r += (!val);
return r;
# endif
- }
- else /* 32 bits */
- {
+ } else /* 32 bits */ {
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
+# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
@@ -357,19 +347,19 @@ static unsigned LZ4_NbCommonBytes (register size_t val)
}
}
+#define STEPSIZE sizeof(reg_t)
static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
- while (likely(pIn<pInLimit-(STEPSIZE-1)))
- {
- size_t diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
+ while (likely(pIn<pInLimit-(STEPSIZE-1))) {
+ reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; }
pIn += LZ4_NbCommonBytes(diff);
return (unsigned)(pIn - pStart);
}
- if (LZ4_64bits()) if ((pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; }
+ if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (unsigned)(pIn - pStart);
@@ -377,29 +367,16 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi
#ifndef LZ4_COMMONDEFS_ONLY
-/**************************************
+/*-************************************
* Local Constants
**************************************/
-#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
-#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
-#define HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
-
static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1));
static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */
-/**************************************
+/*-************************************
* Local Structures and types
**************************************/
-typedef struct {
- U32 hashTable[HASH_SIZE_U32];
- U32 currentOffset;
- U32 initCheck;
- const BYTE* dictionary;
- BYTE* bufferStart; /* obsolete, used for slideInputBuffer */
- U32 dictSize;
-} LZ4_stream_t_internal;
-
typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
typedef enum { byPtr, byU32, byU16 } tableType_t;
@@ -410,44 +387,43 @@ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
-/**************************************
+/*-************************************
* Local Utils
**************************************/
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
+const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
-
-/********************************
+/*-******************************
* Compression functions
********************************/
-
-static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
+static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
{
if (tableType == byU16)
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
+ return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
else
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
+ return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
}
-static const U64 prime5bytes = 889523592379ULL;
-static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType)
+static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
{
+ static const U64 prime5bytes = 889523592379ULL;
+ static const U64 prime8bytes = 11400714785074694791ULL;
const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG;
- const U32 hashMask = (1<<hashLog) - 1;
- return ((sequence * prime5bytes) >> (40 - hashLog)) & hashMask;
+ if (LZ4_isLittleEndian())
+ return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
+ else
+ return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
-static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType)
+FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
{
- if (LZ4_64bits())
- return LZ4_hashSequence64(sequence, tableType);
- return LZ4_hashSequence((U32)sequence, tableType);
+ if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
+ return LZ4_hash4(LZ4_read32(p), tableType);
}
-static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); }
-
static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
{
switch (tableType)
@@ -458,27 +434,30 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
}
}
-static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
- U32 h = LZ4_hashPosition(p, tableType);
+ U32 const h = LZ4_hashPosition(p, tableType);
LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
}
static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
- if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
- { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
+ if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
+ { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
}
-static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
- U32 h = LZ4_hashPosition(p, tableType);
+ U32 const h = LZ4_hashPosition(p, tableType);
return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
}
+
+/** LZ4_compress_generic() :
+ inlined, to ensure branches are decided at compilation time */
FORCE_INLINE int LZ4_compress_generic(
- void* const ctx,
+ LZ4_stream_t_internal* const cctx,
const char* const source,
char* const dest,
const int inputSize,
@@ -489,15 +468,13 @@ FORCE_INLINE int LZ4_compress_generic(
const dictIssue_directive dictIssue,
const U32 acceleration)
{
- LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx;
-
const BYTE* ip = (const BYTE*) source;
const BYTE* base;
const BYTE* lowLimit;
- const BYTE* const lowRefLimit = ip - dictPtr->dictSize;
- const BYTE* const dictionary = dictPtr->dictionary;
- const BYTE* const dictEnd = dictionary + dictPtr->dictSize;
- const size_t dictDelta = dictEnd - (const BYTE*)source;
+ const BYTE* const lowRefLimit = ip - cctx->dictSize;
+ const BYTE* const dictionary = cctx->dictionary;
+ const BYTE* const dictEnd = dictionary + cctx->dictSize;
+ const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
const BYTE* anchor = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
@@ -507,10 +484,9 @@ FORCE_INLINE int LZ4_compress_generic(
BYTE* const olimit = op + maxOutputSize;
U32 forwardH;
- size_t refDelta=0;
/* Init conditions */
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */
switch(dict)
{
case noDict:
@@ -519,11 +495,11 @@ FORCE_INLINE int LZ4_compress_generic(
lowLimit = (const BYTE*)source;
break;
case withPrefix64k:
- base = (const BYTE*)source - dictPtr->currentOffset;
- lowLimit = (const BYTE*)source - dictPtr->dictSize;
+ base = (const BYTE*)source - cctx->currentOffset;
+ lowLimit = (const BYTE*)source - cctx->dictSize;
break;
case usingExtDict:
- base = (const BYTE*)source - dictPtr->currentOffset;
+ base = (const BYTE*)source - cctx->currentOffset;
lowLimit = (const BYTE*)source;
break;
}
@@ -531,44 +507,38 @@ FORCE_INLINE int LZ4_compress_generic(
if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
- LZ4_putPosition(ip, ctx, tableType, base);
+ LZ4_putPosition(ip, cctx->hashTable, tableType, base);
ip++; forwardH = LZ4_hashPosition(ip, tableType);
/* Main Loop */
- for ( ; ; )
- {
+ for ( ; ; ) {
+ ptrdiff_t refDelta = 0;
const BYTE* match;
BYTE* token;
- {
- const BYTE* forwardIp = ip;
+
+ /* Find a match */
+ { const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
-
- /* Find a match */
do {
- U32 h = forwardH;
+ U32 const h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> LZ4_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
- match = LZ4_getPositionOnHash(h, ctx, tableType, base);
- if (dict==usingExtDict)
- {
- if (match<(const BYTE*)source)
- {
+ match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
+ if (dict==usingExtDict) {
+ if (match < (const BYTE*)source) {
refDelta = dictDelta;
lowLimit = dictionary;
- }
- else
- {
+ } else {
refDelta = 0;
lowLimit = (const BYTE*)source;
- }
- }
+ } }
forwardH = LZ4_hashPosition(forwardIp, tableType);
- LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
+ LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
} while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
|| ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
@@ -576,18 +546,17 @@ FORCE_INLINE int LZ4_compress_generic(
}
/* Catch up */
- while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; }
+ while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; }
- {
- /* Encode Literal length */
- unsigned litLength = (unsigned)(ip - anchor);
+ /* Encode Literals */
+ { unsigned const litLength = (unsigned)(ip - anchor);
token = op++;
- if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
- return 0; /* Check output limit */
- if (litLength>=RUN_MASK)
- {
+ if ((outputLimited) && /* Check output buffer overflow */
+ (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
+ return 0;
+ if (litLength >= RUN_MASK) {
int len = (int)litLength-RUN_MASK;
- *token=(RUN_MASK<<ML_BITS);
+ *token = (RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
}
@@ -603,41 +572,37 @@ _next_match:
LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
/* Encode MatchLength */
- {
- unsigned matchLength;
+ { unsigned matchCode;
- if ((dict==usingExtDict) && (lowLimit==dictionary))
- {
+ if ((dict==usingExtDict) && (lowLimit==dictionary)) {
const BYTE* limit;
match += refDelta;
limit = ip + (dictEnd-match);
if (limit > matchlimit) limit = matchlimit;
- matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
- ip += MINMATCH + matchLength;
- if (ip==limit)
- {
- unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit);
- matchLength += more;
+ matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
+ ip += MINMATCH + matchCode;
+ if (ip==limit) {
+ unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit);
+ matchCode += more;
ip += more;
}
- }
- else
- {
- matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
- ip += MINMATCH + matchLength;
+ } else {
+ matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
+ ip += MINMATCH + matchCode;
}
- if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit)))
- return 0; /* Check output limit */
- if (matchLength>=ML_MASK)
- {
+ if ( outputLimited && /* Check output buffer overflow */
+ (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
+ return 0;
+ if (matchCode >= ML_MASK) {
*token += ML_MASK;
- matchLength -= ML_MASK;
- for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; }
- if (matchLength >= 255) { matchLength-=255; *op++ = 255; }
- *op++ = (BYTE)matchLength;
- }
- else *token += (BYTE)(matchLength);
+ matchCode -= ML_MASK;
+ LZ4_write32(op, 0xFFFFFFFF);
+ while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255;
+ op += matchCode / 255;
+ *op++ = (BYTE)(matchCode % 255);
+ } else
+ *token += (BYTE)(matchCode);
}
anchor = ip;
@@ -646,24 +611,19 @@ _next_match:
if (ip > mflimit) break;
/* Fill table */
- LZ4_putPosition(ip-2, ctx, tableType, base);
+ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
/* Test next position */
- match = LZ4_getPosition(ip, ctx, tableType, base);
- if (dict==usingExtDict)
- {
- if (match<(const BYTE*)source)
- {
+ match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
+ if (dict==usingExtDict) {
+ if (match < (const BYTE*)source) {
refDelta = dictDelta;
lowLimit = dictionary;
- }
- else
- {
+ } else {
refDelta = 0;
lowLimit = (const BYTE*)source;
- }
- }
- LZ4_putPosition(ip, ctx, tableType, base);
+ } }
+ LZ4_putPosition(ip, cctx->hashTable, tableType, base);
if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
&& (match+MAX_DISTANCE>=ip)
&& (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
@@ -675,19 +635,16 @@ _next_match:
_last_literals:
/* Encode Last Literals */
- {
- const size_t lastRun = (size_t)(iend - anchor);
- if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
- return 0; /* Check output limit */
- if (lastRun >= RUN_MASK)
- {
+ { size_t const lastRun = (size_t)(iend - anchor);
+ if ( (outputLimited) && /* Check output buffer overflow */
+ ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
+ return 0;
+ if (lastRun >= RUN_MASK) {
size_t accumulator = lastRun - RUN_MASK;
*op++ = RUN_MASK << ML_BITS;
for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
*op++ = (BYTE) accumulator;
- }
- else
- {
+ } else {
*op++ = (BYTE)(lastRun<<ML_BITS);
}
memcpy(op, anchor, lastRun);
@@ -701,22 +658,20 @@ _last_literals:
int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
+ LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
LZ4_resetStream((LZ4_stream_t*)state);
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
- if (maxOutputSize >= LZ4_compressBound(inputSize))
- {
+ if (maxOutputSize >= LZ4_compressBound(inputSize)) {
if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
else
- return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
- }
- else
- {
+ return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+ } else {
if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
- return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
}
@@ -727,10 +682,10 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctx;
- void* ctxPtr = &ctx;
+ void* const ctxPtr = &ctx;
#endif
- int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+ int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
#if (HEAPMODE)
FREEMEM(ctxPtr);
@@ -750,22 +705,21 @@ int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxO
int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
LZ4_stream_t ctx;
-
LZ4_resetStream(&ctx);
if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
- return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+ return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
-/********************************
-* destSize variant
+/*-******************************
+* *_destSize() variant
********************************/
static int LZ4_compress_destSize_generic(
- void* const ctx,
+ LZ4_stream_t_internal* const ctx,
const char* const src,
char* const dst,
int* const srcSizePtr,
@@ -797,32 +751,30 @@ static int LZ4_compress_destSize_generic(
/* First Byte */
*srcSizePtr = 0;
- LZ4_putPosition(ip, ctx, tableType, base);
+ LZ4_putPosition(ip, ctx->hashTable, tableType, base);
ip++; forwardH = LZ4_hashPosition(ip, tableType);
/* Main Loop */
- for ( ; ; )
- {
+ for ( ; ; ) {
const BYTE* match;
BYTE* token;
- {
- const BYTE* forwardIp = ip;
+
+ /* Find a match */
+ { const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = 1 << LZ4_skipTrigger;
- /* Find a match */
do {
U32 h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> LZ4_skipTrigger);
- if (unlikely(forwardIp > mflimit))
- goto _last_literals;
+ if (unlikely(forwardIp > mflimit)) goto _last_literals;
- match = LZ4_getPositionOnHash(h, ctx, tableType, base);
+ match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base);
forwardH = LZ4_hashPosition(forwardIp, tableType);
- LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
+ LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
} while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
|| (LZ4_read32(match) != LZ4_read32(ip)) );
@@ -831,18 +783,15 @@ static int LZ4_compress_destSize_generic(
/* Catch up */
while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
- {
- /* Encode Literal length */
- unsigned litLength = (unsigned)(ip - anchor);
+ /* Encode Literal length */
+ { unsigned litLength = (unsigned)(ip - anchor);
token = op++;
- if (op + ((litLength+240)/255) + litLength > oMaxLit)
- {
+ if (op + ((litLength+240)/255) + litLength > oMaxLit) {
/* Not enough space for a last match */
op--;
goto _last_literals;
}
- if (litLength>=RUN_MASK)
- {
+ if (litLength>=RUN_MASK) {
unsigned len = litLength - RUN_MASK;
*token=(RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
@@ -860,20 +809,15 @@ _next_match:
LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
/* Encode MatchLength */
- {
- size_t matchLength;
+ { size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
- matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
-
- if (op + ((matchLength+240)/255) > oMaxMatch)
- {
+ if (op + ((matchLength+240)/255) > oMaxMatch) {
/* Match description too long : reduce it */
matchLength = (15-1) + (oMaxMatch-op) * 255;
}
ip += MINMATCH + matchLength;
- if (matchLength>=ML_MASK)
- {
+ if (matchLength>=ML_MASK) {
*token += ML_MASK;
matchLength -= ML_MASK;
while (matchLength >= 255) { matchLength-=255; *op++ = 255; }
@@ -889,11 +833,11 @@ _next_match:
if (op > oMaxSeq) break;
/* Fill table */
- LZ4_putPosition(ip-2, ctx, tableType, base);
+ LZ4_putPosition(ip-2, ctx->hashTable, tableType, base);
/* Test next position */
- match = LZ4_getPosition(ip, ctx, tableType, base);
- LZ4_putPosition(ip, ctx, tableType, base);
+ match = LZ4_getPosition(ip, ctx->hashTable, tableType, base);
+ LZ4_putPosition(ip, ctx->hashTable, tableType, base);
if ( (match+MAX_DISTANCE>=ip)
&& (LZ4_read32(match)==LZ4_read32(ip)) )
{ token=op++; *token=0; goto _next_match; }
@@ -904,25 +848,20 @@ _next_match:
_last_literals:
/* Encode Last Literals */
- {
- size_t lastRunSize = (size_t)(iend - anchor);
- if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend)
- {
+ { size_t lastRunSize = (size_t)(iend - anchor);
+ if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) {
/* adapt lastRunSize to fill 'dst' */
lastRunSize = (oend-op) - 1;
lastRunSize -= (lastRunSize+240)/255;
}
ip = anchor + lastRunSize;
- if (lastRunSize >= RUN_MASK)
- {
+ if (lastRunSize >= RUN_MASK) {
size_t accumulator = lastRunSize - RUN_MASK;
*op++ = RUN_MASK << ML_BITS;
for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
*op++ = (BYTE) accumulator;
- }
- else
- {
+ } else {
*op++ = (BYTE)(lastRunSize<<ML_BITS);
}
memcpy(op, anchor, lastRunSize);
@@ -935,20 +874,17 @@ _last_literals:
}
-static int LZ4_compress_destSize_extState (void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
+static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
- LZ4_resetStream((LZ4_stream_t*)state);
+ LZ4_resetStream(state);
- if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */
- {
+ if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
- }
- else
- {
+ } else {
if (*srcSizePtr < LZ4_64Klimit)
- return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16);
+ return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
else
- return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr);
+ return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr);
}
}
@@ -956,10 +892,10 @@ static int LZ4_compress_destSize_extState (void* state, const char* src, char* d
int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
#if (HEAPMODE)
- void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
+ LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctxBody;
- void* ctx = &ctxBody;
+ LZ4_stream_t* ctx = &ctxBody;
#endif
int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
@@ -972,7 +908,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
-/********************************
+/*-******************************
* Streaming functions
********************************/
@@ -996,10 +932,10 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
}
-#define HASH_UNIT sizeof(size_t)
+#define HASH_UNIT sizeof(reg_t)
int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
{
- LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
+ LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse;
const BYTE* p = (const BYTE*)dictionary;
const BYTE* const dictEnd = p + dictSize;
const BYTE* base;
@@ -1007,8 +943,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */
LZ4_resetStream(LZ4_dict);
- if (dictSize < (int)HASH_UNIT)
- {
+ if (dictSize < (int)HASH_UNIT) {
dict->dictionary = NULL;
dict->dictSize = 0;
return 0;
@@ -1021,8 +956,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
dict->dictSize = (U32)(dictEnd - p);
dict->currentOffset += dict->dictSize;
- while (p <= dictEnd-HASH_UNIT)
- {
+ while (p <= dictEnd-HASH_UNIT) {
LZ4_putPosition(p, dict->hashTable, byU32, base);
p+=3;
}
@@ -1034,14 +968,12 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
{
if ((LZ4_dict->currentOffset > 0x80000000) ||
- ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */
- {
+ ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */
/* rescale hash table */
- U32 delta = LZ4_dict->currentOffset - 64 KB;
+ U32 const delta = LZ4_dict->currentOffset - 64 KB;
const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
int i;
- for (i=0; i<HASH_SIZE_U32; i++)
- {
+ for (i=0; i<LZ4_HASH_SIZE_U32; i++) {
if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
else LZ4_dict->hashTable[i] -= delta;
}
@@ -1054,7 +986,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
- LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream;
+ LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
const BYTE* smallest = (const BYTE*) source;
@@ -1064,10 +996,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
/* Check overlapping input/dictionary space */
- {
- const BYTE* sourceEnd = (const BYTE*) source + inputSize;
- if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
- {
+ { const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+ if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) {
streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
@@ -1076,25 +1006,23 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
}
/* prefix mode : source data follows dictionary */
- if (dictEnd == (const BYTE*)source)
- {
+ if (dictEnd == (const BYTE*)source) {
int result;
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
else
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
streamPtr->dictSize += (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
/* external dictionary mode */
- {
- int result;
+ { int result;
if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
else
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
@@ -1106,15 +1034,15 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
/* Hidden debug function, to force external dictionary mode */
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
{
- LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict;
+ LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
int result;
const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
const BYTE* smallest = dictEnd;
if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
- LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest);
+ LZ4_renormDictT(streamPtr, smallest);
- result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+ result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
@@ -1124,10 +1052,17 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
}
+/*! LZ4_saveDict() :
+ * If previously compressed data block is not guaranteed to remain available at its memory location,
+ * save it into a safer place (char* safeBuffer).
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ */
int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
{
- LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
- const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
+ LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
+ const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
@@ -1142,14 +1077,14 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
-/*******************************
+/*-*****************************
* Decompression functions
*******************************/
-/*
- * This generic decompression function cover all use cases.
- * It shall be instantiated several times, using different sets of directives
- * Note that it is essential this generic function is really inlined,
- * in order to remove useless branches during compilation optimization.
+/*! LZ4_decompress_generic() :
+ * This generic decompression function cover all use cases.
+ * It shall be instantiated several times, using different sets of directives
+ * Note that it is important this generic function is really inlined,
+ * in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int LZ4_decompress_generic(
const char* const source,
@@ -1161,7 +1096,7 @@ FORCE_INLINE int LZ4_decompress_generic(
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
- const BYTE* const lowPrefix, /* == dest if dict == noDict */
+ const BYTE* const lowPrefix, /* == dest when no prefix */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize /* note : = 0 if noDict */
)
@@ -1177,7 +1112,7 @@ FORCE_INLINE int LZ4_decompress_generic(
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
- const unsigned dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4};
+ const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};
const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
const int safeDecode = (endOnInput==endOnInputSize);
@@ -1185,46 +1120,37 @@ FORCE_INLINE int LZ4_decompress_generic(
/* Special cases */
- if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
+ if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
-
- /* Main Loop */
- while (1)
- {
- unsigned token;
+ /* Main Loop : decode sequences */
+ while (1) {
size_t length;
const BYTE* match;
size_t offset;
/* get literal length */
- token = *ip++;
- if ((length=(token>>ML_BITS)) == RUN_MASK)
- {
+ unsigned const token = *ip++;
+ if ((length=(token>>ML_BITS)) == RUN_MASK) {
unsigned s;
- do
- {
+ do {
s = *ip++;
length += s;
- }
- while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) && (s==255) );
- if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
- if ((safeDecode) && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
+ } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) );
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
}
/* copy literals */
cpy = op+length;
- if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)))
+ if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
+ || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
{
- if (partialDecoding)
- {
+ if (partialDecoding) {
if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */
if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
- }
- else
- {
+ } else {
if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
}
@@ -1240,59 +1166,49 @@ FORCE_INLINE int LZ4_decompress_generic(
offset = LZ4_readLE16(ip); ip+=2;
match = op - offset;
if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */
+ LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */
/* get matchlength */
length = token & ML_MASK;
- if (length == ML_MASK)
- {
+ if (length == ML_MASK) {
unsigned s;
- do
- {
- if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
+ do {
s = *ip++;
+ if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
length += s;
} while (s==255);
- if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;
/* check external dictionary */
- if ((dict==usingExtDict) && (match < lowPrefix))
- {
+ if ((dict==usingExtDict) && (match < lowPrefix)) {
if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */
- if (length <= (size_t)(lowPrefix-match))
- {
+ if (length <= (size_t)(lowPrefix-match)) {
/* match can be copied as a single segment from external dictionary */
- match = dictEnd - (lowPrefix-match);
- memmove(op, match, length); op += length;
- }
- else
- {
+ memmove(op, dictEnd - (lowPrefix-match), length);
+ op += length;
+ } else {
/* match encompass external dictionary and current block */
- size_t copySize = (size_t)(lowPrefix-match);
+ size_t const copySize = (size_t)(lowPrefix-match);
+ size_t const restSize = length - copySize;
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
- copySize = length - copySize;
- if (copySize > (size_t)(op-lowPrefix)) /* overlap copy */
- {
- BYTE* const endOfMatch = op + copySize;
+ if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */
+ BYTE* const endOfMatch = op + restSize;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
- }
- else
- {
- memcpy(op, lowPrefix, copySize);
- op += copySize;
- }
- }
+ } else {
+ memcpy(op, lowPrefix, restSize);
+ op += restSize;
+ } }
continue;
}
/* copy match within block */
cpy = op + length;
- if (unlikely(offset<8))
- {
+ if (unlikely(offset<8)) {
const int dec64 = dec64table[offset];
op[0] = match[0];
op[1] = match[1];
@@ -1304,20 +1220,19 @@ FORCE_INLINE int LZ4_decompress_generic(
} else { LZ4_copy8(op, match); match+=8; }
op += 8;
- if (unlikely(cpy>oend-12))
- {
+ if (unlikely(cpy>oend-12)) {
BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1);
if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
- if (op < oCopyLimit)
- {
+ if (op < oCopyLimit) {
LZ4_wildCopy(op, match, oCopyLimit);
match += oCopyLimit - op;
op = oCopyLimit;
}
while (op<cpy) *op++ = *match++;
+ } else {
+ LZ4_copy8(op, match);
+ if (length>16) LZ4_wildCopy(op+8, match+8, cpy);
}
- else
- LZ4_wildCopy(op, match, cpy);
op=cpy; /* correction */
}
@@ -1349,15 +1264,7 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
}
-/* streaming decompression functions */
-
-typedef struct
-{
- const BYTE* externalDict;
- size_t extDictSize;
- const BYTE* prefixEnd;
- size_t prefixSize;
-} LZ4_streamDecode_t_internal;
+/*===== streaming decompression functions =====*/
/*
* If you prefer dynamic allocation methods,
@@ -1376,16 +1283,16 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
return 0;
}
-/*
- * LZ4_setStreamDecode
- * Use this function to instruct where to find the dictionary
+/*!
+ * LZ4_setStreamDecode() :
+ * Use this function to instruct where to find the dictionary.
* This function is not necessary if previous data is still available where it was decoded.
* Loading a size of 0 is allowed (same effect as no dictionary).
* Return : 1 if OK, 0 if error
*/
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
{
- LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
lz4sd->prefixSize = (size_t) dictSize;
lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
lz4sd->externalDict = NULL;
@@ -1402,20 +1309,17 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
- LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
int result;
- if (lz4sd->prefixEnd == (BYTE*)dest)
- {
+ if (lz4sd->prefixEnd == (BYTE*)dest) {
result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
endOnInputSize, full, 0,
usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize += result;
lz4sd->prefixEnd += result;
- }
- else
- {
+ } else {
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
@@ -1431,22 +1335,19 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
{
- LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
int result;
- if (lz4sd->prefixEnd == (BYTE*)dest)
- {
+ if (lz4sd->prefixEnd == (BYTE*)dest) {
result = LZ4_decompress_generic(source, dest, 0, originalSize,
endOnOutputSize, full, 0,
usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize += originalSize;
lz4sd->prefixEnd += originalSize;
- }
- else
- {
+ } else {
lz4sd->extDictSize = lz4sd->prefixSize;
- lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize;
+ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
result = LZ4_decompress_generic(source, dest, 0, originalSize,
endOnOutputSize, full, 0,
usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
@@ -1470,8 +1371,7 @@ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest
{
if (dictSize==0)
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
- if (dictStart+dictSize == dest)
- {
+ if (dictStart+dictSize == dest) {
if (dictSize >= (int)(64 KB - 1))
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
@@ -1496,7 +1396,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
}
-/***************************************************
+/*=*************************************************
* Obsolete Functions
***************************************************/
/* obsolete compression functions */
@@ -1521,29 +1421,29 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize,
int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
-static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base)
+static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base)
{
- MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE);
- lz4ds->bufferStart = base;
+ MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t));
+ lz4ds->internal_donotuse.bufferStart = base;
}
int LZ4_resetStreamState(void* state, char* inputBuffer)
{
- if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
- LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer);
+ if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
+ LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer);
return 0;
}
void* LZ4_create (char* inputBuffer)
{
- void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64);
- LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer);
+ LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t));
+ LZ4_init (lz4ds, (BYTE*)inputBuffer);
return lz4ds;
}
char* LZ4_slideInputBuffer (void* LZ4_Data)
{
- LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data;
+ LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse;
int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB);
return (char*)(ctx->bufferStart + dictSize);
}
@@ -1561,4 +1461,3 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin
}
#endif /* LZ4_COMMONDEFS_ONLY */
-
diff --git a/c-blosc/internal-complibs/lz4-1.7.5/lz4.h b/c-blosc/internal-complibs/lz4-1.7.5/lz4.h
new file mode 100644
index 0000000..0aae19c
--- /dev/null
+++ b/c-blosc/internal-complibs/lz4-1.7.5/lz4.h
@@ -0,0 +1,463 @@
+/*
+ * LZ4 - Fast LZ compression algorithm
+ * Header File
+ * Copyright (C) 2011-2016, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://www.lz4.org
+ - LZ4 source repository : https://github.com/lz4/lz4
+*/
+#ifndef LZ4_H_2983827168210
+#define LZ4_H_2983827168210
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* --- Dependency --- */
+#include <stddef.h> /* size_t */
+
+
+/**
+ Introduction
+
+ LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core,
+ scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
+ multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
+
+ The LZ4 compression library provides in-memory compression and decompression functions.
+ Compression can be done in:
+ - a single step (described as Simple Functions)
+ - a single step, reusing a context (described in Advanced Functions)
+ - unbounded multiple steps (described as Streaming compression)
+
+ lz4.h provides block compression functions. It gives full buffer control to user.
+ Decompressing an lz4-compressed block also requires metadata (such as compressed size).
+ Each application is free to encode such metadata in whichever way it wants.
+
+ An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md),
+ take care of encoding standard metadata alongside LZ4-compressed blocks.
+ If your application requires interoperability, it's recommended to use it.
+ A library is provided to take care of it, see lz4frame.h.
+*/
+
+/*^***************************************************************
+* Export parameters
+*****************************************************************/
+/*
+* LZ4_DLL_EXPORT :
+* Enable exporting of functions when building a Windows DLL
+*/
+#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
+# define LZ4LIB_API __declspec(dllexport)
+#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
+# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define LZ4LIB_API
+#endif
+
+
+/*========== Version =========== */
+#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
+#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
+
+#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
+
+#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
+#define LZ4_QUOTE(str) #str
+#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
+#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
+
+LZ4LIB_API int LZ4_versionNumber (void);
+LZ4LIB_API const char* LZ4_versionString (void);
+
+
+/*-************************************
+* Tuning parameter
+**************************************/
+/*!
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+ */
+#define LZ4_MEMORY_USAGE 14
+
+
+/*-************************************
+* Simple Functions
+**************************************/
+/*! LZ4_compress_default() :
+ Compresses 'sourceSize' bytes from buffer 'source'
+ into already allocated 'dest' buffer of size 'maxDestSize'.
+ Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
+ It also runs faster, so it's a recommended setting.
+ If the function cannot compress 'source' into a more limited 'dest' budget,
+ compression stops *immediately*, and the function result is zero.
+ As a consequence, 'dest' content is not valid.
+ This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
+ sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
+ return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
+ or 0 if compression fails */
+LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
+
+/*! LZ4_decompress_safe() :
+ compressedSize : is the precise full size of the compressed block.
+ maxDecompressedSize : is the size of destination buffer, which must be already allocated.
+ return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
+ If destination buffer is not large enough, decoding will stop and output an error code (<0).
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function is protected against buffer overflow exploits, including malicious data packets.
+ It never writes outside output buffer, nor reads outside input buffer.
+*/
+LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+
+
+/*-************************************
+* Advanced Functions
+**************************************/
+#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
+
+/*!
+LZ4_compressBound() :
+ Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
+ This function is primarily useful for memory allocation purposes (destination buffer size).
+ Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
+ Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+ inputSize : max supported value is LZ4_MAX_INPUT_SIZE
+ return : maximum output size in a "worst case" scenario
+ or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+*/
+LZ4LIB_API int LZ4_compressBound(int inputSize);
+
+/*!
+LZ4_compress_fast() :
+ Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+ The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
+ It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
+ An acceleration value of "1" is the same as regular LZ4_compress_default()
+ Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+*/
+LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+
+
+/*!
+LZ4_compress_fast_extState() :
+ Same compression function, just using an externally allocated memory space to store compression state.
+ Use LZ4_sizeofState() to know how much memory must be allocated,
+ and allocate it on 8-bytes boundaries (using malloc() typically).
+ Then, provide it as 'void* state' to compression function.
+*/
+LZ4LIB_API int LZ4_sizeofState(void);
+LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
+
+
+/*!
+LZ4_compress_destSize() :
+ Reverse the logic, by compressing as much data as possible from 'source' buffer
+ into already allocated buffer 'dest' of size 'targetDestSize'.
+ This function either compresses the entire 'source' content into 'dest' if it's large enough,
+ or fill 'dest' buffer completely with as much data as possible from 'source'.
+ *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
+ New value is necessarily <= old value.
+ return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
+ or 0 if compression fails
+*/
+LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
+
+
+/*!
+LZ4_decompress_fast() :
+ originalSize : is the original and therefore uncompressed size
+ return : the number of bytes read from the source buffer (in other words, the compressed size)
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
+ note : This function fully respect memory boundaries for properly formed compressed data.
+ It is a bit faster than LZ4_decompress_safe().
+ However, it does not provide any protection against intentionally modified data stream (malicious input).
+ Use this function in trusted environment only (data to decode comes from a trusted source).
+*/
+LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
+
+/*!
+LZ4_decompress_safe_partial() :
+ This function decompress a compressed block of size 'compressedSize' at position 'source'
+ into destination buffer 'dest' of size 'maxDecompressedSize'.
+ The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+ reducing decompression time.
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
+ Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+ Always control how many bytes were decoded.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
+
+
+/*-*********************************************
+* Streaming Compression Functions
+***********************************************/
+typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
+
+/*! LZ4_createStream() and LZ4_freeStream() :
+ * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
+ * LZ4_freeStream() releases its memory.
+ */
+LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
+LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
+
+/*! LZ4_resetStream() :
+ * An LZ4_stream_t structure can be allocated once and re-used multiple times.
+ * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression.
+ */
+LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
+/*! LZ4_loadDict() :
+ * Use this function to load a static dictionary into LZ4_stream.
+ * Any previous data will be forgotten, only 'dictionary' will remain in memory.
+ * Loading a size of 0 is allowed.
+ * Return : dictionary size, in bytes (necessarily <= 64 KB)
+ */
+LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
+
+/*! LZ4_compress_fast_continue() :
+ * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
+ * Important : Previous data blocks are assumed to still be present and unmodified !
+ * 'dst' buffer must be already allocated.
+ * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ */
+LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+
+/*! LZ4_saveDict() :
+ * If previously compressed data block is not guaranteed to remain available at its memory location,
+ * save it into a safer place (char* safeBuffer).
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ */
+LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
+
+
+/*-**********************************************
+* Streaming Decompression Functions
+* Bufferless synchronous API
+************************************************/
+typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */
+
+/* creation / destruction of streaming decompression tracking structure */
+LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
+LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+
+/*! LZ4_setStreamDecode() :
+ * Use this function to instruct where to find the dictionary.
+ * Setting a size of 0 is allowed (same effect as reset).
+ * @return : 1 if OK, 0 if error
+ */
+LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
+
+/*!
+LZ4_decompress_*_continue() :
+ These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+ In the case of a ring buffers, decoding buffer must be either :
+ - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+ In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ In which case, encoding and decoding buffers do not need to be synchronized,
+ and encoding ring buffer can have any size, including small ones ( < 64 KB).
+ - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ In which case, encoding and decoding buffers do not need to be synchronized,
+ and encoding ring buffer can have any size, including larger than decoding buffer.
+ Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+ and indicate where it is saved using LZ4_setStreamDecode()
+*/
+LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
+
+
+/*! LZ4_decompress_*_usingDict() :
+ * These decoding functions work the same as
+ * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
+ * They are stand-alone, and don't need an LZ4_streamDecode_t structure.
+ */
+LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
+LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+
+
+/*^**********************************************
+ * !!!!!! STATIC LINKING ONLY !!!!!!
+ ***********************************************/
+/*-************************************
+ * Private definitions
+ **************************************
+ * Do not use these definitions.
+ * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
+ * Using these definitions will expose code to API and/or ABI break in future versions of the library.
+ **************************************/
+#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
+#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
+#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
+
+#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#include <stdint.h>
+
+typedef struct {
+ uint32_t hashTable[LZ4_HASH_SIZE_U32];
+ uint32_t currentOffset;
+ uint32_t initCheck;
+ const uint8_t* dictionary;
+ uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */
+ uint32_t dictSize;
+} LZ4_stream_t_internal;
+
+typedef struct {
+ const uint8_t* externalDict;
+ size_t extDictSize;
+ const uint8_t* prefixEnd;
+ size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+
+#else
+
+typedef struct {
+ unsigned int hashTable[LZ4_HASH_SIZE_U32];
+ unsigned int currentOffset;
+ unsigned int initCheck;
+ const unsigned char* dictionary;
+ unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */
+ unsigned int dictSize;
+} LZ4_stream_t_internal;
+
+typedef struct {
+ const unsigned char* externalDict;
+ size_t extDictSize;
+ const unsigned char* prefixEnd;
+ size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+
+#endif
+
+/*!
+ * LZ4_stream_t :
+ * information structure to track an LZ4 stream.
+ * init this structure before first use.
+ * note : only use in association with static linking !
+ * this definition is not API/ABI safe,
+ * and may change in a future version !
+ */
+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
+union LZ4_stream_u {
+ unsigned long long table[LZ4_STREAMSIZE_U64];
+ LZ4_stream_t_internal internal_donotuse;
+} ; /* previously typedef'd to LZ4_stream_t */
+
+
+/*!
+ * LZ4_streamDecode_t :
+ * information structure to track an LZ4 stream during decompression.
+ * init this structure using LZ4_setStreamDecode (or memset()) before first use
+ * note : only use in association with static linking !
+ * this definition is not API/ABI safe,
+ * and may change in a future version !
+ */
+#define LZ4_STREAMDECODESIZE_U64 4
+#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+union LZ4_streamDecode_u {
+ unsigned long long table[LZ4_STREAMDECODESIZE_U64];
+ LZ4_streamDecode_t_internal internal_donotuse;
+} ; /* previously typedef'd to LZ4_streamDecode_t */
+
+
+/*=************************************
+* Obsolete Functions
+**************************************/
+/* Deprecation warnings */
+/* Should these warnings be a problem,
+ it is generally possible to disable them,
+ typically with -Wno-deprecated-declarations for gcc
+ or _CRT_SECURE_NO_WARNINGS in Visual.
+ Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */
+#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
+# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
+#else
+# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define LZ4_DEPRECATED(message) [[deprecated(message)]]
+# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+# elif (LZ4_GCC_VERSION >= 301)
+# define LZ4_DEPRECATED(message) __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
+# define LZ4_DEPRECATED(message)
+# endif
+#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
+
+/* Obsolete compression functions */
+LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Obsolete decompression functions */
+/* These function names are completely deprecated and must no longer be used.
+ They are only provided in lz4.c for compatibility with older programs.
+ - LZ4_uncompress is the same as LZ4_decompress_fast
+ - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
+ These function prototypes are now disabled; uncomment them only if you really need them.
+ It is highly recommended to stop using these prototypes and migrate to maintained ones */
+/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
+/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
+
+/* Obsolete streaming decoding functions */
+LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* LZ4_H_2983827168210 */
diff --git a/c-blosc/internal-complibs/lz4-1.7.2/lz4hc.c b/c-blosc/internal-complibs/lz4-1.7.5/lz4hc.c
similarity index 70%
rename from c-blosc/internal-complibs/lz4-1.7.2/lz4hc.c
rename to c-blosc/internal-complibs/lz4-1.7.5/lz4hc.c
index 80bfa39..5d4ea3e 100644
--- a/c-blosc/internal-complibs/lz4-1.7.2/lz4hc.c
+++ b/c-blosc/internal-complibs/lz4-1.7.5/lz4hc.c
@@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2015, Yann Collet.
+ Copyright (C) 2011-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -28,16 +28,15 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 source repository : https://github.com/Cyan4973/lz4
+ - LZ4 source repository : https://github.com/lz4/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
-
+/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */
/* *************************************
* Tuning Parameter
***************************************/
-static const int LZ4HC_compressionLevel_default = 9;
/*!
* HEAPMODE :
@@ -45,11 +44,13 @@ static const int LZ4HC_compressionLevel_default = 9;
* in stack (0:fastest), or in heap (1:requires malloc()).
* Since workplace is rather large, heap mode is recommended.
*/
-#define LZ4HC_HEAPMODE 0
+#ifndef LZ4HC_HEAPMODE
+# define LZ4HC_HEAPMODE 1
+#endif
/* *************************************
-* Includes
+* Dependency
***************************************/
#include "lz4hc.h"
@@ -76,42 +77,14 @@ static const int LZ4HC_compressionLevel_default = 9;
/* *************************************
* Local Constants
***************************************/
-#define DICTIONARY_LOGSIZE 16
-#define MAXD (1<<DICTIONARY_LOGSIZE)
-#define MAXD_MASK (MAXD - 1)
-
-#define HASH_LOG (DICTIONARY_LOGSIZE-1)
-#define HASHTABLESIZE (1 << HASH_LOG)
-#define HASH_MASK (HASHTABLESIZE - 1)
-
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
-static const int g_maxCompressionLevel = 16;
-
-
-/**************************************
-* Local Types
-**************************************/
-typedef struct
-{
- U32 hashTable[HASHTABLESIZE];
- U16 chainTable[MAXD];
- const BYTE* end; /* next block here to continue on current prefix */
- const BYTE* base; /* All index relative to this position */
- const BYTE* dictBase; /* alternate base for extDict */
- BYTE* inputBuffer; /* deprecated */
- U32 dictLimit; /* below that point, need extDict */
- U32 lowLimit; /* below that point, no more dict */
- U32 nextToUpdate; /* index from which to continue dictionary update */
- U32 compressionLevel;
-} LZ4HC_Data_Structure;
-
/**************************************
* Local Macros
**************************************/
-#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
-//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */
+#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
+#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
@@ -121,7 +94,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
/**************************************
* HC Compression
**************************************/
-static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
+static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
@@ -135,21 +108,20 @@ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
/* Update chains up to ip (excluded) */
-FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
+FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
{
- U16* chainTable = hc4->chainTable;
- U32* HashTable = hc4->hashTable;
+ U16* const chainTable = hc4->chainTable;
+ U32* const hashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
- const U32 target = (U32)(ip - base);
+ U32 const target = (U32)(ip - base);
U32 idx = hc4->nextToUpdate;
- while(idx < target)
- {
- U32 h = LZ4HC_hashPtr(base+idx);
- size_t delta = idx - HashTable[h];
+ while (idx < target) {
+ U32 const h = LZ4HC_hashPtr(base+idx);
+ size_t delta = idx - hashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
DELTANEXTU16(idx) = (U16)delta;
- HashTable[h] = idx;
+ hashTable[h] = idx;
idx++;
}
@@ -157,7 +129,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
}
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
@@ -169,7 +141,6 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I
const U32 dictLimit = hc4->dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
U32 matchIndex;
- const BYTE* match;
int nbAttempts=maxNbAttempts;
size_t ml=0;
@@ -177,24 +148,19 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I
LZ4HC_Insert(hc4, ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
- while ((matchIndex>=lowLimit) && (nbAttempts))
- {
+ while ((matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
- if (matchIndex >= dictLimit)
- {
- match = base + matchIndex;
+ if (matchIndex >= dictLimit) {
+ const BYTE* const match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (LZ4_read32(match) == LZ4_read32(ip)))
{
- size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
+ size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
- }
- else
- {
- match = dictBase + matchIndex;
- if (LZ4_read32(match) == LZ4_read32(ip))
- {
+ } else {
+ const BYTE* const match = dictBase + matchIndex;
+ if (LZ4_read32(match) == LZ4_read32(ip)) {
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
@@ -212,7 +178,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I
FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
- LZ4HC_Data_Structure* hc4,
+ LZ4HC_CCtx_internal* hc4,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
@@ -237,38 +203,32 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
LZ4HC_Insert(hc4, ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
- while ((matchIndex>=lowLimit) && (nbAttempts))
- {
+ while ((matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
- if (matchIndex >= dictLimit)
- {
+ if (matchIndex >= dictLimit) {
const BYTE* matchPtr = base + matchIndex;
- if (*(iLowLimit + longest) == *(matchPtr - delta + longest))
- if (LZ4_read32(matchPtr) == LZ4_read32(ip))
- {
+ if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
+ if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
int back = 0;
- while ((ip+back>iLowLimit)
+ while ((ip+back > iLowLimit)
&& (matchPtr+back > lowPrefixPtr)
&& (ip[back-1] == matchPtr[back-1]))
back--;
mlt -= back;
- if (mlt > longest)
- {
+ if (mlt > longest) {
longest = (int)mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
}
}
- }
- else
- {
- const BYTE* matchPtr = dictBase + matchIndex;
- if (LZ4_read32(matchPtr) == LZ4_read32(ip))
- {
+ }
+ } else {
+ const BYTE* const matchPtr = dictBase + matchIndex;
+ if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
size_t mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
@@ -328,8 +288,15 @@ FORCE_INLINE int LZ4HC_encodeSequence (
/* Encode MatchLength */
length = (int)(matchLength-MINMATCH);
if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
- if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
- else *token += (BYTE)(length);
+ if (length>=(int)ML_MASK) {
+ *token += ML_MASK;
+ length -= ML_MASK;
+ for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; }
+ if (length > 254) { length-=255; *(*op)++ = 255; }
+ *(*op)++ = (BYTE)length;
+ } else {
+ *token += (BYTE)(length);
+ }
/* Prepare next loop */
*ip += matchLength;
@@ -338,18 +305,18 @@ FORCE_INLINE int LZ4HC_encodeSequence (
return 0;
}
+#include "lz4opt.h"
-static int LZ4HC_compress_generic (
- void* ctxvoid,
- const char* source,
- char* dest,
- int inputSize,
- int maxOutputSize,
- int compressionLevel,
+static int LZ4HC_compress_hashChain (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const source,
+ char* const dest,
+ int const inputSize,
+ int const maxOutputSize,
+ unsigned maxNbAttempts,
limitedOutput_directive limit
)
{
- LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + inputSize;
@@ -359,28 +326,22 @@ static int LZ4HC_compress_generic (
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
- unsigned maxNbAttempts;
int ml, ml2, ml3, ml0;
- const BYTE* ref=NULL;
- const BYTE* start2=NULL;
- const BYTE* ref2=NULL;
- const BYTE* start3=NULL;
- const BYTE* ref3=NULL;
+ const BYTE* ref = NULL;
+ const BYTE* start2 = NULL;
+ const BYTE* ref2 = NULL;
+ const BYTE* start3 = NULL;
+ const BYTE* ref3 = NULL;
const BYTE* start0;
const BYTE* ref0;
-
/* init */
- if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
- if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default;
- maxNbAttempts = 1 << (compressionLevel-1);
ctx->end += inputSize;
ip++;
/* Main Loop */
- while (ip < mflimit)
- {
+ while (ip < mflimit) {
ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
if (!ml) { ip++; continue; }
@@ -391,19 +352,16 @@ static int LZ4HC_compress_generic (
_Search2:
if (ip+ml < mflimit)
- ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
+ ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts);
else ml2 = ml;
- if (ml2 == ml) /* No better match */
- {
+ if (ml2 == ml) { /* No better match */
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
continue;
}
- if (start0 < ip)
- {
- if (start2 < ip + ml0) /* empirical */
- {
+ if (start0 < ip) {
+ if (start2 < ip + ml0) { /* empirical */
ip = start0;
ref = ref0;
ml = ml0;
@@ -411,8 +369,7 @@ _Search2:
}
/* Here, start0==ip */
- if ((start2 - ip) < 3) /* First Match too small : removed */
- {
+ if ((start2 - ip) < 3) { /* First Match too small : removed */
ml = ml2;
ip = start2;
ref =ref2;
@@ -425,15 +382,13 @@ _Search3:
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
- if ((start2 - ip) < OPTIMAL_ML)
- {
+ if ((start2 - ip) < OPTIMAL_ML) {
int correction;
int new_ml = ml;
if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
correction = new_ml - (int)(start2 - ip);
- if (correction > 0)
- {
+ if (correction > 0) {
start2 += correction;
ref2 += correction;
ml2 -= correction;
@@ -445,8 +400,7 @@ _Search3:
ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
else ml3 = ml2;
- if (ml3 == ml2) /* No better match : 2 sequences to encode */
- {
+ if (ml3 == ml2) { /* No better match : 2 sequences to encode */
/* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
@@ -456,18 +410,14 @@ _Search3:
continue;
}
- if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */
- {
- if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
- {
- if (start2 < ip+ml)
- {
+ if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */
+ if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
+ if (start2 < ip+ml) {
int correction = (int)(ip+ml - start2);
start2 += correction;
ref2 += correction;
ml2 -= correction;
- if (ml2 < MINMATCH)
- {
+ if (ml2 < MINMATCH) {
start2 = start3;
ref2 = ref3;
ml2 = ml3;
@@ -495,23 +445,18 @@ _Search3:
* OK, now we have 3 ascending matches; let's write at least the first one
* ip & ref are known; Now for ml
*/
- if (start2 < ip+ml)
- {
- if ((start2 - ip) < (int)ML_MASK)
- {
+ if (start2 < ip+ml) {
+ if ((start2 - ip) < (int)ML_MASK) {
int correction;
if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
correction = ml - (int)(start2 - ip);
- if (correction > 0)
- {
+ if (correction > 0) {
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
- }
- else
- {
+ } else {
ml = (int)(start2 - ip);
}
}
@@ -529,8 +474,7 @@ _Search3:
}
/* Encode Last Literals */
- {
- int lastRun = (int)(iend - anchor);
+ { int lastRun = (int)(iend - anchor);
if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
@@ -542,29 +486,61 @@ _Search3:
return (int) (((char*)op)-dest);
}
+static int LZ4HC_getSearchNum(int compressionLevel)
+{
+ switch (compressionLevel) {
+ default: return 0; /* unused */
+ case 11: return 128;
+ case 12: return 1<<10;
+ }
+}
-int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); }
+static int LZ4HC_compress_generic (
+ LZ4HC_CCtx_internal* const ctx,
+ const char* const source,
+ char* const dest,
+ int const inputSize,
+ int const maxOutputSize,
+ int compressionLevel,
+ limitedOutput_directive limit
+ )
+{
+ if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
+ if (compressionLevel > 9) {
+ switch (compressionLevel) {
+ case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
+ case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
+ default:
+ case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
+ }
+ }
+ return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
+}
+
+
+int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
+ LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
- LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src);
+ LZ4HC_init (ctx, (const BYTE*)src);
if (maxDstSize < LZ4_compressBound(srcSize))
- return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
+ return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
else
- return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
+ return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
}
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
-#if LZ4HC_HEAPMODE==1
- LZ4HC_Data_Structure* statePtr = malloc(sizeof(LZ4HC_Data_Structure));
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
+ LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
#else
- LZ4HC_Data_Structure state;
- LZ4HC_Data_Structure* const statePtr = &state;
+ LZ4_streamHC_t state;
+ LZ4_streamHC_t* const statePtr = &state;
#endif
- int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
-#if LZ4HC_HEAPMODE==1
+ int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
+#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
free(statePtr);
#endif
return cSize;
@@ -583,32 +559,38 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_st
/* initialization */
void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
- LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t)); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
- ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL;
- ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel;
+ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+ LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
+ LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
}
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
- LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr;
- if (dictSize > 64 KB)
- {
+ LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+ if (dictSize > 64 KB) {
dictionary += dictSize - 64 KB;
dictSize = 64 KB;
}
LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
- if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
ctxPtr->end = (const BYTE*)dictionary + dictSize;
+ if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
+ LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
+ else
+ if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
return dictSize;
}
/* compression */
-static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
+static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{
- if (ctxPtr->end >= ctxPtr->base + 4)
- LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
+ if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
+ LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
+ else
+ if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
+
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
@@ -618,34 +600,29 @@ static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newB
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
}
-static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
+static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
const char* source, char* dest,
int inputSize, int maxOutputSize, limitedOutput_directive limit)
{
+ LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
/* auto-init if forgotten */
- if (ctxPtr->base == NULL)
- LZ4HC_init (ctxPtr, (const BYTE*) source);
+ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source);
/* Check overflow */
- if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
- {
+ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > 64 KB) dictSize = 64 KB;
-
- LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
+ LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
/* Check if blocks follow each other */
- if ((const BYTE*)source != ctxPtr->end)
- LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
+ if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */
- {
- const BYTE* sourceEnd = (const BYTE*) source + inputSize;
- const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
- const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
- if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd))
- {
+ { const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+ const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
+ const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
+ if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
@@ -658,9 +635,9 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
if (maxOutputSize < LZ4_compressBound(inputSize))
- return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
else
- return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
+ return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
}
@@ -668,14 +645,13 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
- LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr;
- int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+ LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
+ int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
- {
- U32 endIndex = (U32)(streamPtr->end - streamPtr->base);
+ { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
streamPtr->base = streamPtr->end - endIndex;
streamPtr->dictLimit = endIndex - dictSize;
@@ -689,8 +665,8 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
/***********************************
* Deprecated Functions
***********************************/
+/* These functions currently generate deprecation warnings */
/* Deprecated compression functions */
-/* These functions are planned to start generate warnings by r131 approximately */
int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); }
int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
@@ -704,45 +680,41 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src,
/* Deprecated streaming functions */
-/* These functions currently generate deprecation warnings */
int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
{
+ LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
- LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
- ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer;
+ LZ4HC_init(ctx, (const BYTE*)inputBuffer);
+ ctx->inputBuffer = (BYTE*)inputBuffer;
return 0;
}
void* LZ4_createHC (char* inputBuffer)
{
- void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure));
+ LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
if (hc4 == NULL) return NULL; /* not enough memory */
- LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
- ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer;
+ LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
+ hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;
return hc4;
}
-int LZ4_freeHC (void* LZ4HC_Data)
-{
- FREEMEM(LZ4HC_Data);
- return (0);
-}
+int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; }
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+ return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
- LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
- int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
+ LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+ int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
return (char*)(hc4->inputBuffer + dictSize);
}
diff --git a/c-blosc/internal-complibs/lz4-1.7.5/lz4hc.h b/c-blosc/internal-complibs/lz4-1.7.5/lz4hc.h
new file mode 100644
index 0000000..1036fd0
--- /dev/null
+++ b/c-blosc/internal-complibs/lz4-1.7.5/lz4hc.h
@@ -0,0 +1,228 @@
+/*
+ LZ4 HC - High Compression Mode of LZ4
+ Header File
+ Copyright (C) 2011-2016, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 source repository : https://github.com/lz4/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#ifndef LZ4_HC_H_19834876238432
+#define LZ4_HC_H_19834876238432
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* --- Dependency --- */
+/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */
+#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */
+
+
+/* --- Useful constants --- */
+#define LZ4HC_CLEVEL_MIN 3
+#define LZ4HC_CLEVEL_DEFAULT 9
+#define LZ4HC_CLEVEL_OPT_MIN 11
+#define LZ4HC_CLEVEL_MAX 12
+
+
+/*-************************************
+ * Block Compression
+ **************************************/
+/*! LZ4_compress_HC() :
+ * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm.
+ * `dst` must be already allocated.
+ * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
+ * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
+ * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work.
+ * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL.
+ * @return : the number of bytes written into 'dst'
+ * or 0 if compression fails.
+ */
+LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
+
+
+/* Note :
+ * Decompression functions are provided within "lz4.h" (BSD license)
+ */
+
+
+/*! LZ4_compress_HC_extStateHC() :
+ * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.
+ * `state` size is provided by LZ4_sizeofStateHC().
+ * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
+ */
+LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
+LZ4LIB_API int LZ4_sizeofStateHC(void);
+
+
+/*-************************************
+ * Streaming Compression
+ * Bufferless synchronous API
+ **************************************/
+ typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */
+
+/*! LZ4_createStreamHC() and LZ4_freeStreamHC() :
+ * These functions create and release memory for LZ4 HC streaming state.
+ * Newly created states are automatically initialized.
+ * Existing states can be re-used several times, using LZ4_resetStreamHC().
+ * These methods are API and ABI stable, they can be used in combination with a DLL.
+ */
+LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);
+LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
+
+LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
+LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
+
+LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
+
+LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
+
+/*
+ These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+ One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
+ There is an exception for ring buffers, which can be smaller than 64 KB.
+ Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue().
+
+ Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
+ A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
+
+ Then, use LZ4_compress_HC_continue() to compress each successive block.
+ Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
+ 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success.
+
+ If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
+ you must save it to a safer memory space, using LZ4_saveDictHC().
+ Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
+*/
+
+
+/*-******************************************
+ * !!!!! STATIC LINKING ONLY !!!!!
+ *******************************************/
+
+ /*-*************************************
+ * PRIVATE DEFINITIONS :
+ * Do not use these definitions.
+ * They are exposed to allow static allocation of `LZ4_streamHC_t`.
+ * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
+ **************************************/
+#define LZ4HC_DICTIONARY_LOGSIZE 17
+#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
+#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
+
+#define LZ4HC_HASH_LOG 15
+#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG)
+#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
+
+
+#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#include <stdint.h>
+
+typedef struct
+{
+ uint32_t hashTable[LZ4HC_HASHTABLESIZE];
+ uint16_t chainTable[LZ4HC_MAXD];
+ const uint8_t* end; /* next block here to continue on current prefix */
+ const uint8_t* base; /* All index relative to this position */
+ const uint8_t* dictBase; /* alternate base for extDict */
+ uint8_t* inputBuffer; /* deprecated */
+ uint32_t dictLimit; /* below that point, need extDict */
+ uint32_t lowLimit; /* below that point, no more dict */
+ uint32_t nextToUpdate; /* index from which to continue dictionary update */
+ uint32_t searchNum; /* only for optimal parser */
+ uint32_t compressionLevel;
+} LZ4HC_CCtx_internal;
+
+#else
+
+typedef struct
+{
+ unsigned int hashTable[LZ4HC_HASHTABLESIZE];
+ unsigned short chainTable[LZ4HC_MAXD];
+ const unsigned char* end; /* next block here to continue on current prefix */
+ const unsigned char* base; /* All index relative to this position */
+ const unsigned char* dictBase; /* alternate base for extDict */
+ unsigned char* inputBuffer; /* deprecated */
+ unsigned int dictLimit; /* below that point, need extDict */
+ unsigned int lowLimit; /* below that point, no more dict */
+ unsigned int nextToUpdate; /* index from which to continue dictionary update */
+ unsigned int searchNum; /* only for optimal parser */
+ unsigned int compressionLevel;
+} LZ4HC_CCtx_internal;
+
+#endif
+
+#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */
+#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
+union LZ4_streamHC_u {
+ size_t table[LZ4_STREAMHCSIZE_SIZET];
+ LZ4HC_CCtx_internal internal_donotuse;
+}; /* previously typedef'd to LZ4_streamHC_t */
+/*
+ LZ4_streamHC_t :
+ This structure allows static allocation of LZ4 HC streaming state.
+ State must be initialized using LZ4_resetStreamHC() before first use.
+
+ Static allocation shall only be used in combination with static linking.
+ When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable.
+*/
+
+
+/*-************************************
+* Deprecated Functions
+**************************************/
+/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */
+
+/* deprecated compression functions */
+/* these functions will trigger warning messages in future releases */
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Deprecated Streaming functions using older model; should no longer be used */
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
+LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* LZ4_HC_H_19834876238432 */
diff --git a/c-blosc/internal-complibs/lz4-1.7.5/lz4opt.h b/c-blosc/internal-complibs/lz4-1.7.5/lz4opt.h
new file mode 100644
index 0000000..d1913fe
--- /dev/null
+++ b/c-blosc/internal-complibs/lz4-1.7.5/lz4opt.h
@@ -0,0 +1,363 @@
+/*
+ lz4opt.h - Optimal Mode of LZ4
+ Copyright (C) 2015-2016, Przemyslaw Skibinski <inikep at gmail.com>
+ Note : this file is intended to be included within lz4hc.c
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 source repository : https://github.com/lz4/lz4
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+#define LZ4_OPT_NUM (1<<12)
+
+
+typedef struct
+{
+ int off;
+ int len;
+} LZ4HC_match_t;
+
+typedef struct
+{
+ int price;
+ int off;
+ int mlen;
+ int litlen;
+} LZ4HC_optimal_t;
+
+
+/* price in bits */
+FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
+{
+ size_t price = 8*litlen;
+ if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255);
+ return price;
+}
+
+
+/* requires mlen >= MINMATCH */
+FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
+{
+ size_t price = 16 + 8; /* 16-bit offset + token */
+
+ price += LZ4HC_literalsPrice(litlen);
+
+ mlen -= MINMATCH;
+ if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255);
+
+ return price;
+}
+
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
+ LZ4HC_CCtx_internal* ctx,
+ const BYTE* const ip,
+ const BYTE* const iHighLimit,
+ size_t best_mlen,
+ LZ4HC_match_t* matches,
+ int* matchNum)
+{
+ U16* const chainTable = ctx->chainTable;
+ U32* const HashTable = ctx->hashTable;
+ const BYTE* const base = ctx->base;
+ const U32 dictLimit = ctx->dictLimit;
+ const U32 current = (U32)(ip - base);
+ const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1);
+ const BYTE* const dictBase = ctx->dictBase;
+ const BYTE* match;
+ int nbAttempts = ctx->searchNum;
+ int mnum = 0;
+ U16 *ptr0, *ptr1, delta0, delta1;
+ U32 matchIndex;
+ size_t matchLength = 0;
+ U32* HashPos;
+
+ if (ip + MINMATCH > iHighLimit) return 1;
+
+ /* HC4 match finder */
+ HashPos = &HashTable[LZ4HC_hashPtr(ip)];
+ matchIndex = *HashPos;
+ *HashPos = current;
+
+ ptr0 = &DELTANEXTMAXD(current*2+1);
+ ptr1 = &DELTANEXTMAXD(current*2);
+ delta0 = delta1 = (U16)(current - matchIndex);
+
+ while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) {
+ nbAttempts--;
+ if (matchIndex >= dictLimit) {
+ match = base + matchIndex;
+ matchLength = LZ4_count(ip, match, iHighLimit);
+ } else {
+ const BYTE* vLimit = ip + (dictLimit - matchIndex);
+ match = dictBase + matchIndex;
+ if (vLimit > iHighLimit) vLimit = iHighLimit;
+ matchLength = LZ4_count(ip, match, vLimit);
+ if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
+ matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit);
+ }
+
+ if (matchLength > best_mlen) {
+ best_mlen = matchLength;
+ if (matches) {
+ if (matchIndex >= dictLimit)
+ matches[mnum].off = (int)(ip - match);
+ else
+ matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */
+ matches[mnum].len = (int)matchLength;
+ mnum++;
+ }
+ if (best_mlen > LZ4_OPT_NUM) break;
+ }
+
+ if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */
+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
+
+ if (*(ip+matchLength) < *(match+matchLength)) {
+ *ptr0 = delta0;
+ ptr0 = &DELTANEXTMAXD(matchIndex*2);
+ if (*ptr0 == (U16)-1) break;
+ delta0 = *ptr0;
+ delta1 += delta0;
+ matchIndex -= delta0;
+ } else {
+ *ptr1 = delta1;
+ ptr1 = &DELTANEXTMAXD(matchIndex*2+1);
+ if (*ptr1 == (U16)-1) break;
+ delta1 = *ptr1;
+ delta0 += delta1;
+ matchIndex -= delta1;
+ }
+ }
+
+ *ptr0 = (U16)-1;
+ *ptr1 = (U16)-1;
+ if (matchNum) *matchNum = mnum;
+ /* if (best_mlen > 8) return best_mlen-8; */
+ if (!matchNum) return 1;
+ return 1;
+}
+
+
+FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit)
+{
+ const BYTE* const base = ctx->base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = ctx->nextToUpdate;
+ while(idx < target)
+ idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL);
+}
+
+
+/** Tree updater, providing best match */
+FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
+ LZ4HC_CCtx_internal* ctx,
+ const BYTE* const ip, const BYTE* const iHighLimit,
+ size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
+{
+ int mnum = 0;
+ if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */
+ if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit);
+ best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum);
+ ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen);
+ return mnum;
+}
+
+
+#define SET_PRICE(pos, mlen, offset, litlen, price) \
+{ \
+ while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \
+ opt[pos].mlen = (int)mlen; \
+ opt[pos].off = (int)offset; \
+ opt[pos].litlen = (int)litlen; \
+ opt[pos].price = (int)price; \
+}
+
+
+static int LZ4HC_compress_optimal (
+ LZ4HC_CCtx_internal* ctx,
+ const char* const source,
+ char* dest,
+ int inputSize,
+ int maxOutputSize,
+ limitedOutput_directive limit,
+ const size_t sufficient_len,
+ const int fullUpdate
+ )
+{
+ LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];
+ LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
+ const BYTE *inr = NULL;
+ size_t res, cur, cur2;
+ size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
+
+ const BYTE* ip = (const BYTE*) source;
+ const BYTE* anchor = ip;
+ const BYTE* const iend = ip + inputSize;
+ const BYTE* const mflimit = iend - MFLIMIT;
+ const BYTE* const matchlimit = (iend - LASTLITERALS);
+ BYTE* op = (BYTE*) dest;
+ BYTE* const oend = op + maxOutputSize;
+
+ /* init */
+ ctx->end += inputSize;
+ ip++;
+
+ /* Main Loop */
+ while (ip < mflimit) {
+ memset(opt, 0, sizeof(LZ4HC_optimal_t));
+ last_pos = 0;
+ llen = ip - anchor;
+ match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
+ if (!match_num) { ip++; continue; }
+
+ if ((size_t)matches[match_num-1].len > sufficient_len) {
+ best_mlen = matches[match_num-1].len;
+ best_off = matches[match_num-1].off;
+ cur = 0;
+ last_pos = 1;
+ goto encode;
+ }
+
+ /* set prices using matches at position = 0 */
+ for (i = 0; i < match_num; i++) {
+ mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
+ best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM;
+ while (mlen <= best_mlen) {
+ litlen = 0;
+ price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
+ SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
+ mlen++;
+ }
+ }
+
+ if (last_pos < MINMATCH) { ip++; continue; }
+
+ /* check further positions */
+ opt[0].mlen = opt[1].mlen = 1;
+ for (cur = 1; cur <= last_pos; cur++) {
+ inr = ip + cur;
+
+ if (opt[cur-1].mlen == 1) {
+ litlen = opt[cur-1].litlen + 1;
+ if (cur != litlen) {
+ price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
+ } else {
+ price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
+ }
+ } else {
+ litlen = 1;
+ price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen);
+ }
+
+ mlen = 1;
+ best_mlen = 0;
+ if (cur > last_pos || price < (size_t)opt[cur].price)
+ SET_PRICE(cur, mlen, best_mlen, litlen, price);
+
+ if (cur == last_pos || inr >= mflimit) break;
+
+ match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
+ if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
+ best_mlen = matches[match_num-1].len;
+ best_off = matches[match_num-1].off;
+ last_pos = cur + 1;
+ goto encode;
+ }
+
+ /* set prices using matches at position = cur */
+ for (i = 0; i < match_num; i++) {
+ mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
+ cur2 = cur;
+ best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2;
+
+ while (mlen <= best_mlen) {
+ if (opt[cur2].mlen == 1) {
+ litlen = opt[cur2].litlen;
+
+ if (cur2 != litlen)
+ price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen);
+ else
+ price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
+ } else {
+ litlen = 0;
+ price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
+ }
+
+ if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
+ SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
+ }
+ mlen++;
+ }
+ }
+ } /* for (cur = 1; cur <= last_pos; cur++) */
+
+ best_mlen = opt[last_pos].mlen;
+ best_off = opt[last_pos].off;
+ cur = last_pos - best_mlen;
+
+encode: /* cur, last_pos, best_mlen, best_off have to be set */
+ opt[0].mlen = 1;
+ while (1) {
+ mlen = opt[cur].mlen;
+ offset = opt[cur].off;
+ opt[cur].mlen = (int)best_mlen;
+ opt[cur].off = (int)best_off;
+ best_mlen = mlen;
+ best_off = offset;
+ if (mlen > cur) break;
+ cur -= mlen;
+ }
+
+ cur = 0;
+ while (cur < last_pos) {
+ mlen = opt[cur].mlen;
+ if (mlen == 1) { ip++; cur++; continue; }
+ offset = opt[cur].off;
+ cur += mlen;
+
+ res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend);
+ if (res) return 0;
+ }
+ }
+
+ /* Encode Last Literals */
+ { int lastRun = (int)(iend - anchor);
+ if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
+ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
+ else *op++ = (BYTE)(lastRun<<ML_BITS);
+ memcpy(op, anchor, iend - anchor);
+ op += iend-anchor;
+ }
+
+ /* End */
+ return (int) ((char*)op-dest);
+}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/Makefile b/c-blosc/internal-complibs/zstd-1.0.0/Makefile
deleted file mode 100644
index 35522da..0000000
--- a/c-blosc/internal-complibs/zstd-1.0.0/Makefile
+++ /dev/null
@@ -1,115 +0,0 @@
-# ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ################################################################
-
-# Version numbers
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
-LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
-LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
-LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
-LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
-LIBVER := $(shell echo $(LIBVER_SCRIPT))
-VERSION?= $(LIBVER)
-
-DESTDIR?=
-PREFIX ?= /usr/local
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR=$(PREFIX)/include
-
-CPPFLAGS= -I. -I./common
-CFLAGS ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
- -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
-FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
-
-
-ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
-
-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
-CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
-else
-ZSTD_FILES+= legacy/*.c
-CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
-endif
-
-
-# OS X linker doesn't support -soname, and use different extension
-# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
-ifeq ($(shell uname), Darwin)
- SHARED_EXT = dylib
- SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
- SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
- SONAME_FLAGS = -install_name $(PREFIX)/lib/$@.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
-else
- SONAME_FLAGS = -Wl,-soname=$@.$(SHARED_EXT).$(LIBVER_MAJOR)
- SHARED_EXT = so
- SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
- SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
-endif
-
-
-.PHONY: default all clean install uninstall
-
-default: clean libzstd
-
-all: clean libzstd
-
-libzstd: $(ZSTD_FILES)
- @echo compiling static library
- @$(CC) $(FLAGS) -c $^
- @$(AR) rcs $@.a *.o
- @echo compiling dynamic library $(LIBVER)
- @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
- @echo creating versioned links
- @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
- @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
-
-clean:
- @rm -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
- @rm -f decompress/*.o
- @echo Cleaning library completed
-
-#------------------------------------------------------------------------
-#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly))
-
-libzstd.pc:
-libzstd.pc: libzstd.pc.in
- @echo creating pkgconfig
- @sed -e 's|@PREFIX@|$(PREFIX)|' \
- -e 's|@LIBDIR@|$(LIBDIR)|' \
- -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
- -e 's|@VERSION@|$(VERSION)|' \
- $< >$@
-
-install: libzstd libzstd.pc
- @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
- @install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
- @cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
- @cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
- @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
- @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
- @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
- @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
- @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
- @echo zstd static and shared library installed
-
-uninstall:
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
- $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
- [ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
- @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
- @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
- @[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h
- @[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
- @echo zstd libraries successfully uninstalled
-
-endif
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/compress/zbuff_compress.c b/c-blosc/internal-complibs/zstd-1.0.0/compress/zbuff_compress.c
deleted file mode 100644
index 5095b43..0000000
--- a/c-blosc/internal-complibs/zstd-1.0.0/compress/zbuff_compress.c
+++ /dev/null
@@ -1,319 +0,0 @@
-/**
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- */
-
-
-
-/* *************************************
-* Dependencies
-***************************************/
-#include <stdlib.h>
-#include "error_private.h"
-#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
-#define ZBUFF_STATIC_LINKING_ONLY
-#include "zbuff.h"
-
-
-/* *************************************
-* Constants
-***************************************/
-static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
-
-
-/*-***********************************************************
-* Streaming compression
-*
-* A ZBUFF_CCtx object is required to track streaming operation.
-* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
-* Use ZBUFF_compressInit() to start a new compression operation.
-* ZBUFF_CCtx objects can be reused multiple times.
-*
-* Use ZBUFF_compressContinue() repetitively to consume your input.
-* *srcSizePtr and *dstCapacityPtr can be any size.
-* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
-* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
-* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
-* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
-* or an error code, which can be tested using ZBUFF_isError().
-*
-* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
-* Note that it will not output more than *dstCapacityPtr.
-* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
-* @return : nb of bytes still present into internal buffer (0 if it's empty)
-* or an error code, which can be tested using ZBUFF_isError().
-*
-* ZBUFF_compressEnd() instructs to finish a frame.
-* It will perform a flush and write frame epilogue.
-* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
-* @return : nb of bytes still present into internal buffer (0 if it's empty)
-* or an error code, which can be tested using ZBUFF_isError().
-*
-* Hint : recommended buffer sizes (not compulsory)
-* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
-* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
-* ***********************************************************/
-
-typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
-
-/* *** Resources *** */
-struct ZBUFF_CCtx_s {
- ZSTD_CCtx* zc;
- char* inBuff;
- size_t inBuffSize;
- size_t inToCompress;
- size_t inBuffPos;
- size_t inBuffTarget;
- size_t blockSize;
- char* outBuff;
- size_t outBuffSize;
- size_t outBuffContentSize;
- size_t outBuffFlushedSize;
- ZBUFF_cStage stage;
- U32 checksum;
- U32 frameEnded;
- ZSTD_customMem customMem;
-}; /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */
-
-ZBUFF_CCtx* ZBUFF_createCCtx(void)
-{
- return ZBUFF_createCCtx_advanced(defaultCustomMem);
-}
-
-ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
-{
- ZBUFF_CCtx* zbc;
-
- if (!customMem.customAlloc && !customMem.customFree)
- customMem = defaultCustomMem;
-
- if (!customMem.customAlloc || !customMem.customFree)
- return NULL;
-
- zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
- if (zbc==NULL) return NULL;
- memset(zbc, 0, sizeof(ZBUFF_CCtx));
- memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
- zbc->zc = ZSTD_createCCtx_advanced(customMem);
- if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
- return zbc;
-}
-
-size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
-{
- if (zbc==NULL) return 0; /* support free on NULL */
- ZSTD_freeCCtx(zbc->zc);
- if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
- if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
- zbc->customMem.customFree(zbc->customMem.opaque, zbc);
- return 0;
-}
-
-
-/* ====== Initialization ====== */
-
-size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
- const void* dict, size_t dictSize,
- ZSTD_parameters params, unsigned long long pledgedSrcSize)
-{
- /* allocate buffers */
- { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
- if (zbc->inBuffSize < neededInBuffSize) {
- zbc->inBuffSize = neededInBuffSize;
- zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
- zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
- if (zbc->inBuff == NULL) return ERROR(memory_allocation);
- }
- zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
- }
- if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
- zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
- zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
- zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
- if (zbc->outBuff == NULL) return ERROR(memory_allocation);
- }
-
- { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
- if (ZSTD_isError(errorCode)) return errorCode; }
-
- zbc->inToCompress = 0;
- zbc->inBuffPos = 0;
- zbc->inBuffTarget = zbc->blockSize;
- zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
- zbc->stage = ZBUFFcs_load;
- zbc->checksum = params.fParams.checksumFlag > 0;
- zbc->frameEnded = 0;
- return 0; /* ready to go */
-}
-
-
-size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
-{
- ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
- return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
-}
-
-size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
-{
- return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
-}
-
-
-/* internal util function */
-MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- size_t const length = MIN(dstCapacity, srcSize);
- memcpy(dst, src, length);
- return length;
-}
-
-
-/* ====== Compression ====== */
-
-typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e;
-
-static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
- void* dst, size_t* dstCapacityPtr,
- const void* src, size_t* srcSizePtr,
- ZBUFF_flush_e const flush)
-{
- U32 someMoreWork = 1;
- const char* const istart = (const char*)src;
- const char* const iend = istart + *srcSizePtr;
- const char* ip = istart;
- char* const ostart = (char*)dst;
- char* const oend = ostart + *dstCapacityPtr;
- char* op = ostart;
-
- while (someMoreWork) {
- switch(zbc->stage)
- {
- case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
-
- case ZBUFFcs_load:
- /* complete inBuffer */
- { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
- size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
- zbc->inBuffPos += loaded;
- ip += loaded;
- if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
- someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */
- } }
- /* compress current block (note : this stage cannot be stopped in the middle) */
- { void* cDst;
- size_t cSize;
- size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
- size_t oSize = oend-op;
- if (oSize >= ZSTD_compressBound(iSize))
- cDst = op; /* compress directly into output buffer (avoid flush stage) */
- else
- cDst = zbc->outBuff, oSize = zbc->outBuffSize;
- cSize = (flush == zbf_end) ?
- ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) :
- ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
- if (ZSTD_isError(cSize)) return cSize;
- if (flush == zbf_end) zbc->frameEnded = 1;
- /* prepare next block */
- zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
- if (zbc->inBuffTarget > zbc->inBuffSize)
- zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */
- zbc->inToCompress = zbc->inBuffPos;
- if (cDst == op) { op += cSize; break; } /* no need to flush */
- zbc->outBuffContentSize = cSize;
- zbc->outBuffFlushedSize = 0;
- zbc->stage = ZBUFFcs_flush; /* continue to flush stage */
- }
-
- case ZBUFFcs_flush:
- { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
- size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
- op += flushed;
- zbc->outBuffFlushedSize += flushed;
- if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
- zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
- zbc->stage = ZBUFFcs_load;
- break;
- }
-
- case ZBUFFcs_final:
- someMoreWork = 0; /* do nothing */
- break;
-
- default:
- return ERROR(GENERIC); /* impossible */
- }
- }
-
- *srcSizePtr = ip - istart;
- *dstCapacityPtr = op - ostart;
- if (zbc->frameEnded) return 0;
- { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
- if (hintInSize==0) hintInSize = zbc->blockSize;
- return hintInSize;
- }
-}
-
-size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
- void* dst, size_t* dstCapacityPtr,
- const void* src, size_t* srcSizePtr)
-{
- return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather);
-}
-
-
-
-/* ====== Finalize ====== */
-
-size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
-{
- size_t srcSize = 0;
- ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush); /* use a valid src address instead of NULL */
- return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
-}
-
-
-size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
-{
- BYTE* const ostart = (BYTE*)dst;
- BYTE* const oend = ostart + *dstCapacityPtr;
- BYTE* op = ostart;
-
- if (zbc->stage != ZBUFFcs_final) {
- /* flush whatever remains */
- size_t outSize = *dstCapacityPtr;
- size_t srcSize = 0;
- size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end); /* use a valid address instead of NULL */
- size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
- op += outSize;
- if (remainingToFlush) {
- *dstCapacityPtr = op-ostart;
- return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4);
- }
- /* create epilogue */
- zbc->stage = ZBUFFcs_final;
- zbc->outBuffContentSize = !notEnded ? 0 :
- ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* write epilogue into outBuff */
- }
-
- /* flush epilogue */
- { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
- size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
- op += flushed;
- zbc->outBuffFlushedSize += flushed;
- *dstCapacityPtr = op-ostart;
- if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */
- return toFlush - flushed;
- }
-}
-
-
-
-/* *************************************
-* Tool functions
-***************************************/
-size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
-size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/decompress/zbuff_decompress.c b/c-blosc/internal-complibs/zstd-1.0.0/decompress/zbuff_decompress.c
deleted file mode 100644
index b20ee97..0000000
--- a/c-blosc/internal-complibs/zstd-1.0.0/decompress/zbuff_decompress.c
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- */
-
-
-
-/* *************************************
-* Dependencies
-***************************************/
-#include <stdlib.h>
-#include "error_private.h"
-#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
-#define ZBUFF_STATIC_LINKING_ONLY
-#include "zbuff.h"
-
-
-typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
- ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
-
-/* *** Resource management *** */
-struct ZBUFF_DCtx_s {
- ZSTD_DCtx* zd;
- ZSTD_frameParams fParams;
- ZBUFF_dStage stage;
- char* inBuff;
- size_t inBuffSize;
- size_t inPos;
- char* outBuff;
- size_t outBuffSize;
- size_t outStart;
- size_t outEnd;
- size_t blockSize;
- BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
- size_t lhSize;
- ZSTD_customMem customMem;
-}; /* typedef'd to ZBUFF_DCtx within "zbuff.h" */
-
-
-ZBUFF_DCtx* ZBUFF_createDCtx(void)
-{
- return ZBUFF_createDCtx_advanced(defaultCustomMem);
-}
-
-ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
-{
- ZBUFF_DCtx* zbd;
-
- if (!customMem.customAlloc && !customMem.customFree)
- customMem = defaultCustomMem;
-
- if (!customMem.customAlloc || !customMem.customFree)
- return NULL;
-
- zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
- if (zbd==NULL) return NULL;
- memset(zbd, 0, sizeof(ZBUFF_DCtx));
- memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
- zbd->zd = ZSTD_createDCtx_advanced(customMem);
- if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
- zbd->stage = ZBUFFds_init;
- return zbd;
-}
-
-size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
-{
- if (zbd==NULL) return 0; /* support free on null */
- ZSTD_freeDCtx(zbd->zd);
- if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
- if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
- zbd->customMem.customFree(zbd->customMem.opaque, zbd);
- return 0;
-}
-
-
-/* *** Initialization *** */
-
-size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
-{
- zbd->stage = ZBUFFds_loadHeader;
- zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
- return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
-}
-
-size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
-{
- return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
-}
-
-
-/* internal util function */
-MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
-{
- size_t const length = MIN(dstCapacity, srcSize);
- memcpy(dst, src, length);
- return length;
-}
-
-
-/* *** Decompression *** */
-
-size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
- void* dst, size_t* dstCapacityPtr,
- const void* src, size_t* srcSizePtr)
-{
- const char* const istart = (const char*)src;
- const char* const iend = istart + *srcSizePtr;
- const char* ip = istart;
- char* const ostart = (char*)dst;
- char* const oend = ostart + *dstCapacityPtr;
- char* op = ostart;
- U32 someMoreWork = 1;
-
- while (someMoreWork) {
- switch(zbd->stage)
- {
- case ZBUFFds_init :
- return ERROR(init_missing);
-
- case ZBUFFds_loadHeader :
- { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
- if (ZSTD_isError(hSize)) return hSize;
- if (hSize != 0) { /* need more input */
- size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
- if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
- memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
- zbd->lhSize += iend-ip;
- *dstCapacityPtr = 0;
- return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
- }
- memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
- break;
- } }
-
- /* Consume header */
- { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
- size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
- if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
- if (h1Size < zbd->lhSize) { /* long header */
- size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
- if (ZSTD_isError(h2Result)) return h2Result;
- } }
-
- zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
-
- /* Frame header instruct buffer sizes */
- { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
- size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
- zbd->blockSize = blockSize;
- if (zbd->inBuffSize < blockSize) {
- zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
- zbd->inBuffSize = blockSize;
- zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
- if (zbd->inBuff == NULL) return ERROR(memory_allocation);
- }
- if (zbd->outBuffSize < neededOutSize) {
- zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
- zbd->outBuffSize = neededOutSize;
- zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
- if (zbd->outBuff == NULL) return ERROR(memory_allocation);
- } }
- zbd->stage = ZBUFFds_read;
- /* pass-through */
-
- case ZBUFFds_read:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- if (neededInSize==0) { /* end of frame */
- zbd->stage = ZBUFFds_init;
- someMoreWork = 0;
- break;
- }
- if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
- const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
- size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
- zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
- ip, neededInSize);
- if (ZSTD_isError(decodedSize)) return decodedSize;
- ip += neededInSize;
- if (!decodedSize && !isSkipFrame) break; /* this was just a header */
- zbd->outEnd = zbd->outStart + decodedSize;
- zbd->stage = ZBUFFds_flush;
- break;
- }
- if (ip==iend) { someMoreWork = 0; break; } /* no more input */
- zbd->stage = ZBUFFds_load;
- /* pass-through */
- }
-
- case ZBUFFds_load:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
- size_t loadedSize;
- if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
- loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
- ip += loadedSize;
- zbd->inPos += loadedSize;
- if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
-
- /* decode loaded input */
- { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
- size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
- zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
- zbd->inBuff, neededInSize);
- if (ZSTD_isError(decodedSize)) return decodedSize;
- zbd->inPos = 0; /* input is consumed */
- if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
- zbd->outEnd = zbd->outStart + decodedSize;
- zbd->stage = ZBUFFds_flush;
- /* pass-through */
- } }
-
- case ZBUFFds_flush:
- { size_t const toFlushSize = zbd->outEnd - zbd->outStart;
- size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
- op += flushedSize;
- zbd->outStart += flushedSize;
- if (flushedSize == toFlushSize) { /* flush completed */
- zbd->stage = ZBUFFds_read;
- if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
- zbd->outStart = zbd->outEnd = 0;
- break;
- }
- /* cannot flush everything */
- someMoreWork = 0;
- break;
- }
- default: return ERROR(GENERIC); /* impossible */
- } }
-
- /* result */
- *srcSizePtr = ip-istart;
- *dstCapacityPtr = op-ostart;
- { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
- if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */
- nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block);
- if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
- nextSrcSizeHint -= zbd->inPos; /* already loaded*/
- return nextSrcSizeHint;
- }
-}
-
-
-/* *************************************
-* Tool functions
-***************************************/
-size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; }
-size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/zdict.h b/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/zdict.h
deleted file mode 100644
index c84aedd..0000000
--- a/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/zdict.h
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- */
-
-#ifndef DICTBUILDER_H_001
-#define DICTBUILDER_H_001
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/*====== Dependencies ======*/
-#include <stddef.h> /* size_t */
-
-
-/*====== Export for Windows ======*/
-/*!
-* ZSTD_DLL_EXPORT :
-* Enable exporting of functions when building a Windows DLL
-*/
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZDICTLIB_API __declspec(dllexport)
-#else
-# define ZDICTLIB_API
-#endif
-
-
-/*! ZDICT_trainFromBuffer() :
- Train a dictionary from an array of samples.
- Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
- supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
- The resulting dictionary will be saved into `dictBuffer`.
- @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
- or an error code, which can be tested with ZDICT_isError().
- Tips : In general, a reasonable dictionary has a size of ~ 100 KB.
- It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`.
- In general, it's recommended to provide a few thousands samples, but this can vary a lot.
- It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
-*/
-ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
-
-
-/*====== Helper functions ======*/
-ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
-ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
-ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
-
-
-
-#ifdef ZDICT_STATIC_LINKING_ONLY
-
-/* ====================================================================================
- * The definitions in this section are considered experimental.
- * They should never be used with a dynamic library, as they may change in the future.
- * They are provided for advanced usages.
- * Use them only in association with static linking.
- * ==================================================================================== */
-
-typedef struct {
- unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
- int compressionLevel; /* 0 means default; target a specific zstd compression level */
- unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
- unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
- unsigned reserved[2]; /* space for future parameters */
-} ZDICT_params_t;
-
-
-/*! ZDICT_trainFromBuffer_advanced() :
- Same as ZDICT_trainFromBuffer() with control over more parameters.
- `parameters` is optional and can be provided with values set to 0 to mean "default".
- @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`),
- or an error code, which can be tested by ZDICT_isError().
- note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
-*/
-size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
- ZDICT_params_t parameters);
-
-
-/*! ZDICT_addEntropyTablesFromBuffer() :
-
- Given a content-only dictionary (built using any 3rd party algorithm),
- add entropy tables computed from an array of samples.
- Samples must be stored concatenated in a flat buffer `samplesBuffer`,
- supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
-
- The input dictionary content must be stored *at the end* of `dictBuffer`.
- Its size is `dictContentSize`.
- The resulting dictionary with added entropy tables will be *written back to `dictBuffer`*,
- starting from its beginning.
- @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
-*/
-size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
-
-
-
-#endif /* ZDICT_STATIC_LINKING_ONLY */
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* DICTBUILDER_H_001 */
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/BUCK b/c-blosc/internal-complibs/zstd-1.1.3/BUCK
new file mode 100644
index 0000000..6812c1b
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/BUCK
@@ -0,0 +1,186 @@
+cxx_library(
+ name='zstd',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ deps=[
+ ':common',
+ ':compress',
+ ':decompress',
+ ':deprecated',
+ ],
+)
+
+cxx_library(
+ name='compress',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('compress', 'zstdmt_compress.h'),
+ ]),
+ headers=subdir_glob([
+ ('compress', 'zstd_opt.h'),
+ ]),
+ srcs=[
+ 'compress/zstd_compress.c',
+ 'compress/zstdmt_compress.c',
+ ],
+ deps=[':common'],
+)
+
+cxx_library(
+ name='decompress',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ srcs=['decompress/zstd_decompress.c'],
+ deps=[
+ ':common',
+ ':legacy',
+ ],
+)
+
+cxx_library(
+ name='deprecated',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('decprecated', '*.h'),
+ ]),
+ srcs=glob(['deprecated/*.c']),
+ deps=[':common'],
+)
+
+cxx_library(
+ name='legacy',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('legacy', '*.h'),
+ ]),
+ srcs=glob(['legacy/*.c']),
+ deps=[':common'],
+)
+
+cxx_library(
+ name='zdict',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('dictBuilder', 'zdict.h'),
+ ]),
+ headers=subdir_glob([
+ ('dictBuilder', 'divsufsort.h'),
+ ]),
+ srcs=glob(['dictBuilder/*.c']),
+ deps=[':common'],
+)
+
+cxx_library(
+ name='bitstream',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'bitstream.h'),
+ ]),
+)
+
+cxx_library(
+ name='entropy',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'fse.h'),
+ ('common', 'huf.h'),
+ ]),
+ srcs=[
+ 'common/entropy_common.c',
+ 'common/fse_decompress.c',
+ 'compress/fse_compress.c',
+ 'compress/huf_compress.c',
+ 'decompress/huf_decompress.c',
+ ],
+ deps=[
+ ':bitstream',
+ ':errors',
+ ':mem',
+ ],
+)
+
+cxx_library(
+ name='errors',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'error_private.h'),
+ ('common', 'zstd_errors.h'),
+ ]),
+ srcs=['common/error_private.c'],
+)
+
+cxx_library(
+ name='mem',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'mem.h'),
+ ]),
+)
+
+cxx_library(
+ name='pool',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'pool.h'),
+ ]),
+ srcs=['common/pool.c'],
+ deps=[':threading'],
+)
+
+cxx_library(
+ name='threading',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'threading.h'),
+ ]),
+ srcs=['common/threading.c'],
+)
+
+cxx_library(
+ name='xxhash',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('common', 'xxhash.h'),
+ ]),
+ srcs=['common/xxhash.c'],
+)
+
+cxx_library(
+ name='zstd_common',
+ header_namespace='',
+ visibility=['PUBLIC'],
+ exported_headers=subdir_glob([
+ ('', 'zstd.h'),
+ ('common', 'zstd_internal.h'),
+ ]),
+ srcs=['common/zstd_common.c'],
+ deps=[
+ ':errors',
+ ':mem',
+ ],
+)
+
+cxx_library(
+ name='common',
+ deps=[
+ ':bitstream',
+ ':entropy',
+ ':errors',
+ ':mem',
+ ':pool',
+ ':threading',
+ ':xxhash',
+ ':zstd_common',
+ ]
+)
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/Makefile b/c-blosc/internal-complibs/zstd-1.1.3/Makefile
new file mode 100644
index 0000000..05dd2bc
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/Makefile
@@ -0,0 +1,160 @@
+# ##########################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ##########################################################################
+
+# Version numbers
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
+LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
+LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
+LIBVER := $(shell echo $(LIBVER_SCRIPT))
+VERSION?= $(LIBVER)
+
+CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
+CFLAGS ?= -O3
+DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+ -Wstrict-prototypes -Wundef -Wpointer-arith
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS = $(CPPFLAGS) $(CFLAGS)
+
+
+ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
+
+ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
+CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
+else
+CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
+ZSTD_FILES+= $(wildcard legacy/*.c)
+endif
+
+ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES))
+
+# OS X linker doesn't support -soname, and use different extension
+# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
+ifeq ($(shell uname), Darwin)
+ SHARED_EXT = dylib
+ SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
+ SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
+ SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+else
+ SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
+ SHARED_EXT = so
+ SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
+ SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
+endif
+
+LIBZSTD = libzstd.$(SHARED_EXT_VER)
+
+
+.PHONY: default all clean install uninstall
+
+default: lib-release
+
+all: lib
+
+libzstd.a: ARFLAGS = rcs
+libzstd.a: $(ZSTD_OBJ)
+ @echo compiling static library
+ @$(AR) $(ARFLAGS) $@ $^
+
+$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
+$(LIBZSTD): $(ZSTD_FILES)
+ @echo compiling dynamic library $(LIBVER)
+ifneq (,$(filter Windows%,$(OS)))
+ @$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -shared $^ -o dll\libzstd.dll
+ dlltool -D dll\libzstd.dll -d dll\libzstd.def -l dll\libzstd.lib
+else
+ @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+ @echo creating versioned links
+ @ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
+ @ln -sf $@ libzstd.$(SHARED_EXT)
+endif
+
+libzstd : $(LIBZSTD)
+
+lib: libzstd.a libzstd
+
+lib-release: DEBUGFLAGS :=
+lib-release: lib
+
+clean:
+ @$(RM) -r *.dSYM # Mac OS-X specific
+ @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+ @$(RM) dll/libzstd.dll dll/libzstd.lib
+ @$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
+ @echo Cleaning library completed
+
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX ?= /usr/local
+DESTDIR ?=
+LIBDIR ?= $(PREFIX)/lib
+INCLUDEDIR ?= $(PREFIX)/include
+
+ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
+PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+else
+PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+endif
+
+INSTALL_LIB ?= $(INSTALL) -m 755
+INSTALL_DATA ?= $(INSTALL) -m 644
+
+
+libzstd.pc:
+libzstd.pc: libzstd.pc.in
+ @echo creating pkgconfig
+ @sed -e 's|@PREFIX@|$(PREFIX)|' \
+ -e 's|@LIBDIR@|$(LIBDIR)|' \
+ -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
+ -e 's|@VERSION@|$(VERSION)|' \
+ $< >$@
+
+install: libzstd.a libzstd libzstd.pc
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/
+ @$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
+ @echo Installing libraries
+ @$(INSTALL_LIB) libzstd.a $(DESTDIR)$(LIBDIR)
+ @$(INSTALL_LIB) libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
+ @ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+ @ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+ @echo Installing includes
+ @$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
+ @$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
+ @$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings
+ @$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
+ @echo zstd static and shared library installed
+
+uninstall:
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
+ @$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
+ @echo zstd libraries successfully uninstalled
+
+endif
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/README.md b/c-blosc/internal-complibs/zstd-1.1.3/README.md
similarity index 72%
rename from c-blosc/internal-complibs/zstd-1.0.0/README.md
rename to c-blosc/internal-complibs/zstd-1.1.3/README.md
index efcbdc6..3357e3d 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/README.md
+++ b/c-blosc/internal-complibs/zstd-1.1.3/README.md
@@ -43,12 +43,28 @@ Other optional functionalities provided are :
For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` .
+#### Using MinGW+MSYS to create DLL
+
+DLL can be created using MinGW+MSYS with the `make libzstd` command.
+This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`.
+The import library is only required with Visual C++.
+The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to
+compile a project using gcc/MinGW.
+The dynamic library has to be added to linking options.
+It means that if a project that uses ZSTD consists of a single `test-dll.c`
+file it should be linked with `dll\libzstd.dll`. For example:
+```
+ gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll
+```
+The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
+
+
#### Obsolete streaming API
Streaming is now provided within `zstd.h`.
-Older streaming API is still provided within `common/zbuff.h`.
-It is considered obsolete, and will be removed in a future version.
-Consider migrating towards newer streaming API.
+Older streaming API is still available within `deprecated/zbuff.h`.
+It will be removed in a future version.
+Consider migrating code towards newer streaming API in `zstd.h`.
#### Miscellaneous
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/bitstream.h b/c-blosc/internal-complibs/zstd-1.1.3/common/bitstream.h
similarity index 97%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/bitstream.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/bitstream.h
index e96798f..3a45244 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/bitstream.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/bitstream.h
@@ -266,7 +266,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
bitD->bitContainer = MEM_readLEST(bitD->ptr);
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
- bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
} else {
bitD->start = (const char*)srcBuffer;
@@ -298,7 +298,7 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
{
-#if defined(__BMI__) && defined(__GNUC__) /* experimental */
+#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */
# if defined(__x86_64__)
if (sizeof(bitContainer)==8)
return _bextr_u64(bitContainer, start, nbBits);
@@ -367,10 +367,10 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
}
/*! BIT_reloadDStream() :
-* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
+* Refill `bitD` from buffer previously set in BIT_initDStream() .
* This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
- if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+ if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/entropy_common.c b/c-blosc/internal-complibs/zstd-1.1.3/common/entropy_common.c
similarity index 91%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/entropy_common.c
rename to c-blosc/internal-complibs/zstd-1.1.3/common/entropy_common.c
index acd9669..83fd971 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/entropy_common.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/entropy_common.c
@@ -159,6 +159,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
+ `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
@@ -168,9 +169,11 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -185,23 +188,25 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
huffWeight[n+1] = ip[n/2] & 15;
} } }
else { /* header compressed with FSE (normal case) */
+ FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
- oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
+ oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
}
/* collect weight stats */
- memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
+ memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) {
- if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
+ if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
- if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
+ if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
*tableLogPtr = tableLog;
/* determine last weight */
{ U32 const total = 1 << tableLog;
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/error_private.h b/c-blosc/internal-complibs/zstd-1.1.3/common/error_private.c
similarity index 50%
copy from c-blosc/internal-complibs/zstd-1.0.0/common/error_private.h
copy to c-blosc/internal-complibs/zstd-1.1.3/common/error_private.c
index d27e15a..a0fa172 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/error_private.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/error_private.c
@@ -7,64 +7,13 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-/* Note : this module is expected to remain private, do not expose it */
+/* The purpose of this file is to have a single list of error strings embedded in binary */
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
+#include "error_private.h"
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* Dependencies
-******************************************/
-#include <stddef.h> /* size_t */
-#include "error_public.h" /* enum list */
-
-
-/* ****************************************
-* Compiler-specific
-******************************************/
-#if defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/*-****************************************
-* Customization (error_public.h)
-******************************************/
-typedef ZSTD_ErrorCode ERR_enum;
-#define PREFIX(name) ZSTD_error_##name
-
-
-/*-****************************************
-* Error codes handling
-******************************************/
-#ifdef ERROR
-# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
-#endif
-#define ERROR(name) ((size_t)-PREFIX(name))
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
-
-
-/*-****************************************
-* Error Strings
-******************************************/
-
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
+const char* ERR_getErrorString(ERR_enum code)
{
- static const char* notErrorCode = "Unspecified error code";
+ static const char* const notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
@@ -74,6 +23,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
case PREFIX(parameter_unknown): return "Unknown parameter type";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
+ case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
@@ -91,14 +41,3 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
default: return notErrorCode;
}
}
-
-ERR_STATIC const char* ERR_getErrorName(size_t code)
-{
- return ERR_getErrorString(ERR_getErrorCode(code));
-}
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/error_private.h b/c-blosc/internal-complibs/zstd-1.1.3/common/error_private.h
similarity index 53%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/error_private.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/error_private.h
index d27e15a..1bc2e49 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/error_private.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/error_private.h
@@ -21,7 +21,7 @@ extern "C" {
* Dependencies
******************************************/
#include <stddef.h> /* size_t */
-#include "error_public.h" /* enum list */
+#include "zstd_errors.h" /* enum list */
/* ****************************************
@@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu
* Error Strings
******************************************/
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
-{
- static const char* notErrorCode = "Unspecified error code";
- switch( code )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(version_unsupported): return "Version not supported";
- case PREFIX(parameter_unknown): return "Unknown parameter type";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
- case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size incorrect";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(dictionary_wrong): return "Dictionary mismatch";
- case PREFIX(maxCode):
- default: return notErrorCode;
- }
-}
+const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/fse.h b/c-blosc/internal-complibs/zstd-1.1.3/common/fse.h
similarity index 90%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/fse.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/fse.h
index 720d54b..8b07d18 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/fse.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/fse.h
@@ -286,7 +286,7 @@ If there is an error, the function will return an error code, which can be teste
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
-/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
@@ -294,37 +294,72 @@ If there is an error, the function will return an error code, which can be teste
/* *****************************************
* FSE advanced API
*******************************************/
+/* FSE_count_wksp() :
+ * Same as FSE_count(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= `1024` unsigned
+ */
+size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize, unsigned* workSpace);
+
+/** FSE_countFast() :
+ * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
+ */
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
-/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
+
+/* FSE_countFast_wksp() :
+ * Same as FSE_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` must be a table of minimum `1024` unsigned
+ */
+size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
+
+/*! FSE_count_simple
+ * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
+ * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
+*/
+size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
+
+
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */
+/* FSE_compress_wksp() :
+ * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
+ * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
+ */
+#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) )
+size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
-/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
+/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * `wkspSize` must be >= `(1<<tableLog)`.
+ */
+size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
+
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
-/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
+/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
+size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
+
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
- You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
- Visual seems to do it automatically.
- For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
- If none of these solutions is applicable, include "fse.c" directly.
+ Hence their body are included in next section.
*/
-typedef struct
-{
+typedef struct {
ptrdiff_t value;
const void* stateTable;
const void* symbolTT;
@@ -384,8 +419,7 @@ If there is an error, it returns an errorCode (which can be tested using FSE_isE
/* *****************************************
* FSE symbol decompression API
*******************************************/
-typedef struct
-{
+typedef struct {
size_t state;
const void* table; /* precise table may vary, depending on U16 */
} FSE_DState_t;
@@ -503,6 +537,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
BIT_flushBits(bitC);
}
+
/* ====== Decompression ====== */
typedef struct {
@@ -581,14 +616,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
-#define FSE_MAX_MEMORY_USAGE 14
-#define FSE_DEFAULT_MEMORY_USAGE 13
+#ifndef FSE_MAX_MEMORY_USAGE
+# define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+# define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
-#define FSE_MAX_SYMBOL_VALUE 255
-
+#ifndef FSE_MAX_SYMBOL_VALUE
+# define FSE_MAX_SYMBOL_VALUE 255
+#endif
/* **************************************************************
* template functions type & suffix
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/fse_decompress.c b/c-blosc/internal-complibs/zstd-1.1.3/common/fse_decompress.c
similarity index 88%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/fse_decompress.c
rename to c-blosc/internal-complibs/zstd-1.1.3/common/fse_decompress.c
index 032e657..1479a5e 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/fse_decompress.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/fse_decompress.c
@@ -42,12 +42,15 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -73,12 +76,6 @@
/* **************************************************************
-* Complex types
-****************************************************************/
-typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
-
-
-/* **************************************************************
* Templates
****************************************************************/
/*
@@ -297,28 +294,34 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
}
-size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
+size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
{
const BYTE* const istart = (const BYTE*)cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
- DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
- if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
-
/* normal FSE decoding mode */
- { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
- if (FSE_isError(NCountLength)) return NCountLength;
- if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
- ip += NCountLength;
- cSrcSize -= NCountLength;
- }
+ size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
+ if (FSE_isError(NCountLength)) return NCountLength;
+ //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
+ if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
+ ip += NCountLength;
+ cSrcSize -= NCountLength;
- CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) );
+ CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
- return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
+ return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
+}
+
+
+typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
+
+size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
+{
+ DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
+ return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/huf.h b/c-blosc/internal-complibs/zstd-1.1.3/common/huf.h
similarity index 85%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/huf.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/huf.h
index 29bab4b..9427ae8 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/huf.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/huf.h
@@ -62,21 +62,19 @@ size_t HUF_compress(void* dst, size_t dstCapacity,
HUF_decompress() :
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
into already allocated buffer 'dst', of minimum size 'dstSize'.
- `dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
+ `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
because it knows size to regenerate.
- @return : size of regenerated data (== dstSize),
+ @return : size of regenerated data (== originalSize),
or an error code, which can be tested using HUF_isError()
*/
-size_t HUF_decompress(void* dst, size_t dstSize,
+size_t HUF_decompress(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize);
-/* ****************************************
-* Tool functions
-******************************************/
-#define HUF_BLOCKSIZE_MAX (128 * 1024)
+/* *** Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
/* Error Management */
@@ -84,12 +82,18 @@ unsigned HUF_isError(size_t code); /**< tells if a return value is an
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
-/* *** Advanced function *** */
+/* *** Advanced function *** */
/** HUF_compress2() :
-* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */
+ * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` .
+ * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+/** HUF_compress4X_wksp() :
+* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
+size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */
+
+
#ifdef HUF_STATIC_LINKING_ONLY
@@ -98,7 +102,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
/* *** Constants *** */
-#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
+#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
#define HUF_SYMBOLVALUE_MAX 255
@@ -125,9 +129,9 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
typedef U32 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
- HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) }
+ HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
- HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) }
+ HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
/* ****************************************
@@ -141,10 +145,6 @@ size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, con
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
-size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
-size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
-
/* ****************************************
* HUF detailed API
@@ -169,6 +169,12 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, un
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
+ */
+size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
+
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@@ -208,16 +214,20 @@ size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* c
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
+size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
+size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
+
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-
#endif /* HUF_STATIC_LINKING_ONLY */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/mem.h b/c-blosc/internal-complibs/zstd-1.1.3/common/mem.h
similarity index 98%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/mem.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/mem.h
index 681dd35..aff044d 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/mem.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/mem.h
@@ -39,7 +39,7 @@ extern "C" {
#endif
/* code only tested on 32 and 64 bits systems */
-#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
+#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
@@ -55,14 +55,16 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
+ typedef intptr_t iPtrDiff;
#else
- typedef unsigned char BYTE;
+ typedef unsigned char BYTE;
typedef unsigned short U16;
typedef signed short S16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
+ typedef ptrdiff_t iPtrDiff;
#endif
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/common/pool.c b/c-blosc/internal-complibs/zstd-1.1.3/common/pool.c
new file mode 100644
index 0000000..e439fe1
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/pool.c
@@ -0,0 +1,194 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* ====== Dependencies ======= */
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, calloc, free */
+#include "pool.h"
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+#ifdef ZSTD_MULTITHREAD
+
+#include "threading.h" /* pthread adaptation */
+
+/* A job is a function and an opaque argument */
+typedef struct POOL_job_s {
+ POOL_function function;
+ void *opaque;
+} POOL_job;
+
+struct POOL_ctx_s {
+ /* Keep track of the threads */
+ pthread_t *threads;
+ size_t numThreads;
+
+ /* The queue is a circular buffer */
+ POOL_job *queue;
+ size_t queueHead;
+ size_t queueTail;
+ size_t queueSize;
+ /* The mutex protects the queue */
+ pthread_mutex_t queueMutex;
+ /* Condition variable for pushers to wait on when the queue is full */
+ pthread_cond_t queuePushCond;
+ /* Condition variables for poppers to wait on when the queue is empty */
+ pthread_cond_t queuePopCond;
+ /* Indicates if the queue is shutting down */
+ int shutdown;
+};
+
+/* POOL_thread() :
+ Work thread for the thread pool.
+ Waits for jobs and executes them.
+ @returns : NULL on failure else non-null.
+*/
+static void* POOL_thread(void* opaque) {
+ POOL_ctx* const ctx = (POOL_ctx*)opaque;
+ if (!ctx) { return NULL; }
+ for (;;) {
+ /* Lock the mutex and wait for a non-empty queue or until shutdown */
+ pthread_mutex_lock(&ctx->queueMutex);
+ while (ctx->queueHead == ctx->queueTail && !ctx->shutdown) {
+ pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+ }
+ /* empty => shutting down: so stop */
+ if (ctx->queueHead == ctx->queueTail) {
+ pthread_mutex_unlock(&ctx->queueMutex);
+ return opaque;
+ }
+ /* Pop a job off the queue */
+ { POOL_job const job = ctx->queue[ctx->queueHead];
+ ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+ /* Unlock the mutex, signal a pusher, and run the job */
+ pthread_mutex_unlock(&ctx->queueMutex);
+ pthread_cond_signal(&ctx->queuePushCond);
+ job.function(job.opaque);
+ }
+ }
+ /* Unreachable */
+}
+
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+ POOL_ctx *ctx;
+ /* Check the parameters */
+ if (!numThreads || !queueSize) { return NULL; }
+ /* Allocate the context and zero initialize */
+ ctx = (POOL_ctx *)calloc(1, sizeof(POOL_ctx));
+ if (!ctx) { return NULL; }
+ /* Initialize the job queue.
+ * It needs one extra space since one space is wasted to differentiate empty
+ * and full queues.
+ */
+ ctx->queueSize = queueSize + 1;
+ ctx->queue = (POOL_job *)malloc(ctx->queueSize * sizeof(POOL_job));
+ ctx->queueHead = 0;
+ ctx->queueTail = 0;
+ pthread_mutex_init(&ctx->queueMutex, NULL);
+ pthread_cond_init(&ctx->queuePushCond, NULL);
+ pthread_cond_init(&ctx->queuePopCond, NULL);
+ ctx->shutdown = 0;
+ /* Allocate space for the thread handles */
+ ctx->threads = (pthread_t *)malloc(numThreads * sizeof(pthread_t));
+ ctx->numThreads = 0;
+ /* Check for errors */
+ if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
+ /* Initialize the threads */
+ { size_t i;
+ for (i = 0; i < numThreads; ++i) {
+ if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+ ctx->numThreads = i;
+ POOL_free(ctx);
+ return NULL;
+ } }
+ ctx->numThreads = numThreads;
+ }
+ return ctx;
+}
+
+/*! POOL_join() :
+ Shutdown the queue, wake any sleeping threads, and join all of the threads.
+*/
+static void POOL_join(POOL_ctx *ctx) {
+ /* Shut down the queue */
+ pthread_mutex_lock(&ctx->queueMutex);
+ ctx->shutdown = 1;
+ pthread_mutex_unlock(&ctx->queueMutex);
+ /* Wake up sleeping threads */
+ pthread_cond_broadcast(&ctx->queuePushCond);
+ pthread_cond_broadcast(&ctx->queuePopCond);
+ /* Join all of the threads */
+ { size_t i;
+ for (i = 0; i < ctx->numThreads; ++i) {
+ pthread_join(ctx->threads[i], NULL);
+ } }
+}
+
+void POOL_free(POOL_ctx *ctx) {
+ if (!ctx) { return; }
+ POOL_join(ctx);
+ pthread_mutex_destroy(&ctx->queueMutex);
+ pthread_cond_destroy(&ctx->queuePushCond);
+ pthread_cond_destroy(&ctx->queuePopCond);
+ if (ctx->queue) free(ctx->queue);
+ if (ctx->threads) free(ctx->threads);
+ free(ctx);
+}
+
+void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
+ POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
+ if (!ctx) { return; }
+
+ pthread_mutex_lock(&ctx->queueMutex);
+ { POOL_job const job = {function, opaque};
+ /* Wait until there is space in the queue for the new job */
+ size_t newTail = (ctx->queueTail + 1) % ctx->queueSize;
+ while (ctx->queueHead == newTail && !ctx->shutdown) {
+ pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+ newTail = (ctx->queueTail + 1) % ctx->queueSize;
+ }
+ /* The queue is still going => there is space */
+ if (!ctx->shutdown) {
+ ctx->queue[ctx->queueTail] = job;
+ ctx->queueTail = newTail;
+ }
+ }
+ pthread_mutex_unlock(&ctx->queueMutex);
+ pthread_cond_signal(&ctx->queuePopCond);
+}
+
+#else /* ZSTD_MULTITHREAD not defined */
+/* No multi-threading support */
+
+/* We don't need any data, but if it is empty malloc() might return NULL. */
+struct POOL_ctx_s {
+ int data;
+};
+
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
+ (void)numThreads;
+ (void)queueSize;
+ return (POOL_ctx *)malloc(sizeof(POOL_ctx));
+}
+
+void POOL_free(POOL_ctx *ctx) {
+ if (ctx) free(ctx);
+}
+
+void POOL_add(void *ctx, POOL_function function, void *opaque) {
+ (void)ctx;
+ function(opaque);
+}
+
+#endif /* ZSTD_MULTITHREAD */
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/common/pool.h b/c-blosc/internal-complibs/zstd-1.1.3/common/pool.h
new file mode 100644
index 0000000..50cb25b
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/pool.h
@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#ifndef POOL_H
+#define POOL_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+#include <stddef.h> /* size_t */
+
+typedef struct POOL_ctx_s POOL_ctx;
+
+/*! POOL_create() :
+ Create a thread pool with at most `numThreads` threads.
+ `numThreads` must be at least 1.
+ The maximum number of queued jobs before blocking is `queueSize`.
+ `queueSize` must be at least 1.
+ @return : The POOL_ctx pointer on success else NULL.
+*/
+POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
+
+/*! POOL_free() :
+ Free a thread pool returned by POOL_create().
+*/
+void POOL_free(POOL_ctx *ctx);
+
+/*! POOL_function :
+ The function type that can be added to a thread pool.
+*/
+typedef void (*POOL_function)(void *);
+/*! POOL_add_function :
+ The function type for a generic thread pool add function.
+*/
+typedef void (*POOL_add_function)(void *, POOL_function, void *);
+
+/*! POOL_add() :
+ Add the job `function(opaque)` to the thread pool.
+ Possibly blocks until there is room in the queue.
+ Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed.
+*/
+void POOL_add(void *ctx, POOL_function function, void *opaque);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/common/threading.c b/c-blosc/internal-complibs/zstd-1.1.3/common/threading.c
new file mode 100644
index 0000000..b56e594
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/threading.c
@@ -0,0 +1,79 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/**
+ * This file will hold wrapper for systems, which do not support pthreads
+ */
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4206) /* disable: C4206: translation unit is empty (when ZSTD_MULTITHREAD is not defined) */
+#endif
+
+
+#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
+
+/**
+ * Windows minimalist Pthread Wrapper, based on :
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+
+
+/* === Dependencies === */
+#include <process.h>
+#include <errno.h>
+#include "threading.h"
+
+
+/* === Implementation === */
+
+static unsigned __stdcall worker(void *arg)
+{
+ pthread_t* const thread = (pthread_t*) arg;
+ thread->arg = thread->start_routine(thread->arg);
+ return 0;
+}
+
+int pthread_create(pthread_t* thread, const void* unused,
+ void* (*start_routine) (void*), void* arg)
+{
+ (void)unused;
+ thread->arg = arg;
+ thread->start_routine = start_routine;
+ thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
+
+ if (!thread->handle)
+ return errno;
+ else
+ return 0;
+}
+
+int _pthread_join(pthread_t * thread, void **value_ptr)
+{
+ DWORD result;
+
+ if (!thread->handle) return 0;
+
+ result = WaitForSingleObject(thread->handle, INFINITE);
+ switch (result) {
+ case WAIT_OBJECT_0:
+ if (value_ptr) *value_ptr = thread->arg;
+ return 0;
+ case WAIT_ABANDONED:
+ return EINVAL;
+ default:
+ return GetLastError();
+ }
+}
+
+#endif /* ZSTD_MULTITHREAD */
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/common/threading.h b/c-blosc/internal-complibs/zstd-1.1.3/common/threading.h
new file mode 100644
index 0000000..74b2ec0
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/threading.h
@@ -0,0 +1,104 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#ifndef THREADING_H_938743
+#define THREADING_H_938743
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
+
+/**
+ * Windows minimalist Pthread Wrapper, based on :
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+#ifdef WINVER
+# undef WINVER
+#endif
+#define WINVER 0x0600
+
+#ifdef _WIN32_WINNT
+# undef _WIN32_WINNT
+#endif
+#define _WIN32_WINNT 0x0600
+
+#ifndef WIN32_LEAN_AND_MEAN
+# define WIN32_LEAN_AND_MEAN
+#endif
+
+#include <windows.h>
+
+/* mutex */
+#define pthread_mutex_t CRITICAL_SECTION
+#define pthread_mutex_init(a,b) InitializeCriticalSection((a))
+#define pthread_mutex_destroy(a) DeleteCriticalSection((a))
+#define pthread_mutex_lock(a) EnterCriticalSection((a))
+#define pthread_mutex_unlock(a) LeaveCriticalSection((a))
+
+/* condition variable */
+#define pthread_cond_t CONDITION_VARIABLE
+#define pthread_cond_init(a, b) InitializeConditionVariable((a))
+#define pthread_cond_destroy(a) /* No delete */
+#define pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
+#define pthread_cond_signal(a) WakeConditionVariable((a))
+#define pthread_cond_broadcast(a) WakeAllConditionVariable((a))
+
+/* pthread_create() and pthread_join() */
+typedef struct {
+ HANDLE handle;
+ void* (*start_routine)(void*);
+ void* arg;
+} pthread_t;
+
+int pthread_create(pthread_t* thread, const void* unused,
+ void* (*start_routine) (void*), void* arg);
+
+#define pthread_join(a, b) _pthread_join(&(a), (b))
+int _pthread_join(pthread_t* thread, void** value_ptr);
+
+/**
+ * add here more wrappers as required
+ */
+
+
+#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection mathod */
+/* === POSIX Systems === */
+# include <pthread.h>
+
+#else /* ZSTD_MULTITHREAD not defined */
+/* No multithreading support */
+
+#define pthread_mutex_t int /* #define rather than typedef, as sometimes pthread support is implicit, resulting in duplicated symbols */
+#define pthread_mutex_init(a,b)
+#define pthread_mutex_destroy(a)
+#define pthread_mutex_lock(a)
+#define pthread_mutex_unlock(a)
+
+#define pthread_cond_t int
+#define pthread_cond_init(a,b)
+#define pthread_cond_destroy(a)
+#define pthread_cond_wait(a,b)
+#define pthread_cond_signal(a)
+#define pthread_cond_broadcast(a)
+
+/* do not use pthread_t */
+
+#endif /* ZSTD_MULTITHREAD */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* THREADING_H_938743 */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/xxhash.c b/c-blosc/internal-complibs/zstd-1.1.3/common/xxhash.c
similarity index 99%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/xxhash.c
rename to c-blosc/internal-complibs/zstd-1.1.3/common/xxhash.c
index f462c57..29e4fa6 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/xxhash.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/xxhash.c
@@ -115,7 +115,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# define FORCE_INLINE static __forceinline
#else
-# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/xxhash.h b/c-blosc/internal-complibs/zstd-1.1.3/common/xxhash.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/xxhash.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/xxhash.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/zstd_common.c b/c-blosc/internal-complibs/zstd-1.1.3/common/zstd_common.c
similarity index 84%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/zstd_common.c
rename to c-blosc/internal-complibs/zstd-1.1.3/common/zstd_common.c
index 54bc91c..749b287 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/zstd_common.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/zstd_common.c
@@ -16,7 +16,6 @@
#include "error_private.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
-#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
/*-****************************************
@@ -45,15 +44,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
-/* **************************************************************
-* ZBUFF Error Management
-****************************************************************/
-unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-
-const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
-
-
-
/*=**************************************************************
* Custom allocator
****************************************************************/
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/error_public.h b/c-blosc/internal-complibs/zstd-1.1.3/common/zstd_errors.h
similarity index 60%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/error_public.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/zstd_errors.h
index d46abd2..949dbd0 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/error_public.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/zstd_errors.h
@@ -7,8 +7,8 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
@@ -18,6 +18,20 @@ extern "C" {
#include <stddef.h> /* size_t */
+/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+# define ZSTDERRORLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#endif
+
/*-****************************************
* error codes list
******************************************/
@@ -29,6 +43,7 @@ typedef enum {
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
+ ZSTD_error_frameParameter_windowTooLarge,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
@@ -48,12 +63,12 @@ typedef enum {
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)
}
#endif
-#endif /* ERROR_PUBLIC_H_MODULE */
+#endif /* ZSTD_ERRORS_H_398273423 */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/zstd_internal.h b/c-blosc/internal-complibs/zstd-1.1.3/common/zstd_internal.h
similarity index 77%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/zstd_internal.h
rename to c-blosc/internal-complibs/zstd-1.1.3/common/zstd_internal.h
index 51e7170..4b56ce1 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/zstd_internal.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/common/zstd_internal.h
@@ -10,6 +10,38 @@
#ifndef ZSTD_CCOMMON_H_MODULE
#define ZSTD_CCOMMON_H_MODULE
+/*-*******************************************************
+* Compiler specifics
+*********************************************************/
+#ifdef _MSC_VER /* Visual Studio */
+# define FORCE_INLINE static __forceinline
+# include <intrin.h> /* For Visual 2005 */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4324) /* disable: C4324: padded structure */
+# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
+#else
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+# else
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
+#endif
+
+#ifdef _MSC_VER
+# define FORCE_NOINLINE static __declspec(noinline)
+#else
+# ifdef __GNUC__
+# define FORCE_NOINLINE static __attribute__((__noinline__))
+# else
+# define FORCE_NOINLINE static
+# endif
+#endif
+
+
/*-*************************************
* Dependencies
***************************************/
@@ -20,10 +52,12 @@
/*-*************************************
-* Common macros
+* shared macros
***************************************/
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
+#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */
+#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */
/*-*************************************
@@ -84,7 +118,8 @@ static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
-1,-1,-1,-1 };
-static const U32 LL_defaultNormLog = 6;
+#define LL_DEFAULTNORMLOG 6 /* for static allocation */
+static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -94,11 +129,13 @@ static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
-1,-1,-1,-1,-1 };
-static const U32 ML_defaultNormLog = 6;
+#define ML_DEFAULTNORMLOG 6 /* for static allocation */
+static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
-static const U32 OF_defaultNormLog = 5;
+#define OF_DEFAULTNORMLOG 5 /* for static allocation */
+static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
/*-*******************************************
@@ -110,7 +147,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
-MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
+MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -185,6 +222,7 @@ typedef struct {
U32 log2litSum;
U32 log2offCodeSum;
U32 factor;
+ U32 staticPrices;
U32 cachedPrice;
U32 cachedLitLength;
const BYTE* cachedLiterals;
@@ -197,7 +235,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
/* custom memory allocation functions */
void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
void ZSTD_defaultFreeFunction(void* opaque, void* address);
+#ifndef ZSTD_DLL_IMPORT
static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
+#endif
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
void ZSTD_free(void* ptr, ZSTD_customMem customMem);
@@ -227,4 +267,13 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
}
+/* hidden functions */
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
+
+
#endif /* ZSTD_CCOMMON_H_MODULE */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/compress/fse_compress.c b/c-blosc/internal-complibs/zstd-1.1.3/compress/fse_compress.c
similarity index 77%
rename from c-blosc/internal-complibs/zstd-1.0.0/compress/fse_compress.c
rename to c-blosc/internal-complibs/zstd-1.1.3/compress/fse_compress.c
index 386b2c0..6627fac 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/compress/fse_compress.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/compress/fse_compress.c
@@ -41,12 +41,15 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -68,12 +71,6 @@
/* **************************************************************
-* Complex types
-****************************************************************/
-typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
-
-
-/* **************************************************************
* Templates
****************************************************************/
/*
@@ -97,7 +94,13 @@ typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VA
/* Function templates */
-size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
+ * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
+ */
+size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
{
U32 const tableSize = 1 << tableLog;
U32 const tableMask = tableSize - 1;
@@ -108,10 +111,11 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
U32 const step = FSE_TABLESTEP(tableSize);
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
- FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
+ FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace;
U32 highThreshold = tableSize-1;
/* CTable header */
+ if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
tableU16[-2] = (U16) tableLog;
tableU16[-1] = (U16) maxSymbolValue;
@@ -178,6 +182,13 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
}
+size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+{
+ FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
+ return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
+}
+
+
#ifndef FSE_COMMONDEFS_ONLY
@@ -186,7 +197,7 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
****************************************************************/
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
- size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
+ size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
}
@@ -297,21 +308,20 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
* Counting histogram
****************************************************************/
/*! FSE_count_simple
- This function just counts byte values within `src`,
- and store the histogram into table `count`.
- This function is unsafe : it doesn't check that all values within `src` can fit into `count`.
+ This function counts byte values within `src`, and store the histogram into table `count`.
+ It doesn't use any additional memory.
+ But this function is unsafe : it doesn't check that all values within `src` can fit into `count`.
For this reason, prefer using a table `count` with 256 elements.
@return : count of most numerous element
*/
-static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
- const void* src, size_t srcSize)
+size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const end = ip + srcSize;
unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0;
-
memset(count, 0, (maxSymbolValue+1)*sizeof(*count));
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
@@ -326,20 +336,24 @@ static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
}
-static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
+/* FSE_count_parallel_wksp() :
+ * Same as FSE_count_parallel(), but using an externally provided scratch buffer.
+ * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
+static size_t FSE_count_parallel_wksp(
+ unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
- unsigned checkMax)
+ unsigned checkMax, unsigned* const workSpace)
{
const BYTE* ip = (const BYTE*)source;
const BYTE* const iend = ip+sourceSize;
unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0;
+ U32* const Counting1 = workSpace;
+ U32* const Counting2 = Counting1 + 256;
+ U32* const Counting3 = Counting2 + 256;
+ U32* const Counting4 = Counting3 + 256;
-
- U32 Counting1[256] = { 0 };
- U32 Counting2[256] = { 0 };
- U32 Counting3[256] = { 0 };
- U32 Counting4[256] = { 0 };
+ memset(Counting1, 0, 4*256*sizeof(unsigned));
/* safety checks */
if (!sourceSize) {
@@ -385,31 +399,51 @@ static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
} }
- { U32 s; for (s=0; s<=maxSymbolValue; s++) {
- count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
- if (count[s] > max) max = count[s];
- }}
+ { U32 s; for (s=0; s<=maxSymbolValue; s++) {
+ count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
+ if (count[s] > max) max = count[s];
+ } }
while (!count[maxSymbolValue]) maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue;
return (size_t)max;
}
+/* FSE_countFast_wksp() :
+ * Same as FSE_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= `1024` unsigned */
+size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize, unsigned* workSpace)
+{
+ if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
+}
+
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize)
{
- if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
- return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 0);
+ unsigned tmpCounters[1024];
+ return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters);
}
-size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
- const void* source, size_t sourceSize)
+/* FSE_count_wksp() :
+ * Same as FSE_count(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= `1024` unsigned */
+size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* source, size_t sourceSize, unsigned* workSpace)
{
- if (*maxSymbolValuePtr <255)
- return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 1);
+ if (*maxSymbolValuePtr < 255)
+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
*maxSymbolValuePtr = 255;
- return FSE_countFast(count, maxSymbolValuePtr, source, sourceSize);
+ return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
+}
+
+size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
+ const void* src, size_t srcSize)
+{
+ unsigned tmpCounters[1024];
+ return FSE_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters);
}
@@ -425,14 +459,10 @@ size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
`FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
Allocation is manual (C standard does not support variable-size structures).
*/
-
size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
{
- size_t size;
- FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */
- if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC);
- size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
- return size;
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+ return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
}
FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
@@ -483,7 +513,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
U32 ToDistribute;
/* Init */
- U32 lowThreshold = (U32)(total >> tableLog);
+ U32 const lowThreshold = (U32)(total >> tableLog);
U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
for (s=0; s<=maxSymbolValue; s++) {
@@ -531,17 +561,16 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
return 0;
}
- {
- U64 const vStepLog = 62 - tableLog;
+ { U64 const vStepLog = 62 - tableLog;
U64 const mid = (1ULL << (vStepLog-1)) - 1;
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid;
for (s=0; s<=maxSymbolValue; s++) {
if (norm[s]==-2) {
- U64 end = tmpTotal + (count[s] * rStep);
- U32 sStart = (U32)(tmpTotal >> vStepLog);
- U32 sEnd = (U32)(end >> vStepLog);
- U32 weight = sEnd - sStart;
+ U64 const end = tmpTotal + (count[s] * rStep);
+ U32 const sStart = (U32)(tmpTotal >> vStepLog);
+ U32 const sEnd = (U32)(end >> vStepLog);
+ U32 const weight = sEnd - sStart;
if (weight < 1)
return ERROR(GENERIC);
norm[s] = (short)weight;
@@ -563,7 +592,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
-
U64 const scale = 62 - tableLog;
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
U64 const vStep = 1ULL<<(scale-20);
@@ -591,7 +619,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
} }
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
/* corner case, need another normalization method */
- size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
}
else normalizedCounter[largest] += (short)stillToDistribute;
@@ -640,17 +668,15 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
/* Build Symbol Transformation Table */
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
-
for (s=0; s<=maxSymbolValue; s++) {
symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = s-1;
} }
-
return 0;
}
-/* fake FSE_CTable, for rle (100% always same symbol) input */
+/* fake FSE_CTable, for rle input (always same symbol) */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
{
void* ptr = ct;
@@ -682,14 +708,13 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const BYTE* const iend = istart + srcSize;
const BYTE* ip=iend;
-
BIT_CStream_t bitC;
FSE_CState_t CState1, CState2;
/* init */
if (srcSize <= 2) return 0;
- { size_t const errorCode = BIT_initCStream(&bitC, dst, dstSize);
- if (FSE_isError(errorCode)) return 0; }
+ { size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
+ if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
@@ -712,7 +737,7 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
}
/* 2 or 4 encoding per loop */
- for ( ; ip>istart ; ) {
+ while ( ip>istart ) {
FSE_encodeSymbol(&bitC, &CState2, *--ip);
@@ -738,7 +763,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct)
{
- const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
+ unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
if (fast)
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
@@ -749,58 +774,76 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
-size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
-{
- const BYTE* const istart = (const BYTE*) src;
- const BYTE* ip = istart;
+#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
+#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
+/* FSE_compress_wksp() :
+ * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
+ * `wkspSize` size must be `(1<<tableLog)`.
+ */
+size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
+{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 count[FSE_MAX_SYMBOL_VALUE+1];
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
- CTable_max_t ct;
- size_t errorCode;
+ FSE_CTable* CTable = (FSE_CTable*)workSpace;
+ size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
+ void* scratchBuffer = (void*)(CTable + CTableSize);
+ size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
/* init conditions */
- if (srcSize <= 1) return 0; /* Uncompressible */
+ if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
+ if (srcSize <= 1) return 0; /* Not compressible */
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */
- errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
- if (FSE_isError(errorCode)) return errorCode;
- if (errorCode == srcSize) return 1;
- if (errorCode == 1) return 0; /* each symbol only present once */
- if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
+ { CHECK_V_F(maxCount, FSE_count(count, &maxSymbolValue, src, srcSize) );
+ if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
+ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
+ if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
+ }
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
- errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
- if (FSE_isError(errorCode)) return errorCode;
+ CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
/* Write table description header */
- errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog);
- if (FSE_isError(errorCode)) return errorCode;
- op += errorCode;
+ { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
+ op += nc_err;
+ }
/* Compress */
- errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog);
- if (FSE_isError(errorCode)) return errorCode;
- errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
- if (errorCode == 0) return 0; /* not enough space for compressed data */
- op += errorCode;
+ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
+ { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
+ if (cSize == 0) return 0; /* not enough space for compressed data */
+ op += cSize;
+ }
/* check compressibility */
- if ( (size_t)(op-ostart) >= srcSize-1 )
- return 0;
+ if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
return op-ostart;
}
-size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
+typedef struct {
+ FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
+ BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
+} fseWkspMax_t;
+
+size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
+{
+ fseWkspMax_t scratchBuffer;
+ FSE_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
+ if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+ return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
+}
+
+size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
- return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
+ return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/compress/huf_compress.c b/c-blosc/internal-complibs/zstd-1.1.3/compress/huf_compress.c
similarity index 74%
rename from c-blosc/internal-complibs/zstd-1.0.0/compress/huf_compress.c
rename to c-blosc/internal-complibs/zstd-1.1.3/compress/huf_compress.c
index c2dd13c..bf464da 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/compress/huf_compress.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/compress/huf_compress.c
@@ -35,24 +35,8 @@
/* **************************************************************
* Compiler specifics
****************************************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-/* inline is defined */
-#elif defined(_MSC_VER)
-# define inline __inline
-#else
-# define inline /* disable inline */
-#endif
-
-
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -72,6 +56,8 @@
* Error Management
****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
+#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
/* **************************************************************
@@ -86,31 +72,73 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
/* *******************************************************
* HUF : Huffman block compression
*********************************************************/
+/* HUF_compressWeights() :
+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
+ * The use case needs much less stack memory.
+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
+ */
+#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
+size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
+{
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* op = ostart;
+ BYTE* const oend = ostart + dstSize;
+
+ U32 maxSymbolValue = HUF_TABLELOG_MAX;
+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
+
+ FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
+ BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
+
+ U32 count[HUF_TABLELOG_MAX+1];
+ S16 norm[HUF_TABLELOG_MAX+1];
+
+ /* init conditions */
+ if (wtSize <= 1) return 0; /* Not compressible */
+
+ /* Scan input and build symbol stats */
+ { CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
+ if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
+ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
+ }
+
+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
+ CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
+
+ /* Write table description header */
+ { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
+ op += hSize;
+ }
+
+ /* Compress */
+ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
+ { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
+ if (cSize == 0) return 0; /* not enough space for compressed data */
+ op += cSize;
+ }
+
+ return op-ostart;
+}
+
+
struct HUF_CElt_s {
U16 val;
BYTE nbBits;
}; /* typedef'd to HUF_CElt within "huf.h" */
-typedef struct nodeElt_s {
- U32 count;
- U16 parent;
- BYTE byte;
- BYTE nbBits;
-} nodeElt;
-
/*! HUF_writeCTable() :
`CTable` : huffman tree to save, using huf representation.
@return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
- BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
+ BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
BYTE* op = (BYTE*)dst;
U32 n;
/* check conditions */
- if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
/* convert to weight */
bitsToWeight[0] = 0;
@@ -119,38 +147,33 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
for (n=0; n<maxSymbolValue; n++)
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
- { size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
- if (FSE_isError(size)) return size;
- if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
- op[0] = (BYTE)size;
- return size+1;
- }
- }
+ /* attempt weights compression by FSE */
+ { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
+ if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
+ op[0] = (BYTE)hSize;
+ return hSize+1;
+ } }
- /* raw values */
- if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
+ /* write raw values as 4-bits (max : 15) */
+ if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
- huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
-
}
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
+ BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
- size_t readSize;
U32 nbSymbols = 0;
- /*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
/* get symbol weights */
- readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
- if (HUF_isError(readSize)) return readSize;
+ CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
/* check result */
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
@@ -171,13 +194,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
} }
/* fill val */
- { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
- U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
+ { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
+ U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */
+ valPerRank[tableLog+1] = 0; /* for w==0 */
{ U16 min = 0;
- U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) {
- valPerRank[n] = min; /* get starting value within each rank */
+ U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
+ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
@@ -189,6 +213,13 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
}
+typedef struct nodeElt_s {
+ U32 count;
+ U16 parent;
+ BYTE byte;
+ BYTE nbBits;
+} nodeElt;
+
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
{
const U32 largestBits = huffNode[lastNonNull].nbBits;
@@ -294,20 +325,26 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
}
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
+ */
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
-size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
+typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1];
+size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
{
- nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
- nodeElt* huffNode = huffNode0 + 1;
+ nodeElt* const huffNode0 = (nodeElt*)workSpace;
+ nodeElt* const huffNode = huffNode0+1;
U32 n, nonNullRank;
int lowS, lowN;
U16 nodeNb = STARTNODE;
U32 nodeRoot;
/* safety checks */
+ if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
- memset(huffNode0, 0, sizeof(huffNode0));
+ memset(huffNode0, 0, sizeof(huffNodeTable));
/* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue);
@@ -320,7 +357,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
nodeNb++; lowS-=2;
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
- huffNode0[0].count = (U32)(1U<<31);
+ huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
/* create parents */
while (nodeNb <= nodeRoot) {
@@ -363,6 +400,15 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
return maxNbBits;
}
+/** HUF_buildCTable() :
+ * Note : count is used before tree is written, so they can safely overlap
+ */
+size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
+{
+ huffNodeTable nodeTable;
+ return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
+}
+
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
{
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
@@ -390,8 +436,8 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
/* init */
if (dstSize < 8) return 0; /* not enough space to compress */
- { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
- if (HUF_isError(errorCode)) return 0; }
+ { size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
+ if (HUF_isError(initErr)) return 0; }
n = srcSize & ~3; /* join to mod 4 */
switch (srcSize & 3)
@@ -434,32 +480,28 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
if (srcSize < 12) return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
- if (HUF_isError(cSize)) return cSize;
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
if (cSize==0) return 0;
MEM_writeLE16(ostart, (U16)cSize);
op += cSize;
}
ip += segmentSize;
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
- if (HUF_isError(cSize)) return cSize;
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
if (cSize==0) return 0;
MEM_writeLE16(ostart+2, (U16)cSize);
op += cSize;
}
ip += segmentSize;
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
- if (HUF_isError(cSize)) return cSize;
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
if (cSize==0) return 0;
MEM_writeLE16(ostart+4, (U16)cSize);
op += cSize;
}
ip += segmentSize;
- { size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
- if (HUF_isError(cSize)) return cSize;
+ { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) );
if (cSize==0) return 0;
op += cSize;
}
@@ -468,20 +510,25 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
}
+/* `workSpace` must a table of at least 1024 unsigned */
static size_t HUF_compress_internal (
void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
- unsigned singleStream)
+ unsigned singleStream,
+ void* workSpace, size_t wkspSize)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
- U32 count[HUF_SYMBOLVALUE_MAX+1];
- HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
+ union {
+ U32 count[HUF_SYMBOLVALUE_MAX+1];
+ HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
+ } table; /* `count` can overlap with `CTable`; saves 1 KB */
/* checks & inits */
+ if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC);
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
if (!dstSize) return 0; /* cannot fit within dst budget */
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
@@ -490,30 +537,27 @@ static size_t HUF_compress_internal (
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
/* Scan input and build symbol stats */
- { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
- if (HUF_isError(largest)) return largest;
+ { CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
}
/* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
- { size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
- if (HUF_isError(maxBits)) return maxBits;
+ { CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) );
huffLog = (U32)maxBits;
}
/* Write table description header */
- { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
- if (HUF_isError(hSize)) return hSize;
+ { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) );
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
op += hSize;
}
/* Compress */
{ size_t const cSize = (singleStream) ?
- HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
- HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
+ HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) : /* single segment */
+ HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0; /* uncompressible */
op += cSize;
@@ -527,21 +571,38 @@ static size_t HUF_compress_internal (
}
+size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ void* workSpace, size_t wkspSize)
+{
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize);
+}
+
size_t HUF_compress1X (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
+ unsigned workSpace[1024];
+ return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
+}
+
+size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
+ const void* src, size_t srcSize,
+ unsigned maxSymbolValue, unsigned huffLog,
+ void* workSpace, size_t wkspSize)
+{
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize);
}
size_t HUF_compress2 (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
- return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
+ unsigned workSpace[1024];
+ return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
}
-
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/compress/zstd_compress.c b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstd_compress.c
similarity index 84%
rename from c-blosc/internal-complibs/zstd-1.0.0/compress/zstd_compress.c
rename to c-blosc/internal-complibs/zstd-1.1.3/compress/zstd_compress.c
index 0116136..b6cf376 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/compress/zstd_compress.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstd_compress.c
@@ -8,30 +8,13 @@
*/
-
-/*-*******************************************************
-* Compiler specifics
-*********************************************************/
-#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
-# include <intrin.h> /* For Visual 2005 */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-#endif
-
-
/*-*************************************
* Dependencies
***************************************/
#include <string.h> /* memset */
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
-#include "xxhash.h" /* XXH_reset, update, digest */
+#include "xxhash.h" /* XXH_reset, update, digest */
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
@@ -50,6 +33,7 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZS
/*-*************************************
* Helper functions
***************************************/
+#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
@@ -67,8 +51,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
/*-*************************************
* Context memory management
***************************************/
-struct ZSTD_CCtx_s
-{
+struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */
const BYTE* dictBase; /* extDict indexes relative to this position */
@@ -77,10 +60,11 @@ struct ZSTD_CCtx_s
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */
- U32 loadedDictEnd;
+ U32 loadedDictEnd; /* index of end of dictionary */
+ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
ZSTD_compressionStage_e stage;
U32 rep[ZSTD_REP_NUM];
- U32 savedRep[ZSTD_REP_NUM];
+ U32 repToConfirm[ZSTD_REP_NUM];
U32 dictID;
ZSTD_parameters params;
void* workSpace;
@@ -99,6 +83,7 @@ struct ZSTD_CCtx_s
FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+ unsigned tmpCounters[1024];
};
ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -116,7 +101,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL;
memset(cctx, 0, sizeof(ZSTD_CCtx));
- memcpy(&(cctx->customMem), &customMem, sizeof(ZSTD_customMem));
+ cctx->customMem = customMem;
return cctx;
}
@@ -130,51 +115,59 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
{
+ if (cctx==NULL) return 0; /* support sizeof on NULL */
return sizeof(*cctx) + cctx->workSpaceSize;
}
+size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
+{
+ switch(param)
+ {
+ case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0;
+ default: return ERROR(parameter_unknown);
+ }
+}
+
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */
{
return &(ctx->seqStore);
}
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
+{
+ return cctx->params;
+}
-#define CLAMP(val,min,max) { if (val<min) val=min; else if (val>max) val=max; }
-#define CLAMPCHECK(val,min,max) { if ((val<min) || (val>max)) return ERROR(compressionParameter_unsupported); }
/** ZSTD_checkParams() :
ensure param values remain within authorized range.
@return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{
+# define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); }
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
- { U32 const searchLengthMin = (cParams.strategy == ZSTD_fast || cParams.strategy == ZSTD_greedy) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN;
+ { U32 const searchLengthMin = ((cParams.strategy == ZSTD_fast) | (cParams.strategy == ZSTD_greedy)) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN;
U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1;
CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); }
CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
- if ((U32)(cParams.strategy) > (U32)ZSTD_btopt) return ERROR(compressionParameter_unsupported);
+ if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported);
return 0;
}
-/** ZSTD_checkCParams_advanced() :
- temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
-size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
+/** ZSTD_cycleLog() :
+ * condition for correct operation : hashLog > 1 */
+static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
{
- if (srcSize > (1ULL << ZSTD_WINDOWLOG_MIN)) return ZSTD_checkCParams(cParams);
- if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported);
- if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */
- if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */
- if ((srcSize <= (1ULL << cParams.hashLog)) && ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */
- return ZSTD_checkCParams(cParams);
+ U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
+ return hashLog - btScale;
}
-
/** ZSTD_adjustCParams() :
- optimize cPar for a given input (`srcSize` and `dictSize`).
+ optimize `cPar` for a given input (`srcSize` and `dictSize`).
mostly downsizing to reduce memory consumption and initialization.
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
but if both are 0, no optimization can be done.
@@ -187,16 +180,15 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
- U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
+ U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
- { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt);
- U32 const maxChainLog = cPar.windowLog+btPlus;
- if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
+ { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
+ if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog);
+ }
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
- if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
return cPar;
}
@@ -218,100 +210,142 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
- + ((cParams.strategy == ZSTD_btopt) ? optSpace : 0);
+ + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
return sizeof(ZSTD_CCtx) + neededSpace;
}
+
+static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
+{
+ return (param1.cParams.hashLog == param2.cParams.hashLog)
+ & (param1.cParams.chainLog == param2.cParams.chainLog)
+ & (param1.cParams.strategy == param2.cParams.strategy)
+ & ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3));
+}
+
+/*! ZSTD_continueCCtx() :
+ reuse CCtx without reset (note : requires no dictionary) */
+static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize)
+{
+ U32 const end = (U32)(cctx->nextSrc - cctx->base);
+ cctx->params = params;
+ cctx->frameContentSize = frameContentSize;
+ cctx->lowLimit = end;
+ cctx->dictLimit = end;
+ cctx->nextToUpdate = end+1;
+ cctx->stage = ZSTDcs_init;
+ cctx->dictID = 0;
+ cctx->loadedDictEnd = 0;
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = repStartValue[i]; }
+ cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
+ XXH64_reset(&cctx->xxhState, 0);
+ return 0;
+}
+
+typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
+
/*! ZSTD_resetCCtx_advanced() :
- note : 'params' is expected to be validated */
+ note : 'params' must be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params, U64 frameContentSize,
- U32 reset)
-{ /* note : params considered validated here */
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
- U32 const divider = (params.cParams.searchLength==3) ? 3 : 4;
- size_t const maxNbSeq = blockSize / divider;
- size_t const tokenSpace = blockSize + 11*maxNbSeq;
- size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
- size_t const hSize = ((size_t)1) << params.cParams.hashLog;
- U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
- size_t const h3Size = ((size_t)1) << hashLog3;
- size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
- void* ptr;
-
- /* Check if workSpace is large enough, alloc a new one if needed */
- { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
- + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
- size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
- + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
- if (zc->workSpaceSize < neededSpace) {
- ZSTD_free(zc->workSpace, zc->customMem);
- zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
- if (zc->workSpace == NULL) return ERROR(memory_allocation);
- zc->workSpaceSize = neededSpace;
- } }
+ ZSTD_compResetPolicy_e const crp)
+{
+ if (crp == ZSTDcrp_continue)
+ if (ZSTD_equivalentParams(params, zc->params))
+ return ZSTD_continueCCtx(zc, params, frameContentSize);
+
+ { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
+ U32 const divider = (params.cParams.searchLength==3) ? 3 : 4;
+ size_t const maxNbSeq = blockSize / divider;
+ size_t const tokenSpace = blockSize + 11*maxNbSeq;
+ size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
+ size_t const hSize = ((size_t)1) << params.cParams.hashLog;
+ U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
+ size_t const h3Size = ((size_t)1) << hashLog3;
+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
+ void* ptr;
+
+ /* Check if workSpace is large enough, alloc a new one if needed */
+ { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
+ + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
+ size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ + (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
+ if (zc->workSpaceSize < neededSpace) {
+ ZSTD_free(zc->workSpace, zc->customMem);
+ zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
+ if (zc->workSpace == NULL) return ERROR(memory_allocation);
+ zc->workSpaceSize = neededSpace;
+ } }
- if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
- XXH64_reset(&zc->xxhState, 0);
- zc->hashLog3 = hashLog3;
- zc->hashTable = (U32*)(zc->workSpace);
- zc->chainTable = zc->hashTable + hSize;
- zc->hashTable3 = zc->chainTable + chainSize;
- ptr = zc->hashTable3 + h3Size;
- zc->hufTable = (HUF_CElt*)ptr;
- zc->flagStaticTables = 0;
- ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
-
- zc->nextToUpdate = 1;
- zc->nextSrc = NULL;
- zc->base = NULL;
- zc->dictBase = NULL;
- zc->dictLimit = 0;
- zc->lowLimit = 0;
- zc->params = params;
- zc->blockSize = blockSize;
- zc->frameContentSize = frameContentSize;
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; }
-
- if (params.cParams.strategy == ZSTD_btopt) {
- zc->seqStore.litFreq = (U32*)ptr;
- zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
- zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
- zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
- ptr = zc->seqStore.offCodeFreq + (MaxOff+1);
- zc->seqStore.matchTable = (ZSTD_match_t*)ptr;
- ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
- zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr;
- ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
- zc->seqStore.litLengthSum = 0;
- }
- zc->seqStore.sequencesStart = (seqDef*)ptr;
- ptr = zc->seqStore.sequencesStart + maxNbSeq;
- zc->seqStore.llCode = (BYTE*) ptr;
- zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
- zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
- zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
-
- zc->stage = ZSTDcs_init;
- zc->dictID = 0;
- zc->loadedDictEnd = 0;
+ if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */
+ XXH64_reset(&zc->xxhState, 0);
+ zc->hashLog3 = hashLog3;
+ zc->hashTable = (U32*)(zc->workSpace);
+ zc->chainTable = zc->hashTable + hSize;
+ zc->hashTable3 = zc->chainTable + chainSize;
+ ptr = zc->hashTable3 + h3Size;
+ zc->hufTable = (HUF_CElt*)ptr;
+ zc->flagStaticTables = 0;
+ ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
+
+ zc->nextToUpdate = 1;
+ zc->nextSrc = NULL;
+ zc->base = NULL;
+ zc->dictBase = NULL;
+ zc->dictLimit = 0;
+ zc->lowLimit = 0;
+ zc->params = params;
+ zc->blockSize = blockSize;
+ zc->frameContentSize = frameContentSize;
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; }
+
+ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
+ zc->seqStore.litFreq = (U32*)ptr;
+ zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
+ zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
+ zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
+ ptr = zc->seqStore.offCodeFreq + (MaxOff+1);
+ zc->seqStore.matchTable = (ZSTD_match_t*)ptr;
+ ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
+ zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr;
+ ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
+ zc->seqStore.litLengthSum = 0;
+ }
+ zc->seqStore.sequencesStart = (seqDef*)ptr;
+ ptr = zc->seqStore.sequencesStart + maxNbSeq;
+ zc->seqStore.llCode = (BYTE*) ptr;
+ zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
+ zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
+ zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
- return 0;
+ zc->stage = ZSTDcs_init;
+ zc->dictID = 0;
+ zc->loadedDictEnd = 0;
+
+ return 0;
+ }
}
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
+ int i;
+ for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
+}
/*! ZSTD_copyCCtx() :
* Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
* @return : 0, or an error code */
-size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
+size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
{
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
- ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0);
- dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
+ ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset);
/* copy tables */
{ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
@@ -375,7 +409,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
* Block entropic compression
*********************************************************/
-/* See zstd_compression_format.md for detailed format description */
+/* See doc/zstd_compression_format.md for detailed format description */
size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
@@ -463,8 +497,8 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
singleStream = 1;
cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable);
} else {
- cLitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11)
- : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11);
+ cLitSize = singleStream ? HUF_compress1X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters))
+ : HUF_compress4X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters));
}
if ((cLitSize==0) | (cLitSize >= srcSize - minGain))
@@ -559,6 +593,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
BYTE* op = ostart;
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
BYTE* seqHead;
+ BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)];
/* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart;
@@ -586,7 +621,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
/* CTable for Literal Lengths */
{ U32 max = MaxLL;
- size_t const mostFrequent = FSE_countFast(count, &max, llCodeTable, nbSeq);
+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = llCodeTable[0];
FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
@@ -594,7 +629,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
LLtype = set_repeat;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) {
- FSE_buildCTable(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog);
+ FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
LLtype = set_basic;
} else {
size_t nbSeq_1 = nbSeq;
@@ -604,13 +639,13 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
op += NCountSize; }
- FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
+ FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
LLtype = set_compressed;
} }
/* CTable for Offsets */
{ U32 max = MaxOff;
- size_t const mostFrequent = FSE_countFast(count, &max, ofCodeTable, nbSeq);
+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = ofCodeTable[0];
FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
@@ -618,7 +653,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
Offtype = set_repeat;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) {
- FSE_buildCTable(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog);
+ FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
Offtype = set_basic;
} else {
size_t nbSeq_1 = nbSeq;
@@ -628,13 +663,13 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
op += NCountSize; }
- FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
+ FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
Offtype = set_compressed;
} }
/* CTable for MatchLengths */
{ U32 max = MaxML;
- size_t const mostFrequent = FSE_countFast(count, &max, mlCodeTable, nbSeq);
+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters);
if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
*op++ = *mlCodeTable;
FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
@@ -642,7 +677,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
} else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
MLtype = set_repeat;
} else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) {
- FSE_buildCTable(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog);
+ FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
MLtype = set_basic;
} else {
size_t nbSeq_1 = nbSeq;
@@ -652,7 +687,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
{ size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
op += NCountSize; }
- FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
+ FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
MLtype = set_compressed;
} }
@@ -665,8 +700,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
FSE_CState_t stateOffsetBits;
FSE_CState_t stateLitLength;
- { size_t const errorCode = BIT_initCStream(&blockStream, op, oend-op);
- if (ERR_isError(errorCode)) return ERROR(dstSize_tooSmall); } /* not enough space remaining */
+ CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */
/* first symbols */
FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
@@ -718,12 +752,19 @@ _check_compressibility:
if ((size_t)(op-ostart) >= maxCSize) return 0; }
/* confirm repcodes */
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->savedRep[i]; }
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; }
return op - ostart;
}
+#if 0 /* for debug */
+# define STORESEQ_DEBUG
+#include <stdio.h> /* fprintf */
+U32 g_startDebug = 0;
+const BYTE* g_start = NULL;
+#endif
+
/*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
`offsetCode` : distance to match, or 0 == repCode.
@@ -731,13 +772,14 @@ _check_compressibility:
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
{
-#if 0 /* for debug */
- static const BYTE* g_start = NULL;
- const U32 pos = (U32)(literals - g_start);
- if (g_start==NULL) g_start = literals;
- //if ((pos > 1) && (pos < 50000))
- printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
- pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+#ifdef STORESEQ_DEBUG
+ if (g_startDebug) {
+ const U32 pos = (U32)((const BYTE*)literals - g_start);
+ if (g_start==NULL) g_start = (const BYTE*)literals;
+ if ((pos > 1895000) && (pos < 1895300))
+ fprintf(stderr, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
+ pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
+ }
#endif
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@@ -987,8 +1029,8 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
} } }
/* save reps for next block */
- cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
- cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;
+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1080,7 +1122,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
if (ip <= ilimit) {
/* Fill Table */
- hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2;
+ hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2;
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
/* check immediate repcode */
while (ip <= ilimit) {
@@ -1102,7 +1144,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
} } }
/* save reps for next block */
- ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+ ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1256,8 +1298,8 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
} } }
/* save reps for next block */
- cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
- cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;
+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1406,7 +1448,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
} } }
/* save reps for next block */
- ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+ ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -1457,7 +1499,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
- const BYTE* match = base + matchIndex;
+ const BYTE* match;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
@@ -1476,8 +1518,9 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) {
- U32* nextPtr = bt + 2*(matchIndex & btMask);
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+
#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
if (matchIndex == predictedSmall) {
@@ -1573,7 +1616,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) {
- U32* nextPtr = bt + 2*(matchIndex & btMask);
+ U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match;
@@ -1937,8 +1980,8 @@ _storeSequence:
} }
/* Save reps for next block */
- ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset;
- ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset;
+ ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
+ ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -2132,7 +2175,7 @@ _storeSequence:
} }
/* Save reps for next block */
- ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
+ ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -2169,7 +2212,17 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
#ifdef ZSTD_OPT_H_91842398743
- ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
+#else
+ (void)ctx; (void)src; (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
#else
(void)ctx; (void)src; (void)srcSize;
return;
@@ -2179,7 +2232,17 @@ static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t src
static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
#ifdef ZSTD_OPT_H_91842398743
- ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize);
+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
+#else
+ (void)ctx; (void)src; (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
#else
(void)ctx; (void)src; (void)srcSize;
return;
@@ -2191,9 +2254,9 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{
- static const ZSTD_blockCompressor blockCompressor[2][7] = {
- { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt },
- { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }
+ static const ZSTD_blockCompressor blockCompressor[2][8] = {
+ { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 },
+ { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict }
};
return blockCompressor[extDict][(U32)strat];
@@ -2234,7 +2297,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
BYTE* op = ostart;
U32 const maxDist = 1 << cctx->params.cParams.windowLog;
- if (cctx->params.fParams.checksumFlag)
+ if (cctx->params.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
@@ -2245,16 +2308,16 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (remaining < blockSize) blockSize = remaining;
/* preemptive overflow correction */
- if (cctx->lowLimit > (1<<30)) {
- U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
- U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
- U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
- U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */
- U32 const correction = cctx->lowLimit - newLowLimit;
+ if (cctx->lowLimit > (2U<<30)) {
+ U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
+ U32 const current = (U32)(ip - cctx->base);
+ U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);
+ U32 const correction = current - newCurrent;
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
ZSTD_reduceIndex(cctx, correction);
cctx->base += correction;
cctx->dictBase += correction;
- cctx->lowLimit = newLowLimit;
+ cctx->lowLimit -= correction;
cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
else cctx->nextToUpdate -= correction;
@@ -2371,12 +2434,14 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->nextSrc = ip + srcSize;
- { size_t const cSize = frame ?
+ if (srcSize) {
+ size_t const cSize = frame ?
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
return cSize + fhSize;
- }
+ } else
+ return fhSize;
}
@@ -2412,7 +2477,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
- zc->loadedDictEnd = (U32)(iend - zc->base);
+ zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
zc->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0;
@@ -2435,6 +2500,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
case ZSTD_btlazy2:
case ZSTD_btopt:
+ case ZSTD_btopt2:
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
break;
@@ -2447,14 +2513,28 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
}
+/* Dictionaries that assign zero probability to symbols that show up causes problems
+ when FSE encoding. Refuse dictionaries that assign zero probability to symbols
+ that we may encounter during compression.
+ NOTE: This behavior is not standard and could be improved in the future. */
+static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) {
+ U32 s;
+ if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted);
+ for (s = 0; s <= maxSymbolValue; ++s) {
+ if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted);
+ }
+ return 0;
+}
+
+
/* Dictionary format :
- Magic == ZSTD_DICT_MAGIC (4 bytes)
- HUF_writeCTable(256)
- FSE_writeNCount(off)
- FSE_writeNCount(ml)
- FSE_writeNCount(ll)
- RepOffsets
- Dictionary content
+ Magic == ZSTD_DICT_MAGIC (4 bytes)
+ HUF_writeCTable(256)
+ FSE_writeNCount(off)
+ FSE_writeNCount(ml)
+ FSE_writeNCount(ll)
+ RepOffsets
+ Dictionary content
*/
/*! ZSTD_loadDictEntropyStats() :
@return : size read from dictionary
@@ -2463,45 +2543,62 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
{
const BYTE* dictPtr = (const BYTE*)dict;
const BYTE* const dictEnd = dictPtr + dictSize;
+ short offcodeNCount[MaxOff+1];
+ unsigned offcodeMaxValue = MaxOff;
+ BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)];
{ size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
dictPtr += hufHeaderSize;
}
- { short offcodeNCount[MaxOff+1];
- unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
+ { unsigned offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
- { size_t const errorCode = FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
- if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
+ CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
- { size_t const errorCode = FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
- if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
+ /* Every match length code must have non-zero probability */
+ CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
+ CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
- { size_t const errorCode = FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
- if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
+ /* Every literal length code must have non-zero probability */
+ CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
+ CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
- cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
- cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
- cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+ cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+ cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+ cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
+ { U32 offcodeMax = MaxOff;
+ if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) {
+ U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */
+ /* Calculate minimum offset code required to represent maxOffset */
+ offcodeMax = ZSTD_highbit32(maxOffset);
+ }
+ /* Every possible supported offset <= dictContentSize + 128 KB must be representable */
+ CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
+ }
+
cctx->flagStaticTables = 1;
return dictPtr - (const BYTE*)dict;
}
@@ -2517,24 +2614,22 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4);
/* known magic number : dict is parsed for entropy stats and content */
- { size_t const eSize_8 = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8);
- size_t const eSize = eSize_8 + 8;
- if (ZSTD_isError(eSize_8)) return eSize_8;
+ { size_t const loadError = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8);
+ size_t const eSize = loadError + 8;
+ if (ZSTD_isError(loadError)) return loadError;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
}
}
-
/*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */
-static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{
- size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1);
- if (ZSTD_isError(resetError)) return resetError;
-
- return ZSTD_compress_insertDictionary(zc, dict, dictSize);
+ ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
+ CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
+ return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
}
@@ -2545,9 +2640,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
/* compression parameters verification and optimization */
- { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize);
- if (ZSTD_isError(errorCode)) return errorCode; }
-
+ CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
}
@@ -2559,9 +2652,9 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
}
-size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
+size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
{
- return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel);
+ return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
}
@@ -2625,9 +2718,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
const void* dict,size_t dictSize,
ZSTD_parameters params)
{
- size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize);
- if(ZSTD_isError(errorCode)) return errorCode;
-
+ CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
@@ -2637,14 +2728,13 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
const void* dict,size_t dictSize,
ZSTD_parameters params)
{
- size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, srcSize);
- if (ZSTD_isError(errorCode)) return errorCode;
+ CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
}
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{
- ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize);
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0);
params.fParams.contentSizeFlag = 1;
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
}
@@ -2669,39 +2759,54 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/* ===== Dictionary API ===== */
struct ZSTD_CDict_s {
- void* dictContent;
+ void* dictBuffer;
+ const void* dictContent;
size_t dictContentSize;
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
-ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
+{
+ if (cdict==NULL) return 0; /* support sizeof on NULL */
+ return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
+}
+
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
+ ZSTD_parameters params, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
- void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
- if (!dictContent || !cdict || !cctx) {
- ZSTD_free(dictContent, customMem);
+ if (!cdict || !cctx) {
ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem);
return NULL;
}
- memcpy(dictContent, dict, dictSize);
- { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
+ if ((byReference) || (!dictBuffer) || (!dictSize)) {
+ cdict->dictBuffer = NULL;
+ cdict->dictContent = dictBuffer;
+ } else {
+ void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
+ if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
+ memcpy(internalBuffer, dictBuffer, dictSize);
+ cdict->dictBuffer = internalBuffer;
+ cdict->dictContent = internalBuffer;
+ }
+
+ { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
- ZSTD_free(dictContent, customMem);
- ZSTD_free(cdict, customMem);
+ ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cctx, customMem);
+ ZSTD_free(cdict, customMem);
return NULL;
} }
- cdict->dictContent = dictContent;
- cdict->dictContentSize = dictSize;
cdict->refContext = cctx;
+ cdict->dictContentSize = dictSize;
return cdict;
}
}
@@ -2711,31 +2816,49 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
- return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
+ return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
+}
+
+ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
+ params.fParams.contentSizeFlag = 1;
+ return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
}
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support free on NULL */
- { ZSTD_customMem cMem = cdict->refContext->customMem;
+ { ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext);
- ZSTD_free(cdict->dictContent, cMem);
+ ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem);
return 0;
}
}
+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
+ return ZSTD_getParamsFromCCtx(cdict->refContext);
+}
+
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
+{
+ if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
+ else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
+ return 0;
+}
+
/*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */
-ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_CDict* cdict)
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_CDict* cdict)
{
- size_t const errorCode = ZSTD_copyCCtx(cctx, cdict->refContext);
- if (ZSTD_isError(errorCode)) return errorCode;
+ CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
if (cdict->refContext->params.fParams.contentSizeFlag==1) {
cctx->params.fParams.contentSizeFlag = 1;
@@ -2754,7 +2877,9 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s {
- ZSTD_CCtx* zc;
+ ZSTD_CCtx* cctx;
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@@ -2768,6 +2893,9 @@ struct ZSTD_CStream_s {
ZSTD_cStreamStage stage;
U32 checksum;
U32 frameEnded;
+ U64 pledgedSrcSize;
+ U64 inputProcessed;
+ ZSTD_parameters params;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -2787,8 +2915,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
if (zcs==NULL) return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
- zcs->zc = ZSTD_createCCtx_advanced(customMem);
- if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; }
+ zcs->cctx = ZSTD_createCCtx_advanced(customMem);
+ if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; }
return zcs;
}
@@ -2796,7 +2924,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
{
if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem;
- ZSTD_freeCCtx(zcs->zc);
+ ZSTD_freeCCtx(zcs->cctx);
+ ZSTD_freeCDict(zcs->cdictLocal);
ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem);
@@ -2810,6 +2939,24 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+{
+ if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
+
+ if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
+ else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
+
+ zcs->inToCompress = 0;
+ zcs->inBuffPos = 0;
+ zcs->inBuffTarget = zcs->blockSize;
+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+ zcs->stage = zcss_load;
+ zcs->frameEnded = 0;
+ zcs->pledgedSrcSize = pledgedSrcSize;
+ zcs->inputProcessed = 0;
+ return 0; /* ready to go */
+}
+
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -2818,7 +2965,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = neededInBuffSize;
- ZSTD_free(zcs->inBuff, zcs->customMem); /* should not be necessary */
+ ZSTD_free(zcs->inBuff, zcs->customMem);
zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem);
if (zcs->inBuff == NULL) return ERROR(memory_allocation);
}
@@ -2826,22 +2973,32 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
}
if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
- ZSTD_free(zcs->outBuff, zcs->customMem); /* should not be necessary */
+ ZSTD_free(zcs->outBuff, zcs->customMem);
zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
- { size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize);
- if (ZSTD_isError(errorCode)) return errorCode; }
+ if (dict && dictSize >= 8) {
+ ZSTD_freeCDict(zcs->cdictLocal);
+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
+ if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+ zcs->cdict = zcs->cdictLocal;
+ } else zcs->cdict = NULL;
- zcs->inToCompress = 0;
- zcs->inBuffPos = 0;
- zcs->inBuffTarget = zcs->blockSize;
- zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
- zcs->stage = zcss_load;
zcs->checksum = params.fParams.checksumFlag > 0;
- zcs->frameEnded = 0;
- return 0; /* ready to go */
+ zcs->params = params;
+
+ return ZSTD_resetCStream(zcs, pledgedSrcSize);
+}
+
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+ ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+ size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
+ zcs->cdict = cdict;
+ zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID;
+ return initError;
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
@@ -2850,6 +3007,13 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
}
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
+{
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+ if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
+ return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
+}
+
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
{
return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
@@ -2857,7 +3021,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{
- return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize;
+ if (zcs==NULL) return 0; /* support sizeof on NULL */
+ return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
}
/*====== Compression ======*/
@@ -2908,8 +3073,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ?
- ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
- ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
+ ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
+ ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
if (flush == zsf_end) zcs->frameEnded = 1;
/* prepare next block */
@@ -2945,6 +3110,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
*srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart;
+ zcs->inputProcessed += *srcSizePtr;
if (zcs->frameEnded) return 0;
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
if (hintInSize==0) hintInSize = zcs->blockSize;
@@ -2974,8 +3140,8 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs,
- (char*)(output->dst) + output->pos, &sizeWritten,
- &srcSize, &srcSize, /* use a valid src address instead of NULL */
+ (char*)(output->dst) + output->pos, &sizeWritten,
+ &srcSize, &srcSize, /* use a valid src address instead of NULL */
zsf_flush);
output->pos += sizeWritten;
if (ZSTD_isError(result)) return result;
@@ -2989,6 +3155,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
+ if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
+ return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
+
if (zcs->stage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0;
@@ -3003,7 +3172,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
/* create epilogue */
zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 :
- ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
+ ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
}
/* flush epilogue */
@@ -3048,9 +3217,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
{ 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */
- { 25, 25, 23, 7, 3, 64, ZSTD_btopt }, /* level 20 */
- { 26, 26, 23, 7, 3,256, ZSTD_btopt }, /* level 21 */
- { 27, 27, 25, 9, 3,512, ZSTD_btopt }, /* level 22 */
+ { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */
+ { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */
+ { 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */
},
{ /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */
@@ -3074,9 +3243,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/
- { 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/
- { 18, 19, 18, 12, 3,512, ZSTD_btopt }, /* level 21.*/
- { 18, 19, 18, 13, 3,512, ZSTD_btopt }, /* level 22.*/
+ { 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/
+ { 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/
+ { 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/
},
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */
@@ -3100,9 +3269,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/
- { 17, 18, 17, 9, 3,256, ZSTD_btopt }, /* level 20.*/
- { 17, 18, 17, 10, 3,256, ZSTD_btopt }, /* level 21.*/
- { 17, 18, 17, 11, 3,512, ZSTD_btopt }, /* level 22.*/
+ { 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/
+ { 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/
+ { 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/
},
{ /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */
@@ -3126,9 +3295,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/
{ 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/
- { 14, 15, 15, 8, 3,256, ZSTD_btopt }, /* level 20.*/
- { 14, 15, 15, 9, 3,256, ZSTD_btopt }, /* level 21.*/
- { 14, 15, 15, 10, 3,256, ZSTD_btopt }, /* level 22.*/
+ { 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/
+ { 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/
+ { 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/
},
};
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/compress/zstd_opt.h b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstd_opt.h
similarity index 88%
rename from c-blosc/internal-complibs/zstd-1.0.0/compress/zstd_opt.h
rename to c-blosc/internal-complibs/zstd-1.1.3/compress/zstd_opt.h
index cb58729..8862bbd 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/compress/zstd_opt.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstd_opt.h
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
@@ -15,7 +15,9 @@
#define ZSTD_OPT_H_91842398743
-#define ZSTD_FREQ_DIV 5
+#define ZSTD_LITFREQ_ADD 2
+#define ZSTD_FREQ_DIV 4
+#define ZSTD_MAX_PRICE (1<<30)
/*-*************************************
* Price functions for optimal parser
@@ -30,22 +32,32 @@ FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
}
-MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
+MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t srcSize)
{
unsigned u;
ssPtr->cachedLiterals = NULL;
ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
+ ssPtr->staticPrices = 0;
if (ssPtr->litLengthSum == 0) {
- ssPtr->litSum = (2<<Litbits);
+ if (srcSize <= 1024) ssPtr->staticPrices = 1;
+
+ for (u=0; u<=MaxLit; u++)
+ ssPtr->litFreq[u] = 0;
+ for (u=0; u<srcSize; u++)
+ ssPtr->litFreq[src[u]]++;
+
+ ssPtr->litSum = 0;
ssPtr->litLengthSum = MaxLL+1;
ssPtr->matchLengthSum = MaxML+1;
ssPtr->offCodeSum = (MaxOff+1);
- ssPtr->matchSum = (2<<Litbits);
+ ssPtr->matchSum = (ZSTD_LITFREQ_ADD<<Litbits);
- for (u=0; u<=MaxLit; u++)
- ssPtr->litFreq[u] = 2;
+ for (u=0; u<=MaxLit; u++) {
+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV);
+ ssPtr->litSum += ssPtr->litFreq[u];
+ }
for (u=0; u<=MaxLL; u++)
ssPtr->litLengthFreq[u] = 1;
for (u=0; u<=MaxML; u++)
@@ -60,11 +72,11 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
ssPtr->litSum = 0;
for (u=0; u<=MaxLit; u++) {
- ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV);
+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1));
ssPtr->litSum += ssPtr->litFreq[u];
}
for (u=0; u<=MaxLL; u++) {
- ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV);
+ ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1));
ssPtr->litLengthSum += ssPtr->litLengthFreq[u];
}
for (u=0; u<=MaxML; u++) {
@@ -72,6 +84,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u];
ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3);
}
+ ssPtr->matchSum *= ZSTD_LITFREQ_ADD;
for (u=0; u<=MaxOff; u++) {
ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV);
ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
@@ -86,6 +99,9 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
{
U32 price, u;
+ if (ssPtr->staticPrices)
+ return ZSTD_highbit32((U32)litLength+1) + (litLength*6);
+
if (litLength == 0)
return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1);
@@ -120,11 +136,17 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
}
-FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
+FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
{
/* offset */
+ U32 price;
BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
- U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
+
+ if (seqStorePtr->staticPrices)
+ return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode;
+
+ price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
+ if (!ultra && offCode >= 20) price += (offCode-19)*2;
/* match Length */
{ const BYTE ML_deltaCode = 36;
@@ -141,9 +163,9 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
U32 u;
/* literals */
- seqStorePtr->litSum += litLength;
+ seqStorePtr->litSum += litLength*ZSTD_LITFREQ_ADD;
for (u=0; u < litLength; u++)
- seqStorePtr->litFreq[literals[u]]++;
+ seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
/* literal Length */
{ const BYTE LL_deltaCode = 19;
@@ -171,7 +193,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
{ \
- while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \
+ while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \
opt[pos].mlen = mlen_; \
opt[pos].off = offset_; \
opt[pos].litlen = litlen_; \
@@ -375,7 +397,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
*********************************/
FORCE_INLINE
void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, const int ultra)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
@@ -398,10 +420,9 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
/* init */
ctx->nextToUpdate3 = ctx->nextToUpdate;
- ZSTD_rescaleFreqs(seqStorePtr);
+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize);
ip += (ip==prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
- inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -414,7 +435,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
/* check repCode */
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
for (i=(ip == anchor); i<last_i; i++) {
- const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i];
+ const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart))
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) {
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
@@ -424,7 +445,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
}
best_off = i - (ip == anchor);
do {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--;
@@ -449,7 +470,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
best_mlen = matches[u].len;
while (mlen <= best_mlen) {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
mlen++;
@@ -496,10 +517,10 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
}
- best_mlen = minMatch;
+ best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */
- const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
+ const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart))
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) {
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
@@ -510,21 +531,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
}
best_off = i - (opt[cur].mlen != 1);
+ if (mlen > best_mlen) best_mlen = mlen;
+
+ do {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
+ } else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+ }
- if (opt[cur].mlen == 1) {
- litlen = opt[cur].litlen;
- if (cur > litlen) {
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
- } else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
- } else {
- litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
- }
-
- if (mlen > best_mlen) best_mlen = mlen;
-
- do {
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
@@ -549,12 +569,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen)
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
} else {
litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
}
if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -600,7 +620,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
offset--;
} else {
if (offset != 0) {
- best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]);
+ best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = best_off;
@@ -614,7 +634,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
/* Last Literals */
{ size_t const lastLLSize = iend - anchor;
@@ -626,7 +646,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
FORCE_INLINE
void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, const int ultra)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
@@ -655,9 +675,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
ctx->nextToUpdate3 = ctx->nextToUpdate;
- ZSTD_rescaleFreqs(seqStorePtr);
+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize);
ip += (ip==prefixStart);
- inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -666,13 +685,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0;
- inr = ip;
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */
{ U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor);
for (i = (ip==anchor); i<last_i; i++) {
- const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (ip==anchor)) ? (rep[0] - 1) : rep[i];
+ const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
const U32 repIndex = (U32)(current - repCur);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
@@ -691,7 +709,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_off = i - (ip==anchor);
litlen = opt[0].litlen;
do {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--;
@@ -721,7 +739,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_mlen = matches[u].len;
litlen = opt[0].litlen;
while (mlen <= best_mlen) {
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++;
@@ -765,11 +783,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
}
- best_mlen = 0;
-
+ best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (mlen != 1); i<last_i; i++) {
- const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
+ const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
const U32 repIndex = (U32)(current+cur - repCur);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
@@ -786,20 +803,20 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
}
best_off = i - (opt[cur].mlen != 1);
- if (opt[cur].mlen == 1) {
- litlen = opt[cur].litlen;
- if (cur > litlen) {
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
- } else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
- } else {
- litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
- }
-
- best_mlen = mlen;
+ if (mlen > best_mlen) best_mlen = mlen;
do {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
+ } else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+ }
+
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
@@ -808,30 +825,28 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
- if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
+ if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
last_pos = cur + 1;
goto _storeSequence;
}
- best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch;
-
/* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
- best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur;
+ best_mlen = matches[u].len;
while (mlen <= best_mlen) {
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen)
- price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
else
- price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
} else {
litlen = 0;
- price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
}
if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -877,7 +892,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
offset--;
} else {
if (offset != 0) {
- best_off = ((offset==ZSTD_REP_MOVE_OPT) && (litLength==0)) ? (rep[0] - 1) : (rep[offset]);
+ best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
if (offset != 1) rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = best_off;
@@ -892,7 +907,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */
- { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
+ { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
/* Last Literals */
{ size_t lastLLSize = iend - anchor;
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/compress/zstdmt_compress.c b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstdmt_compress.c
new file mode 100644
index 0000000..97d2b38
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstdmt_compress.c
@@ -0,0 +1,740 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* ====== Tuning parameters ====== */
+#define ZSTDMT_NBTHREADS_MAX 128
+
+
+/* ====== Compiler specifics ====== */
+#if defined(_MSC_VER)
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+#endif
+
+
+/* ====== Dependencies ====== */
+#include <stdlib.h> /* malloc */
+#include <string.h> /* memcpy */
+#include "pool.h" /* threadpool */
+#include "threading.h" /* mutex */
+#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
+#include "zstdmt_compress.h"
+#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+#include "xxhash.h"
+
+
+/* ====== Debug ====== */
+#if 0
+
+# include <stdio.h>
+# include <unistd.h>
+# include <sys/times.h>
+ static unsigned g_debugLevel = 3;
+# define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); }
+# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
+
+# define DEBUG_PRINTHEX(l,p,n) { \
+ unsigned debug_u; \
+ for (debug_u=0; debug_u<(n); debug_u++) \
+ DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \
+ DEBUGLOGRAW(l, " \n"); \
+}
+
+static unsigned long long GetCurrentClockTimeMicroseconds()
+{
+ static clock_t _ticksPerSecond = 0;
+ if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK);
+
+ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
+ return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond);
+}
+
+#define MUTEX_WAIT_TIME_DLEVEL 5
+#define PTHREAD_MUTEX_LOCK(mutex) \
+if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \
+ unsigned long long beforeTime = GetCurrentClockTimeMicroseconds(); \
+ pthread_mutex_lock(mutex); \
+ unsigned long long afterTime = GetCurrentClockTimeMicroseconds(); \
+ unsigned long long elapsedTime = (afterTime-beforeTime); \
+ if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \
+ DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
+ elapsedTime, #mutex); \
+ } \
+} else pthread_mutex_lock(mutex);
+
+#else
+
+# define DEBUGLOG(l, ...) {} /* disabled */
+# define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
+# define DEBUG_PRINTHEX(l,p,n) {}
+
+#endif
+
+
+/* ===== Buffer Pool ===== */
+
+typedef struct buffer_s {
+ void* start;
+ size_t size;
+} buffer_t;
+
+static const buffer_t g_nullBuffer = { NULL, 0 };
+
+typedef struct ZSTDMT_bufferPool_s {
+ unsigned totalBuffers;
+ unsigned nbBuffers;
+ buffer_t bTable[1]; /* variable size */
+} ZSTDMT_bufferPool;
+
+static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads)
+{
+ unsigned const maxNbBuffers = 2*nbThreads + 2;
+ ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)calloc(1, sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t));
+ if (bufPool==NULL) return NULL;
+ bufPool->totalBuffers = maxNbBuffers;
+ bufPool->nbBuffers = 0;
+ return bufPool;
+}
+
+static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
+{
+ unsigned u;
+ if (!bufPool) return; /* compatibility with free on NULL */
+ for (u=0; u<bufPool->totalBuffers; u++)
+ free(bufPool->bTable[u].start);
+ free(bufPool);
+}
+
+/* assumption : invocation from main thread only ! */
+static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
+{
+ if (pool->nbBuffers) { /* try to use an existing buffer */
+ buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
+ size_t const availBufferSize = buf.size;
+ if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) /* large enough, but not too much */
+ return buf;
+ free(buf.start); /* size conditions not respected : scratch this buffer and create a new one */
+ }
+ /* create new buffer */
+ { buffer_t buffer;
+ void* const start = malloc(bSize);
+ if (start==NULL) bSize = 0;
+ buffer.start = start; /* note : start can be NULL if malloc fails ! */
+ buffer.size = bSize;
+ return buffer;
+ }
+}
+
+/* store buffer for later re-use, up to pool capacity */
+static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
+{
+ if (buf.start == NULL) return; /* release on NULL */
+ if (pool->nbBuffers < pool->totalBuffers) {
+ pool->bTable[pool->nbBuffers++] = buf; /* store for later re-use */
+ return;
+ }
+ /* Reached bufferPool capacity (should not happen) */
+ free(buf.start);
+}
+
+
+/* ===== CCtx Pool ===== */
+
+typedef struct {
+ unsigned totalCCtx;
+ unsigned availCCtx;
+ ZSTD_CCtx* cctx[1]; /* variable size */
+} ZSTDMT_CCtxPool;
+
+/* assumption : CCtxPool invocation only from main thread */
+
+/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
+static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
+{
+ unsigned u;
+ for (u=0; u<pool->totalCCtx; u++)
+ ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
+ free(pool);
+}
+
+/* ZSTDMT_createCCtxPool() :
+ * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
+static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads)
+{
+ ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) calloc(1, sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*));
+ if (!cctxPool) return NULL;
+ cctxPool->totalCCtx = nbThreads;
+ cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */
+ cctxPool->cctx[0] = ZSTD_createCCtx();
+ if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
+ DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads);
+ return cctxPool;
+}
+
+static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
+{
+ if (pool->availCCtx) {
+ pool->availCCtx--;
+ return pool->cctx[pool->availCCtx];
+ }
+ return ZSTD_createCCtx(); /* note : can be NULL, when creation fails ! */
+}
+
+static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
+{
+ if (cctx==NULL) return; /* compatibility with release on NULL */
+ if (pool->availCCtx < pool->totalCCtx)
+ pool->cctx[pool->availCCtx++] = cctx;
+ else
+ /* pool overflow : should not happen, since totalCCtx==nbThreads */
+ ZSTD_freeCCtx(cctx);
+}
+
+
+/* ===== Thread worker ===== */
+
+typedef struct {
+ buffer_t buffer;
+ size_t filled;
+} inBuff_t;
+
+typedef struct {
+ ZSTD_CCtx* cctx;
+ buffer_t src;
+ const void* srcStart;
+ size_t srcSize;
+ size_t dictSize;
+ buffer_t dstBuff;
+ size_t cSize;
+ size_t dstFlushed;
+ unsigned firstChunk;
+ unsigned lastChunk;
+ unsigned jobCompleted;
+ unsigned jobScanned;
+ pthread_mutex_t* jobCompleted_mutex;
+ pthread_cond_t* jobCompleted_cond;
+ ZSTD_parameters params;
+ ZSTD_CDict* cdict;
+ unsigned long long fullFrameSize;
+} ZSTDMT_jobDescription;
+
+/* ZSTDMT_compressChunk() : POOL_function type */
+void ZSTDMT_compressChunk(void* jobDescription)
+{
+ ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
+ const void* const src = (const char*)job->srcStart + job->dictSize;
+ buffer_t const dstBuff = job->dstBuff;
+ DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
+ if (job->cdict) {
+ size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize);
+ if (job->cdict) DEBUGLOG(3, "using CDict ");
+ if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+ } else {
+ size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
+ if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+ ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
+ }
+ if (!job->firstChunk) { /* flush frame header */
+ size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
+ if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
+ ZSTD_invalidateRepCodes(job->cctx);
+ }
+
+ DEBUGLOG(4, "Compressing : ");
+ DEBUG_PRINTHEX(4, job->srcStart, 12);
+ job->cSize = (job->lastChunk) ? /* last chunk signal */
+ ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
+ ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
+ DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
+
+_endJob:
+ PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
+ job->jobCompleted = 1;
+ job->jobScanned = 0;
+ pthread_cond_signal(job->jobCompleted_cond);
+ pthread_mutex_unlock(job->jobCompleted_mutex);
+}
+
+
+/* ------------------------------------------ */
+/* ===== Multi-threaded compression ===== */
+/* ------------------------------------------ */
+
+struct ZSTDMT_CCtx_s {
+ POOL_ctx* factory;
+ ZSTDMT_bufferPool* buffPool;
+ ZSTDMT_CCtxPool* cctxPool;
+ pthread_mutex_t jobCompleted_mutex;
+ pthread_cond_t jobCompleted_cond;
+ size_t targetSectionSize;
+ size_t marginSize;
+ size_t inBuffSize;
+ size_t dictSize;
+ size_t targetDictSize;
+ inBuff_t inBuff;
+ ZSTD_parameters params;
+ XXH64_state_t xxhState;
+ unsigned nbThreads;
+ unsigned jobIDMask;
+ unsigned doneJobID;
+ unsigned nextJobID;
+ unsigned frameEnded;
+ unsigned allJobsCompleted;
+ unsigned overlapRLog;
+ unsigned long long frameContentSize;
+ size_t sectionSize;
+ ZSTD_CDict* cdict;
+ ZSTD_CStream* cstream;
+ ZSTDMT_jobDescription jobs[1]; /* variable size (must lies at the end) */
+};
+
+ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
+{
+ ZSTDMT_CCtx* cctx;
+ U32 const minNbJobs = nbThreads + 2;
+ U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
+ U32 const nbJobs = 1 << nbJobsLog2;
+ DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n",
+ nbThreads, minNbJobs, nbJobsLog2, nbJobs);
+ if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
+ cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription));
+ if (!cctx) return NULL;
+ cctx->nbThreads = nbThreads;
+ cctx->jobIDMask = nbJobs - 1;
+ cctx->allJobsCompleted = 1;
+ cctx->sectionSize = 0;
+ cctx->overlapRLog = 3;
+ cctx->factory = POOL_create(nbThreads, 1);
+ cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
+ cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
+ if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) { /* one object was not created */
+ ZSTDMT_freeCCtx(cctx);
+ return NULL;
+ }
+ if (nbThreads==1) {
+ cctx->cstream = ZSTD_createCStream();
+ if (!cctx->cstream) {
+ ZSTDMT_freeCCtx(cctx); return NULL;
+ } }
+ pthread_mutex_init(&cctx->jobCompleted_mutex, NULL); /* Todo : check init function return */
+ pthread_cond_init(&cctx->jobCompleted_cond, NULL);
+ DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads);
+ return cctx;
+}
+
+/* ZSTDMT_releaseAllJobResources() :
+ * Ensure all workers are killed first. */
+static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
+{
+ unsigned jobID;
+ for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].dstBuff);
+ mtctx->jobs[jobID].dstBuff = g_nullBuffer;
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].src);
+ mtctx->jobs[jobID].src = g_nullBuffer;
+ ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[jobID].cctx);
+ mtctx->jobs[jobID].cctx = NULL;
+ }
+ memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer);
+ mtctx->inBuff.buffer = g_nullBuffer;
+ mtctx->allJobsCompleted = 1;
+}
+
+size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
+{
+ if (mtctx==NULL) return 0; /* compatible with free on NULL */
+ POOL_free(mtctx->factory);
+ if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
+ ZSTDMT_freeBufferPool(mtctx->buffPool); /* release job resources into pools first */
+ ZSTDMT_freeCCtxPool(mtctx->cctxPool);
+ ZSTD_freeCDict(mtctx->cdict);
+ ZSTD_freeCStream(mtctx->cstream);
+ pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
+ pthread_cond_destroy(&mtctx->jobCompleted_cond);
+ free(mtctx);
+ return 0;
+}
+
+size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
+{
+ switch(parameter)
+ {
+ case ZSTDMT_p_sectionSize :
+ mtctx->sectionSize = value;
+ return 0;
+ case ZSTDMT_p_overlapSectionLog :
+ DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
+ mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
+ return 0;
+ default :
+ return ERROR(compressionParameter_unsupported);
+ }
+}
+
+
+/* ------------------------------------------ */
+/* ===== Multi-threaded compression ===== */
+/* ------------------------------------------ */
+
+size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel)
+{
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
+ size_t const chunkTargetSize = (size_t)1 << (params.cParams.windowLog + 2);
+ unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + (srcSize < chunkTargetSize) /* min 1 */;
+ unsigned nbChunks = MIN(nbChunksMax, mtctx->nbThreads);
+ size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
+ size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0xFFFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */
+ size_t remainingSrcSize = srcSize;
+ const char* const srcStart = (const char*)src;
+ size_t frameStartPos = 0;
+
+ DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes ", params.cParams.windowLog, (U32)chunkTargetSize);
+ DEBUGLOG(2, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
+ params.fParams.contentSizeFlag = 1;
+
+ if (nbChunks==1) { /* fallback to single-thread mode */
+ ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
+ return ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
+ }
+
+ { unsigned u;
+ for (u=0; u<nbChunks; u++) {
+ size_t const chunkSize = MIN(remainingSrcSize, avgChunkSize);
+ size_t const dstBufferCapacity = u ? ZSTD_compressBound(chunkSize) : dstCapacity;
+ buffer_t const dstAsBuffer = { dst, dstCapacity };
+ buffer_t const dstBuffer = u ? ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity) : dstAsBuffer;
+ ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(mtctx->cctxPool);
+
+ if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+ mtctx->jobs[u].cSize = ERROR(memory_allocation); /* job result */
+ mtctx->jobs[u].jobCompleted = 1;
+ nbChunks = u+1;
+ break; /* let's wait for previous jobs to complete, but don't start new ones */
+ }
+
+ mtctx->jobs[u].srcStart = srcStart + frameStartPos;
+ mtctx->jobs[u].srcSize = chunkSize;
+ mtctx->jobs[u].fullFrameSize = srcSize;
+ mtctx->jobs[u].params = params;
+ mtctx->jobs[u].dstBuff = dstBuffer;
+ mtctx->jobs[u].cctx = cctx;
+ mtctx->jobs[u].firstChunk = (u==0);
+ mtctx->jobs[u].lastChunk = (u==nbChunks-1);
+ mtctx->jobs[u].jobCompleted = 0;
+ mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
+ mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;
+
+ DEBUGLOG(3, "posting job %u (%u bytes)", u, (U32)chunkSize);
+ DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12);
+ POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
+
+ frameStartPos += chunkSize;
+ remainingSrcSize -= chunkSize;
+ } }
+ /* note : since nbChunks <= nbThreads, all jobs should be running immediately in parallel */
+
+ { unsigned chunkID;
+ size_t error = 0, dstPos = 0;
+ for (chunkID=0; chunkID<nbChunks; chunkID++) {
+ DEBUGLOG(3, "waiting for chunk %u ", chunkID);
+ PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
+ while (mtctx->jobs[chunkID].jobCompleted==0) {
+ DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID);
+ pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
+ }
+ pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
+ DEBUGLOG(3, "ready to write chunk %u ", chunkID);
+
+ ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
+ mtctx->jobs[chunkID].cctx = NULL;
+ mtctx->jobs[chunkID].srcStart = NULL;
+ { size_t const cSize = mtctx->jobs[chunkID].cSize;
+ if (ZSTD_isError(cSize)) error = cSize;
+ if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall);
+ if (chunkID) { /* note : chunk 0 is already written directly into dst */
+ if (!error) memcpy((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize);
+ ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff);
+ mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
+ }
+ dstPos += cSize ;
+ }
+ }
+ if (!error) DEBUGLOG(3, "compressed size : %u ", (U32)dstPos);
+ return error ? error : dstPos;
+ }
+
+}
+
+
+/* ====================================== */
+/* ======= Streaming API ======= */
+/* ====================================== */
+
+static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) {
+ while (zcs->doneJobID < zcs->nextJobID) {
+ unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
+ PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+ while (zcs->jobs[jobID].jobCompleted==0) {
+ DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */
+ pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
+ }
+ pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+ zcs->doneJobID++;
+ }
+}
+
+
+static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
+ const void* dict, size_t dictSize, unsigned updateDict,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ ZSTD_customMem const cmem = { NULL, NULL, NULL };
+ DEBUGLOG(3, "Started new compression, with windowLog : %u", params.cParams.windowLog);
+ if (zcs->nbThreads==1) return ZSTD_initCStream_advanced(zcs->cstream, dict, dictSize, params, pledgedSrcSize);
+ if (zcs->allJobsCompleted == 0) { /* previous job not correctly finished */
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ zcs->allJobsCompleted = 1;
+ }
+ zcs->params = params;
+ if (updateDict) {
+ ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
+ if (dict && dictSize) {
+ zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params, cmem);
+ if (zcs->cdict == NULL) return ERROR(memory_allocation);
+ } }
+ zcs->frameContentSize = pledgedSrcSize;
+ zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
+ DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
+ DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
+ zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
+ zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
+ zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
+ DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
+ zcs->marginSize = zcs->targetSectionSize >> 2;
+ zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
+ zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+ if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
+ zcs->inBuff.filled = 0;
+ zcs->dictSize = 0;
+ zcs->doneJobID = 0;
+ zcs->nextJobID = 0;
+ zcs->frameEnded = 0;
+ zcs->allJobsCompleted = 0;
+ if (params.fParams.checksumFlag) XXH64_reset(&zcs->xxhState, 0);
+ return 0;
+}
+
+size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ return ZSTDMT_initCStream_internal(zcs, dict, dictSize, 1, params, pledgedSrcSize);
+}
+
+/* ZSTDMT_resetCStream() :
+ * pledgedSrcSize is optional and can be zero == unknown */
+size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
+{
+ if (zcs->nbThreads==1) return ZSTD_resetCStream(zcs->cstream, pledgedSrcSize);
+ return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, pledgedSrcSize);
+}
+
+size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
+ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
+ return ZSTDMT_initCStream_internal(zcs, NULL, 0, 1, params, 0);
+}
+
+
+static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsigned endFrame)
+{
+ size_t const dstBufferCapacity = ZSTD_compressBound(srcSize);
+ buffer_t const dstBuffer = ZSTDMT_getBuffer(zcs->buffPool, dstBufferCapacity);
+ ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(zcs->cctxPool);
+ unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;
+
+ if ((cctx==NULL) || (dstBuffer.start==NULL)) {
+ zcs->jobs[jobID].jobCompleted = 1;
+ zcs->nextJobID++;
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ return ERROR(memory_allocation);
+ }
+
+ DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
+ zcs->jobs[jobID].src = zcs->inBuff.buffer;
+ zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
+ zcs->jobs[jobID].srcSize = srcSize;
+ zcs->jobs[jobID].dictSize = zcs->dictSize; /* note : zcs->inBuff.filled is presumed >= srcSize + dictSize */
+ zcs->jobs[jobID].params = zcs->params;
+ if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0; /* do not calculate checksum within sections, just keep it in header for first section */
+ zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
+ zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
+ zcs->jobs[jobID].dstBuff = dstBuffer;
+ zcs->jobs[jobID].cctx = cctx;
+ zcs->jobs[jobID].firstChunk = (zcs->nextJobID==0);
+ zcs->jobs[jobID].lastChunk = endFrame;
+ zcs->jobs[jobID].jobCompleted = 0;
+ zcs->jobs[jobID].dstFlushed = 0;
+ zcs->jobs[jobID].jobCompleted_mutex = &zcs->jobCompleted_mutex;
+ zcs->jobs[jobID].jobCompleted_cond = &zcs->jobCompleted_cond;
+
+ /* get a new buffer for next input */
+ if (!endFrame) {
+ size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
+ zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+ if (zcs->inBuff.buffer.start == NULL) { /* not enough memory to allocate next input buffer */
+ zcs->jobs[jobID].jobCompleted = 1;
+ zcs->nextJobID++;
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ return ERROR(memory_allocation);
+ }
+ DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled);
+ zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
+ DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src", (U32)zcs->inBuff.filled, (U32)newDictSize, (U32)(zcs->inBuff.filled - newDictSize));
+ memmove(zcs->inBuff.buffer.start, (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, zcs->inBuff.filled);
+ DEBUGLOG(5, "new inBuff pre-filled");
+ zcs->dictSize = newDictSize;
+ } else {
+ zcs->inBuff.buffer = g_nullBuffer;
+ zcs->inBuff.filled = 0;
+ zcs->dictSize = 0;
+ zcs->frameEnded = 1;
+ if (zcs->nextJobID == 0)
+ zcs->params.fParams.checksumFlag = 0; /* single chunk : checksum is calculated directly within worker thread */
+ }
+
+ DEBUGLOG(3, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)", zcs->nextJobID, (U32)zcs->jobs[jobID].srcSize, zcs->jobs[jobID].lastChunk, zcs->doneJobID, zcs->doneJobID & zcs->jobIDMask);
+ POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]); /* this call is blocking when thread worker pool is exhausted */
+ zcs->nextJobID++;
+ return 0;
+}
+
+
+/* ZSTDMT_flushNextJob() :
+ * output : will be updated with amount of data flushed .
+ * blockToFlush : if >0, the function will block and wait if there is no data available to flush .
+ * @return : amount of data remaining within internal buffer, 1 if unknown but > 0, 0 if no more, or an error code */
+static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush)
+{
+ unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
+ if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */
+ PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
+ while (zcs->jobs[wJobID].jobCompleted==0) {
+ DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
+ if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */
+ pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */
+ }
+ pthread_mutex_unlock(&zcs->jobCompleted_mutex);
+ /* compression job completed : output can be flushed */
+ { ZSTDMT_jobDescription job = zcs->jobs[wJobID];
+ if (!job.jobScanned) {
+ if (ZSTD_isError(job.cSize)) {
+ DEBUGLOG(5, "compression error detected ");
+ ZSTDMT_waitForAllJobsCompleted(zcs);
+ ZSTDMT_releaseAllJobResources(zcs);
+ return job.cSize;
+ }
+ ZSTDMT_releaseCCtx(zcs->cctxPool, job.cctx);
+ zcs->jobs[wJobID].cctx = NULL;
+ DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag);
+ if (zcs->params.fParams.checksumFlag) {
+ XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
+ if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */
+ U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
+ DEBUGLOG(4, "writing checksum : %08X \n", checksum);
+ MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum);
+ job.cSize += 4;
+ zcs->jobs[wJobID].cSize += 4;
+ } }
+ ZSTDMT_releaseBuffer(zcs->buffPool, job.src);
+ zcs->jobs[wJobID].srcStart = NULL;
+ zcs->jobs[wJobID].src = g_nullBuffer;
+ zcs->jobs[wJobID].jobScanned = 1;
+ }
+ { size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
+ DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
+ memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
+ output->pos += toWrite;
+ job.dstFlushed += toWrite;
+ }
+ if (job.dstFlushed == job.cSize) { /* output buffer fully flushed => move to next one */
+ ZSTDMT_releaseBuffer(zcs->buffPool, job.dstBuff);
+ zcs->jobs[wJobID].dstBuff = g_nullBuffer;
+ zcs->jobs[wJobID].jobCompleted = 0;
+ zcs->doneJobID++;
+ } else {
+ zcs->jobs[wJobID].dstFlushed = job.dstFlushed;
+ }
+ /* return value : how many bytes left in buffer ; fake it to 1 if unknown but >0 */
+ if (job.cSize > job.dstFlushed) return (job.cSize - job.dstFlushed);
+ if (zcs->doneJobID < zcs->nextJobID) return 1; /* still some buffer to flush */
+ zcs->allJobsCompleted = zcs->frameEnded; /* frame completed and entirely flushed */
+ return 0; /* everything flushed */
+} }
+
+
+size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+ size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize;
+ if (zcs->frameEnded) return ERROR(stage_wrong); /* current frame being ended. Only flush is allowed. Restart with init */
+ if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input);
+
+ /* fill input buffer */
+ { size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
+ memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, input->src, toLoad);
+ input->pos += toLoad;
+ zcs->inBuff.filled += toLoad;
+ }
+
+ if ( (zcs->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */
+ && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { /* avoid overwriting job round buffer */
+ CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) );
+ }
+
+ /* check for data to flush */
+ CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize)) ); /* block if it wasn't possible to create new job due to saturation */
+
+ /* recommended next input size : fill current input buffer */
+ return zcs->inBuffSize - zcs->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */
+}
+
+
+static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned endFrame)
+{
+ size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
+
+ if (srcSize) DEBUGLOG(4, "flushing : %u bytes left to compress", (U32)srcSize);
+ if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
+ && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
+ CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
+ }
+
+ /* check if there is any data available to flush */
+ DEBUGLOG(5, "zcs->doneJobID : %u ; zcs->nextJobID : %u ", zcs->doneJobID, zcs->nextJobID);
+ return ZSTDMT_flushNextJob(zcs, output, 1);
+}
+
+
+size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+{
+ if (zcs->nbThreads==1) return ZSTD_flushStream(zcs->cstream, output);
+ return ZSTDMT_flushStream_internal(zcs, output, 0);
+}
+
+size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
+{
+ if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output);
+ return ZSTDMT_flushStream_internal(zcs, output, 1);
+}
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/compress/zstdmt_compress.h b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstdmt_compress.h
new file mode 100644
index 0000000..27f78ee
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/compress/zstdmt_compress.h
@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+ #ifndef ZSTDMT_COMPRESS_H
+ #define ZSTDMT_COMPRESS_H
+
+ #if defined (__cplusplus)
+ extern "C" {
+ #endif
+
+
+/* Note : All prototypes defined in this file shall be considered experimental.
+ * There is no guarantee of API continuity (yet) on any of these prototypes */
+
+/* === Dependencies === */
+#include <stddef.h> /* size_t */
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
+#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
+
+
+/* === Simple one-pass functions === */
+
+typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
+ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
+ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
+
+ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ int compressionLevel);
+
+
+/* === Streaming functions === */
+
+ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+
+ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
+
+
+/* === Advanced functions and parameters === */
+
+#ifndef ZSTDMT_SECTION_SIZE_MIN
+# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
+#endif
+
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+
+/* ZSDTMT_parameter :
+ * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
+typedef enum {
+ ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
+ ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
+} ZSDTMT_parameter;
+
+/* ZSTDMT_setMTCtxParameter() :
+ * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
+ * The function must be called typically after ZSTD_createCCtx().
+ * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTDMT_COMPRESS_H */
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/decompress/huf_decompress.c b/c-blosc/internal-complibs/zstd-1.1.3/decompress/huf_decompress.c
similarity index 98%
rename from c-blosc/internal-complibs/zstd-1.0.0/decompress/huf_decompress.c
rename to c-blosc/internal-complibs/zstd-1.1.3/decompress/huf_decompress.c
index a5521bd..a342dfb 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/decompress/huf_decompress.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/decompress/huf_decompress.c
@@ -43,16 +43,8 @@
# define inline /* disable inline */
#endif
-
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -366,13 +358,15 @@ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* doubl
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
+/* HUF_fillDTableX4Level2() :
+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{
HUF_DEltX4 DElt;
- U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
+ U32 rankVal[HUF_TABLELOG_MAX + 1];
/* get pre-calculated rankVal */
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
@@ -406,14 +400,14 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
} }
}
-typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1];
+typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
const sortedSymbol_t* sortedList, const U32 sortedListSize,
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline)
{
- U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
+ U32 rankVal[HUF_TABLELOG_MAX + 1];
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
const U32 minBits = nbBitsBaseline - maxWeight;
U32 s;
@@ -454,8 +448,8 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
- U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 };
- U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 };
+ U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 };
+ U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 };
U32* const rankStart = rankStart0+1;
rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
@@ -466,7 +460,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */
- if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
+ if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/decompress/zstd_decompress.c b/c-blosc/internal-complibs/zstd-1.1.3/decompress/zstd_decompress.c
similarity index 58%
rename from c-blosc/internal-complibs/zstd-1.0.0/decompress/zstd_decompress.c
rename to c-blosc/internal-complibs/zstd-1.1.3/decompress/zstd_decompress.c
index fb1ee35..9c04503 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/decompress/zstd_decompress.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/decompress/zstd_decompress.c
@@ -28,14 +28,13 @@
# define ZSTD_LEGACY_SUPPORT 0
#endif
-
/*!
* MAXWINDOWSIZE_DEFAULT :
* maximum window size accepted by DStream, by default.
* Frames requiring more memory will be rejected.
*/
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-# define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */
+# define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
#endif
@@ -57,24 +56,15 @@
#endif
-/*-*******************************************************
-* Compiler specifics
-*********************************************************/
-#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
-# include <intrin.h> /* For Visual 2005 */
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
+#if defined(_MSC_VER)
+# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+# define ZSTD_PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
+#elif defined(__GNUC__)
+# define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
+# define ZSTD_PREFETCH(ptr) /* disabled */
#endif
-
/*-*************************************
* Macros
***************************************/
@@ -99,8 +89,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
struct ZSTD_DCtx_s
{
+ const FSE_DTable* LLTptr;
+ const FSE_DTable* MLTptr;
+ const FSE_DTable* OFTptr;
+ const HUF_DTable* HUFptr;
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
- FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+ FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
const void* previousDstEnd;
@@ -119,30 +113,33 @@ struct ZSTD_DCtx_s
U32 dictID;
const BYTE* litPtr;
ZSTD_customMem customMem;
- size_t litBufSize;
size_t litSize;
size_t rleSize;
BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
-size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); }
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); }
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{
- dctx->expected = ZSTD_frameHeaderSize_min;
+ dctx->expected = ZSTD_frameHeaderSize_prefix;
dctx->stage = ZSTDds_getFrameHeaderSize;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
- dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
+ dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
dctx->litEntropy = dctx->fseEntropy = 0;
dctx->dictID = 0;
- MEM_STATIC_ASSERT(sizeof(dctx->rep)==sizeof(repStartValue));
- memcpy(dctx->rep, repStartValue, sizeof(repStartValue));
+ MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
+ memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
+ dctx->LLTptr = dctx->LLTable;
+ dctx->MLTptr = dctx->MLTable;
+ dctx->OFTptr = dctx->OFTable;
+ dctx->HUFptr = dctx->hufTable;
return 0;
}
@@ -153,7 +150,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
- dctx = (ZSTD_DCtx*) ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
+ dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx);
@@ -178,24 +175,62 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
}
+static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
+{
+ ZSTD_decompressBegin(dstDCtx); /* init */
+ if (srcDCtx) { /* support refDCtx on NULL */
+ dstDCtx->dictEnd = srcDCtx->dictEnd;
+ dstDCtx->vBase = srcDCtx->vBase;
+ dstDCtx->base = srcDCtx->base;
+ dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
+ dstDCtx->dictID = srcDCtx->dictID;
+ dstDCtx->litEntropy = srcDCtx->litEntropy;
+ dstDCtx->fseEntropy = srcDCtx->fseEntropy;
+ dstDCtx->LLTptr = srcDCtx->LLTable;
+ dstDCtx->MLTptr = srcDCtx->MLTable;
+ dstDCtx->OFTptr = srcDCtx->OFTable;
+ dstDCtx->HUFptr = srcDCtx->hufTable;
+ dstDCtx->rep[0] = srcDCtx->rep[0];
+ dstDCtx->rep[1] = srcDCtx->rep[1];
+ dstDCtx->rep[2] = srcDCtx->rep[2];
+ }
+}
+
/*-*************************************************************
* Decompression section
***************************************************************/
-/* See compression format details in : zstd_compression_format.md */
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+unsigned ZSTD_isFrame(const void* buffer, size_t size)
+{
+ if (size < 4) return 0;
+ { U32 const magic = MEM_readLE32(buffer);
+ if (magic == ZSTD_MAGICNUMBER) return 1;
+ if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+ }
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+ if (ZSTD_isLegacy(buffer, size)) return 1;
+#endif
+ return 0;
+}
+
/** ZSTD_frameHeaderSize() :
-* srcSize must be >= ZSTD_frameHeaderSize_min.
+* srcSize must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{
- if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
+ if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
{ BYTE const fhd = ((const BYTE*)src)[4];
U32 const dictID= fhd & 3;
U32 const singleSegment = (fhd >> 5) & 1;
U32 const fcsId = fhd >> 6;
- return ZSTD_frameHeaderSize_min + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ (singleSegment && !fcsId);
}
}
@@ -210,7 +245,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
{
const BYTE* ip = (const BYTE*)src;
- if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
+ if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
@@ -240,7 +275,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
if (!singleSegment) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
- if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
+ if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
@@ -262,7 +297,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
}
if (!windowSize) windowSize = (U32)frameContentSize;
- if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
+ if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID;
@@ -293,14 +328,16 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
/** ZSTD_decodeFrameHeader() :
-* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
+* `headerSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
-static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
{
- size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
+ if (ZSTD_isError(result)) return result; /* invalid header */
+ if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
- return result;
+ return 0;
}
@@ -369,43 +406,41 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
{
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */
- { singleStream = !lhlCode;
- lhSize = 3;
- litSize = (lhc >> 4) & 0x3FF;
- litCSize = (lhc >> 14) & 0x3FF;
- break;
- }
+ singleStream = !lhlCode;
+ lhSize = 3;
+ litSize = (lhc >> 4) & 0x3FF;
+ litCSize = (lhc >> 14) & 0x3FF;
+ break;
case 2:
/* 2 - 2 - 14 - 14 */
- { lhSize = 4;
- litSize = (lhc >> 4) & 0x3FFF;
- litCSize = lhc >> 18;
- break;
- }
+ lhSize = 4;
+ litSize = (lhc >> 4) & 0x3FFF;
+ litCSize = lhc >> 18;
+ break;
case 3:
/* 2 - 2 - 18 - 18 */
- { lhSize = 5;
- litSize = (lhc >> 4) & 0x3FFFF;
- litCSize = (lhc >> 22) + (istart[4] << 10);
- break;
- }
+ lhSize = 5;
+ litSize = (lhc >> 4) & 0x3FFFF;
+ litCSize = (lhc >> 22) + (istart[4] << 10);
+ break;
}
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ?
( singleStream ?
- HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) :
- HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) :
+ HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
+ HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
( singleStream ?
HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
dctx->litEntropy = 1;
+ if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
@@ -432,13 +467,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
- dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@@ -463,37 +497,215 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break;
}
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[lhSize], litSize);
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
default:
return ERROR(corruption_detected); /* impossible */
}
-
}
}
+typedef union {
+ FSE_decode_t realData;
+ U32 alignedBy4;
+} FSE_decode_t4;
+
+static const FSE_decode_t4 LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
+ { { LL_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */
+ { { 0, 0, 4 } }, /* 0 : base, symbol, bits */
+ { { 16, 0, 4 } },
+ { { 32, 1, 5 } },
+ { { 0, 3, 5 } },
+ { { 0, 4, 5 } },
+ { { 0, 6, 5 } },
+ { { 0, 7, 5 } },
+ { { 0, 9, 5 } },
+ { { 0, 10, 5 } },
+ { { 0, 12, 5 } },
+ { { 0, 14, 6 } },
+ { { 0, 16, 5 } },
+ { { 0, 18, 5 } },
+ { { 0, 19, 5 } },
+ { { 0, 21, 5 } },
+ { { 0, 22, 5 } },
+ { { 0, 24, 5 } },
+ { { 32, 25, 5 } },
+ { { 0, 26, 5 } },
+ { { 0, 27, 6 } },
+ { { 0, 29, 6 } },
+ { { 0, 31, 6 } },
+ { { 32, 0, 4 } },
+ { { 0, 1, 4 } },
+ { { 0, 2, 5 } },
+ { { 32, 4, 5 } },
+ { { 0, 5, 5 } },
+ { { 32, 7, 5 } },
+ { { 0, 8, 5 } },
+ { { 32, 10, 5 } },
+ { { 0, 11, 5 } },
+ { { 0, 13, 6 } },
+ { { 32, 16, 5 } },
+ { { 0, 17, 5 } },
+ { { 32, 19, 5 } },
+ { { 0, 20, 5 } },
+ { { 32, 22, 5 } },
+ { { 0, 23, 5 } },
+ { { 0, 25, 4 } },
+ { { 16, 25, 4 } },
+ { { 32, 26, 5 } },
+ { { 0, 28, 6 } },
+ { { 0, 30, 6 } },
+ { { 48, 0, 4 } },
+ { { 16, 1, 4 } },
+ { { 32, 2, 5 } },
+ { { 32, 3, 5 } },
+ { { 32, 5, 5 } },
+ { { 32, 6, 5 } },
+ { { 32, 8, 5 } },
+ { { 32, 9, 5 } },
+ { { 32, 11, 5 } },
+ { { 32, 12, 5 } },
+ { { 0, 15, 6 } },
+ { { 32, 17, 5 } },
+ { { 32, 18, 5 } },
+ { { 32, 20, 5 } },
+ { { 32, 21, 5 } },
+ { { 32, 23, 5 } },
+ { { 32, 24, 5 } },
+ { { 0, 35, 6 } },
+ { { 0, 34, 6 } },
+ { { 0, 33, 6 } },
+ { { 0, 32, 6 } },
+}; /* LL_defaultDTable */
+
+static const FSE_decode_t4 ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
+ { { ML_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */
+ { { 0, 0, 6 } }, /* 0 : base, symbol, bits */
+ { { 0, 1, 4 } },
+ { { 32, 2, 5 } },
+ { { 0, 3, 5 } },
+ { { 0, 5, 5 } },
+ { { 0, 6, 5 } },
+ { { 0, 8, 5 } },
+ { { 0, 10, 6 } },
+ { { 0, 13, 6 } },
+ { { 0, 16, 6 } },
+ { { 0, 19, 6 } },
+ { { 0, 22, 6 } },
+ { { 0, 25, 6 } },
+ { { 0, 28, 6 } },
+ { { 0, 31, 6 } },
+ { { 0, 33, 6 } },
+ { { 0, 35, 6 } },
+ { { 0, 37, 6 } },
+ { { 0, 39, 6 } },
+ { { 0, 41, 6 } },
+ { { 0, 43, 6 } },
+ { { 0, 45, 6 } },
+ { { 16, 1, 4 } },
+ { { 0, 2, 4 } },
+ { { 32, 3, 5 } },
+ { { 0, 4, 5 } },
+ { { 32, 6, 5 } },
+ { { 0, 7, 5 } },
+ { { 0, 9, 6 } },
+ { { 0, 12, 6 } },
+ { { 0, 15, 6 } },
+ { { 0, 18, 6 } },
+ { { 0, 21, 6 } },
+ { { 0, 24, 6 } },
+ { { 0, 27, 6 } },
+ { { 0, 30, 6 } },
+ { { 0, 32, 6 } },
+ { { 0, 34, 6 } },
+ { { 0, 36, 6 } },
+ { { 0, 38, 6 } },
+ { { 0, 40, 6 } },
+ { { 0, 42, 6 } },
+ { { 0, 44, 6 } },
+ { { 32, 1, 4 } },
+ { { 48, 1, 4 } },
+ { { 16, 2, 4 } },
+ { { 32, 4, 5 } },
+ { { 32, 5, 5 } },
+ { { 32, 7, 5 } },
+ { { 32, 8, 5 } },
+ { { 0, 11, 6 } },
+ { { 0, 14, 6 } },
+ { { 0, 17, 6 } },
+ { { 0, 20, 6 } },
+ { { 0, 23, 6 } },
+ { { 0, 26, 6 } },
+ { { 0, 29, 6 } },
+ { { 0, 52, 6 } },
+ { { 0, 51, 6 } },
+ { { 0, 50, 6 } },
+ { { 0, 49, 6 } },
+ { { 0, 48, 6 } },
+ { { 0, 47, 6 } },
+ { { 0, 46, 6 } },
+}; /* ML_defaultDTable */
+
+static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
+ { { OF_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */
+ { { 0, 0, 5 } }, /* 0 : base, symbol, bits */
+ { { 0, 6, 4 } },
+ { { 0, 9, 5 } },
+ { { 0, 15, 5 } },
+ { { 0, 21, 5 } },
+ { { 0, 3, 5 } },
+ { { 0, 7, 4 } },
+ { { 0, 12, 5 } },
+ { { 0, 18, 5 } },
+ { { 0, 23, 5 } },
+ { { 0, 5, 5 } },
+ { { 0, 8, 4 } },
+ { { 0, 14, 5 } },
+ { { 0, 20, 5 } },
+ { { 0, 2, 5 } },
+ { { 16, 7, 4 } },
+ { { 0, 11, 5 } },
+ { { 0, 17, 5 } },
+ { { 0, 22, 5 } },
+ { { 0, 4, 5 } },
+ { { 16, 8, 4 } },
+ { { 0, 13, 5 } },
+ { { 0, 19, 5 } },
+ { { 0, 1, 5 } },
+ { { 16, 6, 4 } },
+ { { 0, 10, 5 } },
+ { { 0, 16, 5 } },
+ { { 0, 28, 5 } },
+ { { 0, 27, 5 } },
+ { { 0, 26, 5 } },
+ { { 0, 25, 5 } },
+ { { 0, 24, 5 } },
+}; /* OF_defaultDTable */
+
/*! ZSTD_buildSeqTable() :
@return : nb bytes read from src,
or an error code if it fails, testable with ZSTD_isError()
*/
-FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
+static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr,
+ symbolEncodingType_e type, U32 max, U32 maxLog,
const void* src, size_t srcSize,
- const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
+ const FSE_decode_t4* defaultTable, U32 flagRepeatTable)
{
+ const void* const tmpPtr = defaultTable; /* bypass strict aliasing */
switch(type)
{
case set_rle :
if (!srcSize) return ERROR(srcSize_wrong);
if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected);
- FSE_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */
+ FSE_buildDTable_rle(DTableSpace, *(const BYTE*)src);
+ *DTablePtr = DTableSpace;
return 1;
case set_basic :
- FSE_buildDTable(DTable, defaultNorm, max, defaultLog);
+ *DTablePtr = (const FSE_DTable*)tmpPtr;
return 0;
case set_repeat:
if (!flagRepeatTable) return ERROR(corruption_detected);
@@ -505,14 +717,13 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
if (FSE_isError(headerSize)) return ERROR(corruption_detected);
if (tableLog > maxLog) return ERROR(corruption_detected);
- FSE_buildDTable(DTable, norm, max, tableLog);
+ FSE_buildDTable(DTableSpace, norm, max, tableLog);
+ *DTablePtr = DTableSpace;
return headerSize;
} }
}
-
-size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
- FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable,
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const void* src, size_t srcSize)
{
const BYTE* const istart = (const BYTE* const)src;
@@ -526,10 +737,13 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF)
+ if (nbSeq == 0xFF) {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
- else
+ } else {
+ if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
}
*nbSeqPtr = nbSeq;
}
@@ -542,18 +756,25 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
ip++;
/* Build DTables */
- { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
+ { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr,
+ LLtype, MaxLL, LLFSELog,
+ ip, iend-ip, LL_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
ip += llhSize;
}
- { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
+ { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr,
+ OFtype, MaxOff, OffFSELog,
+ ip, iend-ip, OF_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
ip += ofhSize;
}
- { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
+ { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr,
+ MLtype, MaxML, MLFSELog,
+ ip, iend-ip, ML_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
ip += mlhSize;
- } }
+ }
+ }
return ip-istart;
}
@@ -563,6 +784,7 @@ typedef struct {
size_t litLength;
size_t matchLength;
size_t offset;
+ const BYTE* match;
} seq_t;
typedef struct {
@@ -571,9 +793,61 @@ typedef struct {
FSE_DState_t stateOffb;
FSE_DState_t stateML;
size_t prevOffset[ZSTD_REP_NUM];
+ const BYTE* base;
+ size_t pos;
+ iPtrDiff gotoDict;
} seqState_t;
+FORCE_NOINLINE
+size_t ZSTD_execSequenceLast7(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = oLitEnd - sequence.offset;
+
+ /* check */
+ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */
+
+ /* copy literals */
+ if (op < oend_w) {
+ ZSTD_wildcopy(op, *litPtr, oend_w - op);
+ *litPtr += oend_w - op;
+ op = oend_w;
+ }
+ while (op < oLitEnd) *op++ = *(*litPtr)++;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+ match = dictEnd - (base-match);
+ if (match + sequence.matchLength <= dictEnd) {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ } }
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+}
+
+
+
+
static seq_t ZSTD_decodeSequence(seqState_t* seqState)
{
seq_t seq;
@@ -599,24 +873,25 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
static const U32 OF_base[MaxOff+1] = {
- 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
- 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
- 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
- 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
+ 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
+ 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
+ 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
+ 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
/* sequence */
{ size_t offset;
if (!ofCode)
offset = 0;
else {
- offset = OF_base[ofCode] + BIT_readBits(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
}
if (ofCode <= 1) {
offset += (llCode==0);
if (offset) {
- size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp;
@@ -631,10 +906,10 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
seq.offset = offset;
}
- seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
+ seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
- seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&seqState->DStream, llBits) : 0); /* <= 16 bits */
+ seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if (MEM_32bits() ||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream);
@@ -651,7 +926,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit_w,
+ const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@@ -662,8 +937,9 @@ size_t ZSTD_execSequence(BYTE* op,
const BYTE* match = oLitEnd - sequence.offset;
/* check */
- if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
- if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
/* copy Literals */
ZSTD_copy8(op, *litPtr);
@@ -676,7 +952,7 @@ size_t ZSTD_execSequence(BYTE* op,
if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
- match = dictEnd - (base-match);
+ match += (dictEnd-base);
if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
@@ -687,7 +963,13 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_w || sequence.matchLength < MINMATCH) {
+ U32 i;
+ for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */
if (sequence.offset < 8) {
@@ -715,7 +997,7 @@ size_t ZSTD_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -732,18 +1014,14 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize;
- FSE_DTable* DTableLL = dctx->LLTable;
- FSE_DTable* DTableML = dctx->MLTable;
- FSE_DTable* DTableOffb = dctx->OffTable;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq;
/* Build Decoding Tables */
- { size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
+ { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize;
}
@@ -753,16 +1031,15 @@ static size_t ZSTD_decompressSequences(
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
- { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
- if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
- FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
- FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
- FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState);
- size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }
@@ -784,14 +1061,247 @@ static size_t ZSTD_decompressSequences(
}
-static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
+static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
{
- if (dst != dctx->previousDstEnd) { /* not contiguous */
- dctx->dictEnd = dctx->previousDstEnd;
- dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
- dctx->base = dst;
- dctx->previousDstEnd = dst;
+ seq_t seq;
+
+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
+
+ U32 const llBits = LL_bits[llCode];
+ U32 const mlBits = ML_bits[mlCode];
+ U32 const ofBits = ofCode;
+ U32 const totalBits = llBits+mlBits+ofBits;
+
+ static const U32 LL_base[MaxLL+1] = {
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
+ 0x2000, 0x4000, 0x8000, 0x10000 };
+
+ static const U32 ML_base[MaxML+1] = {
+ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
+ 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
+ 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
+
+ static const U32 OF_base[MaxOff+1] = {
+ 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
+ 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
+ 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
+ 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
+
+ /* sequence */
+ { size_t offset;
+ if (!ofCode)
+ offset = 0;
+ else {
+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
+ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+ }
+
+ if (ofCode <= 1) {
+ offset += (llCode==0);
+ if (offset) {
+ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
+ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset = temp;
+ } else {
+ offset = seqState->prevOffset[0];
+ }
+ } else {
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset;
+ }
+ seq.offset = offset;
+ }
+
+ seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
+ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
+
+ seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
+ if (MEM_32bits() ||
+ (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream);
+
+ { size_t const pos = seqState->pos + seq.litLength;
+ seq.match = seqState->base + pos - seq.offset; /* single memory segment */
+ if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */
+ seqState->pos = pos + seq.matchLength;
}
+
+ /* ANS state update */
+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
+ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
+
+ return seq;
+}
+
+FORCE_INLINE
+size_t ZSTD_execSequenceLong(BYTE* op,
+ BYTE* const oend, seq_t sequence,
+ const BYTE** litPtr, const BYTE* const litLimit,
+ const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+{
+ BYTE* const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
+ const BYTE* const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE* match = sequence.match;
+
+ /* check */
+#if 1
+ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
+#endif
+
+ /* copy Literals */
+ ZSTD_copy8(op, *litPtr);
+ if (sequence.litLength > 8)
+ ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* copy Match */
+#if 1
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+ if (match + sequence.matchLength <= dictEnd) {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currentPrefixSegment */
+ { size_t const length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ if (op > oend_w || sequence.matchLength < MINMATCH) {
+ U32 i;
+ for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
+ return sequenceLength;
+ }
+ } }
+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
+#endif
+
+ /* match within prefix */
+ if (sequence.offset < 8) {
+ /* close range match, overlap */
+ static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
+ static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
+ int const sub2 = dec64table[sequence.offset];
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += dec32table[sequence.offset];
+ ZSTD_copy4(op+4, match);
+ match -= sub2;
+ } else {
+ ZSTD_copy8(op, match);
+ }
+ op += 8; match += 8;
+
+ if (oMatchEnd > oend-(16-MINMATCH)) {
+ if (op < oend_w) {
+ ZSTD_wildcopy(op, match, oend_w - op);
+ match += oend_w - op;
+ op = oend_w;
+ }
+ while (op < oMatchEnd) *op++ = *match++;
+ } else {
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
+ }
+ return sequenceLength;
+}
+
+static size_t ZSTD_decompressSequencesLong(
+ ZSTD_DCtx* dctx,
+ void* dst, size_t maxDstSize,
+ const void* seqStart, size_t seqSize)
+{
+ const BYTE* ip = (const BYTE*)seqStart;
+ const BYTE* const iend = ip + seqSize;
+ BYTE* const ostart = (BYTE* const)dst;
+ BYTE* const oend = ostart + maxDstSize;
+ BYTE* op = ostart;
+ const BYTE* litPtr = dctx->litPtr;
+ const BYTE* const litEnd = litPtr + dctx->litSize;
+ const BYTE* const base = (const BYTE*) (dctx->base);
+ const BYTE* const vBase = (const BYTE*) (dctx->vBase);
+ const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+ int nbSeq;
+
+ /* Build Decoding Tables */
+ { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
+ if (ZSTD_isError(seqHSize)) return seqHSize;
+ ip += seqHSize;
+ }
+
+ /* Regen sequences */
+ if (nbSeq) {
+#define STORED_SEQS 4
+#define STOSEQ_MASK (STORED_SEQS-1)
+#define ADVANCED_SEQS 4
+ seq_t sequences[STORED_SEQS];
+ int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
+ seqState_t seqState;
+ int seqNb;
+ dctx->fseEntropy = 1;
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
+ seqState.base = base;
+ seqState.pos = (size_t)(op-base);
+ seqState.gotoDict = (iPtrDiff)(dictEnd - base);
+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+
+ /* prepare in advance */
+ for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb<seqAdvance; seqNb++) {
+ sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState);
+ }
+ if (seqNb<seqAdvance) return ERROR(corruption_detected);
+
+ /* decode and decompress */
+ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
+ seq_t const sequence = ZSTD_decodeSequenceLong(&seqState);
+ size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ ZSTD_PREFETCH(sequence.match);
+ sequences[seqNb&STOSEQ_MASK] = sequence;
+ op += oneSeqSize;
+ }
+ if (seqNb<nbSeq) return ERROR(corruption_detected);
+
+ /* finish queue */
+ seqNb -= seqAdvance;
+ for ( ; seqNb<nbSeq ; seqNb++) {
+ size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
+ if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ op += oneSeqSize;
+ }
+
+ /* save reps for next block */
+ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
+ }
+
+ /* last literal segment */
+ { size_t const lastLLSize = litEnd - litPtr;
+ if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
+ memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+
+ return op-ostart;
}
@@ -809,10 +1319,21 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
ip += litCSize;
srcSize -= litCSize;
}
+ if (dctx->fParams.windowSize > (1<<23)) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
}
+static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst)
+{
+ if (dst != dctx->previousDstEnd) { /* not contiguous */
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
+ dctx->base = dst;
+ dctx->previousDstEnd = dst;
+ }
+}
+
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
@@ -859,12 +1380,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
/* Frame Header */
- { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
- size_t result;
+ { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
- result = ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize);
- if (ZSTD_isError(result)) return result;
+ CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize));
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
}
@@ -917,30 +1436,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
}
-/*! ZSTD_decompress_usingPreparedDCtx() :
-* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
-* It avoids reloading the dictionary each time.
-* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
-* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
-size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize)
-{
- ZSTD_copyDCtx(dctx, refDCtx);
- ZSTD_checkContinuity(dctx, dst);
- return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
-}
-
-
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict, size_t dictSize)
+ const void* src, size_t srcSize,
+ const void* dict, size_t dictSize)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
#endif
- ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
+ CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
@@ -1009,32 +1513,30 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
switch (dctx->stage)
{
case ZSTDds_getFrameHeaderSize :
- if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
- if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
- memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
- dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
+ if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */
+ if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
+ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
dctx->stage = ZSTDds_decodeSkippableHeader;
return 0;
}
- dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
+ dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
- memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
- if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
- dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
+ if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
+ dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
dctx->stage = ZSTDds_decodeFrameHeader;
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
case ZSTDds_decodeFrameHeader:
- { size_t result;
- memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
- result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
- if (ZSTD_isError(result)) return result;
- dctx->expected = ZSTD_blockHeaderSize;
- dctx->stage = ZSTDds_decodeBlockHeader;
- return 0;
- }
+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
+ CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
+ dctx->expected = ZSTD_blockHeaderSize;
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ return 0;
+
case ZSTDds_decodeBlockHeader:
{ blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
@@ -1106,7 +1608,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
return 0;
}
case ZSTDds_decodeSkippableHeader:
- { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
+ { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
dctx->stage = ZSTDds_skipFrame;
return 0;
@@ -1142,36 +1644,36 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
}
{ short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
- { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
- if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
+ CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
- { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
- if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
+ CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize;
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
- { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
- if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
+ CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize;
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
- dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
- dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
- dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1;
@@ -1200,53 +1702,55 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
return ZSTD_refDictContent(dctx, dict, dictSize);
}
-
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
- { size_t const errorCode = ZSTD_decompressBegin(dctx);
- if (ZSTD_isError(errorCode)) return errorCode; }
-
- if (dict && dictSize) {
- size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
- if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
- }
-
+ CHECK_F(ZSTD_decompressBegin(dctx));
+ if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
return 0;
}
+/* ====== ZSTD_DDict ====== */
+
struct ZSTD_DDict_s {
- void* dict;
+ void* dictBuffer;
+ const void* dictContent;
size_t dictSize;
ZSTD_DCtx* refContext;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
-ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
+ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
- void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
- if (!dictContent || !ddict || !dctx) {
- ZSTD_free(dictContent, customMem);
+ if (!ddict || !dctx) {
ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem);
return NULL;
}
- memcpy(dictContent, dict, dictSize);
- { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
+ if ((byReference) || (!dict) || (!dictSize)) {
+ ddict->dictBuffer = NULL;
+ ddict->dictContent = dict;
+ } else {
+ void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
+ if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; }
+ memcpy(internalBuffer, dict, dictSize);
+ ddict->dictBuffer = internalBuffer;
+ ddict->dictContent = internalBuffer;
+ }
+ { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
- ZSTD_free(dictContent, customMem);
+ ZSTD_free(ddict->dictBuffer, customMem);
ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem);
return NULL;
} }
- ddict->dict = dictContent;
ddict->dictSize = dictSize;
ddict->refContext = dctx;
return ddict;
@@ -1259,34 +1763,92 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
- return ZSTD_createDDict_advanced(dict, dictSize, allocator);
+ return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
+}
+
+
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
+{
+ ZSTD_customMem const allocator = { NULL, NULL, NULL };
+ return ZSTD_createDDict_advanced(dictBuffer, dictSize, 1, allocator);
}
+
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->refContext->customMem;
ZSTD_freeDCtx(ddict->refContext);
- ZSTD_free(ddict->dict, cMem);
+ ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
}
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0; /* support sizeof on NULL */
+ return sizeof(*ddict) + ZSTD_sizeof_DCtx(ddict->refContext) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
+}
+
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
+{
+ if (dictSize < 8) return 0;
+ if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
+ return MEM_readLE32((const char*)dict + 4);
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0;
+ return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
+{
+ ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 };
+ size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
+ if (ZSTD_isError(hError)) return 0;
+ return zfp.dictID;
+}
+
+
/*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */
-ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_DDict* ddict)
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTD_DDict* ddict)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
- if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
+ if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize);
#endif
- return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
- dst, dstCapacity,
- src, srcSize);
+ ZSTD_refDCtx(dctx, ddict->refContext);
+ ZSTD_checkContinuity(dctx, dst);
+ return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
@@ -1299,7 +1861,9 @@ typedef enum { zdss_init, zdss_loadHeader,
/* *** Resource management *** */
struct ZSTD_DStream_s {
- ZSTD_DCtx* zd;
+ ZSTD_DCtx* dctx;
+ ZSTD_DDict* ddictLocal;
+ const ZSTD_DDict* ddict;
ZSTD_frameParams fParams;
ZSTD_dStreamStage stage;
char* inBuff;
@@ -1311,15 +1875,13 @@ struct ZSTD_DStream_s {
size_t outStart;
size_t outEnd;
size_t blockSize;
- BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
size_t lhSize;
ZSTD_customMem customMem;
- void* dictContent;
- size_t dictSize;
- const void* dictSource;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
+ U32 hostageByte;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
@@ -1339,8 +1901,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
if (zds==NULL) return NULL;
memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
- zds->zd = ZSTD_createDCtx_advanced(customMem);
- if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
+ zds->dctx = ZSTD_createDCtx_advanced(customMem);
+ if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds;
@@ -1350,10 +1912,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support free on null */
{ ZSTD_customMem const cMem = zds->customMem;
- ZSTD_freeDCtx(zds->zd);
+ ZSTD_freeDCtx(zds->dctx);
+ ZSTD_freeDDict(zds->ddictLocal);
ZSTD_free(zds->inBuff, cMem);
ZSTD_free(zds->outBuff, cMem);
- ZSTD_free(zds->dictContent, cMem);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (zds->legacyContext)
ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
@@ -1373,17 +1935,15 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
- if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */
- if (dictSize > zds->dictSize) {
- ZSTD_free(zds->dictContent, zds->customMem);
- zds->dictContent = ZSTD_malloc(dictSize, zds->customMem);
- if (zds->dictContent == NULL) return ERROR(memory_allocation);
- }
- memcpy(zds->dictContent, dict, dictSize);
- zds->dictSize = dictSize;
- }
+ ZSTD_freeDDict(zds->ddictLocal);
+ if (dict && dictSize >= 8) {
+ zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
+ if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
+ } else zds->ddictLocal = NULL;
+ zds->ddict = zds->ddictLocal;
zds->legacyVersion = 0;
- return 0;
+ zds->hostageByte = 0;
+ return ZSTD_frameHeaderSize_prefix;
}
size_t ZSTD_initDStream(ZSTD_DStream* zds)
@@ -1391,13 +1951,29 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0);
}
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */
+{
+ size_t const initResult = ZSTD_initDStream(zds);
+ zds->ddict = ddict;
+ return initResult;
+}
+
+size_t ZSTD_resetDStream(ZSTD_DStream* zds)
+{
+ zds->stage = zdss_loadHeader;
+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+ zds->legacyVersion = 0;
+ zds->hostageByte = 0;
+ return ZSTD_frameHeaderSize_prefix;
+}
+
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
ZSTD_DStreamParameter_e paramType, unsigned paramValue)
{
switch(paramType)
{
default : return ERROR(parameter_unknown);
- case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
+ case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
}
return 0;
}
@@ -1405,11 +1981,12 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{
- return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize;
+ if (zds==NULL) return 0; /* support sizeof on NULL */
+ return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize;
}
-/* *** Decompression *** */
+/* ***** Decompression ***** */
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
@@ -1438,7 +2015,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
switch(zds->stage)
{
case zdss_init :
- return ERROR(init_missing);
+ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
+ /* fall-through */
case zdss_loadHeader :
{ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
@@ -1446,17 +2024,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
- size_t initResult;
- initResult = ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
- zds->dictContent, zds->dictSize);
- if (ZSTD_isError(initResult)) return initResult;
+ const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
+ size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
+ CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
+ dict, dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
} else {
return hSize; /* error */
} }
#else
- return hSize;
+ return hSize;
#endif
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
@@ -1464,27 +2042,26 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
zds->lhSize += iend-ip;
input->pos = input->size;
- return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
+ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
- ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize);
- { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */
- size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
- if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
- if (h1Size < zds->lhSize) { /* long header */
- size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
- size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size);
- if (ZSTD_isError(h2Result)) return h2Result;
+ { const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL;
+ ZSTD_refDCtx(zds->dctx, refContext);
+ }
+ { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
+ { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size));
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
- if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
+ if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
- /* Frame header instruct buffer sizes */
+ /* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize;
zds->blockSize = blockSize;
@@ -1504,21 +2081,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* pass-through */
case zdss_read:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (neededInSize==0) { /* end of frame */
zds->stage = zdss_init;
someMoreWork = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
- const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
- size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
- zds->outEnd = zds->outStart + decodedSize;
+ zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
break;
}
@@ -1528,7 +2105,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
}
case zdss_load:
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
@@ -1538,8 +2115,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
- { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
- size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
+ { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -1561,7 +2138,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->outStart = zds->outEnd = 0;
break;
}
- /* cannot flush everything */
+ /* cannot complete flush */
someMoreWork = 0;
break;
}
@@ -1571,9 +2148,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* result */
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
- { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
- if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
- nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
+ { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+ if (!nextSrcSizeHint) { /* frame fully decoded */
+ if (zds->outEnd == zds->outStart) { /* output fully flushed */
+ if (zds->hostageByte) {
+ if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */
+ input->pos++; /* release hostage */
+ }
+ return 0;
+ }
+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
+ zds->hostageByte=1;
+ }
+ return 1;
+ }
+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint;
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/common/zbuff.h b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff.h
similarity index 66%
rename from c-blosc/internal-complibs/zstd-1.0.0/common/zbuff.h
rename to c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff.h
index f99e061..85f9735 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/common/zbuff.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff.h
@@ -9,35 +9,50 @@
/* ***************************************************************
* NOTES/WARNINGS
-*****************************************************************/
-/* The streaming API defined here will soon be deprecated by the
-* new one in 'zstd.h'; consider migrating towards newer streaming
-* API. See 'lib/README.md'.
-*****************************************************************/
+******************************************************************/
+/* The streaming API defined here is deprecated.
+ * Consider migrating towards ZSTD_compressStream() API in `zstd.h`
+ * See 'lib/README.md'.
+ *****************************************************************/
-#ifndef ZSTD_BUFFERED_H_23987
-#define ZSTD_BUFFERED_H_23987
#if defined (__cplusplus)
extern "C" {
#endif
+#ifndef ZSTD_BUFFERED_H_23987
+#define ZSTD_BUFFERED_H_23987
+
/* *************************************
* Dependencies
***************************************/
#include <stddef.h> /* size_t */
+#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
/* ***************************************************************
* Compiler specifics
*****************************************************************/
-/* ZSTD_DLL_EXPORT :
-* Enable exporting of functions when building a Windows DLL */
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+/* Deprecation warnings */
+/* Should these warnings be a problem,
+ it is generally possible to disable them,
+ typically with -Wno-deprecated-declarations for gcc
+ or _CRT_SECURE_NO_WARNINGS in Visual.
+ Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */
+#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
+# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
#else
-# define ZSTDLIB_API
-#endif
+# if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
+# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
+# elif defined(__GNUC__) && (__GNUC__ >= 3)
+# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler")
+# define ZBUFF_DEPRECATED(message) ZSTDLIB_API
+# endif
+#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */
/* *************************************
@@ -49,16 +64,16 @@ extern "C" {
* ZBUFF and ZSTD are 100% interoperable,
* frames created by one can be decoded by the other one */
-typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
-ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
-ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
+typedef ZSTD_CStream ZBUFF_CCtx;
+ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
+ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
-ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
-ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
+ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
-ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
-ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
+ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
+ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
+ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
/*-*************************************************
* Streaming compression - howto
@@ -101,14 +116,14 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap
* **************************************************/
-typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
-ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
-ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
+typedef ZSTD_DStream ZBUFF_DCtx;
+ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
+ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
+ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
+ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
+ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
@@ -141,18 +156,22 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
/* *************************************
* Tool functions
***************************************/
-ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
-ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
+ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode);
+ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
-ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
-ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
-ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
-ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
+ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void);
+ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void);
+ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
+ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
+
+#endif /* ZSTD_BUFFERED_H_23987 */
#ifdef ZBUFF_STATIC_LINKING_ONLY
+#ifndef ZBUFF_STATIC_H_30298098432
+#define ZBUFF_STATIC_H_30298098432
/* ====================================================================================
* The definitions in this section are considered experimental.
@@ -169,23 +188,23 @@ ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
/*--- Custom memory allocator ---*/
/*! ZBUFF_createCCtx_advanced() :
* Create a ZBUFF compression context using external alloc and free functions */
-ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
+ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZBUFF_createDCtx_advanced() :
* Create a ZBUFF decompression context using external alloc and free functions */
-ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
+ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
/*--- Advanced Streaming Initialization ---*/
-ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
+ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
-#endif /* ZBUFF_STATIC_LINKING_ONLY */
+
+#endif /* ZBUFF_STATIC_H_30298098432 */
+#endif /* ZBUFF_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
-
-#endif /* ZSTD_BUFFERED_H_23987 */
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_common.c b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_common.c
new file mode 100644
index 0000000..9fff6eb
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_common.c
@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "error_private.h"
+#include "zbuff.h"
+
+/*-****************************************
+* ZBUFF Error Management (deprecated)
+******************************************/
+
+/*! ZBUFF_isError() :
+* tells if a return value is an error code */
+unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
+/*! ZBUFF_getErrorName() :
+* provides error code string from function result (useful for debugging) */
+const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_compress.c b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_compress.c
new file mode 100644
index 0000000..5a37a00
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_compress.c
@@ -0,0 +1,145 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+
+/* *************************************
+* Dependencies
+***************************************/
+#define ZBUFF_STATIC_LINKING_ONLY
+#include "zbuff.h"
+
+
+/*-***********************************************************
+* Streaming compression
+*
+* A ZBUFF_CCtx object is required to track streaming operation.
+* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
+* Use ZBUFF_compressInit() to start a new compression operation.
+* ZBUFF_CCtx objects can be reused multiple times.
+*
+* Use ZBUFF_compressContinue() repetitively to consume your input.
+* *srcSizePtr and *dstCapacityPtr can be any size.
+* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
+* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
+* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
+* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
+* Note that it will not output more than *dstCapacityPtr.
+* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
+* @return : nb of bytes still present into internal buffer (0 if it's empty)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* ZBUFF_compressEnd() instructs to finish a frame.
+* It will perform a flush and write frame epilogue.
+* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
+* @return : nb of bytes still present into internal buffer (0 if it's empty)
+* or an error code, which can be tested using ZBUFF_isError().
+*
+* Hint : recommended buffer sizes (not compulsory)
+* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
+* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
+* ***********************************************************/
+
+ZBUFF_CCtx* ZBUFF_createCCtx(void)
+{
+ return ZSTD_createCStream();
+}
+
+ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
+{
+ return ZSTD_createCStream_advanced(customMem);
+}
+
+size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
+{
+ return ZSTD_freeCStream(zbc);
+}
+
+
+/* ====== Initialization ====== */
+
+size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
+ const void* dict, size_t dictSize,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ return ZSTD_initCStream_advanced(zbc, dict, dictSize, params, pledgedSrcSize);
+}
+
+
+size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
+{
+ return ZSTD_initCStream_usingDict(zbc, dict, dictSize, compressionLevel);
+}
+
+size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
+{
+ return ZSTD_initCStream(zbc, compressionLevel);
+}
+
+/* ====== Compression ====== */
+
+
+size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
+ void* dst, size_t* dstCapacityPtr,
+ const void* src, size_t* srcSizePtr)
+{
+ size_t result;
+ ZSTD_outBuffer outBuff;
+ ZSTD_inBuffer inBuff;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ inBuff.src = src;
+ inBuff.pos = 0;
+ inBuff.size = *srcSizePtr;
+ result = ZSTD_compressStream(zbc, &outBuff, &inBuff);
+ *dstCapacityPtr = outBuff.pos;
+ *srcSizePtr = inBuff.pos;
+ return result;
+}
+
+
+
+/* ====== Finalize ====== */
+
+size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
+{
+ size_t result;
+ ZSTD_outBuffer outBuff;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ result = ZSTD_flushStream(zbc, &outBuff);
+ *dstCapacityPtr = outBuff.pos;
+ return result;
+}
+
+
+size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
+{
+ size_t result;
+ ZSTD_outBuffer outBuff;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ result = ZSTD_endStream(zbc, &outBuff);
+ *dstCapacityPtr = outBuff.pos;
+ return result;
+}
+
+
+
+/* *************************************
+* Tool functions
+***************************************/
+size_t ZBUFF_recommendedCInSize(void) { return ZSTD_CStreamInSize(); }
+size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_CStreamOutSize(); }
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_decompress.c b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_decompress.c
new file mode 100644
index 0000000..d9c155e
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/deprecated/zbuff_decompress.c
@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+
+/* *************************************
+* Dependencies
+***************************************/
+#define ZBUFF_STATIC_LINKING_ONLY
+#include "zbuff.h"
+
+
+ZBUFF_DCtx* ZBUFF_createDCtx(void)
+{
+ return ZSTD_createDStream();
+}
+
+ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
+{
+ return ZSTD_createDStream_advanced(customMem);
+}
+
+size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
+{
+ return ZSTD_freeDStream(zbd);
+}
+
+
+/* *** Initialization *** */
+
+size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
+{
+ return ZSTD_initDStream_usingDict(zbd, dict, dictSize);
+}
+
+size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
+{
+ return ZSTD_initDStream(zbd);
+}
+
+
+/* *** Decompression *** */
+
+size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
+ void* dst, size_t* dstCapacityPtr,
+ const void* src, size_t* srcSizePtr)
+{
+ ZSTD_outBuffer outBuff;
+ ZSTD_inBuffer inBuff;
+ size_t result;
+ outBuff.dst = dst;
+ outBuff.pos = 0;
+ outBuff.size = *dstCapacityPtr;
+ inBuff.src = src;
+ inBuff.pos = 0;
+ inBuff.size = *srcSizePtr;
+ result = ZSTD_decompressStream(zbd, &outBuff, &inBuff);
+ *dstCapacityPtr = outBuff.pos;
+ *srcSizePtr = inBuff.pos;
+ return result;
+}
+
+
+/* *************************************
+* Tool functions
+***************************************/
+size_t ZBUFF_recommendedDInSize(void) { return ZSTD_DStreamInSize(); }
+size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); }
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/cover.c b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/cover.c
new file mode 100644
index 0000000..c5b606d
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/cover.c
@@ -0,0 +1,1021 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include <stdio.h> /* fprintf */
+#include <stdlib.h> /* malloc, free, qsort */
+#include <string.h> /* memset */
+#include <time.h> /* clock */
+
+#include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#ifndef ZDICT_STATIC_LINKING_ONLY
+#define ZDICT_STATIC_LINKING_ONLY
+#endif
+#include "zdict.h"
+
+/*-*************************************
+* Constants
+***************************************/
+#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
+
+/*-*************************************
+* Console display
+***************************************/
+static int g_displayLevel = 2;
+#define DISPLAY(...) \
+ { \
+ fprintf(stderr, __VA_ARGS__); \
+ fflush(stderr); \
+ }
+#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
+ if (displayLevel >= l) { \
+ DISPLAY(__VA_ARGS__); \
+ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
+#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
+
+#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
+ if (displayLevel >= l) { \
+ if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
+ g_time = clock(); \
+ DISPLAY(__VA_ARGS__); \
+ if (displayLevel >= 4) \
+ fflush(stdout); \
+ } \
+ }
+#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+/*-*************************************
+* Hash table
+***************************************
+* A small specialized hash map for storing activeDmers.
+* The map does not resize, so if it becomes full it will loop forever.
+* Thus, the map must be large enough to store every value.
+* The map implements linear probing and keeps its load less than 0.5.
+*/
+
+#define MAP_EMPTY_VALUE ((U32)-1)
+typedef struct COVER_map_pair_t_s {
+ U32 key;
+ U32 value;
+} COVER_map_pair_t;
+
+typedef struct COVER_map_s {
+ COVER_map_pair_t *data;
+ U32 sizeLog;
+ U32 size;
+ U32 sizeMask;
+} COVER_map_t;
+
+/**
+ * Clear the map.
+ */
+static void COVER_map_clear(COVER_map_t *map) {
+ memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t));
+}
+
+/**
+ * Initializes a map of the given size.
+ * Returns 1 on success and 0 on failure.
+ * The map must be destroyed with COVER_map_destroy().
+ * The map is only guaranteed to be large enough to hold size elements.
+ */
+static int COVER_map_init(COVER_map_t *map, U32 size) {
+ map->sizeLog = ZSTD_highbit32(size) + 2;
+ map->size = (U32)1 << map->sizeLog;
+ map->sizeMask = map->size - 1;
+ map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t));
+ if (!map->data) {
+ map->sizeLog = 0;
+ map->size = 0;
+ return 0;
+ }
+ COVER_map_clear(map);
+ return 1;
+}
+
+/**
+ * Internal hash function
+ */
+static const U32 prime4bytes = 2654435761U;
+static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
+ return (key * prime4bytes) >> (32 - map->sizeLog);
+}
+
+/**
+ * Helper function that returns the index that a key should be placed into.
+ */
+static U32 COVER_map_index(COVER_map_t *map, U32 key) {
+ const U32 hash = COVER_map_hash(map, key);
+ U32 i;
+ for (i = hash;; i = (i + 1) & map->sizeMask) {
+ COVER_map_pair_t *pos = &map->data[i];
+ if (pos->value == MAP_EMPTY_VALUE) {
+ return i;
+ }
+ if (pos->key == key) {
+ return i;
+ }
+ }
+}
+
+/**
+ * Returns the pointer to the value for key.
+ * If key is not in the map, it is inserted and the value is set to 0.
+ * The map must not be full.
+ */
+static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
+ COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)];
+ if (pos->value == MAP_EMPTY_VALUE) {
+ pos->key = key;
+ pos->value = 0;
+ }
+ return &pos->value;
+}
+
+/**
+ * Deletes key from the map if present.
+ */
+static void COVER_map_remove(COVER_map_t *map, U32 key) {
+ U32 i = COVER_map_index(map, key);
+ COVER_map_pair_t *del = &map->data[i];
+ U32 shift = 1;
+ if (del->value == MAP_EMPTY_VALUE) {
+ return;
+ }
+ for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) {
+ COVER_map_pair_t *const pos = &map->data[i];
+ /* If the position is empty we are done */
+ if (pos->value == MAP_EMPTY_VALUE) {
+ del->value = MAP_EMPTY_VALUE;
+ return;
+ }
+ /* If pos can be moved to del do so */
+ if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) {
+ del->key = pos->key;
+ del->value = pos->value;
+ del = pos;
+ shift = 1;
+ } else {
+ ++shift;
+ }
+ }
+}
+
+/**
+ * Destroyes a map that is inited with COVER_map_init().
+ */
+static void COVER_map_destroy(COVER_map_t *map) {
+ if (map->data) {
+ free(map->data);
+ }
+ map->data = NULL;
+ map->size = 0;
+}
+
+/*-*************************************
+* Context
+***************************************/
+
+typedef struct {
+ const BYTE *samples;
+ size_t *offsets;
+ const size_t *samplesSizes;
+ size_t nbSamples;
+ U32 *suffix;
+ size_t suffixSize;
+ U32 *freqs;
+ U32 *dmerAt;
+ unsigned d;
+} COVER_ctx_t;
+
+/* We need a global context for qsort... */
+static COVER_ctx_t *g_ctx = NULL;
+
+/*-*************************************
+* Helper functions
+***************************************/
+
+/**
+ * Returns the sum of the sample sizes.
+ */
+static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
+ size_t sum = 0;
+ size_t i;
+ for (i = 0; i < nbSamples; ++i) {
+ sum += samplesSizes[i];
+ }
+ return sum;
+}
+
+/**
+ * Returns -1 if the dmer at lp is less than the dmer at rp.
+ * Return 0 if the dmers at lp and rp are equal.
+ * Returns 1 if the dmer at lp is greater than the dmer at rp.
+ */
+static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) {
+ const U32 lhs = *(const U32 *)lp;
+ const U32 rhs = *(const U32 *)rp;
+ return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d);
+}
+
+/**
+ * Same as COVER_cmp() except ties are broken by pointer value
+ * NOTE: g_ctx must be set to call this function. A global is required because
+ * qsort doesn't take an opaque pointer.
+ */
+static int COVER_strict_cmp(const void *lp, const void *rp) {
+ int result = COVER_cmp(g_ctx, lp, rp);
+ if (result == 0) {
+ result = lp < rp ? -1 : 1;
+ }
+ return result;
+}
+
+/**
+ * Returns the first pointer in [first, last) whose element does not compare
+ * less than value. If no such element exists it returns last.
+ */
+static const size_t *COVER_lower_bound(const size_t *first, const size_t *last,
+ size_t value) {
+ size_t count = last - first;
+ while (count != 0) {
+ size_t step = count / 2;
+ const size_t *ptr = first;
+ ptr += step;
+ if (*ptr < value) {
+ first = ++ptr;
+ count -= step + 1;
+ } else {
+ count = step;
+ }
+ }
+ return first;
+}
+
+/**
+ * Generic groupBy function.
+ * Groups an array sorted by cmp into groups with equivalent values.
+ * Calls grp for each group.
+ */
+static void
+COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx,
+ int (*cmp)(COVER_ctx_t *, const void *, const void *),
+ void (*grp)(COVER_ctx_t *, const void *, const void *)) {
+ const BYTE *ptr = (const BYTE *)data;
+ size_t num = 0;
+ while (num < count) {
+ const BYTE *grpEnd = ptr + size;
+ ++num;
+ while (num < count && cmp(ctx, ptr, grpEnd) == 0) {
+ grpEnd += size;
+ ++num;
+ }
+ grp(ctx, ptr, grpEnd);
+ ptr = grpEnd;
+ }
+}
+
+/*-*************************************
+* Cover functions
+***************************************/
+
+/**
+ * Called on each group of positions with the same dmer.
+ * Counts the frequency of each dmer and saves it in the suffix array.
+ * Fills `ctx->dmerAt`.
+ */
+static void COVER_group(COVER_ctx_t *ctx, const void *group,
+ const void *groupEnd) {
+ /* The group consists of all the positions with the same first d bytes. */
+ const U32 *grpPtr = (const U32 *)group;
+ const U32 *grpEnd = (const U32 *)groupEnd;
+ /* The dmerId is how we will reference this dmer.
+ * This allows us to map the whole dmer space to a much smaller space, the
+ * size of the suffix array.
+ */
+ const U32 dmerId = (U32)(grpPtr - ctx->suffix);
+ /* Count the number of samples this dmer shows up in */
+ U32 freq = 0;
+ /* Details */
+ const size_t *curOffsetPtr = ctx->offsets;
+ const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples;
+ /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a
+ * different sample than the last.
+ */
+ size_t curSampleEnd = ctx->offsets[0];
+ for (; grpPtr != grpEnd; ++grpPtr) {
+ /* Save the dmerId for this position so we can get back to it. */
+ ctx->dmerAt[*grpPtr] = dmerId;
+ /* Dictionaries only help for the first reference to the dmer.
+ * After that zstd can reference the match from the previous reference.
+ * So only count each dmer once for each sample it is in.
+ */
+ if (*grpPtr < curSampleEnd) {
+ continue;
+ }
+ freq += 1;
+ /* Binary search to find the end of the sample *grpPtr is in.
+ * In the common case that grpPtr + 1 == grpEnd we can skip the binary
+ * search because the loop is over.
+ */
+ if (grpPtr + 1 != grpEnd) {
+ const size_t *sampleEndPtr =
+ COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr);
+ curSampleEnd = *sampleEndPtr;
+ curOffsetPtr = sampleEndPtr + 1;
+ }
+ }
+ /* At this point we are never going to look at this segment of the suffix
+ * array again. We take advantage of this fact to save memory.
+ * We store the frequency of the dmer in the first position of the group,
+ * which is dmerId.
+ */
+ ctx->suffix[dmerId] = freq;
+}
+
+/**
+ * A segment is a range in the source as well as the score of the segment.
+ */
+typedef struct {
+ U32 begin;
+ U32 end;
+ double score;
+} COVER_segment_t;
+
+/**
+ * Selects the best segment in an epoch.
+ * Segments of are scored according to the function:
+ *
+ * Let F(d) be the frequency of dmer d.
+ * Let S_i be the dmer at position i of segment S which has length k.
+ *
+ * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
+ *
+ * Once the dmer d is in the dictionay we set F(d) = 0.
+ */
+static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
+ COVER_map_t *activeDmers, U32 begin,
+ U32 end, COVER_params_t parameters) {
+ /* Constants */
+ const U32 k = parameters.k;
+ const U32 d = parameters.d;
+ const U32 dmersInK = k - d + 1;
+ /* Try each segment (activeSegment) and save the best (bestSegment) */
+ COVER_segment_t bestSegment = {0, 0, 0};
+ COVER_segment_t activeSegment;
+ /* Reset the activeDmers in the segment */
+ COVER_map_clear(activeDmers);
+ /* The activeSegment starts at the beginning of the epoch. */
+ activeSegment.begin = begin;
+ activeSegment.end = begin;
+ activeSegment.score = 0;
+ /* Slide the activeSegment through the whole epoch.
+ * Save the best segment in bestSegment.
+ */
+ while (activeSegment.end < end) {
+ /* The dmerId for the dmer at the next position */
+ U32 newDmer = ctx->dmerAt[activeSegment.end];
+ /* The entry in activeDmers for this dmerId */
+ U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer);
+ /* If the dmer isn't already present in the segment add its score. */
+ if (*newDmerOcc == 0) {
+ /* The paper suggest using the L-0.5 norm, but experiments show that it
+ * doesn't help.
+ */
+ activeSegment.score += freqs[newDmer];
+ }
+ /* Add the dmer to the segment */
+ activeSegment.end += 1;
+ *newDmerOcc += 1;
+
+ /* If the window is now too large, drop the first position */
+ if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
+ U32 delDmer = ctx->dmerAt[activeSegment.begin];
+ U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
+ activeSegment.begin += 1;
+ *delDmerOcc -= 1;
+ /* If this is the last occurence of the dmer, subtract its score */
+ if (*delDmerOcc == 0) {
+ COVER_map_remove(activeDmers, delDmer);
+ activeSegment.score -= freqs[delDmer];
+ }
+ }
+
+ /* If this segment is the best so far save it */
+ if (activeSegment.score > bestSegment.score) {
+ bestSegment = activeSegment;
+ }
+ }
+ {
+ /* Trim off the zero frequency head and tail from the segment. */
+ U32 newBegin = bestSegment.end;
+ U32 newEnd = bestSegment.begin;
+ U32 pos;
+ for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+ U32 freq = freqs[ctx->dmerAt[pos]];
+ if (freq != 0) {
+ newBegin = MIN(newBegin, pos);
+ newEnd = pos + 1;
+ }
+ }
+ bestSegment.begin = newBegin;
+ bestSegment.end = newEnd;
+ }
+ {
+ /* Zero out the frequency of each dmer covered by the chosen segment. */
+ U32 pos;
+ for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
+ freqs[ctx->dmerAt[pos]] = 0;
+ }
+ }
+ return bestSegment;
+}
+
+/**
+ * Check the validity of the parameters.
+ * Returns non-zero if the parameters are valid and 0 otherwise.
+ */
+static int COVER_checkParameters(COVER_params_t parameters) {
+ /* k and d are required parameters */
+ if (parameters.d == 0 || parameters.k == 0) {
+ return 0;
+ }
+ /* d <= k */
+ if (parameters.d > parameters.k) {
+ return 0;
+ }
+ return 1;
+}
+
+/**
+ * Clean up a context initialized with `COVER_ctx_init()`.
+ */
+static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
+ if (!ctx) {
+ return;
+ }
+ if (ctx->suffix) {
+ free(ctx->suffix);
+ ctx->suffix = NULL;
+ }
+ if (ctx->freqs) {
+ free(ctx->freqs);
+ ctx->freqs = NULL;
+ }
+ if (ctx->dmerAt) {
+ free(ctx->dmerAt);
+ ctx->dmerAt = NULL;
+ }
+ if (ctx->offsets) {
+ free(ctx->offsets);
+ ctx->offsets = NULL;
+ }
+}
+
+/**
+ * Prepare a context for dictionary building.
+ * The context is only dependent on the parameter `d` and can used multiple
+ * times.
+ * Returns 1 on success or zero on error.
+ * The context must be destroyed with `COVER_ctx_destroy()`.
+ */
+static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
+ const size_t *samplesSizes, unsigned nbSamples,
+ unsigned d) {
+ const BYTE *const samples = (const BYTE *)samplesBuffer;
+ const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
+ /* Checks */
+ if (totalSamplesSize < d ||
+ totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
+ DISPLAYLEVEL(1, "Total samples size is too large, maximum size is %u MB\n",
+ (COVER_MAX_SAMPLES_SIZE >> 20));
+ return 0;
+ }
+ /* Zero the context */
+ memset(ctx, 0, sizeof(*ctx));
+ DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
+ (U32)totalSamplesSize);
+ ctx->samples = samples;
+ ctx->samplesSizes = samplesSizes;
+ ctx->nbSamples = nbSamples;
+ /* Partial suffix array */
+ ctx->suffixSize = totalSamplesSize - d + 1;
+ ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+ /* Maps index to the dmerID */
+ ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+ /* The offsets of each file */
+ ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
+ if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) {
+ DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
+ COVER_ctx_destroy(ctx);
+ return 0;
+ }
+ ctx->freqs = NULL;
+ ctx->d = d;
+
+ /* Fill offsets from the samlesSizes */
+ {
+ U32 i;
+ ctx->offsets[0] = 0;
+ for (i = 1; i <= nbSamples; ++i) {
+ ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
+ }
+ }
+ DISPLAYLEVEL(2, "Constructing partial suffix array\n");
+ {
+ /* suffix is a partial suffix array.
+ * It only sorts suffixes by their first parameters.d bytes.
+ * The sort is stable, so each dmer group is sorted by position in input.
+ */
+ U32 i;
+ for (i = 0; i < ctx->suffixSize; ++i) {
+ ctx->suffix[i] = i;
+ }
+ /* qsort doesn't take an opaque pointer, so pass as a global */
+ g_ctx = ctx;
+ qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), &COVER_strict_cmp);
+ }
+ DISPLAYLEVEL(2, "Computing frequencies\n");
+ /* For each dmer group (group of positions with the same first d bytes):
+ * 1. For each position we set dmerAt[position] = dmerID. The dmerID is
+ * (groupBeginPtr - suffix). This allows us to go from position to
+ * dmerID so we can look up values in freq.
+ * 2. We calculate how many samples the dmer occurs in and save it in
+ * freqs[dmerId].
+ */
+ COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, &COVER_cmp,
+ &COVER_group);
+ ctx->freqs = ctx->suffix;
+ ctx->suffix = NULL;
+ return 1;
+}
+
+/**
+ * Given the prepared context build the dictionary.
+ */
+static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
+ COVER_map_t *activeDmers, void *dictBuffer,
+ size_t dictBufferCapacity,
+ COVER_params_t parameters) {
+ BYTE *const dict = (BYTE *)dictBuffer;
+ size_t tail = dictBufferCapacity;
+ /* Divide the data up into epochs of equal size.
+ * We will select at least one segment from each epoch.
+ */
+ const U32 epochs = (U32)(dictBufferCapacity / parameters.k);
+ const U32 epochSize = (U32)(ctx->suffixSize / epochs);
+ size_t epoch;
+ DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
+ epochSize);
+ /* Loop through the epochs until there are no more segments or the dictionary
+ * is full.
+ */
+ for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
+ const U32 epochBegin = (U32)(epoch * epochSize);
+ const U32 epochEnd = epochBegin + epochSize;
+ size_t segmentSize;
+ /* Select a segment */
+ COVER_segment_t segment = COVER_selectSegment(
+ ctx, freqs, activeDmers, epochBegin, epochEnd, parameters);
+ /* Trim the segment if necessary and if it is empty then we are done */
+ segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
+ if (segmentSize == 0) {
+ break;
+ }
+ /* We fill the dictionary from the back to allow the best segments to be
+ * referenced with the smallest offsets.
+ */
+ tail -= segmentSize;
+ memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
+ DISPLAYUPDATE(
+ 2, "\r%u%% ",
+ (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
+ }
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ return tail;
+}
+
+/**
+ * Translate from COVER_params_t to ZDICT_params_t required for finalizing the
+ * dictionary.
+ */
+static ZDICT_params_t COVER_translateParams(COVER_params_t parameters) {
+ ZDICT_params_t zdictParams;
+ memset(&zdictParams, 0, sizeof(zdictParams));
+ zdictParams.notificationLevel = 1;
+ zdictParams.dictID = parameters.dictID;
+ zdictParams.compressionLevel = parameters.compressionLevel;
+ return zdictParams;
+}
+
+/**
+ * Constructs a dictionary using a heuristic based on the following paper:
+ *
+ * Liao, Petri, Moffat, Wirth
+ * Effective Construction of Relative Lempel-Ziv Dictionaries
+ * Published in WWW 2016.
+ */
+ZDICTLIB_API size_t COVER_trainFromBuffer(
+ void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
+ const size_t *samplesSizes, unsigned nbSamples, COVER_params_t parameters) {
+ BYTE *const dict = (BYTE *)dictBuffer;
+ COVER_ctx_t ctx;
+ COVER_map_t activeDmers;
+ /* Checks */
+ if (!COVER_checkParameters(parameters)) {
+ DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+ return ERROR(GENERIC);
+ }
+ if (nbSamples == 0) {
+ DISPLAYLEVEL(1, "Cover must have at least one input file\n");
+ return ERROR(GENERIC);
+ }
+ if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+ DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+ ZDICT_DICTSIZE_MIN);
+ return ERROR(dstSize_tooSmall);
+ }
+ /* Initialize global data */
+ g_displayLevel = parameters.notificationLevel;
+ /* Initialize context and activeDmers */
+ if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+ parameters.d)) {
+ return ERROR(GENERIC);
+ }
+ if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
+ DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
+ COVER_ctx_destroy(&ctx);
+ return ERROR(GENERIC);
+ }
+
+ DISPLAYLEVEL(2, "Building dictionary\n");
+ {
+ const size_t tail =
+ COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer,
+ dictBufferCapacity, parameters);
+ ZDICT_params_t zdictParams = COVER_translateParams(parameters);
+ const size_t dictionarySize = ZDICT_finalizeDictionary(
+ dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+ samplesBuffer, samplesSizes, nbSamples, zdictParams);
+ if (!ZSTD_isError(dictionarySize)) {
+ DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
+ (U32)dictionarySize);
+ }
+ COVER_ctx_destroy(&ctx);
+ COVER_map_destroy(&activeDmers);
+ return dictionarySize;
+ }
+}
+
+/**
+ * COVER_best_t is used for two purposes:
+ * 1. Synchronizing threads.
+ * 2. Saving the best parameters and dictionary.
+ *
+ * All of the methods except COVER_best_init() are thread safe if zstd is
+ * compiled with multithreaded support.
+ */
+typedef struct COVER_best_s {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond;
+ size_t liveJobs;
+ void *dict;
+ size_t dictSize;
+ COVER_params_t parameters;
+ size_t compressedSize;
+} COVER_best_t;
+
+/**
+ * Initialize the `COVER_best_t`.
+ */
+static void COVER_best_init(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ pthread_mutex_init(&best->mutex, NULL);
+ pthread_cond_init(&best->cond, NULL);
+ best->liveJobs = 0;
+ best->dict = NULL;
+ best->dictSize = 0;
+ best->compressedSize = (size_t)-1;
+ memset(&best->parameters, 0, sizeof(best->parameters));
+}
+
+/**
+ * Wait until liveJobs == 0.
+ */
+static void COVER_best_wait(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ pthread_mutex_lock(&best->mutex);
+ while (best->liveJobs != 0) {
+ pthread_cond_wait(&best->cond, &best->mutex);
+ }
+ pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Call COVER_best_wait() and then destroy the COVER_best_t.
+ */
+static void COVER_best_destroy(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ COVER_best_wait(best);
+ if (best->dict) {
+ free(best->dict);
+ }
+ pthread_mutex_destroy(&best->mutex);
+ pthread_cond_destroy(&best->cond);
+}
+
+/**
+ * Called when a thread is about to be launched.
+ * Increments liveJobs.
+ */
+static void COVER_best_start(COVER_best_t *best) {
+ if (!best) {
+ return;
+ }
+ pthread_mutex_lock(&best->mutex);
+ ++best->liveJobs;
+ pthread_mutex_unlock(&best->mutex);
+}
+
+/**
+ * Called when a thread finishes executing, both on error or success.
+ * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
+ * If this dictionary is the best so far save it and its parameters.
+ */
+static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
+ COVER_params_t parameters, void *dict,
+ size_t dictSize) {
+ if (!best) {
+ return;
+ }
+ {
+ size_t liveJobs;
+ pthread_mutex_lock(&best->mutex);
+ --best->liveJobs;
+ liveJobs = best->liveJobs;
+ /* If the new dictionary is better */
+ if (compressedSize < best->compressedSize) {
+ /* Allocate space if necessary */
+ if (!best->dict || best->dictSize < dictSize) {
+ if (best->dict) {
+ free(best->dict);
+ }
+ best->dict = malloc(dictSize);
+ if (!best->dict) {
+ best->compressedSize = ERROR(GENERIC);
+ best->dictSize = 0;
+ return;
+ }
+ }
+ /* Save the dictionary, parameters, and size */
+ memcpy(best->dict, dict, dictSize);
+ best->dictSize = dictSize;
+ best->parameters = parameters;
+ best->compressedSize = compressedSize;
+ }
+ pthread_mutex_unlock(&best->mutex);
+ if (liveJobs == 0) {
+ pthread_cond_broadcast(&best->cond);
+ }
+ }
+}
+
+/**
+ * Parameters for COVER_tryParameters().
+ */
+typedef struct COVER_tryParameters_data_s {
+ const COVER_ctx_t *ctx;
+ COVER_best_t *best;
+ size_t dictBufferCapacity;
+ COVER_params_t parameters;
+} COVER_tryParameters_data_t;
+
+/**
+ * Tries a set of parameters and upates the COVER_best_t with the results.
+ * This function is thread safe if zstd is compiled with multithreaded support.
+ * It takes its parameters as an *OWNING* opaque pointer to support threading.
+ */
+static void COVER_tryParameters(void *opaque) {
+ /* Save parameters as local variables */
+ COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
+ const COVER_ctx_t *const ctx = data->ctx;
+ const COVER_params_t parameters = data->parameters;
+ size_t dictBufferCapacity = data->dictBufferCapacity;
+ size_t totalCompressedSize = ERROR(GENERIC);
+ /* Allocate space for hash table, dict, and freqs */
+ COVER_map_t activeDmers;
+ BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+ U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
+ if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
+ DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
+ goto _cleanup;
+ }
+ if (!dict || !freqs) {
+ DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
+ goto _cleanup;
+ }
+ /* Copy the frequencies because we need to modify them */
+ memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32));
+ /* Build the dictionary */
+ {
+ const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
+ dictBufferCapacity, parameters);
+ const ZDICT_params_t zdictParams = COVER_translateParams(parameters);
+ dictBufferCapacity = ZDICT_finalizeDictionary(
+ dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
+ ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, zdictParams);
+ if (ZDICT_isError(dictBufferCapacity)) {
+ DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+ goto _cleanup;
+ }
+ }
+ /* Check total compressed size */
+ {
+ /* Pointers */
+ ZSTD_CCtx *cctx;
+ ZSTD_CDict *cdict;
+ void *dst;
+ /* Local variables */
+ size_t dstCapacity;
+ size_t i;
+ /* Allocate dst with enough space to compress the maximum sized sample */
+ {
+ size_t maxSampleSize = 0;
+ for (i = 0; i < ctx->nbSamples; ++i) {
+ maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
+ }
+ dstCapacity = ZSTD_compressBound(maxSampleSize);
+ dst = malloc(dstCapacity);
+ }
+ /* Create the cctx and cdict */
+ cctx = ZSTD_createCCtx();
+ cdict =
+ ZSTD_createCDict(dict, dictBufferCapacity, parameters.compressionLevel);
+ if (!dst || !cctx || !cdict) {
+ goto _compressCleanup;
+ }
+ /* Compress each sample and sum their sizes (or error) */
+ totalCompressedSize = 0;
+ for (i = 0; i < ctx->nbSamples; ++i) {
+ const size_t size = ZSTD_compress_usingCDict(
+ cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
+ ctx->samplesSizes[i], cdict);
+ if (ZSTD_isError(size)) {
+ totalCompressedSize = ERROR(GENERIC);
+ goto _compressCleanup;
+ }
+ totalCompressedSize += size;
+ }
+ _compressCleanup:
+ ZSTD_freeCCtx(cctx);
+ ZSTD_freeCDict(cdict);
+ if (dst) {
+ free(dst);
+ }
+ }
+
+_cleanup:
+ COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
+ dictBufferCapacity);
+ free(data);
+ COVER_map_destroy(&activeDmers);
+ if (dict) {
+ free(dict);
+ }
+ if (freqs) {
+ free(freqs);
+ }
+}
+
+ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
+ size_t dictBufferCapacity,
+ const void *samplesBuffer,
+ const size_t *samplesSizes,
+ unsigned nbSamples,
+ COVER_params_t *parameters) {
+ /* constants */
+ const unsigned nbThreads = parameters->nbThreads;
+ const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
+ const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d;
+ const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k;
+ const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k;
+ const unsigned kSteps = parameters->steps == 0 ? 32 : parameters->steps;
+ const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
+ const unsigned kIterations =
+ (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
+ /* Local variables */
+ const int displayLevel = parameters->notificationLevel;
+ unsigned iteration = 1;
+ unsigned d;
+ unsigned k;
+ COVER_best_t best;
+ POOL_ctx *pool = NULL;
+ /* Checks */
+ if (kMinK < kMaxD || kMaxK < kMinK) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
+ return ERROR(GENERIC);
+ }
+ if (nbSamples == 0) {
+ DISPLAYLEVEL(1, "Cover must have at least one input file\n");
+ return ERROR(GENERIC);
+ }
+ if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
+ DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
+ ZDICT_DICTSIZE_MIN);
+ return ERROR(dstSize_tooSmall);
+ }
+ if (nbThreads > 1) {
+ pool = POOL_create(nbThreads, 1);
+ if (!pool) {
+ return ERROR(memory_allocation);
+ }
+ }
+ /* Initialization */
+ COVER_best_init(&best);
+ /* Turn down global display level to clean up display at level 2 and below */
+ g_displayLevel = parameters->notificationLevel - 1;
+ /* Loop through d first because each new value needs a new context */
+ LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
+ kIterations);
+ for (d = kMinD; d <= kMaxD; d += 2) {
+ /* Initialize the context for this value of d */
+ COVER_ctx_t ctx;
+ LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
+ if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+ COVER_best_destroy(&best);
+ return ERROR(GENERIC);
+ }
+ /* Loop through k reusing the same context */
+ for (k = kMinK; k <= kMaxK; k += kStepSize) {
+ /* Prepare the arguments */
+ COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
+ sizeof(COVER_tryParameters_data_t));
+ LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
+ if (!data) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
+ COVER_best_destroy(&best);
+ COVER_ctx_destroy(&ctx);
+ return ERROR(GENERIC);
+ }
+ data->ctx = &ctx;
+ data->best = &best;
+ data->dictBufferCapacity = dictBufferCapacity;
+ data->parameters = *parameters;
+ data->parameters.k = k;
+ data->parameters.d = d;
+ data->parameters.steps = kSteps;
+ /* Check the parameters */
+ if (!COVER_checkParameters(data->parameters)) {
+ DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+ continue;
+ }
+ /* Call the function and pass ownership of data to it */
+ COVER_best_start(&best);
+ if (pool) {
+ POOL_add(pool, &COVER_tryParameters, data);
+ } else {
+ COVER_tryParameters(data);
+ }
+ /* Print status */
+ LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
+ (U32)((iteration * 100) / kIterations));
+ ++iteration;
+ }
+ COVER_best_wait(&best);
+ COVER_ctx_destroy(&ctx);
+ }
+ LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
+ /* Fill the output buffer and parameters with output of the best parameters */
+ {
+ const size_t dictSize = best.dictSize;
+ if (ZSTD_isError(best.compressedSize)) {
+ COVER_best_destroy(&best);
+ return best.compressedSize;
+ }
+ *parameters = best.parameters;
+ memcpy(dictBuffer, best.dict, dictSize);
+ COVER_best_destroy(&best);
+ POOL_free(pool);
+ return dictSize;
+ }
+}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/divsufsort.c b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/divsufsort.c
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/divsufsort.c
rename to c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/divsufsort.c
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/divsufsort.h b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/divsufsort.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/divsufsort.h
rename to c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/divsufsort.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/zdict.c b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/zdict.c
similarity index 84%
rename from c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/zdict.c
rename to c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/zdict.c
index adfe55c..0757dbb 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/dictBuilder/zdict.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/zdict.c
@@ -36,12 +36,11 @@
#include <time.h> /* clock */
#include "mem.h" /* read */
-#include "error_private.h"
#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */
#define HUF_STATIC_LINKING_ONLY
-#include "huf.h"
+#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */
#include "zstd_internal.h" /* includes zstd.h */
-#include "xxhash.h"
+#include "xxhash.h" /* XXH64 */
#include "divsufsort.h"
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
@@ -61,7 +60,7 @@
#define NOISELENGTH 32
#define MINRATIO 4
-static const int g_compressionLevel_default = 5;
+static const int g_compressionLevel_default = 6;
static const U32 g_selectivity_default = 9;
static const size_t g_provision_entropySize = 200;
static const size_t g_min_fast_dictContent = 192;
@@ -71,26 +70,18 @@ static const size_t g_min_fast_dictContent = 192;
* Console display
***************************************/
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
-
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if (ZDICT_clockSpan(g_time) > refreshRate) \
- { g_time = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 3 / 10;
-static clock_t g_time = 0;
+#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
-static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
+static void ZDICT_printHex(const void* ptr, size_t length)
{
const BYTE* const b = (const BYTE*)ptr;
size_t u;
for (u=0; u<length; u++) {
BYTE c = b[u];
if (c<32 || c>126) c = '.'; /* non-printable char */
- DISPLAYLEVEL(dlevel, "%c", c);
+ DISPLAY("%c", c);
}
}
@@ -211,7 +202,7 @@ static void ZDICT_initDictItem(dictItem* d)
static dictItem ZDICT_analyzePos(
BYTE* doneMarks,
const int* suffix, U32 start,
- const void* buffer, U32 minRatio)
+ const void* buffer, U32 minRatio, U32 notificationLevel)
{
U32 lengthList[LLIMIT] = {0};
U32 cumulLength[LLIMIT] = {0};
@@ -315,13 +306,13 @@ static dictItem ZDICT_analyzePos(
} while (length >=MINMATCHLENGTH);
/* look backward */
- length = MINMATCHLENGTH;
- while ((length >= MINMATCHLENGTH) & (start > 0)) {
- length = ZDICT_count(b + pos, b + suffix[start - 1]);
- if (length >= LLIMIT) length = LLIMIT - 1;
- lengthList[length]++;
- if (length >= MINMATCHLENGTH) start--;
- }
+ length = MINMATCHLENGTH;
+ while ((length >= MINMATCHLENGTH) & (start > 0)) {
+ length = ZDICT_count(b + pos, b + suffix[start - 1]);
+ if (length >= LLIMIT) length = LLIMIT - 1;
+ lengthList[length]++;
+ if (length >= MINMATCHLENGTH) start--;
+ }
/* largest useful length */
memset(cumulLength, 0, sizeof(cumulLength));
@@ -379,21 +370,22 @@ static dictItem ZDICT_analyzePos(
static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
{
const U32 tableSize = table->pos;
- const U32 max = elt.pos + (elt.length-1);
+ const U32 eltEnd = elt.pos + elt.length;
/* tail overlap */
U32 u; for (u=1; u<tableSize; u++) {
if (u==eltNbToSkip) continue;
- if ((table[u].pos > elt.pos) && (table[u].pos < max)) { /* overlap */
+ if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */
/* append */
U32 addedLength = table[u].pos - elt.pos;
table[u].length += addedLength;
table[u].pos = elt.pos;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
- table[u].savings += elt.length / 8; /* rough approx */
+ table[u].savings += elt.length / 8; /* rough approx bonus */
elt = table[u];
+ /* sort : improve rank */
while ((u>1) && (table[u-1].savings < elt.savings))
- table[u] = table[u-1], u--;
+ table[u] = table[u-1], u--;
table[u] = elt;
return u;
} }
@@ -401,14 +393,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
/* front overlap */
for (u=1; u<tableSize; u++) {
if (u==eltNbToSkip) continue;
- if ((table[u].pos + table[u].length > elt.pos) && (table[u].pos < elt.pos)) { /* overlap */
+ if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
- int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length);
- table[u].savings += elt.length / 8; /* rough approx */
- if (addedLength > 0) { /* otherwise, already included */
+ int addedLength = (int)eltEnd - (table[u].pos + table[u].length);
+ table[u].savings += elt.length / 8; /* rough approx bonus */
+ if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
}
+ /* sort : improve rank */
elt = table[u];
while ((u>1) && (table[u-1].savings < elt.savings))
table[u] = table[u-1], u--;
@@ -473,7 +466,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */
const size_t* fileSizes, unsigned nbFiles,
- U32 minRatio)
+ U32 minRatio, U32 notificationLevel)
{
int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));
int* const suffix = suffix0+1;
@@ -481,6 +474,13 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks)); /* +16 for overflow security */
U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));
size_t result = 0;
+ clock_t displayClock = 0;
+ clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
+
+# define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
+ if (ZDICT_clockSpan(displayClock) > refreshRate) \
+ { displayClock = clock(); DISPLAY(__VA_ARGS__); \
+ if (notificationLevel>=4) fflush(stdout); } }
/* init */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
@@ -506,7 +506,8 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
{ size_t pos;
for (pos=0; pos < bufferSize; pos++)
reverseSuffix[suffix[pos]] = (U32)pos;
- /* build file pos */
+ /* note filePos tracks borders between samples.
+ It's not used at this stage, but planned to become useful in a later update */
filePos[0] = 0;
for (pos=1; pos<nbFiles; pos++)
filePos[pos] = (U32)(filePos[pos-1] + fileSizes[pos-1]);
@@ -518,7 +519,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
{ U32 cursor; for (cursor=0; cursor < bufferSize; ) {
dictItem solution;
if (doneMarks[cursor]) { cursor++; continue; }
- solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
+ solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel);
if (solution.length==0) { cursor++; continue; }
ZDICT_insertDictItem(dictList, dictListSize, solution);
cursor += solution.length;
@@ -558,17 +559,17 @@ typedef struct
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, U32 notificationLevel)
{
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog);
size_t cSize;
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
- { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
- if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
- }
+ { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
+ if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
+ }
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
- if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
+ if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
@@ -647,9 +648,10 @@ static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val,
#define OFFCODE_MAX 30 /* only applicable to first block */
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
- unsigned compressionLevel,
- const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
- const void* dictBuffer, size_t dictBufferSize)
+ unsigned compressionLevel,
+ const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
+ const void* dictBuffer, size_t dictBufferSize,
+ unsigned notificationLevel)
{
U32 countLit[256];
HUF_CREATE_STATIC_CTABLE(hufTable, 255);
@@ -690,18 +692,19 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
memset(bestRepOffset, 0, sizeof(bestRepOffset));
if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
- { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
- if (ZSTD_isError(beginResult)) {
- eSize = ERROR(GENERIC);
- DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
- goto _cleanup;
- } }
+ { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
+ if (ZSTD_isError(beginResult)) {
+ eSize = ERROR(GENERIC);
+ DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
+ goto _cleanup;
+ } }
/* collect stats on all files */
for (u=0; u<nbFiles; u++) {
ZDICT_countEStats(esr, params,
- countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
- (const char*)srcBuffer + pos, fileSizes[u]);
+ countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
+ (const char*)srcBuffer + pos, fileSizes[u],
+ notificationLevel);
pos += fileSizes[u];
}
@@ -748,7 +751,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
}
llLog = (U32)errorCode;
-
/* write result to buffer */
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
if (HUF_isError(hhSize)) {
@@ -810,7 +812,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]);
#endif
- dstPtr += 12;
+ //dstPtr += 12;
eSize += 12;
_cleanup:
@@ -822,12 +824,62 @@ _cleanup:
}
+
+size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
+ const void* customDictContent, size_t dictContentSize,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t params)
+{
+ size_t hSize;
+#define HBUFFSIZE 256
+ BYTE header[HBUFFSIZE];
+ int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
+ U32 const notificationLevel = params.notificationLevel;
+
+ /* check conditions */
+ if (dictBufferCapacity < dictContentSize) return ERROR(dstSize_tooSmall);
+ if (dictContentSize < ZDICT_CONTENTSIZE_MIN) return ERROR(srcSize_wrong);
+ if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
+
+ /* dictionary header */
+ MEM_writeLE32(header, ZSTD_DICT_MAGIC);
+ { U64 const randomID = XXH64(customDictContent, dictContentSize, 0);
+ U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
+ U32 const dictID = params.dictID ? params.dictID : compliantID;
+ MEM_writeLE32(header+4, dictID);
+ }
+ hSize = 8;
+
+ /* entropy tables */
+ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
+ DISPLAYLEVEL(2, "statistics ... \n");
+ { size_t const eSize = ZDICT_analyzeEntropy(header+hSize, HBUFFSIZE-hSize,
+ compressionLevel,
+ samplesBuffer, samplesSizes, nbSamples,
+ customDictContent, dictContentSize,
+ notificationLevel);
+ if (ZDICT_isError(eSize)) return eSize;
+ hSize += eSize;
+ }
+
+ /* copy elements in final buffer ; note : src and dst buffer can overlap */
+ if (hSize + dictContentSize > dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize;
+ { size_t const dictSize = hSize + dictContentSize;
+ char* dictEnd = (char*)dictBuffer + dictSize;
+ memmove(dictEnd - dictContentSize, customDictContent, dictContentSize);
+ memcpy(dictBuffer, header, hSize);
+ return dictSize;
+ }
+}
+
+
size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params)
{
size_t hSize;
int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
+ U32 const notificationLevel = params.notificationLevel;
/* dictionary header */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
@@ -844,7 +896,8 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
{ size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
compressionLevel,
samplesBuffer, samplesSizes, nbSamples,
- (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
+ (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize,
+ notificationLevel);
if (ZDICT_isError(eSize)) return eSize;
hSize += eSize;
}
@@ -872,6 +925,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
size_t const targetDictSize = maxDictSize;
size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples);
size_t dictSize = 0;
+ U32 const notificationLevel = params.notificationLevel;
/* checks */
if (!dictList) return ERROR(memory_allocation);
@@ -880,35 +934,36 @@ size_t ZDICT_trainFromBuffer_unsafe(
/* init */
ZDICT_initDictItem(dictList);
- g_displayLevel = params.notificationLevel;
/* build dictionary */
ZDICT_trainBuffer(dictList, dictListSize,
samplesBuffer, samplesBuffSize,
samplesSizes, nbSamples,
- minRep);
+ minRep, notificationLevel);
/* display best matches */
- if (g_displayLevel>= 3) {
+ if (params.notificationLevel>= 3) {
U32 const nb = MIN(25, dictList[0].pos);
U32 const dictContentSize = ZDICT_dictSize(dictList);
U32 u;
- DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
- DISPLAYLEVEL(3, "list %u best segments \n", nb);
- for (u=1; u<=nb; u++) {
- U32 pos = dictList[u].pos;
- U32 length = dictList[u].length;
- U32 printedLength = MIN(40, length);
+ DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize);
+ DISPLAYLEVEL(3, "list %u best segments \n", nb-1);
+ for (u=1; u<nb; u++) {
+ U32 const pos = dictList[u].pos;
+ U32 const length = dictList[u].length;
+ U32 const printedLength = MIN(40, length);
+ if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize))
+ return ERROR(GENERIC); /* should never happen */
DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
u, length, pos, dictList[u].savings);
- ZDICT_printHex(3, (const char*)samplesBuffer+pos, printedLength);
+ ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
DISPLAYLEVEL(3, "| \n");
} }
/* create dictionary */
{ U32 dictContentSize = ZDICT_dictSize(dictList);
- if (dictContentSize < targetDictSize/2) {
+ if (dictContentSize < targetDictSize/3) {
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
if (minRep > MINRATIO) {
DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
@@ -918,12 +973,12 @@ size_t ZDICT_trainFromBuffer_unsafe(
DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20));
}
- if ((dictContentSize > targetDictSize*2) && (nbSamples > 2*MINRATIO) && (selectivity>1)) {
+ if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) {
U32 proposedSelectivity = selectivity-1;
while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; }
DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize);
- DISPLAYLEVEL(2, "! you may consider decreasing selectivity to produce denser dictionary (-s%u) \n", proposedSelectivity);
- DISPLAYLEVEL(2, "! but test its efficiency on samples \n");
+ DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity);
+ DISPLAYLEVEL(2, "! always test dictionary efficiency on samples \n");
}
/* limit dictionary size */
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/zdict.h b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/zdict.h
new file mode 100644
index 0000000..4d0a62a
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dictBuilder/zdict.h
@@ -0,0 +1,201 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef DICTBUILDER_H_001
+#define DICTBUILDER_H_001
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/*====== Dependencies ======*/
+#include <stddef.h> /* size_t */
+
+
+/* ===== ZDICTLIB_API : control library symbols visibility ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+# define ZDICTLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+# define ZDICTLIB_API ZDICTLIB_VISIBILITY
+#endif
+
+
+/*! ZDICT_trainFromBuffer() :
+ Train a dictionary from an array of samples.
+ Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ The resulting dictionary will be saved into `dictBuffer`.
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code, which can be tested with ZDICT_isError().
+ Tips : In general, a reasonable dictionary has a size of ~ 100 KB.
+ It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`.
+ In general, it's recommended to provide a few thousands samples, but this can vary a lot.
+ It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+*/
+ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+
+
+/*====== Helper functions ======*/
+ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
+ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
+ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
+
+
+
+#ifdef ZDICT_STATIC_LINKING_ONLY
+
+/* ====================================================================================
+ * The definitions in this section are considered experimental.
+ * They should never be used with a dynamic library, as they may change in the future.
+ * They are provided for advanced usages.
+ * Use them only in association with static linking.
+ * ==================================================================================== */
+
+typedef struct {
+ unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
+ int compressionLevel; /* 0 means default; target a specific zstd compression level */
+ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+ unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
+ unsigned reserved[2]; /* reserved space for future parameters */
+} ZDICT_params_t;
+
+
+/*! ZDICT_trainFromBuffer_advanced() :
+ Same as ZDICT_trainFromBuffer() with control over more parameters.
+ `parameters` is optional and can be provided with values set to 0 to mean "default".
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`),
+ or an error code, which can be tested by ZDICT_isError().
+ note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
+*/
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+/*! COVER_params_t :
+ For all values 0 means default.
+ kMin and d are the only required parameters.
+*/
+typedef struct {
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
+
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
+ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+ unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
+ int compressionLevel; /* 0 means default; target a specific zstd compression level */
+} COVER_params_t;
+
+
+/*! COVER_trainFromBuffer() :
+ Train a dictionary from an array of samples using the COVER algorithm.
+ Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
+ supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
+ The resulting dictionary will be saved into `dictBuffer`.
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code, which can be tested with ZDICT_isError().
+ Note : COVER_trainFromBuffer() requires about 9 bytes of memory for each input byte.
+ Tips : In general, a reasonable dictionary has a size of ~ 100 KB.
+ It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`.
+ In general, it's recommended to provide a few thousands samples, but this can vary a lot.
+ It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
+*/
+ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ COVER_params_t parameters);
+
+/*! COVER_optimizeTrainFromBuffer() :
+ The same requirements as above hold for all the parameters except `parameters`.
+ This function tries many parameter combinations and picks the best parameters.
+ `*parameters` is filled with the best parameters found, and the dictionary
+ constructed with those parameters is stored in `dictBuffer`.
+
+ All of the parameters d, k, steps are optional.
+ If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
+ if steps is zero it defaults to its default value.
+ If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
+
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code, which can be tested with ZDICT_isError().
+ On success `*parameters` contains the parameters selected.
+ Note : COVER_optimizeTrainFromBuffer() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
+*/
+ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
+ COVER_params_t *parameters);
+
+/*! ZDICT_finalizeDictionary() :
+
+ Given a custom content as a basis for dictionary, and a set of samples,
+ finalize dictionary by adding headers and statistics.
+
+ Samples must be stored concatenated in a flat buffer `samplesBuffer`,
+ supplied with an array of sizes `samplesSizes`, providing the size of each sample in order.
+
+ dictContentSize must be > ZDICT_CONTENTSIZE_MIN bytes.
+ maxDictSize must be >= dictContentSize, and must be > ZDICT_DICTSIZE_MIN bytes.
+
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`),
+ or an error code, which can be tested by ZDICT_isError().
+ note : ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0.
+ note 2 : dictBuffer and customDictContent can overlap
+*/
+#define ZDICT_CONTENTSIZE_MIN 256
+#define ZDICT_DICTSIZE_MIN 512
+ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
+ const void* customDictContent, size_t dictContentSize,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+
+
+/* Deprecation warnings */
+/* It is generally possible to disable deprecation warnings from compiler,
+ for example with -Wno-deprecated-declarations for gcc
+ or _CRT_SECURE_NO_WARNINGS in Visual.
+ Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
+#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */
+#else
+# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API [[deprecated(message)]]
+# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
+# elif (ZDICT_GCC_VERSION >= 301)
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated))
+# elif defined(_MSC_VER)
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message))
+# else
+# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
+# define ZDICT_DEPRECATED(message) ZDICTLIB_API
+# endif
+#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
+
+ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
+size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+
+
+#endif /* ZDICT_STATIC_LINKING_ONLY */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* DICTBUILDER_H_001 */
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dll/example/Makefile b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/Makefile
new file mode 100644
index 0000000..36041a0
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/Makefile
@@ -0,0 +1,47 @@
+# ##########################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ##########################################################################
+
+VOID := /dev/null
+ZSTDDIR := ../include
+LIBDIR := ../static
+DLLDIR := ../dll
+
+CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make
+CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
+ -Wdeclaration-after-statement -Wstrict-prototypes \
+ -Wpointer-arith -Wstrict-aliasing=1
+CFLAGS += $(MOREFLAGS)
+CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_
+FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+.PHONY: default fullbench-dll fullbench-lib
+
+
+default: all
+
+all: fullbench-dll fullbench-lib
+
+
+fullbench-lib: fullbench.c datagen.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib
+
+fullbench-dll: fullbench.c datagen.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll
+
+clean:
+ @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \
+ @echo Cleaning completed
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dll/example/README.md b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/README.md
new file mode 100644
index 0000000..957a29f
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/README.md
@@ -0,0 +1,69 @@
+ZSTD Windows binary package
+====================================
+
+#### The package contents
+
+- `zstd.exe` : Command Line Utility, supporting gzip-like arguments
+- `dll\libzstd.dll` : The DLL of ZSTD library
+- `dll\libzstd.lib` : The import library of ZSTD library for Visual C++
+- `example\` : The example of usage of ZSTD library
+- `include\` : Header files required with ZSTD library
+- `static\libzstd_static.lib` : The static ZSTD library
+
+
+#### Usage of Command Line Interface
+
+Command Line Interface (CLI) supports gzip-like arguments.
+By default CLI takes an input file and compresses it to an output file:
+```
+ Usage: zstd [arg] [input] [output]
+```
+The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can
+be improved with commands from `-3` to `-16` but higher levels also have slower
+compression. CLI includes in-memory compression benchmark module with compression
+levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds.
+CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined
+into `-b1e18i1`.
+
+
+#### The example of usage of static and dynamic ZSTD libraries with gcc/MinGW
+
+Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`.
+`fullbench-dll` uses a dynamic ZSTD library from the `dll` directory.
+`fullbench-lib` uses a static ZSTD library from the `lib` directory.
+
+
+#### Using ZSTD DLL with gcc/MinGW
+
+The header files from `include\` and the dynamic library `dll\libzstd.dll`
+are required to compile a project using gcc/MinGW.
+The dynamic library has to be added to linking options.
+It means that if a project that uses ZSTD consists of a single `test-dll.c`
+file it should be linked with `dll\libzstd.dll`. For example:
+```
+ gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\libzstd.dll
+```
+The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
+
+
+#### The example of usage of static and dynamic ZSTD libraries with Visual C++
+
+Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a
+dynamic ZSTD library from the `dll` directory. The solution works with Visual C++
+2010 or newer. When one will open the solution with Visual C++ newer than 2010
+then the solution will upgraded to the current version.
+
+
+#### Using ZSTD DLL with Visual C++
+
+The header files from `include\` and the import library `dll\libzstd.lib`
+are required to compile a project using Visual C++.
+
+1. The path to header files should be added to `Additional Include Directories` that can
+ be found in project properties `C/C++` then `General`.
+2. The import library has to be added to `Additional Dependencies` that can
+ be found in project properties `Linker` then `Input`.
+ If one will provide only the name `libzstd.lib` without a full path to the library
+ the directory has to be added to `Linker\General\Additional Library Directories`.
+
+The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dll/example/build_package.bat b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/build_package.bat
new file mode 100644
index 0000000..a7a7ab0
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/build_package.bat
@@ -0,0 +1,17 @@
+ at ECHO OFF
+MKDIR bin\dll bin\static bin\example bin\include
+COPY tests\fullbench.c bin\example\
+COPY programs\datagen.c bin\example\
+COPY programs\datagen.h bin\example\
+COPY programs\util.h bin\example\
+COPY lib\common\mem.h bin\example\
+COPY lib\common\zstd_errors.h bin\example\
+COPY lib\common\zstd_internal.h bin\example\
+COPY lib\common\error_private.h bin\example\
+COPY lib\zstd.h bin\include\
+COPY lib\libzstd.a bin\static\libzstd_static.lib
+COPY lib\dll\libzstd.* bin\dll\
+COPY lib\dll\example\Makefile bin\example\
+COPY lib\dll\example\fullbench-dll.* bin\example\
+COPY lib\dll\example\README.md bin\
+COPY programs\zstd.exe bin\zstd.exe
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dll/example/fullbench-dll.sln b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/fullbench-dll.sln
new file mode 100644
index 0000000..ef8d4c0
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/fullbench-dll.sln
@@ -0,0 +1,25 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Express 2012 for Windows Desktop
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
+ Debug|x64 = Debug|x64
+ Release|Win32 = Release|Win32
+ Release|x64 = Release|x64
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
+ {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dll/example/fullbench-dll.vcxproj b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/fullbench-dll.vcxproj
new file mode 100644
index 0000000..fbea783
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dll/example/fullbench-dll.vcxproj
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>fullbench-dll</RootNamespace>
+ <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+ <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>MultiByte</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>MultiByte</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>MultiByte</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>MultiByte</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <EnablePREfast>false</EnablePREfast>
+ <AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>false</TreatWarningAsError>
+ <EnablePREfast>false</EnablePREfast>
+ <AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
+ <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="datagen.c" />
+ <ClCompile Include="fullbench.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\include\zstd.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/c-blosc/internal-complibs/zstd-1.1.3/dll/libzstd.def b/c-blosc/internal-complibs/zstd-1.1.3/dll/libzstd.def
new file mode 100644
index 0000000..0a3259e
--- /dev/null
+++ b/c-blosc/internal-complibs/zstd-1.1.3/dll/libzstd.def
@@ -0,0 +1,86 @@
+LIBRARY libzstd.dll
+EXPORTS
+ ZDICT_getDictID
+ ZDICT_getErrorName
+ ZDICT_isError
+ ZDICT_trainFromBuffer
+ ZSTD_CStreamInSize
+ ZSTD_CStreamOutSize
+ ZSTD_DStreamInSize
+ ZSTD_DStreamOutSize
+ ZSTD_adjustCParams
+ ZSTD_checkCParams
+ ZSTD_compress
+ ZSTD_compressBegin
+ ZSTD_compressBegin_advanced
+ ZSTD_compressBegin_usingDict
+ ZSTD_compressBlock
+ ZSTD_compressBound
+ ZSTD_compressCCtx
+ ZSTD_compressContinue
+ ZSTD_compressEnd
+ ZSTD_compressStream
+ ZSTD_compress_advanced
+ ZSTD_compress_usingCDict
+ ZSTD_compress_usingDict
+ ZSTD_copyCCtx
+ ZSTD_copyDCtx
+ ZSTD_createCCtx
+ ZSTD_createCCtx_advanced
+ ZSTD_createCDict
+ ZSTD_createCDict_advanced
+ ZSTD_createCStream
+ ZSTD_createCStream_advanced
+ ZSTD_createDCtx
+ ZSTD_createDCtx_advanced
+ ZSTD_createDDict
+ ZSTD_createDStream
+ ZSTD_createDStream_advanced
+ ZSTD_decompress
+ ZSTD_decompressBegin
+ ZSTD_decompressBegin_usingDict
+ ZSTD_decompressBlock
+ ZSTD_decompressContinue
+ ZSTD_decompressDCtx
+ ZSTD_decompressStream
+ ZSTD_decompress_usingDDict
+ ZSTD_decompress_usingDict
+ ZSTD_endStream
+ ZSTD_estimateCCtxSize
+ ZSTD_estimateDCtxSize
+ ZSTD_flushStream
+ ZSTD_freeCCtx
+ ZSTD_freeCDict
+ ZSTD_freeCStream
+ ZSTD_freeDCtx
+ ZSTD_freeDDict
+ ZSTD_freeDStream
+ ZSTD_getBlockSizeMax
+ ZSTD_getCParams
+ ZSTD_getDecompressedSize
+ ZSTD_getErrorName
+ ZSTD_getFrameParams
+ ZSTD_getParams
+ ZSTD_initCStream
+ ZSTD_initCStream_advanced
+ ZSTD_initCStream_usingCDict
+ ZSTD_initCStream_usingDict
+ ZSTD_initDStream
+ ZSTD_initDStream_usingDDict
+ ZSTD_initDStream_usingDict
+ ZSTD_insertBlock
+ ZSTD_isError
+ ZSTD_isFrame
+ ZSTD_maxCLevel
+ ZSTD_nextInputType
+ ZSTD_nextSrcSizeToDecompress
+ ZSTD_resetCStream
+ ZSTD_resetDStream
+ ZSTD_setDStreamParameter
+ ZSTD_sizeof_CCtx
+ ZSTD_sizeof_CDict
+ ZSTD_sizeof_CStream
+ ZSTD_sizeof_DCtx
+ ZSTD_sizeof_DDict
+ ZSTD_sizeof_DStream
+ ZSTD_versionNumber
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_legacy.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_legacy.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_legacy.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_legacy.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v01.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v01.c
similarity index 89%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v01.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v01.c
index 94847d5..6fd30c0 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v01.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v01.c
@@ -13,6 +13,7 @@
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
#include "zstd_v01.h"
+#include "error_private.h"
/******************************************
@@ -139,11 +140,15 @@ typedef struct
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -953,13 +958,16 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
U32 weightTotal;
U32 maxBits;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
U32 nextRankStart;
void* ptr = DTable+1;
HUF_DElt* const dt = (HUF_DElt*)ptr;
+ if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
+ iSize = ip[0];
+
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1000,6 +1008,7 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
rankVal[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected;
/* get last non-null symbol weight (implied, total must be 2^n) */
maxBits = FSE_highbit32(weightTotal) + 1;
@@ -1173,57 +1182,6 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si
/*
zstd - standard compression library
- Header File for static linking only
- Copyright (C) 2014-2015, Yann Collet.
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - zstd source repository : https://github.com/Cyan4973/zstd
- - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-
-/* The objects defined into this file should be considered experimental.
- * They are not labelled stable, as their prototype may change in the future.
- * You can use them for tests, provide feedback, or if you can endure risk of future changes.
- */
-
-/**************************************
-* Error management
-**************************************/
-#define ZSTD_LIST_ERRORS(ITEM) \
- ITEM(ZSTD_OK_NoError) ITEM(ZSTD_ERROR_GENERIC) \
- ITEM(ZSTD_ERROR_MagicNumber) \
- ITEM(ZSTD_ERROR_SrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
- ITEM(ZSTD_ERROR_corruption) \
- ITEM(ZSTD_ERROR_maxCode)
-
-#define ZSTD_GENERATE_ENUM(ENUM) ENUM,
-typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* exposed list of errors; static linking only */
-
-/*
- zstd - standard compression library
Copyright (C) 2014-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -1300,17 +1258,9 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* expo
#endif
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#else
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -1404,7 +1354,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
-static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
+static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -1491,11 +1441,8 @@ typedef struct ZSTD_Cctx_s
/**************************************
* Error Management
**************************************/
-/* tells if a return value is an error code */
-static unsigned ZSTD_isError(size_t code) { return (code > (size_t)(-ZSTD_ERROR_maxCode)); }
-
/* published entry point */
-unsigned ZSTDv01_isError(size_t code) { return ZSTD_isError(code); }
+unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); }
/**************************************
@@ -1510,13 +1457,13 @@ unsigned ZSTDv01_isError(size_t code) { return ZSTD_isError(code); }
* Decompression code
**************************************************************/
-static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
+size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
{
const BYTE* const in = (const BYTE* const)src;
BYTE headerFlags;
U32 cSize;
- if (srcSize < 3) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (srcSize < 3) return ERROR(srcSize_wrong);
headerFlags = *in;
cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
@@ -1532,7 +1479,7 @@ static size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockPropertie
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
- if (srcSize > maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+ if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
memcpy(dst, src, srcSize);
return srcSize;
}
@@ -1549,21 +1496,21 @@ static size_t ZSTD_decompressLiterals(void* ctx,
size_t litSize;
/* check : minimum 2, for litSize, +1, for content */
- if (srcSize <= 3) return (size_t)-ZSTD_ERROR_corruption;
+ if (srcSize <= 3) return ERROR(corruption_detected);
litSize = ip[1] + (ip[0]<<8);
litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
op = oend - litSize;
(void)ctx;
- if (litSize > maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+ if (litSize > maxDstSize) return ERROR(dstSize_tooSmall);
errorCode = HUF_decompress(op, litSize, ip+2, srcSize-2);
- if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
+ if (FSE_isError(errorCode)) return ERROR(GENERIC);
return litSize;
}
-static size_t ZSTD_decodeLiteralsBlock(void* ctx,
+size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
void* dst, size_t maxDstSize,
const BYTE** litStart, size_t* litSize,
const void* src, size_t srcSize)
@@ -1574,9 +1521,9 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
BYTE* const oend = ostart + maxDstSize;
blockProperties_t litbp;
- size_t litcSize = ZSTD_getcBlockSize(src, srcSize, &litbp);
- if (ZSTD_isError(litcSize)) return litcSize;
- if (litcSize > srcSize - ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_SrcSize;
+ size_t litcSize = ZSTDv01_getcBlockSize(src, srcSize, &litbp);
+ if (ZSTDv01_isError(litcSize)) return litcSize;
+ if (litcSize > srcSize - ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
ip += ZSTD_blockHeaderSize;
switch(litbp.blockType)
@@ -1589,7 +1536,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
case bt_rle:
{
size_t rleSize = litbp.origSize;
- if (rleSize>maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+ if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
+ if (!srcSize) return ERROR(srcSize_wrong);
memset(oend - rleSize, *ip, rleSize);
*litStart = oend - rleSize;
*litSize = rleSize;
@@ -1599,7 +1547,7 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
case bt_compressed:
{
size_t decodedLitSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize);
- if (ZSTD_isError(decodedLitSize)) return decodedLitSize;
+ if (ZSTDv01_isError(decodedLitSize)) return decodedLitSize;
*litStart = oend - decodedLitSize;
*litSize = decodedLitSize;
ip += litcSize;
@@ -1607,14 +1555,14 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
}
case bt_end:
default:
- return (size_t)-ZSTD_ERROR_GENERIC;
+ return ERROR(GENERIC);
}
return ip-istart;
}
-static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
+size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
const void* src, size_t srcSize)
{
@@ -1626,7 +1574,7 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
size_t dumpsLength;
/* check */
- if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (srcSize < 5) return ERROR(srcSize_wrong);
/* SeqHead */
*nbSeq = ZSTD_readLE16(ip); ip+=2;
@@ -1650,7 +1598,7 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
*dumpsLengthPtr = dumpsLength;
/* check */
- if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
+ if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
/* sequences */
{
@@ -1669,8 +1617,8 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
default :
{ U32 max = MaxLL;
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
- if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
- if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+ if (FSE_isError(headerSize)) return ERROR(GENERIC);
+ if (LLlog > LLFSELog) return ERROR(corruption_detected);
ip += headerSize;
FSE_buildDTable(DTableLL, norm, max, LLlog);
} }
@@ -1679,7 +1627,7 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
{
case bt_rle :
Offlog = 0;
- if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */
+ if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
FSE_buildDTable_rle(DTableOffb, *ip++); break;
case bt_raw :
Offlog = Offbits;
@@ -1687,8 +1635,8 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
default :
{ U32 max = MaxOff;
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
- if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
- if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
+ if (FSE_isError(headerSize)) return ERROR(GENERIC);
+ if (Offlog > OffFSELog) return ERROR(corruption_detected);
ip += headerSize;
FSE_buildDTable(DTableOffb, norm, max, Offlog);
} }
@@ -1697,7 +1645,7 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
{
case bt_rle :
MLlog = 0;
- if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */
+ if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
FSE_buildDTable_rle(DTableML, *ip++); break;
case bt_raw :
MLlog = MLbits;
@@ -1705,8 +1653,8 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
default :
{ U32 max = MaxML;
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
- if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
- if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+ if (FSE_isError(headerSize)) return ERROR(GENERIC);
+ if (MLlog > MLFSELog) return ERROR(corruption_detected);
ip += headerSize;
FSE_buildDTable(DTableML, norm, max, MLlog);
} } }
@@ -1809,9 +1757,9 @@ static size_t ZSTD_execSequence(BYTE* op,
const BYTE* const litEnd = *litPtr + litLength;
/* check */
- if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */
- if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
- if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */
+ if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
+ if (litEnd > litLimit) return ERROR(corruption_detected);
+ if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
/* copy Literals */
if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
@@ -1822,7 +1770,7 @@ static size_t ZSTD_execSequence(BYTE* op,
*litPtr = litEnd; /* update for next sequence */
/* check : last match must be at a minimum distance of 8 from end of dest buffer */
- if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+ if (oend-op < 8) return ERROR(dstSize_tooSmall);
/* copy Match */
{
@@ -1832,8 +1780,8 @@ static size_t ZSTD_execSequence(BYTE* op,
U64 saved[2];
/* check */
- if (match < base) return (size_t)-ZSTD_ERROR_corruption;
- if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
+ if (match < base) return ERROR(corruption_detected);
+ if (sequence.offset > (size_t)base) return ERROR(corruption_detected);
/* save beginning of literal sequence, in case of write overlap */
if (overlapRisk)
@@ -1855,7 +1803,7 @@ static size_t ZSTD_execSequence(BYTE* op,
} else { ZSTD_copy8(op, match); }
op += 8; match += 8;
- if (endMatch > oend-12)
+ if (endMatch > oend-(16-MINMATCH))
{
if (op < oend-8)
{
@@ -1866,7 +1814,7 @@ static size_t ZSTD_execSequence(BYTE* op,
while (op<endMatch) *op++ = *match++;
}
else
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
/* restore, in case of overlap */
if (overlapRisk) memcpy(endMatch, saved, qutt);
@@ -1911,10 +1859,10 @@ static size_t ZSTD_decompressSequences(
BYTE* const base = (BYTE*) (dctx->base);
/* Build Decoding Tables */
- errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
+ errorCode = ZSTDv01_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb,
ip, iend-ip);
- if (ZSTD_isError(errorCode)) return errorCode;
+ if (ZSTDv01_isError(errorCode)) return errorCode;
ip += errorCode;
/* Regen sequences */
@@ -1927,7 +1875,7 @@ static size_t ZSTD_decompressSequences(
seqState.dumpsEnd = dumps + dumpsLength;
seqState.prevOffset = 1;
errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip);
- if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_corruption;
+ if (FSE_isError(errorCode)) return ERROR(corruption_detected);
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
@@ -1938,18 +1886,18 @@ static size_t ZSTD_decompressSequences(
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
- if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
+ if (ZSTDv01_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
/* check if reached exact end */
- if ( !FSE_endOfDStream(&(seqState.DStream)) ) return (size_t)-ZSTD_ERROR_corruption; /* requested too much : data is corrupted */
- if (nbSeq<0) return (size_t)-ZSTD_ERROR_corruption; /* requested too many sequences : data is corrupted */
+ if ( !FSE_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* requested too much : data is corrupted */
+ if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */
/* last literal segment */
{
size_t lastLLSize = litEnd - litPtr;
- if (op+lastLLSize > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
+ if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
if (op != litPtr) memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
@@ -1971,8 +1919,8 @@ static size_t ZSTD_decompressBlock(
size_t errorCode;
/* Decode literals sub-block */
- errorCode = ZSTD_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, &litSize, src, srcSize);
- if (ZSTD_isError(errorCode)) return errorCode;
+ errorCode = ZSTDv01_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, &litSize, src, srcSize);
+ if (ZSTDv01_isError(errorCode)) return errorCode;
ip += errorCode;
srcSize -= errorCode;
@@ -1993,20 +1941,20 @@ size_t ZSTDv01_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const voi
blockProperties_t blockProperties;
/* Frame Header */
- if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
magicNumber = ZSTD_readBE32(src);
- if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_MagicNumber;
+ if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
/* Loop on each block */
while (1)
{
- size_t blockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
- if (ZSTD_isError(blockSize)) return blockSize;
+ size_t blockSize = ZSTDv01_getcBlockSize(ip, iend-ip, &blockProperties);
+ if (ZSTDv01_isError(blockSize)) return blockSize;
ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize;
- if (blockSize > remainingSize) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (blockSize > remainingSize) return ERROR(srcSize_wrong);
switch(blockProperties.blockType)
{
@@ -2017,18 +1965,18 @@ size_t ZSTDv01_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const voi
errorCode = ZSTD_copyUncompressedBlock(op, oend-op, ip, blockSize);
break;
case bt_rle :
- return (size_t)-ZSTD_ERROR_GENERIC; /* not yet supported */
+ return ERROR(GENERIC); /* not yet supported */
break;
case bt_end :
/* end of frame */
- if (remainingSize) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (remainingSize) return ERROR(srcSize_wrong);
break;
default:
- return (size_t)-ZSTD_ERROR_GENERIC;
+ return ERROR(GENERIC);
}
if (blockSize == 0) break; /* bt_end */
- if (ZSTD_isError(errorCode)) return errorCode;
+ if (ZSTDv01_isError(errorCode)) return errorCode;
op += errorCode;
ip += blockSize;
remainingSize -= blockSize;
@@ -2082,7 +2030,7 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
dctx_t* ctx = (dctx_t*)dctx;
/* Sanity check */
- if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
+ if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
if (dst != ctx->previousDstEnd) /* not contiguous */
ctx->base = dst;
@@ -2091,7 +2039,7 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
{
/* Check frame magic header */
U32 magicNumber = ZSTD_readBE32(src);
- if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_MagicNumber;
+ if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
return 0;
@@ -2101,8 +2049,8 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
if (ctx->phase == 1)
{
blockProperties_t bp;
- size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
- if (ZSTD_isError(blockSize)) return blockSize;
+ size_t blockSize = ZSTDv01_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+ if (ZSTDv01_isError(blockSize)) return blockSize;
if (bp.blockType == bt_end)
{
ctx->expected = 0;
@@ -2130,13 +2078,13 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
break;
case bt_rle :
- return (size_t)-ZSTD_ERROR_GENERIC; /* not yet handled */
+ return ERROR(GENERIC); /* not yet handled */
break;
case bt_end : /* should never happen (filtered at phase 1) */
rSize = 0;
break;
default:
- return (size_t)-ZSTD_ERROR_GENERIC;
+ return ERROR(GENERIC);
}
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v01.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v01.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v01.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v01.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v02.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v02.c
similarity index 97%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v02.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v02.c
index af13662..b8a12ab 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v02.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v02.c
@@ -7,15 +7,11 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
#include <stddef.h> /* size_t, ptrdiff_t */
#include "zstd_v02.h"
+#include "error_private.h"
+
/******************************************
* Compiler-specific
@@ -24,46 +20,6 @@ extern "C" {
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#elif defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/******************************************
-* Error Management
-******************************************/
-#define PREFIX(name) ZSTD_error_##name
-
-#define ERROR(name) (size_t)-PREFIX(name)
-
-#define ERROR_LIST(ITEM) \
- ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
- ITEM(PREFIX(memory_allocation)) \
- ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
- ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
- ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \
- ITEM(PREFIX(maxCode))
-
-#define ERROR_GENERATE_ENUM(ENUM) ENUM,
-typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
-
-#define ERROR_CONVERTTOSTRING(STRING) #STRING,
-#define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR)
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
/* ******************************************************************
@@ -1122,12 +1078,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -1595,15 +1554,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -1656,10 +1607,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1701,6 +1654,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1;
@@ -2790,17 +2744,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
#endif
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#else
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2862,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
-static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
+static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -2922,7 +2868,6 @@ struct ZSTD_DCtx_s
blockType_t bType;
U32 phase;
const BYTE* litPtr;
- size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2994,8 +2939,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */
}
case IS_RAW:
@@ -3006,13 +2951,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+3;
- dctx->litBufSize = srcSize-3;
dctx->litSize = litSize;
return litSize+3;
}
@@ -3020,9 +2964,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
{
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[3], litSize);
+ memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
return 4;
}
@@ -3229,7 +3172,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* checks */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
/* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3263,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
- if (oMatchEnd > oend-12)
+ if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@@ -3275,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
}
@@ -3295,7 +3238,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litMax = litPtr + dctx->litBufSize;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@@ -3331,7 +3273,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v02.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v02.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v02.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v02.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v03.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v03.c
similarity index 97%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v03.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v03.c
index 637204d..6459da3 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v03.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v03.c
@@ -7,15 +7,11 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
#include <stddef.h> /* size_t, ptrdiff_t */
#include "zstd_v03.h"
+#include "error_private.h"
+
/******************************************
* Compiler-specific
@@ -24,46 +20,7 @@ extern "C" {
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#elif defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/******************************************
-* Error Management
-******************************************/
-#define PREFIX(name) ZSTD_error_##name
-
-#define ERROR(name) (size_t)-PREFIX(name)
-
-#define ERROR_LIST(ITEM) \
- ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
- ITEM(PREFIX(memory_allocation)) \
- ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
- ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
- ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \
- ITEM(PREFIX(maxCode))
-
-#define ERROR_GENERATE_ENUM(ENUM) ENUM,
-typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
-
-#define ERROR_CONVERTTOSTRING(STRING) #STRING,
-#define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR)
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-
-#if defined (__cplusplus)
-}
-#endif
-#endif /* ERROR_H_MODULE */
/* ******************************************************************
@@ -1122,12 +1079,15 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx;
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -1588,25 +1548,13 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
/* inline is defined */
#elif defined(_MSC_VER)
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# define inline __inline
#else
# define inline /* disable inline */
#endif
-#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-#endif
-
-
/****************************************************************
* Includes
****************************************************************/
@@ -1656,10 +1604,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -1701,6 +1651,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1;
@@ -2432,17 +2383,11 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
#endif
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
#else
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2504,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
-static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
+static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -2564,7 +2509,6 @@ struct ZSTD_DCtx_s
blockType_t bType;
U32 phase;
const BYTE* litPtr;
- size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
@@ -2636,8 +2580,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */
}
case IS_RAW:
@@ -2648,13 +2592,12 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+3;
- dctx->litBufSize = srcSize-3;
dctx->litSize = litSize;
return litSize+3;
}
@@ -2662,9 +2605,8 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
{
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[3], litSize);
+ memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE;
dctx->litSize = litSize;
return 4;
}
@@ -2871,7 +2813,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* checks */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
- if (litEnd > litLimit-8) return ERROR(corruption_detected); /* overRead beyond lit buffer */
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */
/* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -2905,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
- if (oMatchEnd > oend-12)
+ if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@@ -2917,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
}
@@ -2937,7 +2879,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litMax = litPtr + dctx->litBufSize;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@@ -2973,7 +2914,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
+ oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v03.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v03.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v03.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v03.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v04.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v04.c
similarity index 93%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v04.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v04.c
index 1239bdb..723242c 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v04.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v04.c
@@ -10,6 +10,7 @@
/*- Dependencies -*/
#include "zstd_v04.h"
+#include "error_private.h"
/* ******************************************************************
@@ -250,79 +251,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif /* MEM_H_MODULE */
-/* ******************************************************************
- Error codes list
- Copyright (C) 2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Source repository : https://github.com/Cyan4973/zstd
-****************************************************************** */
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* error list
-******************************************/
-enum {
- ZSTD_error_No_Error,
- ZSTD_error_GENERIC,
- ZSTD_error_prefix_unknown,
- ZSTD_error_frameParameter_unsupported,
- ZSTD_error_frameParameter_unsupportedBy32bitsImplementation,
- ZSTD_error_init_missing,
- ZSTD_error_memory_allocation,
- ZSTD_error_stage_wrong,
- ZSTD_error_dstSize_tooSmall,
- ZSTD_error_srcSize_wrong,
- ZSTD_error_corruption_detected,
- ZSTD_error_tableLog_tooLarge,
- ZSTD_error_maxSymbolValue_tooLarge,
- ZSTD_error_maxSymbolValue_tooSmall,
- ZSTD_error_maxCode
-};
-
-/* note : functions provide error codes in reverse negative order,
- so compare with (size_t)(0-enum) */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_PUBLIC_H_MODULE */
-
-
-
/*
zstd - standard compression library
Header File for static linking only
@@ -456,115 +384,6 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstS
}
#endif
-/* ******************************************************************
- Error codes and messages
- Copyright (C) 2013-2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Source repository : https://github.com/Cyan4973/zstd
-****************************************************************** */
-/* Note : this module is expected to remain private, do not expose it */
-
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* *****************************************
-* Includes
-******************************************/
-#include <stddef.h> /* size_t, ptrdiff_t */
-
-
-/* *****************************************
-* Compiler-specific
-******************************************/
-#if defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/* *****************************************
-* Error Codes
-******************************************/
-#define PREFIX(name) ZSTD_error_##name
-
-#ifdef ERROR
-# undef ERROR /* reported already defined on VS 2015 by Rich Geldreich */
-#endif
-#define ERROR(name) (size_t)-PREFIX(name)
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-
-/* *****************************************
-* Error Strings
-******************************************/
-
-ERR_STATIC const char* ERR_getErrorName(size_t code)
-{
- static const char* codeError = "Unspecified error code";
- switch( (size_t)(0-code) )
- {
- case ZSTD_error_No_Error: return "No error detected";
- case ZSTD_error_GENERIC: return "Error (generic)";
- case ZSTD_error_prefix_unknown: return "Unknown frame descriptor";
- case ZSTD_error_frameParameter_unsupported: return "Unsupported frame parameter";
- case ZSTD_error_frameParameter_unsupportedBy32bitsImplementation: return "Frame parameter unsupported in 32-bits mode";
- case ZSTD_error_init_missing: return "Context should be init first";
- case ZSTD_error_memory_allocation: return "Allocation error : not enough memory";
- case ZSTD_error_dstSize_tooSmall: return "Destination buffer is too small";
- case ZSTD_error_srcSize_wrong: return "Src size incorrect";
- case ZSTD_error_corruption_detected: return "Corrupted block detected";
- case ZSTD_error_tableLog_tooLarge: return "tableLog requires too much memory";
- case ZSTD_error_maxSymbolValue_tooLarge: return "Unsupported max possible Symbol Value : too large";
- case ZSTD_error_maxSymbolValue_tooSmall: return "Specified maxSymbolValue is too small";
- case ZSTD_error_maxCode:
- default: return codeError;
- }
-}
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
-
#endif /* ZSTD_STATIC_H */
@@ -668,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
-static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
+static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -1380,12 +1199,15 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -2021,15 +1843,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2082,10 +1896,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
@@ -2127,6 +1943,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1;
@@ -2837,17 +2654,9 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#else
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2897,7 +2706,6 @@ struct ZSTDv04_Dctx_s
blockType_t bType;
ZSTD_dStage stage;
const BYTE* litPtr;
- size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
@@ -2968,7 +2776,7 @@ static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_
size_t result;
if (srcSize != zc->headerSize) return ERROR(srcSize_wrong);
result = ZSTD_getFrameParams(&(zc->params), src, srcSize);
- if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bitsImplementation);
+ if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
return result;
}
@@ -3038,8 +2846,8 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
size_t litSize = BLOCKSIZE;
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
return readSize; /* works if it's an error too */
}
case IS_RAW:
@@ -3050,22 +2858,20 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, 8);
return litSize+3;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+3;
- dctx->litBufSize = srcSize-3;
dctx->litSize = litSize;
return litSize+3; }
case IS_RLE:
{
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[3], litSize);
+ memset(dctx->litBuffer, istart[3], litSize + 8);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
return 4;
}
@@ -3206,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
/* Literal length */
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
prevOffset = litLength ? seq->offset : seqState->prevOffset;
- if (litLength == MaxLL)
- {
+ if (litLength == MaxLL) {
U32 add = *dumps++;
if (add < 255) litLength += add;
- else
- {
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ else {
+ litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
dumps += 3;
}
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
+ if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
/* Offset */
- {
- static const U32 offsetPrefix[MaxOff+1] = {
+ { static const U32 offsetPrefix[MaxOff+1] = {
1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
@@ -3237,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
/* MatchLength */
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
- if (matchLength == MaxML)
- {
+ if (matchLength == MaxML) {
U32 add = *dumps++;
if (add < 255) matchLength += add;
- else
- {
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ else {
+ matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
dumps += 3;
}
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
+ if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
matchLength += MINMATCH;
@@ -3260,7 +3063,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit_8,
+ const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@@ -3275,7 +3078,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3301,12 +3104,16 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
}
}
+ /* Requirement: op <= oend_8 */
/* match within prefix */
- if (sequence.offset < 8)
- {
+ if (sequence.offset < 8) {
/* close range match, overlap */
const int sub2 = dec64table[sequence.offset];
op[0] = match[0];
@@ -3316,14 +3123,12 @@ static size_t ZSTD_execSequence(BYTE* op,
match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match);
match -= sub2;
- }
- else
- {
+ } else {
ZSTD_copy8(op, match);
}
op += 8; match += 8;
- if (oMatchEnd > oend-12)
+ if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@@ -3335,7 +3140,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
- ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -3353,7 +3158,6 @@ static size_t ZSTD_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@@ -3392,7 +3196,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+ oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v04.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v04.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v04.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v04.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v05.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v05.c
similarity index 94%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v05.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v05.c
index 96fdf35..f135924 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v05.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v05.c
@@ -10,6 +10,7 @@
/*- Dependencies -*/
#include "zstd_v05.h"
+#include "error_private.h"
/* ******************************************************************
@@ -254,79 +255,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif /* MEM_H_MODULE */
-/* ******************************************************************
- Error codes list
- Copyright (C) 2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Source repository : https://github.com/Cyan4973/zstd
-****************************************************************** */
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* error codes list
-******************************************/
-typedef enum {
- ZSTDv05_error_no_error,
- ZSTDv05_error_GENERIC,
- ZSTDv05_error_prefix_unknown,
- ZSTDv05_error_frameParameter_unsupported,
- ZSTDv05_error_frameParameter_unsupportedBy32bits,
- ZSTDv05_error_init_missing,
- ZSTDv05_error_memory_allocation,
- ZSTDv05_error_stage_wrong,
- ZSTDv05_error_dstSize_tooSmall,
- ZSTDv05_error_srcSize_wrong,
- ZSTDv05_error_corruption_detected,
- ZSTDv05_error_tableLog_tooLarge,
- ZSTDv05_error_maxSymbolValue_tooLarge,
- ZSTDv05_error_maxSymbolValue_tooSmall,
- ZSTDv05_error_dictionary_corrupted,
- ZSTDv05_error_maxCode
-} ZSTDv05_ErrorCode;
-
-/* note : functions provide error codes in reverse negative order,
- so compare with (size_t)(0-enum) */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_PUBLIC_H_MODULE */
-
-
/*
zstd - standard compression library
Header File for static linking only
@@ -470,119 +398,6 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity
#endif /* ZSTDv05_STATIC_H */
-
-/* ******************************************************************
- Error codes and messages
- Copyright (C) 2013-2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Source repository : https://github.com/Cyan4973/zstd
-****************************************************************** */
-/* Note : this module is expected to remain private, do not expose it */
-
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-
-/* ****************************************
-* Compiler-specific
-******************************************/
-#if defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/*-****************************************
-* Customization
-******************************************/
-typedef ZSTDv05_ErrorCode ERR_enum;
-#define PREFIX(name) ZSTDv05_error_##name
-
-
-/*-****************************************
-* Error codes handling
-******************************************/
-#ifdef ERROR
-# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
-#endif
-#define ERROR(name) (size_t)-PREFIX(name)
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-ERR_STATIC ERR_enum ERR_getError(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
-
-
-/*-****************************************
-* Error Strings
-******************************************/
-
-ERR_STATIC const char* ERR_getErrorName(size_t code)
-{
- static const char* notErrorCode = "Unspecified error code";
- switch( ERR_getError(code) )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size incorrect";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max possible Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(maxCode):
- default: return notErrorCode; /* should be impossible, due to ERR_getError() */
- }
-}
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
/*
zstd_internal - common functions to include
Header File for include
@@ -694,7 +509,7 @@ static void ZSTDv05_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv05_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
-MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
+MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -1380,12 +1195,15 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -2002,14 +1820,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2062,10 +1873,12 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
U32 n;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -2099,6 +1912,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BITv05_highbit32(weightTotal) + 1;
@@ -2221,13 +2035,14 @@ size_t HUFv05_decompress1X2_usingDTable(
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
- size_t errorCode;
const U32 dtLog = DTable[0];
const void* dtPtr = DTable;
const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1;
BITv05_DStream_t bitD;
- errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
- if (HUFv05_isError(errorCode)) return errorCode;
+
+ if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall);
+ { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
+ if (HUFv05_isError(errorCode)) return errorCode; }
HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog);
@@ -2858,17 +2673,9 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#else
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2895,9 +2702,6 @@ static void ZSTDv05_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
* tells if a return value is an error code */
unsigned ZSTDv05_isError(size_t code) { return ERR_isError(code); }
-/*! ZSTDv05_getError() :
-* convert a `size_t` function result into a proper ZSTDv05_errorCode enum */
-ZSTDv05_ErrorCode ZSTDv05_getError(size_t code) { return ERR_getError(code); }
/*! ZSTDv05_getErrorName() :
* provides error code string (useful for debugging) */
@@ -2927,7 +2731,6 @@ struct ZSTDv05_DCtx_s
ZSTDv05_dStage stage;
U32 flagStaticTables;
const BYTE* litPtr;
- size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv05_frameHeaderSize_max];
@@ -3142,6 +2945,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
{
size_t litSize, litCSize, singleStream=0;
U32 lhSize = ((istart[0]) >> 4) & 3;
+ if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
switch(lhSize)
{
case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
@@ -3165,6 +2969,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break;
}
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUFv05_isError(singleStream ?
HUFv05_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
@@ -3172,8 +2977,8 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_PCH:
@@ -3190,13 +2995,14 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_RAW:
@@ -3221,13 +3027,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
- dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@@ -3246,12 +3051,12 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break;
case 3:
litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
+ if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break;
}
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[lhSize], litSize);
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@@ -3263,7 +3068,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, U32 flagStaticTable)
{
const BYTE* const istart = (const BYTE* const)src;
const BYTE* ip = istart;
@@ -3279,17 +3084,22 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
/* SeqHead */
*nbSeq = *ip++;
if (*nbSeq==0) return 1;
- if (*nbSeq >= 128)
+ if (*nbSeq >= 128) {
+ if (ip >= iend) return ERROR(srcSize_wrong);
*nbSeq = ((nbSeq[0]-128)<<8) + *ip++;
+ }
+ if (ip >= iend) return ERROR(srcSize_wrong);
LLtype = *ip >> 6;
Offtype = (*ip >> 4) & 3;
MLtype = (*ip >> 2) & 3;
if (*ip & 2) {
+ if (ip+3 > iend) return ERROR(srcSize_wrong);
dumpsLength = ip[2];
dumpsLength += ip[1] << 8;
ip += 3;
} else {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
dumpsLength = ip[1];
dumpsLength += (ip[0] & 1) << 8;
ip += 2;
@@ -3318,6 +3128,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableLL, LLbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3341,6 +3152,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableOffb, Offbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3364,6 +3176,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableML, MLbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3417,7 +3230,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
if (litLength&1) litLength>>=1, dumps += 3;
else litLength = (U16)(litLength)>>1, dumps += 2;
}
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
+ if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
/* Offset */
@@ -3450,7 +3264,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
if (matchLength&1) matchLength>>=1, dumps += 3;
else matchLength = (U16)(matchLength)>>1, dumps += 2;
}
- if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
+ if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
+ if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
matchLength += MINMATCH;
@@ -3473,7 +3288,7 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTDv05_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit_8,
+ const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
@@ -3488,7 +3303,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
/* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
- if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
+ if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */
ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3512,7 +3327,12 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
@@ -3530,7 +3350,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
}
op += 8; match += 8;
- if (oMatchEnd > oend-12) {
+ if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_8) {
ZSTDv05_wildcopy(op, match, oend_8 - op);
match += oend_8 - op;
@@ -3539,7 +3359,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
while (op < oMatchEnd)
*op++ = *match++;
} else {
- ZSTDv05_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTDv05_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -3557,7 +3377,6 @@ static size_t ZSTDv05_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
size_t errorCode, dumpsLength;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq;
const BYTE* dumps;
@@ -3571,7 +3390,7 @@ static size_t ZSTDv05_decompressSequences(
/* Build Decoding Tables */
errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb,
- ip, seqSize);
+ ip, seqSize, dctx->flagStaticTables);
if (ZSTDv05_isError(errorCode)) return errorCode;
ip += errorCode;
@@ -3595,7 +3414,7 @@ static size_t ZSTDv05_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTDv05_decodeSequence(&sequence, &seqState);
- oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+ oneSeqSize = ZSTDv05_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv05_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
@@ -3860,11 +3679,11 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
{
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSEv05Log;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSEv05Log;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSEv05Log;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
hSize = HUFv05_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUFv05_isError(hSize)) return ERROR(dictionary_corrupted);
@@ -3873,6 +3692,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
offcodeHeaderSize = FSEv05_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSEv05_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSEv05Log) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + offcodeHeaderSize;
@@ -3880,12 +3700,14 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
matchlengthHeaderSize = FSEv05_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSEv05_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSEv05Log) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + matchlengthHeaderSize;
dictSize -= matchlengthHeaderSize;
litlengthHeaderSize = FSEv05_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
+ if (litlengthLog > LLFSEv05Log) return ERROR(dictionary_corrupted);
if (FSEv05_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v05.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v05.h
similarity index 100%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v05.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v05.h
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v06.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v06.c
similarity index 94%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v06.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v06.c
index 6bd463b..8be4bc3 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v06.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v06.c
@@ -13,6 +13,7 @@
#include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */
#include <stdlib.h> /* malloc, free, qsort */
+#include "error_private.h"
@@ -273,77 +274,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif /* MEM_H_MODULE */
-/* ******************************************************************
- Error codes list
- Copyright (C) 2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Homepage : http://www.zstd.net
-****************************************************************** */
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* error codes list
-******************************************/
-typedef enum {
- ZSTDv06_error_no_error,
- ZSTDv06_error_GENERIC,
- ZSTDv06_error_prefix_unknown,
- ZSTDv06_error_frameParameter_unsupported,
- ZSTDv06_error_frameParameter_unsupportedBy32bits,
- ZSTDv06_error_compressionParameter_unsupported,
- ZSTDv06_error_init_missing,
- ZSTDv06_error_memory_allocation,
- ZSTDv06_error_stage_wrong,
- ZSTDv06_error_dstSize_tooSmall,
- ZSTDv06_error_srcSize_wrong,
- ZSTDv06_error_corruption_detected,
- ZSTDv06_error_tableLog_tooLarge,
- ZSTDv06_error_maxSymbolValue_tooLarge,
- ZSTDv06_error_maxSymbolValue_tooSmall,
- ZSTDv06_error_dictionary_corrupted,
- ZSTDv06_error_maxCode
-} ZSTDv06_ErrorCode;
-
-/* note : compare with size_t function results using ZSTDv06_getError() */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_PUBLIC_H_MODULE */
/*
zstd - standard compression library
Header File for static linking only
@@ -396,7 +326,7 @@ extern "C" {
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTDv06_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
-ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
+ZSTDLIBv06_API size_t ZSTDv06_decompress_usingPreparedDCtx(
ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
@@ -407,7 +337,7 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
static const size_t ZSTDv06_frameHeaderSize_min = 5;
static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX;
-ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
+ZSTDLIBv06_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
/*
Streaming decompression, direct mode (bufferless)
@@ -466,140 +396,15 @@ ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
*/
#define ZSTDv06_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
-ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-/*-*************************************
-* Error management
-***************************************/
-/*! ZSTDv06_getErrorCode() :
- convert a `size_t` function result into a `ZSTDv06_ErrorCode` enum type,
- which can be used to compare directly with enum list published into "error_public.h" */
-ZSTDLIB_API ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t functionResult);
-ZSTDLIB_API const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code);
-
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDv06_STATIC_H */
-/* ******************************************************************
- Error codes and messages
- Copyright (C) 2013-2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Homepage : http://www.zstd.net
-****************************************************************** */
-/* Note : this module is expected to remain private, do not expose it */
-
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* Compiler-specific
-******************************************/
-#if defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/*-****************************************
-* Customization (error_public.h)
-******************************************/
-typedef ZSTDv06_ErrorCode ERR_enum;
-#define PREFIX(name) ZSTDv06_error_##name
-
-
-/*-****************************************
-* Error codes handling
-******************************************/
-#ifdef ERROR
-# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
-#endif
-#define ERROR(name) ((size_t)-PREFIX(name))
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
-
-
-/*-****************************************
-* Error Strings
-******************************************/
-
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
-{
- static const char* notErrorCode = "Unspecified error code";
- switch( code )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
- case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size incorrect";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(maxCode):
- default: return notErrorCode;
- }
-}
-
-ERR_STATIC const char* ERR_getErrorName(size_t code)
-{
- return ERR_getErrorString(ERR_getErrorCode(code));
-}
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
/*
zstd_internal - common functions to include
Header File for include
@@ -732,7 +537,7 @@ static void ZSTDv06_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv06_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
-MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
+MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -1665,12 +1470,15 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
@@ -2124,9 +1932,11 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -2161,6 +1971,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv06_highbit32(weightTotal) + 1;
@@ -2238,14 +2049,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2990,14 +2794,6 @@ unsigned ZSTDv06_isError(size_t code) { return ERR_isError(code); }
* provides error code string from function result (useful for debugging) */
const char* ZSTDv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
-/*! ZSTDv06_getError() :
-* convert a `size_t` function result into a proper ZSTDv06_errorCode enum */
-ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
-
-/*! ZSTDv06_getErrorString() :
-* provides error code string from enum */
-const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code) { return ERR_getErrorName(code); }
-
/* **************************************************************
* ZBUFF Error Management
@@ -3054,16 +2850,9 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -3104,7 +2893,6 @@ struct ZSTDv06_DCtx_s
ZSTDv06_dStage stage;
U32 flagRepeatTable;
const BYTE* litPtr;
- size_t litBufSize;
size_t litSize;
BYTE litBuffer[ZSTDv06_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
@@ -3381,8 +3169,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_PCH:
@@ -3397,13 +3185,14 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
+ if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
{ size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
}
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case IS_RAW:
@@ -3427,13 +3216,12 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
- dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@@ -3455,9 +3243,8 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
break;
}
if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[lhSize], litSize);
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@@ -3516,10 +3303,13 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF)
+ if (nbSeq == 0xFF) {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
- else
+ } else {
+ if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
}
*nbSeqPtr = nbSeq;
}
@@ -3645,7 +3435,7 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
size_t ZSTDv06_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit_8,
+ const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@@ -3658,7 +3448,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
/* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
- if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
/* copy Literals */
ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
@@ -3680,7 +3470,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8 || sequence.matchLength < MINMATCH) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
@@ -3708,7 +3503,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
- ZSTDv06_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTDv06_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -3725,7 +3520,6 @@ static size_t ZSTDv06_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
FSEv06_DTable* DTableLL = dctx->LLTable;
FSEv06_DTable* DTableML = dctx->MLTable;
@@ -3769,7 +3563,7 @@ static size_t ZSTDv06_decompressSequences(
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
#endif
- { size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+ { size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv06_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }
@@ -4036,9 +3830,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
dictSize -= hSize;
{ short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + offcodeHeaderSize;
@@ -4046,9 +3841,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + matchlengthHeaderSize;
@@ -4056,9 +3852,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v06.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v06.h
similarity index 63%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v06.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v06.h
index bcc6efb..14040ab 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v06.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v06.h
@@ -14,23 +14,19 @@
extern "C" {
#endif
-/*-*************************************
-* Dependencies
-***************************************/
+/*====== Dependency ======*/
#include <stddef.h> /* size_t */
-/*-***************************************************************
-* Export parameters
-*****************************************************************/
+/*====== Export for Windows ======*/
/*!
* ZSTDv06_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+# define ZSTDLIBv06_API __declspec(dllexport)
#else
-# define ZSTDLIB_API
+# define ZSTDLIBv06_API
#endif
@@ -42,18 +38,18 @@ extern "C" {
`dstCapacity` must be large enough, equal or larger than originalSize.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */
-ZSTDLIB_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
- const void* src, size_t compressedSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
/* *************************************
* Helper functions
***************************************/
-ZSTDLIB_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */
+ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */
/* Error Management */
-ZSTDLIB_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */
+ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */
/* *************************************
@@ -61,12 +57,12 @@ ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides rea
***************************************/
/** Decompression context */
typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx;
-ZSTDLIB_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);
-ZSTDLIB_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */
+ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);
+ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv06_decompressDCtx() :
* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */
-ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-***********************
@@ -76,10 +72,10 @@ ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t d
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */
-ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
/*-************************
@@ -88,12 +84,12 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; };
typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams;
-ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
-ZSTDLIB_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx);
+ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx);
-ZSTDLIB_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx);
-ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx);
+ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -102,15 +98,15 @@ ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, siz
***************************************/
typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx;
-ZSTDLIB_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void);
-ZSTDLIB_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx);
+ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void);
+ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx);
+ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
- void* dst, size_t* dstCapacityPtr,
- const void* src, size_t* srcSizePtr);
+ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
+ void* dst, size_t* dstCapacityPtr,
+ const void* src, size_t* srcSizePtr);
/*-***************************************************************************
* Streaming decompression howto
@@ -140,13 +136,13 @@ ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
/* *************************************
* Tool functions
***************************************/
-ZSTDLIB_API unsigned ZBUFFv06_isError(size_t errorCode);
-ZSTDLIB_API const char* ZBUFFv06_getErrorName(size_t errorCode);
+ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode);
+ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
-ZSTDLIB_API size_t ZBUFFv06_recommendedDInSize(void);
-ZSTDLIB_API size_t ZBUFFv06_recommendedDOutSize(void);
+ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void);
+ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void);
/*-*************************************
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v07.c b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v07.c
similarity index 94%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v07.c
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v07.c
index f948069..b1fcdc7 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v07.c
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v07.c
@@ -21,6 +21,8 @@
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
#define ZSTDv07_STATIC_LINKING_ONLY
+#include "error_private.h"
+
#ifdef ZSTDv07_STATIC_LINKING_ONLY
@@ -66,27 +68,27 @@ typedef struct { ZSTDv07_allocFunction customAlloc; ZSTDv07_freeFunction customF
/*! ZSTDv07_estimateDCtxSize() :
* Gives the potential amount of memory allocated to create a ZSTDv07_DCtx */
-ZSTDLIB_API size_t ZSTDv07_estimateDCtxSize(void);
+ZSTDLIBv07_API size_t ZSTDv07_estimateDCtxSize(void);
/*! ZSTDv07_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
-ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem);
+ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem);
/*! ZSTDv07_sizeofDCtx() :
* Gives the amount of memory used by a given ZSTDv07_DCtx */
-ZSTDLIB_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx);
/* ******************************************************************
* Buffer-less streaming functions (synchronous mode)
********************************************************************/
-ZSTDLIB_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx);
-ZSTDLIB_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx);
+ZSTDLIBv07_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv07_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx);
-ZSTDLIB_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx);
-ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv07_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*
Buffer-less streaming decompression (synchronous mode)
@@ -167,8 +169,8 @@ ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, siz
*/
#define ZSTDv07_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
-ZSTDLIB_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
+ZSTDLIBv07_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
#endif /* ZSTDv07_STATIC_LINKING_ONLY */
@@ -427,203 +429,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif
#endif /* MEM_H_MODULE */
-
-/* ******************************************************************
- Error codes list
- Copyright (C) 2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Homepage : http://www.zstd.net
-****************************************************************** */
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-* error codes list
-******************************************/
-typedef enum {
- ZSTDv07_error_no_error,
- ZSTDv07_error_GENERIC,
- ZSTDv07_error_prefix_unknown,
- ZSTDv07_error_frameParameter_unsupported,
- ZSTDv07_error_frameParameter_unsupportedBy32bits,
- ZSTDv07_error_compressionParameter_unsupported,
- ZSTDv07_error_init_missing,
- ZSTDv07_error_memory_allocation,
- ZSTDv07_error_stage_wrong,
- ZSTDv07_error_dstSize_tooSmall,
- ZSTDv07_error_srcSize_wrong,
- ZSTDv07_error_corruption_detected,
- ZSTDv07_error_checksum_wrong,
- ZSTDv07_error_tableLog_tooLarge,
- ZSTDv07_error_maxSymbolValue_tooLarge,
- ZSTDv07_error_maxSymbolValue_tooSmall,
- ZSTDv07_error_dictionary_corrupted,
- ZSTDv07_error_dictionary_wrong,
- ZSTDv07_error_maxCode
-} ZSTDv07_ErrorCode;
-
-/*! ZSTDv07_getErrorCode() :
- convert a `size_t` function result into a `ZSTDv07_ErrorCode` enum type,
- which can be used to compare directly with enum list published into "error_public.h" */
-ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t functionResult);
-const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code);
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_PUBLIC_H_MODULE */
-/* ******************************************************************
- Error codes and messages
- Copyright (C) 2013-2016, Yann Collet
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - Homepage : http://www.zstd.net
-****************************************************************** */
-/* Note : this module is expected to remain private, do not expose it */
-
-#ifndef ERROR_H_MODULE
-#define ERROR_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-
-/* ****************************************
-* Compiler-specific
-******************************************/
-#if defined(__GNUC__)
-# define ERR_STATIC static __attribute__((unused))
-#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-# define ERR_STATIC static inline
-#elif defined(_MSC_VER)
-# define ERR_STATIC static __inline
-#else
-# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
-#endif
-
-
-/*-****************************************
-* Customization (error_public.h)
-******************************************/
-typedef ZSTDv07_ErrorCode ERR_enum;
-#define PREFIX(name) ZSTDv07_error_##name
-
-
-/*-****************************************
-* Error codes handling
-******************************************/
-#ifdef ERROR
-# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
-#endif
-#define ERROR(name) ((size_t)-PREFIX(name))
-
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-
-ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
-
-
-/*-****************************************
-* Error Strings
-******************************************/
-
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
-{
- static const char* notErrorCode = "Unspecified error code";
- switch( code )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
- case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size incorrect";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(dictionary_wrong): return "Dictionary mismatch";
- case PREFIX(maxCode):
- default: return notErrorCode;
- }
-}
-
-ERR_STATIC const char* ERR_getErrorName(size_t code)
-{
- return ERR_getErrorString(ERR_getErrorCode(code));
-}
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_H_MODULE */
/* ******************************************************************
bitstream
Part of FSE library
@@ -845,8 +650,8 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits)
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD)
{
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
- return BITv07_DStream_overflow;
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
+ return BITv07_DStream_overflow;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
bitD->ptr -= bitD->bitsConsumed >> 3;
@@ -1577,9 +1382,11 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
- size_t iSize = ip[0];
+ size_t iSize;
size_t oSize;
+ if (!srcSize) return ERROR(srcSize_wrong);
+ iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
@@ -1614,6 +1421,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
+ if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv07_highbit32(weightTotal) + 1;
@@ -1680,17 +1488,18 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
-# ifdef __GNUC__
-# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# define FORCE_INLINE static inline
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
-
-
/* **************************************************************
* Error Management
****************************************************************/
@@ -2006,14 +1815,7 @@ size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -2888,13 +2690,6 @@ unsigned ZSTDv07_isError(size_t code) { return ERR_isError(code); }
* provides error code string from function result (useful for debugging) */
const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); }
-/*! ZSTDv07_getError() :
-* convert a `size_t` function result into a proper ZSTDv07_errorCode enum */
-ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
-
-/*! ZSTDv07_getErrorString() :
-* provides error code string from enum */
-const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code) { return ERR_getErrorName(code); }
/* **************************************************************
@@ -3050,7 +2845,7 @@ static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv07_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
-MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length)
+MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@@ -3176,17 +2971,10 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
#endif
@@ -3233,7 +3021,6 @@ struct ZSTDv07_DCtx_s
U32 dictID;
const BYTE* litPtr;
ZSTDv07_customMem customMem;
- size_t litBufSize;
size_t litSize;
BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX];
@@ -3607,9 +3394,9 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize;
dctx->litEntropy = 1;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case lbt_repeat:
@@ -3630,8 +3417,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
if (HUFv07_isError(errorCode)) return ERROR(corruption_detected);
}
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize;
}
case lbt_raw:
@@ -3655,13 +3442,12 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart+lhSize, litSize);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
- dctx->litBufSize = srcSize-lhSize;
dctx->litSize = litSize;
return lhSize+litSize;
}
@@ -3683,9 +3469,8 @@ size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
break;
}
if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
- memset(dctx->litBuffer, istart[lhSize], litSize);
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
- dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
@@ -3744,10 +3529,13 @@ size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) {
- if (nbSeq == 0xFF)
+ if (nbSeq == 0xFF) {
+ if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
- else
+ } else {
+ if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
+ }
}
*nbSeqPtr = nbSeq;
}
@@ -3871,7 +3659,7 @@ static seq_t ZSTDv07_decodeSequence(seqState_t* seqState)
static
size_t ZSTDv07_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
- const BYTE** litPtr, const BYTE* const litLimit_w,
+ const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
@@ -3883,7 +3671,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
/* check */
if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
- if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
/* copy Literals */
ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
@@ -3905,7 +3693,12 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_w || sequence.matchLength < MINMATCH) {
+ while (op < oMatchEnd) *op++ = *match++;
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
@@ -3933,7 +3726,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
- ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTDv07_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -3950,7 +3743,6 @@ static size_t ZSTDv07_decompressSequences(
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
- const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize;
FSEv07_DTable* DTableLL = dctx->LLTable;
FSEv07_DTable* DTableML = dctx->MLTable;
@@ -3980,7 +3772,7 @@ static size_t ZSTDv07_decompressSequences(
for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) {
nbSeq--;
{ seq_t const sequence = ZSTDv07_decodeSequence(&seqState);
- size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
+ size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }
@@ -4046,7 +3838,7 @@ size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx,
/** ZSTDv07_insertBlock() :
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
-ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize)
+ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize)
{
ZSTDv07_checkContinuity(dctx, blockStart);
dctx->previousDstEnd = (const char*)blockStart + blockSize;
@@ -4312,36 +4104,39 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si
}
{ short offcodeNCount[MaxOff+1];
- U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
+ U32 offcodeMaxValue=MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += offcodeHeaderSize;
}
{ short matchlengthNCount[MaxML+1];
- unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += matchlengthHeaderSize;
}
{ short litlengthNCount[MaxLL+1];
- unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += litlengthHeaderSize;
}
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
- dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
- dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
- dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
+ dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1;
@@ -4448,7 +4243,7 @@ size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict)
/*! ZSTDv07_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */
-ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
+ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTDv07_DDict* ddict)
@@ -4535,7 +4330,7 @@ struct ZBUFFv07_DCtx_s {
ZSTDv07_customMem customMem;
}; /* typedef'd to ZBUFFv07_DCtx within "zstd_buffered.h" */
-ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem);
+ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem);
ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void)
{
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v07.h b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v07.h
similarity index 70%
rename from c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v07.h
rename to c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v07.h
index d1fbc08..30725dc 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/legacy/zstd_v07.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/legacy/zstd_v07.h
@@ -24,13 +24,12 @@ extern "C" {
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+# define ZSTDLIBv07_API __declspec(dllexport)
#else
-# define ZSTDLIB_API
+# define ZSTDLIBv07_API
#endif
-
/* *************************************
* Simple API
***************************************/
@@ -46,12 +45,12 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize);
`dstCapacity` must be equal or larger than originalSize.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */
-ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
- const void* src, size_t compressedSize);
+ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
/*====== Helper functions ======*/
-ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */
+ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */
/*-*************************************
@@ -59,12 +58,12 @@ ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides rea
***************************************/
/** Decompression context */
typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx;
-ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void);
-ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */
+ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void);
+ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv07_decompressDCtx() :
* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */
-ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-************************
@@ -74,10 +73,10 @@ ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t d
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression.
* Note : This function load the dictionary, resulting in a significant startup time */
-ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize);
+ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
/*-**************************
@@ -87,16 +86,16 @@ ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */
typedef struct ZSTDv07_DDict_s ZSTDv07_DDict;
-ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict);
+ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize);
+ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict);
/*! ZSTDv07_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */
-ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTDv07_DDict* ddict);
+ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTDv07_DDict* ddict);
typedef struct {
unsigned long long frameContentSize;
@@ -105,7 +104,7 @@ typedef struct {
unsigned checksumFlag;
} ZSTDv07_frameParams;
-ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
@@ -114,13 +113,13 @@ ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const
* Streaming functions
***************************************/
typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx;
-ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void);
-ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx);
+ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void);
+ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx,
+ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
@@ -152,13 +151,13 @@ ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx,
/* *************************************
* Tool functions
***************************************/
-ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode);
-ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode);
+ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode);
+ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
-ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void);
-ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void);
+ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void);
+ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void);
/*-*************************************
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/libzstd.pc.in b/c-blosc/internal-complibs/zstd-1.1.3/libzstd.pc.in
similarity index 53%
rename from c-blosc/internal-complibs/zstd-1.0.0/libzstd.pc.in
rename to c-blosc/internal-complibs/zstd-1.1.3/libzstd.pc.in
index 28afc3a..1d07b91 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/libzstd.pc.in
+++ b/c-blosc/internal-complibs/zstd-1.1.3/libzstd.pc.in
@@ -1,5 +1,5 @@
# ZSTD - standard compression algorithm
-# Copyright (C) 2014-2015, Yann Collet.
+# Copyright (C) 2014-2016, Yann Collet, Facebook
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@
@@ -7,8 +7,8 @@ libdir=@LIBDIR@
includedir=@INCLUDEDIR@
Name: zstd
-Description: lossless compression algorithm library
-URL: https://github.com/Cyan4973/zstd
+Description: fast lossless compression algorithm library
+URL: http://www.zstd.net/
Version: @VERSION@
-Libs: -L at LIBDIR@ -lzstd
-Cflags: -I at INCLUDEDIR@
+Libs: -L${libdir} -lzstd
+Cflags: -I${includedir}
diff --git a/c-blosc/internal-complibs/zstd-1.0.0/zstd.h b/c-blosc/internal-complibs/zstd-1.1.3/zstd.h
similarity index 61%
rename from c-blosc/internal-complibs/zstd-1.0.0/zstd.h
rename to c-blosc/internal-complibs/zstd-1.1.3/zstd.h
index d768ded..f5cbf4b 100644
--- a/c-blosc/internal-complibs/zstd-1.0.0/zstd.h
+++ b/c-blosc/internal-complibs/zstd-1.1.3/zstd.h
@@ -1,4 +1,4 @@
-/**
+/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
@@ -7,33 +7,56 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#ifndef ZSTD_H_235446
-#define ZSTD_H_235446
-
#if defined (__cplusplus)
extern "C" {
#endif
-/*====== Dependency ======*/
+#ifndef ZSTD_H_235446
+#define ZSTD_H_235446
+
+/* ====== Dependency ======*/
#include <stddef.h> /* size_t */
-/*====== Export for Windows ======*/
-/*!
-* ZSTD_DLL_EXPORT :
-* Enable exporting of functions when building a Windows DLL
-*/
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+/* ===== ZSTDLIB_API : control library symbols visibility ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+# define ZSTDLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
-# define ZSTDLIB_API
+# define ZSTDLIB_API ZSTDLIB_VISIBILITY
#endif
-/*======= Version =======*/
+/*******************************************************************************************************
+ Introduction
+
+ zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+ at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
+ decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+ Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+ Compression can be done in:
+ - a single step (described as Simple API)
+ - a single step, reusing a context (described as Explicit memory management)
+ - unbounded multiple steps (described as Streaming compression)
+ The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+ - a single step (described as Simple dictionary API)
+ - a single step, reusing a dictionary (described as Fast dictionary API)
+
+ Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+ These APIs shall never be used with a dynamic library.
+ They are not "stable", their definition may change in the future. Only static linking is allowed.
+*********************************************************************************************************/
+
+/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
-#define ZSTD_VERSION_MINOR 0
-#define ZSTD_VERSION_RELEASE 0
+#define ZSTD_VERSION_MINOR 1
+#define ZSTD_VERSION_RELEASE 3
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@@ -41,47 +64,48 @@ extern "C" {
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-ZSTDLIB_API unsigned ZSTD_versionNumber (void);
+ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */
-/* *************************************
+/***************************************
* Simple API
***************************************/
/*! ZSTD_compress() :
- Compresses `src` buffer into already allocated `dst`.
+ Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
- @return : the number of bytes written into `dst` (<= `dstCapacity),
- or an error code if it fails (which can be tested using ZSTD_isError()) */
+ @return : compressed size written into `dst` (<= `dstCapacity),
+ or an error code if it fails (which can be tested using ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
+/*! ZSTD_decompress() :
+ `compressedSize` : must be the _exact_ size of a single compressed frame.
+ `dstCapacity` is an upper bound of originalSize.
+ If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
+ @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+ or an errorCode if it fails (which can be tested using ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
+
/*! ZSTD_getDecompressedSize() :
-* @return : decompressed size as a 64-bits value _if known_, 0 otherwise.
-* note 1 : decompressed size can be very large (64-bits value),
-* potentially larger than what local system can handle as a single memory segment.
+* 'src' is the start of a zstd compressed frame.
+* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
+* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+* When `return==0`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
-* note 2 : decompressed size is an optional field, that may not be present.
-* When `return==0`, data to decompress can have any size.
+* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+* (For example, data may be necessarily cut into blocks <= 16 KB).
+* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+* note 3 : decompressed size can be very large (64-bits value),
+* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
-* Optionally, application may rely on its own implied limits.
-* (For example, application data could be necessarily cut into blocks <= 16 KB).
-* note 3 : decompressed size could be wrong or intentionally modified !
-* Always ensure result fits within application's authorized limits !
+* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+* Always ensure result fits within application's authorized limits.
* Each application can set its own limits.
-* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
+* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-/*! ZSTD_decompress() :
- `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
- `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ).
- If originalSize is unknown, and if there is no implied application-specific limitations,
- it's preferable to use streaming mode to decompress data.
- @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
- or an errorCode if it fails (which can be tested using ZSTD_isError()) */
-ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
- const void* src, size_t compressedSize);
-
/*====== Helper functions ======*/
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
@@ -90,34 +114,39 @@ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `siz
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
-/*-*************************************
+/***************************************
* Explicit memory management
***************************************/
-/** Compression context */
+/*= Compression context
+* When compressing many times,
+* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+* This will make workload friendlier for system's memory.
+* Use one context per thread for parallel execution in multi-threaded environments. */
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-/** ZSTD_compressCCtx() :
- Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */
+/*! ZSTD_compressCCtx() :
+ Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
-/** Decompression context */
+/*= Decompression context */
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-/** ZSTD_decompressDCtx() :
-* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
+/*! ZSTD_decompressDCtx() :
+* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-/*-************************
+/**************************
* Simple dictionary API
***************************/
/*! ZSTD_compress_usingDict() :
* Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-* Note : This function load the dictionary, resulting in significant startup delay. */
+* Note : This function loads the dictionary, resulting in significant startup delay.
+* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -127,41 +156,53 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
/*! ZSTD_decompress_usingDict() :
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
* Dictionary must be identical to the one used during compression.
-* Note : This function load the dictionary, resulting in significant startup delay */
+* Note : This function loads the dictionary, resulting in significant startup delay.
+* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
-/*-**************************
-* Fast Dictionary API
+/****************************
+* Fast dictionary API
****************************/
-/*! ZSTD_createCDict() :
-* Create a digested dictionary, ready to start compression operation without startup delay.
-* `dict` can be released after ZSTD_CDict creation */
typedef struct ZSTD_CDict_s ZSTD_CDict;
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_createCDict() :
+* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
+* `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
+/*! ZSTD_freeCDict() :
+* Function frees memory allocated by ZSTD_createCDict(). */
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
/*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-* Note that compression level is decided during dictionary creation */
+* Note that compression level is decided during dictionary creation. */
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict);
+
+typedef struct ZSTD_DDict_s ZSTD_DDict;
+
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
-* `dict` can be released after creation */
-typedef struct ZSTD_DDict_s ZSTD_DDict;
-ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+* dictBuffer can be released after DDict creation, as its content is copied inside DDict */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
+
+/*! ZSTD_freeDDict() :
+* Function frees memory allocated with ZSTD_createDDict() */
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
/*! ZSTD_decompress_usingDDict() :
-* Decompression using a digested Dictionary
+* Decompression using a digested Dictionary.
* Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
@@ -169,7 +210,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict);
-/*-**************************
+/****************************
* Streaming
****************************/
@@ -186,31 +227,33 @@ typedef struct ZSTD_outBuffer_s {
} ZSTD_outBuffer;
-/*====== streaming compression ======*/
/*-***********************************************************************
-* Streaming compression - howto
+* Streaming compression - HowTo
*
* A ZSTD_CStream object is required to track streaming operation.
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+* since it will play nicer with system's memory, by re-using already allocated memory.
+* Use one separate ZSTD_CStream per thread for parallel execution.
*
-* Start by initializing ZSTD_CStream.
+* Start a new compression by initializing ZSTD_CStream.
* Use ZSTD_initCStream() to start a new compression operation.
-* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+* Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
*
* Use ZSTD_compressStream() repetitively to consume input stream.
-* The function will automatically update both `pos`.
+* The function will automatically update both `pos` fields.
* Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data.
* @return : a size hint, preferred nb of bytes to use as input for next function call
-* (it's just a hint, to help latency a little, any other value will work fine)
-* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
* or an error code, which can be tested using ZSTD_isError().
+* Note 1 : it's just a hint, to help latency a little, any other value will work fine.
+* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
*
-* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+* At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
* `output->pos` will be updated.
-* Note some content might still be left within internal buffer if `output->size` is too small.
+* Note that some content might still be left within internal buffer if `output->size` is too small.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
*
@@ -219,7 +262,7 @@ typedef struct ZSTD_outBuffer_s {
* The epilogue is required for decoders to consider a frame completed.
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
* In which case, call again ZSTD_endStream() to complete the flush.
-* @return : nb of bytes still present within internal buffer (0 if it's empty)
+* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************/
@@ -228,19 +271,18 @@ typedef struct ZSTD_CStream_s ZSTD_CStream;
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */
-
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+
-/*====== decompression ======*/
/*-***************************************************************************
-* Streaming decompression howto
+* Streaming decompression - HowTo
*
* A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
@@ -248,67 +290,73 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
*
* Use ZSTD_initDStream() to start a new decompression operation,
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
+* @return : recommended first input size
*
* Use ZSTD_decompressStream() repetitively to consume your input.
-* The function will update both `pos`.
-* Note that it may not consume the entire input (pos < size),
-* in which case it's up to the caller to present remaining input again.
+* The function will update both `pos` fields.
+* If `input.pos < input.size`, some input has not been consumed.
+* It's up to the caller to present again remaining data.
+* If `output.pos < output.size`, decoder has flushed everything it could.
* @return : 0 when a frame is completely decoded and fully flushed,
-* 1 when there is still some data left within internal buffer to flush,
-* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted),
-* or an error code, which can be tested using ZSTD_isError().
-*
+* an error code, which can be tested using ZSTD_isError(),
+* any other value > 0, which means there is still some decoding to do to complete current frame.
+* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
* *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */
-
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
+#endif /* ZSTD_H_235446 */
-#ifdef ZSTD_STATIC_LINKING_ONLY
-/* ====================================================================================
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
+/****************************************************************************************
+ * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
* The definitions in this section are considered experimental.
* They should never be used with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
- * ==================================================================================== */
+ * ***************************************************************************************/
-/*--- Constants ---*/
-#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */
+/* --- Constants ---*/
+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX_32 25
#define ZSTD_WINDOWLOG_MAX_64 27
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
-#define ZSTD_WINDOWLOG_MIN 18
-#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
-#define ZSTD_CHAINLOG_MIN 4
+#define ZSTD_WINDOWLOG_MIN 10
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
-#define ZSTD_HASHLOG_MIN 12
+#define ZSTD_HASHLOG_MIN 6
+#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
+#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
#define ZSTD_HASHLOG3_MAX 17
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
-#define ZSTD_SEARCHLENGTH_MAX 7
-#define ZSTD_SEARCHLENGTH_MIN 3
+#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
+#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MIN 4
#define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
-static const size_t ZSTD_frameHeaderSize_min = 5;
+#define ZSTD_FRAMEHEADERSIZE_MIN 6
+static const size_t ZSTD_frameHeaderSize_prefix = 5;
+static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
-/*--- Types ---*/
-typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */
+/*--- Advanced types ---*/
+typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
typedef struct {
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -321,9 +369,9 @@ typedef struct {
} ZSTD_compressionParameters;
typedef struct {
- unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
- unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
- unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+ unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+ unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */
+ unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@@ -331,13 +379,13 @@ typedef struct {
ZSTD_frameParameters fParams;
} ZSTD_parameters;
-/* custom memory allocation functions */
+/*= Custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-/*-*************************************
+/***************************************
* Advanced compression functions
***************************************/
/*! ZSTD_estimateCCtxSize() :
@@ -349,24 +397,42 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+/*! ZSTD_sizeofCCtx() :
+ * Gives the amount of memory used by a given ZSTD_CCtx */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+
+typedef enum {
+ ZSTD_p_forceWindow /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
+} ZSTD_CCtxParameter;
+/*! ZSTD_setCCtxParameter() :
+ * Set advanced parameters, selected through enum ZSTD_CCtxParameter
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
+
+/*! ZSTD_createCDict_byReference() :
+ * Create a digested dictionary for compression
+ * Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem);
-/*! ZSTD_sizeofCCtx() :
- * Gives the amount of memory used by a given ZSTD_CCtx */
-ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+/*! ZSTD_sizeof_CDict() :
+ * Gives the amount of memory used by a given ZSTD_sizeof_CDict */
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
+
+/*! ZSTD_getCParams() :
+* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+* `estimatedSrcSize` value is optional, select 0 if not known */
+ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
/*! ZSTD_getParams() :
-* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
* All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
-
-/*! ZSTD_getCParams() :
-* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
-* `srcSize` value is optional, select 0 if not known */
-ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
/*! ZSTD_checkCParams() :
* Ensure param values remain within authorized range */
@@ -386,7 +452,14 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
ZSTD_parameters params);
-/*--- Advanced Decompression functions ---*/
+/*--- Advanced decompression functions ---*/
+
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
/*! ZSTD_estimateDCtxSize() :
* Gives the potential amount of memory allocated to create a ZSTD_DCtx */
@@ -396,50 +469,84 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
-/*! ZSTD_sizeofDCtx() :
+/*! ZSTD_sizeof_DCtx() :
* Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-
-/* ******************************************************************
-* Advanced Streaming functions
+/*! ZSTD_createDDict_byReference() :
+ * Create a digested dictionary, ready to start decompression operation without startup delay.
+ * Dictionary content is simply referenced, and therefore stays in dictBuffer.
+ * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
+ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+ unsigned byReference, ZSTD_customMem customMem);
+
+/*! ZSTD_sizeof_DDict() :
+ * Gives the amount of memory used by a given ZSTD_DDict */
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
+
+/********************************************************************
+* Advanced streaming functions
********************************************************************/
-/*====== compression ======*/
-
+/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
- ZSTD_parameters params, unsigned long long pledgedSrcSize);
+ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
+ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-/*====== decompression ======*/
-
-typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
-
+/*===== Advanced Streaming decompression functions =====*/
+typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
+ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-/* ******************************************************************
+/*********************************************************************
* Buffer-less and synchronous inner streaming functions
-********************************************************************/
-/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+*
+* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with many restrictions (documented below).
-* Prefer using normal streaming API for an easier experience */
+* Prefer using normal streaming API for an easier experience
+********************************************************************* */
-ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
-
-ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+/**
+ Buffer-less streaming compression (synchronous mode)
-/*
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -462,32 +569,24 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
- It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
- Without last block mark, frames will be considered unfinished (broken) by decoders.
+ It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+ Without last block mark, frames will be considered unfinished (corrupted) by decoders.
- You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+ `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
*/
-typedef struct {
- unsigned long long frameContentSize;
- unsigned windowSize;
- unsigned dictID;
- unsigned checksumFlag;
-} ZSTD_frameParams;
-
-ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
-
-ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+/*===== Buffer-less streaming compression functions =====*/
+ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
-ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
-ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-/*
+/*-
Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations.
@@ -544,11 +643,27 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
It also returns Frame Size as fparamsPtr->frameContentSize.
*/
+typedef struct {
+ unsigned long long frameContentSize;
+ unsigned windowSize;
+ unsigned dictID;
+ unsigned checksumFlag;
+} ZSTD_frameParams;
+
+/*===== Buffer-less streaming decompression functions =====*/
+ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
+ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
+ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-/* **************************************
-* Block functions
-****************************************/
-/*! Block functions produce and decode raw zstd blocks, without frame metadata.
+/**
+ Block functions
+
+ Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -572,16 +687,15 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
*/
#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
+/*===== Raw zstd block functions =====*/
ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
-#endif /* ZSTD_STATIC_LINKING_ONLY */
+#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
-
-#endif /* ZSTD_H_235446 */
diff --git a/c-blosc/tests/Makefile b/c-blosc/tests/Makefile
index 7234803..bd93c85 100644
--- a/c-blosc/tests/Makefile
+++ b/c-blosc/tests/Makefile
@@ -9,7 +9,7 @@ SOURCES := $(wildcard *.c)
EXECUTABLES := $(patsubst %.c, %.exe, $(SOURCES))
# Support for internal LZ4 and LZ4HC
-LZ4_DIR = ../internal-complibs/lz4-1.7.2
+LZ4_DIR = ../internal-complibs/lz4-1.7.5
CFLAGS += -DHAVE_LZ4 -I$(LZ4_DIR)
BLOSC_LIB += $(wildcard $(LZ4_DIR)/*.c)
@@ -34,7 +34,7 @@ LDFLAGS += -DHAVE_ZLIB -lz
# Support for internal Zstd
-ZSTD_DIR = ../internal-complibs/zstd-1.0.0
+ZSTD_DIR = ../internal-complibs/zstd-1.1.3
CFLAGS += -DHAVE_ZSTD -I$(ZSTD_DIR) -I$(ZSTD_DIR)/common
BLOSC_LIB += $(wildcard $(ZSTD_DIR)/*/*.c)
diff --git a/c-blosc/tests/test_common.h b/c-blosc/tests/test_common.h
index 204ff82..4e40533 100644
--- a/c-blosc/tests/test_common.h
+++ b/c-blosc/tests/test_common.h
@@ -62,7 +62,7 @@ static void* blosc_test_malloc(const size_t alignment, const size_t size)
void *block = NULL;
int32_t res = 0;
-#if _ISOC11_SOURCE
+#if _ISOC11_SOURCE || (__STDC_VERSION__ >= 201112L && !defined(__APPLE__))
/* C11 aligned allocation. 'size' must be a multiple of the alignment. */
block = aligned_alloc(alignment, size);
#elif defined(_WIN32)
diff --git a/cpuinfo.py b/cpuinfo.py
index d0bcfa9..32e603e 100644
--- a/cpuinfo.py
+++ b/cpuinfo.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
-# Copyright (c) 2014-2016, Matthew Brennan Jones <matthew.brennan.jones at gmail.com>
+# Copyright (c) 2014-2017, Matthew Brennan Jones <matthew.brennan.jones at gmail.com>
# Py-cpuinfo is a Python module to show the cpuinfo of a processor
# It uses a MIT style license
# It is hosted at: https://github.com/workhorsy/py-cpuinfo
@@ -25,6 +25,7 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CPUINFO_VERSION = (3, 0, 0)
import os, sys
import re
@@ -52,6 +53,7 @@ class DataSource(object):
cpu_count = multiprocessing.cpu_count()
is_windows = platform.system().lower() == 'windows'
raw_arch_string = platform.machine()
+ can_cpuid = True
@staticmethod
def has_proc_cpuinfo():
@@ -62,6 +64,10 @@ class DataSource(object):
return len(program_paths('dmesg')) > 0
@staticmethod
+ def has_var_run_dmesg_boot():
+ return os.path.exists('/var/run/dmesg.boot')
+
+ @staticmethod
def has_cpufreq_info():
return len(program_paths('cpufreq-info')) > 0
@@ -110,6 +116,10 @@ class DataSource(object):
return run_and_get_stdout(['dmesg', '-a'])
@staticmethod
+ def cat_var_run_dmesg_boot():
+ return run_and_get_stdout(['cat', '/var/run/dmesg.boot'])
+
+ @staticmethod
def sysctl_machdep_cpu_hw_cpufrequency():
return run_and_get_stdout(['sysctl', 'machdep.cpu', 'hw.cpufrequency'])
@@ -263,46 +273,6 @@ def _get_hz_string_from_brand(processor_brand):
return (scale, hz_brand)
-def _get_hz_string_from_beagle_bone():
- scale, hz_brand = 1, '0.0'
-
- if not DataSource.has_cpufreq_info():
- return scale, hz_brand
-
- returncode, output = DataSource.cpufreq_info()
- if returncode != 0:
- return (scale, hz_brand)
-
- hz_brand = output.split('current CPU frequency is')[1].split('.')[0].lower()
-
- if hz_brand.endswith('mhz'):
- scale = 6
- elif hz_brand.endswith('ghz'):
- scale = 9
- hz_brand = hz_brand.rstrip('mhz').rstrip('ghz').strip()
- hz_brand = to_hz_string(hz_brand)
-
- return (scale, hz_brand)
-
-def _get_hz_string_from_lscpu():
- scale, hz_brand = 1, '0.0'
-
- if not DataSource.has_lscpu():
- return scale, hz_brand
-
- returncode, output = DataSource.lscpu()
- if returncode != 0:
- return (scale, hz_brand)
-
- new_hz = _get_field(False, output, None, None, 'CPU max MHz', 'CPU MHz')
- if new_hz == None:
- return (scale, hz_brand)
-
- new_hz = to_hz_string(new_hz)
- scale = 6
-
- return (scale, new_hz)
-
def to_friendly_hz(ticks, scale):
# Get the raw Hz as a string
left, right = to_raw_hz(ticks, scale)
@@ -364,6 +334,145 @@ def to_hz_string(ticks):
return ticks
+def _parse_cpu_string(cpu_string):
+ # Get location of fields at end of string
+ fields_index = cpu_string.find('(', cpu_string.find('@'))
+ #print(fields_index)
+
+ # Processor Brand
+ processor_brand = cpu_string
+ if fields_index != -1:
+ processor_brand = cpu_string[0 : fields_index].strip()
+ #print('processor_brand: ', processor_brand)
+
+ fields = None
+ if fields_index != -1:
+ fields = cpu_string[fields_index : ]
+ #print('fields: ', fields)
+
+ # Hz
+ scale, hz_brand = _get_hz_string_from_brand(processor_brand)
+
+ # Various fields
+ vendor_id, stepping, model, family = (None, None, None, None)
+ if fields:
+ try:
+ fields = fields.rsplit('(', 1)[1].split(')')[0].split(',')
+ fields = [f.strip().lower() for f in fields]
+ fields = [f.split(':') for f in fields]
+ fields = [{f[0].strip() : f[1].strip()} for f in fields]
+ #print('fields: ', fields)
+ for field in fields:
+ name = list(field.keys())[0]
+ value = list(field.values())[0]
+ #print('name:{0}, value:{1}'.format(name, value))
+ if name == 'origin':
+ vendor_id = value.strip('"')
+ elif name == 'stepping':
+ stepping = int(value.lstrip('0x'), 16)
+ elif name == 'model':
+ model = int(value.lstrip('0x'), 16)
+ elif name in ['fam', 'family']:
+ family = int(value.lstrip('0x'), 16)
+ except:
+ #raise
+ pass
+
+ return (processor_brand, hz_brand, scale, vendor_id, stepping, model, family)
+
+def _parse_dmesg_output(output):
+ try:
+ # Get all the dmesg lines that might contain a CPU string
+ lines = output.split(' CPU0:')[1:] + \
+ output.split(' CPU1:')[1:] + \
+ output.split(' CPU:')[1:] + \
+ output.split('\nCPU0:')[1:] + \
+ output.split('\nCPU1:')[1:] + \
+ output.split('\nCPU:')[1:]
+ lines = [l.split('\n')[0].strip() for l in lines]
+
+ # Convert the lines to CPU strings
+ cpu_strings = [_parse_cpu_string(l) for l in lines]
+
+ # Find the CPU string that has the most fields
+ best_string = None
+ highest_count = 0
+ for cpu_string in cpu_strings:
+ count = sum([n is not None for n in cpu_string])
+ if count > highest_count:
+ highest_count = count
+ best_string = cpu_string
+
+ # If no CPU string was found, return {}
+ if not best_string:
+ return {}
+
+ processor_brand, hz_actual, scale, vendor_id, stepping, model, family = best_string
+
+ # Origin
+ if ' Origin=' in output:
+ fields = output[output.find(' Origin=') : ].split('\n')[0]
+ fields = fields.strip().split()
+ fields = [n.strip().split('=') for n in fields]
+ fields = [{n[0].strip().lower() : n[1].strip()} for n in fields]
+ #print('fields: ', fields)
+
+ for field in fields:
+ name = list(field.keys())[0]
+ value = list(field.values())[0]
+ #print('name:{0}, value:{1}'.format(name, value))
+ if name == 'origin':
+ vendor_id = value.strip('"')
+ elif name == 'stepping':
+ stepping = int(value.lstrip('0x'), 16)
+ elif name == 'model':
+ model = int(value.lstrip('0x'), 16)
+ elif name in ['fam', 'family']:
+ family = int(value.lstrip('0x'), 16)
+
+ #print('FIELDS: ', (vendor_id, stepping, model, family))
+
+ # Features
+ flag_lines = []
+ for category in [' Features=', ' Features2=', ' AMD Features=', ' AMD Features2=']:
+ if category in output:
+ flag_lines.append(output.split(category)[1].split('\n')[0])
+
+ flags = []
+ for line in flag_lines:
+ line = line.split('<')[1].split('>')[0].lower()
+ for flag in line.split(','):
+ flags.append(flag)
+ flags.sort()
+
+ # Convert from GHz/MHz string to Hz
+ scale, hz_advertised = _get_hz_string_from_brand(processor_brand)
+
+ info = {
+ 'vendor_id' : vendor_id,
+ 'brand' : processor_brand,
+
+ 'stepping' : stepping,
+ 'model' : model,
+ 'family' : family,
+ 'flags' : flags
+ }
+
+ if hz_advertised and hz_advertised != '0.0':
+ info['hz_advertised'] = to_friendly_hz(hz_advertised, scale)
+ info['hz_actual'] = to_friendly_hz(hz_actual, scale)
+
+ if hz_advertised and hz_advertised != '0.0':
+ info['hz_advertised_raw'] = to_raw_hz(hz_advertised, scale)
+ info['hz_actual_raw'] = to_raw_hz(hz_actual, scale)
+
+ return {k: v for k, v in info.items() if v}
+ except:
+ #raise
+ pass
+
+ return {}
+
def parse_arch(raw_arch_string):
arch, bits = None, None
raw_arch_string = raw_arch_string.lower()
@@ -376,7 +485,7 @@ def parse_arch(raw_arch_string):
arch = 'X86_64'
bits = 64
# ARM
- elif re.match('^armv8-a$', raw_arch_string):
+ elif re.match('^armv8-a|aarch64$', raw_arch_string):
arch = 'ARM_8'
bits = 64
elif re.match('^armv7$|^armv7[a-z]$|^armv7-[a-z]$|^armv6[a-z]$', raw_arch_string):
@@ -389,7 +498,7 @@ def parse_arch(raw_arch_string):
elif re.match('^ppc32$|^prep$|^pmac$|^powermac$', raw_arch_string):
arch = 'PPC_32'
bits = 32
- elif re.match('^powerpc$|^ppc64$', raw_arch_string):
+ elif re.match('^powerpc$|^ppc64$|^ppc64le$', raw_arch_string):
arch = 'PPC_64'
bits = 64
# SPARC
@@ -580,7 +689,7 @@ class CPUID(object):
'extended_family' : extended_family
}
- # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000000h:_Get_Highest_Extended_Function_Supported
+ # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000000h:_Get_Highest_Extended_Function_Supported
def get_max_extension_support(self):
# Check for extension support
max_extension_support = self._run_asm(
@@ -684,13 +793,13 @@ class CPUID(object):
# Get the Extended CPU flags
extended_flags = {}
- # https://en.wikipedia.org/wiki/CPUID#EAX.3D7.2C_ECX.3D0:_Extended_Features
+ # http://en.wikipedia.org/wiki/CPUID#EAX.3D7.2C_ECX.3D0:_Extended_Features
if max_extension_support == 7:
pass
# FIXME: Are we missing all these flags too?
# avx2 et cetera ...
- # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000001h:_Extended_Processor_Info_and_Feature_Bits
+ # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000001h:_Extended_Processor_Info_and_Feature_Bits
if max_extension_support >= 0x80000001:
# EBX # FIXME: This may need to be EDX instead
ebx = self._run_asm(
@@ -784,7 +893,7 @@ class CPUID(object):
flags.sort()
return flags
- # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000002h.2C80000003h.2C80000004h:_Processor_Brand_String
+ # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000002h.2C80000003h.2C80000004h:_Processor_Brand_String
def get_processor_brand(self, max_extension_support):
processor_brand = ""
@@ -838,7 +947,7 @@ class CPUID(object):
return processor_brand
- # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000006h:_Extended_L2_Cache_Features
+ # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000006h:_Extended_L2_Cache_Features
def get_cache(self, max_extension_support):
cache_info = {}
@@ -922,31 +1031,23 @@ class CPUID(object):
return ticks
-def get_cpu_info_from_cpuid():
+def actual_get_cpu_info_from_cpuid():
'''
- Returns the CPU info gathered by querying the X86 cpuid register in a new process.
- Returns None of non X86 cpus.
- Returns None if SELinux is in enforcing mode.
+ Warning! This function has the potential to crash the Python runtime.
+ Do not call it directly. Use the _get_cpu_info_from_cpuid function instead.
+ It will safely call this function in another process.
'''
-
- returncode, output = run_and_get_stdout([sys.executable, "-c", "import cpuinfo; print(cpuinfo.actual_get_cpu_info_from_cpuid())"])
- if returncode != 0:
- return None
- info = b64_to_obj(output)
- return info
-
-def actual_get_cpu_info_from_cpuid():
# Get the CPU arch and bits
arch, bits = parse_arch(DataSource.raw_arch_string)
# Return none if this is not an X86 CPU
if not arch in ['X86_32', 'X86_64']:
- return None
+ return obj_to_b64({})
# Return none if SE Linux is in enforcing mode
cpuid = CPUID()
if cpuid.is_selinux_enforcing:
- return None
+ return obj_to_b64({})
# Get the cpu info from the CPUID register
max_extension_support = cpuid.get_max_extension_support()
@@ -972,11 +1073,6 @@ def actual_get_cpu_info_from_cpuid():
'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
'hz_actual_raw' : to_raw_hz(hz_actual, 6),
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : DataSource.raw_arch_string,
-
'l2_cache_size' : cache_info['size_kb'],
'l2_cache_line_size' : cache_info['line_size_b'],
'l2_cache_associativity' : hex(cache_info['associativity']),
@@ -989,21 +1085,47 @@ def actual_get_cpu_info_from_cpuid():
'extended_family' : info['extended_family'],
'flags' : cpuid.get_flags(max_extension_support)
}
+
+ info = {k: v for k, v in info.items() if v}
return obj_to_b64(info)
-def get_cpu_info_from_proc_cpuinfo():
+def _get_cpu_info_from_cpuid():
'''
- Returns the CPU info gathered from /proc/cpuinfo. Will return None if
- /proc/cpuinfo is not found.
+ Returns the CPU info gathered by querying the X86 cpuid register in a new process.
+ Returns {} on non X86 cpus.
+ Returns {} if SELinux is in enforcing mode.
+ '''
+
+ # Return {} if can't cpuid
+ if not DataSource.can_cpuid:
+ return {}
+
+ # Get the CPU arch and bits
+ arch, bits = parse_arch(DataSource.raw_arch_string)
+
+ # Return {} if this is not an X86 CPU
+ if not arch in ['X86_32', 'X86_64']:
+ return {}
+
+ returncode, output = run_and_get_stdout([sys.executable, "-c", "import cpuinfo; print(cpuinfo.actual_get_cpu_info_from_cpuid())"])
+ if returncode != 0:
+ return {}
+ info = b64_to_obj(output)
+ return info
+
+def _get_cpu_info_from_proc_cpuinfo():
+ '''
+ Returns the CPU info gathered from /proc/cpuinfo.
+ Returns {} if /proc/cpuinfo is not found.
'''
try:
- # Just return None if there is no cpuinfo
+ # Just return {} if there is no cpuinfo
if not DataSource.has_proc_cpuinfo():
- return None
+ return {}
returncode, output = DataSource.cat_proc_cpuinfo()
if returncode != 0:
- return None
+ return {}
# Various fields
vendor_id = _get_field(False, output, None, '', 'vendor_id', 'vendor id', 'vendor')
@@ -1014,8 +1136,10 @@ def get_cpu_info_from_proc_cpuinfo():
family = _get_field(False, output, int, 0, 'cpu family')
hardware = _get_field(False, output, None, '', 'Hardware')
# Flags
- flags = _get_field(False, output, None, None, 'flags', 'Features').split()
- flags.sort()
+ flags = _get_field(False, output, None, None, 'flags', 'Features')
+ if flags:
+ flags = flags.split()
+ flags.sort()
# Convert from MHz string to Hz
hz_actual = _get_field(False, output, None, '', 'cpu MHz', 'cpu speed', 'clock')
@@ -1023,164 +1147,190 @@ def get_cpu_info_from_proc_cpuinfo():
hz_actual = to_hz_string(hz_actual)
# Convert from GHz/MHz string to Hz
- scale, hz_advertised = _get_hz_string_from_brand(processor_brand)
-
- # Try getting the Hz for a BeagleBone
- if hz_advertised == '0.0':
- scale, hz_advertised = _get_hz_string_from_beagle_bone()
- hz_actual = hz_advertised
-
- # Try getting the Hz for a lscpu
- if hz_advertised == '0.0':
- scale, hz_advertised = _get_hz_string_from_lscpu()
- hz_actual = hz_advertised
-
- # Get the CPU arch and bits
- arch, bits = parse_arch(DataSource.raw_arch_string)
+ scale, hz_advertised = (0, None)
+ try:
+ scale, hz_advertised = _get_hz_string_from_brand(processor_brand)
+ except Exception:
+ pass
- return {
- 'vendor_id' : vendor_id,
+ info = {
'hardware' : hardware,
'brand' : processor_brand,
- 'hz_advertised' : to_friendly_hz(hz_advertised, scale),
- 'hz_actual' : to_friendly_hz(hz_actual, 6),
- 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
- 'hz_actual_raw' : to_raw_hz(hz_actual, 6),
-
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : DataSource.raw_arch_string,
-
'l2_cache_size' : cache_size,
- 'l2_cache_line_size' : 0,
- 'l2_cache_associativity' : 0,
-
+ 'flags' : flags,
+ 'vendor_id' : vendor_id,
'stepping' : stepping,
'model' : model,
'family' : family,
- 'processor_type' : 0,
- 'extended_model' : 0,
- 'extended_family' : 0,
- 'flags' : flags
}
+
+ # Make the Hz the same for actual and advertised if missing any
+ if not hz_advertised or hz_advertised == '0.0':
+ hz_advertised = hz_actual
+ scale = 6
+ elif not hz_actual or hz_actual == '0.0':
+ hz_actual = hz_advertised
+
+ # Add the Hz if there is one
+ if to_raw_hz(hz_advertised, scale) > (0, 0):
+ info['hz_advertised'] = to_friendly_hz(hz_advertised, scale)
+ info['hz_advertised_raw'] = to_raw_hz(hz_advertised, scale)
+ if to_raw_hz(hz_actual, scale) > (0, 0):
+ info['hz_actual'] = to_friendly_hz(hz_actual, 6)
+ info['hz_actual_raw'] = to_raw_hz(hz_actual, 6)
+
+ info = {k: v for k, v in info.items() if v}
+ return info
except:
#raise # NOTE: To have this throw on error, uncomment this line
- return None
+ return {}
-def get_cpu_info_from_dmesg():
+def _get_cpu_info_from_cpufreq_info():
'''
- Returns the CPU info gathered from dmesg. Will return None if
- dmesg is not found or does not have the desired info.
+ Returns the CPU info gathered from cpufreq-info.
+ Returns {} if cpufreq-info is not found.
'''
try:
- # Just return None if there is no dmesg
- if not DataSource.has_dmesg():
- return None
+ scale, hz_brand = 1, '0.0'
- # If dmesg fails return None
- returncode, output = DataSource.dmesg_a()
- if output == None or returncode != 0:
- return None
+ if not DataSource.has_cpufreq_info():
+ return {}
- # Processor Brand
- long_brand = output.split('CPU: ')[1].split('\n')[0]
- processor_brand = long_brand.rsplit('(', 1)[0]
- processor_brand = processor_brand.strip()
+ returncode, output = DataSource.cpufreq_info()
+ if returncode != 0:
+ return {}
- # Hz
- scale = 0
- hz_actual = long_brand.rsplit('(', 1)[1].split(' ')[0].lower()
- if hz_actual.endswith('mhz'):
+ hz_brand = output.split('current CPU frequency is')[1].split('\n')[0].rsplit('.', 1)[0].lower().strip()
+
+ if hz_brand.endswith('mhz'):
scale = 6
- elif hz_actual.endswith('ghz'):
+ elif hz_brand.endswith('ghz'):
scale = 9
- hz_actual = hz_actual.split('-')[0]
- hz_actual = to_hz_string(hz_actual)
+ hz_brand = hz_brand.rstrip('mhz').rstrip('ghz').strip()
+ hz_brand = to_hz_string(hz_brand)
+
+ info = {
+ 'hz_advertised' : to_friendly_hz(hz_brand, scale),
+ 'hz_actual' : to_friendly_hz(hz_brand, scale),
+ 'hz_advertised_raw' : to_raw_hz(hz_brand, scale),
+ 'hz_actual_raw' : to_raw_hz(hz_brand, scale),
+ }
- # Various fields
- fields = output.split('CPU: ')[1].split('\n')[1].split('\n')[0].strip().split(' ')
- vendor_id = None
- stepping = None
- model = None
- family = None
- for field in fields:
- name, value = field.split('=')
- name = name.strip().lower()
- value = value.strip()
- if name == 'origin':
- vendor_id = value.strip('"')
- elif name == 'stepping':
- stepping = int(value)
- elif name == 'model':
- model = int(value, 16)
- elif name == 'family':
- family = int(value, 16)
+ info = {k: v for k, v in info.items() if v}
+ return info
+ except:
+ #raise # NOTE: To have this throw on error, uncomment this line
+ return {}
- # Flags
- flag_lines = []
- for category in [' Features=', ' Features2=', ' AMD Features=', ' AMD Features2=']:
- if category in output:
- flag_lines.append(output.split(category)[1].split('\n')[0])
+def _get_cpu_info_from_lscpu():
+ '''
+ Returns the CPU info gathered from lscpu.
+ Returns {} if lscpu is not found.
+ '''
+ try:
+ if not DataSource.has_lscpu():
+ return {}
- flags = []
- for line in flag_lines:
- line = line.split('<')[1].split('>')[0].lower()
- for flag in line.split(','):
- flags.append(flag)
- flags.sort()
+ returncode, output = DataSource.lscpu()
+ if returncode != 0:
+ return {}
- # Convert from GHz/MHz string to Hz
- scale, hz_advertised = _get_hz_string_from_brand(processor_brand)
+ info = {}
- # Get the CPU arch and bits
- arch, bits = parse_arch(DataSource.raw_arch_string)
+ new_hz = _get_field(False, output, None, None, 'CPU max MHz', 'CPU MHz')
+ if new_hz:
+ new_hz = to_hz_string(new_hz)
+ scale = 6
+ info['hz_advertised'] = to_friendly_hz(new_hz, scale)
+ info['hz_actual'] = to_friendly_hz(new_hz, scale)
+ info['hz_advertised_raw'] = to_raw_hz(new_hz, scale)
+ info['hz_actual_raw'] = to_raw_hz(new_hz, scale)
- return {
- 'vendor_id' : vendor_id,
- 'hardware' : '',
- 'brand' : processor_brand,
+ vendor_id = _get_field(False, output, None, None, 'Vendor ID')
+ if vendor_id:
+ info['vendor_id'] = vendor_id
- 'hz_advertised' : to_friendly_hz(hz_advertised, scale),
- 'hz_actual' : to_friendly_hz(hz_actual, 6),
- 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
- 'hz_actual_raw' : to_raw_hz(hz_actual, 6),
+ brand = _get_field(False, output, None, None, 'Model name')
+ if brand:
+ info['brand'] = brand
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : DataSource.raw_arch_string,
+ brand = _get_field(False, output, None, None, 'Model name')
+ if brand:
+ info['brand'] = brand
- 'l2_cache_size' : 0,
- 'l2_cache_line_size' : 0,
- 'l2_cache_associativity' : 0,
+ family = _get_field(False, output, None, None, 'CPU family')
+ if family and family.isdigit():
+ info['family'] = int(family)
- 'stepping' : stepping,
- 'model' : model,
- 'family' : family,
- 'processor_type' : 0,
- 'extended_model' : 0,
- 'extended_family' : 0,
- 'flags' : flags
- }
+ stepping = _get_field(False, output, None, None, 'Stepping')
+ if stepping and stepping.isdigit():
+ info['stepping'] = int(stepping)
+
+ model = _get_field(False, output, None, None, 'Model')
+ if model and model.isdigit():
+ info['model'] = int(model)
+
+ # Flags
+ flags = _get_field(False, output, None, None, 'flags', 'Features')
+ if flags:
+ flags = flags.split()
+ flags.sort()
+ info['flags'] = flags
+
+ info = {k: v for k, v in info.items() if v}
+ return info
except:
- return None
+ #raise # NOTE: To have this throw on error, uncomment this line
+ return {}
+
+def _get_cpu_info_from_dmesg():
+ '''
+ Returns the CPU info gathered from dmesg.
+ Returns {} if dmesg is not found or does not have the desired info.
+ '''
+ # Just return {} if there is no dmesg
+ if not DataSource.has_dmesg():
+ return {}
+
+ # If dmesg fails return {}
+ returncode, output = DataSource.dmesg_a()
+ if output == None or returncode != 0:
+ return {}
+
+ return _parse_dmesg_output(output)
-def get_cpu_info_from_sysctl():
+def _get_cpu_info_from_cat_var_run_dmesg_boot():
'''
- Returns the CPU info gathered from sysctl. Will return None if
- sysctl is not found.
+ Returns the CPU info gathered from /var/run/dmesg.boot.
+ Returns {} if dmesg is not found or does not have the desired info.
+ '''
+ # Just return {} if there is no /var/run/dmesg.boot
+ if not DataSource.has_var_run_dmesg_boot():
+ return {}
+
+ # If dmesg.boot fails return {}
+ returncode, output = DataSource.cat_var_run_dmesg_boot()
+ if output == None or returncode != 0:
+ return {}
+
+ return _parse_dmesg_output(output)
+
+
+def _get_cpu_info_from_sysctl():
+ '''
+ Returns the CPU info gathered from sysctl.
+ Returns {} if sysctl is not found.
'''
try:
- # Just return None if there is no sysctl
+ # Just return {} if there is no sysctl
if not DataSource.has_sysctl():
- return None
+ return {}
- # If sysctl fails return None
+ # If sysctl fails return {}
returncode, output = DataSource.sysctl_machdep_cpu_hw_cpufrequency()
if output == None or returncode != 0:
- return None
+ return {}
# Various fields
vendor_id = _get_field(False, output, None, None, 'machdep.cpu.vendor')
@@ -1191,7 +1341,9 @@ def get_cpu_info_from_sysctl():
family = _get_field(False, output, int, 0, 'machdep.cpu.family')
# Flags
- flags = _get_field(False, output, None, None, 'machdep.cpu.features').lower().split()
+ flags = _get_field(False, output, None, '', 'machdep.cpu.features').lower().split()
+ flags.extend(_get_field(False, output, None, '', 'machdep.cpu.leaf7_features').lower().split())
+ flags.extend(_get_field(False, output, None, '', 'machdep.cpu.extfeatures').lower().split())
flags.sort()
# Convert from GHz/MHz string to Hz
@@ -1199,12 +1351,8 @@ def get_cpu_info_from_sysctl():
hz_actual = _get_field(False, output, None, None, 'hw.cpufrequency')
hz_actual = to_hz_string(hz_actual)
- # Get the CPU arch and bits
- arch, bits = parse_arch(DataSource.raw_arch_string)
-
- return {
+ info = {
'vendor_id' : vendor_id,
- 'hardware' : '',
'brand' : processor_brand,
'hz_advertised' : to_friendly_hz(hz_advertised, scale),
@@ -1212,40 +1360,33 @@ def get_cpu_info_from_sysctl():
'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
'hz_actual_raw' : to_raw_hz(hz_actual, 0),
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : DataSource.raw_arch_string,
-
'l2_cache_size' : cache_size,
- 'l2_cache_line_size' : 0,
- 'l2_cache_associativity' : 0,
'stepping' : stepping,
'model' : model,
'family' : family,
- 'processor_type' : 0,
- 'extended_model' : 0,
- 'extended_family' : 0,
'flags' : flags
}
+
+ info = {k: v for k, v in info.items() if v}
+ return info
except:
- return None
+ return {}
-def get_cpu_info_from_sysinfo():
+def _get_cpu_info_from_sysinfo():
'''
- Returns the CPU info gathered from sysinfo. Will return None if
- sysinfo is not found.
+ Returns the CPU info gathered from sysinfo.
+ Returns {} if sysinfo is not found.
'''
try:
- # Just return None if there is no sysinfo
+ # Just return {} if there is no sysinfo
if not DataSource.has_sysinfo():
- return None
+ return {}
- # If sysinfo fails return None
+ # If sysinfo fails return {}
returncode, output = DataSource.sysinfo_cpu()
if output == None or returncode != 0:
- return None
+ return {}
# Various fields
vendor_id = '' #_get_field(False, output, None, None, 'CPU #0: ')
@@ -1267,12 +1408,8 @@ def get_cpu_info_from_sysinfo():
scale, hz_advertised = _get_hz_string_from_brand(processor_brand)
hz_actual = hz_advertised
- # Get the CPU arch and bits
- arch, bits = parse_arch(DataSource.raw_arch_string)
-
- return {
+ info = {
'vendor_id' : vendor_id,
- 'hardware' : '',
'brand' : processor_brand,
'hz_advertised' : to_friendly_hz(hz_advertised, scale),
@@ -1280,36 +1417,29 @@ def get_cpu_info_from_sysinfo():
'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
'hz_actual_raw' : to_raw_hz(hz_actual, scale),
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : DataSource.raw_arch_string,
-
'l2_cache_size' : cache_size,
- 'l2_cache_line_size' : 0,
- 'l2_cache_associativity' : 0,
'stepping' : stepping,
'model' : model,
'family' : family,
- 'processor_type' : 0,
- 'extended_model' : 0,
- 'extended_family' : 0,
'flags' : flags
}
+
+ info = {k: v for k, v in info.items() if v}
+ return info
except:
- return None
+ return {}
-def get_cpu_info_from_registry():
+def _get_cpu_info_from_registry():
'''
FIXME: Is missing many of the newer CPU flags like sse3
- Returns the CPU info gathered from the Windows Registry. Will return None if
- not on Windows.
+ Returns the CPU info gathered from the Windows Registry.
+ Returns {} if not on Windows.
'''
try:
- # Just return None if not on Windows
+ # Just return {} if not on Windows
if not DataSource.is_windows:
- return None
+ return {}
# Get the CPU name
processor_brand = DataSource.winreg_processor_brand()
@@ -1378,9 +1508,8 @@ def get_cpu_info_from_registry():
flags = [k for k, v in flags.items() if v]
flags.sort()
- return {
+ info = {
'vendor_id' : vendor_id,
- 'hardware' : '',
'brand' : processor_brand,
'hz_advertised' : to_friendly_hz(hz_advertised, scale),
@@ -1388,50 +1517,37 @@ def get_cpu_info_from_registry():
'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
'hz_actual_raw' : to_raw_hz(hz_actual, 6),
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : raw_arch_string,
-
- 'l2_cache_size' : 0,
- 'l2_cache_line_size' : 0,
- 'l2_cache_associativity' : 0,
-
- 'stepping' : 0,
- 'model' : 0,
- 'family' : 0,
- 'processor_type' : 0,
- 'extended_model' : 0,
- 'extended_family' : 0,
'flags' : flags
}
+
+ info = {k: v for k, v in info.items() if v}
+ return info
except:
- return None
+ return {}
-def get_cpu_info_from_kstat():
+def _get_cpu_info_from_kstat():
'''
- Returns the CPU info gathered from isainfo and kstat. Will
- return None if isainfo or kstat are not found.
+ Returns the CPU info gathered from isainfo and kstat.
+ Returns {} if isainfo or kstat are not found.
'''
try:
- # Just return None if there is no isainfo or kstat
+ # Just return {} if there is no isainfo or kstat
if not DataSource.has_isainfo() or not DataSource.has_kstat():
- return None
+ return {}
- # If isainfo fails return None
+ # If isainfo fails return {}
returncode, flag_output = DataSource.isainfo_vb()
if flag_output == None or returncode != 0:
- return None
+ return {}
- # If kstat fails return None
+ # If kstat fails return {}
returncode, kstat = DataSource.kstat_m_cpu_info()
if kstat == None or returncode != 0:
- return None
+ return {}
# Various fields
vendor_id = kstat.split('\tvendor_id ')[1].split('\n')[0].strip()
processor_brand = kstat.split('\tbrand ')[1].split('\n')[0].strip()
- cache_size = 0
stepping = int(kstat.split('\tstepping ')[1].split('\n')[0].strip())
model = int(kstat.split('\tmodel ')[1].split('\n')[0].strip())
family = int(kstat.split('\tfamily ')[1].split('\n')[0].strip())
@@ -1449,12 +1565,8 @@ def get_cpu_info_from_kstat():
hz_actual = kstat.split('\tcurrent_clock_Hz ')[1].split('\n')[0].strip()
hz_actual = to_hz_string(hz_actual)
- # Get the CPU arch and bits
- arch, bits = parse_arch(DataSource.raw_arch_string)
-
- return {
+ info = {
'vendor_id' : vendor_id,
- 'hardware' : '',
'brand' : processor_brand,
'hz_advertised' : to_friendly_hz(hz_advertised, scale),
@@ -1462,64 +1574,84 @@ def get_cpu_info_from_kstat():
'hz_advertised_raw' : to_raw_hz(hz_advertised, scale),
'hz_actual_raw' : to_raw_hz(hz_actual, 0),
- 'arch' : arch,
- 'bits' : bits,
- 'count' : DataSource.cpu_count,
- 'raw_arch_string' : DataSource.raw_arch_string,
-
- 'l2_cache_size' : cache_size,
- 'l2_cache_line_size' : 0,
- 'l2_cache_associativity' : 0,
-
'stepping' : stepping,
'model' : model,
'family' : family,
- 'processor_type' : 0,
- 'extended_model' : 0,
- 'extended_family' : 0,
'flags' : flags
}
+
+ info = {k: v for k, v in info.items() if v}
+ return info
except:
- return None
+ return {}
+
+def CopyNewFields(info, new_info):
+ keys = [
+ 'vendor_id', 'hardware', 'brand', 'hz_advertised', 'hz_actual',
+ 'hz_advertised_raw', 'hz_actual_raw', 'arch', 'bits', 'count',
+ 'raw_arch_string', 'l2_cache_size', 'l2_cache_line_size',
+ 'l2_cache_associativity', 'stepping', 'model', 'family',
+ 'processor_type', 'extended_model', 'extended_family', 'flags'
+ ]
+
+ for key in keys:
+ if new_info.get(key, None) and not info.get(key, None):
+ info[key] = new_info[key]
def get_cpu_info():
- info = None
+ '''
+ Returns the CPU info by using the best sources of information for your OS.
+ Returns {} if nothing is found.
+ '''
+
+ # Get the CPU arch and bits
+ arch, bits = parse_arch(DataSource.raw_arch_string)
+
+ info = {
+ 'cpuinfo_version' : CPUINFO_VERSION,
+ 'arch' : arch,
+ 'bits' : bits,
+ 'count' : DataSource.cpu_count,
+ 'raw_arch_string' : DataSource.raw_arch_string,
+ }
# Try the Windows registry
- if not info:
- info = get_cpu_info_from_registry()
+ CopyNewFields(info, _get_cpu_info_from_registry())
# Try /proc/cpuinfo
- if not info:
- info = get_cpu_info_from_proc_cpuinfo()
+ CopyNewFields(info, _get_cpu_info_from_proc_cpuinfo())
+
+ # Try cpufreq-info
+ CopyNewFields(info, _get_cpu_info_from_cpufreq_info())
+
+ # Try LSCPU
+ CopyNewFields(info, _get_cpu_info_from_lscpu())
# Try sysctl
- if not info:
- info = get_cpu_info_from_sysctl()
+ CopyNewFields(info, _get_cpu_info_from_sysctl())
# Try kstat
- if not info:
- info = get_cpu_info_from_kstat()
+ CopyNewFields(info, _get_cpu_info_from_kstat())
# Try dmesg
- if not info:
- info = get_cpu_info_from_dmesg()
+ CopyNewFields(info, _get_cpu_info_from_dmesg())
+
+ # Try /var/run/dmesg.boot
+ CopyNewFields(info, _get_cpu_info_from_cat_var_run_dmesg_boot())
# Try sysinfo
- if not info:
- info = get_cpu_info_from_sysinfo()
+ CopyNewFields(info, _get_cpu_info_from_sysinfo())
# Try querying the CPU cpuid register
- if not info:
- info = get_cpu_info_from_cpuid()
+ CopyNewFields(info, _get_cpu_info_from_cpuid())
return info
# Make sure we are running on a supported system
def _check_arch():
arch, bits = parse_arch(DataSource.raw_arch_string)
- if not arch in ['X86_32', 'X86_64', 'ARM_7', 'ARM_8']:
- raise Exception("py-cpuinfo currently only works on X86 and some ARM CPUs.")
+ if not arch in ['X86_32', 'X86_64', 'ARM_7', 'ARM_8', 'PPC_64']:
+ raise Exception("py-cpuinfo currently only works on X86 and some PPC and ARM CPUs.")
def main():
try:
@@ -1530,6 +1662,7 @@ def main():
info = get_cpu_info()
if info:
+ print('cpuinfo version: {0}'.format(info.get('cpuinfo_version', '')))
print('Vendor ID: {0}'.format(info.get('vendor_id', '')))
print('Hardware Raw: {0}'.format(info.get('hardware', '')))
print('Brand: {0}'.format(info.get('brand', '')))
diff --git a/doc/source/_templates/travis-ci.html b/doc/source/_templates/travis-ci.html
deleted file mode 100644
index 6a6d185..0000000
--- a/doc/source/_templates/travis-ci.html
+++ /dev/null
@@ -1,3 +0,0 @@
-<div>
-<a href="https://travis-ci.org/PyTables/PyTables"><img src="https://travis-ci.org/PyTables/PyTables.png" alt="travis-ci status page"/></a>
-</div>
diff --git a/doc/source/_theme/altered_nature/static/altered_nature.css_t b/doc/source/_theme/altered_nature/static/altered_nature.css_t
deleted file mode 100644
index 7a5e6d9..0000000
--- a/doc/source/_theme/altered_nature/static/altered_nature.css_t
+++ /dev/null
@@ -1,299 +0,0 @@
-/**
- * Sphinx stylesheet -- default theme
- * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- */
-
- at import url("basic.css");
-
-/* -- page layout ----------------------------------------------------------- */
-
-body {
- font-family: Times, serif;
- font-size: 100%;
- background-color: #111;
- color: #555;
- margin: 0;
- padding: 0;
-}
-
-div.documentwrapper {
- float: left;
- width: 100%;
-}
-
-div.bodywrapper {
- margin: 0 0 0 230px;
-}
-
-hr{
- border: 1px solid #B1B4B6;
-}
-
-div.document {
- background-color: #eee;
-}
-
-div.body {
- background-color: #ffffff;
- color: #3E4349;
- padding: 0 30px 30px 30px;
- font-size: 1em;
-}
-
-div.footer {
- color: #555;
- width: 100%;
- padding: 13px 0;
- text-align: center;
- font-size: 75%;
-}
-
-div.footer a {
- color: #444;
- text-decoration: underline;
-}
-
-div.related {
- background-color: #6BA81E;
- line-height: 32px;
- color: #fff;
- text-shadow: 0px 1px 0 #444;
- font-size: 0.80em;
-}
-
-div.related a {
- color: #E2F3CC;
-}
-
-div.sphinxsidebar {
- font-size: 0.75em;
- line-height: 1.5em;
-}
-
-div.sphinxsidebarwrapper{
- padding: 20px 0;
-}
-
-div.sphinxsidebar h3,
-div.sphinxsidebar h4 {
- font-family: Times, serif;
- color: #222;
- font-size: 1.2em;
- font-weight: normal;
- margin: 0;
- padding: 5px 10px;
- background-color: #ddd;
- text-shadow: 1px 1px 0 white
-}
-
-div.sphinxsidebar h4{
- font-size: 1.1em;
-}
-
-div.sphinxsidebar h3 a {
- color: #444;
-}
-
-
-div.sphinxsidebar p {
- color: #888;
- padding: 5px 20px;
-}
-
-div.sphinxsidebar p.topless {
-}
-
-div.sphinxsidebar ul {
- margin: 10px 20px;
- padding: 0;
- color: #000;
-}
-
-div.sphinxsidebar a {
- color: #444;
-}
-
-div.sphinxsidebar input {
- border: 1px solid #ccc;
- font-family: serif;
- font-size: 1em;
-}
-
-div.sphinxsidebar input[type=text]{
- margin-left: 20px;
-}
-
-/* -- body styles ----------------------------------------------------------- */
-
-a {
- color: #005B81;
- text-decoration: none;
-}
-
-a:hover {
- color: #E32E00;
- text-decoration: underline;
-}
-
-div.body h1,
-div.body h2,
-div.body h3,
-div.body h4,
-div.body h5,
-div.body h6 {
- font-family: Times, serif;
- background-color: #BED4EB;
- font-weight: normal;
- color: #212224;
- margin: 30px 0px 10px 0px;
- padding: 5px 0 5px 10px;
- text-shadow: 0px 1px 0 white
-}
-
-div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; }
-div.body h2 { font-size: 150%; background-color: #C8D5E3; }
-div.body h3 { font-size: 120%; background-color: #D8DEE3; }
-div.body h4 { font-size: 110%; background-color: #D8DEE3; }
-div.body h5 { font-size: 100%; background-color: #D8DEE3; }
-div.body h6 { font-size: 100%; background-color: #D8DEE3; }
-
-/* make p.rubric look like h2*/
-p.rubric {
- font-family: Times, serif;
- background-color: #BED4EB;
- font-weight: normal;
- color: #212224;
- margin: 30px 0px 10px 0px;
- padding: 5px 0 5px 10px;
- text-shadow: 0px 1px 0 white;
- font-size: 120%; background-color: #D8DEE3; }
-
-a.headerlink {
- color: #c60f0f;
- font-size: 0.8em;
- padding: 0 4px 0 4px;
- text-decoration: none;
-}
-
-a.headerlink:hover {
- background-color: #c60f0f;
- color: white;
-}
-
-div.body p, div.body dd, div.body li {
- line-height: 1.5em;
-}
-
-/*
-div.highlight{
- background-color: white;
-}
-*/
-
-pre {
- padding: 10px;
- background-color: White;
- color: #222;
- line-height: 1.2em;
- border: 1px solid #C6C9CB;
- font-size: 1.2em;
- margin: 1.5em 0 1.5em 0;
- -webkit-box-shadow: 1px 1px 1px #d8d8d8;
- -moz-box-shadow: 1px 1px 1px #d8d8d8;
-}
-
-
-tt {
- background-color: #ecf0f3;
- color: #222;
- padding: 1px 2px;
- font-size: 1.2em;
- font-family: monospace;
-}
-
-div.admonition, div.warning {
- font-size: 0.9em;
- margin: 1em 0.5em 1em 0;
- border: 1px solid #2a441b;
- background-color: #e6f6dc;
- padding: 0;
-}
-
-div.admonition p, div.warning p {
- margin: 0.5em 1em 0.5em 1em;
- padding: 0;
-}
-
-div.admonition pre, div.warning pre {
- margin: 0.4em 1em 0.4em 1em;
-}
-
-div.admonition p.admonition-title,
-div.warning p.admonition-title {
- margin: 0;
- padding: 0.1em 0 0.1em 0.5em;
- border-bottom: 1px solid #2a441b;
- font-weight: bold;
- background-color: #fdfcae;
-}
-
-div.warning {
- border: 1px solid #940000;
-}
-
-div.warning p.admonition-title {
- color: black;
- background-color: #CF0000;
- border-bottom-color: #940000;
-}
-
-div.admonition ul, div.admonition ol,
-div.warning ul, div.warning ol {
- margin: 0.1em 0.5em 0.5em 3em;
- padding: 0;
-}
-
-div.versioninfo {
- margin: 1em 0 0 0;
- border: 1px solid #ccc;
- background-color: #DDEAF0;
- padding: 8px;
- line-height: 1.3em;
- font-size: 0.9em;
-}
-
-
-/**
- * Styling for field lists
- */
-
-table.field-list th {
- border-left: 1px solid #aaa !important;
- padding-left: 5px;
-}
-
-table.field-list {
- border-collapse: separate;
- border-spacing: 10px;
-}
-
-
-th {
- background-color: #ede;
-}
-
-.field-list th {
-/* color: rgb(0,102,204); */
- color: #993333;
- white-space: nowrap;
-}
-
-.first {
-margin-top: 0 !important;
-}
-
-
-td.field-body > blockquote {
-margin-top: 0.1em;
-margin-bottom: 0.5em;
-}
diff --git a/doc/source/_theme/altered_nature/static/old_pygments.css b/doc/source/_theme/altered_nature/static/old_pygments.css
deleted file mode 100644
index 652b761..0000000
--- a/doc/source/_theme/altered_nature/static/old_pygments.css
+++ /dev/null
@@ -1,54 +0,0 @@
-.c { color: #999988; font-style: italic } /* Comment */
-.k { font-weight: bold } /* Keyword */
-.o { font-weight: bold } /* Operator */
-.cm { color: #999988; font-style: italic } /* Comment.Multiline */
-.cp { color: #999999; font-weight: bold } /* Comment.preproc */
-.c1 { color: #999988; font-style: italic } /* Comment.Single */
-.gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
-.ge { font-style: italic } /* Generic.Emph */
-.gr { color: #aa0000 } /* Generic.Error */
-.gh { color: #999999 } /* Generic.Heading */
-.gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
-.go { color: #111 } /* Generic.Output */
-.gp { color: #555555 } /* Generic.Prompt */
-.gs { font-weight: bold } /* Generic.Strong */
-.gu { color: #aaaaaa } /* Generic.Subheading */
-.gt { color: #aa0000 } /* Generic.Traceback */
-.kc { font-weight: bold } /* Keyword.Constant */
-.kd { font-weight: bold } /* Keyword.Declaration */
-.kp { font-weight: bold } /* Keyword.Pseudo */
-.kr { font-weight: bold } /* Keyword.Reserved */
-.kt { color: #445588; font-weight: bold } /* Keyword.Type */
-.m { color: #009999 } /* Literal.Number */
-.s { color: #bb8844 } /* Literal.String */
-.na { color: #008080 } /* Name.Attribute */
-.nb { color: #999999 } /* Name.Builtin */
-.nc { color: #445588; font-weight: bold } /* Name.Class */
-.no { color: #ff99ff } /* Name.Constant */
-.ni { color: #800080 } /* Name.Entity */
-.ne { color: #990000; font-weight: bold } /* Name.Exception */
-.nf { color: #990000; font-weight: bold } /* Name.Function */
-.nn { color: #555555 } /* Name.Namespace */
-.nt { color: #000080 } /* Name.Tag */
-.nv { color: purple } /* Name.Variable */
-.ow { font-weight: bold } /* Operator.Word */
-.mf { color: #009999 } /* Literal.Number.Float */
-.mh { color: #009999 } /* Literal.Number.Hex */
-.mi { color: #009999 } /* Literal.Number.Integer */
-.mo { color: #009999 } /* Literal.Number.Oct */
-.sb { color: #bb8844 } /* Literal.String.Backtick */
-.sc { color: #bb8844 } /* Literal.String.Char */
-.sd { color: #bb8844 } /* Literal.String.Doc */
-.s2 { color: #bb8844 } /* Literal.String.Double */
-.se { color: #bb8844 } /* Literal.String.Escape */
-.sh { color: #bb8844 } /* Literal.String.Heredoc */
-.si { color: #bb8844 } /* Literal.String.Interpol */
-.sx { color: #bb8844 } /* Literal.String.Other */
-.sr { color: #808000 } /* Literal.String.Regex */
-.s1 { color: #bb8844 } /* Literal.String.Single */
-.ss { color: #bb8844 } /* Literal.String.Symbol */
-.bp { color: #999999 } /* Name.Builtin.Pseudo */
-.vc { color: #ff99ff } /* Name.Variable.Class */
-.vg { color: #ff99ff } /* Name.Variable.Global */
-.vi { color: #ff99ff } /* Name.Variable.Instance */
-.il { color: #009999 } /* Literal.Number.Integer.Long */
\ No newline at end of file
diff --git a/doc/source/_theme/altered_nature/theme.conf b/doc/source/_theme/altered_nature/theme.conf
deleted file mode 100644
index 4a45eda..0000000
--- a/doc/source/_theme/altered_nature/theme.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-[theme]
-inherit = basic
-stylesheet = altered_nature.css
-pygments_style = tango
diff --git a/doc/source/_theme/cloud/layout.html b/doc/source/_theme/cloud/layout.html
deleted file mode 100644
index 3b08bfc..0000000
--- a/doc/source/_theme/cloud/layout.html
+++ /dev/null
@@ -1,98 +0,0 @@
-{#
- cloud/layout.html
- ~~~~~~~~~~~~~~~~~
-
- Sphinx layout template for the cloud theme.
-
- :copyright: Copyright 2009-2011 Assurance Technologies LLC
- :license: BSD
-
- roottarget - optional target for root link (default to toc)
- logotarget - optional target for logo link (defaults to root target)
- googleanalytics_id - if set, enabled google analytics snippet & footer
- googleanalytics_path - optional subpatch for GA cookie
- collapsiblesidebar - enable collapsible sidebar support
-
-#}
-
-{% extends "basic/layout.html" %}
-
-{% set reldelim2 = reldim2 is not defined and ' ' or reldelim2 %}
-
-{# add script file to instrument collapsable sections and other features #}
-{% set script_files = script_files + ['_static/jquery.cookie.js',
- '_static/toggle_sections.js'] %}
-{% if theme_collapsiblesidebar|tobool %}
-{% set script_files = script_files + ['_static/toggle_sidebar.js'] %}
-{% endif %}
-
-{# add font stylesheets #}
-{% set css_files = css_files + [theme_fontcssurl] %}
-
-{# make root link redirectable #}
-{%- set theme_roottarget = (theme_roottarget == "<toc>" and master_doc or theme_roottarget) %}
-{% block rootrellink %}
- <li><a href="{{ pathto(theme_roottarget) }}">{{shorttitle|e}}</a>{{reldelim1}}</li>
-{% endblock %}
-
-{# make logo link redirectable #}
-{%- set theme_logotarget = (theme_logotarget == "<root>" and theme_roottarget or (theme_logotarget == "<toc>" and master_doc or theme_logotarget)) %}
-{%- block sidebarlogo %}
- {%- if logo %}
- <p class="logo"><a href="{{ pathto(theme_logotarget) }}">
- <img class="logo" src="{{ pathto('_static/' + logo, 1) }}" alt="Logo"/>
- </a></p>
- {%- endif %}
-{%- endblock %}
-
-{# wrap relbars in distinct classes to make themeing easier #}
-{%- block relbar1 %}
- <div class="relbar-top">
- {{ super() }}
- </div>
-{% endblock %}
-
-{%- block relbar2 %}
- <div class="relbar-bottom">
- {{ super() }}
- </div>
-{% endblock %}
-
-{% block extrahead -%}
- {{ super() }}
- {%- if theme_googleanalytics_id -%}
-<script type="text/javascript">
-var ga_enabled = !$.cookie('disable-ga');
-if(ga_enabled){
- var _gaq = _gaq || [];
- _gaq.push(['_setAccount', '{{ theme_googleanalytics_id }}']);
- _gaq.push(['_setCookiePath', '{{ theme_googleanalytics_path }}']);
- _gaq.push(['_setDetectFlash', false]);
- _gaq.push(['_trackPageview']);
- (function() {
- var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
- ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
- var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
- })();
-}
-</script>
- {%- endif %}
-{%- endblock %}
-
-{% block footer -%}
- {{ super() }}
-
- {%- if theme_googleanalytics_id -%}
-<script type="text/javascript">
- if(ga_enabled){
- document.write("<div class=\"footer\">This page uses <a href=\"http://analytics.google.com\">Google Analytics</a> to collect statistics. ");
- document.write("Click <button title=\"set cookie to disable analytics for this site\" class=\"link\" onclick=\"$.cookie('disable-ga', 'true', {expires: 3650, path: '{{ theme_googleanalytics_path }}'}); window.location.reload(); return false; \">here</button> to disable analytics for this site.");
- document.write("</div>");
- }else{
- document.write("<div class=\"footer\">Google Analytics has been disabled. ");
- document.write("Click <button title=\"set cookie to re-enable analytics for this site\" class=\"link\" onclick=\"$.cookie('disable-ga', null, {path: '{{ theme_googleanalytics_path }}'}); window.location.reload(); return false; \">here</button> to re-enable analytics for this site.");
- };
-</script>
- {%- endif %}
- <!-- cloud_sptheme 1.3 -->
-{%- endblock %}
diff --git a/doc/source/_theme/cloud/static/cloud.css_t b/doc/source/_theme/cloud/static/cloud.css_t
deleted file mode 100644
index 5e08332..0000000
--- a/doc/source/_theme/cloud/static/cloud.css_t
+++ /dev/null
@@ -1,969 +0,0 @@
-/*****************************************************
- * cloud.css_t
- * ~~~~~~~~~~~
- *
- * stylesheet for "Cloud" theme for Sphinx.
- *
- * :copyright: Copyright 2011 by Assurance Technologies
- * :license: BSD
- *
- *****************************************************/
-
- at import url("basic.css");
-
-/*****************************************************
- * page layout
- *****************************************************/
-
-body
-{
- font-family: {{ theme_bodyfont }};
- font-size: 100%;
- color: #000;
- margin: 1em 1em 0 1em;
- padding: 0;
- #background-color: {{ theme_footerbgcolor }};
- #background:{{ theme_footerbgcolor }} url('sun.png') no-repeat right top fixed;
- background-color: {{ theme_footerbgcolor }};
-/*
- background-image:url('sun.png'), url('cld.png'), url('grass.png'), url('mount.png'), url('pyne_icon_small.png'), url('cld1.png'), url('cld1.png'), url('cld.png'), url('cld1.png');
- background-repeat: no-repeat, no-repeat, repeat-x, no-repeat, no-repeat, no-repeat, no-repeat, no-repeat, no-repeat;
- background-position: top right, 2% 40%, left bottom, 100.5% bottom, -1% bottom, 9% 12%, 4% 80%, 93% 450px, 99% 25%;
- background-attachment: fixed, fixed, scroll, scroll, scroll, fixed, fixed, scroll, fixed;
-*/
-}
-
-/*****************************************************
- * page layout - relbars (top & bottom)
- *****************************************************/
-
-div.related
-{
- margin: 0 auto;
- max-width: {{ theme_max_width }};
- background-color: {{ theme_relbarbgcolor }};
- line-height: 30px;
- color: {{ theme_relbartextcolor }};
-}
-
-div.relbar-top div.related
-{
- border-radius: .7em .7em 0 0;
- -moz-border-radius: .7em .7em 0 0;
- -webkit-border-radius: .7em .7em 0 0;
-}
-
-div.relbar-bottom div.related
-{
- border-radius: 0 0 .7em .7em;
- -moz-border-radius: 0 0 .7em .7em;
- -webkit-border-radius: 0 0 .7em .7em;
-}
-
-div.related a
-{
- color: {{ theme_relbarlinkcolor }};
-}
-
-/*****************************************************
- * page layout - document
- *****************************************************/
-
-div.document
-{
- /* note: relative used by div.sidebartoggle */
- position: relative;
- margin: 0 auto;
- max-width: {{ theme_max_width }};
- background-color: {{ theme_sidebarbgcolor }};
-}
-
-div.documentwrapper
-{
- float: left;
- width: 100%;
-}
-
-div.bodywrapper
-{
- margin: 0 0 0 {{ theme_sidebarwidth }};
-}
-
-{% if theme_collapsiblesidebar|tobool %}
-div.document.collapsed-sidebar div.bodywrapper { margin-left: 0; }
-{% endif %}
-
-div.body
-{
- min-height: {{ theme_min_height }};
- /* note: this ^ is just a hack to prevent body from being shorter than sidebar */
-
- background-color: {{ theme_bgcolor }};
- border-left: 1px solid {{theme_bodytrimcolor}};
-
- color: {{ theme_textcolor }};
- padding: 0 20px 30px 20px;
-}
-
-/*****************************************************
- * page layout - sidebar
- *****************************************************/
-
-div.sphinxsidebar
-{
- /* it's nasty sometimes, but looks prettier than letting too-long module names break into document body*/
- overflow: hidden;
-
- width: {{ theme_sidebarwidth }};
-}
-
-{% if theme_collapsiblesidebar|tobool %}
-div.document.collapsed-sidebar div.sphinxsidebar { display: none; }
-{% endif %}
-
-div.sphinxsidebar h3,
-div.sphinxsidebar h4
-{
- padding: 0;
- margin: 24px 16px 0 0;
-
- border-bottom: 1px dashed {{ theme_sidebartrimcolor }};
-
- font-family: {{ theme_headfont }};
- font-weight: normal;
- color: {{ theme_sidebartextcolor }};
-}
-
-div.sphinxsidebar h3
-{
- font-size: 1.4em;
-}
-
-div.sphinxsidebar h4
-{
- font-size: 1.3em;
-}
-
-div.sphinxsidebar h3 a
-{
- color: {{ theme_sidebartextcolor }};
-}
-
-div.sphinxsidebar p
-{
- color: {{ theme_sidebartextcolor }};
-}
-
-div.sphinxsidebar p.topless
-{
- margin: 5px 10px 10px 10px;
-}
-
-div.sphinxsidebar ul
-{
- margin: 10px;
- padding: 0;
- color: {{ theme_sidebartextcolor }};
-}
-
-div.sphinxsidebar a
-{
- color: {{ theme_sidebarlinkcolor }};
-}
-
-div.sphinxsidebar input
-{
- border: 1px solid {{ theme_sidebartrimcolor }};
- font-family: sans-serif;
- font-size: 1em;
-}
-
-div#searchbox input[type="submit"]
-{
- border-radius: 0 0 .7em 0;
- -webkit-border-radius: 0 0 .7em 0;
- -moz-border-radius: 0 0 .7em 0;
-}
-
-div.sphinxsidebar input[type="submit"]:hover,
-div.sidebartoggle button:hover
-{
- background: {{ theme_sidebarhighcolor }};
- box-shadow: 0 1px 2px rgba(0,0,0,.2);
- -webkit-box-shadow: 0 1px 2px rgba(0,0,0,.2);
- -moz-box-shadow: 0 1px 2px rgba(0,0,0,.2);
-}
-
-div.sphinxsidebar input[type="submit"]:active,
-div.sidebartoggle button:active
-{
- box-shadow: none;
- -webkit-box-shadow: none;
- -moz-box-shadow: none;
-}
-
-
-div.sphinxsidebar p.logo
-{
- margin: 16px 0 0 0;
- text-align: center;
-}
-
-div.sphinxsidebar .searchtip
-{
- color: {{ theme_sidebartrimcolor }};
-}
-
-/*--------------------------------------------------
- * collapsible sidebar buttons
- *--------------------------------------------------*/
-{% if theme_collapsiblesidebar|tobool %}
-
-div.sidebartoggle
-{
- position: absolute;
- width: {{ theme_sidebarwidth }};
- top: 0;
- left: 0;
-}
-
-div.document.collapsed-sidebar div.sidebartoggle
-{
- width: auto;
-}
-
-div.sidebartoggle button
-{
- position: absolute;
- right: -1px;
- top: 0;
-
- margin: 0;
- padding: 0 6px;
-
- background: {{ theme_sidebarbgcolor }};
- border: 1px solid {{ theme_sidebartrimcolor }};
- border-top: 0;
- color: {{ theme_sidebartextcolor }};
-
- border-radius: 0 0 0 .7em;
- -webkit-border-radius: 0 0 0 .7em;
- -moz-border-radius: 0 0 0 .7em;
-
- white-space: nowrap;
-}
-
-div.document.collapsed-sidebar div.sidebartoggle button
-{
- right: auto;
- left: 0;
- border-radius: 0 0 .7em 0;
- -webkit-border-radius: 0 0 .7em 0;
- -moz-border-radius: 0 0 .7em 0;
-}
-
-{% endif %}
-/*****************************************************
- * page layout - footer
- *****************************************************/
-
-div.footer
-{
- color: {{ theme_footertextcolor }};
- width: 100%;
- padding: 9px 0;
- text-align: center;
- font-size: 75%;
-}
-
-div.footer button.link
-{
- margin: 0 -1px;
- padding: 0;
- background: none;
- border: none;
- font-size: inherit;
- font-family: inherit;
-}
-
-div.footer a,
-div.footer button.link
-{
- color: {{ theme_footertextcolor }};
- text-decoration: underline;
-}
-
-div.footer a:hover,
-div.footer button.link:hover
-{
- color: white;
-}
-
-div.footer + div.footer,
-div.footer + script + div.footer
-{
- margin-top: -12px;
-}
-
-/*****************************************************
- * adaptive page layout - if too small, hide some things
- *****************************************************/
-
- at media only screen and (max-width: {{ theme_compact_width }}),
- only screen and (max-width: {{ theme_minimal_width }}),
- only screen and (max-device-width: {{ theme_minimal_width }}),
- handheld
-{
- body {
- margin: 0;
- }
-
- div.relbar-top div.related,
- div.relbar-bottom div.related
- {
- border-radius: 0;
- -moz-border-radius: 0;
- -webkit-border-radius: 0;
- }
-}
-
- at media only screen and (max-width: {{ theme_minimal_width }}),
- only screen and (max-device-width: {{ theme_minimal_width }}),
- handheld
-{
- /* hide the sidebar */
- div.sphinxsidebar
- {
- display: none;
- }
-
- {% if theme_collapsiblesidebar|tobool %}
- /* FIXME: would like to make this work some how, hiding it til it works */
- div.sidebartoggle { display: none; }
- {% endif %}
-
- div.bodywrapper
- {
- margin-left: 0;
- }
-
- /* hide all lefthand links but first one, to save space*/
- div.related > ul > li:not(.right) + li
- {
- display: none;
- }
-
- /* remove spacing on headers and reduce body padding */
- body div.body h2, body div.body p.rubric
- {
- border-radius: 0;
- -moz-border-radius: 0;
- -webkit-border-radius: 0;
- }
-
- div.body
- {
- padding: 0 10px 30px;
- }
-}
-
-/*****************************************************
- * adapt for print media
- *****************************************************/
- at media print
-{
- div.body { border: 0; }
-}
-
-/*****************************************************
- * document - link styles
- *****************************************************/
-
-a
-{
- color: {{ theme_linkcolor }};
- text-decoration: none;
-}
-
-a:visited
-{
- color: {{ theme_visitedlinkcolor }};
- text-decoration: none;
-}
-
-a:hover
-{
- text-decoration: underline;
-}
-
-a.biglink
-{
- font-size: 130%;
-}
-
-{% if theme_externalrefs|tobool %}
-a.external:not(.issue):before
-{
- content: {{ theme_externalicon }};
- padding-right: .1em;
-}
-a.issue:before
-{
- content: {{ theme_issueicon }};
- padding-right: .2em;
-}
-{% endif %}
-
-dt:target,
-.footnote:target,
-.highlighted
-{
- padding: 4px;
- margin: -4px;
-
- border-radius: 4px;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
-}
-
-/*****************************************************
- * document - header styles
- *****************************************************/
-
-div.body h1,
-div.body h2,
-div.body h3,
-div.body h4,
-div.body h5,
-div.body h6,
-div.body p.rubric
-{
- margin: 1em 0 -.25em;
- padding: 0;
-
- color: {{ theme_headtextcolor }};
-
- font-family: {{ theme_headfont }};
- font-weight: normal;
- text-shadow: 1px 1px 0 rgba(0,0,0,.1);
-}
-
-div.body h1 {
- margin: 0;
- padding-top: .25em;
- text-align: center;
- font-size: 200%;
-}
-
-div.body h2,
-div.body p.rubric
-{
- margin: 1em -10px 0 -10px;
- padding: 7px;
-
- background: {{ theme_sectionbgcolor }};
- border: 1px solid {{ theme_sectiontrimcolor }};
- border-width: 0 1px 1px 0;
-
- border-radius: 10px;
- -moz-border-radius: 10px;
- -webkit-border-radius: 10px;
-
- font-size: 120%;
- color: {{ theme_sectiontextcolor }};
- text-shadow: 1px 1px 0 rgba(0,0,0,.1);
-}
-
-div.body p.rubric,
-div.body div.section.dimmed h2
-{
- background: {{ theme_rubricbgcolor }};
-}
-
-div.body h3 { font-size: 140%; }
-div.body h4 { font-size: 120%; }
-div.body h5 { font-size: 110%; }
-div.body h6 { font-size: 100%; }
-
-a.headerlink
-{
- color: {{ theme_headlinkcolor }};
- font-size: 0.8em;
- margin: 0 0 0 1px;
- padding: 0 5px 0 4px;
- text-decoration: none;
- border-radius: 5px;
-}
-
-a.headerlink:hover
-{
- background-color: {{ theme_headlinkcolor }};
- color: white;
-
- box-shadow: 2px 2px 2px rgba(0,0,0,.2);
- -webkit-box-shadow: 2px 2px 2px rgba(0,0,0,.2);
- -moz-box-shadow: 2px 2px 2px rgba(0,0,0,.2);
-}
-
-/*****************************************************
- * document - toggleable h1/h2 sections
- *****************************************************/
-
-.html-toggle-button
-{
- position: relative;
-}
-
-.html-toggle-button:hover
-{
- /* give a little bit of hint that this is clickable */
- cursor: pointer;
- box-shadow: 2px 2px 2px {{ theme_headlinkcolor }};
- -moz-box-shadow: 2px 2px 2px {{ theme_headlinkcolor }};
- -webkit-box-shadow: 2px 2px 2px {{ theme_headlinkcolor }};
-}
-
-.html-toggle.collapsed > .html-toggle-button
-{
- margin-bottom: 1em;
-}
-
-.html-toggle.collapsed > .html-toggle-button:after
-{
- content: "[+ show section]";
- position: absolute;
- right: 1em;
- margin: 0;
- padding: .15em 0 0 0;
- text-shadow: none;
- color: {{ theme_linkcolor }};
- font-size: 65%;
- font-style: italic;
-}
-
-.html-toggle.expanded > .html-toggle-button:after
-{
- content: "[- hide section]";
- position: absolute;
- right: 1em;
- margin: 0;
- padding: .15em 0 0 0;
- text-shadow: none;
- color: {{ theme_linkcolor }};
- font-size: 65%;
- font-style: italic;
-}
-
-/*****************************************************
- * document - admonitions
- *****************************************************/
-
-div.admonition
-{
- background: #f2f2f2;
- border: 1px solid rgba(0,0,0,.05);
- border-width: 0 1px 1px 0;
- margin: 1em 0;
- padding: .5em;
- border-radius: 0 10px 0 10px;
- -webkit-border-radius: 0 10px 0 10px;
- -moz-border-radius: 0 10px 0 10px;
-}
-
-div.note, div.seealso, div.warning, div.admonition-todo
-{
- background-position: 8px center;
- background-repeat: no-repeat;
- padding-left: 38px;
-}
-
-div.note { background-color: #E7F0FE; background-image: url(icon-note.png); }
-
-div.seealso { background-color: #FFF7E0; background-image: url(icon-seealso.png); }
-div.topic { background-color: #eee; }
-div.warning { background-color: #ffe4e4; background-image: url(icon-warning.png); }
-div.admonition-todo { background-color: #FFF7E0; background-image: url(icon-todo.png); }
-
-div.admonition.caution { background-color: green; }
-/*div.admonition.caution { background-color: #FFF0E4; }*/
-
-div.admonition p,
-div.admonition pre,
-div.admonition ul,
-div.admonition ol
-{
- margin-bottom: 5px;
-}
-
-/* next 3 rules merge the title into the first paragraph */
-p.admonition-title
-{
- display: inline;
- margin-right: 0;
-}
-
-p.admonition-title + p
-{
- display: inline;
-}
-
-p.admonition-title:after
-{
- content: ":";
-}
-
-p.admonition-title + ul
-{
- margin-top: 0;
-}
-
-/* don't indent admonitions inside definitions */
-dl div.admonition
-{
- margin-left: 0;
-}
-
-/*****************************************************
- * document - misc body styles
- *****************************************************/
-
-div.body p,
-div.body dd,
-div.body li
-{
- text-align: justify;
- line-height: 140%;
-}
-
-/*dl.function,
-dl.method,
-dl.attribute,
-dl.class,
-dl.exception,
-dl.data */
-
-dl
-{
- margin-bottom: 1.5em;
-}
-
-
-a.footnote-reference
-{
- font-size: 70%;
- position: relative;
- top: -.75em;
-}
-
-table.docutils.footnote
-{
- margin: 1em 0 0 1em;
-}
-
- /* join separate ULs together */
-ul + ul,
-ul + div > ul:only-child,
-div.toctree-wrapper + ul
-{
- margin-top: -1em;
-}
-
-table.html-plain-table
-{
- border: 0;
-}
-
-table.docutils.html-plain-table td
-{
- border: 0;
-}
-
-h2 + table.docutils
-{
- margin-top: 1em;
-}
-
-/* styling for OLs */
-dd ol
-{
- margin-bottom: 10px;
-}
-
-div.frontpage-images.container
-{
- text-align: center;
- margin-top: 2.5em;
- margin-bottom: 2.5em;
-}
-
-
-/*****************************************************
- * document - quoted text
- *****************************************************/
-
-div.highlight .c
-{
- font-size: 90%;
-}
-
-pre
-{
- padding: 5px;
-
- background-color: {{ theme_codebgcolor }};
- border: 1px solid #80858a;
- border-width: 1px 1px 1px 4px;
-
- color: {{ theme_codetextcolor }};
- line-height: 120%;
-}
-
-tt
-{
- color: {{ theme_codetextcolor }};
- font-size: 0.95em;
-}
-
-tt.literal
-{
- background-color: {{ theme_quotebgcolor }};
- border: 1px solid {{ theme_quotetrimcolor }};
- padding: 1px 2px;
- margin: 0 1px;
-
- color: {{ theme_codetextcolor }};
- border-radius: 4px;
- -moz-border-radius: 4px;
- -webkit-border-radius: 4px;
-}
-
-tt.samp.literal > em
-{
- font-family: {{ theme_codevarfont }};
- font-size: 1em;
- padding: 0 1px;
-}
-
-h1 tt.literal
-{
- background-color: inherit;
- border: 0;
- font-weight: bold;
-}
-
-h1 tt.samp.literal > em
-{
- font-weight: normal;
-}
-
-div.sphinxsidebar tt.literal,
-a.reference tt.literal,
-div.related a tt.literal,
-tt.literal.xref
-{
- background-color: transparent;
- border: none;
- padding: 0;
- margin: 0;
-}
-
-th
-{
- background-color: #ede;
-}
-
-.viewcode-back
-{
- font-family: {{ theme_bodyfont }};
-}
-
-div.viewcode-block:target
-{
- background-color: #f4debf;
- border-top: 1px solid #ac9;
- border-bottom: 1px solid #ac9;
-}
-
-/*****************************************************
- * nested sections -
- * (requires css classes set by astdoc.ext.nested_sections extension)
- *****************************************************/
-
-dl.nested-section > dt
-{
- color: {{ theme_headtextcolor }};
-
- font-family: {{ theme_headfont }};
- font-weight: normal;
- font-size: 100%;
-}
-
-dl.nested-section-1 > dt { font-size: 140%; }
-dl.nested-section-2 > dt { font-size: 120%; }
-dl.nested-section-3 > dt { font-size: 110%; }
-
-dl.nested-section > dd
-{
- margin-left: 2em;
- margin-top: 1em;
-}
-
-/*****************************************************
- * index page - category colorization
- * (requires css classes set by astdoc.ext.index_styles extension)
- *****************************************************/
-
-table.indextable span.category
-{
- font-size: 80%;
- color: #84ADBE;
-}
-
-table.indextable span.category.function,
-table.indextable span.category.classmethod,
-table.indextable span.category.method
-{ color: #9AB9CE; }
-
-table.indextable span.category.attribute { color: #8BC38B; }
-table.indextable span.category.class { color: #9996c2; }
-
-/*table.indextable span.category.module { color: #8BC38B; }*/
-
-table.indextable span.subject
-{
- font-weight: bold;
-}
-
-table.indextable td > dl > dt
-{
- margin-bottom: .5em;
-}
-
-table.indextable td > dl > dd > dl
-{
- margin-top: -.5em;
- margin-bottom: .5em;
-}
-
-/*#################################################################
- unorganized bits
- #################################################################*/
-
-
-/*****************************************************
- * provide styling for TODO
- *****************************************************/
-
-div#todos p.admonition-title
-{
- font-weight: normal;
- color: #AAA;
- font-size: 70%;
-}
-
-div#todos div.admonition-todo + p
-{
- font-size: 70%;
- text-align: right;
- margin-top: -.5em;
- margin-bottom: 1.5em;
- color: #AAA;
-}
-
-div#todos div.admonition-todo + p a
-{
- font-size: 130%;
-}
-
-/*****************************************************
- * add more whitespace to parameter lists
- *****************************************************/
-td.field-body > ul.first.simple > li,
-td.field-body > p.first
-{
- margin-bottom: 1em;
-}
-
-td.field-body > p.first:only-child
-{
- margin: 0; /* work around Chrome bug */
-}
-
-
-td.field-body > ul.first.simple > li > em,
-td.field-body > em
-{
- padding: 2px 4px;
-/* border-bottom: 1px solid #DEE6ED;
- background: #ECF0F3;
- */
-}
-
-/*****************************************************
- * misc tweaks to parameter lists
- *****************************************************/
-
-th {
- background: inherit;
-}
-
-
-table.field-list {
- border-collapse: separate;
- border-spacing: 10px;
-}
-
-table.field-list th {
- !border-left: 1px solid {{ theme_sidebartrimcolor }} !important;
- border: 1px solid {{ theme_sidebartrimcolor }} !important;
- padding-left: 5px;
-}
-
-th.field-name {
- background: {{ theme_sidebarbgcolor }};
- border-top: 0;
- color: {{ theme_linkcolor }};
- white-space: nowrap;
-}
-
-td.field-body > p.first:empty
-{
- display: none;
-}
-
-td.field-body > p:last-child:empty
-{
- margin-bottom: 1em;
-}
-
-/*****************************************************
- * css colorization of object definitions,
- * adds colored line underneath definition title.
- * color scheme used:
- * functions, methods - blue
- * classes - purple
- * exceptions - red
- * objects, attributes, data - green
- *****************************************************/
-/*
-dl.classmethod > dt,
-dl.function > dt,
-dl.method > dt
-{
- background: #E2ECF3;
-}
-
-dl.exception > dt { background: #F2E3E1; }
-dl.class > dt { background: #E7E6F6; }
-
-dl.attribute > dt,
-dl.object > dt,
-dl.data > dt
-{
- background: #E2F2E2;
-}*/
-
-
-/*****************************************************
- * EOF
- *****************************************************/
diff --git a/doc/source/_theme/cloud/static/icon-note.png b/doc/source/_theme/cloud/static/icon-note.png
deleted file mode 100644
index 07cf010..0000000
Binary files a/doc/source/_theme/cloud/static/icon-note.png and /dev/null differ
diff --git a/doc/source/_theme/cloud/static/icon-seealso.png b/doc/source/_theme/cloud/static/icon-seealso.png
deleted file mode 100644
index 5a0d618..0000000
Binary files a/doc/source/_theme/cloud/static/icon-seealso.png and /dev/null differ
diff --git a/doc/source/_theme/cloud/static/icon-todo.png b/doc/source/_theme/cloud/static/icon-todo.png
deleted file mode 100644
index 8ef08e2..0000000
Binary files a/doc/source/_theme/cloud/static/icon-todo.png and /dev/null differ
diff --git a/doc/source/_theme/cloud/static/icon-warning.png b/doc/source/_theme/cloud/static/icon-warning.png
deleted file mode 100644
index 0ce38d9..0000000
Binary files a/doc/source/_theme/cloud/static/icon-warning.png and /dev/null differ
diff --git a/doc/source/_theme/cloud/static/jquery.cookie.js b/doc/source/_theme/cloud/static/jquery.cookie.js
deleted file mode 100644
index 6a3e394..0000000
--- a/doc/source/_theme/cloud/static/jquery.cookie.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * jQuery Cookie plugin
- *
- * Copyright (c) 2010 Klaus Hartl (stilbuero.de)
- * Dual licensed under the MIT and GPL licenses:
- * http://www.opensource.org/licenses/mit-license.php
- * http://www.gnu.org/licenses/gpl.html
- *
- */
-jQuery.cookie = function (key, value, options) {
-
- // key and at least value given, set cookie...
- if (arguments.length > 1 && String(value) !== "[object Object]") {
- options = jQuery.extend({}, options);
-
- if (value === null || value === undefined) {
- options.expires = -1;
- }
-
- if (typeof options.expires === 'number') {
- var days = options.expires, t = options.expires = new Date();
- t.setDate(t.getDate() + days);
- }
-
- value = String(value);
-
- return (document.cookie = [
- encodeURIComponent(key), '=',
- options.raw ? value : encodeURIComponent(value),
- options.expires ? '; expires=' + options.expires.toUTCString() : '', // use expires attribute, max-age is not supported by IE
- options.path ? '; path=' + options.path : '',
- options.domain ? '; domain=' + options.domain : '',
- options.secure ? '; secure' : ''
- ].join(''));
- }
-
- // key and possibly options given, get cookie...
- options = value || {};
- var result, decode = options.raw ? function (s) { return s; } : decodeURIComponent;
- return (result = new RegExp('(?:^|; )' + encodeURIComponent(key) + '=([^;]*)').exec(document.cookie)) ? decode(result[1]) : null;
-};
diff --git a/doc/source/_theme/cloud/static/toggle_sections.js b/doc/source/_theme/cloud/static/toggle_sections.js
deleted file mode 100644
index 33f0d3b..0000000
--- a/doc/source/_theme/cloud/static/toggle_sections.js
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * toggle_sections.js
- * ~~~~~~~~~~~~~~
- *
- * Sphinx JavaScript helper for collapsible sections.
- * looks for sections with css class "html-toggle",
- * with optional additional classes "expanded" or "collapsed"
- * (defaults to "collapsed")
- *
- * :copyright: Copyright 2011 by Assurance Technologies
- * :license: BSD
- *
- * NOTE: while this provides full javascript instrumentation,
- * css styling should be applied to .html-toggle > .html-toggle-button
- */
-
-$(document).ready(function (){
- function init(){
- var jobj = $(this);
- var parent = jobj.parent()
-
- /* add class for styling purposes */
- jobj.addClass("html-toggle-button");
-
- /* initialize state */
- _setState(jobj, parent.hasClass("expanded") || _containsHash(parent));
-
- /* bind toggle callback */
- jobj.click(function (){
- _setState(jobj, !parent.hasClass("expanded"));
- });
-
- /* check for hash changes - older browsers may not have this evt */
- $(window).bind("hashchange", function () {
- var hash = document.location.hash;
- if(!hash)
- return;
- if(_containsHash(parent))
- _setState(jobj, true);
- var pos = $(hash).position();
- window.scrollTo(pos.left, pos.top);
- });
- }
-
- function _containsHash(parent){
- var hash = document.location.hash;
- if(!hash)
- return false;
- return parent[0].id == hash.substr(1) || parent.find(hash).length>0;
- }
-
- function _setState(jobj, expanded){
- var parent = jobj.parent();
- if(expanded){
- parent.addClass("expanded").removeClass("collapsed");
- parent.children().show();
- }else{
- parent.addClass("collapsed").removeClass("expanded");
- parent.children().hide();
- parent.children("span:first-child:empty").show(); /* for :ref: span tag */
- jobj.show();
- }
- }
-
- $(".html-toggle.section > h2, .html-toggle.section > h3, .html-toggle.section > h4, .html-toggle.section > h5, .html-toggle.section > h6").each(init);
-
-});
diff --git a/doc/source/_theme/cloud/static/toggle_sidebar.js_t b/doc/source/_theme/cloud/static/toggle_sidebar.js_t
deleted file mode 100644
index 0ec0149..0000000
--- a/doc/source/_theme/cloud/static/toggle_sidebar.js_t
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * toggle_sidebar.js
- * ~~~~~~~~~~~~~~
- *
- * Sphinx JavaScript helper for collapsible sidebar.
- * adds button into sidebar, and toggles document.collapsed-sidebar
- * all the actual collapsing is done via css.
- *
- * :copyright: Copyright 2011 by Assurance Technologies
- * :license: BSD
- *
- */
-
-$(document).ready(function (){
- var holder = $('<div class="sidebartoggle"><button id="sidebar-hide" title="click to hide the sidebar">«</button><button id="sidebar-show" style="display: none" title="click to show the sidebar">sidebar »</button></div>');
- var doc = $('div.document');
-
- var show_btn = $('#sidebar-show', holder);
- var hide_btn = $('#sidebar-hide', holder);
- var copts = { expires: 7, path: DOCUMENTATION_OPTIONS.url_root };
-
- show_btn.click(function (){
- doc.removeClass("collapsed-sidebar");
- hide_btn.show();
- show_btn.hide();
- $.cookie("sidebar", "expanded", copts);
- });
-
- hide_btn.click(function (){
- doc.addClass("collapsed-sidebar");
- show_btn.show();
- hide_btn.hide();
- $.cookie("sidebar", "collapsed", copts);
- });
-
- var state = $.cookie("sidebar");
- if(!state && {{ theme_defaultcollapsed | tobool | lower }}){
- state = "collapsed";
- }
-
- doc.append(holder);
-
- if (state == "collapsed"){
- doc.addClass("collapsed-sidebar");
- show_btn.show();
- hide_btn.hide();
- }
-
-});
diff --git a/doc/source/_theme/cloud/theme.conf b/doc/source/_theme/cloud/theme.conf
deleted file mode 100644
index 1fce86a..0000000
--- a/doc/source/_theme/cloud/theme.conf
+++ /dev/null
@@ -1,74 +0,0 @@
-[theme]
-inherit = basic
-stylesheet = cloud.css
-pygments_style = sphinx
-
-[options]
-#configuration options
-collapsiblesidebar = false
-defaultcollapsed = false
-externalrefs = true
-externalicon = "\21D7"
-issueicon = "\25CE"
-
-#link targets
-#NOTE: roottarget can be name of document (eg: 'index', or "<toc>")
-# logotarget is same, or can be "<root>" to reflect root value
-roottarget = <toc>
-logotarget = <root>
-
-#document dimensions
-max_width = 11in ; max width document will expand to.
-compact_width = 960px ; max width of "compact" mode (hides extra padding).
-minimal_width = 700px ; max width of "minimal" mode (hides sidebar and other extras).
-
-min_height = 6in ; min height of document
-
-#styling for document body
-bodyfont = Georgia, "Bitstream Vera Serif", "New York", Palatino, serif
-bodytrimcolor = #D0D0D0
-bgcolor = #ffffff
-textcolor = #000000
-linkcolor = #003469
-quotebgcolor = rgba(0,0,0,.075)
-quotetrimcolor = rgba(0,0,0,.05)
-codebgcolor = #eeffcc
-codetextcolor = #333333
-codevarfont = Georgia, "Bitstream Vera Serif", "New York", Palatino, serif
-
-#styling for document headers
-headfont = Georgia, "Bitstream Vera Serif", "New York", Palatino, serif
-headtextcolor = #000000
-headlinkcolor = #003469
-
-#styling for section headers
-sectionbgcolor = #84A6C7
-sectiontrimcolor = rgba(0,0,0,.125)
-sectiontextcolor = #ffffff
-rubricbgcolor = #B2CCE0
-
-#styling for footer / html background
-footerbgcolor = #1A4162
-footertextcolor = #B0B0B0
-
-#styling for sidebar
-sidebarwidth = 230px
-sidebarbgcolor = #F2F2F2
-sidebartextcolor= #777777
-sidebarlinkcolor= #003469
-sidebartrimcolor= #C0C0C0
-sidebarhighcolor= #F9F9F9
-
-#styling for top & bottom relbars
-relbarbgcolor = #5682AD
-relbartextcolor = #ffffff
-relbarlinkcolor = #ffffff
-relbartrimcolor = #777777
-
-#font css url - for loading in fonts (eg google font dir)
-#fontcssurl = http://fonts.googleapis.com/css?family=Crimson+Text
-fontcssurl =
-
-# set google analytics tracker
-googleanalytics_id =
-googleanalytics_path = /
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 2c88394..013ab5d 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -12,19 +12,17 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
-import sys
-import os
-
-# If extensions (or modules to document with autodoc) are in another directory,
+# If extensions (or modules to document with autodoc) are in another
+# directory,
# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('../sphinxext'))
-#sys.path.insert(0, os.path.abspath('../..'))
+# documentation root, use os.path.abspath to make it absolute, like shown
+# here.
+#sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+needs_sphinx = '1.1'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -33,12 +31,12 @@ extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.autosummary',
'sphinx.ext.doctest',
- 'sphinx.ext.pngmath',
+ 'sphinx.ext.mathjax',
'sphinx.ext.inheritance_diagram',
'sphinx.ext.extlinks',
'sphinx.ext.todo',
'sphinx.ext.viewcode',
- 'ipython_console_highlighting',
+ 'IPython.sphinxext.ipython_console_highlighting',
'numpydoc',
]
@@ -55,8 +53,8 @@ source_encoding = 'utf-8'
master_doc = 'index'
# General information about the project.
-project = u'PyTables'
-copyright = u'2011-2014, PyTables maintainers'
+project = 'PyTables'
+copyright = '2011-2017, PyTables maintainers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -65,6 +63,7 @@ copyright = u'2011-2014, PyTables maintainers'
# The short X.Y version.
VERSION = open('../../VERSION').read().strip()
version = VERSION
+
# The full version, including alpha/beta/rc tags.
release = VERSION
@@ -98,138 +97,32 @@ exclude_patterns = []
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
-#pygments_style = 'sphinx'
-#pygments_style = 'friendly'
-#pygments_style = 'bw'
-#pygments_style = 'fruity'
-#pygments_style = 'manni'
-pygments_style = 'tango'
-
-
-# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+pygments_style = 'sphinx'
# -- Options for HTML output ----------------------------------------------
+
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
-#html_theme = 'default'
-#html_theme = 'altered_nature'
-html_theme = 'cloud'
-#html_theme = 'sphinxdoc'
+html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
-#html_theme_options = {}
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-linkcolor = '#9F1E1E'
-trimcolor = '#511755'
-trimcolor = '#000000'
-
-html_theme_options = {'sidebarbgcolor': 'rgba(213, 197, 229, 0.15)',
- 'sidebartextcolor': '#280941',
- 'sidebarlinkcolor': linkcolor,
- 'sidebartrimcolor': trimcolor,
- 'collapsiblesidebar': True,
- 'relbarbgcolor': '#006FFF',
- 'footerbgcolor': 'rgba(252, 255, 0, 0.125)',
- 'footertextcolor': '#504A4B',
- 'bodytrimcolor': trimcolor,
- 'linkcolor': linkcolor,
- 'textcolor': '#323039',
- 'sectionbgcolor': '#3CAD1C',
- #'sectiontextcolor': '#777777',
- #'sectiontrimcolor': trimcolor,
- 'codebgcolor': '#F1FFF0',
- 'codetextcolor': '#000000',
- 'quotebgcolor': '#f6fcfc',
- 'rubricbgcolor': '#D00000',
- #'min_height': 'bottom',
- }
+html_theme_options = {
+ # 'sticky_navigation': True # Set to False to disable the sticky nav while scrolling.
+ 'logo_only': True, # if we have a html_logo below, this shows /only/ the logo with no title text
+}
# Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = ["_theme"]
-
-# The name for this set of Sphinx documents. If None, it defaults to
-# "<project> v<release> documentation".
-#html_title = None
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
+import sphinx_rtd_theme
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '_static/logo-pytables-small.png'
-# The name of an image file (within the static path) to use as favicon of the
-# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-html_favicon = 'images/favicon.ico'
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#html_extra_path = []
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-html_sidebars = {
- 'index': ['globaltoc.html', 'relations.html', 'sourcelink.html',
- 'searchbox.html', 'travis-ci.html']
-}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
-# If false, no module index is generated.
-#html_domain_indices = True
-
-# If false, no index is generated.
-#html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it. The value of this option must be the
-# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'pytablesdoc'
-
# -- Options for LaTeX output ---------------------------------------------
@@ -290,10 +183,10 @@ autosummary_generate = []
# -- Options for Epub output ----------------------------------------------
# Bibliographic Dublin Core info.
-epub_title = u'PyTables'
-epub_author = u'PyTables maintainers'
-epub_publisher = u'PyTables maintainers'
-epub_copyright = u'2011-2014, PyTables maintainers'
+epub_title = 'PyTables'
+epub_author = 'PyTables maintainers'
+epub_publisher = 'PyTables maintainers'
+epub_copyright = copyright
# The basename for the epub file. It defaults to the project name.
#epub_basename = u'PyTables'
diff --git a/doc/source/cookbook/py2exe_howto/pytables_test.py b/doc/source/cookbook/py2exe_howto/pytables_test.py
index f7e2c20..5ea37c6 100644
--- a/doc/source/cookbook/py2exe_howto/pytables_test.py
+++ b/doc/source/cookbook/py2exe_howto/pytables_test.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
from tables import *
import numarray
@@ -15,7 +16,7 @@ h5file = openFile("tutorial.h5", mode="w", title="Test file")
group = h5file.createGroup("/", "detector", "Detector information")
table = h5file.createTable(group, "readout", Particle, "Readout example")
-print h5file
+print(h5file)
particle = table.row
@@ -36,6 +37,6 @@ table = h5file.root.detector.readout
pressure = [x['pressure'] for x in table.iterrows() if x['TDCcount']>3 and
20<=x['pressure']<50]
-print pressure
+print(pressure)
h5file.close()
diff --git a/doc/source/cookbook/threading.rst b/doc/source/cookbook/threading.rst
index aca40eb..ca55903 100644
--- a/doc/source/cookbook/threading.rst
+++ b/doc/source/cookbook/threading.rst
@@ -181,7 +181,7 @@ the **drawback** that the user code have to be modified to replace
:func:`open_file` and :meth:`File.close` calls with their safe version
(:func:`synchronized_open_file` and :func:`synchronized_close_file`).
-Also, the solution showed in the example does not cover the entire PyTables
+Also, the solution shown in the example does not cover the entire PyTables
API (e.g. although not recommended HDF5 files can be opened using the
:class:`File` constructor) and makes it impossible to use *pythonic*
constructs like the *with* statement::
@@ -248,7 +248,7 @@ themselves::
_tables_file._open_files = ThreadsafeFileRegistry()
-At this point PyTables can be used transparently in example program presented
+At this point PyTables can be used transparently in the example program presented
in the previous section.
In particular the standard PyTables API (including *with* statements) can be
used in the *thread function*::
diff --git a/doc/source/dev_team.rst b/doc/source/dev_team.rst
index 3310baf..3a209cb 100644
--- a/doc/source/dev_team.rst
+++ b/doc/source/dev_team.rst
@@ -14,9 +14,10 @@ The PyTables team includes:
* `Josh Moore`_
* `Anthony Scopatz`_
* `Andrea Bedini`_
+* `Tom Kooij`_
.. _Anthony Scopatz: http://www.scopatz.com/
.. _Antonio Valentino: https://github.com/avalentino
.. _Josh Moore: https://github.com/joshmoore
.. _Andrea Bedini: https://github.com/andreabedini
-
+.. _Tom Kooij: https://github.com/tomkooij
diff --git a/doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst b/doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst
index f152d2e..4401990 100644
--- a/doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst
+++ b/doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst
@@ -1 +1,33 @@
-.. include:: ../../../RELEASE_NOTES.txt
+=======================================
+ Release notes for PyTables 3.3 series
+=======================================
+
+Changes from 3.2.3.1 to 3.3.0
+=============================
+
+Improvements
+------------
+
+- Single codebase Python 2 and 3 support (PR #493).
+- Internal Blosc version updated to 1.11.1 (closes :issue:`541`)
+- Full BitShuffle support for new Blosc versions (>= 1.8).
+- It is now possible to remove all rows from a table.
+- It is now possible to read reference types by dereferencing them as
+ numpy array of objects (closes :issue:`518` and :issue:`519`).
+ Thanks to Ehsan Azar
+- Get rid of the `-native` compile flag (closes :issue:`503`)
+- The default number of threads to run numexpr (MAX_NUMEXPR_THREADS)
+ internally has been raised from 2 to 4. This is because we are in
+ 2016 and 4 core configurations are becoming common.
+- In order to avoid locking issues when using PyTables concurrently in
+ several process, MAX_BLOSC_THREADS has been set to 1 by default. If
+ you are running PyTables in one single process, you may want to
+ experiment if higher values (like 2 or 4) bring better performance for
+ you.
+
+Bugs fixed
+----------
+
+- On python 3 try 'latin1' encoding before 'bytes' encoding during unpickling
+ of node attributes pickled on python 2. Better fix for :issue:`560`.
+- Fixed Windows 32 and 64-bit builds.
diff --git a/doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst b/doc/source/release-notes/RELEASE_NOTES_v3.4.x.rst
similarity index 100%
copy from doc/source/release-notes/RELEASE_NOTES_v3.3.x.rst
copy to doc/source/release-notes/RELEASE_NOTES_v3.4.x.rst
diff --git a/doc/source/release_notes.rst b/doc/source/release_notes.rst
index 2e5e57a..354a4ea 100644
--- a/doc/source/release_notes.rst
+++ b/doc/source/release_notes.rst
@@ -17,9 +17,9 @@ PyTables
.. toctree::
:maxdepth: 1
+ release-notes/RELEASE_NOTES_v3.4.x
release-notes/RELEASE_NOTES_v3.3.x
release-notes/RELEASE_NOTES_v3.2.x
- release-notes/RELEASE_NOTES_v3.2.x
release-notes/RELEASE_NOTES_v3.1.x
release-notes/RELEASE_NOTES_v3.0.x
release-notes/RELEASE_NOTES_v2.4.x
@@ -61,7 +61,10 @@ Release timeline
----------------
=============== =========== ==========
-PyTables 3.3 2016-09-12
+PyTables 3.4.2 2017-04-19
+PyTables 3.4.1 2017-04-12
+PyTables 3.4.0 2017-04-11
+PyTables 3.3.0 2016-09-12
PyTables 3.2.3.1 2016-07-05
PyTables 3.2.3 2016-07-04
PyTables 3.2.2 2015-09-22
diff --git a/doc/source/usersguide/index.rst b/doc/source/usersguide/index.rst
index 6ab2d3f..19f82e4 100644
--- a/doc/source/usersguide/index.rst
+++ b/doc/source/usersguide/index.rst
@@ -13,7 +13,7 @@ Hierarchical datasets in Python
|copy| 2008, 2009, 2010 - Francesc Alted
- |copy| 2011-2015 - PyTables maintainers
+ |copy| 2011-2017 - PyTables maintainers
--------
Contents
diff --git a/doc/source/usersguide/installation.rst b/doc/source/usersguide/installation.rst
index 692e3bc..a4cb7fa 100644
--- a/doc/source/usersguide/installation.rst
+++ b/doc/source/usersguide/installation.rst
@@ -49,7 +49,7 @@ Prerequisites
First, make sure that you have
* Python_ >= 2.7 including Python 3.x
-* HDF5_ >= 1.8.4 (>=1.8.15 is strongly recommended, HDF5 v1.10 not supported)
+* HDF5_ >= 1.8.4 (>=1.8.15 is strongly recommended)
* NumPy_ >= 1.8.1
* Numexpr_ >= 2.5.2
* Cython_ >= 0.21
@@ -162,6 +162,39 @@ worry too much ;)
--lzo=c:\\Program Files (x86)\\GnuWin32
--bzip2=c:\\Program Files (x86)\\GnuWin32
+**Conda**
+
+ Pre-built packages for PyTables are available in the anaconda (default)
+ channel::
+
+ conda install pytables
+
+ The most recent version is usually available in the conda-forge
+ channel::
+
+ conda config --add channels conda-forge
+ conda install pytables
+
+ The HDF5 libraries and other helper packages are automatically found in
+ a conda environment. During installation setup.py uses the `CONDA_PREFIX`
+ environment variable to detect a conda enviroment. If detected it will
+ try to find all packages within this enviroment. PyTables needs at least
+ the hdf5 package:
+
+ conda install hdf5
+ python setup.py install
+
+ It is still possible to override package locations using the
+ :envvar:`HDF5_DIR`, :envvar:`LZO_DIR`, :envvar:`BZIP2_DIR` or
+ :envvar:`BLOSC_DIR` environment variables.
+
+ When inside a conda environment *pkg-config* will not work. To disable
+ using the conda enviroment and fall back to *pkg-config* use `--no-conda`::
+
+ python setup.py install --no-conda
+
+ When the `--use-pkgconfig` flag is used, `--no-conda` is assumed.
+
**Development version (Unix)**
Installation of the development version is very similar to installation
diff --git a/doc/source/usersguide/libref/homogenous_storage.rst b/doc/source/usersguide/libref/homogenous_storage.rst
index 18709f1..04d926c 100644
--- a/doc/source/usersguide/libref/homogenous_storage.rst
+++ b/doc/source/usersguide/libref/homogenous_storage.rst
@@ -32,7 +32,7 @@ Array methods
.. automethod:: Array.iterrows
-.. automethod:: Array.next
+.. automethod:: Array.__next__
.. automethod:: Array.read
@@ -105,7 +105,7 @@ VLArray methods
.. automethod:: VLArray.iterrows
-.. automethod:: VLArray.next
+.. automethod:: VLArray.__next__
.. automethod:: VLArray.read
diff --git a/doc/source/usersguide/usersguide.rst b/doc/source/usersguide/usersguide.rst
index e7ca46f..1bd1151 100644
--- a/doc/source/usersguide/usersguide.rst
+++ b/doc/source/usersguide/usersguide.rst
@@ -19,7 +19,7 @@ PyTables User's Guide
|copy| 2008, 2009, 2010 - Francesc Alted
- |copy| 2011-2015 - PyTables maintainers
+ |copy| 2011-2017 - PyTables maintainers
:Date: |today|
:Version: |version|
:Home Page: http://www.pytables.org
@@ -114,4 +114,3 @@ Appendixes
:maxdepth: 1
bibliography
-
diff --git a/doc/sphinxext/LICENSE.txt b/doc/sphinxext/LICENSE.txt
deleted file mode 100644
index c569c87..0000000
--- a/doc/sphinxext/LICENSE.txt
+++ /dev/null
@@ -1,141 +0,0 @@
--------------------------------------------------------------------------------
- The files
- - ipython_console_highlighting.py
- - ipython_directive.py
- have the following license:
-
- The IPython licensing terms
-
-IPython is licensed under the terms of the Modified BSD License (also known as
-New or Revised BSD), as follows:
-
-Copyright (c) 2008-2010, IPython Development Team
-Copyright (c) 2001-2007, Fernando Perez. <fernando.perez at colorado.edu>
-Copyright (c) 2001, Janko Hauser <jhauser at zscout.de>
-Copyright (c) 2001, Nathaniel Gray <n8gray at caltech.edu>
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice, this
-list of conditions and the following disclaimer in the documentation and/or
-other materials provided with the distribution.
-
-Neither the name of the IPython Development Team nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
--------------------------------------------------------------------------------
- The files
- - numpydoc.py
- - autosummary.py
- - autosummary_generate.py
- - docscrape.py
- - docscrape_sphinx.py
- - phantom_import.py
- have the following license:
-
-Copyright (C) 2008 Stefan van der Walt <stefan at mentat.za.net>, Pauli Virtanen <pav at iki.fi>
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- 1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in
- the documentation and/or other materials provided with the
- distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
--------------------------------------------------------------------------------
- The files
- - compiler_unparse.py
- - comment_eater.py
- - traitsdoc.py
- have the following license:
-
-This software is OSI Certified Open Source Software.
-OSI Certified is a certification mark of the Open Source Initiative.
-
-Copyright (c) 2006, Enthought, Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of Enthought, Inc. nor the names of its contributors may
- be used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
--------------------------------------------------------------------------------
- The files
- - only_directives.py
- - plot_directive.py
- originate from Matplotlib (http://matplotlib.sf.net/) which has
- the following license:
-
-Copyright (c) 2002-2008 John D. Hunter; All Rights Reserved.
-
-1. This LICENSE AGREEMENT is between John D. Hunter (“JDH”), and the Individual or Organization (“Licensee”) accessing and otherwise using matplotlib software in source or binary form and its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, JDH hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use matplotlib 0.98.3 alone or in any derivative version, provided, however, that JDH’s License Agreement and JDH’s notice of copyright, i.e., “Copyright (c) 2002-2008 John D. Hunter; All Rights Reserved” are retained in matplotlib 0.98 [...]
-
-3. In the event Licensee prepares a derivative work that is based on or incorporates matplotlib 0.98.3 or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to matplotlib 0.98.3.
-
-4. JDH is making matplotlib 0.98.3 available to Licensee on an “AS IS” basis. JDH MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, JDH MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB 0.98.3 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. JDH SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB 0.98.3 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING MATPLOTLIB 0.98.3, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between JDH and Licensee. This License Agreement does not grant permission to use JDH trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using matplotlib 0.98.3, Licensee agrees to be bound by the terms and conditions of this License Agreement.
-
diff --git a/doc/sphinxext/__init__.py b/doc/sphinxext/__init__.py
deleted file mode 100644
index ae9073b..0000000
--- a/doc/sphinxext/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from numpydoc import setup
diff --git a/doc/sphinxext/apigen.py b/doc/sphinxext/apigen.py
deleted file mode 100644
index 9b14151..0000000
--- a/doc/sphinxext/apigen.py
+++ /dev/null
@@ -1,427 +0,0 @@
-"""Attempt to generate templates for module reference with Sphinx
-
-XXX - we exclude extension modules
-
-To include extension modules, first identify them as valid in the
-``_uri2path`` method, then handle them in the ``_parse_module`` script.
-
-We get functions and classes by parsing the text of .py files.
-Alternatively we could import the modules for discovery, and we'd have
-to do that for extension modules. This would involve changing the
-``_parse_module`` method to work via import and introspection, and
-might involve changing ``discover_modules`` (which determines which
-files are modules, and therefore which module URIs will be passed to
-``_parse_module``).
-
-NOTE: this is a modified version of a script originally shipped with the
-PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
-project."""
-
-# Stdlib imports
-import os
-import re
-
-# Functions and classes
-class ApiDocWriter(object):
- ''' Class for automatic detection and parsing of API docs
- to Sphinx-parsable reST format'''
-
- # only separating first two levels
- rst_section_levels = ['*', '=', '-', '~', '^']
-
- def __init__(self,
- package_name,
- rst_extension='.rst',
- package_skip_patterns=None,
- module_skip_patterns=None,
- ):
- ''' Initialize package for parsing
-
- Parameters
- ----------
- package_name : string
- Name of the top-level package. *package_name* must be the
- name of an importable package
- rst_extension : string, optional
- Extension for reST files, default '.rst'
- package_skip_patterns : None or sequence of {strings, regexps}
- Sequence of strings giving URIs of packages to be excluded
- Operates on the package path, starting at (including) the
- first dot in the package path, after *package_name* - so,
- if *package_name* is ``sphinx``, then ``sphinx.util`` will
- result in ``.util`` being passed for earching by these
- regexps. If is None, gives default. Default is:
- ['\.tests$']
- module_skip_patterns : None or sequence
- Sequence of strings giving URIs of modules to be excluded
- Operates on the module name including preceding URI path,
- back to the first dot after *package_name*. For example
- ``sphinx.util.console`` results in the string to search of
- ``.util.console``
- If is None, gives default. Default is:
- ['\.setup$', '\._']
- '''
- if package_skip_patterns is None:
- package_skip_patterns = ['\\.tests$']
- if module_skip_patterns is None:
- module_skip_patterns = ['\\.setup$', '\\._']
- self.package_name = package_name
- self.rst_extension = rst_extension
- self.package_skip_patterns = package_skip_patterns
- self.module_skip_patterns = module_skip_patterns
-
- def get_package_name(self):
- return self._package_name
-
- def set_package_name(self, package_name):
- ''' Set package_name
-
- >>> docwriter = ApiDocWriter('sphinx')
- >>> import sphinx
- >>> docwriter.root_path == sphinx.__path__[0]
- True
- >>> docwriter.package_name = 'docutils'
- >>> import docutils
- >>> docwriter.root_path == docutils.__path__[0]
- True
- '''
- # It's also possible to imagine caching the module parsing here
- self._package_name = package_name
- self.root_module = __import__(package_name)
- self.root_path = self.root_module.__path__[0]
- self.written_modules = None
-
- package_name = property(get_package_name, set_package_name, None,
- 'get/set package_name')
-
- def _get_object_name(self, line):
- ''' Get second token in line
- >>> docwriter = ApiDocWriter('sphinx')
- >>> docwriter._get_object_name(" def func(): ")
- 'func'
- >>> docwriter._get_object_name(" class Klass(object): ")
- 'Klass'
- >>> docwriter._get_object_name(" class Klass: ")
- 'Klass'
- '''
- name = line.split()[1].split('(')[0].strip()
- # in case we have classes which are not derived from object
- # ie. old style classes
- return name.rstrip(':')
-
- def _uri2path(self, uri):
- ''' Convert uri to absolute filepath
-
- Parameters
- ----------
- uri : string
- URI of python module to return path for
-
- Returns
- -------
- path : None or string
- Returns None if there is no valid path for this URI
- Otherwise returns absolute file system path for URI
-
- Examples
- --------
- >>> docwriter = ApiDocWriter('sphinx')
- >>> import sphinx
- >>> modpath = sphinx.__path__[0]
- >>> res = docwriter._uri2path('sphinx.builder')
- >>> res == os.path.join(modpath, 'builder.py')
- True
- >>> res = docwriter._uri2path('sphinx')
- >>> res == os.path.join(modpath, '__init__.py')
- True
- >>> docwriter._uri2path('sphinx.does_not_exist')
-
- '''
- if uri == self.package_name:
- return os.path.join(self.root_path, '__init__.py')
- path = uri.replace('.', os.path.sep)
- path = path.replace(self.package_name + os.path.sep, '')
- path = os.path.join(self.root_path, path)
- # XXX maybe check for extensions as well?
- if os.path.exists(path + '.py'): # file
- path += '.py'
- elif os.path.exists(os.path.join(path, '__init__.py')):
- path = os.path.join(path, '__init__.py')
- else:
- return None
- return path
-
- def _path2uri(self, dirpath):
- ''' Convert directory path to uri '''
- relpath = dirpath.replace(self.root_path, self.package_name)
- if relpath.startswith(os.path.sep):
- relpath = relpath[1:]
- return relpath.replace(os.path.sep, '.')
-
- def _parse_module(self, uri):
- ''' Parse module defined in *uri* '''
- filename = self._uri2path(uri)
- if filename is None:
- # nothing that we could handle here.
- return ([], [])
- f = open(filename, 'rt')
- functions, classes = self._parse_lines(f)
- f.close()
- return functions, classes
-
- def _parse_lines(self, linesource):
- ''' Parse lines of text for functions and classes '''
- functions = []
- classes = []
- for line in linesource:
- if line.startswith('def ') and line.count('('):
- # exclude private stuff
- name = self._get_object_name(line)
- if not name.startswith('_'):
- functions.append(name)
- elif line.startswith('class '):
- # exclude private stuff
- name = self._get_object_name(line)
- if not name.startswith('_'):
- classes.append(name)
- else:
- pass
- functions.sort()
- classes.sort()
- return functions, classes
-
- def generate_api_doc(self, uri):
- '''Make autodoc documentation template string for a module
-
- Parameters
- ----------
- uri : string
- python location of module - e.g 'sphinx.builder'
-
- Returns
- -------
- S : string
- Contents of API doc
- '''
- # get the names of all classes and functions
- functions, classes = self._parse_module(uri)
- if not len(functions) and not len(classes):
- print 'WARNING: Empty -', uri # dbg
- return ''
-
- # Make a shorter version of the uri that omits the package name for
- # titles
- uri_short = re.sub(r'^%s\.' % self.package_name, '', uri)
-
- ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
-
- chap_title = uri_short
- ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title)
- + '\n\n')
-
- # Set the chapter title to read 'module' for all modules except for the
- # main packages
- if '.' in uri:
- title = 'Module: :mod:`' + uri_short + '`'
- else:
- title = ':mod:`' + uri_short + '`'
- ad += title + '\n' + self.rst_section_levels[2] * len(title)
-
- if len(classes):
- ad += '\nInheritance diagram for ``%s``:\n\n' % uri
- ad += '.. inheritance-diagram:: %s \n' % uri
- ad += ' :parts: 3\n'
-
- ad += '\n.. automodule:: ' + uri + '\n'
- ad += '\n.. currentmodule:: ' + uri + '\n'
- multi_class = len(classes) > 1
- multi_fx = len(functions) > 1
- if multi_class:
- ad += '\n' + 'Classes' + '\n' + \
- self.rst_section_levels[2] * 7 + '\n'
- elif len(classes) and multi_fx:
- ad += '\n' + 'Class' + '\n' + \
- self.rst_section_levels[2] * 5 + '\n'
- for c in classes:
- ad += '\n:class:`' + c + '`\n' \
- + self.rst_section_levels[multi_class + 2 ] * \
- (len(c)+9) + '\n\n'
- ad += '\n.. autoclass:: ' + c + '\n'
- # must NOT exclude from index to keep cross-refs working
- ad += ' :members:\n' \
- ' :undoc-members:\n' \
- ' :show-inheritance:\n' \
- ' :inherited-members:\n' \
- '\n' \
- ' .. automethod:: __init__\n'
- if multi_fx:
- ad += '\n' + 'Functions' + '\n' + \
- self.rst_section_levels[2] * 9 + '\n\n'
- elif len(functions) and multi_class:
- ad += '\n' + 'Function' + '\n' + \
- self.rst_section_levels[2] * 8 + '\n\n'
- for f in functions:
- # must NOT exclude from index to keep cross-refs working
- ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
- return ad
-
- def _survives_exclude(self, matchstr, match_type):
- ''' Returns True if *matchstr* does not match patterns
-
- ``self.package_name`` removed from front of string if present
-
- Examples
- --------
- >>> dw = ApiDocWriter('sphinx')
- >>> dw._survives_exclude('sphinx.okpkg', 'package')
- True
- >>> dw.package_skip_patterns.append('^\\.badpkg$')
- >>> dw._survives_exclude('sphinx.badpkg', 'package')
- False
- >>> dw._survives_exclude('sphinx.badpkg', 'module')
- True
- >>> dw._survives_exclude('sphinx.badmod', 'module')
- True
- >>> dw.module_skip_patterns.append('^\\.badmod$')
- >>> dw._survives_exclude('sphinx.badmod', 'module')
- False
- '''
- if match_type == 'module':
- patterns = self.module_skip_patterns
- elif match_type == 'package':
- patterns = self.package_skip_patterns
- else:
- raise ValueError('Cannot interpret match type "%s"'
- % match_type)
- # Match to URI without package name
- L = len(self.package_name)
- if matchstr[:L] == self.package_name:
- matchstr = matchstr[L:]
- for pat in patterns:
- try:
- pat.search
- except AttributeError:
- pat = re.compile(pat)
- if pat.search(matchstr):
- return False
- return True
-
- def discover_modules(self):
- ''' Return module sequence discovered from ``self.package_name``
-
-
- Parameters
- ----------
- None
-
- Returns
- -------
- mods : sequence
- Sequence of module names within ``self.package_name``
-
- Examples
- --------
- >>> dw = ApiDocWriter('sphinx')
- >>> mods = dw.discover_modules()
- >>> 'sphinx.util' in mods
- True
- >>> dw.package_skip_patterns.append('\.util$')
- >>> 'sphinx.util' in dw.discover_modules()
- False
- >>>
- '''
- modules = [self.package_name]
- # raw directory parsing
- for dirpath, dirnames, filenames in os.walk(self.root_path):
- # Check directory names for packages
- root_uri = self._path2uri(os.path.join(self.root_path,
- dirpath))
- for dirname in dirnames[:]: # copy list - we modify inplace
- package_uri = '.'.join((root_uri, dirname))
- if (self._uri2path(package_uri) and
- self._survives_exclude(package_uri, 'package')):
- modules.append(package_uri)
- else:
- dirnames.remove(dirname)
- # Check filenames for modules
- for filename in filenames:
- module_name = filename[:-3]
- module_uri = '.'.join((root_uri, module_name))
- if (self._uri2path(module_uri) and
- self._survives_exclude(module_uri, 'module')):
- modules.append(module_uri)
- return sorted(modules)
-
- def write_modules_api(self, modules, outdir):
- # write the list
- written_modules = []
- for m in modules:
- api_str = self.generate_api_doc(m)
- if not api_str:
- continue
- # write out to file
- outfile = os.path.join(outdir,
- m + self.rst_extension)
- fileobj = open(outfile, 'wt')
- fileobj.write(api_str)
- fileobj.close()
- written_modules.append(m)
- self.written_modules = written_modules
-
- def write_api_docs(self, outdir):
- """Generate API reST files.
-
- Parameters
- ----------
- outdir : string
- Directory name in which to store files
- We create automatic filenames for each module
-
- Returns
- -------
- None
-
- Notes
- -----
- Sets self.written_modules to list of written modules
- """
- if not os.path.exists(outdir):
- os.mkdir(outdir)
- # compose list of modules
- modules = self.discover_modules()
- self.write_modules_api(modules, outdir)
-
- def write_index(self, outdir, froot='gen', relative_to=None):
- """Make a reST API index file from written files
-
- Parameters
- ----------
- path : string
- Filename to write index to
- outdir : string
- Directory to which to write generated index file
- froot : string, optional
- root (filename without extension) of filename to write to
- Defaults to 'gen'. We add ``self.rst_extension``.
- relative_to : string
- path to which written filenames are relative. This
- component of the written file path will be removed from
- outdir, in the generated index. Default is None, meaning,
- leave path as it is.
- """
- if self.written_modules is None:
- raise ValueError('No modules written')
- # Get full filename path
- path = os.path.join(outdir, froot+self.rst_extension)
- # Path written into index is relative to rootpath
- if relative_to is not None:
- relpath = outdir.replace(relative_to + os.path.sep, '')
- else:
- relpath = outdir
- idx = open(path, 'wt')
- w = idx.write
- w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
- w('.. toctree::\n\n')
- for f in self.written_modules:
- w(' %s\n' % os.path.join(relpath, f))
- idx.close()
diff --git a/doc/sphinxext/comment_eater.py b/doc/sphinxext/comment_eater.py
deleted file mode 100644
index e11eea9..0000000
--- a/doc/sphinxext/comment_eater.py
+++ /dev/null
@@ -1,158 +0,0 @@
-from cStringIO import StringIO
-import compiler
-import inspect
-import textwrap
-import tokenize
-
-from compiler_unparse import unparse
-
-
-class Comment(object):
- """ A comment block.
- """
- is_comment = True
- def __init__(self, start_lineno, end_lineno, text):
- # int : The first line number in the block. 1-indexed.
- self.start_lineno = start_lineno
- # int : The last line number. Inclusive!
- self.end_lineno = end_lineno
- # str : The text block including '#' character but not any leading spaces.
- self.text = text
-
- def add(self, string, start, end, line):
- """ Add a new comment line.
- """
- self.start_lineno = min(self.start_lineno, start[0])
- self.end_lineno = max(self.end_lineno, end[0])
- self.text += string
-
- def __repr__(self):
- return '%s(%r, %r, %r)' % (self.__class__.__name__, self.start_lineno,
- self.end_lineno, self.text)
-
-
-class NonComment(object):
- """ A non-comment block of code.
- """
- is_comment = False
- def __init__(self, start_lineno, end_lineno):
- self.start_lineno = start_lineno
- self.end_lineno = end_lineno
-
- def add(self, string, start, end, line):
- """ Add lines to the block.
- """
- if string.strip():
- # Only add if not entirely whitespace.
- self.start_lineno = min(self.start_lineno, start[0])
- self.end_lineno = max(self.end_lineno, end[0])
-
- def __repr__(self):
- return '%s(%r, %r)' % (self.__class__.__name__, self.start_lineno,
- self.end_lineno)
-
-
-class CommentBlocker(object):
- """ Pull out contiguous comment blocks.
- """
- def __init__(self):
- # Start with a dummy.
- self.current_block = NonComment(0, 0)
-
- # All of the blocks seen so far.
- self.blocks = []
-
- # The index mapping lines of code to their associated comment blocks.
- self.index = {}
-
- def process_file(self, file):
- """ Process a file object.
- """
- for token in tokenize.generate_tokens(file.next):
- self.process_token(*token)
- self.make_index()
-
- def process_token(self, kind, string, start, end, line):
- """ Process a single token.
- """
- if self.current_block.is_comment:
- if kind == tokenize.COMMENT:
- self.current_block.add(string, start, end, line)
- else:
- self.new_noncomment(start[0], end[0])
- else:
- if kind == tokenize.COMMENT:
- self.new_comment(string, start, end, line)
- else:
- self.current_block.add(string, start, end, line)
-
- def new_noncomment(self, start_lineno, end_lineno):
- """ We are transitioning from a noncomment to a comment.
- """
- block = NonComment(start_lineno, end_lineno)
- self.blocks.append(block)
- self.current_block = block
-
- def new_comment(self, string, start, end, line):
- """ Possibly add a new comment.
-
- Only adds a new comment if this comment is the only thing on the line.
- Otherwise, it extends the noncomment block.
- """
- prefix = line[:start[1]]
- if prefix.strip():
- # Oops! Trailing comment, not a comment block.
- self.current_block.add(string, start, end, line)
- else:
- # A comment block.
- block = Comment(start[0], end[0], string)
- self.blocks.append(block)
- self.current_block = block
-
- def make_index(self):
- """ Make the index mapping lines of actual code to their associated
- prefix comments.
- """
- for prev, block in zip(self.blocks[:-1], self.blocks[1:]):
- if not block.is_comment:
- self.index[block.start_lineno] = prev
-
- def search_for_comment(self, lineno, default=None):
- """ Find the comment block just before the given line number.
-
- Returns None (or the specified default) if there is no such block.
- """
- if not self.index:
- self.make_index()
- block = self.index.get(lineno, None)
- text = getattr(block, 'text', default)
- return text
-
-
-def strip_comment_marker(text):
- """ Strip # markers at the front of a block of comment text.
- """
- lines = []
- for line in text.splitlines():
- lines.append(line.lstrip('#'))
- text = textwrap.dedent('\n'.join(lines))
- return text
-
-
-def get_class_traits(klass):
- """ Yield all of the documentation for trait definitions on a class object.
- """
- # FIXME: gracefully handle errors here or in the caller?
- source = inspect.getsource(klass)
- cb = CommentBlocker()
- cb.process_file(StringIO(source))
- mod_ast = compiler.parse(source)
- class_ast = mod_ast.node.nodes[0]
- for node in class_ast.code.nodes:
- # FIXME: handle other kinds of assignments?
- if isinstance(node, compiler.ast.Assign):
- name = node.nodes[0].name
- rhs = unparse(node.expr).strip()
- doc = strip_comment_marker(cb.search_for_comment(node.lineno, default=''))
- yield name, rhs, doc
-
diff --git a/doc/sphinxext/compiler_unparse.py b/doc/sphinxext/compiler_unparse.py
deleted file mode 100644
index bf34f5e..0000000
--- a/doc/sphinxext/compiler_unparse.py
+++ /dev/null
@@ -1,860 +0,0 @@
-""" Turn compiler.ast structures back into executable python code.
-
- The unparse method takes a compiler.ast tree and transforms it back into
- valid python code. It is incomplete and currently only works for
- import statements, function calls, function definitions, assignments, and
- basic expressions.
-
- Inspired by python-2.5-svn/Demo/parser/unparse.py
-
- fixme: We may want to move to using _ast trees because the compiler for
- them is about 6 times faster than compiler.compile.
-"""
-
-import sys
-import cStringIO
-from compiler.ast import Const, Name, Tuple, Div, Mul, Sub, Add
-
-def unparse(ast, single_line_functions=False):
- s = cStringIO.StringIO()
- UnparseCompilerAst(ast, s, single_line_functions)
- return s.getvalue().lstrip()
-
-op_precedence = { 'compiler.ast.Power':3, 'compiler.ast.Mul':2, 'compiler.ast.Div':2,
- 'compiler.ast.Add':1, 'compiler.ast.Sub':1 }
-
-class UnparseCompilerAst:
- """ Methods in this class recursively traverse an AST and
- output source code for the abstract syntax; original formatting
- is disregarged.
- """
-
- #########################################################################
- # object interface.
- #########################################################################
-
- def __init__(self, tree, file = sys.stdout, single_line_functions=False):
- """ Unparser(tree, file=sys.stdout) -> None.
-
- Print the source for tree to file.
- """
- self.f = file
- self._single_func = single_line_functions
- self._do_indent = True
- self._indent = 0
- self._dispatch(tree)
- self._write("\n")
- self.f.flush()
-
- #########################################################################
- # Unparser private interface.
- #########################################################################
-
- ### format, output, and dispatch methods ################################
-
- def _fill(self, text = ""):
- "Indent a piece of text, according to the current indentation level"
- if self._do_indent:
- self._write("\n"+" "*self._indent + text)
- else:
- self._write(text)
-
- def _write(self, text):
- "Append a piece of text to the current line."
- self.f.write(text)
-
- def _enter(self):
- "Print ':', and increase the indentation."
- self._write(": ")
- self._indent += 1
-
- def _leave(self):
- "Decrease the indentation level."
- self._indent -= 1
-
- def _dispatch(self, tree):
- "_dispatcher function, _dispatching tree type T to method _T."
- if isinstance(tree, list):
- for t in tree:
- self._dispatch(t)
- return
- meth = getattr(self, "_"+tree.__class__.__name__)
- if tree.__class__.__name__ == 'NoneType' and not self._do_indent:
- return
- meth(tree)
-
-
- #########################################################################
- # compiler.ast unparsing methods.
- #
- # There should be one method per concrete grammar type. They are
- # organized in alphabetical order.
- #########################################################################
-
- def _Add(self, t):
- self.__binary_op(t, '+')
-
- def _And(self, t):
- self._write(" (")
- for i, node in enumerate(t.nodes):
- self._dispatch(node)
- if i != len(t.nodes)-1:
- self._write(") and (")
- self._write(")")
-
- def _AssAttr(self, t):
- """ Handle assigning an attribute of an object
- """
- self._dispatch(t.expr)
- self._write('.'+t.attrname)
-
- def _Assign(self, t):
- """ Expression Assignment such as "a = 1".
-
- This only handles assignment in expressions. Keyword assignment
- is handled separately.
- """
- self._fill()
- for target in t.nodes:
- self._dispatch(target)
- self._write(" = ")
- self._dispatch(t.expr)
- if not self._do_indent:
- self._write('; ')
-
- def _AssName(self, t):
- """ Name on left hand side of expression.
-
- Treat just like a name on the right side of an expression.
- """
- self._Name(t)
-
- def _AssTuple(self, t):
- """ Tuple on left hand side of an expression.
- """
-
- # _write each elements, separated by a comma.
- for element in t.nodes[:-1]:
- self._dispatch(element)
- self._write(", ")
-
- # Handle the last one without writing comma
- last_element = t.nodes[-1]
- self._dispatch(last_element)
-
- def _AugAssign(self, t):
- """ +=,-=,*=,/=,**=, etc. operations
- """
-
- self._fill()
- self._dispatch(t.node)
- self._write(' '+t.op+' ')
- self._dispatch(t.expr)
- if not self._do_indent:
- self._write(';')
-
- def _Bitand(self, t):
- """ Bit and operation.
- """
-
- for i, node in enumerate(t.nodes):
- self._write("(")
- self._dispatch(node)
- self._write(")")
- if i != len(t.nodes)-1:
- self._write(" & ")
-
- def _Bitor(self, t):
- """ Bit or operation
- """
-
- for i, node in enumerate(t.nodes):
- self._write("(")
- self._dispatch(node)
- self._write(")")
- if i != len(t.nodes)-1:
- self._write(" | ")
-
- def _CallFunc(self, t):
- """ Function call.
- """
- self._dispatch(t.node)
- self._write("(")
- comma = False
- for e in t.args:
- if comma: self._write(", ")
- else: comma = True
- self._dispatch(e)
- if t.star_args:
- if comma: self._write(", ")
- else: comma = True
- self._write("*")
- self._dispatch(t.star_args)
- if t.dstar_args:
- if comma: self._write(", ")
- else: comma = True
- self._write("**")
- self._dispatch(t.dstar_args)
- self._write(")")
-
- def _Compare(self, t):
- self._dispatch(t.expr)
- for op, expr in t.ops:
- self._write(" " + op + " ")
- self._dispatch(expr)
-
- def _Const(self, t):
- """ A constant value such as an integer value, 3, or a string, "hello".
- """
- self._dispatch(t.value)
-
- def _Decorators(self, t):
- """ Handle function decorators (eg. @has_units)
- """
- for node in t.nodes:
- self._dispatch(node)
-
- def _Dict(self, t):
- self._write("{")
- for i, (k, v) in enumerate(t.items):
- self._dispatch(k)
- self._write(": ")
- self._dispatch(v)
- if i < len(t.items)-1:
- self._write(", ")
- self._write("}")
-
- def _Discard(self, t):
- """ Node for when return value is ignored such as in "foo(a)".
- """
- self._fill()
- self._dispatch(t.expr)
-
- def _Div(self, t):
- self.__binary_op(t, '/')
-
- def _Ellipsis(self, t):
- self._write("...")
-
- def _From(self, t):
- """ Handle "from xyz import foo, bar as baz".
- """
- # fixme: Are From and ImportFrom handled differently?
- self._fill("from ")
- self._write(t.modname)
- self._write(" import ")
- for i, (name, asname) in enumerate(t.names):
- if i != 0:
- self._write(", ")
- self._write(name)
- if asname is not None:
- self._write(" as "+asname)
-
- def _Function(self, t):
- """ Handle function definitions
- """
- if t.decorators is not None:
- self._fill("@")
- self._dispatch(t.decorators)
- self._fill("def "+t.name + "(")
- defaults = [None] * (len(t.argnames) - len(t.defaults)) + list(t.defaults)
- for i, arg in enumerate(zip(t.argnames, defaults)):
- self._write(arg[0])
- if arg[1] is not None:
- self._write('=')
- self._dispatch(arg[1])
- if i < len(t.argnames)-1:
- self._write(', ')
- self._write(")")
- if self._single_func:
- self._do_indent = False
- self._enter()
- self._dispatch(t.code)
- self._leave()
- self._do_indent = True
-
- def _Getattr(self, t):
- """ Handle getting an attribute of an object
- """
- if isinstance(t.expr, (Div, Mul, Sub, Add)):
- self._write('(')
- self._dispatch(t.expr)
- self._write(')')
- else:
- self._dispatch(t.expr)
-
- self._write('.'+t.attrname)
-
- def _If(self, t):
- self._fill()
-
- for i, (compare, code) in enumerate(t.tests):
- if i == 0:
- self._write("if ")
- else:
- self._write("elif ")
- self._dispatch(compare)
- self._enter()
- self._fill()
- self._dispatch(code)
- self._leave()
- self._write("\n")
-
- if t.else_ is not None:
- self._write("else")
- self._enter()
- self._fill()
- self._dispatch(t.else_)
- self._leave()
- self._write("\n")
-
- def _IfExp(self, t):
- self._dispatch(t.then)
- self._write(" if ")
- self._dispatch(t.test)
-
- if t.else_ is not None:
- self._write(" else (")
- self._dispatch(t.else_)
- self._write(")")
-
- def _Import(self, t):
- """ Handle "import xyz.foo".
- """
- self._fill("import ")
-
- for i, (name, asname) in enumerate(t.names):
- if i != 0:
- self._write(", ")
- self._write(name)
- if asname is not None:
- self._write(" as "+asname)
-
- def _Keyword(self, t):
- """ Keyword value assignment within function calls and definitions.
- """
- self._write(t.name)
- self._write("=")
- self._dispatch(t.expr)
-
- def _List(self, t):
- self._write("[")
- for i, node in enumerate(t.nodes):
- self._dispatch(node)
- if i < len(t.nodes)-1:
- self._write(", ")
- self._write("]")
-
- def _Module(self, t):
- if t.doc is not None:
- self._dispatch(t.doc)
- self._dispatch(t.node)
-
- def _Mul(self, t):
- self.__binary_op(t, '*')
-
- def _Name(self, t):
- self._write(t.name)
-
- def _NoneType(self, t):
- self._write("None")
-
- def _Not(self, t):
- self._write('not (')
- self._dispatch(t.expr)
- self._write(')')
-
- def _Or(self, t):
- self._write(" (")
- for i, node in enumerate(t.nodes):
- self._dispatch(node)
- if i != len(t.nodes)-1:
- self._write(") or (")
- self._write(")")
-
- def _Pass(self, t):
- self._write("pass\n")
-
- def _Printnl(self, t):
- self._fill("print ")
- if t.dest:
- self._write(">> ")
- self._dispatch(t.dest)
- self._write(", ")
- comma = False
- for node in t.nodes:
- if comma: self._write(', ')
- else: comma = True
- self._dispatch(node)
-
- def _Power(self, t):
- self.__binary_op(t, '**')
-
- def _Return(self, t):
- self._fill("return ")
- if t.value:
- if isinstance(t.value, Tuple):
- text = ', '.join([ name.name for name in t.value.asList() ])
- self._write(text)
- else:
- self._dispatch(t.value)
- if not self._do_indent:
- self._write('; ')
-
- def _Slice(self, t):
- self._dispatch(t.expr)
- self._write("[")
- if t.lower:
- self._dispatch(t.lower)
- self._write(":")
- if t.upper:
- self._dispatch(t.upper)
- #if t.step:
- # self._write(":")
- # self._dispatch(t.step)
- self._write("]")
-
- def _Sliceobj(self, t):
- for i, node in enumerate(t.nodes):
- if i != 0:
- self._write(":")
- if not (isinstance(node, Const) and node.value is None):
- self._dispatch(node)
-
- def _Stmt(self, tree):
- for node in tree.nodes:
- self._dispatch(node)
-
- def _Sub(self, t):
- self.__binary_op(t, '-')
-
- def _Subscript(self, t):
- self._dispatch(t.expr)
- self._write("[")
- for i, value in enumerate(t.subs):
- if i != 0:
- self._write(",")
- self._dispatch(value)
- self._write("]")
-
- def _TryExcept(self, t):
- self._fill("try")
- self._enter()
- self._dispatch(t.body)
- self._leave()
-
- for handler in t.handlers:
- self._fill('except ')
- self._dispatch(handler[0])
- if handler[1] is not None:
- self._write(', ')
- self._dispatch(handler[1])
- self._enter()
- self._dispatch(handler[2])
- self._leave()
-
- if t.else_:
- self._fill("else")
- self._enter()
- self._dispatch(t.else_)
- self._leave()
-
- def _Tuple(self, t):
-
- if not t.nodes:
- # Empty tuple.
- self._write("()")
- else:
- self._write("(")
-
- # _write each elements, separated by a comma.
- for element in t.nodes[:-1]:
- self._dispatch(element)
- self._write(", ")
-
- # Handle the last one without writing comma
- last_element = t.nodes[-1]
- self._dispatch(last_element)
-
- self._write(")")
-
- def _UnaryAdd(self, t):
- self._write("+")
- self._dispatch(t.expr)
-
- def _UnarySub(self, t):
- self._write("-")
- self._dispatch(t.expr)
-
- def _With(self, t):
- self._fill('with ')
- self._dispatch(t.expr)
- if t.vars:
- self._write(' as ')
- self._dispatch(t.vars.name)
- self._enter()
- self._dispatch(t.body)
- self._leave()
- self._write('\n')
-
- def _int(self, t):
- self._write(repr(t))
-
- def __binary_op(self, t, symbol):
- # Check if parenthesis are needed on left side and then dispatch
- has_paren = False
- left_class = str(t.left.__class__)
- if (left_class in op_precedence.keys() and
- op_precedence[left_class] < op_precedence[str(t.__class__)]):
- has_paren = True
- if has_paren:
- self._write('(')
- self._dispatch(t.left)
- if has_paren:
- self._write(')')
- # Write the appropriate symbol for operator
- self._write(symbol)
- # Check if parenthesis are needed on the right side and then dispatch
- has_paren = False
- right_class = str(t.right.__class__)
- if (right_class in op_precedence.keys() and
- op_precedence[right_class] < op_precedence[str(t.__class__)]):
- has_paren = True
- if has_paren:
- self._write('(')
- self._dispatch(t.right)
- if has_paren:
- self._write(')')
-
- def _float(self, t):
- # if t is 0.1, str(t)->'0.1' while repr(t)->'0.1000000000001'
- # We prefer str here.
- self._write(str(t))
-
- def _str(self, t):
- self._write(repr(t))
-
- def _tuple(self, t):
- self._write(str(t))
-
- #########################################################################
- # These are the methods from the _ast modules unparse.
- #
- # As our needs to handle more advanced code increase, we may want to
- # modify some of the methods below so that they work for compiler.ast.
- #########################################################################
-
-# # stmt
-# def _Expr(self, tree):
-# self._fill()
-# self._dispatch(tree.value)
-#
-# def _Import(self, t):
-# self._fill("import ")
-# first = True
-# for a in t.names:
-# if first:
-# first = False
-# else:
-# self._write(", ")
-# self._write(a.name)
-# if a.asname:
-# self._write(" as "+a.asname)
-#
-## def _ImportFrom(self, t):
-## self._fill("from ")
-## self._write(t.module)
-## self._write(" import ")
-## for i, a in enumerate(t.names):
-## if i == 0:
-## self._write(", ")
-## self._write(a.name)
-## if a.asname:
-## self._write(" as "+a.asname)
-## # XXX(jpe) what is level for?
-##
-#
-# def _Break(self, t):
-# self._fill("break")
-#
-# def _Continue(self, t):
-# self._fill("continue")
-#
-# def _Delete(self, t):
-# self._fill("del ")
-# self._dispatch(t.targets)
-#
-# def _Assert(self, t):
-# self._fill("assert ")
-# self._dispatch(t.test)
-# if t.msg:
-# self._write(", ")
-# self._dispatch(t.msg)
-#
-# def _Exec(self, t):
-# self._fill("exec ")
-# self._dispatch(t.body)
-# if t.globals:
-# self._write(" in ")
-# self._dispatch(t.globals)
-# if t.locals:
-# self._write(", ")
-# self._dispatch(t.locals)
-#
-# def _Print(self, t):
-# self._fill("print ")
-# do_comma = False
-# if t.dest:
-# self._write(">>")
-# self._dispatch(t.dest)
-# do_comma = True
-# for e in t.values:
-# if do_comma:self._write(", ")
-# else:do_comma=True
-# self._dispatch(e)
-# if not t.nl:
-# self._write(",")
-#
-# def _Global(self, t):
-# self._fill("global")
-# for i, n in enumerate(t.names):
-# if i != 0:
-# self._write(",")
-# self._write(" " + n)
-#
-# def _Yield(self, t):
-# self._fill("yield")
-# if t.value:
-# self._write(" (")
-# self._dispatch(t.value)
-# self._write(")")
-#
-# def _Raise(self, t):
-# self._fill('raise ')
-# if t.type:
-# self._dispatch(t.type)
-# if t.inst:
-# self._write(", ")
-# self._dispatch(t.inst)
-# if t.tback:
-# self._write(", ")
-# self._dispatch(t.tback)
-#
-#
-# def _TryFinally(self, t):
-# self._fill("try")
-# self._enter()
-# self._dispatch(t.body)
-# self._leave()
-#
-# self._fill("finally")
-# self._enter()
-# self._dispatch(t.finalbody)
-# self._leave()
-#
-# def _excepthandler(self, t):
-# self._fill("except ")
-# if t.type:
-# self._dispatch(t.type)
-# if t.name:
-# self._write(", ")
-# self._dispatch(t.name)
-# self._enter()
-# self._dispatch(t.body)
-# self._leave()
-#
-# def _ClassDef(self, t):
-# self._write("\n")
-# self._fill("class "+t.name)
-# if t.bases:
-# self._write("(")
-# for a in t.bases:
-# self._dispatch(a)
-# self._write(", ")
-# self._write(")")
-# self._enter()
-# self._dispatch(t.body)
-# self._leave()
-#
-# def _FunctionDef(self, t):
-# self._write("\n")
-# for deco in t.decorators:
-# self._fill("@")
-# self._dispatch(deco)
-# self._fill("def "+t.name + "(")
-# self._dispatch(t.args)
-# self._write(")")
-# self._enter()
-# self._dispatch(t.body)
-# self._leave()
-#
-# def _For(self, t):
-# self._fill("for ")
-# self._dispatch(t.target)
-# self._write(" in ")
-# self._dispatch(t.iter)
-# self._enter()
-# self._dispatch(t.body)
-# self._leave()
-# if t.orelse:
-# self._fill("else")
-# self._enter()
-# self._dispatch(t.orelse)
-# self._leave
-#
-# def _While(self, t):
-# self._fill("while ")
-# self._dispatch(t.test)
-# self._enter()
-# self._dispatch(t.body)
-# self._leave()
-# if t.orelse:
-# self._fill("else")
-# self._enter()
-# self._dispatch(t.orelse)
-# self._leave
-#
-# # expr
-# def _Str(self, tree):
-# self._write(repr(tree.s))
-##
-# def _Repr(self, t):
-# self._write("`")
-# self._dispatch(t.value)
-# self._write("`")
-#
-# def _Num(self, t):
-# self._write(repr(t.n))
-#
-# def _ListComp(self, t):
-# self._write("[")
-# self._dispatch(t.elt)
-# for gen in t.generators:
-# self._dispatch(gen)
-# self._write("]")
-#
-# def _GeneratorExp(self, t):
-# self._write("(")
-# self._dispatch(t.elt)
-# for gen in t.generators:
-# self._dispatch(gen)
-# self._write(")")
-#
-# def _comprehension(self, t):
-# self._write(" for ")
-# self._dispatch(t.target)
-# self._write(" in ")
-# self._dispatch(t.iter)
-# for if_clause in t.ifs:
-# self._write(" if ")
-# self._dispatch(if_clause)
-#
-# def _IfExp(self, t):
-# self._dispatch(t.body)
-# self._write(" if ")
-# self._dispatch(t.test)
-# if t.orelse:
-# self._write(" else ")
-# self._dispatch(t.orelse)
-#
-# unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"}
-# def _UnaryOp(self, t):
-# self._write(self.unop[t.op.__class__.__name__])
-# self._write("(")
-# self._dispatch(t.operand)
-# self._write(")")
-#
-# binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
-# "LShift":">>", "RShift":"<<", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
-# "FloorDiv":"//", "Pow": "**"}
-# def _BinOp(self, t):
-# self._write("(")
-# self._dispatch(t.left)
-# self._write(")" + self.binop[t.op.__class__.__name__] + "(")
-# self._dispatch(t.right)
-# self._write(")")
-#
-# boolops = {_ast.And: 'and', _ast.Or: 'or'}
-# def _BoolOp(self, t):
-# self._write("(")
-# self._dispatch(t.values[0])
-# for v in t.values[1:]:
-# self._write(" %s " % self.boolops[t.op.__class__])
-# self._dispatch(v)
-# self._write(")")
-#
-# def _Attribute(self,t):
-# self._dispatch(t.value)
-# self._write(".")
-# self._write(t.attr)
-#
-## def _Call(self, t):
-## self._dispatch(t.func)
-## self._write("(")
-## comma = False
-## for e in t.args:
-## if comma: self._write(", ")
-## else: comma = True
-## self._dispatch(e)
-## for e in t.keywords:
-## if comma: self._write(", ")
-## else: comma = True
-## self._dispatch(e)
-## if t.starargs:
-## if comma: self._write(", ")
-## else: comma = True
-## self._write("*")
-## self._dispatch(t.starargs)
-## if t.kwargs:
-## if comma: self._write(", ")
-## else: comma = True
-## self._write("**")
-## self._dispatch(t.kwargs)
-## self._write(")")
-#
-# # slice
-# def _Index(self, t):
-# self._dispatch(t.value)
-#
-# def _ExtSlice(self, t):
-# for i, d in enumerate(t.dims):
-# if i != 0:
-# self._write(': ')
-# self._dispatch(d)
-#
-# # others
-# def _arguments(self, t):
-# first = True
-# nonDef = len(t.args)-len(t.defaults)
-# for a in t.args[0:nonDef]:
-# if first:first = False
-# else: self._write(", ")
-# self._dispatch(a)
-# for a,d in zip(t.args[nonDef:], t.defaults):
-# if first:first = False
-# else: self._write(", ")
-# self._dispatch(a),
-# self._write("=")
-# self._dispatch(d)
-# if t.vararg:
-# if first:first = False
-# else: self._write(", ")
-# self._write("*"+t.vararg)
-# if t.kwarg:
-# if first:first = False
-# else: self._write(", ")
-# self._write("**"+t.kwarg)
-#
-## def _keyword(self, t):
-## self._write(t.arg)
-## self._write("=")
-## self._dispatch(t.value)
-#
-# def _Lambda(self, t):
-# self._write("lambda ")
-# self._dispatch(t.args)
-# self._write(": ")
-# self._dispatch(t.body)
-
-
-
diff --git a/doc/sphinxext/docscrape.py b/doc/sphinxext/docscrape.py
deleted file mode 100644
index 8a99937..0000000
--- a/doc/sphinxext/docscrape.py
+++ /dev/null
@@ -1,500 +0,0 @@
-"""Extract reference documentation from the NumPy source tree.
-
-"""
-
-import inspect
-import textwrap
-import re
-import pydoc
-from StringIO import StringIO
-from warnings import warn
-
-class Reader(object):
- """A line-based string reader.
-
- """
- def __init__(self, data):
- """
- Parameters
- ----------
- data : str
- String with lines separated by '\n'.
-
- """
- if isinstance(data, list):
- self._str = data
- else:
- self._str = data.split('\n') # store string as list of lines
-
- self.reset()
-
- def __getitem__(self, n):
- return self._str[n]
-
- def reset(self):
- self._l = 0 # current line nr
-
- def read(self):
- if not self.eof():
- out = self[self._l]
- self._l += 1
- return out
- else:
- return ''
-
- def seek_next_non_empty_line(self):
- for l in self[self._l:]:
- if l.strip():
- break
- else:
- self._l += 1
-
- def eof(self):
- return self._l >= len(self._str)
-
- def read_to_condition(self, condition_func):
- start = self._l
- for line in self[start:]:
- if condition_func(line):
- return self[start:self._l]
- self._l += 1
- if self.eof():
- return self[start:self._l+1]
- return []
-
- def read_to_next_empty_line(self):
- self.seek_next_non_empty_line()
- def is_empty(line):
- return not line.strip()
- return self.read_to_condition(is_empty)
-
- def read_to_next_unindented_line(self):
- def is_unindented(line):
- return (line.strip() and (len(line.lstrip()) == len(line)))
- return self.read_to_condition(is_unindented)
-
- def peek(self,n=0):
- if self._l + n < len(self._str):
- return self[self._l + n]
- else:
- return ''
-
- def is_empty(self):
- return not ''.join(self._str).strip()
-
-
-class NumpyDocString(object):
- def __init__(self, docstring, config={}):
- docstring = textwrap.dedent(docstring).split('\n')
-
- self._doc = Reader(docstring)
- self._parsed_data = {
- 'Signature': '',
- 'Summary': [''],
- 'Extended Summary': [],
- 'Parameters': [],
- 'Returns': [],
- 'Raises': [],
- 'Warns': [],
- 'Other Parameters': [],
- 'Attributes': [],
- 'Methods': [],
- 'See Also': [],
- 'Notes': [],
- 'Warnings': [],
- 'References': '',
- 'Examples': '',
- 'index': {}
- }
-
- self._parse()
-
- def __getitem__(self, key):
- return self._parsed_data[key]
-
- def __setitem__(self, key, val):
- if not self._parsed_data.has_key(key):
- warn("Unknown section %s" % key)
- else:
- self._parsed_data[key] = val
-
- def _is_at_section(self):
- self._doc.seek_next_non_empty_line()
-
- if self._doc.eof():
- return False
-
- l1 = self._doc.peek().strip() # e.g. Parameters
-
- if l1.startswith('.. index::'):
- return True
-
- l2 = self._doc.peek(1).strip() # ---------- or ==========
- return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1))
-
- def _strip(self, doc):
- i = 0
- j = 0
- for i, line in enumerate(doc):
- if line.strip(): break
-
- for j, line in enumerate(doc[::-1]):
- if line.strip(): break
-
- return doc[i:len(doc)-j]
-
- def _read_to_next_section(self):
- section = self._doc.read_to_next_empty_line()
-
- while not self._is_at_section() and not self._doc.eof():
- if not self._doc.peek(-1).strip(): # previous line was empty
- section += ['']
-
- section += self._doc.read_to_next_empty_line()
-
- return section
-
- def _read_sections(self):
- while not self._doc.eof():
- data = self._read_to_next_section()
- name = data[0].strip()
-
- if name.startswith('..'): # index section
- yield name, data[1:]
- elif len(data) < 2:
- yield StopIteration
- else:
- yield name, self._strip(data[2:])
-
- def _parse_param_list(self, content):
- r = Reader(content)
- params = []
- while not r.eof():
- header = r.read().strip()
- if ' : ' in header:
- arg_name, arg_type = header.split(' : ')[:2]
- else:
- arg_name, arg_type = header, ''
-
- desc = r.read_to_next_unindented_line()
- desc = dedent_lines(desc)
-
- params.append((arg_name, arg_type, desc))
-
- return params
-
-
- _name_rgx = re.compile(r"^\s*(:(?P<role>\w+):`(?P<name>[a-zA-Z0-9_.-]+)`|"
- r" (?P<name2>[a-zA-Z0-9_.-]+))\s*", re.X)
- def _parse_see_also(self, content):
- """
- func_name : Descriptive text
- continued text
- another_func_name : Descriptive text
- func_name1, func_name2, :meth:`func_name`, func_name3
-
- """
- items = []
-
- def parse_item_name(text):
- """Match ':role:`name`' or 'name'"""
- m = self._name_rgx.match(text)
- if m:
- g = m.groups()
- if g[1] is None:
- return g[3], None
- else:
- return g[2], g[1]
- raise ValueError("%s is not a item name" % text)
-
- def push_item(name, rest):
- if not name:
- return
- name, role = parse_item_name(name)
- items.append((name, list(rest), role))
- del rest[:]
-
- current_func = None
- rest = []
-
- for line in content:
- if not line.strip(): continue
-
- m = self._name_rgx.match(line)
- if m and line[m.end():].strip().startswith(':'):
- push_item(current_func, rest)
- current_func, line = line[:m.end()], line[m.end():]
- rest = [line.split(':', 1)[1].strip()]
- if not rest[0]:
- rest = []
- elif not line.startswith(' '):
- push_item(current_func, rest)
- current_func = None
- if ',' in line:
- for func in line.split(','):
- if func.strip():
- push_item(func, [])
- elif line.strip():
- current_func = line
- elif current_func is not None:
- rest.append(line.strip())
- push_item(current_func, rest)
- return items
-
- def _parse_index(self, section, content):
- """
- .. index: default
- :refguide: something, else, and more
-
- """
- def strip_each_in(lst):
- return [s.strip() for s in lst]
-
- out = {}
- section = section.split('::')
- if len(section) > 1:
- out['default'] = strip_each_in(section[1].split(','))[0]
- for line in content:
- line = line.split(':')
- if len(line) > 2:
- out[line[1]] = strip_each_in(line[2].split(','))
- return out
-
- def _parse_summary(self):
- """Grab signature (if given) and summary"""
- if self._is_at_section():
- return
-
- summary = self._doc.read_to_next_empty_line()
- summary_str = " ".join([s.strip() for s in summary]).strip()
- if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str):
- self['Signature'] = summary_str
- if not self._is_at_section():
- self['Summary'] = self._doc.read_to_next_empty_line()
- else:
- self['Summary'] = summary
-
- if not self._is_at_section():
- self['Extended Summary'] = self._read_to_next_section()
-
- def _parse(self):
- self._doc.reset()
- self._parse_summary()
-
- for (section, content) in self._read_sections():
- if not section.startswith('..'):
- section = ' '.join([s.capitalize() for s in section.split(' ')])
- if section in ('Parameters', 'Returns', 'Raises', 'Warns',
- 'Other Parameters', 'Attributes', 'Methods'):
- self[section] = self._parse_param_list(content)
- elif section.startswith('.. index::'):
- self['index'] = self._parse_index(section, content)
- elif section == 'See Also':
- self['See Also'] = self._parse_see_also(content)
- else:
- self[section] = content
-
- # string conversion routines
-
- def _str_header(self, name, symbol='-'):
- return [name, len(name)*symbol]
-
- def _str_indent(self, doc, indent=4):
- out = []
- for line in doc:
- out += [' '*indent + line]
- return out
-
- def _str_signature(self):
- if self['Signature']:
- return [self['Signature'].replace('*', '\*')] + ['']
- else:
- return ['']
-
- def _str_summary(self):
- if self['Summary']:
- return self['Summary'] + ['']
- else:
- return []
-
- def _str_extended_summary(self):
- if self['Extended Summary']:
- return self['Extended Summary'] + ['']
- else:
- return []
-
- def _str_param_list(self, name):
- out = []
- if self[name]:
- out += self._str_header(name)
- for param, param_type, desc in self[name]:
- out += ['%s : %s' % (param, param_type)]
- out += self._str_indent(desc)
- out += ['']
- return out
-
- def _str_section(self, name):
- out = []
- if self[name]:
- out += self._str_header(name)
- out += self[name]
- out += ['']
- return out
-
- def _str_see_also(self, func_role):
- if not self['See Also']: return []
- out = []
- out += self._str_header("See Also")
- last_had_desc = True
- for func, desc, role in self['See Also']:
- if role:
- link = ':%s:`%s`' % (role, func)
- elif func_role:
- link = ':%s:`%s`' % (func_role, func)
- else:
- link = "`%s`_" % func
- if desc or last_had_desc:
- out += ['']
- out += [link]
- else:
- out[-1] += ", %s" % link
- if desc:
- out += self._str_indent([' '.join(desc)])
- last_had_desc = True
- else:
- last_had_desc = False
- out += ['']
- return out
-
- def _str_index(self):
- idx = self['index']
- out = []
- out += ['.. index:: %s' % idx.get('default', '')]
- for section, references in idx.iteritems():
- if section == 'default':
- continue
- out += [' :%s: %s' % (section, ', '.join(references))]
- return out
-
- def __str__(self, func_role=''):
- out = []
- out += self._str_signature()
- out += self._str_summary()
- out += self._str_extended_summary()
- for param_list in ('Parameters', 'Returns', 'Other Parameters',
- 'Raises', 'Warns'):
- out += self._str_param_list(param_list)
- out += self._str_section('Warnings')
- out += self._str_see_also(func_role)
- for s in ('Notes', 'References', 'Examples'):
- out += self._str_section(s)
- for param_list in ('Attributes', 'Methods'):
- out += self._str_param_list(param_list)
- out += self._str_index()
- return '\n'.join(out)
-
-
-def indent(str,indent=4):
- indent_str = ' '*indent
- if str is None:
- return indent_str
- lines = str.split('\n')
- return '\n'.join(indent_str + l for l in lines)
-
-def dedent_lines(lines):
- """Deindent a list of lines maximally"""
- return textwrap.dedent("\n".join(lines)).split("\n")
-
-def header(text, style='-'):
- return text + '\n' + style*len(text) + '\n'
-
-
-class FunctionDoc(NumpyDocString):
- def __init__(self, func, role='func', doc=None, config={}):
- self._f = func
- self._role = role # e.g. "func" or "meth"
-
- if doc is None:
- if func is None:
- raise ValueError("No function or docstring given")
- doc = inspect.getdoc(func) or ''
- NumpyDocString.__init__(self, doc)
-
- if not self['Signature'] and func is not None:
- func, func_name = self.get_func()
- try:
- # try to read signature
- argspec = inspect.getargspec(func)
- argspec = inspect.formatargspec(*argspec)
- argspec = argspec.replace('*', '\*')
- signature = '%s%s' % (func_name, argspec)
- except TypeError, e:
- signature = '%s()' % func_name
- self['Signature'] = signature
-
- def get_func(self):
- func_name = getattr(self._f, '__name__', self.__class__.__name__)
- if inspect.isclass(self._f):
- func = getattr(self._f, '__call__', self._f.__init__)
- else:
- func = self._f
- return func, func_name
-
- def __str__(self):
- out = ''
-
- func, func_name = self.get_func()
- signature = self['Signature'].replace('*', '\*')
-
- roles = {'func': 'function',
- 'meth': 'method'}
-
- if self._role:
- if not roles.has_key(self._role):
- print "Warning: invalid role %s" % self._role
- out += '.. %s:: %s\n \n\n' % (roles.get(self._role, ''),
- func_name)
-
- out += super(FunctionDoc, self).__str__(func_role=self._role)
- return out
-
-
-class ClassDoc(NumpyDocString):
- def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc,
- config={}):
- if not inspect.isclass(cls) and cls is not None:
- raise ValueError("Expected a class or None, but got %r" % cls)
- self._cls = cls
-
- if modulename and not modulename.endswith('.'):
- modulename += '.'
- self._mod = modulename
-
- if doc is None:
- if cls is None:
- raise ValueError("No class or documentation string given")
- doc = pydoc.getdoc(cls)
-
- NumpyDocString.__init__(self, doc)
-
- if config.get('show_class_members', True):
- if not self['Methods']:
- self['Methods'] = [(name, '', '')
- for name in sorted(self.methods)]
- if not self['Attributes']:
- self['Attributes'] = [(name, '', '')
- for name in sorted(self.properties)]
-
- @property
- def methods(self):
- if self._cls is None:
- return []
- return [name for name, func in inspect.getmembers(self._cls)
- if not name.startswith('_') and callable(func)]
-
- @property
- def properties(self):
- if self._cls is None:
- return []
- return [name for name, func in inspect.getmembers(self._cls)
- if not name.startswith('_') and func is None]
diff --git a/doc/sphinxext/docscrape_sphinx.py b/doc/sphinxext/docscrape_sphinx.py
deleted file mode 100644
index e9da950..0000000
--- a/doc/sphinxext/docscrape_sphinx.py
+++ /dev/null
@@ -1,227 +0,0 @@
-import re, inspect, textwrap, pydoc
-import sphinx
-from docscrape import NumpyDocString, FunctionDoc, ClassDoc
-
-class SphinxDocString(NumpyDocString):
- def __init__(self, docstring, config={}):
- self.use_plots = config.get('use_plots', False)
- NumpyDocString.__init__(self, docstring, config=config)
-
- # string conversion routines
- def _str_header(self, name, symbol='`'):
- return ['.. rubric:: ' + name, '']
-
- def _str_field_list(self, name):
- return [':' + name + ':']
-
- def _str_indent(self, doc, indent=4):
- out = []
- for line in doc:
- out += [' '*indent + line]
- return out
-
- def _str_signature(self):
- return ['']
- if self['Signature']:
- return ['``%s``' % self['Signature']] + ['']
- else:
- return ['']
-
- def _str_summary(self):
- return self['Summary'] + ['']
-
- def _str_extended_summary(self):
- return self['Extended Summary'] + ['']
-
- def _str_param_list(self, name):
- out = []
- if self[name]:
- out += self._str_field_list(name)
- out += ['']
- for param, param_type, desc in self[name]:
- out += self._str_indent(['**%s** : %s' % (param.strip(),
- param_type)])
- out += ['']
- out += self._str_indent(desc, 8)
- out += ['']
- return out
-
- @property
- def _obj(self):
- if hasattr(self, '_cls'):
- return self._cls
- elif hasattr(self, '_f'):
- return self._f
- return None
-
- def _str_member_list(self, name):
- """
- Generate a member listing, autosummary:: table where possible,
- and a table where not.
-
- """
- out = []
- if self[name]:
- out += ['.. rubric:: %s' % name, '']
- prefix = getattr(self, '_name', '')
-
- if prefix:
- prefix = '~%s.' % prefix
-
- autosum = []
- others = []
- for param, param_type, desc in self[name]:
- param = param.strip()
- if not self._obj or hasattr(self._obj, param):
- autosum += [" %s%s" % (prefix, param)]
- else:
- others.append((param, param_type, desc))
-
- if autosum:
- out += ['.. autosummary::', ' :toctree:', '']
- out += autosum
-
- if others:
- maxlen_0 = max([len(x[0]) for x in others])
- maxlen_1 = max([len(x[1]) for x in others])
- hdr = "="*maxlen_0 + " " + "="*maxlen_1 + " " + "="*10
- fmt = '%%%ds %%%ds ' % (maxlen_0, maxlen_1)
- n_indent = maxlen_0 + maxlen_1 + 4
- out += [hdr]
- for param, param_type, desc in others:
- out += [fmt % (param.strip(), param_type)]
- out += self._str_indent(desc, n_indent)
- out += [hdr]
- out += ['']
- return out
-
- def _str_section(self, name):
- out = []
- if self[name]:
- out += self._str_header(name)
- out += ['']
- content = textwrap.dedent("\n".join(self[name])).split("\n")
- out += content
- out += ['']
- return out
-
- def _str_see_also(self, func_role):
- out = []
- if self['See Also']:
- see_also = super(SphinxDocString, self)._str_see_also(func_role)
- out = ['.. seealso::', '']
- out += self._str_indent(see_also[2:])
- return out
-
- def _str_warnings(self):
- out = []
- if self['Warnings']:
- out = ['.. warning::', '']
- out += self._str_indent(self['Warnings'])
- return out
-
- def _str_index(self):
- idx = self['index']
- out = []
- if len(idx) == 0:
- return out
-
- out += ['.. index:: %s' % idx.get('default', '')]
- for section, references in idx.iteritems():
- if section == 'default':
- continue
- elif section == 'refguide':
- out += [' single: %s' % (', '.join(references))]
- else:
- out += [' %s: %s' % (section, ','.join(references))]
- return out
-
- def _str_references(self):
- out = []
- if self['References']:
- out += self._str_header('References')
- if isinstance(self['References'], str):
- self['References'] = [self['References']]
- out.extend(self['References'])
- out += ['']
- # Latex collects all references to a separate bibliography,
- # so we need to insert links to it
- if sphinx.__version__ >= "0.6":
- out += ['.. only:: latex', '']
- else:
- out += ['.. latexonly::', '']
- items = []
- for line in self['References']:
- m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I)
- if m:
- items.append(m.group(1))
- out += [' ' + ", ".join(["[%s]_" % item for item in items]), '']
- return out
-
- def _str_examples(self):
- examples_str = "\n".join(self['Examples'])
-
- if (self.use_plots and 'import matplotlib' in examples_str
- and 'plot::' not in examples_str):
- out = []
- out += self._str_header('Examples')
- out += ['.. plot::', '']
- out += self._str_indent(self['Examples'])
- out += ['']
- return out
- else:
- return self._str_section('Examples')
-
- def __str__(self, indent=0, func_role="obj"):
- out = []
- out += self._str_signature()
- out += self._str_index() + ['']
- out += self._str_summary()
- out += self._str_extended_summary()
- for param_list in ('Parameters', 'Returns', 'Other Parameters',
- 'Raises', 'Warns'):
- out += self._str_param_list(param_list)
- out += self._str_warnings()
- out += self._str_see_also(func_role)
- out += self._str_section('Notes')
- out += self._str_references()
- out += self._str_examples()
- #for param_list in ('Attributes', 'Methods'):
- # out += self._str_member_list(param_list)
- out = self._str_indent(out, indent)
- return '\n'.join(out)
-
-class SphinxFunctionDoc(SphinxDocString, FunctionDoc):
- def __init__(self, obj, doc=None, config={}):
- self.use_plots = config.get('use_plots', False)
- FunctionDoc.__init__(self, obj, doc=doc, config=config)
-
-class SphinxClassDoc(SphinxDocString, ClassDoc):
- def __init__(self, obj, doc=None, func_doc=None, config={}):
- self.use_plots = config.get('use_plots', False)
- ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config)
-
-class SphinxObjDoc(SphinxDocString):
- def __init__(self, obj, doc=None, config={}):
- self._f = obj
- SphinxDocString.__init__(self, doc, config=config)
-
-def get_doc_object(obj, what=None, doc=None, config={}):
- if what is None:
- if inspect.isclass(obj):
- what = 'class'
- elif inspect.ismodule(obj):
- what = 'module'
- elif callable(obj):
- what = 'function'
- else:
- what = 'object'
- if what == 'class':
- return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc,
- config=config)
- elif what in ('function', 'method'):
- return SphinxFunctionDoc(obj, doc=doc, config=config)
- else:
- if doc is None:
- doc = pydoc.getdoc(obj)
- return SphinxObjDoc(obj, doc, config=config)
diff --git a/doc/sphinxext/inheritance_diagram.py b/doc/sphinxext/inheritance_diagram.py
deleted file mode 100644
index 407fc13..0000000
--- a/doc/sphinxext/inheritance_diagram.py
+++ /dev/null
@@ -1,407 +0,0 @@
-"""
-Defines a docutils directive for inserting inheritance diagrams.
-
-Provide the directive with one or more classes or modules (separated
-by whitespace). For modules, all of the classes in that module will
-be used.
-
-Example::
-
- Given the following classes:
-
- class A: pass
- class B(A): pass
- class C(A): pass
- class D(B, C): pass
- class E(B): pass
-
- .. inheritance-diagram: D E
-
- Produces a graph like the following:
-
- A
- / \
- B C
- / \ /
- E D
-
-The graph is inserted as a PNG+image map into HTML and a PDF in
-LaTeX.
-"""
-
-import inspect
-import os
-import re
-import subprocess
-try:
- from hashlib import md5
-except ImportError:
- from md5 import md5
-
-from docutils.nodes import Body, Element
-from docutils.parsers.rst import directives
-from sphinx.roles import xfileref_role
-
-def my_import(name):
- """Module importer - taken from the python documentation.
-
- This function allows importing names with dots in them."""
-
- mod = __import__(name)
- components = name.split('.')
- for comp in components[1:]:
- mod = getattr(mod, comp)
- return mod
-
-class DotException(Exception):
- pass
-
-class InheritanceGraph(object):
- """
- Given a list of classes, determines the set of classes that
- they inherit from all the way to the root "object", and then
- is able to generate a graphviz dot graph from them.
- """
- def __init__(self, class_names, show_builtins=False):
- """
- *class_names* is a list of child classes to show bases from.
-
- If *show_builtins* is True, then Python builtins will be shown
- in the graph.
- """
- self.class_names = class_names
- self.classes = self._import_classes(class_names)
- self.all_classes = self._all_classes(self.classes)
- if len(self.all_classes) == 0:
- raise ValueError("No classes found for inheritance diagram")
- self.show_builtins = show_builtins
-
- py_sig_re = re.compile(r'''^([\w.]*\.)? # class names
- (\w+) \s* $ # optionally arguments
- ''', re.VERBOSE)
-
- def _import_class_or_module(self, name):
- """
- Import a class using its fully-qualified *name*.
- """
- try:
- path, base = self.py_sig_re.match(name).groups()
- except:
- raise ValueError(
- "Invalid class or module '%s' specified for inheritance diagram" % name)
- fullname = (path or '') + base
- path = (path and path.rstrip('.'))
- if not path:
- path = base
- try:
- module = __import__(path, None, None, [])
- # We must do an import of the fully qualified name. Otherwise if a
- # subpackage 'a.b' is requested where 'import a' does NOT provide
- # 'a.b' automatically, then 'a.b' will not be found below. This
- # second call will force the equivalent of 'import a.b' to happen
- # after the top-level import above.
- my_import(fullname)
-
- except ImportError:
- raise ValueError(
- "Could not import class or module '%s' specified for inheritance diagram" % name)
-
- try:
- todoc = module
- for comp in fullname.split('.')[1:]:
- todoc = getattr(todoc, comp)
- except AttributeError:
- raise ValueError(
- "Could not find class or module '%s' specified for inheritance diagram" % name)
-
- # If a class, just return it
- if inspect.isclass(todoc):
- return [todoc]
- elif inspect.ismodule(todoc):
- classes = []
- for cls in todoc.__dict__.values():
- if inspect.isclass(cls) and cls.__module__ == todoc.__name__:
- classes.append(cls)
- return classes
- raise ValueError(
- "'%s' does not resolve to a class or module" % name)
-
- def _import_classes(self, class_names):
- """
- Import a list of classes.
- """
- classes = []
- for name in class_names:
- classes.extend(self._import_class_or_module(name))
- return classes
-
- def _all_classes(self, classes):
- """
- Return a list of all classes that are ancestors of *classes*.
- """
- all_classes = {}
-
- def recurse(cls):
- all_classes[cls] = None
- for c in cls.__bases__:
- if c not in all_classes:
- recurse(c)
-
- for cls in classes:
- recurse(cls)
-
- return all_classes.keys()
-
- def class_name(self, cls, parts=0):
- """
- Given a class object, return a fully-qualified name. This
- works for things I've tested in matplotlib so far, but may not
- be completely general.
- """
- module = cls.__module__
- if module == '__builtin__':
- fullname = cls.__name__
- else:
- fullname = "%s.%s" % (module, cls.__name__)
- if parts == 0:
- return fullname
- name_parts = fullname.split('.')
- return '.'.join(name_parts[-parts:])
-
- def get_all_class_names(self):
- """
- Get all of the class names involved in the graph.
- """
- return [self.class_name(x) for x in self.all_classes]
-
- # These are the default options for graphviz
- default_graph_options = {
- "rankdir": "LR",
- "size": '"8.0, 12.0"'
- }
- default_node_options = {
- "shape": "box",
- "fontsize": 10,
- "height": 0.25,
- "fontname": "Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",
- "style": '"setlinewidth(0.5)"'
- }
- default_edge_options = {
- "arrowsize": 0.5,
- "style": '"setlinewidth(0.5)"'
- }
-
- def _format_node_options(self, options):
- return ','.join(["%s=%s" % x for x in options.items()])
- def _format_graph_options(self, options):
- return ''.join(["%s=%s;\n" % x for x in options.items()])
-
- def generate_dot(self, fd, name, parts=0, urls={},
- graph_options={}, node_options={},
- edge_options={}):
- """
- Generate a graphviz dot graph from the classes that
- were passed in to __init__.
-
- *fd* is a Python file-like object to write to.
-
- *name* is the name of the graph
-
- *urls* is a dictionary mapping class names to http urls
-
- *graph_options*, *node_options*, *edge_options* are
- dictionaries containing key/value pairs to pass on as graphviz
- properties.
- """
- g_options = self.default_graph_options.copy()
- g_options.update(graph_options)
- n_options = self.default_node_options.copy()
- n_options.update(node_options)
- e_options = self.default_edge_options.copy()
- e_options.update(edge_options)
-
- fd.write('digraph %s {\n' % name)
- fd.write(self._format_graph_options(g_options))
-
- for cls in self.all_classes:
- if not self.show_builtins and cls in __builtins__.values():
- continue
-
- name = self.class_name(cls, parts)
-
- # Write the node
- this_node_options = n_options.copy()
- url = urls.get(self.class_name(cls))
- if url is not None:
- this_node_options['URL'] = '"%s"' % url
- fd.write(' "%s" [%s];\n' %
- (name, self._format_node_options(this_node_options)))
-
- # Write the edges
- for base in cls.__bases__:
- if not self.show_builtins and base in __builtins__.values():
- continue
-
- base_name = self.class_name(base, parts)
- fd.write(' "%s" -> "%s" [%s];\n' %
- (base_name, name,
- self._format_node_options(e_options)))
- fd.write('}\n')
-
- def run_dot(self, args, name, parts=0, urls={},
- graph_options={}, node_options={}, edge_options={}):
- """
- Run graphviz 'dot' over this graph, returning whatever 'dot'
- writes to stdout.
-
- *args* will be passed along as commandline arguments.
-
- *name* is the name of the graph
-
- *urls* is a dictionary mapping class names to http urls
-
- Raises DotException for any of the many os and
- installation-related errors that may occur.
- """
- try:
- dot = subprocess.Popen(['dot'] + list(args),
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- close_fds=True)
- except OSError:
- raise DotException("Could not execute 'dot'. Are you sure you have 'graphviz' installed?")
- except ValueError:
- raise DotException("'dot' called with invalid arguments")
- except:
- raise DotException("Unexpected error calling 'dot'")
-
- self.generate_dot(dot.stdin, name, parts, urls, graph_options,
- node_options, edge_options)
- dot.stdin.close()
- result = dot.stdout.read()
- returncode = dot.wait()
- if returncode != 0:
- raise DotException("'dot' returned the errorcode %d" % returncode)
- return result
-
-class inheritance_diagram(Body, Element):
- """
- A docutils node to use as a placeholder for the inheritance
- diagram.
- """
- pass
-
-def inheritance_diagram_directive(name, arguments, options, content, lineno,
- content_offset, block_text, state,
- state_machine):
- """
- Run when the inheritance_diagram directive is first encountered.
- """
- node = inheritance_diagram()
-
- class_names = arguments
-
- # Create a graph starting with the list of classes
- graph = InheritanceGraph(class_names)
-
- # Create xref nodes for each target of the graph's image map and
- # add them to the doc tree so that Sphinx can resolve the
- # references to real URLs later. These nodes will eventually be
- # removed from the doctree after we're done with them.
- for name in graph.get_all_class_names():
- refnodes, x = xfileref_role(
- 'class', ':class:`%s`' % name, name, 0, state)
- node.extend(refnodes)
- # Store the graph object so we can use it to generate the
- # dot file later
- node['graph'] = graph
- # Store the original content for use as a hash
- node['parts'] = options.get('parts', 0)
- node['content'] = " ".join(class_names)
- return [node]
-
-def get_graph_hash(node):
- return md5(node['content'] + str(node['parts'])).hexdigest()[-10:]
-
-def html_output_graph(self, node):
- """
- Output the graph for HTML. This will insert a PNG with clickable
- image map.
- """
- graph = node['graph']
- parts = node['parts']
-
- graph_hash = get_graph_hash(node)
- name = "inheritance%s" % graph_hash
- path = '_images'
- dest_path = os.path.join(setup.app.builder.outdir, path)
- if not os.path.exists(dest_path):
- os.makedirs(dest_path)
- png_path = os.path.join(dest_path, name + ".png")
- path = setup.app.builder.imgpath
-
- # Create a mapping from fully-qualified class names to URLs.
- urls = {}
- for child in node:
- if child.get('refuri') is not None:
- urls[child['reftitle']] = child.get('refuri')
- elif child.get('refid') is not None:
- urls[child['reftitle']] = '#' + child.get('refid')
-
- # These arguments to dot will save a PNG file to disk and write
- # an HTML image map to stdout.
- image_map = graph.run_dot(['-Tpng', '-o%s' % png_path, '-Tcmapx'],
- name, parts, urls)
- return ('<img src="%s/%s.png" usemap="#%s" class="inheritance"/>%s' %
- (path, name, name, image_map))
-
-def latex_output_graph(self, node):
- """
- Output the graph for LaTeX. This will insert a PDF.
- """
- graph = node['graph']
- parts = node['parts']
-
- graph_hash = get_graph_hash(node)
- name = "inheritance%s" % graph_hash
- dest_path = os.path.abspath(os.path.join(setup.app.builder.outdir, '_images'))
- if not os.path.exists(dest_path):
- os.makedirs(dest_path)
- pdf_path = os.path.abspath(os.path.join(dest_path, name + ".pdf"))
-
- graph.run_dot(['-Tpdf', '-o%s' % pdf_path],
- name, parts, graph_options={'size': '"6.0,6.0"'})
- return '\n\\includegraphics{%s}\n\n' % pdf_path
-
-def visit_inheritance_diagram(inner_func):
- """
- This is just a wrapper around html/latex_output_graph to make it
- easier to handle errors and insert warnings.
- """
- def visitor(self, node):
- try:
- content = inner_func(self, node)
- except DotException, e:
- # Insert the exception as a warning in the document
- warning = self.document.reporter.warning(str(e), line=node.line)
- warning.parent = node
- node.children = [warning]
- else:
- source = self.document.attributes['source']
- self.body.append(content)
- node.children = []
- return visitor
-
-def do_nothing(self, node):
- pass
-
-def setup(app):
- setup.app = app
- setup.confdir = app.confdir
-
- app.add_node(
- inheritance_diagram,
- latex=(visit_inheritance_diagram(latex_output_graph), do_nothing),
- html=(visit_inheritance_diagram(html_output_graph), do_nothing))
- app.add_directive(
- 'inheritance-diagram', inheritance_diagram_directive,
- False, (1, 100, 0), parts = directives.nonnegative_int)
diff --git a/doc/sphinxext/ipython_console_highlighting.py b/doc/sphinxext/ipython_console_highlighting.py
deleted file mode 100644
index 217b779..0000000
--- a/doc/sphinxext/ipython_console_highlighting.py
+++ /dev/null
@@ -1,114 +0,0 @@
-"""reST directive for syntax-highlighting ipython interactive sessions.
-
-XXX - See what improvements can be made based on the new (as of Sept 2009)
-'pycon' lexer for the python console. At the very least it will give better
-highlighted tracebacks.
-"""
-
-#-----------------------------------------------------------------------------
-# Needed modules
-
-# Standard library
-import re
-
-# Third party
-from pygments.lexer import Lexer, do_insertions
-from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer,
- PythonTracebackLexer)
-from pygments.token import Comment, Generic
-
-from sphinx import highlighting
-
-#-----------------------------------------------------------------------------
-# Global constants
-line_re = re.compile('.*?\n')
-
-#-----------------------------------------------------------------------------
-# Code begins - classes and functions
-
-class IPythonConsoleLexer(Lexer):
- """
- For IPython console output or doctests, such as:
-
- .. sourcecode:: ipython
-
- In [1]: a = 'foo'
-
- In [2]: a
- Out[2]: 'foo'
-
- In [3]: print a
- foo
-
- In [4]: 1 / 0
-
- Notes:
-
- - Tracebacks are not currently supported.
-
- - It assumes the default IPython prompts, not customized ones.
- """
-
- name = 'IPython console session'
- aliases = ['ipython']
- mimetypes = ['text/x-ipython-console']
- input_prompt = re.compile("(In \[[0-9]+\]: )|( \.\.\.+:)")
- output_prompt = re.compile("(Out\[[0-9]+\]: )|( \.\.\.+:)")
- continue_prompt = re.compile(" \.\.\.+:")
- tb_start = re.compile("\-+")
-
- def get_tokens_unprocessed(self, text):
- pylexer = PythonLexer(**self.options)
- tblexer = PythonTracebackLexer(**self.options)
-
- curcode = ''
- insertions = []
- for match in line_re.finditer(text):
- line = match.group()
- input_prompt = self.input_prompt.match(line)
- continue_prompt = self.continue_prompt.match(line.rstrip())
- output_prompt = self.output_prompt.match(line)
- if line.startswith("#"):
- insertions.append((len(curcode),
- [(0, Comment, line)]))
- elif input_prompt is not None:
- insertions.append((len(curcode),
- [(0, Generic.Prompt, input_prompt.group())]))
- curcode += line[input_prompt.end():]
- elif continue_prompt is not None:
- insertions.append((len(curcode),
- [(0, Generic.Prompt, continue_prompt.group())]))
- curcode += line[continue_prompt.end():]
- elif output_prompt is not None:
- # Use the 'error' token for output. We should probably make
- # our own token, but error is typicaly in a bright color like
- # red, so it works fine for our output prompts.
- insertions.append((len(curcode),
- [(0, Generic.Error, output_prompt.group())]))
- curcode += line[output_prompt.end():]
- else:
- if curcode:
- for item in do_insertions(insertions,
- pylexer.get_tokens_unprocessed(curcode)):
- yield item
- curcode = ''
- insertions = []
- yield match.start(), Generic.Output, line
- if curcode:
- for item in do_insertions(insertions,
- pylexer.get_tokens_unprocessed(curcode)):
- yield item
-
-
-def setup(app):
- """Setup as a sphinx extension."""
-
- # This is only a lexer, so adding it below to pygments appears sufficient.
- # But if somebody knows that the right API usage should be to do that via
- # sphinx, by all means fix it here. At least having this setup.py
- # suppresses the sphinx warning we'd get without it.
- pass
-
-#-----------------------------------------------------------------------------
-# Register the extension as a valid pygments lexer
-highlighting.lexers['ipython'] = IPythonConsoleLexer()
diff --git a/doc/sphinxext/ipython_directive.py b/doc/sphinxext/ipython_directive.py
deleted file mode 100644
index b75fdeb..0000000
--- a/doc/sphinxext/ipython_directive.py
+++ /dev/null
@@ -1,635 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Sphinx directive to support embedded IPython code.
-
-This directive allows pasting of entire interactive IPython sessions, prompts
-and all, and their code will actually get re-executed at doc build time, with
-all prompts renumbered sequentially.
-
-To enable this directive, simply list it in your Sphinx ``conf.py`` file
-(making sure the directory where you placed it is visible to sphinx, as is
-needed for all Sphinx directives).
-
-By default this directive assumes that your prompts are unchanged IPython ones,
-but this can be customized. For example, the following code in your Sphinx
-config file will configure this directive for the following input/output
-prompts ``Yade [1]:`` and ``-> [1]:``::
-
- import ipython_directive as id
- id.rgxin =re.compile(r'(?:In |Yade )\[(\d+)\]:\s?(.*)\s*')
- id.rgxout=re.compile(r'(?:Out| -> )\[(\d+)\]:\s?(.*)\s*')
- id.fmtin ='Yade [%d]:'
- id.fmtout=' -> [%d]:'
-
- from IPython import Config
- id.CONFIG = Config(
- prompt_in1="Yade [\#]:",
- prompt_in2=" .\D..",
- prompt_out=" -> [\#]:"
- )
- id.reconfig_shell()
-
- import ipython_console_highlighting as ich
- ich.IPythonConsoleLexer.input_prompt=
- re.compile("(Yade \[[0-9]+\]: )|( \.\.\.+:)")
- ich.IPythonConsoleLexer.output_prompt=
- re.compile("(( -> )|(Out)\[[0-9]+\]: )|( \.\.\.+:)")
- ich.IPythonConsoleLexer.continue_prompt=re.compile(" \.\.\.+:")
-
-
-ToDo
-----
-
-- Turn the ad-hoc test() function into a real test suite.
-- Break up ipython-specific functionality from matplotlib stuff into better
- separated code.
-- Make sure %bookmarks used internally are removed on exit.
-
-
-Authors
--------
-
-- John D Hunter: orignal author.
-- Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
-- VáclavŠmilauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
-"""
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-# Stdlib
-import cStringIO
-import os
-import re
-import sys
-
-# To keep compatibility with various python versions
-try:
- from hashlib import md5
-except ImportError:
- from md5 import md5
-
-# Third-party
-import matplotlib
-import sphinx
-from docutils.parsers.rst import directives
-
-matplotlib.use('Agg')
-
-# Our own
-from IPython import Config, InteractiveShell
-from IPython.utils.io import Term
-
-#-----------------------------------------------------------------------------
-# Globals
-#-----------------------------------------------------------------------------
-
-sphinx_version = sphinx.__version__.split(".")
-# The split is necessary for sphinx beta versions where the string is
-# '6b1'
-sphinx_version = tuple([int(re.split('[a-z]', x)[0])
- for x in sphinx_version[:2]])
-
-COMMENT, INPUT, OUTPUT = range(3)
-CONFIG = Config()
-rgxin = re.compile('In \[(\d+)\]:\s?(.*)\s*')
-rgxout = re.compile('Out\[(\d+)\]:\s?(.*)\s*')
-fmtin = 'In [%d]:'
-fmtout = 'Out[%d]:'
-
-#-----------------------------------------------------------------------------
-# Functions and class declarations
-#-----------------------------------------------------------------------------
-def block_parser(part):
- """
- part is a string of ipython text, comprised of at most one
- input, one ouput, comments, and blank lines. The block parser
- parses the text into a list of::
-
- blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]
-
- where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
- data is, depending on the type of token::
-
- COMMENT : the comment string
-
- INPUT: the (DECORATOR, INPUT_LINE, REST) where
- DECORATOR: the input decorator (or None)
- INPUT_LINE: the input as string (possibly multi-line)
- REST : any stdout generated by the input line (not OUTPUT)
-
-
- OUTPUT: the output string, possibly multi-line
- """
-
- block = []
- lines = part.split('\n')
- N = len(lines)
- i = 0
- decorator = None
- while True:
-
- if i==N:
- # nothing left to parse -- the last line
- break
-
- line = lines[i]
- i += 1
- line_stripped = line.strip()
- if line_stripped.startswith('#'):
- block.append((COMMENT, line))
- continue
-
- if line_stripped.startswith('@'):
- # we're assuming at most one decorator -- may need to
- # rethink
- decorator = line_stripped
- continue
-
- # does this look like an input line?
- matchin = rgxin.match(line)
- if matchin:
- lineno, inputline = int(matchin.group(1)), matchin.group(2)
-
- # the ....: continuation string
- continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
- Nc = len(continuation)
- # input lines can continue on for more than one line, if
- # we have a '\' line continuation char or a function call
- # echo line 'print'. The input line can only be
- # terminated by the end of the block or an output line, so
- # we parse out the rest of the input line if it is
- # multiline as well as any echo text
-
- rest = []
- while i<N:
-
- # look ahead; if the next line is blank, or a comment, or
- # an output line, we're done
-
- nextline = lines[i]
- matchout = rgxout.match(nextline)
- #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
- if matchout or nextline.startswith('#'):
- break
- elif nextline.startswith(continuation):
- inputline += '\n' + nextline[Nc:]
- else:
- rest.append(nextline)
- i+= 1
-
- block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
- continue
-
- # if it looks like an output line grab all the text to the end
- # of the block
- matchout = rgxout.match(line)
- if matchout:
- lineno, output = int(matchout.group(1)), matchout.group(2)
- if i<N-1:
- output = '\n'.join([output] + lines[i:])
-
- block.append((OUTPUT, output))
- break
-
- return block
-
-
-class EmbeddedSphinxShell(object):
- """An embedded IPython instance to run inside Sphinx"""
-
- def __init__(self):
-
- self.cout = cStringIO.StringIO()
- Term.cout = self.cout
- Term.cerr = self.cout
-
- # For debugging, so we can see normal output, use this:
- # from IPython.utils.io import Tee
- #Term.cout = Tee(self.cout, channel='stdout') # dbg
- #Term.cerr = Tee(self.cout, channel='stderr') # dbg
-
- # Create config object for IPython
- config = Config()
- config.Global.display_banner = False
- config.Global.exec_lines = ['import numpy as np',
- 'from pylab import *'
- ]
- config.InteractiveShell.autocall = False
- config.InteractiveShell.autoindent = False
- config.InteractiveShell.colors = 'NoColor'
-
- # Create and initialize ipython, but don't start its mainloop
- IP = InteractiveShell.instance(config=config)
-
- # Store a few parts of IPython we'll need.
- self.IP = IP
- self.user_ns = self.IP.user_ns
- self.user_global_ns = self.IP.user_global_ns
-
- self.input = ''
- self.output = ''
-
- self.is_verbatim = False
- self.is_doctest = False
- self.is_suppress = False
-
- # on the first call to the savefig decorator, we'll import
- # pyplot as plt so we can make a call to the plt.gcf().savefig
- self._pyplot_imported = False
-
- # we need bookmark the current dir first so we can save
- # relative to it
- self.process_input_line('bookmark ipy_basedir')
- self.cout.seek(0)
- self.cout.truncate(0)
-
- def process_input_line(self, line):
- """process the input, capturing stdout"""
- #print "input='%s'"%self.input
- stdout = sys.stdout
- try:
- sys.stdout = self.cout
- self.IP.push_line(line)
- finally:
- sys.stdout = stdout
-
- # Callbacks for each type of token
- def process_input(self, data, input_prompt, lineno):
- """Process data block for INPUT token."""
- decorator, input, rest = data
- image_file = None
- #print 'INPUT:', data # dbg
- is_verbatim = decorator=='@verbatim' or self.is_verbatim
- is_doctest = decorator=='@doctest' or self.is_doctest
- is_suppress = decorator=='@suppress' or self.is_suppress
- is_savefig = decorator is not None and \
- decorator.startswith('@savefig')
-
- input_lines = input.split('\n')
-
- continuation = ' %s:'%''.join(['.']*(len(str(lineno))+2))
- Nc = len(continuation)
-
- if is_savefig:
- saveargs = decorator.split(' ')
- filename = saveargs[1]
- outfile = os.path.join('_static/%s'%filename)
- # build out an image directive like
- # .. image:: somefile.png
- # :width 4in
- #
- # from an input like
- # savefig somefile.png width=4in
- imagerows = ['.. image:: %s'%outfile]
-
- for kwarg in saveargs[2:]:
- arg, val = kwarg.split('=')
- arg = arg.strip()
- val = val.strip()
- imagerows.append(' :%s: %s'%(arg, val))
-
- image_file = outfile
- image_directive = '\n'.join(imagerows)
-
- # TODO: can we get "rest" from ipython
- #self.process_input_line('\n'.join(input_lines))
-
- ret = []
- is_semicolon = False
-
- for i, line in enumerate(input_lines):
- if line.endswith(';'):
- is_semicolon = True
-
- if i==0:
- # process the first input line
- if is_verbatim:
- self.process_input_line('')
- else:
- # only submit the line in non-verbatim mode
- self.process_input_line(line)
- formatted_line = '%s %s'%(input_prompt, line)
- else:
- # process a continuation line
- if not is_verbatim:
- self.process_input_line(line)
-
- formatted_line = '%s %s'%(continuation, line)
-
- if not is_suppress:
- ret.append(formatted_line)
-
- if not is_suppress:
- if len(rest.strip()):
- if is_verbatim:
- # the "rest" is the standard output of the
- # input, which needs to be added in
- # verbatim mode
- ret.append(rest)
-
- self.cout.seek(0)
- output = self.cout.read()
- if not is_suppress and not is_semicolon:
- ret.append(output)
-
- self.cout.truncate(0)
- return ret, input_lines, output, is_doctest, image_file
- #print 'OUTPUT', output # dbg
-
- def process_output(self, data, output_prompt,
- input_lines, output, is_doctest, image_file):
- """Process data block for OUTPUT token."""
- if is_doctest:
- submitted = data.strip()
- found = output
- if found is not None:
- found = found.strip()
-
- # XXX - fperez: in 0.11, 'output' never comes with the prompt
- # in it, just the actual output text. So I think all this code
- # can be nuked...
- ## ind = found.find(output_prompt)
- ## if ind<0:
- ## e='output prompt="%s" does not match out line=%s' % \
- ## (output_prompt, found)
- ## raise RuntimeError(e)
- ## found = found[len(output_prompt):].strip()
-
- if found!=submitted:
- e = ('doctest failure for input_lines="%s" with '
- 'found_output="%s" and submitted output="%s"' %
- (input_lines, found, submitted) )
- raise RuntimeError(e)
- #print 'doctest PASSED for input_lines="%s" with found_output="%s" and submitted output="%s"'%(input_lines, found, submitted)
-
- def process_comment(self, data):
- """Process data block for COMMENT token."""
- if not self.is_suppress:
- return [data]
-
- def process_block(self, block):
- """
- process block from the block_parser and return a list of processed lines
- """
-
- ret = []
- output = None
- input_lines = None
-
- m = rgxin.match(str(self.IP.outputcache.prompt1).strip())
- lineno = int(m.group(1))
-
- input_prompt = fmtin%lineno
- output_prompt = fmtout%lineno
- image_file = None
- image_directive = None
- # XXX - This needs a second refactor. There's too much state being
- # held globally, which makes for a very awkward interface and large,
- # hard to test functions. I've already broken this up at least into
- # three separate processors to isolate the logic better, but this only
- # serves to highlight the coupling. Next we need to clean it up...
- for token, data in block:
- if token==COMMENT:
- out_data = self.process_comment(data)
- elif token==INPUT:
- out_data, input_lines, output, is_doctest, image_file= \
- self.process_input(data, input_prompt, lineno)
- elif token==OUTPUT:
- out_data = \
- self.process_output(data, output_prompt,
- input_lines, output, is_doctest,
- image_file)
- if out_data:
- ret.extend(out_data)
-
- if image_file is not None:
- self.ensure_pyplot()
- command = 'plt.gcf().savefig("%s")'%image_file
- print 'SAVEFIG', command # dbg
- self.process_input_line('bookmark ipy_thisdir')
- self.process_input_line('cd -b ipy_basedir')
- self.process_input_line(command)
- self.process_input_line('cd -b ipy_thisdir')
- self.cout.seek(0)
- self.cout.truncate(0)
- return ret, image_directive
-
- def ensure_pyplot(self):
- if self._pyplot_imported:
- return
- self.process_input_line('import matplotlib.pyplot as plt')
-
-# A global instance used below. XXX: not sure why this can't be created inside
-# ipython_directive itself.
-shell = EmbeddedSphinxShell()
-
-def reconfig_shell():
- """Called after setting module-level variables to re-instantiate
- with the set values (since shell is instantiated first at import-time
- when module variables have default values)"""
- global shell
- shell = EmbeddedSphinxShell()
-
-
-def ipython_directive(name, arguments, options, content, lineno,
- content_offset, block_text, state, state_machine,
- ):
-
- debug = ipython_directive.DEBUG
- shell.is_suppress = options.has_key('suppress')
- shell.is_doctest = options.has_key('doctest')
- shell.is_verbatim = options.has_key('verbatim')
-
- #print 'ipy', shell.is_suppress, options
- parts = '\n'.join(content).split('\n\n')
- lines = ['.. sourcecode:: ipython', '']
-
- figures = []
- for part in parts:
- block = block_parser(part)
-
- if len(block):
- rows, figure = shell.process_block(block)
- for row in rows:
- lines.extend([' %s'%line for line in row.split('\n')])
-
- if figure is not None:
- figures.append(figure)
-
- for figure in figures:
- lines.append('')
- lines.extend(figure.split('\n'))
- lines.append('')
-
- #print lines
- if len(lines)>2:
- if debug:
- print '\n'.join(lines)
- else:
- #print 'INSERTING %d lines'%len(lines)
- state_machine.insert_input(
- lines, state_machine.input_lines.source(0))
-
- return []
-
-ipython_directive.DEBUG = False
-ipython_directive.DEBUG = True # dbg
-
-# Enable as a proper Sphinx directive
-def setup(app):
- setup.app = app
- options = {'suppress': directives.flag,
- 'doctest': directives.flag,
- 'verbatim': directives.flag,
- }
-
- app.add_directive('ipython', ipython_directive, True, (0, 2, 0), **options)
-
-
-# Simple smoke test, needs to be converted to a proper automatic test.
-def test():
-
- examples = [
- r"""
-In [9]: pwd
-Out[9]: '/home/jdhunter/py4science/book'
-
-In [10]: cd bookdata/
-/home/jdhunter/py4science/book/bookdata
-
-In [2]: from pylab import *
-
-In [2]: ion()
-
-In [3]: im = imread('stinkbug.png')
-
- at savefig mystinkbug.png width=4in
-In [4]: imshow(im)
-Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>
-
-""",
- r"""
-
-In [1]: x = 'hello world'
-
-# string methods can be
-# used to alter the string
- at doctest
-In [2]: x.upper()
-Out[2]: 'HELLO WORLD'
-
- at verbatim
-In [3]: x.st<TAB>
-x.startswith x.strip
-""",
- r"""
-
-In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
- .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'
-
-In [131]: print url.split('&')
-['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']
-
-In [60]: import urllib
-
-""",
- r"""\
-
-In [133]: import numpy.random
-
- at suppress
-In [134]: numpy.random.seed(2358)
-
- at doctest
-In [135]: np.random.rand(10,2)
-Out[135]:
-array([[ 0.64524308, 0.59943846],
- [ 0.47102322, 0.8715456 ],
- [ 0.29370834, 0.74776844],
- [ 0.99539577, 0.1313423 ],
- [ 0.16250302, 0.21103583],
- [ 0.81626524, 0.1312433 ],
- [ 0.67338089, 0.72302393],
- [ 0.7566368 , 0.07033696],
- [ 0.22591016, 0.77731835],
- [ 0.0072729 , 0.34273127]])
-
-""",
-
- r"""
-In [106]: print x
-jdh
-
-In [109]: for i in range(10):
- .....: print i
- .....:
- .....:
-0
-1
-2
-3
-4
-5
-6
-7
-8
-9
-""",
-
- r"""
-
-In [144]: from pylab import *
-
-In [145]: ion()
-
-# use a semicolon to suppress the output
- at savefig test_hist.png width=4in
-In [151]: hist(np.random.randn(10000), 100);
-
-
- at savefig test_plot.png width=4in
-In [151]: plot(np.random.randn(10000), 'o');
- """,
-
- r"""
-# use a semicolon to suppress the output
-In [151]: plt.clf()
-
- at savefig plot_simple.png width=4in
-In [151]: plot([1,2,3])
-
- at savefig hist_simple.png width=4in
-In [151]: hist(np.random.randn(10000), 100);
-
-""",
- r"""
-# update the current fig
-In [151]: ylabel('number')
-
-In [152]: title('normal distribution')
-
-
- at savefig hist_with_text.png
-In [153]: grid(True)
-
- """,
- ]
-
- #ipython_directive.DEBUG = True # dbg
- #options = dict(suppress=True) # dbg
- options = dict()
- for example in examples:
- content = example.split('\n')
- ipython_directive('debug', arguments=None, options=options,
- content=content, lineno=0,
- content_offset=None, block_text=None,
- state=None, state_machine=None,
- )
-
-# Run test suite as a script
-if __name__=='__main__':
- if not os.path.isdir('_static'):
- os.mkdir('_static')
- test()
- print 'All OK? Check figures in _static/'
diff --git a/doc/sphinxext/numpydoc.py b/doc/sphinxext/numpydoc.py
deleted file mode 100644
index 43c6733..0000000
--- a/doc/sphinxext/numpydoc.py
+++ /dev/null
@@ -1,169 +0,0 @@
-"""
-========
-numpydoc
-========
-
-Sphinx extension that handles docstrings in the Numpy standard format. [1]
-
-It will:
-
-- Convert Parameters etc. sections to field lists.
-- Convert See Also section to a See also entry.
-- Renumber references.
-- Extract the signature from the docstring, if it can't be determined otherwise.
-
-.. [1] http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard
-
-"""
-
-import sphinx
-
-if sphinx.__version__ < '1.0.1':
- raise RuntimeError("Sphinx 1.0.1 or newer is required")
-
-import os, re, pydoc
-from docscrape_sphinx import get_doc_object, SphinxDocString
-from sphinx.util.compat import Directive
-import inspect
-
-def mangle_docstrings(app, what, name, obj, options, lines,
- reference_offset=[0]):
-
- cfg = dict(use_plots=app.config.numpydoc_use_plots,
- show_class_members=app.config.numpydoc_show_class_members)
-
- if what == 'module':
- # Strip top title
- title_re = re.compile(ur'^\s*[#*=]{4,}\n[a-z0-9 -]+\n[#*=]{4,}\s*',
- re.I|re.S)
- lines[:] = title_re.sub(u'', u"\n".join(lines)).split(u"\n")
- else:
- doc = get_doc_object(obj, what, u"\n".join(lines), config=cfg)
- lines[:] = unicode(doc).split(u"\n")
-
- if app.config.numpydoc_edit_link and hasattr(obj, '__name__') and \
- obj.__name__:
- if hasattr(obj, '__module__'):
- v = dict(full_name=u"%s.%s" % (obj.__module__, obj.__name__))
- else:
- v = dict(full_name=obj.__name__)
- lines += [u'', u'.. htmlonly::', '']
- lines += [u' %s' % x for x in
- (app.config.numpydoc_edit_link % v).split("\n")]
-
- # replace reference numbers so that there are no duplicates
- references = []
- for line in lines:
- line = line.strip()
- m = re.match(ur'^.. \[([a-z0-9_.-])\]', line, re.I)
- if m:
- references.append(m.group(1))
-
- # start renaming from the longest string, to avoid overwriting parts
- references.sort(key=lambda x: -len(x))
- if references:
- for i, line in enumerate(lines):
- for r in references:
- if re.match(ur'^\d+$', r):
- new_r = u"R%d" % (reference_offset[0] + int(r))
- else:
- new_r = u"%s%d" % (r, reference_offset[0])
- lines[i] = lines[i].replace(u'[%s]_' % r,
- u'[%s]_' % new_r)
- lines[i] = lines[i].replace(u'.. [%s]' % r,
- u'.. [%s]' % new_r)
-
- reference_offset[0] += len(references)
-
-def mangle_signature(app, what, name, obj, options, sig, retann):
- # Do not try to inspect classes that don't define `__init__`
- if (inspect.isclass(obj) and
- (not hasattr(obj, '__init__') or
- 'initializes x; see ' in pydoc.getdoc(obj.__init__))):
- return '', ''
-
- if not (callable(obj) or hasattr(obj, '__argspec_is_invalid_')): return
- if not hasattr(obj, '__doc__'): return
-
- doc = SphinxDocString(pydoc.getdoc(obj))
- if doc['Signature']:
- sig = re.sub(u"^[^(]*", u"", doc['Signature'])
- return sig, u''
-
-def setup(app, get_doc_object_=get_doc_object):
- global get_doc_object
- get_doc_object = get_doc_object_
-
- app.connect('autodoc-process-docstring', mangle_docstrings)
- app.connect('autodoc-process-signature', mangle_signature)
- app.add_config_value('numpydoc_edit_link', None, False)
- app.add_config_value('numpydoc_use_plots', None, False)
- app.add_config_value('numpydoc_show_class_members', True, True)
-
- # Extra mangling domains
- app.add_domain(NumpyPythonDomain)
- app.add_domain(NumpyCDomain)
-
-#------------------------------------------------------------------------------
-# Docstring-mangling domains
-#------------------------------------------------------------------------------
-
-from docutils.statemachine import ViewList
-from sphinx.domains.c import CDomain
-from sphinx.domains.python import PythonDomain
-
-class ManglingDomainBase(object):
- directive_mangling_map = {}
-
- def __init__(self, *a, **kw):
- super(ManglingDomainBase, self).__init__(*a, **kw)
- self.wrap_mangling_directives()
-
- def wrap_mangling_directives(self):
- for name, objtype in self.directive_mangling_map.items():
- self.directives[name] = wrap_mangling_directive(
- self.directives[name], objtype)
-
-class NumpyPythonDomain(ManglingDomainBase, PythonDomain):
- name = 'np'
- directive_mangling_map = {
- 'function': 'function',
- 'class': 'class',
- 'exception': 'class',
- 'method': 'function',
- 'classmethod': 'function',
- 'staticmethod': 'function',
- 'attribute': 'attribute',
- }
-
-class NumpyCDomain(ManglingDomainBase, CDomain):
- name = 'np-c'
- directive_mangling_map = {
- 'function': 'function',
- 'member': 'attribute',
- 'macro': 'function',
- 'type': 'class',
- 'var': 'object',
- }
-
-def wrap_mangling_directive(base_directive, objtype):
- class directive(base_directive):
- def run(self):
- env = self.state.document.settings.env
-
- name = None
- if self.arguments:
- m = re.match(r'^(.*\s+)?(.*?)(\(.*)?', self.arguments[0])
- name = m.group(2).strip()
-
- if not name:
- name = self.arguments[0]
-
- lines = list(self.content)
- mangle_docstrings(env.app, objtype, name, None, None, lines)
- self.content = ViewList(lines, self.content.parent)
-
- return base_directive.run(self)
-
- return directive
-
diff --git a/doc/sphinxext/phantom_import.py b/doc/sphinxext/phantom_import.py
deleted file mode 100644
index c77eeb5..0000000
--- a/doc/sphinxext/phantom_import.py
+++ /dev/null
@@ -1,162 +0,0 @@
-"""
-==============
-phantom_import
-==============
-
-Sphinx extension to make directives from ``sphinx.ext.autodoc`` and similar
-extensions to use docstrings loaded from an XML file.
-
-This extension loads an XML file in the Pydocweb format [1] and
-creates a dummy module that contains the specified docstrings. This
-can be used to get the current docstrings from a Pydocweb instance
-without needing to rebuild the documented module.
-
-.. [1] http://code.google.com/p/pydocweb
-
-"""
-import imp, sys, compiler, types, os, inspect, re
-
-def setup(app):
- app.connect('builder-inited', initialize)
- app.add_config_value('phantom_import_file', None, True)
-
-def initialize(app):
- fn = app.config.phantom_import_file
- if (fn and os.path.isfile(fn)):
- print "[numpydoc] Phantom importing modules from", fn, "..."
- import_phantom_module(fn)
-
-#------------------------------------------------------------------------------
-# Creating 'phantom' modules from an XML description
-#------------------------------------------------------------------------------
-def import_phantom_module(xml_file):
- """
- Insert a fake Python module to sys.modules, based on a XML file.
-
- The XML file is expected to conform to Pydocweb DTD. The fake
- module will contain dummy objects, which guarantee the following:
-
- - Docstrings are correct.
- - Class inheritance relationships are correct (if present in XML).
- - Function argspec is *NOT* correct (even if present in XML).
- Instead, the function signature is prepended to the function docstring.
- - Class attributes are *NOT* correct; instead, they are dummy objects.
-
- Parameters
- ----------
- xml_file : str
- Name of an XML file to read
-
- """
- import lxml.etree as etree
-
- object_cache = {}
-
- tree = etree.parse(xml_file)
- root = tree.getroot()
-
- # Sort items so that
- # - Base classes come before classes inherited from them
- # - Modules come before their contents
- all_nodes = dict([(n.attrib['id'], n) for n in root])
-
- def _get_bases(node, recurse=False):
- bases = [x.attrib['ref'] for x in node.findall('base')]
- if recurse:
- j = 0
- while True:
- try:
- b = bases[j]
- except IndexError: break
- if b in all_nodes:
- bases.extend(_get_bases(all_nodes[b]))
- j += 1
- return bases
-
- type_index = ['module', 'class', 'callable', 'object']
-
- def base_cmp(a, b):
- x = cmp(type_index.index(a.tag), type_index.index(b.tag))
- if x != 0: return x
-
- if a.tag == 'class' and b.tag == 'class':
- a_bases = _get_bases(a, recurse=True)
- b_bases = _get_bases(b, recurse=True)
- x = cmp(len(a_bases), len(b_bases))
- if x != 0: return x
- if a.attrib['id'] in b_bases: return -1
- if b.attrib['id'] in a_bases: return 1
-
- return cmp(a.attrib['id'].count('.'), b.attrib['id'].count('.'))
-
- nodes = root.getchildren()
- nodes.sort(base_cmp)
-
- # Create phantom items
- for node in nodes:
- name = node.attrib['id']
- doc = (node.text or '').decode('string-escape') + "\n"
- if doc == "\n": doc = ""
-
- # create parent, if missing
- parent = name
- while True:
- parent = '.'.join(parent.split('.')[:-1])
- if not parent: break
- if parent in object_cache: break
- obj = imp.new_module(parent)
- object_cache[parent] = obj
- sys.modules[parent] = obj
-
- # create object
- if node.tag == 'module':
- obj = imp.new_module(name)
- obj.__doc__ = doc
- sys.modules[name] = obj
- elif node.tag == 'class':
- bases = [object_cache[b] for b in _get_bases(node)
- if b in object_cache]
- bases.append(object)
- init = lambda self: None
- init.__doc__ = doc
- obj = type(name, tuple(bases), {'__doc__': doc, '__init__': init})
- obj.__name__ = name.split('.')[-1]
- elif node.tag == 'callable':
- funcname = node.attrib['id'].split('.')[-1]
- argspec = node.attrib.get('argspec')
- if argspec:
- argspec = re.sub('^[^(]*', '', argspec)
- doc = "%s%s\n\n%s" % (funcname, argspec, doc)
- obj = lambda: 0
- obj.__argspec_is_invalid_ = True
- obj.func_name = funcname
- obj.__name__ = name
- obj.__doc__ = doc
- if inspect.isclass(object_cache[parent]):
- obj.__objclass__ = object_cache[parent]
- else:
- class Dummy(object): pass
- obj = Dummy()
- obj.__name__ = name
- obj.__doc__ = doc
- if inspect.isclass(object_cache[parent]):
- obj.__get__ = lambda: None
- object_cache[name] = obj
-
- if parent:
- if inspect.ismodule(object_cache[parent]):
- obj.__module__ = parent
- setattr(object_cache[parent], name.split('.')[-1], obj)
-
- # Populate items
- for node in root:
- obj = object_cache.get(node.attrib['id'])
- if obj is None: continue
- for ref in node.findall('ref'):
- if node.tag == 'class':
- if ref.attrib['ref'].startswith(node.attrib['id'] + '.'):
- setattr(obj, ref.attrib['name'],
- object_cache.get(ref.attrib['ref']))
- else:
- setattr(obj, ref.attrib['name'],
- object_cache.get(ref.attrib['ref']))
diff --git a/doc/sphinxext/plot_directive.py b/doc/sphinxext/plot_directive.py
deleted file mode 100644
index 80801e7..0000000
--- a/doc/sphinxext/plot_directive.py
+++ /dev/null
@@ -1,636 +0,0 @@
-"""
-A special directive for generating a matplotlib plot.
-
-.. warning::
-
- This is a hacked version of plot_directive.py from Matplotlib.
- It's very much subject to change!
-
-
-Usage
------
-
-Can be used like this::
-
- .. plot:: examples/example.py
-
- .. plot::
-
- import matplotlib.pyplot as plt
- plt.plot([1,2,3], [4,5,6])
-
- .. plot::
-
- A plotting example:
-
- >>> import matplotlib.pyplot as plt
- >>> plt.plot([1,2,3], [4,5,6])
-
-The content is interpreted as doctest formatted if it has a line starting
-with ``>>>``.
-
-The ``plot`` directive supports the options
-
- format : {'python', 'doctest'}
- Specify the format of the input
-
- include-source : bool
- Whether to display the source code. Default can be changed in conf.py
-
-and the ``image`` directive options ``alt``, ``height``, ``width``,
-``scale``, ``align``, ``class``.
-
-Configuration options
----------------------
-
-The plot directive has the following configuration options:
-
- plot_include_source
- Default value for the include-source option
-
- plot_pre_code
- Code that should be executed before each plot.
-
- plot_basedir
- Base directory, to which plot:: file names are relative to.
- (If None or empty, file names are relative to the directoly where
- the file containing the directive is.)
-
- plot_formats
- File formats to generate. List of tuples or strings::
-
- [(suffix, dpi), suffix, ...]
-
- that determine the file format and the DPI. For entries whose
- DPI was omitted, sensible defaults are chosen.
-
- plot_html_show_formats
- Whether to show links to the files in HTML.
-
-TODO
-----
-
-* Refactor Latex output; now it's plain images, but it would be nice
- to make them appear side-by-side, or in floats.
-
-"""
-
-import sys, os, glob, shutil, imp, warnings, cStringIO, re, textwrap, traceback
-import sphinx
-
-import warnings
-warnings.warn("A plot_directive module is also available under "
- "matplotlib.sphinxext; expect this numpydoc.plot_directive "
- "module to be deprecated after relevant features have been "
- "integrated there.",
- FutureWarning, stacklevel=2)
-
-
-#------------------------------------------------------------------------------
-# Registration hook
-#------------------------------------------------------------------------------
-
-def setup(app):
- setup.app = app
- setup.config = app.config
- setup.confdir = app.confdir
-
- app.add_config_value('plot_pre_code', '', True)
- app.add_config_value('plot_include_source', False, True)
- app.add_config_value('plot_formats', ['png', 'hires.png', 'pdf'], True)
- app.add_config_value('plot_basedir', None, True)
- app.add_config_value('plot_html_show_formats', True, True)
-
- app.add_directive('plot', plot_directive, True, (0, 1, False),
- **plot_directive_options)
-
-#------------------------------------------------------------------------------
-# plot:: directive
-#------------------------------------------------------------------------------
-from docutils.parsers.rst import directives
-from docutils import nodes
-
-def plot_directive(name, arguments, options, content, lineno,
- content_offset, block_text, state, state_machine):
- return run(arguments, content, options, state_machine, state, lineno)
-plot_directive.__doc__ = __doc__
-
-def _option_boolean(arg):
- if not arg or not arg.strip():
- # no argument given, assume used as a flag
- return True
- elif arg.strip().lower() in ('no', '0', 'false'):
- return False
- elif arg.strip().lower() in ('yes', '1', 'true'):
- return True
- else:
- raise ValueError('"%s" unknown boolean' % arg)
-
-def _option_format(arg):
- return directives.choice(arg, ('python', 'lisp'))
-
-def _option_align(arg):
- return directives.choice(arg, ("top", "middle", "bottom", "left", "center",
- "right"))
-
-plot_directive_options = {'alt': directives.unchanged,
- 'height': directives.length_or_unitless,
- 'width': directives.length_or_percentage_or_unitless,
- 'scale': directives.nonnegative_int,
- 'align': _option_align,
- 'class': directives.class_option,
- 'include-source': _option_boolean,
- 'format': _option_format,
- }
-
-#------------------------------------------------------------------------------
-# Generating output
-#------------------------------------------------------------------------------
-
-from docutils import nodes, utils
-
-try:
- # Sphinx depends on either Jinja or Jinja2
- import jinja2
- def format_template(template, **kw):
- return jinja2.Template(template).render(**kw)
-except ImportError:
- import jinja
- def format_template(template, **kw):
- return jinja.from_string(template, **kw)
-
-TEMPLATE = """
-{{ source_code }}
-
-{{ only_html }}
-
- {% if source_link or (html_show_formats and not multi_image) %}
- (
- {%- if source_link -%}
- `Source code <{{ source_link }}>`__
- {%- endif -%}
- {%- if html_show_formats and not multi_image -%}
- {%- for img in images -%}
- {%- for fmt in img.formats -%}
- {%- if source_link or not loop.first -%}, {% endif -%}
- `{{ fmt }} <{{ dest_dir }}/{{ img.basename }}.{{ fmt }}>`__
- {%- endfor -%}
- {%- endfor -%}
- {%- endif -%}
- )
- {% endif %}
-
- {% for img in images %}
- .. figure:: {{ build_dir }}/{{ img.basename }}.png
- {%- for option in options %}
- {{ option }}
- {% endfor %}
-
- {% if html_show_formats and multi_image -%}
- (
- {%- for fmt in img.formats -%}
- {%- if not loop.first -%}, {% endif -%}
- `{{ fmt }} <{{ dest_dir }}/{{ img.basename }}.{{ fmt }}>`__
- {%- endfor -%}
- )
- {%- endif -%}
- {% endfor %}
-
-{{ only_latex }}
-
- {% for img in images %}
- .. image:: {{ build_dir }}/{{ img.basename }}.pdf
- {% endfor %}
-
-"""
-
-class ImageFile(object):
- def __init__(self, basename, dirname):
- self.basename = basename
- self.dirname = dirname
- self.formats = []
-
- def filename(self, format):
- return os.path.join(self.dirname, "%s.%s" % (self.basename, format))
-
- def filenames(self):
- return [self.filename(fmt) for fmt in self.formats]
-
-def run(arguments, content, options, state_machine, state, lineno):
- if arguments and content:
- raise RuntimeError("plot:: directive can't have both args and content")
-
- document = state_machine.document
- config = document.settings.env.config
-
- options.setdefault('include-source', config.plot_include_source)
-
- # determine input
- rst_file = document.attributes['source']
- rst_dir = os.path.dirname(rst_file)
-
- if arguments:
- if not config.plot_basedir:
- source_file_name = os.path.join(rst_dir,
- directives.uri(arguments[0]))
- else:
- source_file_name = os.path.join(setup.confdir, config.plot_basedir,
- directives.uri(arguments[0]))
- code = open(source_file_name, 'r').read()
- output_base = os.path.basename(source_file_name)
- else:
- source_file_name = rst_file
- code = textwrap.dedent("\n".join(map(str, content)))
- counter = document.attributes.get('_plot_counter', 0) + 1
- document.attributes['_plot_counter'] = counter
- base, ext = os.path.splitext(os.path.basename(source_file_name))
- output_base = '%s-%d.py' % (base, counter)
-
- base, source_ext = os.path.splitext(output_base)
- if source_ext in ('.py', '.rst', '.txt'):
- output_base = base
- else:
- source_ext = ''
-
- # ensure that LaTeX includegraphics doesn't choke in foo.bar.pdf filenames
- output_base = output_base.replace('.', '-')
-
- # is it in doctest format?
- is_doctest = contains_doctest(code)
- if options.has_key('format'):
- if options['format'] == 'python':
- is_doctest = False
- else:
- is_doctest = True
-
- # determine output directory name fragment
- source_rel_name = relpath(source_file_name, setup.confdir)
- source_rel_dir = os.path.dirname(source_rel_name)
- while source_rel_dir.startswith(os.path.sep):
- source_rel_dir = source_rel_dir[1:]
-
- # build_dir: where to place output files (temporarily)
- build_dir = os.path.join(os.path.dirname(setup.app.doctreedir),
- 'plot_directive',
- source_rel_dir)
- if not os.path.exists(build_dir):
- os.makedirs(build_dir)
-
- # output_dir: final location in the builder's directory
- dest_dir = os.path.abspath(os.path.join(setup.app.builder.outdir,
- source_rel_dir))
-
- # how to link to files from the RST file
- dest_dir_link = os.path.join(relpath(setup.confdir, rst_dir),
- source_rel_dir).replace(os.path.sep, '/')
- build_dir_link = relpath(build_dir, rst_dir).replace(os.path.sep, '/')
- source_link = dest_dir_link + '/' + output_base + source_ext
-
- # make figures
- try:
- results = makefig(code, source_file_name, build_dir, output_base,
- config)
- errors = []
- except PlotError, err:
- reporter = state.memo.reporter
- sm = reporter.system_message(
- 2, "Exception occurred in plotting %s: %s" % (output_base, err),
- line=lineno)
- results = [(code, [])]
- errors = [sm]
-
- # generate output restructuredtext
- total_lines = []
- for j, (code_piece, images) in enumerate(results):
- if options['include-source']:
- if is_doctest:
- lines = ['']
- lines += [row.rstrip() for row in code_piece.split('\n')]
- else:
- lines = ['.. code-block:: python', '']
- lines += [' %s' % row.rstrip()
- for row in code_piece.split('\n')]
- source_code = "\n".join(lines)
- else:
- source_code = ""
-
- opts = [':%s: %s' % (key, val) for key, val in options.items()
- if key in ('alt', 'height', 'width', 'scale', 'align', 'class')]
-
- only_html = ".. only:: html"
- only_latex = ".. only:: latex"
-
- if j == 0:
- src_link = source_link
- else:
- src_link = None
-
- result = format_template(
- TEMPLATE,
- dest_dir=dest_dir_link,
- build_dir=build_dir_link,
- source_link=src_link,
- multi_image=len(images) > 1,
- only_html=only_html,
- only_latex=only_latex,
- options=opts,
- images=images,
- source_code=source_code,
- html_show_formats=config.plot_html_show_formats)
-
- total_lines.extend(result.split("\n"))
- total_lines.extend("\n")
-
- if total_lines:
- state_machine.insert_input(total_lines, source=source_file_name)
-
- # copy image files to builder's output directory
- if not os.path.exists(dest_dir):
- os.makedirs(dest_dir)
-
- for code_piece, images in results:
- for img in images:
- for fn in img.filenames():
- shutil.copyfile(fn, os.path.join(dest_dir,
- os.path.basename(fn)))
-
- # copy script (if necessary)
- if source_file_name == rst_file:
- target_name = os.path.join(dest_dir, output_base + source_ext)
- f = open(target_name, 'w')
- f.write(unescape_doctest(code))
- f.close()
-
- return errors
-
-
-#------------------------------------------------------------------------------
-# Run code and capture figures
-#------------------------------------------------------------------------------
-
-import matplotlib
-matplotlib.use('Agg')
-import matplotlib.pyplot as plt
-import matplotlib.image as image
-from matplotlib import _pylab_helpers
-
-import exceptions
-
-def contains_doctest(text):
- try:
- # check if it's valid Python as-is
- compile(text, '<string>', 'exec')
- return False
- except SyntaxError:
- pass
- r = re.compile(r'^\s*>>>', re.M)
- m = r.search(text)
- return bool(m)
-
-def unescape_doctest(text):
- """
- Extract code from a piece of text, which contains either Python code
- or doctests.
-
- """
- if not contains_doctest(text):
- return text
-
- code = ""
- for line in text.split("\n"):
- m = re.match(r'^\s*(>>>|\.\.\.) (.*)$', line)
- if m:
- code += m.group(2) + "\n"
- elif line.strip():
- code += "# " + line.strip() + "\n"
- else:
- code += "\n"
- return code
-
-def split_code_at_show(text):
- """
- Split code at plt.show()
-
- """
-
- parts = []
- is_doctest = contains_doctest(text)
-
- part = []
- for line in text.split("\n"):
- if (not is_doctest and line.strip() == 'plt.show()') or \
- (is_doctest and line.strip() == '>>> plt.show()'):
- part.append(line)
- parts.append("\n".join(part))
- part = []
- else:
- part.append(line)
- if "\n".join(part).strip():
- parts.append("\n".join(part))
- return parts
-
-class PlotError(RuntimeError):
- pass
-
-def run_code(code, code_path, ns=None):
- # Change the working directory to the directory of the example, so
- # it can get at its data files, if any.
- pwd = os.getcwd()
- old_sys_path = list(sys.path)
- if code_path is not None:
- dirname = os.path.abspath(os.path.dirname(code_path))
- os.chdir(dirname)
- sys.path.insert(0, dirname)
-
- # Redirect stdout
- stdout = sys.stdout
- sys.stdout = cStringIO.StringIO()
-
- # Reset sys.argv
- old_sys_argv = sys.argv
- sys.argv = [code_path]
-
- try:
- try:
- code = unescape_doctest(code)
- if ns is None:
- ns = {}
- if not ns:
- exec setup.config.plot_pre_code in ns
- exec code in ns
- except (Exception, SystemExit), err:
- raise PlotError(traceback.format_exc())
- finally:
- os.chdir(pwd)
- sys.argv = old_sys_argv
- sys.path[:] = old_sys_path
- sys.stdout = stdout
- return ns
-
-
-#------------------------------------------------------------------------------
-# Generating figures
-#------------------------------------------------------------------------------
-
-def out_of_date(original, derived):
- """
- Returns True if derivative is out-of-date wrt original,
- both of which are full file paths.
- """
- return (not os.path.exists(derived)
- or os.stat(derived).st_mtime < os.stat(original).st_mtime)
-
-
-def makefig(code, code_path, output_dir, output_base, config):
- """
- Run a pyplot script *code* and save the images under *output_dir*
- with file names derived from *output_base*
-
- """
-
- # -- Parse format list
- default_dpi = {'png': 80, 'hires.png': 200, 'pdf': 50}
- formats = []
- for fmt in config.plot_formats:
- if isinstance(fmt, str):
- formats.append((fmt, default_dpi.get(fmt, 80)))
- elif type(fmt) in (tuple, list) and len(fmt)==2:
- formats.append((str(fmt[0]), int(fmt[1])))
- else:
- raise PlotError('invalid image format "%r" in plot_formats' % fmt)
-
- # -- Try to determine if all images already exist
-
- code_pieces = split_code_at_show(code)
-
- # Look for single-figure output files first
- all_exists = True
- img = ImageFile(output_base, output_dir)
- for format, dpi in formats:
- if out_of_date(code_path, img.filename(format)):
- all_exists = False
- break
- img.formats.append(format)
-
- if all_exists:
- return [(code, [img])]
-
- # Then look for multi-figure output files
- results = []
- all_exists = True
- for i, code_piece in enumerate(code_pieces):
- images = []
- for j in xrange(1000):
- img = ImageFile('%s_%02d_%02d' % (output_base, i, j), output_dir)
- for format, dpi in formats:
- if out_of_date(code_path, img.filename(format)):
- all_exists = False
- break
- img.formats.append(format)
-
- # assume that if we have one, we have them all
- if not all_exists:
- all_exists = (j > 0)
- break
- images.append(img)
- if not all_exists:
- break
- results.append((code_piece, images))
-
- if all_exists:
- return results
-
- # -- We didn't find the files, so build them
-
- results = []
- ns = {}
-
- for i, code_piece in enumerate(code_pieces):
- # Clear between runs
- plt.close('all')
-
- # Run code
- run_code(code_piece, code_path, ns)
-
- # Collect images
- images = []
- fig_managers = _pylab_helpers.Gcf.get_all_fig_managers()
- for j, figman in enumerate(fig_managers):
- if len(fig_managers) == 1 and len(code_pieces) == 1:
- img = ImageFile(output_base, output_dir)
- else:
- img = ImageFile("%s_%02d_%02d" % (output_base, i, j),
- output_dir)
- images.append(img)
- for format, dpi in formats:
- try:
- figman.canvas.figure.savefig(img.filename(format), dpi=dpi)
- except exceptions.BaseException, err:
- raise PlotError(traceback.format_exc())
- img.formats.append(format)
-
- # Results
- results.append((code_piece, images))
-
- return results
-
-
-#------------------------------------------------------------------------------
-# Relative pathnames
-#------------------------------------------------------------------------------
-
-try:
- from os.path import relpath
-except ImportError:
- # Copied from Python 2.7
- if 'posix' in sys.builtin_module_names:
- def relpath(path, start=os.path.curdir):
- """Return a relative version of a path"""
- from os.path import sep, curdir, join, abspath, commonprefix, \
- pardir
-
- if not path:
- raise ValueError("no path specified")
-
- start_list = abspath(start).split(sep)
- path_list = abspath(path).split(sep)
-
- # Work out how much of the filepath is shared by start and path.
- i = len(commonprefix([start_list, path_list]))
-
- rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
- if not rel_list:
- return curdir
- return join(*rel_list)
- elif 'nt' in sys.builtin_module_names:
- def relpath(path, start=os.path.curdir):
- """Return a relative version of a path"""
- from os.path import sep, curdir, join, abspath, commonprefix, \
- pardir, splitunc
-
- if not path:
- raise ValueError("no path specified")
- start_list = abspath(start).split(sep)
- path_list = abspath(path).split(sep)
- if start_list[0].lower() != path_list[0].lower():
- unc_path, rest = splitunc(path)
- unc_start, rest = splitunc(start)
- if bool(unc_path) ^ bool(unc_start):
- raise ValueError("Cannot mix UNC and non-UNC paths (%s and %s)"
- % (path, start))
- else:
- raise ValueError("path is on drive %s, start on drive %s"
- % (path_list[0], start_list[0]))
- # Work out how much of the filepath is shared by start and path.
- for i in range(min(len(start_list), len(path_list))):
- if start_list[i].lower() != path_list[i].lower():
- break
- else:
- i += 1
-
- rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
- if not rel_list:
- return curdir
- return join(*rel_list)
- else:
- raise RuntimeError("Unsupported platform (no relpath available!)")
diff --git a/doc/sphinxext/setup.py b/doc/sphinxext/setup.py
deleted file mode 100644
index 76e3fd8..0000000
--- a/doc/sphinxext/setup.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from distutils.core import setup
-import setuptools
-import sys, os
-
-version = "0.4"
-
-setup(
- name="numpydoc",
- packages=["numpydoc"],
- package_dir={"numpydoc": ""},
- version=version,
- description="Sphinx extension to support docstrings in Numpy format",
- # classifiers from http://pypi.python.org/pypi?%3Aaction=list_classifiers
- classifiers=["Development Status :: 3 - Alpha",
- "Environment :: Plugins",
- "License :: OSI Approved :: BSD License",
- "Topic :: Documentation"],
- keywords="sphinx numpy",
- author="Pauli Virtanen and others",
- author_email="pav at iki.fi",
- url="http://github.com/numpy/numpy/tree/master/doc/sphinxext",
- license="BSD",
- zip_safe=False,
- install_requires=["Sphinx >= 1.0.1"],
- package_data={'numpydoc': 'tests', '': ''},
- entry_points={
- "console_scripts": [
- "autosummary_generate = numpydoc.autosummary_generate:main",
- ],
- },
-)
diff --git a/doc/sphinxext/traitsdoc.py b/doc/sphinxext/traitsdoc.py
deleted file mode 100644
index 66dbe69..0000000
--- a/doc/sphinxext/traitsdoc.py
+++ /dev/null
@@ -1,140 +0,0 @@
-"""
-=========
-traitsdoc
-=========
-
-Sphinx extension that handles docstrings in the Numpy standard format, [1]
-and support Traits [2].
-
-This extension can be used as a replacement for ``numpydoc`` when support
-for Traits is required.
-
-.. [1] http://projects.scipy.org/numpy/wiki/CodingStyleGuidelines#docstring-standard
-.. [2] http://code.enthought.com/projects/traits/
-
-"""
-
-import inspect
-import os
-import pydoc
-
-import docscrape
-import docscrape_sphinx
-from docscrape_sphinx import SphinxClassDoc, SphinxFunctionDoc, SphinxDocString
-
-import numpydoc
-
-import comment_eater
-
-class SphinxTraitsDoc(SphinxClassDoc):
- def __init__(self, cls, modulename='', func_doc=SphinxFunctionDoc):
- if not inspect.isclass(cls):
- raise ValueError("Initialise using a class. Got %r" % cls)
- self._cls = cls
-
- if modulename and not modulename.endswith('.'):
- modulename += '.'
- self._mod = modulename
- self._name = cls.__name__
- self._func_doc = func_doc
-
- docstring = pydoc.getdoc(cls)
- docstring = docstring.split('\n')
-
- # De-indent paragraph
- try:
- indent = min(len(s) - len(s.lstrip()) for s in docstring
- if s.strip())
- except ValueError:
- indent = 0
-
- for n, line in enumerate(docstring):
- docstring[n] = docstring[n][indent:]
-
- self._doc = docscrape.Reader(docstring)
- self._parsed_data = {
- 'Signature': '',
- 'Summary': '',
- 'Description': [],
- 'Extended Summary': [],
- 'Parameters': [],
- 'Returns': [],
- 'Raises': [],
- 'Warns': [],
- 'Other Parameters': [],
- 'Traits': [],
- 'Methods': [],
- 'See Also': [],
- 'Notes': [],
- 'References': '',
- 'Example': '',
- 'Examples': '',
- 'index': {}
- }
-
- self._parse()
-
- def _str_summary(self):
- return self['Summary'] + ['']
-
- def _str_extended_summary(self):
- return self['Description'] + self['Extended Summary'] + ['']
-
- def __str__(self, indent=0, func_role="func"):
- out = []
- out += self._str_signature()
- out += self._str_index() + ['']
- out += self._str_summary()
- out += self._str_extended_summary()
- for param_list in ('Parameters', 'Traits', 'Methods',
- 'Returns', 'Raises'):
- out += self._str_param_list(param_list)
- out += self._str_see_also("obj")
- out += self._str_section('Notes')
- out += self._str_references()
- out += self._str_section('Example')
- out += self._str_section('Examples')
- out = self._str_indent(out, indent)
- return '\n'.join(out)
-
-def looks_like_issubclass(obj, classname):
- """ Return True if the object has a class or superclass with the given class
- name.
-
- Ignores old-style classes.
- """
- t = obj
- if t.__name__ == classname:
- return True
- for klass in t.__mro__:
- if klass.__name__ == classname:
- return True
- return False
-
-def get_doc_object(obj, what=None, config=None):
- if what is None:
- if inspect.isclass(obj):
- what = 'class'
- elif inspect.ismodule(obj):
- what = 'module'
- elif callable(obj):
- what = 'function'
- else:
- what = 'object'
- if what == 'class':
- doc = SphinxTraitsDoc(obj, '', func_doc=SphinxFunctionDoc, config=config)
- if looks_like_issubclass(obj, 'HasTraits'):
- for name, trait, comment in comment_eater.get_class_traits(obj):
- # Exclude private traits.
- if not name.startswith('_'):
- doc['Traits'].append((name, trait, comment.splitlines()))
- return doc
- elif what in ('function', 'method'):
- return SphinxFunctionDoc(obj, '', config=config)
- else:
- return SphinxDocString(pydoc.getdoc(obj), config=config)
-
-def setup(app):
- # init numpydoc
- numpydoc.setup(app, get_doc_object)
-
diff --git a/environment.yml b/environment.yml
new file mode 100644
index 0000000..0128a88
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,3 @@
+name: PyTables
+dependencies:
+ - hdf5
diff --git a/requirements.txt b/requirements.txt
index 97f7e4c..1c3d846 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@
# doc/source/usersguide/installation.rst
numpy>=1.8.0
numexpr>=2.5.2
-six
+six>=1.9.0
diff --git a/setup.py b/setup.py
index 6b64a57..b5c55ba 100755
--- a/setup.py
+++ b/setup.py
@@ -32,8 +32,13 @@ import distutils.spawn
from setuptools.command.build_ext import build_ext
# For guessing the capabilities of the CPU for C-Blosc
-import cpuinfo
-cpu_info = cpuinfo.get_cpu_info()
+try:
+ import cpuinfo
+ cpu_info = cpuinfo.get_cpu_info()
+ cpu_flags = cpu_info['flags']
+except Exception as e:
+ print('cpuinfo failed, assuming no CPU features:', e)
+ cpu_flags = []
# The name for the pkg-config utility
PKG_CONFIG = 'pkg-config'
@@ -98,10 +103,12 @@ if sys.version_info < min_python_version:
print("* Using Python %s" % sys.version.splitlines()[0])
# Minumum equired versions for numpy, numexpr and HDF5
-exec(open(os.path.join('tables', 'req_versions.py')).read())
+with open(os.path.join('tables', 'req_versions.py')) as fd:
+ exec(fd.read())
-VERSION = open('VERSION').read().strip()
+with open('VERSION') as fd:
+ VERSION = fd.read().strip()
# ----------------------------------------------------------------------
@@ -130,6 +137,7 @@ def add_from_flags(envname, flag_key, dirs):
if flag.startswith(flag_key):
dirs.append(flag[len(flag_key):])
+
if os.name == 'posix':
prefixes = ('/usr/local', '/sw', '/opt', '/opt/local', '/usr', '/')
@@ -336,22 +344,24 @@ def get_hdf5_version(headername):
major_version = -1
minor_version = -1
release_version = -1
- for line in open(headername):
- if 'H5_VERS_MAJOR' in line:
- major_version = int(re.split("\s*", line)[2])
- if 'H5_VERS_MINOR' in line:
- minor_version = int(re.split("\s*", line)[2])
- if 'H5_VERS_RELEASE' in line:
- release_version = int(re.split("\s*", line)[2])
- if (major_version != -1 and minor_version != -1 and
- release_version != -1):
- break
+ with open(headername) as fd:
+ for line in fd:
+ if 'H5_VERS_MAJOR' in line:
+ major_version = int(re.split("\s*", line)[2])
+ if 'H5_VERS_MINOR' in line:
+ minor_version = int(re.split("\s*", line)[2])
+ if 'H5_VERS_RELEASE' in line:
+ release_version = int(re.split("\s*", line)[2])
+ if (major_version != -1 and minor_version != -1 and
+ release_version != -1):
+ break
if (major_version == -1 or minor_version == -1 or
release_version == -1):
exit_with_error("Unable to detect HDF5 library version!")
return LooseVersion("%s.%s.%s" % (major_version, minor_version,
release_version))
+
# Get the Blosc version provided the 'blosc.h' header
def get_blosc_version(headername):
major_version = -1
@@ -400,12 +410,12 @@ elif os.name == 'nt':
if os.environ['APPVEYOR'] and os.environ['BUILDWHEEL']:
# Conda HDF5 is linked to zlib.dll (from conda package zlib)
# but szip.dll is not included in conda
- dll_files = [os.environ['PYTHON']+'\\Library\\bin\\zlib.dll']
+ dll_files = [os.environ['CONDA_PREFIX']+'\\Library\\bin\\zlib.dll']
except KeyError:
# Update these paths for your own system!
dll_files = [
- #'\\windows\\system\\zlib1.dll',
- #'\\windows\\system\\szip.dll',
+ # '\\windows\\system\\zlib1.dll',
+ # '\\windows\\system\\szip.dll',
]
if debug:
@@ -446,6 +456,7 @@ LFLAGS = os.environ.get('LFLAGS', '').split()
# is not a good idea.
CFLAGS = os.environ.get('CFLAGS', '').split()
LIBS = os.environ.get('LIBS', '').split()
+CONDA_PREFIX = os.environ.get('CONDA_PREFIX', '')
# We start using pkg-config since some distributions are putting HDF5
# (and possibly other libraries) in exotic locations. See issue #442.
if distutils.spawn.find_executable(PKG_CONFIG):
@@ -485,6 +496,10 @@ for arg in args:
# sys.argv.remove(arg)
elif arg.find('--use-pkgconfig') == 0:
USE_PKGCONFIG = arg.split('=')[1]
+ CONDA_PREFIX = ''
+ sys.argv.remove(arg)
+ elif arg.find('--no-conda') == 0:
+ CONDA_PREFIX = ''
sys.argv.remove(arg)
USE_PKGCONFIG = True if USE_PKGCONFIG.upper() == 'TRUE' else False
@@ -497,9 +512,11 @@ print('* USE_PKGCONFIG:', USE_PKGCONFIG)
if not HDF5_DIR and os.name == 'nt':
import ctypes.util
if not debug:
- libdir = ctypes.util.find_library('hdf5.dll') or ctypes.util.find_library('hdf5dll.dll')
+ libdir = (ctypes.util.find_library('hdf5.dll') or
+ ctypes.util.find_library('hdf5dll.dll'))
else:
- libdir = ctypes.util.find_library('hdf5_D.dll') or ctypes.util.find_library('hdf5ddll.dll')
+ libdir = (ctypes.util.find_library('hdf5_D.dll') or
+ ctypes.util.find_library('hdf5ddll.dll'))
# Like 'C:\\Program Files\\HDF Group\\HDF5\\1.8.8\\bin\\hdf5dll.dll'
if libdir:
# Strip off the filename
@@ -508,6 +525,12 @@ if not HDF5_DIR and os.name == 'nt':
HDF5_DIR = os.path.dirname(libdir)
print("* Found HDF5 using system PATH ('%s')" % libdir)
+
+if CONDA_PREFIX:
+ print('* Found conda env: ``%s``' % CONDA_PREFIX)
+ if os.name == 'nt':
+ CONDA_PREFIX += '\\Library'
+
# The next flag for the C compiler is needed for finding the C headers for
# the Cython extensions
CFLAGS.append("-Isrc")
@@ -515,6 +538,7 @@ CFLAGS.append("-Isrc")
# Force the 1.8.x HDF5 API even if the library as been compiled to use the
# 1.6.x API by default
CFLAGS.extend([
+ "-DH5_USE_18_API",
"-DH5Acreate_vers=2",
"-DH5Aiterate_vers=2",
"-DH5Dcreate_vers=2",
@@ -560,6 +584,10 @@ for (package, location) in [(hdf5_package, HDF5_DIR),
% (lzo1_package.name, lzo2_package.name))
continue # do not use LZO 1 if LZO 2 is available
+ # if a package location is not specified, try to find it in conda env
+ if not location and CONDA_PREFIX:
+ location = CONDA_PREFIX
+
(hdrdir, libdir, rundir) = package.find_directories(
location, use_pkgconfig=USE_PKGCONFIG)
@@ -572,18 +600,14 @@ for (package, location) in [(hdf5_package, HDF5_DIR),
"Unsupported HDF5 version! HDF5 v%s+ required. "
"Found version v%s" % (min_hdf5_version, hdf5_version))
- if hdf5_version >= "1.10":
- exit_with_error(
- "HDF5 1.10 release not supported. HDF5 v1.8 release required. "
- "Found version v%s" % (hdf5_version))
-
if os.name == 'nt' and hdf5_version < "1.8.10":
# Change in DLL naming happened in 1.8.10
hdf5_old_dll_name = 'hdf5dll' if not debug else 'hdf5ddll'
package.library_name = hdf5_old_dll_name
package.runtime_name = hdf5_old_dll_name
_platdep['HDF5'] = [hdf5_old_dll_name, hdf5_old_dll_name]
- _, libdir, rundir = package.find_directories(location, use_pkgconfig=USE_PKGCONFIG)
+ _, libdir, rundir = package.find_directories(
+ location, use_pkgconfig=USE_PKGCONFIG)
# check if the library is in the standard compiler paths
if not libdir and package.target_function:
@@ -644,7 +668,7 @@ for (package, location) in [(hdf5_package, HDF5_DIR),
print_warning(
"This Blosc version does not support the BitShuffle filter. "
"Minimum desirable version is %s. Found version: %s" % (
- min_blosc_bitshuffle_version, blosc_version))
+ min_blosc_bitshuffle_version, blosc_version))
if not rundir:
loc = {
@@ -677,7 +701,6 @@ if lzo2_enabled:
else:
lzo_package = lzo1_package
-
# ------------------------------------------------------------------------------
cython_extnames = [
@@ -716,8 +739,8 @@ cython_extfiles = get_cython_extfiles(cython_extnames)
# Update the version.h file if this file is newer
if newer('VERSION', 'src/version.h'):
- open('src/version.h', 'w').write(
- '#define PYTABLES_VERSION "%s"\n' % VERSION)
+ with open('src/version.h', 'w') as fd:
+ fd.write('#define PYTABLES_VERSION "%s"\n' % VERSION)
# --------------------------------------------------------------------
@@ -725,10 +748,11 @@ if newer('VERSION', 'src/version.h'):
# PyTables contains data files for tests.
setuptools_kwargs['zip_safe'] = False
-setuptools_kwargs['extras_require'] = {}
setuptools_kwargs['install_requires'] = requirements
+
# Detect packages automatically.
setuptools_kwargs['packages'] = find_packages(exclude=['*.bench'])
+
# Entry points for automatic creation of scripts.
setuptools_kwargs['entry_points'] = {
'console_scripts': [
@@ -812,7 +836,7 @@ if 'BLOSC' not in optional_libs:
os.remove(fd.name)
# SSE2
- if 'sse2' in cpu_info['flags']:
+ if 'sse2' in cpu_flags:
print('SSE2 detected')
CFLAGS.append('-DSHUFFLE_SSE2_ENABLED')
if os.name == 'nt':
@@ -826,13 +850,13 @@ if 'BLOSC' not in optional_libs:
if 'sse2' in f]
# AVX2
# Detection code for AVX2 only works for gcc/clang, not for MSVC yet
- if ('avx2' in cpu_info['flags'] and
- compiler_has_flags(compiler, ["-mavx2"])):
- print('AVX2 detected')
- CFLAGS.append('-DSHUFFLE_AVX2_ENABLED')
- CFLAGS.append('-mavx2')
- blosc_sources += [f for f in glob.glob('c-blosc/blosc/*.c')
- if 'avx2' in f]
+ if ('avx2' in cpu_flags and
+ compiler_has_flags(compiler, ["-mavx2"])):
+ print('AVX2 detected')
+ CFLAGS.append('-DSHUFFLE_AVX2_ENABLED')
+ CFLAGS.append('-mavx2')
+ blosc_sources += [f for f in glob.glob('c-blosc/blosc/*.c')
+ if 'avx2' in f]
else:
ADDLIBS += ['blosc']
@@ -991,5 +1015,11 @@ interactively save and retrieve large amounts of data.
ext_modules=extensions,
cmdclass=cmdclass,
data_files=data_files,
+ extra_require={
+ 'doc': [
+ 'sphinx >= 1.1',
+ 'sphinx_rtd_theme',
+ 'numpydoc',
+ 'ipython']},
**setuptools_kwargs
)
diff --git a/src/H5ARRAY.c b/src/H5ARRAY.c
index 4b29c05..3ca25c5 100644
--- a/src/H5ARRAY.c
+++ b/src/H5ARRAY.c
@@ -31,7 +31,7 @@
*-------------------------------------------------------------------------
*/
-herr_t H5ARRAYmake( hid_t loc_id,
+hid_t H5ARRAYmake( hid_t loc_id,
const char *dset_name,
const char *obversion,
const int rank,
@@ -138,7 +138,7 @@ herr_t H5ARRAYmake( hid_t loc_id,
blosc_compcode = blosc_compname_to_compcode(blosc_compname);
cd_values[6] = blosc_compcode;
if ( H5Pset_filter( plist_id, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 7, cd_values) < 0 )
- return -1;
+ return -1;
}
/* The LZO compressor does accept parameters */
else if (strcmp(complib, "lzo") == 0) {
diff --git a/src/H5ARRAY.h b/src/H5ARRAY.h
index 2c1fd40..d49e4fb 100644
--- a/src/H5ARRAY.h
+++ b/src/H5ARRAY.h
@@ -13,7 +13,7 @@
extern "C" {
#endif
-herr_t H5ARRAYmake( hid_t loc_id,
+hid_t H5ARRAYmake( hid_t loc_id,
const char *dset_name,
const char *obversion,
const int rank,
diff --git a/src/H5TB-opt.c b/src/H5TB-opt.c
index 31e7948..c1c1b34 100644
--- a/src/H5TB-opt.c
+++ b/src/H5TB-opt.c
@@ -72,7 +72,7 @@
*/
-herr_t H5TBOmake_table( const char *table_title,
+hid_t H5TBOmake_table( const char *table_title,
hid_t loc_id,
const char *dset_name,
char *version,
diff --git a/src/H5TB-opt.h b/src/H5TB-opt.h
index 8bb2847..4c2243d 100644
--- a/src/H5TB-opt.h
+++ b/src/H5TB-opt.h
@@ -4,7 +4,7 @@
extern "C" {
#endif
-herr_t H5TBOmake_table( const char *table_title,
+hid_t H5TBOmake_table( const char *table_title,
hid_t loc_id,
const char *dset_name,
char *version,
diff --git a/src/H5VLARRAY.c b/src/H5VLARRAY.c
index 86ff545..431b436 100644
--- a/src/H5VLARRAY.c
+++ b/src/H5VLARRAY.c
@@ -29,7 +29,7 @@
*-------------------------------------------------------------------------
*/
-herr_t H5VLARRAYmake( hid_t loc_id,
+hid_t H5VLARRAYmake( hid_t loc_id,
const char *dset_name,
const char *obversion,
const int rank,
diff --git a/src/H5VLARRAY.h b/src/H5VLARRAY.h
index c0913b5..a900857 100644
--- a/src/H5VLARRAY.h
+++ b/src/H5VLARRAY.h
@@ -7,7 +7,7 @@
extern "C" {
#endif
-herr_t H5VLARRAYmake( hid_t loc_id,
+hid_t H5VLARRAYmake( hid_t loc_id,
const char *dset_name,
const char *obversion,
const int rank,
diff --git a/tables/atom.py b/tables/atom.py
index d538698..2d8c767 100644
--- a/tables/atom.py
+++ b/tables/atom.py
@@ -120,7 +120,7 @@ def _abstract_atom_init(deftype, defvalue):
def _normalize_shape(shape):
"""Check that the `shape` is safe to be used and return it as a tuple."""
- if isinstance(shape, (int, numpy.integer, int)):
+ if isinstance(shape, (numpy.integer, six.integer_types)):
if shape < 1:
raise ValueError("shape value must be greater than 0: %d"
% shape)
diff --git a/tables/attributeset.py b/tables/attributeset.py
index 7d7f475..66c7658 100644
--- a/tables/attributeset.py
+++ b/tables/attributeset.py
@@ -72,6 +72,7 @@ def issysattrname(name):
return False
+ at six.python_2_unicode_compatible
class AttributeSet(hdf5extension.AttributeSet, object):
"""Container for the HDF5 attributes of a Node.
diff --git a/tables/definitions.pxd b/tables/definitions.pxd
index 61d75cb..5a94c09 100644
--- a/tables/definitions.pxd
+++ b/tables/definitions.pxd
@@ -15,8 +15,6 @@
import sys
cdef extern from *:
- ctypedef char const_char 'const char'
- #ctypedef long size_t
ctypedef long uintptr_t
# Standard C functions.
@@ -40,16 +38,16 @@ cdef extern from "numpy/arrayobject.h":
# Structs and types from HDF5
cdef extern from "hdf5.h" nogil:
- ctypedef int hid_t # In H5Ipublic.h
+ ctypedef long long hid_t # In H5Ipublic.h
ctypedef int hbool_t
ctypedef int herr_t
ctypedef int htri_t
# hsize_t should be unsigned, but Windows platform does not support
# such an unsigned long long type.
- ctypedef long long hsize_t
+ ctypedef unsigned long long hsize_t
ctypedef signed long long hssize_t
ctypedef long long int64_t
- ctypedef long long haddr_t
+ ctypedef unsigned long long haddr_t
ctypedef haddr_t hobj_ref_t
ctypedef struct hvl_t:
@@ -172,61 +170,58 @@ cdef extern from "hdf5.h" nogil:
H5T_NCLASSES # this must be last
# Native types
- cdef enum:
- H5T_C_S1
- H5T_NATIVE_B8
- H5T_NATIVE_CHAR
- H5T_NATIVE_SCHAR
- H5T_NATIVE_UCHAR
- H5T_NATIVE_SHORT
- H5T_NATIVE_USHORT
- H5T_NATIVE_INT
- H5T_NATIVE_UINT
- H5T_NATIVE_LONG
- H5T_NATIVE_ULONG
- H5T_NATIVE_LLONG
- H5T_NATIVE_ULLONG
- H5T_NATIVE_FLOAT
- H5T_NATIVE_DOUBLE
- H5T_NATIVE_LDOUBLE
+ hid_t H5T_C_S1
+ hid_t H5T_NATIVE_B8
+ hid_t H5T_NATIVE_CHAR
+ hid_t H5T_NATIVE_SCHAR
+ hid_t H5T_NATIVE_UCHAR
+ hid_t H5T_NATIVE_SHORT
+ hid_t H5T_NATIVE_USHORT
+ hid_t H5T_NATIVE_INT
+ hid_t H5T_NATIVE_UINT
+ hid_t H5T_NATIVE_LONG
+ hid_t H5T_NATIVE_ULONG
+ hid_t H5T_NATIVE_LLONG
+ hid_t H5T_NATIVE_ULLONG
+ hid_t H5T_NATIVE_FLOAT
+ hid_t H5T_NATIVE_DOUBLE
+ hid_t H5T_NATIVE_LDOUBLE
# "Standard" types
- cdef enum:
- H5T_STD_I8LE
- H5T_STD_I16LE
- H5T_STD_I32LE
- H5T_STD_I64LE
- H5T_STD_U8LE
- H5T_STD_U16LE
- H5T_STD_U32LE
- H5T_STD_U64LE
- H5T_STD_B8LE
- H5T_STD_B16LE
- H5T_STD_B32LE
- H5T_STD_B64LE
- H5T_IEEE_F32LE
- H5T_IEEE_F64LE
- H5T_STD_I8BE
- H5T_STD_I16BE
- H5T_STD_I32BE
- H5T_STD_I64BE
- H5T_STD_U8BE
- H5T_STD_U16BE
- H5T_STD_U32BE
- H5T_STD_U64BE
- H5T_STD_B8BE
- H5T_STD_B16BE
- H5T_STD_B32BE
- H5T_STD_B64BE
- H5T_IEEE_F32BE
- H5T_IEEE_F64BE
+ hid_t H5T_STD_I8LE
+ hid_t H5T_STD_I16LE
+ hid_t H5T_STD_I32LE
+ hid_t H5T_STD_I64LE
+ hid_t H5T_STD_U8LE
+ hid_t H5T_STD_U16LE
+ hid_t H5T_STD_U32LE
+ hid_t H5T_STD_U64LE
+ hid_t H5T_STD_B8LE
+ hid_t H5T_STD_B16LE
+ hid_t H5T_STD_B32LE
+ hid_t H5T_STD_B64LE
+ hid_t H5T_IEEE_F32LE
+ hid_t H5T_IEEE_F64LE
+ hid_t H5T_STD_I8BE
+ hid_t H5T_STD_I16BE
+ hid_t H5T_STD_I32BE
+ hid_t H5T_STD_I64BE
+ hid_t H5T_STD_U8BE
+ hid_t H5T_STD_U16BE
+ hid_t H5T_STD_U32BE
+ hid_t H5T_STD_U64BE
+ hid_t H5T_STD_B8BE
+ hid_t H5T_STD_B16BE
+ hid_t H5T_STD_B32BE
+ hid_t H5T_STD_B64BE
+ hid_t H5T_IEEE_F32BE
+ hid_t H5T_IEEE_F64BE
# Types which are particular to UNIX (for Time types)
- cdef enum:
- H5T_UNIX_D32LE
- H5T_UNIX_D64LE
- H5T_UNIX_D32BE
- H5T_UNIX_D64BE
+ hid_t H5T_UNIX_D32LE
+ hid_t H5T_UNIX_D64LE
+ hid_t H5T_UNIX_D32BE
+ hid_t H5T_UNIX_D64BE
# The order to retrieve atomic native datatype
cdef enum H5T_direction_t:
@@ -281,9 +276,9 @@ cdef extern from "hdf5.h" nogil:
hid_t maj_num # major error ID
hid_t min_num # minor error number
unsigned line # line in file where error occurs
- const_char *func_name # function in which error occurred
- const_char *file_name # file in which error occurred
- const_char *desc # optional supplied description
+ const char *func_name # function in which error occurred
+ const char *file_name # file in which error occurred
+ const char *desc # optional supplied description
ctypedef herr_t (*H5E_walk_t)(unsigned n, H5E_error_t *err, void *data)
ctypedef herr_t (*H5E_auto_t)(hid_t estack, void *data)
@@ -433,7 +428,7 @@ cdef extern from "hdf5.h" nogil:
# size_t *block_size, size_t *cbuf_size)
#herr_t H5Pset_fapl_direct(hid_t fapl_id, size_t alignment,
# size_t block_size, size_t cbuf_size)
- herr_t H5Pset_fapl_log(hid_t fapl_id, const_char *logfile,
+ herr_t H5Pset_fapl_log(hid_t fapl_id, const char *logfile,
unsigned long long flags, size_t buf_size)
#herr_t H5Pset_fapl_windows(hid_t fapl_id)
herr_t H5Pset_fapl_stdio(hid_t fapl_id)
@@ -446,7 +441,7 @@ cdef extern from "hdf5.h" nogil:
herr_t H5Pset_fapl_family(hid_t fapl_id, hsize_t memb_size,
hid_t memb_fapl_id)
#herr_t H5Pget_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map,
- # hid_t *memb_fapl, const_char **memb_name,
+ # hid_t *memb_fapl, const char **memb_name,
# haddr_t *memb_addr, hbool_t *relax)
herr_t H5Pset_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map,
hid_t *memb_fapl, char **memb_name,
diff --git a/tables/description.py b/tables/description.py
index 3bcaa35..bc5652e 100644
--- a/tables/description.py
+++ b/tables/description.py
@@ -313,6 +313,7 @@ Time64Col = Col._subclass_from_prefix('Time64')
# Table description classes
# =========================
+ at six.python_2_unicode_compatible
class Description(object):
"""This class represents descriptions of the structure of tables.
diff --git a/tables/exceptions.py b/tables/exceptions.py
index 9306b6a..5aca7b2 100644
--- a/tables/exceptions.py
+++ b/tables/exceptions.py
@@ -23,6 +23,7 @@ import warnings
import traceback
+ at six.python_2_unicode_compatible
class HDF5ExtError(RuntimeError):
"""A low level HDF5 operation failed.
diff --git a/tables/file.py b/tables/file.py
index 9afa144..aa5520f 100644
--- a/tables/file.py
+++ b/tables/file.py
@@ -576,6 +576,7 @@ class NodeManager(object):
node._f_close()
+ at six.python_2_unicode_compatible
class File(hdf5extension.File, object):
"""The in-memory representation of a PyTables file.
diff --git a/tables/filters.py b/tables/filters.py
index 0877fb8..199e26d 100644
--- a/tables/filters.py
+++ b/tables/filters.py
@@ -57,6 +57,7 @@ _bitshuffle_flag = 0x8
# Classes
# =======
+ at six.python_2_unicode_compatible
class Filters(object):
"""Container for filter properties.
diff --git a/tables/group.py b/tables/group.py
index 28bc654..e29323a 100644
--- a/tables/group.py
+++ b/tables/group.py
@@ -41,7 +41,7 @@ class _ChildrenDict(ProxyDict):
return container._f_get_child(key)
-
+ at six.python_2_unicode_compatible
class Group(hdf5extension.Group, Node):
"""Basic PyTables grouping structure.
diff --git a/tables/hdf5extension.pyx b/tables/hdf5extension.pyx
index b496db4..0e00801 100644
--- a/tables/hdf5extension.pyx
+++ b/tables/hdf5extension.pyx
@@ -68,7 +68,7 @@ from cpython.bytes cimport (PyBytes_AsString, PyBytes_FromStringAndSize,
from cpython.unicode cimport PyUnicode_DecodeUTF8
-from definitions cimport (const_char, uintptr_t, hid_t, herr_t, hsize_t, hvl_t,
+from definitions cimport (uintptr_t, hid_t, herr_t, hsize_t, hvl_t,
H5S_seloper_t, H5D_FILL_VALUE_UNDEFINED,
H5O_TYPE_UNKNOWN, H5O_TYPE_GROUP, H5O_TYPE_DATASET, H5O_TYPE_NAMED_DATATYPE,
H5L_TYPE_ERROR, H5L_TYPE_HARD, H5L_TYPE_SOFT, H5L_TYPE_EXTERNAL,
@@ -1217,6 +1217,10 @@ cdef class Array(Leaf):
atom_ = atom
shape = shape[:-len(atom_.shape)]
self.disk_type_id = atom_to_hdf5_type(atom_, self.byteorder)
+ if self.disk_type_id < 0:
+ raise HDF5ExtError(
+ "Problems creating the %s: invalid disk type ID for atom %s" % (
+ self.__class__.__name__, atom_))
# Allocate space for the dimension axis info and fill it
dims = numpy.array(shape, dtype=numpy.intp)
@@ -1885,6 +1889,10 @@ cdef class VLArray(Leaf):
# Get the HDF5 type of the *scalar* atom
scatom = atom.copy(shape=())
self.base_type_id = atom_to_hdf5_type(scatom, self.byteorder)
+ if self.base_type_id < 0:
+ raise HDF5ExtError(
+ "Problems creating the %s: invalid base type ID for atom %s" % (
+ self.__class__.__name__, scatom))
# Allocate space for the dimension axis info
rank = len(atom.shape)
diff --git a/tables/idxutils.py b/tables/idxutils.py
index 99d85d1..470ea4c 100644
--- a/tables/idxutils.py
+++ b/tables/idxutils.py
@@ -13,6 +13,7 @@
"""Utilities to be used mainly by the Index class."""
from __future__ import absolute_import
+import six
import sys
import math
import numpy
@@ -471,7 +472,7 @@ def nextafter(x, direction, dtype, itemsize):
direction."""
assert direction in [-1, 0, +1]
- assert dtype.kind == "S" or type(x) in (bool, int, float)
+ assert dtype.kind == "S" or type(x) in (bool, float) + six.integer_types
if direction == 0:
return x
diff --git a/tables/index.py b/tables/index.py
index 22314b0..8909c00 100644
--- a/tables/index.py
+++ b/tables/index.py
@@ -45,6 +45,7 @@ from .utilsextension import (nan_aware_gt, nan_aware_ge,
nan_aware_lt, nan_aware_le,
bisect_left, bisect_right)
from .lrucacheextension import ObjectCache
+import six
from six.moves import range
@@ -105,7 +106,7 @@ def _table_column_pathname_of_index(indexpathname):
return (tablepathname, colpathname)
-
+ at six.python_2_unicode_compatible
class Index(NotLoggedMixin, Group, indexesextension.Index):
"""Represents the index of a column in a table.
diff --git a/tables/indexes.py b/tables/indexes.py
index 4074d51..58f755f 100644
--- a/tables/indexes.py
+++ b/tables/indexes.py
@@ -20,20 +20,20 @@ from .node import NotLoggedMixin
from .carray import CArray
from .earray import EArray
from . import indexesextension
+import six
# Declarations for inheriting
-class CacheArray(NotLoggedMixin, EArray, indexesextension.CacheArray):
+class CacheArray(indexesextension.CacheArray, NotLoggedMixin, EArray):
"""Container for keeping index caches of 1st and 2nd level."""
# Class identifier.
_c_classid = 'CACHEARRAY'
-
-class LastRowArray(NotLoggedMixin, CArray, indexesextension.LastRowArray):
+class LastRowArray(indexesextension.LastRowArray, NotLoggedMixin, CArray):
"""Container for keeping sorted and indices values of last row of an
index."""
@@ -41,8 +41,8 @@ class LastRowArray(NotLoggedMixin, CArray, indexesextension.LastRowArray):
_c_classid = 'LASTROWARRAY'
-
-class IndexArray(NotLoggedMixin, EArray, indexesextension.IndexArray):
+ at six.python_2_unicode_compatible
+class IndexArray(indexesextension.IndexArray, NotLoggedMixin, EArray):
"""Represent the index (sorted or reverse index) dataset in HDF5 file.
All NumPy typecodes are supported except for complex datatypes.
@@ -73,7 +73,6 @@ class IndexArray(NotLoggedMixin, EArray, indexesextension.IndexArray):
# Class identifier.
_c_classid = 'INDEXARRAY'
-
# Properties
# ~~~~~~~~~~
@property
@@ -174,7 +173,6 @@ class IndexArray(NotLoggedMixin, EArray, indexesextension.IndexArray):
result2 += chunksize * nchunk2
return (result1, result2)
-
def __str__(self):
"A compact representation of this class"
return "IndexArray(path=%s)" % self._v_pathname
diff --git a/tables/leaf.py b/tables/leaf.py
index 1424cee..e05469f 100644
--- a/tables/leaf.py
+++ b/tables/leaf.py
@@ -25,6 +25,7 @@ from .filters import Filters
from .utils import byteorders, lazyattr, SizeType
from .exceptions import PerformanceWarning
from . import utilsextension
+import six
from six.moves import range
@@ -73,6 +74,7 @@ def calc_chunksize(expected_mb):
return chunksize * 8
+ at six.python_2_unicode_compatible
class Leaf(Node):
"""Abstract base class for all PyTables leaves.
diff --git a/tables/link.py b/tables/link.py
index 5ed2915..679d182 100644
--- a/tables/link.py
+++ b/tables/link.py
@@ -34,6 +34,7 @@ from . import linkextension
from .node import Node
from .utils import lazyattr
from .attributeset import AttributeSet
+import six
import tables.file
@@ -138,6 +139,7 @@ class Link(Node):
return str(self)
+ at six.python_2_unicode_compatible
class SoftLink(linkextension.SoftLink, Link):
"""Represents a soft link (aka symbolic link).
@@ -198,7 +200,7 @@ class SoftLink(linkextension.SoftLink, Link):
# SoftLink rather than the target node
_link_attrnames = ('target', 'dereference', 'is_dangling', 'copy', 'move',
'remove', 'rename', '__init__', '__str__', '__repr__',
- '__class__', '__dict__')
+ '__unicode__', '__class__', '__dict__')
_link_attrprefixes = ('_f_', '_c_', '_g_', '_v_')
@@ -323,6 +325,7 @@ class SoftLink(linkextension.SoftLink, Link):
self.target, dangling)
+ at six.python_2_unicode_compatible
class ExternalLink(linkextension.ExternalLink, Link):
"""Represents an external link.
diff --git a/tables/linkextension.pyx b/tables/linkextension.pyx
index 2aa3a00..b188a3d 100644
--- a/tables/linkextension.pyx
+++ b/tables/linkextension.pyx
@@ -22,7 +22,7 @@ from libc.string cimport strlen
from cpython cimport PY_MAJOR_VERSION
from cpython.unicode cimport PyUnicode_DecodeUTF8
from definitions cimport (H5P_DEFAULT,
- const_char, hid_t, herr_t, hbool_t, int64_t, H5T_cset_t, haddr_t)
+ hid_t, herr_t, hbool_t, int64_t, H5T_cset_t, haddr_t)
@@ -74,7 +74,7 @@ cdef extern from "H5Lpublic.h" nogil:
herr_t H5Lunpack_elink_val(
char *ext_linkval, size_t link_size, unsigned *flags,
- const_char **filename, const_char **obj_path)
+ const char **filename, const char **obj_path)
herr_t H5Lcopy(
hid_t src_loc_id, char *src_name, hid_t dest_loc_id, char *dest_name,
@@ -255,8 +255,8 @@ cdef class ExternalLink(Link):
raise HDF5ExtError("failed to get target value")
ret = H5Lunpack_elink_val(clinkval, val_size, &flags,
- <const_char **>&cfilename,
- <const_char **>&c_obj_path)
+ <const char **>&cfilename,
+ <const char **>&c_obj_path)
if ret < 0:
raise HDF5ExtError("failed to unpack external link value")
diff --git a/tables/scripts/ptdump.py b/tables/scripts/ptdump.py
index 1c26b4d..067ff8e 100644
--- a/tables/scripts/ptdump.py
+++ b/tables/scripts/ptdump.py
@@ -167,7 +167,7 @@ def main():
args.dump = 1
# Catch the files passed as the last arguments
- src = args.src.split(':')
+ src = args.src.rsplit(':', 1)
if len(src) == 1:
filename, nodename = src[0], "/"
else:
diff --git a/tables/scripts/ptrepack.py b/tables/scripts/ptrepack.py
index 2939165..796c4a4 100644
--- a/tables/scripts/ptrepack.py
+++ b/tables/scripts/ptrepack.py
@@ -63,7 +63,6 @@ def newdst_group(dstfileh, dstgroup, title, filters):
return group
-
def recreate_indexes(table, dstfileh, dsttable):
listoldindexes = table._listoldindexes
if listoldindexes != []:
@@ -84,7 +83,6 @@ def recreate_indexes(table, dstfileh, dsttable):
colobj.create_index(filters=None)
-
def copy_leaf(srcfile, dstfile, srcnode, dstnode, title,
filters, copyuserattrs, overwritefile, overwrtnodes, stats,
start, stop, step, chunkshape, sortby, check_CSI,
@@ -177,7 +175,6 @@ def copy_leaf(srcfile, dstfile, srcnode, dstnode, title,
dstfileh.close()
-
def copy_children(srcfile, dstfile, srcgroup, dstgroup, title,
recursive, filters, copyuserattrs, overwritefile,
overwrtnodes, stats, start, stop, step,
@@ -271,7 +268,6 @@ def copy_children(srcfile, dstfile, srcgroup, dstgroup, title,
dstfileh.close()
-
def _get_parser():
parser = argparse.ArgumentParser(
description='''This utility is very powerful and lets you copy any
@@ -426,8 +422,8 @@ def main():
)
# Catch the files passed as the last arguments
- src = args.src.split(':')
- dst = args.dst.split(':')
+ src = args.src.rsplit(':', 1)
+ dst = args.dst.rsplit(':', 1)
if len(src) == 1:
srcfile, srcnode = src[0], "/"
else:
@@ -548,7 +544,10 @@ def main():
cpu2 = time.clock()
tcopy = round(t2 - t1, 3)
cpucopy = round(cpu2 - cpu1, 3)
- tpercent = int(round(cpucopy / tcopy, 2) * 100)
+ try:
+ tpercent = int(round(cpucopy / tcopy, 2) * 100)
+ except ZeroDivisionError:
+ tpercent = 'NaN'
if verbose:
ngroups = stats['groups']
diff --git a/tables/scripts/pttree.py b/tables/scripts/pttree.py
index a507c0b..c8f6635 100644
--- a/tables/scripts/pttree.py
+++ b/tables/scripts/pttree.py
@@ -103,7 +103,7 @@ def main():
args = parser.parse_args()
# Catch the files passed as the last arguments
- src = args.__dict__.pop('src').split(':')
+ src = args.__dict__.pop('src').rsplit(':', 1)
if len(src) == 1:
filename, nodename = src[0], "/"
else:
diff --git a/tables/table.py b/tables/table.py
index 522315c..70b0a64 100644
--- a/tables/table.py
+++ b/tables/table.py
@@ -2991,6 +2991,7 @@ very small/large chunksize, you may want to increase/decrease it."""
(str(self), self.description, self.byteorder, self.chunkshape)
+ at six.python_2_unicode_compatible
class Cols(object):
"""Container for columns in a table or nested column.
@@ -3265,6 +3266,7 @@ class Cols(object):
return out
+ at six.python_2_unicode_compatible
class Column(object):
"""Accessor for a non-nested column in a table.
diff --git a/tables/tests/test_all.py b/tables/tests/test_all.py
index 8cb6b37..57a505b 100644
--- a/tables/tests/test_all.py
+++ b/tables/tests/test_all.py
@@ -50,6 +50,7 @@ def suite():
'tables.tests.test_indexvalues',
'tables.tests.test_index_backcompat',
'tables.tests.test_aux',
+ 'tables.tests.test_utils',
# Sub-packages
'tables.nodes.tests.test_filenode',
]
diff --git a/tables/tests/test_create.py b/tables/tests/test_create.py
index 8d25020..14b9dc7 100644
--- a/tables/tests/test_create.py
+++ b/tables/tests/test_create.py
@@ -624,18 +624,24 @@ class FiltersCaseBloscBloscLZ(FiltersTreeTestCase):
'BLOSC compression library not available')
@unittest.skipIf('lz4' not in tables.blosc_compressor_list(), 'lz4 required')
class FiltersCaseBloscLZ4(FiltersTreeTestCase):
- filters = Filters(shuffle=False, complevel=1, complib="blosc:lz4")
- gfilters = Filters(complevel=5, shuffle=True, complib="blosc:lz4")
- open_kwargs = dict(filters=filters)
+ def setUp(self):
+ self.filters = Filters(shuffle=False, complevel=1, complib="blosc:lz4")
+ self.gfilters = Filters(complevel=5, shuffle=True, complib="blosc:lz4")
+ self.open_kwargs = dict(filters=self.filters)
+ super(FiltersCaseBloscLZ4, self).setUp()
@unittest.skipIf(not common.blosc_avail,
'BLOSC compression library not available')
@unittest.skipIf('lz4' not in tables.blosc_compressor_list(), 'lz4 required')
class FiltersCaseBloscLZ4HC(FiltersTreeTestCase):
- filters = Filters(shuffle=False, complevel=1, complib="blosc:lz4hc")
- gfilters = Filters(complevel=5, shuffle=True, complib="blosc:lz4hc")
- open_kwargs = dict(filters=filters)
+ def setUp(self):
+ self.filters = Filters(
+ shuffle=False, complevel=1, complib="blosc:lz4hc")
+ self.gfilters = Filters(
+ complevel=5, shuffle=True, complib="blosc:lz4hc")
+ self.open_kwargs = dict(filters=self.filters)
+ super(FiltersCaseBloscLZ4HC, self).setUp()
@unittest.skipIf(not common.blosc_avail,
@@ -643,26 +649,36 @@ class FiltersCaseBloscLZ4HC(FiltersTreeTestCase):
@unittest.skipIf('snappy' not in tables.blosc_compressor_list(),
'snappy required')
class FiltersCaseBloscSnappy(FiltersTreeTestCase):
- filters = Filters(shuffle=False, complevel=1, complib="blosc:snappy")
- gfilters = Filters(complevel=5, shuffle=True, complib="blosc:snappy")
- open_kwargs = dict(filters=filters)
+ def setUp(self):
+ self.filters = Filters(
+ shuffle=False, complevel=1, complib="blosc:snappy")
+ self.gfilters = Filters(
+ complevel=5, shuffle=True, complib="blosc:snappy")
+ self.open_kwargs = dict(filters=self.filters)
+ super(FiltersCaseBloscSnappy, self).setUp()
@unittest.skipIf(not common.blosc_avail,
'BLOSC compression library not available')
@unittest.skipIf('zlib' not in tables.blosc_compressor_list(), 'zlib required')
class FiltersCaseBloscZlib(FiltersTreeTestCase):
- filters = Filters(shuffle=False, complevel=1, complib="blosc:zlib")
- gfilters = Filters(complevel=5, shuffle=True, complib="blosc:zlib")
- open_kwargs = dict(filters=filters)
+ def setUp(self):
+ self.filters = Filters(shuffle=False, complevel=1, complib="blosc:zlib")
+ self.gfilters = Filters(complevel=5, shuffle=True, complib="blosc:zlib")
+ self.open_kwargs = dict(filters=self.filters)
+ super(FiltersCaseBloscZlib, self).setUp()
+
@unittest.skipIf(not common.blosc_avail,
'BLOSC compression library not available')
@unittest.skipIf('zstd' not in tables.blosc_compressor_list(), 'zstd required')
class FiltersCaseBloscZstd(FiltersTreeTestCase):
- filters = Filters(shuffle=False, complevel=1, complib="blosc:zstd")
- gfilters = Filters(complevel=5, shuffle=True, complib="blosc:zstd")
- open_kwargs = dict(filters=filters)
+ def setUp(self):
+ self.filters = Filters(shuffle=False, complevel=1, complib="blosc:zstd")
+ self.gfilters = Filters(complevel=5, shuffle=True, complib="blosc:zstd")
+ self.open_kwargs = dict(filters=self.filters)
+ super(FiltersCaseBloscZstd, self).setUp()
+
@unittest.skipIf(not common.blosc_avail,
'BLOSC compression library not available')
diff --git a/tables/tests/test_utils.py b/tables/tests/test_utils.py
new file mode 100644
index 0000000..ac33918
--- /dev/null
+++ b/tables/tests/test_utils.py
@@ -0,0 +1,95 @@
+import sys
+from mock import patch
+
+import six
+from tables.tests import common
+from tables.tests.common import unittest
+from tables.tests.common import PyTablesTestCase as TestCase
+
+import tables.scripts.ptrepack as ptrepack
+import tables.scripts.ptdump as ptdump
+import tables.scripts.pttree as pttree
+
+
+class ptrepackTestCase(TestCase):
+ """Test ptrepack"""
+
+ @patch.object(ptrepack, 'copy_leaf')
+ @patch.object(ptrepack, 'open_file')
+ def test_paths_windows(self, mock_open_file, mock_copy_leaf):
+ """Checking handling of windows filenames: test gh-616"""
+
+ # this filename has a semi-colon to check for
+ # regression of gh-616
+ src_fn = 'D:\\window~1\\path\\000\\infile'
+ src_path = '/'
+ dst_fn = 'another\\path\\'
+ dst_path = '/path/in/outfile'
+
+ argv = ['ptrepack', src_fn + ':' + src_path, dst_fn + ':' + dst_path]
+ with patch.object(sys, 'argv', argv):
+ ptrepack.main()
+
+ args, kwargs = mock_open_file.call_args_list[0]
+ self.assertEqual(args, (src_fn, 'r'))
+
+ args, kwargs = mock_copy_leaf.call_args_list[0]
+ self.assertEqual(args, (src_fn, dst_fn, src_path, dst_path))
+
+
+class ptdumpTestCase(TestCase):
+ """Test ptdump"""
+
+ @patch.object(ptdump, 'open_file')
+ @patch('sys.stdout', new_callable=six.StringIO)
+ def test_paths_windows(self, _, mock_open_file):
+ """Checking handling of windows filenames: test gh-616"""
+
+ # this filename has a semi-colon to check for
+ # regression of gh-616 (in ptdump)
+ src_fn = 'D:\\window~1\\path\\000\\ptdump'
+ src_path = '/'
+
+ argv = ['ptdump', src_fn + ':' + src_path]
+ with patch.object(sys, 'argv', argv):
+ ptdump.main()
+
+ args, kwargs = mock_open_file.call_args_list[0]
+ self.assertEqual(args, (src_fn, 'r'))
+
+
+class pttreeTestCase(TestCase):
+ """Test ptdump"""
+
+ @patch.object(pttree.tables, 'open_file')
+ @patch.object(pttree, 'get_tree_str')
+ @patch('sys.stdout', new_callable=six.StringIO)
+ def test_paths_windows(self, _, mock_get_tree_str, mock_open_file):
+ """Checking handling of windows filenames: test gh-616"""
+
+ # this filename has a semi-colon to check for
+ # regression of gh-616 (in pttree)
+ src_fn = 'D:\\window~1\\path\\000\\pttree'
+ src_path = '/'
+
+ argv = ['pttree', src_fn + ':' + src_path]
+ with patch.object(sys, 'argv', argv):
+ pttree.main()
+
+ args, kwargs = mock_open_file.call_args_list[0]
+ self.assertEqual(args, (src_fn, 'r'))
+
+
+def suite():
+ theSuite = unittest.TestSuite()
+
+ theSuite.addTest(unittest.makeSuite(ptrepackTestCase))
+ theSuite.addTest(unittest.makeSuite(ptdumpTestCase))
+ theSuite.addTest(unittest.makeSuite(pttreeTestCase))
+
+ return theSuite
+
+if __name__ == '__main__':
+ common.parse_argv(sys.argv)
+ common.print_versions()
+ unittest.main(defaultTest='suite')
diff --git a/tables/unimplemented.py b/tables/unimplemented.py
index 4b64491..30bc4ec 100644
--- a/tables/unimplemented.py
+++ b/tables/unimplemented.py
@@ -13,6 +13,7 @@
"""Here is defined the UnImplemented class."""
from __future__ import absolute_import
+import six
import warnings
from . import hdf5extension
@@ -114,6 +115,7 @@ class UnImplemented(hdf5extension.UnImplemented, Leaf):
# Classes reported as H5G_UNKNOWN by HDF5
+ at six.python_2_unicode_compatible
class Unknown(Node):
"""This class represents nodes reported as *unknown* by the underlying
HDF5 library.
diff --git a/tables/utilsextension.pxd b/tables/utilsextension.pxd
index aabe0e2..762e47e 100644
--- a/tables/utilsextension.pxd
+++ b/tables/utilsextension.pxd
@@ -15,10 +15,10 @@ These are declarations for functions in utilsextension.pyx that have to
be shared with other extensions.
"""
-from definitions cimport hsize_t, hid_t, const_char, hobj_ref_t
+from definitions cimport hsize_t, hid_t, hobj_ref_t
from numpy cimport ndarray
cdef hsize_t *malloc_dims(object)
cdef hid_t get_native_type(hid_t) nogil
-cdef str cstr_to_pystr(const_char*)
+cdef str cstr_to_pystr(const char*)
cdef int load_reference(hid_t dataset_id, hobj_ref_t *refbuf, size_t item_size, ndarray nparr) except -1
diff --git a/tables/utilsextension.pyx b/tables/utilsextension.pyx
index 7d801e0..0a33e4e 100644
--- a/tables/utilsextension.pyx
+++ b/tables/utilsextension.pyx
@@ -201,7 +201,7 @@ cdef extern from "H5ARRAY.h" nogil:
# @TODO: use the c_string_type and c_string_encoding global directives
# (new in cython 0.19)
-cdef str cstr_to_pystr(const_char* cstring):
+cdef str cstr_to_pystr(const char* cstring):
if PY_MAJOR_VERSION > 2:
pystring = PyUnicode_DecodeUTF8(cstring, strlen(cstring), NULL)
else:
@@ -356,9 +356,7 @@ else: # Unix systems
#---------------------------------------------------------------------
# Error handling helpers
-# XXX: silence warning about incompatible pointer types
-#ctypedef H5E_error_t* const_H5E_error_t_ptr "const H5E_error_t*"
-cdef herr_t e_walk_cb(unsigned n, H5E_error_t *err, void *data) with gil:
+cdef herr_t e_walk_cb(unsigned n, const H5E_error_t *err, void *data) with gil:
cdef object bt = <object>data # list
#cdef char major_msg[256]
#cdef char minor_msg[256]
@@ -799,7 +797,8 @@ def blosc_get_complib_info_():
"""
- cdef char *complib, *version
+ cdef char *complib
+ cdef char *version
cinfo = {}
for name in blosc_list_compressors().split(b','):
@@ -1156,6 +1155,7 @@ def enum_to_hdf5(object enum_atom, str byteorder):
def atom_to_hdf5_type(atom, str byteorder):
cdef hid_t tid = -1
+ cdef hid_t tid2 = -1
cdef hsize_t *dims = NULL
cdef bytes encoded_byteorder
cdef char *cbyteorder = NULL
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/pytables.git
More information about the debian-science-commits
mailing list