[med-svn] [libzstd] 02/07: Merge branch 'upstream'
Kevin Murray
daube-guest at moszumanska.debian.org
Thu Jul 21 02:46:57 UTC 2016
This is an automated email from the git hooks/post-receive script.
daube-guest pushed a commit to branch master
in repository libzstd.
commit 61101f9bb35da85b4728dd90623bbdf43a9f11a1
Merge: 73cb715 6837d46
Author: Kevin Murray <spam at kdmurray.id.au>
Date: Thu Jul 21 10:47:21 2016 +1000
Merge branch 'upstream'
There were many conflicts, this needs review
.gitattributes | 10 +-
.gitignore | 39 +-
.travis.yml | 109 +-
Makefile | 154 +-
NEWS | 42 +-
README.md | 89 +-
appveyor.yml | 104 +
images/Cspeed4.png | Bin 47376 -> 47294 bytes
images/DCspeed5.png | Bin 69388 -> 69278 bytes
images/Dspeed4.png | Bin 9499 -> 9927 bytes
images/smallData.png | Bin 0 -> 36133 bytes
lib/Makefile | 40 +-
lib/README.md | 69 +-
lib/{ => common}/bitstream.h | 233 +-
lib/common/entropy_common.c | 231 +
lib/{ => common}/error_private.h | 25 +-
lib/{ => common}/error_public.h | 8 +-
lib/common/fse.h | 628 +++
lib/common/fse_decompress.c | 331 ++
lib/common/huf.h | 228 +
lib/{ => common}/mem.h | 161 +-
{programs => lib/common}/xxhash.c | 613 ++-
{programs => lib/common}/xxhash.h | 97 +-
lib/{ => common}/zbuff.h | 70 +-
lib/common/zstd.h | 446 ++
lib/common/zstd_common.c | 91 +
lib/common/zstd_internal.h | 238 +
lib/compress/.debug/zstd_stats.h | 162 +
lib/{fse.c => compress/fse_compress.c} | 547 +--
lib/compress/huf_compress.c | 576 +++
lib/compress/zbuff_compress.c | 329 ++
lib/compress/zstd_compress.c | 2774 ++++++++++++
lib/compress/zstd_opt.h | 1041 +++++
lib/decompress/huf_decompress.c | 894 ++++
lib/decompress/zbuff_decompress.c | 294 ++
lib/decompress/zstd_decompress.c | 1346 ++++++
lib/dictBuilder/divsufsort.c | 1913 ++++++++
lib/dictBuilder/divsufsort.h | 67 +
lib/dictBuilder/zdict.c | 1036 +++++
lib/dictBuilder/zdict.h | 120 +
lib/fse.h | 295 --
lib/fse_static.h | 336 --
lib/huff0.c | 1728 --------
lib/huff0.h | 97 -
lib/huff0_static.h | 139 -
lib/legacy/zstd_legacy.h | 72 +-
lib/legacy/zstd_v01.c | 4 +-
lib/legacy/zstd_v02.c | 4 +-
lib/legacy/zstd_v03.c | 364 +-
lib/legacy/zstd_v04.c | 361 +-
lib/legacy/zstd_v04.h | 2 +-
lib/legacy/{zstd_v04.c => zstd_v05.c} | 3319 +++++++-------
lib/legacy/zstd_v05.h | 171 +
lib/legacy/zstd_v06.c | 4583 ++++++++++++++++++++
lib/legacy/zstd_v06.h | 185 +
lib/zbuff.c | 548 ---
lib/zstd.h | 149 -
lib/zstd_compress.c | 2394 ----------
lib/zstd_decompress.c | 1166 -----
lib/zstd_internal.h | 182 -
lib/zstd_static.h | 249 --
programs/.gitignore | 22 +
programs/Makefile | 142 +-
programs/bench.c | 509 +--
programs/bench.h | 18 +-
programs/datagen.c | 179 +-
programs/datagencli.c | 80 +-
programs/dibio.c | 162 +-
programs/dibio.h | 3 +-
programs/fileio.c | 460 +-
programs/fileio.h | 11 +-
programs/fullbench.c | 482 +-
programs/fuzzer.c | 897 ++--
programs/legacy/fileio_legacy.c | 247 +-
programs/legacy/fileio_legacy.h | 4 +-
programs/paramgrill.c | 284 +-
programs/playTests.sh | 199 +-
programs/roundTripCrash.c | 193 +
programs/util.h | 401 ++
programs/zbufftest.c | 515 ++-
programs/zstd.1 | 42 +-
programs/zstdcli.c | 279 +-
projects/.gitignore | 2 +
projects/README.md | 9 +
projects/VS2008/fullbench/fullbench.vcproj | 449 ++
projects/VS2008/fuzzer/fuzzer.vcproj | 461 ++
projects/VS2008/zstd.sln | 56 +
projects/VS2008/zstd/zstd.vcproj | 545 +++
projects/VS2008/zstdlib/zstdlib.vcproj | 495 +++
.../VS2010/datagen/datagen.vcxproj | 351 +-
projects/VS2010/datagen/datagen.vcxproj.filters | 26 +
.../VS2010}/fullbench/fullbench.vcxproj | 371 +-
.../VS2010/fullbench/fullbench.vcxproj.filters | 86 +
.../2013 => projects/VS2010}/fuzzer/fuzzer.vcxproj | 379 +-
projects/VS2010/fuzzer/fuzzer.vcxproj.filters | 92 +
{visual/2013 => projects/VS2010}/zstd.sln | 124 +-
{visual/2013 => projects/VS2010}/zstd/zstd.vcxproj | 429 +-
projects/VS2010/zstd/zstd.vcxproj.filters | 158 +
projects/VS2010/zstdlib/zstdlib.rc | 51 +
.../VS2010}/zstdlib/zstdlib.vcxproj | 426 +-
projects/VS2010/zstdlib/zstdlib.vcxproj.filters | 95 +
projects/build/README.md | 51 +
projects/build/build.VS2010.cmd | 7 +
projects/build/build.VS2012.cmd | 6 +
projects/build/build.VS2013.cmd | 7 +
projects/build/build.VS2015.cmd | 7 +
projects/build/build.generic.cmd | 52 +
projects/cmake/.gitignore | 6 +
{contrib => projects}/cmake/CMakeLists.txt | 12 +-
.../CMakeModules/AddExtraCompilationFlags.cmake | 0
.../cmake/cmake_uninstall.cmake.in | 0
{contrib => projects}/cmake/lib/CMakeLists.txt | 96 +-
projects/cmake/programs/.gitignore | 8 +
.../cmake/programs/CMakeLists.txt | 24 +-
tests/.gitignore | 4 +
tests/Makefile | 41 +
tests/README.md | 55 +
tests/test-zstd-speed.py | 265 ++
tests/test-zstd-versions.py | 266 ++
visual/2013/fullbench/fullbench.vcxproj.filters | 72 -
visual/2013/fuzzer/fuzzer.vcxproj.filters | 90 -
visual/2013/zstd/zstd.vcxproj.filters | 150 -
visual/2013/zstdlib/resource.h | Bin 812 -> 0 bytes
visual/2013/zstdlib/zstdlib.rc | Bin 5204 -> 0 bytes
visual/2013/zstdlib/zstdlib.vcxproj.filters | 83 -
{programs => zlibWrapper}/.gitignore | 22 +-
zlibWrapper/Makefile | 61 +
zlibWrapper/README.md | 105 +
zlibWrapper/examples/example.c | 610 +++
zlibWrapper/examples/example_original.c | 601 +++
zlibWrapper/zstd_zlibwrapper.c | 981 +++++
.../zstd_zlibwrapper.h | 39 +-
132 files changed, 30412 insertions(+), 14546 deletions(-)
diff --cc programs/fileio.c
index 028c7db,5e7b26d..befe71d
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@@ -381,10 -367,11 +367,31 @@@ static int FIO_compressFilename_interna
/* Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu =>%6llu bytes, %s) \n", srcFileName,
+ (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize,
+ dstFileName);
-- return 0;
++
++/*! FIO_compressFilename_extRess() :
++ * @return : 0 : compression completed correctly,
++ * 1 : missing or pb opening srcFileName
++ */
++static int FIO_compressFilename_extRess(cRess_t ress,
++ const char* dstFileName, const char* srcFileName,
++ int cLevel)
++{
++ int result;
++
++ ress.srcFile = FIO_openSrcFile(srcFileName);
++ if (ress.srcFile==0) return 1;
++ ress.dstFile = FIO_openDstFile(dstFileName);
++ if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
++
++ result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
++
++ fclose(ress.srcFile); /* no pb to expect : only reading */
++ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
++ return result;
}
diff --cc programs/zstd.1
index 27d607f,d7760f7..84ab946
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@@ -71,26 -71,41 +71,58 @@@ It also features a very fast decoder, w
.TP
.BR \-c ", " --stdout
force write to standard output, even if it is the console
+
+.SH DICTIONARY
+.PP
+\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages.
+It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`.
+Then during compression and decompression, make reference to the same dictionary.
+It will improve compression ratio of small files.
+Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
+.TP
+.B \--train FILEs
+ use FILEs as training set to create a dictionary.
+ The training set should contain a lot of small files (> 100).
+ and weight typically 100x the target dictionary size
+ (for example, 10 MB for a 100 KB dictionary)
+.TP
+.B \-o file
+ dictionary saved into `file` (default: dictionary)
.TP
+ .BR \-C ", " --check
+ add integrity check computed from uncompressed data
+ .TP
+ .BR \-t ", " --test
+ Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
+ No files are created or removed.
+
+ .SH DICTIONARY
+ .PP
+ \fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages.
+ It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`.
+ Then during compression and decompression, make reference to the same dictionary.
+ It will improve compression ratio of small files.
+ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
+ .TP
+ .B \--train FILEs
+ use FILEs as training set to create a dictionary.
+ The training set should contain a lot of small files (> 100).
+ and weight typically 100x the target dictionary size
+ (for example, 10 MB for a 100 KB dictionary)
+ .TP
+ .B \-o file
+ dictionary saved into `file` (default: dictionary)
+ .TP
.B \--maxdict #
- limit dictionary to specified size (default : 112640)
+ limit dictionary to specified size (default : 112640)
+ .TP
+ .B \--dictID #
+ A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
+ By default, zstd will create a 4-bytes random number ID.
+ It's possible to give a precise number instead.
+ Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
+ and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
+ However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
.TP
.B \-s#
dictionary selectivity level (default: 9)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/libzstd.git
More information about the debian-med-commit
mailing list