Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev

This commit is contained in:
Przemyslaw Skibinski 2016-11-08 08:44:19 +01:00
commit 90981a35c9
7 changed files with 76 additions and 137 deletions

View File

@ -4,57 +4,29 @@ matrix:
include:
# OS X Mavericks
- os: osx
env: Ubu="OS X Mavericks" PARAMS='-C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang
env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang
# Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS="install PREFIX=~/install_test_dir" COMPILER=cc
env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean && make cmake && make clean && make travis-install" COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS="examples" COMPILER=cc
env: Ubu=12.04cont Cmd='make test' COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS="cmake" COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS=clangtest COMPILER=clang
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS=gpptest COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS='-C tests test-lz4' COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS='-C tests test-lz4 MOREFLAGS="-std=c99"' COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS='-C tests test-lz4c' COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS='-C tests test-fullbench' COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS='-C tests test-fuzzer' COMPILER=cc
- os: linux
sudo: false
env: Ubu="12.04cont" PARAMS='-C tests test-frametest' COMPILER=cc
env: Ubu=12.04cont Cmd='make clangtest' COMPILER=clang
# 14.04 LTS Server Edition 64 bit
- env: Ubu="14.04" PARAMS=sanitize COMPILER=clang
- env: Ubu=14.04 Cmd='make staticAnalyze sanitize' COMPILER=clang
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
- env: Ubu="14.04" PARAMS=staticAnalyze COMPILER=clang
dist: trusty
sudo: required
addons:
apt:
packages:
- clang
- env: Ubu="14.04" PARAMS=armtest COMPILER=arm-linux-gnueabi-gcc
- env: Ubu=14.04 Cmd='make armtest' COMPILER=arm-linux-gnueabi-gcc
dist: trusty
sudo: required
addons:
@ -63,7 +35,14 @@ matrix:
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- linux-libc-dev-armel-cross
- env: Ubu="14.04" PARAMS=versionsTest COMPILER=cc
- env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
- env: Ubu=14.04 Cmd='make -C tests test32 versionsTest' COMPILER=cc
dist: trusty
sudo: required
addons:
@ -72,27 +51,7 @@ matrix:
- python3
- libc6-dev-i386
- gcc-multilib
- env: Ubu="14.04" PARAMS='-C tests test-lz4' COMPILER=cc
dist: trusty
sudo: required
- env: Ubu="14.04" PARAMS='-C tests test-lz4c32' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-5' COMPILER=gcc-5
dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- env: Ubu="14.04" PARAMS='-C tests test-lz4c32 CC=gcc-5' COMPILER=gcc-5
- env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5 MOREFLAGS="-Werror"' COMPILER=gcc-5
dist: trusty
sudo: required
addons:
@ -104,7 +63,7 @@ matrix:
- gcc-multilib
- gcc-5
- gcc-5-multilib
- env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=gcc-6' COMPILER=gcc-6
- env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS="-Werror"' COMPILER=gcc-6
dist: trusty
sudo: required
addons:
@ -113,7 +72,7 @@ matrix:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- env: Ubu="14.04" PARAMS='-C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8
- env: Ubu=14.04 Cmd='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8
dist: trusty
sudo: required
addons:
@ -123,39 +82,8 @@ matrix:
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
- env: Ubu="14.04" PARAMS='-C tests test-fullbench32' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu="14.04" PARAMS='-C tests test-fuzzer32' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu="14.04" PARAMS='-C tests test-frametest32' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Ubu="14.04" PARAMS='-C tests test-mem' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
script:
- echo PARAMS=$PARAMS
- echo Cmd=$Cmd
- $COMPILER -v
- sh -c "make $PARAMS"
- sh -c "$Cmd"

View File

@ -85,7 +85,7 @@ uninstall:
@$(MAKE) -C $(PRGDIR) $@
travis-install:
sudo $(MAKE) install
$(MAKE) install PREFIX=~/install_test_dir
test:
$(MAKE) -C $(TESTDIR) test
@ -96,19 +96,29 @@ cmake:
gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -I../lib -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
c_standards: clean
$(MAKE) all CFLAGS="-std=gnu90"
$(MAKE) clean
$(MAKE) all CFLAGS="-std=c99"
$(MAKE) clean
$(MAKE) all CFLAGS="-std=gnu99"
$(MAKE) clean
$(MAKE) all CFLAGS="-std=c11"
$(MAKE) clean
clangtest: clean
CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) all CC=clang
$(MAKE) all CC=clang CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion"
sanitize: clean
CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1
$(MAKE) test CC=clang FUZZER_TIME="-T1mn" NB_LOOPS=-i1 CFLAGS="-O3 -g -fsanitize=undefined"
staticAnalyze: clean
CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
scan-build --status-bugs -v $(MAKE) all CFLAGS=-g
armtest: clean
CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc
CFLAGS="-O3 -Werror" $(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc
CFLAGS="-O3 -Werror" $(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc
$(MAKE) -C $(LZ4DIR) all CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror"
$(MAKE) -C $(PRGDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror"
$(MAKE) -C $(TESTDIR) bins CC=arm-linux-gnueabi-gcc CFLAGS="-O3 -Werror"
versionsTest: clean
$(MAKE) -C $(TESTDIR) $@

7
NEWS
View File

@ -2,12 +2,17 @@ v1.7.2
Changed : moved to versioning; package, cli and library have same version number
Improved: Small decompression speed boost
Improved: Small compression speed improvement on 64-bits systems
Improved: Performance on ARMv6 and ARMv7
Improved: Small compression ratio and speed improvement on small files
Improved: Significant speed boost on ARMv6 and ARMv7
New : cli : --rm command
New : cli : file attributes are preserved, by Przemyslaw Skibinki
Added : Debianization, by Evgeniy Polyakov
Makefile: Generates object files (*.o) for faster (re)compilation on low power systems
New : recursive mode in benchmark
Fix : cli : crash on some invalid inputs
Fix : cli : -t correctly validates lz4-compressed files, by Nick Terrell
Fix : better ratio on 64-bits big-endian targets
Fix : cli : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243
r131
New : Dos/DJGPP target, thanks to Louis Santillan (#114)

View File

@ -412,7 +412,7 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
/*-******************************
* Compression functions
********************************/
static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
{
if (tableType == byU16)
return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
@ -420,7 +420,7 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
}
static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType)
static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
{
static const U64 prime5bytes = 889523592379ULL;
static const U64 prime8bytes = 11400714785074694791ULL;
@ -431,15 +431,10 @@ static U32 LZ4_hashSequence64(U64 sequence, tableType_t const tableType)
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType)
FORCE_INLINE U32 LZ4_hashPosition(const void* p, tableType_t const tableType)
{
if (LZ4_64bits()) return LZ4_hashSequence64(sequence, tableType);
return LZ4_hashSequence((U32)sequence, tableType);
}
static U32 LZ4_hashPosition(const void* p, tableType_t tableType)
{
return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType);
if ((LZ4_64bits()) && (tableType == byU32)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
return LZ4_hash4(LZ4_read32(p), tableType);
}
static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
@ -452,7 +447,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
}
}
static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
U32 const h = LZ4_hashPosition(p, tableType);
LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
@ -465,7 +460,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab
{ const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
}
static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
{
U32 const h = LZ4_hashPosition(p, tableType);
return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@ -655,7 +650,7 @@ _next_match:
_last_literals:
/* Encode Last Literals */
{ const size_t lastRun = (size_t)(iend - anchor);
{ size_t const lastRun = (size_t)(iend - anchor);
if ( (outputLimited) && /* Check output buffer overflow */
((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
return 0;
@ -683,12 +678,12 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
if (maxOutputSize >= LZ4_compressBound(inputSize)) {
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
} else {
if (inputSize < LZ4_64Klimit)
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
else
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
}
@ -701,10 +696,10 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
#else
LZ4_stream_t ctx;
void* ctxPtr = &ctx;
void* const ctxPtr = &ctx;
#endif
int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
#if (HEAPMODE)
FREEMEM(ctxPtr);

View File

@ -332,11 +332,11 @@ FORCE_INLINE int LZ4HC_encodeSequence (
static int LZ4HC_compress_generic (
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
void* const ctxvoid,
const char* const source,
char* const dest,
int const inputSize,
int const maxOutputSize,
int compressionLevel,
limitedOutput_directive limit
)
@ -381,7 +381,7 @@ static int LZ4HC_compress_generic (
_Search2:
if (ip+ml < mflimit)
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts);
else ml2 = ml;
if (ml2 == ml) { /* No better match */

View File

@ -466,20 +466,21 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* read first block */
readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile);
if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName);
filesize += readSize;
/* single-block file */
if (readSize < blockSize) {
/* Compress in single pass */
size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs);
if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize));
if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
compressedfilesize = cSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
(unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */
/* Write Block */
{ size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block");
if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block");
} }
else
@ -488,9 +489,9 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
{
/* Write Archive Header */
size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs);
if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
{ size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); }
if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
compressedfilesize += headerSize;
/* Main Loop */
@ -499,26 +500,26 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
/* Compress Block */
outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize));
if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
compressedfilesize += outSize;
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
/* Write Block */
{ size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile);
if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); }
if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
/* Read next block */
readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
filesize += readSize;
}
if (ferror(srcFile)) EXM_THROW(36, "Error reading %s ", srcFileName);
if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
/* End of Stream mark */
headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
if (LZ4F_isError(headerSize)) EXM_THROW(37, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize));
{ size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
if (sizeCheck!=headerSize) EXM_THROW(38, "Write error : cannot write end of stream"); }
if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
compressedfilesize += headerSize;
}
@ -532,7 +533,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
UTIL_setFileStat(dstFileName, &statbuf);
}
if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(39, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");

View File

@ -122,9 +122,9 @@ versionsTest:
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test-mem
test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32 test-mem32
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32
test-all: test test32