mirrors/lz4
0
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2024-11-23 18:04:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix 32-bits mode.

Browse Source 
Large File support for Mac OS-X in 32-bits mode
Fixed potential undefined behavior
Changed makefile for 32-bits mode
This commit is contained in:
Yann Collet 2016-11-17 13:02:06 -08:00
parent 7fde7438d3
commit 1abecbc33c
11 changed files with 174 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.travis.yml
View File

@ -17,7 +17,7 @@ matrix:
- os: linux
sudo: false
env: Ubu=12.04cont Cmd="make gpptest-native && make clean && make examples && make clean && make cmake && make clean && make travis-install && make clean && make clangtest-native" COMPILER=cc
env: Ubu=12.04cont Cmd="make gpptest-native && make clean examples && make clean cmake && make clean travis-install && make clean clangtest-native" COMPILER=cc
# 14.04 LTS Server Edition 64 bit
@ -87,7 +87,7 @@ matrix:
packages:
- valgrind
- env: Ubu=14.04 Cmd='make -C tests test-lz4 test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror' COMPILER=gcc-5
- env: Ubu=14.04 Cmd='make -C tests test-lz4 clean test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror' COMPILER=gcc-5
dist: trusty
sudo: required
addons:

11
Makefile
View File

@ -36,8 +36,8 @@ VOID := /dev/null
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
PRGDIR = programs
LZ4DIR = lib
PRGDIR = programs
TESTDIR = tests
@ -51,12 +51,15 @@ endif
.PHONY: default all lib lz4 clean test versionsTest examples
default: lz4
default: lib lz4
all: lib lz4
all:
@$(MAKE) -C $(LZ4DIR) $@
@$(MAKE) -C $(PRGDIR) $@
@$(MAKE) -C $(TESTDIR) $@
lib:
@$(MAKE) -C $(LZ4DIR) all
@$(MAKE) -C $(LZ4DIR)
lz4:
@$(MAKE) -C $(PRGDIR)

3
NEWS
View File

@ -1,3 +1,6 @@
v1.7.4
cli : fix : Large file support in 32-bits mode on Mac OS-X
v1.7.3
Changed : moved to versioning; package, cli and library have same version number
Improved: Small decompression speed boost

10
lib/Makefile
View File

@ -70,12 +70,18 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
default: lib
default: lib-release
all: lib
all32: CFLAGS+=-m32
all32: all
lib: liblz4.a liblz4
lib-release: CFLAGS := -O3
lib-release: lib
liblz4.a: *.c
ifeq ($(BUILD_STATIC),yes)
@echo compiling static library
@ -96,7 +102,7 @@ else
endif
clean:
@$(RM) -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* liblz4.pc dll/liblz4.dll dll/liblz4.lib
@$(RM) core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* liblz4.pc dll/liblz4.dll dll/liblz4.lib
@echo Cleaning library completed

8
lib/lz4.c
View File

@ -141,12 +141,14 @@
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef uintptr_t uptrval;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef size_t uptrval; /* generally true, except OpenVMS-64 */
#endif
@ -1139,8 +1141,8 @@ FORCE_INLINE int LZ4_decompress_generic(
s = *ip++;
length += s;
} while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) );
if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */
}
/* copy literals */
@ -1178,7 +1180,7 @@ FORCE_INLINE int LZ4_decompress_generic(
if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
length += s;
} while (s==255);
if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
}
length += MINMATCH;

2
lib/lz4.h
View File

@ -85,7 +85,7 @@ extern "C" {
/*========== Version =========== */
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)

26
programs/Makefile
View File

@ -29,18 +29,18 @@
DESTDIR ?=
PREFIX ?= /usr/local
VOID := /dev/null
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
VOID := /dev/null
CFLAGS ?= -O3 # allows custom optimization flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
# Define *.exe as extension for Windows systems
@ -52,27 +52,29 @@ endif
default: lz4
default: lz4-release
m32: lz4c32
all: lz4 lz4c
native: lz4 lz4c
all: native m32
all32: CFLAGS+=-m32
all32: all
lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
$(CC) $(FLAGS) $^ -o $@$(EXT)
lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o
lz4-release: CFLAGS := -O3
lz4-release: lz4
lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o
$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c datagen.c
$(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
lz4c32: CFLAGS += -m32
lz4c32: lz4
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
lz4$(EXT) lz4c$(EXT)
@echo Cleaning completed

23
programs/lz4io.c
View File

@ -34,6 +34,8 @@
* Compiler Options
**************************************/
#define _LARGE_FILES /* Large file support on 32-bits AIX */
#define _FILE_OFFSET_BITS 64 /* off_t width */
#define _LARGEFILE_SOURCE
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
@ -74,6 +76,9 @@
# define SET_SPARSE_FILE_MODE(file)
# endif
#else
# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__))
# define fseek fseeko
# endif
# define SET_BINARY_MODE(file)
# define SET_SPARSE_FILE_MODE(file)
#endif
@ -103,9 +108,6 @@
#define MIN_STREAM_BUFSIZE (192 KB)
#define LZ4IO_BLOCKSIZEID_DEFAULT 7
#define sizeT sizeof(size_t)
#define maskT (sizeT - 1)
/**************************************
* Macros
@ -623,11 +625,14 @@ static unsigned LZ4IO_readLE32 (const void* s)
return value32;
}
#define sizeT sizeof(size_t)
#define maskT (sizeT - 1)
static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
{
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
const size_t* ptrT = bufferT;
size_t bufferSizeT = bufferSize / sizeT;
size_t bufferSizeT = bufferSize / sizeT;
const size_t* const bufferTEnd = bufferT + bufferSizeT;
static const size_t segmentSizeT = (32 KB) / sizeT;
@ -716,11 +721,11 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
unsigned int blockSize;
/* Block Size */
{ size_t const sizeCheck = fread(in_buff, 1, 4, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
{ size_t const sizeCheck = fread(in_buff, 1, 4, finput);
if (sizeCheck == 0) break; /* Nothing to read : file read is completed */
if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
/* Cannot read next block : maybe new stream ? */
g_magicRead = blockSize;
break;

222
tests/Makefile
View File

@ -23,21 +23,18 @@
# - LZ4 source repository : https://github.com/lz4/lz4
# ##########################################################################
# fuzzer : Test tool, to check lz4 integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
# frametest : Test tool, to check lz4frame integrity on target platform
# frametest32: Same as frametest, but forced to compile in 32-bits mode
# fullbench : Precisely measure speed for each LZ4 function variant
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# datagen : generates synthetic data samples for tests & benchmarks
# ##########################################################################
DESTDIR ?=
PREFIX ?= /usr/local
VOID := /dev/null
BINDIR := $(PREFIX)/bin
MANDIR := $(PREFIX)/share/man/man1
LZ4DIR := ../lib
LIBDIR := ../lib
PRGDIR := ../programs
VOID := /dev/null
TESTDIR := versionsTest
PYTHON ?= python3
@ -46,8 +43,8 @@ CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum
-Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
# Define *.exe as extension for Windows systems
@ -56,6 +53,7 @@ EXT =.exe
else
EXT =
endif
LZ4 := $(PRGDIR)/lz4$(EXT)
# Default test parameters
@ -64,58 +62,48 @@ FUZZER_TIME := -T3mn
NB_LOOPS ?= -i1
default: native
default: all
m32: fullbench32 fuzzer32 frametest32
native: fullbench fuzzer frametest datagen fasttest
all: native m32
all: fullbench fuzzer frametest datagen fasttest
all32: CFLAGS+=-m32
all32: all
lz4:
$(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
lz4c:
$(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
lz4c32:
$(MAKE) -C $(PRGDIR) $@
lz4c32: # create a 32-bits version for 32/64 interop tests
$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
cp $(LZ4) $(LZ4)c32
fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
fullbench : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench-lib: fullbench.c $(LIBDIR)/xxhash.c
$(MAKE) -C $(LIBDIR) liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4.a
fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
fullbench-dll: fullbench.c $(LIBDIR)/xxhash.c
$(MAKE) -C $(LIBDIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LIBDIR)/dll/liblz4.dll
fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c
fuzzer : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c
frametest: $(LIBDIR)/lz4frame.o $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fasttest: $(LZ4DIR)/lz4.o fasttest.c
fasttest: $(LIBDIR)/lz4.o fasttest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
datagen : $(PRGDIR)/datagen.c datagencli.c
$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(MAKE) -C $(LIBDIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
fullbench-dll$(EXT) fullbench-lib$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
@ -134,40 +122,44 @@ versionsTest:
#FreeBSD targets
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
MD5:=md5sum
ifneq (,$(filter $(shell uname), Darwin ))
MD5:=md5 -r
endif
test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer
test32: test-lz4c32 test-frametest32 test-fullbench32 test-fuzzer32
test-all: test test32
test32: CFLAGS+=-m32
test32: test
test-lz4-sparse: lz4 datagen
@echo "\n ---- test sparse file support ----"
./datagen -g5M -P100 > tmpSrc
$(PRGDIR)/lz4 -B4D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB4
$(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4
diff -s tmpSrc tmpB4
$(PRGDIR)/lz4 -B5D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB5
$(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5
diff -s tmpSrc tmpB5
$(PRGDIR)/lz4 -B6D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB6
$(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6
diff -s tmpSrc tmpB6
$(PRGDIR)/lz4 -B7D tmpSrc | $(PRGDIR)/lz4 -dv --sparse > tmpB7
$(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7
diff -s tmpSrc tmpB7
$(PRGDIR)/lz4 tmpSrc | $(PRGDIR)/lz4 -dv --no-sparse > tmpNoSparse
$(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse
diff -s tmpSrc tmpNoSparse
ls -ls tmp*
./datagen -s1 -g1200007 -P100 | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -dv --sparse > tmpOdd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block)
./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
ls -ls tmpOdd
@$(RM) tmp*
@echo "\n Compatibility with Console :"
echo "Hello World 1 !" | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -d -c
echo "Hello World 2 !" | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -d | cat
echo "Hello World 3 !" | $(PRGDIR)/lz4 --no-frame-crc | $(PRGDIR)/lz4 -d -c
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
@echo "\n Compatibility with Append :"
./datagen -P100 -g1M > tmp1M
cat tmp1M tmp1M > tmp2M
$(PRGDIR)/lz4 -B5 -v tmp1M tmpC
$(PRGDIR)/lz4 -d -v tmpC tmpR
$(PRGDIR)/lz4 -d -v tmpC >> tmpR
$(LZ4) -B5 -v tmp1M tmpC
$(LZ4) -d -v tmpC tmpR
$(LZ4) -d -v tmpC >> tmpR
ls -ls tmp*
diff tmp2M tmpR
@$(RM) tmp*
@ -175,14 +167,15 @@ test-lz4-sparse: lz4 datagen
test-lz4-contentSize: lz4 datagen
@echo "\n ---- test original size support ----"
./datagen -g15M > tmp
$(PRGDIR)/lz4 -v tmp | $(PRGDIR)/lz4 -t
$(PRGDIR)/lz4 -v --content-size tmp | $(PRGDIR)/lz4 -d > tmp2
$(LZ4) -v tmp | $(LZ4) -t
$(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2
diff -s tmp tmp2
# test large size [2-4] GB
@./datagen -g3G -P100 | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 --decompress --force --sparse - tmp
@./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp
@ls -ls tmp
$(PRGDIR)/lz4 --quiet --content-size tmp | $(PRGDIR)/lz4 --verbose --decompress --force --sparse - tmp2
@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2
@ls -ls tmp2
diff -s tmp tmp2
@$(RM) tmp*
test-lz4-frame-concatenation: lz4 datagen
@ -190,10 +183,10 @@ test-lz4-frame-concatenation: lz4 datagen
@echo -n > empty.test
@echo hi > nonempty.test
cat nonempty.test empty.test nonempty.test > orig.test
@$(PRGDIR)/lz4 -zq empty.test > empty.lz4.test
@$(PRGDIR)/lz4 -zq nonempty.test > nonempty.lz4.test
@$(LZ4) -zq empty.test > empty.lz4.test
@$(LZ4) -zq nonempty.test > nonempty.lz4.test
cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test
$(PRGDIR)/lz4 -d concat.lz4.test > result.test
$(LZ4) -d concat.lz4.test > result.test
sdiff orig.test result.test
@$(RM) *.test
@echo frame concatenation test completed
@ -203,54 +196,54 @@ test-lz4-multiple: lz4 datagen
@./datagen -s1 > tmp1 2> $(VOID)
@./datagen -s2 -g100K > tmp2 2> $(VOID)
@./datagen -s3 -g1M > tmp3 2> $(VOID)
$(PRGDIR)/lz4 -f -m tmp*
$(LZ4) -f -m tmp*
ls -ls tmp*
@$(RM) tmp1 tmp2 tmp3
$(PRGDIR)/lz4 -df -m *.lz4
$(LZ4) -df -m *.lz4
ls -ls tmp*
$(PRGDIR)/lz4 -f -m tmp1 notHere tmp2; echo $$?
$(LZ4) -f -m tmp1 notHere tmp2; echo $$?
@$(RM) tmp*
test-lz4-basic: lz4 datagen
@echo "\n ---- test lz4 basic compression/decompression ----"
./datagen -g0 | $(PRGDIR)/lz4 -v | $(PRGDIR)/lz4 -t
./datagen -g16KB | $(PRGDIR)/lz4 -9 | $(PRGDIR)/lz4 -t
./datagen -g0 | $(LZ4) -v | $(LZ4) -t
./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t
./datagen -g20KB > tmpSrc
$(PRGDIR)/lz4 < tmpSrc | $(PRGDIR)/lz4 -d > tmpRes
$(LZ4) < tmpSrc | $(LZ4) -d > tmpRes
diff -q tmpSrc tmpRes
$(PRGDIR)/lz4 --no-frame-crc < tmpSrc | $(PRGDIR)/lz4 -d > tmpRes
$(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes
diff -q tmpSrc tmpRes
./datagen | $(PRGDIR)/lz4 | $(PRGDIR)/lz4 -t
./datagen -g6M -P99 | $(PRGDIR)/lz4 -9BD | $(PRGDIR)/lz4 -t
./datagen -g17M | $(PRGDIR)/lz4 -9v | $(PRGDIR)/lz4 -qt
./datagen -g33M | $(PRGDIR)/lz4 --no-frame-crc | $(PRGDIR)/lz4 -t
./datagen -g256MB | $(PRGDIR)/lz4 -vqB4D | $(PRGDIR)/lz4 -t
./datagen | $(LZ4) | $(LZ4) -t
./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt
./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
@$(RM) tmp*
test-lz4-hugefile: lz4 datagen
@echo "\n ---- test huge files compression/decompression ----"
./datagen -g6GB | $(PRGDIR)/lz4 -vB5D | $(PRGDIR)/lz4 -qt
./datagen -g6GB | $(PRGDIR)/lz4 -v5BD | $(PRGDIR)/lz4 -qt
./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt
@$(RM) tmp*
test-lz4-testmode: lz4 datagen
@echo "\n ---- bench mode ----"
$(PRGDIR)/lz4 -bi1
$(LZ4) -bi1
@echo "\n ---- test mode ----"
./datagen | $(PRGDIR)/lz4 -t && false || true
./datagen | $(PRGDIR)/lz4 -tf && false || true
./datagen | $(LZ4) -t && false || true
./datagen | $(LZ4) -tf && false || true
@echo "\n ---- pass-through mode ----"
./datagen | $(PRGDIR)/lz4 -d > $(VOID) && false || true
./datagen | $(PRGDIR)/lz4 -df > $(VOID)
./datagen | $(LZ4) -d > $(VOID) && false || true
./datagen | $(LZ4) -df > $(VOID)
@echo "Hello World !" > tmp1
$(PRGDIR)/lz4 -dcf tmp1
$(LZ4) -dcf tmp1
@echo "from underground..." > tmp2
$(PRGDIR)/lz4 -dcfm tmp1 tmp2
$(LZ4) -dcfm tmp1 tmp2
@echo "\n ---- test cli ----"
$(PRGDIR)/lz4 file-does-not-exist && false || true
$(PRGDIR)/lz4 -f file-does-not-exist && false || true
$(PRGDIR)/lz4 -fm file1-dne file2-dne && false || true
$(PRGDIR)/lz4 -fm file1-dne file2-dne && false || true
$(LZ4) file-does-not-exist && false || true
$(LZ4) -f file-does-not-exist && false || true
$(LZ4) -fm file1-dne file2-dne && false || true
$(LZ4) -fm file1-dne file2-dne && false || true
test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \
@ -258,53 +251,54 @@ test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
test-lz4c: lz4c datagen
@echo "\n ---- test lz4c version ----"
./datagen -g256MB | $(PRGDIR)/lz4c -l -v | $(PRGDIR)/lz4c -t
./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
test-lz4c32: CFLAGS+=-m32
test-lz4c32: test-lz4
test-interop-32-64: lz4 lz4c32 datagen
@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
./datagen -g16KB | $(PRGDIR)/lz4c32 -9 | $(PRGDIR)/lz4 -t
./datagen -P10 | $(PRGDIR)/lz4 -9B4 | $(PRGDIR)/lz4c32 -t
./datagen | $(PRGDIR)/lz4c32 | $(PRGDIR)/lz4 -t
./datagen -g1M | $(PRGDIR)/lz4 -3B5 | $(PRGDIR)/lz4c32 -t
./datagen -g256MB | $(PRGDIR)/lz4c32 -vqB4D | $(PRGDIR)/lz4 -qt
./datagen -g1G -P90 | $(PRGDIR)/lz4 | $(PRGDIR)/lz4c32 -t
./datagen -g6GB | $(PRGDIR)/lz4c32 -vq9BD | $(PRGDIR)/lz4 -qt
./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4) -t
./datagen -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t
./datagen | $(LZ4)c32 | $(LZ4) -t
./datagen -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t
./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt
./datagen -g1G -P90 | $(LZ4) | $(LZ4)c32 -t
./datagen -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt
test-lz4c32-basic: lz4c32 datagen
@echo "\n ---- test lz4c32 32-bits version ----"
./datagen -g16KB | $(PRGDIR)/lz4c32 -9 | $(PRGDIR)/lz4c32 -t
./datagen | $(PRGDIR)/lz4c32 | $(PRGDIR)/lz4c32 -t
./datagen -g256MB | $(PRGDIR)/lz4c32 -vqB4D | $(PRGDIR)/lz4c32 -qt
./datagen -g6GB | $(PRGDIR)/lz4c32 -vqB5D | $(PRGDIR)/lz4c32 -qt
./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t
./datagen | $(LZ4)c32 | $(LZ4)c32 -t
./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt
./datagen -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt
test-platform:
@echo "\n ---- test lz4 $(QEMU_SYS) platform ----"
$(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(PRGDIR)/lz4 -9 | $(QEMU_SYS) $(PRGDIR)/lz4 -t
$(QEMU_SYS) ./datagen | $(QEMU_SYS) $(PRGDIR)/lz4 | $(QEMU_SYS) $(PRGDIR)/lz4 -t
$(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(PRGDIR)/lz4 -vqB4D | $(QEMU_SYS) $(PRGDIR)/lz4 -qt
$(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t
$(QEMU_SYS) ./datagen | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t
$(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt
ifneq ($(QEMU_SYS),qemu-arm-static)
$(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(PRGDIR)/lz4 -vqB5D | $(QEMU_SYS) $(PRGDIR)/lz4 -qt
$(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt
endif
test-lz4c32: test-lz4c32-basic test-interop-32-64
test-fullbench: fullbench
./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES)
test-fullbench32: CFLAGS += -m32
test-fullbench32: test-fullbench
test-fuzzer: fuzzer
./fuzzer $(FUZZER_TIME)
test-fuzzer32: fuzzer32
./fuzzer32 $(FUZZER_TIME)
test-fuzzer32: CFLAGS += -m32
test-fuzzer32: test-fuzzer
test-frametest: frametest
./frametest $(FUZZER_TIME)
test-frametest32: frametest32
./frametest32 $(FUZZER_TIME)
test-frametest32: CFLAGS += -m32
test-frametest32: test-frametest
test-fasttest: fasttest
./fasttest
@ -313,17 +307,17 @@ test-mem: lz4 datagen fuzzer frametest fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
./datagen -g16KB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -9 -BD -f tmp $(VOID)
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f tmp $(VOID)
./datagen -g16KB -s2 > tmp2
./datagen -g16KB -s3 > tmp3
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 --force --multiple tmp tmp2 tmp3
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple tmp tmp2 tmp3
./datagen -g16MB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -t tmp2
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f tmp tmp2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t tmp2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 tmp
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
./datagen -g256MB > tmp
valgrind --leak-check=yes --error-exitcode=1 $(PRGDIR)/lz4 -B4D -f -vq tmp $(VOID)
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq tmp $(VOID)
$(RM) tmp*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256

63
tests/fullbench.c
View File

@ -229,16 +229,6 @@ static int local_LZ4_saveDict(const char* in, char* out, int inSize)
return LZ4_saveDict(&LZ4_stream, out, inSize);
}
static int local_LZ4_compress_default(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
}
static int local_LZ4_compress_default_large(const char* in, char* out, int inSize)
{
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
@ -279,26 +269,6 @@ static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSi
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
}
static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 1);
}
static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1, 1);
}
static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 1);
}
static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1, 1);
}
#ifndef LZ4_DLL_IMPORT
/* declare hidden function */
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
@ -328,31 +298,16 @@ static int local_LZ4_compress_HC(const char* in, char* out, int inSize)
return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize), 9);
}
static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
static int local_LZ4_compress_HC_extStateHC(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize), 9);
}
static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1, 9);
}
static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
{
return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize)-1, 9);
}
static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
static int local_LZ4_compress_HC_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize));
}
static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{
return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1);
}
/* decompression functions */
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
@ -530,11 +485,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break;
case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break;
case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 12: compressionFunction = local_LZ4_compress_HC_extStateHC; compressorName = "LZ4_compress_HC_extStateHC"; break;
case 14: compressionFunction = local_LZ4_compress_HC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compress_HC_continue"; break;
#ifndef LZ4_DLL_IMPORT
case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
#endif
@ -549,13 +501,6 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; }
LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize);
break;
case 60: DISPLAY("Obsolete compression functions : \n"); continue;
case 61: compressionFunction = local_LZ4_compress_default; compressorName = "LZ4_compress_default"; break;
case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break;
case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break;
default :
continue; /* unknown ID : just skip */
}

15
tests/fuzzer.c
View File

@ -170,7 +170,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
static int FUZ_AddressOverflow(void)
{
char* buffers[MAX_NB_BUFF_I134+1];
int i, nbBuff=0;
int nbBuff=0;
int highAddress = 0;
DISPLAY("Overflow tests : ");
@ -185,12 +185,13 @@ static int FUZ_AddressOverflow(void)
buffers[0] = (char*)malloc(BLOCKSIZE_I134);
buffers[1] = (char*)malloc(BLOCKSIZE_I134);
if ((!buffers[0]) || (!buffers[1])) {
free(buffers[0]); free(buffers[1]);
DISPLAY("not enough memory for tests \n");
return 0;
}
for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) {
DISPLAY("%3i \b\b\b\b", nbBuff);
DISPLAY("%3i \b\b\b\b", nbBuff); fflush(stdout);
buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
if (buffers[nbBuff]==NULL) goto _endOfTests;
@ -201,7 +202,7 @@ static int FUZ_AddressOverflow(void)
}
{ size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512);
int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1);
char* const input = buffers[nbBuff-1];
char* output = buffers[nbBuff];
int r;
@ -209,15 +210,15 @@ static int FUZ_AddressOverflow(void)
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
for(i = 4; i <= nbOf255+4; i++) input[i] = (char)0xff;
{ unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
input[0] = (char)0x1F; /* Match length overflow */
input[1] = (char)0x01;
input[2] = (char)0x01;
input[3] = (char)0x00;
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) goto _overflowError;
if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
output = buffers[nbBuff-2]; /* Reverse in/out pointer order */
input[0] = (char)0xF0; /* Literal length overflow */
@ -237,7 +238,7 @@ static int FUZ_AddressOverflow(void)
nbBuff++;
_endOfTests:
for (i=0 ; i<nbBuff; i++) free(buffers[i]);
{ int i; for (i=0 ; i<nbBuff; i++) free(buffers[i]); }
if (!highAddress) DISPLAY("high address not possible \n");
else DISPLAY("all overflows correctly detected \n");
return 0;