mirrors/lz4
0
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2024-12-04 15:26:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7 from Cyan4973/dev

Browse Source 
LZ4 streaming API
This commit is contained in:
Cyan4973 2014-06-24 20:57:37 +02:00
commit 9d8e1f58a0
15 changed files with 1875 additions and 734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.travis.yml
View File

@ -4,3 +4,11 @@ script: make test
before_install: before_install:
- sudo apt-get update -qq - sudo apt-get update -qq
- sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq gcc-multilib
- sudo apt-get install -qq valgrind
env:
- LZ4_TRAVIS_CI_ENV=-m32
- LZ4_TRAVIS_CI_ENV=-m64
matrix:
fast_finish: true

19
Makefile
View File

@ -30,9 +30,10 @@
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################ # ################################################################
export RELEASE=r117 # Version numbers
export RELEASE=r118
LIBVER_MAJOR=1 LIBVER_MAJOR=1
LIBVER_MINOR=0 LIBVER_MINOR=2
LIBVER_PATCH=0 LIBVER_PATCH=0
LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
@ -41,7 +42,7 @@ PREFIX=/usr
CC := $(CC) CC := $(CC)
CFLAGS+= -I. -std=c99 -O3 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\" CFLAGS+= -I. -std=c99 -O3 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\"
LIBDIR=$(PREFIX)/lib LIBDIR?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include INCLUDEDIR=$(PREFIX)/include
PRGDIR = programs PRGDIR = programs
DISTRIBNAME=lz4-$(RELEASE).tar.gz DISTRIBNAME=lz4-$(RELEASE).tar.gz
@ -92,7 +93,7 @@ lz4programs: lz4.c lz4hc.c
liblz4: lz4.c lz4hc.c liblz4: lz4.c lz4hc.c
@echo compiling static library @echo compiling static library
@$(CC) $(CFLAGS) -c $^ @$(CC) $(CFLAGS) -c $^
@ar rcs liblz4.a lz4.o lz4hc.o @$(AR) rcs liblz4.a lz4.o lz4hc.o
@echo compiling dynamic library @echo compiling dynamic library
@$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) @$(CC) $(CFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@echo creating versioned links @echo creating versioned links
@ -105,25 +106,25 @@ clean:
@echo Cleaning completed @echo Cleaning completed
#make install option is reserved to Linux & OSX targets #make install option is designed for Linux & OSX targets only
ifneq (,$(filter $(shell uname),Linux Darwin)) ifneq (,$(filter $(shell uname),Linux Darwin))
install: liblz4 install: liblz4
@install -d -m 755 $(DESTDIR)$(LIBDIR)/ $(DESTDIR)$(INCLUDEDIR)/ @install -d -m 755 $(DESTDIR)$(LIBDIR)/ $(DESTDIR)$(INCLUDEDIR)/
@install -m 755 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
@install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
@cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) @cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
@cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) @cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
@install -m 755 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
@install -m 755 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
@install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@echo lz4 static and shared library installed @echo lz4 static and shared library installed
@cd $(PRGDIR); $(MAKE) -e install @cd $(PRGDIR); $(MAKE) -e install
uninstall: uninstall:
[ -x $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
[ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) [ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h [ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h
[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h [ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
@echo lz4 libraries successfully uninstalled @echo lz4 libraries successfully uninstalled

5
README.md
View File

@ -4,6 +4,11 @@ LZ4 - Extremely fast compression
LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio. A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio.
|Branch |Status |
|------------|---------|
|master | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=master)](https://travis-ci.org/Cyan4973/lz4) |
|dev | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=dev)](https://travis-ci.org/Cyan4973/lz4) |
This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/). This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/).
The intention is to offer github's capabilities to lz4 users, such as cloning, branch, or source download. The intention is to offer github's capabilities to lz4 users, such as cloning, branch, or source download.

774
lz4.c
View File

File diff suppressed because it is too large Load Diff

250
lz4.h
View File

@ -28,8 +28,8 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at : You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/ - LZ4 source repository : http://code.google.com/p/lz4/
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/ */
#pragma once #pragma once
@ -42,16 +42,21 @@ extern "C" {
Version Version
**************************************/ **************************************/
#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ #define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */
#define LZ4_VERSION_MINOR 1 /* for minor interface/format changes */ #define LZ4_VERSION_MINOR 2 /* for minor interface/format changes */
#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
/************************************** /**************************************
Compiler Options Tuning parameter
**************************************/ **************************************/
#if (defined(__GNUC__) && defined(__STRICT_ANSI__)) || (defined(_MSC_VER) && !defined(__cplusplus)) /* Visual Studio */ /*
# define inline __inline /* Visual C is not C99, but supports some kind of inline */ * LZ4_MEMORY_USAGE :
#endif * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#define LZ4_MEMORY_USAGE 14
/************************************** /**************************************
@ -59,7 +64,7 @@ extern "C" {
**************************************/ **************************************/
int LZ4_compress (const char* source, char* dest, int inputSize); int LZ4_compress (const char* source, char* dest, int inputSize);
int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize); int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxOutputSize);
/* /*
LZ4_compress() : LZ4_compress() :
@ -72,10 +77,21 @@ LZ4_compress() :
or 0 if the compression fails or 0 if the compression fails
LZ4_decompress_safe() : LZ4_decompress_safe() :
maxOutputSize : is the size of the destination buffer (which must be already allocated) compressedSize : is obviously the source size
maxOutputSize : is the size of the destination buffer, which must be already allocated.
return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
If the destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result. If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets This function is protected against buffer overflow exploits :
it never writes outside of output buffer, and never reads outside of input buffer.
Therefore, it is protected against malicious data packets.
*/
/*
Note :
Should you prefer to explicitly allocate compression-table memory using your own allocation method,
use the streaming functions provided below, simply reset the memory area between each call to LZ4_compress_continue()
*/ */
@ -89,7 +105,6 @@ LZ4_decompress_safe() :
LZ4_compressBound() : LZ4_compressBound() :
Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible) Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
primarily useful for memory allocation of output buffer. primarily useful for memory allocation of output buffer.
inline function is recommended for the general case,
macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation). macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
@ -118,17 +133,18 @@ LZ4_decompress_fast() :
originalSize : is the original and therefore uncompressed size originalSize : is the original and therefore uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size) return : the number of bytes read from the source buffer (in other words, the compressed size)
If the source stream is malformed, the function will stop decoding and return a negative result. If the source stream is malformed, the function will stop decoding and return a negative result.
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
note : This function is a bit faster than LZ4_decompress_safe() note : This function is a bit faster than LZ4_decompress_safe()
This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet. It provides fast decompression and fully respect memory boundaries for properly formed compressed data.
Use this function preferably into a trusted environment (data to decode comes from a trusted source). It does not provide full protection against intentionnally modified data stream.
Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes. Use this function in a trusted environment (data to decode comes from a trusted source).
*/ */
int LZ4_decompress_fast (const char* source, char* dest, int originalSize); int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
/* /*
LZ4_decompress_safe_partial() : LZ4_decompress_safe_partial() :
This function decompress a compressed block of size 'inputSize' at position 'source' This function decompress a compressed block of size 'compressedSize' at position 'source'
into output buffer 'dest' of size 'maxOutputSize'. into output buffer 'dest' of size 'maxOutputSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time. reducing decompression time.
@ -138,110 +154,152 @@ LZ4_decompress_safe_partial() :
If the source stream is detected malformed, the function will stop decoding and return a negative result. If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
*/ */
int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize); int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize);
/***********************************************
Experimental Streaming Compression Functions
***********************************************/
#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U32 * sizeof(unsigned int))
/*
* LZ4_stream_t
* information structure to track an LZ4 stream.
* important : set this structure content to zero before first use !
*/
typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t;
/*
* If you prefer dynamic allocation methods,
* LZ4_createStream
* provides a pointer (void*) towards an initialized LZ4_stream_t structure.
* LZ4_free just frees it.
*/
void* LZ4_createStream();
int LZ4_free (void* LZ4_stream);
/* /*
These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. * LZ4_loadDict
To know how much memory must be allocated for the compression tables, use : * Use this function to load a static dictionary into LZ4_stream.
int LZ4_sizeofState(); * Any previous data will be forgotten, only 'dictionary' will remain in memory.
* Loading a size of 0 is allowed (same effect as init).
Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0). * Return : 1 if OK, 0 if error
The allocated memory can be provided to the compressions functions using 'void* state' parameter.
LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
*/ */
int LZ4_sizeofState(void); int LZ4_loadDict (void* LZ4_stream, const char* dictionary, int dictSize);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
/**************************************
Streaming Functions
**************************************/
void* LZ4_create (const char* inputBuffer);
int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
char* LZ4_slideInputBuffer (void* LZ4_Data);
int LZ4_free (void* LZ4_Data);
/* /*
These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. * LZ4_compress_continue
In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function : * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
* Previous data blocks are assumed to still be present at their previous location.
void* LZ4_create (const char* inputBuffer);
The result of the function is the (void*) pointer on the LZ4 Data Structure.
This pointer will be needed in all other functions.
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
The input buffer must be already allocated, and size at least 192KB.
'inputBuffer' will also be the 'const char* source' of the first block.
All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
If next block does not begin immediately after the previous one, the compression will fail (return 0).
When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
char* LZ4_slideInputBuffer(void* LZ4_Data);
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
==> The memory position where the next input data block must start is provided as the result of the function.
Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
*/ */
int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize);
int LZ4_sizeofStreamState(void);
int LZ4_resetStreamState(void* state, const char* inputBuffer);
/* /*
These functions achieve the same result as : * LZ4_compress_limitedOutput_continue
void* LZ4_create (const char* inputBuffer); * Same as before, but also specify a maximum target compressed size (maxOutputSize)
* If objective cannot be met, compression exits, and returns a zero.
They are provided here to allow the user program to allocate memory using its own routines.
To know how much space must be allocated, use LZ4_sizeofStreamState();
Note also that space must be 4-bytes aligned.
Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
void* state is a pointer to the space allocated.
It must be aligned on 4-bytes boundaries, and be large enough.
The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
The input buffer must be already allocated, and size at least 192KB.
'inputBuffer' will also be the 'const char* source' of the first block.
The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
return value of LZ4_resetStreamState() must be 0 is OK.
Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
*/ */
int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
/* /*
*_withPrefix64k() : * LZ4_saveDict
These decoding functions work the same as their "normal name" versions, * If previously compressed data block is not guaranteed to remain at its previous memory location
but can use up to 64KB of data in front of 'char* dest'. * save it into a safe place (char* safeBuffer)
These functions are necessary to decode inter-dependant blocks. * Note : you don't need to call LZ4_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call again LZ4_compress_continue()
* Return : 1 if OK, 0 if error
* Note : any dictSize > 64 KB will be interpreted as 64KB.
*/ */
int LZ4_saveDict (void* LZ4_stream, char* safeBuffer, int dictSize);
/************************************************
Experimental Streaming Decompression Functions
************************************************/
#define LZ4_STREAMDECODESIZE_U32 4
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
/*
* LZ4_streamDecode_t
* information structure to track an LZ4 stream.
* important : set this structure content to zero before first use !
*/
typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
/*
* If you prefer dynamic allocation methods,
* LZ4_createStreamDecode()
* provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
* LZ4_free just frees it.
*/
void* LZ4_createStreamDecode();
int LZ4_free (void* LZ4_stream); /* yes, it's the same one as for compression */
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks must still be available at the memory position where they were decoded.
If it's not possible, save the relevant part of decoded data into a safe buffer,
and indicate where it stands using LZ4_setDictDecode()
*/
int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize);
int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, char* dest, int originalSize);
/*
* LZ4_setDictDecode
* Use this function to instruct where to find the dictionary.
* This function can be used to specify a static dictionary,
* or to instruct where to find some previously decoded data saved into a different memory space.
* Setting a size of 0 is allowed (same effect as no dictionary).
* Return : 1 if OK, 0 if error
*/
int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictSize);
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue()
all together into a single function call.
It doesn't use nor update an LZ4_streamDecode_t structure.
*/
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
/************************************** /**************************************
Obsolete Functions Obsolete Functions
**************************************/ **************************************/
/* /*
These functions are deprecated and should no longer be used. Obsolete decompression functions
They are provided here for compatibility with existing user programs. These function names are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
- LZ4_uncompress is the same as LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
*/ */
int LZ4_uncompress (const char* source, char* dest, int outputSize); int LZ4_uncompress (const char* source, char* dest, int outputSize);
int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/* Obsolete functions for externally allocated state; use streaming interface instead */
int LZ4_sizeofState(void);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete streaming functions; use new streaming interface whenever possible */
void* LZ4_create (const char* inputBuffer);
int LZ4_sizeofStreamState(void);
int LZ4_resetStreamState(void* state, const char* inputBuffer);
char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize);
int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
#if defined (__cplusplus) #if defined (__cplusplus)
} }

6
lz4hc.c
View File

@ -54,8 +54,9 @@
**************************************/ **************************************/
/* 32 or 64 bits ? */ /* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|| defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ || defined(__powerpc64__) || defined(__powerpc64le__) \
|| defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ || defined(__ppc64__) || defined(__ppc64le__) \
|| defined(__PPC64__) || defined(__PPC64LE__) \
|| defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1 # define LZ4_ARCH64 1
#else #else
@ -66,6 +67,7 @@
* Little Endian or Big Endian ? * Little Endian or Big Endian ?
* Overwrite the #define below if you know your architecture endianess * Overwrite the #define below if you know your architecture endianess
*/ */
#include <stdlib.h> /* Apparently required to detect endianess */
#if defined (__GLIBC__) #if defined (__GLIBC__)
# include <endian.h> # include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN) # if (__BYTE_ORDER == __BIG_ENDIAN)

1
lz4hc.h
View File

@ -104,6 +104,7 @@ They just use the externally allocated memory area instead of allocating their o
/************************************** /**************************************
Streaming Functions Streaming Functions
**************************************/ **************************************/
/* Note : these streaming functions still follows the older model */
void* LZ4_createHC (const char* inputBuffer); void* LZ4_createHC (const char* inputBuffer);
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);

77
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ################################################################ # ################################################################
RELEASE=r116 RELEASE=r118
DESTDIR= DESTDIR=
PREFIX=/usr PREFIX=/usr
CC:=$(CC) CC:=$(CC)
@ -42,18 +42,32 @@ MANDIR=$(PREFIX)/share/man/man1
LZ4DIR=.. LZ4DIR=..
TEST_FILES = COPYING TEST_FILES = COPYING
TEST_TARGETS=test-64 test-32
BENCH_NB=-i5
# Minimize test target for Travis CI's Build Matrix
ifeq ($(LZ4_TRAVIS_CI_ENV),-m32)
TEST_TARGETS=test-32
BENCH_NB=-i1
else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64)
TEST_TARGETS=test-64
BENCH_NB=-i1
endif
# Define *.exe as extension for Windows systems # Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS))) ifneq (,$(filter Windows%,$(OS)))
EXT =.exe EXT =.exe
VOID = nul
else else
EXT = EXT =
VOID = /dev/null
endif endif
default: lz4 lz4c default: lz4 lz4c
all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32 all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 datagen
lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) $(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
@ -64,22 +78,27 @@ lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
datagen : datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean: clean:
@rm -f core *.o \ @rm -f core *.o \
lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) datagen$(EXT)
@echo Cleaning completed @echo Cleaning completed
@ -102,15 +121,35 @@ uninstall:
[ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4 [ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4
[ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c [ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c
[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1 [ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
[ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1
[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
@echo lz4 successfully uninstalled @echo lz4 successfully uninstalled
test: test-lz4 test-lz4c test-lz4c32 test-fuzzer test-fuzzer32 test-fullbench test-fullbench32 test: $(TEST_TARGETS)
test-lz4: test-64: test-lz4 test-lz4c test-fullbench test-fuzzer test-mem
test-lz4c: test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32 test-mem32
test-lz4c32: test-lz4: lz4 datagen
./datagen | ./lz4 | ./lz4 -vdq > $(VOID)
./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
test-lz4c: lz4c datagen
test-lz4c32: lz4c32 lz4 datagen
./datagen | ./lz4c32 | ./lz4c32 -vdq > $(VOID)
./datagen | ./lz4c32 | ./lz4 -vdq > $(VOID)
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -vdq > $(VOID)
./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
test-fullbench: fullbench
./fullbench --no-prompt $(BENCH_NB) $(TEST_FILES)
test-fullbench32: fullbench32
./fullbench32 --no-prompt $(BENCH_NB) $(TEST_FILES)
test-fuzzer: fuzzer test-fuzzer: fuzzer
./fuzzer --no-prompt ./fuzzer --no-prompt
@ -118,10 +157,14 @@ test-fuzzer: fuzzer
test-fuzzer32: fuzzer32 test-fuzzer32: fuzzer32
./fuzzer32 --no-prompt ./fuzzer32 --no-prompt
test-fullbench: fullbench test-mem: lz4 datagen
./fullbench --no-prompt $(TEST_FILES) ./datagen -g256MB > tmp
valgrind ./lz4 -B4D -f tmp /dev/null
./datagen -g16MB > tmp
valgrind ./lz4 -9 -B5D -f tmp /dev/null
rm tmp
test-fullbench32: fullbench32 test-mem32: lz4c32 datagen
./fullbench32 --no-prompt $(TEST_FILES) # unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system...
endif endif

283
programs/datagen.c Normal file
View File

@ -0,0 +1,283 @@
/*
datagen.c - compressible data generator test tool
Copyright (C) Yann Collet 2012-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
*/
/**************************************
Remove Visual warning messages
**************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets
/**************************************
Includes
**************************************/
//#include <stdlib.h>
#include <stdio.h> // fgets, sscanf
#include <string.h> // strcmp
/**************************************
Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
Constants
**************************************/
#ifndef LZ4_VERSION
# define LZ4_VERSION "rc118"
#endif
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
#define CDG_SIZE_DEFAULT (64 KB)
#define CDG_SEED_DEFAULT 0
#define CDG_COMPRESSIBILITY_DEFAULT 50
#define PRIME1 2654435761U
#define PRIME2 2246822519U
/**************************************
Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/**************************************
Local Parameters
**************************************/
static int no_prompt = 0;
static char* programName;
static int displayLevel = 2;
/*********************************************************
Fuzzer functions
*********************************************************/
#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int CDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 = CDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
}
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
#define CDG_RANDCHAR (((CDG_rand(seed) >> 9) & 63) + '0')
static void CDG_generate(U64 size, U32* seed, double proba)
{
BYTE fullbuff[32 KB + 128 KB + 1];
BYTE* buff = fullbuff + 32 KB;
U64 total=0;
U32 P32 = (U32)(32768 * proba);
U32 pos=0;
U32 genBlockSize = 128 KB;
// Build initial prefix
while (pos<32 KB)
{
// Select : Literal (char) or Match (within 32K)
if (CDG_RAND15BITS < P32)
{
// Copy (within 64K)
U32 d;
int ref;
int length = CDG_RANDLENGTH + 4;
U32 offset = CDG_RAND15BITS + 1;
if (offset > pos) offset = pos;
ref = pos - offset;
d = pos + length;
while (pos < d) fullbuff[pos++] = fullbuff[ref++];
}
else
{
// Literal (noise)
U32 d;
int length = CDG_RANDLENGTH;
d = pos + length;
while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
}
}
// Generate compressible data
pos = 0;
while (total < size)
{
if (size-total < 128 KB) genBlockSize = (U32)(size-total);
total += genBlockSize;
buff[genBlockSize] = 0;
pos = 0;
while (pos<genBlockSize)
{
// Select : Literal (char) or Match (within 32K)
if (CDG_RAND15BITS < P32)
{
// Copy (within 64K)
int ref;
U32 d;
int length = CDG_RANDLENGTH + 4;
U32 offset = CDG_RAND15BITS + 1;
if (pos + length > genBlockSize ) length = genBlockSize - pos;
ref = pos - offset;
d = pos + length;
while (pos < d) buff[pos++] = buff[ref++];
}
else
{
// Literal (noise)
U32 d;
int length = CDG_RANDLENGTH;
if (pos + length > genBlockSize) length = genBlockSize - pos;
d = pos + length;
while (pos < d) buff[pos++] = CDG_RANDCHAR;
}
}
pos=0;
for (;pos+512<=genBlockSize;pos+=512) printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
// Regenerate prefix
memcpy(fullbuff, buff + 96 KB, 32 KB);
}
}
int CDG_usage()
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
DISPLAY( " %s [size] [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -g# : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
DISPLAY( " -s# : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, char** argv)
{
int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT;
U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT;
// Check command line
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
while (argument[1]!=0)
{
argument++;
switch(*argument)
{
case 'h':
return CDG_usage();
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
{
size *= 10;
size += *argument - '0';
argument++;
}
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 'p':
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
case 'v':
displayLevel = 4;
break;
default: ;
}
}
}
}
// Get Seed
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
CDG_generate(size, &seed, ((double)proba) / 100);
return 0;
}

187
programs/fullbench.c
View File

@ -118,8 +118,8 @@
#define MAX_MEM (1984<<20) #define MAX_MEM (1984<<20)
#define DEFAULT_CHUNKSIZE (4<<20) #define DEFAULT_CHUNKSIZE (4<<20)
#define ALL_COMPRESSORS -1 #define ALL_COMPRESSORS 0
#define ALL_DECOMPRESSORS -1 #define ALL_DECOMPRESSORS 0
//************************************** //**************************************
@ -250,78 +250,108 @@ static U64 BMK_GetFileSize(char* infilename)
} }
//********************************************************* /*********************************************************
// Public function Benchmark function
//********************************************************* *********************************************************/
static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
{ {
return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
} }
static void* stateLZ4; static void* stateLZ4;
static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize) static int local_LZ4_compress_withState(const char* in, char* out, int inSize)
{ {
return LZ4_compress_withState(stateLZ4, in, out, inSize); return LZ4_compress_withState(stateLZ4, in, out, inSize);
} }
static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
{ {
return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
} }
static void* ctx; static void* ctx;
static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize) static int local_LZ4_compress_continue(const char* in, char* out, int inSize)
{ {
return LZ4_compress_continue(ctx, in, out, inSize); return LZ4_compress_continue(ctx, in, out, inSize);
} }
static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
{ {
return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
} }
LZ4_stream_t LZ4_dict;
static void* local_LZ4_resetDictT(const char* fake)
{
(void)fake;
memset(&LZ4_dict, 0, sizeof(LZ4_stream_t));
return NULL;
}
int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize);
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
{
return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize);
}
static void* stateLZ4HC; static void* stateLZ4HC;
static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
{ {
return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize); return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize);
} }
static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
{ {
return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
} }
static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
{ {
return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
} }
static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
{ {
return LZ4_compressHC_continue(ctx, in, out, inSize); return LZ4_compressHC_continue(ctx, in, out, inSize);
} }
static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
{ {
return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
} }
static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
{ {
(void)inSize; (void)inSize;
LZ4_decompress_fast(in, out, outSize); LZ4_decompress_fast(in, out, outSize);
return outSize; return outSize;
} }
static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize) static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
{ {
(void)inSize; (void)inSize;
LZ4_decompress_fast_withPrefix64k(in, out, outSize); LZ4_decompress_fast_withPrefix64k(in, out, outSize);
return outSize; return outSize;
} }
static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_fast_usingDict(in, out, outSize, in - 65536, 65536);
return outSize;
}
static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
{
(void)inSize;
LZ4_decompress_safe_usingDict(in, out, inSize, outSize, in - 65536, 65536);
return outSize;
}
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
{ {
return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
} }
@ -330,24 +360,16 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
{ {
int fileIdx=0; int fileIdx=0;
char* orig_buff; char* orig_buff;
# define NB_COMPRESSION_ALGORITHMS 12 # define NB_COMPRESSION_ALGORITHMS 13
# define MINCOMPRESSIONCHAR '0' # define MINCOMPRESSIONCHAR '0'
# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0};
static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0};
"LZ4_compress_withState", "LZ4_compress_limitedOutput_withState", # define NB_DECOMPRESSION_ALGORITHMS 7
"LZ4_compress_continue", "LZ4_compress_limitedOutput_continue",
"LZ4_compressHC", "LZ4_compressHC_limitedOutput",
"LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC",
"LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
# define NB_DECOMPRESSION_ALGORITHMS 5
# define MINDECOMPRESSIONCHAR '0' # define MINDECOMPRESSIONCHAR '0'
# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS) # define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)
static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" }; static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_fast_usingDict",
double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0}; "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_usingDict", "LZ4_decompress_safe_partial" };
double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0};
U64 totals = 0;
// Loop for each file // Loop for each file
@ -450,9 +472,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
DISPLAY(" %s : \n", inFileName); DISPLAY(" %s : \n", inFileName);
// Compression Algorithms // Compression Algorithms
for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
{ {
char* cName = compressionNames[cAlgNb]; char* compressorName;
int (*compressionFunction)(const char*, char*, int); int (*compressionFunction)(const char*, char*, int);
void* (*initFunction)(const char*) = NULL; void* (*initFunction)(const char*) = NULL;
double bestTime = 100000000.; double bestTime = 100000000.;
@ -461,18 +483,19 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
switch(cAlgNb) switch(cAlgNb)
{ {
case 0 : compressionFunction = LZ4_compress; break; case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break;
case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break; case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break;
case 2 : compressionFunction = local_LZ4_compress_withState; break; case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break;
case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break; case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break;
case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break; case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break;
case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break; case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break;
case 6 : compressionFunction = LZ4_compressHC; break; case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break;
case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break; case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break;
case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break; case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break;
case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break; case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break;
case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break; case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break;
case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break; case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break;
case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
} }
@ -481,7 +504,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
double averageTime; double averageTime;
int milliTime; int milliTime;
PROGRESS("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize); PROGRESS("%1i-%-26.26s : %9i ->\r", loopNb, compressorName, (int)benchedSize);
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
nb_loops = 0; nb_loops = 0;
@ -494,7 +517,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{ {
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1); if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
} }
if (initFunction!=NULL) free(ctx); if (initFunction!=NULL) free(ctx);
nb_loops++; nb_loops++;
@ -505,13 +528,13 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
if (averageTime < bestTime) bestTime = averageTime; if (averageTime < bestTime) bestTime = averageTime;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.; ratio = (double)cSize/(double)benchedSize*100.;
PROGRESS("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); PROGRESS("%1i-%-26.26s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
} }
if (ratio<100.) if (ratio<100.)
DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); DISPLAY("%-28.28s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
else else
DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); DISPLAY("%-28.28s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
totalCTime[cAlgNb] += bestTime; totalCTime[cAlgNb] += bestTime;
totalCSize[cAlgNb] += cSize; totalCSize[cAlgNb] += cSize;
@ -521,7 +544,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
{ {
chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1); if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
} }
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
@ -532,16 +555,18 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
int (*decompressionFunction)(const char*, char*, int, int); int (*decompressionFunction)(const char*, char*, int, int);
double bestTime = 100000000.; double bestTime = 100000000.;
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue; if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb+1)) continue;
switch(dAlgNb) switch(dAlgNb)
{ {
case 0: decompressionFunction = local_LZ4_decompress_fast; break; case 0: decompressionFunction = local_LZ4_decompress_fast; break;
case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break; case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break;
case 2: decompressionFunction = LZ4_decompress_safe; break; case 2: decompressionFunction = local_LZ4_decompress_fast_usingDict; break;
case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break; case 3: decompressionFunction = LZ4_decompress_safe; break;
case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break; case 4: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; case 5: decompressionFunction = local_LZ4_decompress_safe_usingDict; break;
case 6: decompressionFunction = local_LZ4_decompress_safe_partial; break;
default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1;
} }
for (loopNb = 1; loopNb <= nbIterations; loopNb++) for (loopNb = 1; loopNb <= nbIterations; loopNb++)
@ -550,7 +575,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
int milliTime; int milliTime;
U32 crcDecoded; U32 crcDecoded;
PROGRESS("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize); PROGRESS("%1i-%-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize);
nb_loops = 0; nb_loops = 0;
milliTime = BMK_GetMilliStart(); milliTime = BMK_GetMilliStart();
@ -570,19 +595,18 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
averageTime = (double)milliTime / nb_loops; averageTime = (double)milliTime / nb_loops;
if (averageTime < bestTime) bestTime = averageTime; if (averageTime < bestTime) bestTime = averageTime;
PROGRESS("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); PROGRESS("%1i-%-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
// CRC Checking // CRC Checking
crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
} }
DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); DISPLAY("%-31.31s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
totalDTime[dAlgNb] += bestTime; totalDTime[dAlgNb] += bestTime;
} }
totals += benchedSize;
} }
free(orig_buff); free(orig_buff);
@ -590,25 +614,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
free(chunkP); free(chunkP);
} }
if (nbFiles > 1)
{
int AlgNb;
DISPLAY(" ** TOTAL ** : \n");
for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
{
char* cName = compressionNames[AlgNb];
if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
}
for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
{
char* dName = decompressionNames[AlgNb];
if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
}
}
if (BMK_pause) { printf("press enter...\n"); getchar(); } if (BMK_pause) { printf("press enter...\n"); getchar(); }
return 0; return 0;
@ -629,8 +634,8 @@ int usage(char* exename)
int usage_advanced() int usage_advanced()
{ {
DISPLAY( "\nAdvanced options :\n"); DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR); DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS);
DISPLAY( " -d# : test only decompression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR); DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS);
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
DISPLAY( " -B# : Block size [4-7](default : 7)\n"); DISPLAY( " -B# : Block size [4-7](default : 7)\n");
//DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); //DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
@ -679,15 +684,23 @@ int main(int argc, char** argv)
// Select compression algorithm only // Select compression algorithm only
case 'c': case 'c':
decompressionTest = 0; decompressionTest = 0;
if ((argument[1]>= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR)) while ((argument[1]>= '0') && (argument[1]<= '9'))
compressionAlgo = argument[1] - '0', argument++; {
compressionAlgo *= 10;
compressionAlgo += argument[1] - '0';
argument++;
}
break; break;
// Select decompression algorithm only // Select decompression algorithm only
case 'd': case 'd':
compressionTest = 0; compressionTest = 0;
if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR)) while ((argument[1]>= '0') && (argument[1]<= '9'))
decompressionAlgo = argument[1] - '0', argument++; {
decompressionAlgo *= 10;
decompressionAlgo += argument[1] - '0';
argument++;
}
break; break;
// Display help on usage // Display help on usage

764
programs/fuzzer.c
View File

@ -1,6 +1,6 @@
/* /*
fuzzer.c - Fuzzer test tool for LZ4 fuzzer.c - Fuzzer test tool for LZ4
Copyright (C) Yann Collet - Andrew Mahone 2012-2014 Copyright (C) Yann Collet 2012-2014
GPL v2 License GPL v2 License
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
@ -22,47 +22,78 @@
- LZ4 source repository : http://code.google.com/p/lz4/ - LZ4 source repository : http://code.google.com/p/lz4/
*/ */
//************************************** /**************************************
// Remove Visual warning messages Remove Visual warning messages
//************************************** **************************************/
#define _CRT_SECURE_NO_WARNINGS // fgets #define _CRT_SECURE_NO_WARNINGS // fgets
//************************************** /**************************************
// Includes Includes
//************************************** **************************************/
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> // fgets, sscanf #include <stdio.h> // fgets, sscanf
#include <sys/timeb.h> // timeb #include <sys/timeb.h> // timeb
#include <string.h> // strcmp #include <string.h> // strcmp
#include "lz4.h" #include "lz4.h"
#include "lz4hc.h" #include "lz4hc.h"
#include "xxhash.h"
//************************************** /**************************************
// Constants Basic Types
//************************************** **************************************/
#ifndef LZ4_VERSION #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# define LZ4_VERSION "" # include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif #endif
#define NB_ATTEMPTS (1<<17)
#define LEN ((1<<15)) /**************************************
#define SEQ_POW 2 Constants
#define NUM_SEQ (1 << SEQ_POW) **************************************/
#define SEQ_MSK ((NUM_SEQ) - 1) #ifndef LZ4_VERSION
#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0) # define LZ4_VERSION "rc118"
#define NEW_SEQ(x) ((((x) >> 10) %10) == 0) #endif
#define PAGE_SIZE 4096
#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) #define NB_ATTEMPTS (1<<16)
#define COMPRESSIBLE_NOISE_LENGTH (1 << 21)
#define FUZ_MAX_BLOCK_SIZE (1 << 17)
#define FUZ_MAX_DICT_SIZE (1 << 15)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
#define PRIME1 2654435761U #define PRIME1 2654435761U
#define PRIME2 2246822519U #define PRIME2 2246822519U
#define PRIME3 3266489917U #define PRIME3 3266489917U
//********************************************************* //**************************************
// Functions // Macros
//********************************************************* //**************************************
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/*****************************************
Local Parameters
*****************************************/
static int no_prompt = 0;
static char* programName;
static int displayLevel = 2;
/*********************************************************
Fuzzer functions
*********************************************************/
static int FUZ_GetMilliStart() static int FUZ_GetMilliStart()
{ {
struct timeb tb; struct timeb tb;
@ -82,20 +113,54 @@ static int FUZ_GetMilliSpan( int nTimeStart )
} }
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src) unsigned int FUZ_rand(unsigned int* src)
{ {
*src = ((*src) * PRIME1) + PRIME2; U32 rand32 = *src;
return *src; rand32 *= PRIME1;
rand32 += PRIME2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
return rand32 >> 3;
} }
int test_canary(unsigned char *buf) #define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767)
#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 14) : (FUZ_rand(seed) & 511) + 15)
void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed)
{ {
int i; BYTE* BBuffer = (BYTE*)buffer;
for (i = 0; i < 2048; i++) int pos = 0;
if (buf[i] != buf[i + 2048]) U32 P32 = (U32)(32768 * proba);
return 0;
return 1; // First Byte
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
// Select : Literal (noise) or copy (within 64K)
if (FUZ_RAND15BITS < P32)
{
// Copy (within 64K)
int ref, d;
int length = FUZ_RANDLENGTH + 4;
int offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos;
if (pos + length > bufferSize) length = bufferSize - pos;
ref = pos - offset;
d = pos + length;
while (pos < d) BBuffer[pos++] = BBuffer[ref++];
}
else
{
// Literal (noise)
int d;
int length = FUZ_RANDLENGTH;
if (pos + length > bufferSize) length = bufferSize - pos;
d = pos + length;
while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
}
}
} }
@ -105,7 +170,8 @@ int FUZ_SecurityTest()
char* input; char* input;
int i, r; int i, r;
printf("Overflow test (issue 52)...\n"); // Overflow test, by Ludwig Strigeus
printf("Overflow test (issue 52)...");
input = (char*) malloc (20<<20); input = (char*) malloc (20<<20);
output = (char*) malloc (20<<20); output = (char*) malloc (20<<20);
input[0] = 0x0F; input[0] = 0x0F;
@ -121,27 +187,471 @@ int FUZ_SecurityTest()
return 0; return 0;
} }
#define FUZ_MAX(a,b) (a>b?a:b)
int main(int argc, char** argv) { int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
const int no_prompt = (argc > 1) && (!strcmp(argv[1], "--no-prompt"));
unsigned long long bytes = 0; unsigned long long bytes = 0;
unsigned long long cbytes = 0; unsigned long long cbytes = 0;
unsigned long long hcbytes = 0; unsigned long long hcbytes = 0;
unsigned char buf[LEN]; unsigned long long ccbytes = 0;
unsigned char testOut[LEN+1]; void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN) # define FUZ_max LZ4_COMPRESSBOUND(LEN)
# define FUZ_avail ROUND_PAGE(FUZ_max) unsigned int randState=seed;
const int off_full = FUZ_avail - FUZ_max; int ret, cycleNb;
unsigned char cbuf[FUZ_avail + PAGE_SIZE]; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
int i, j, k, ret, len, lenHC, attemptNb; # define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
char userInput[30] = {0};
# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }
# define FUZ_DISPLAYTEST testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb);
void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
U32 crcOrig, crcCheck;
int displayRefresh;
printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);
// init
memset(&LZ4dict, 0, sizeof(LZ4dict));
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// display refresh rate
switch(displayLevel)
{
case 0: displayRefresh = nbCycles+1; break;
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
case 2: displayRefresh=89; break;
default : displayRefresh=1;
}
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
// synd rand & dict
int dictSize, blockSize, blockStart;
char* dict;
char* block;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
if ((cycleNb%displayRefresh) == 0)
{
printf("\r%7i /%7i - ", cycleNb, nbCycles);
fflush(stdout);
}
// Select block to test
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
/* Compression tests */
// Test compression HC
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
compressedSize = ret;
/* Decompression tests */
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with more than enough output size => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
decodedBuffer[blockSize+1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
// Test decoding with output size being 10 bytes too short => must fail
FUZ_DISPLAYTEST;
if (blockSize>10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
}
// Test decoding with input size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
// Test decoding with input size being one byte too large => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
// Test partial decoding with target output size being max/2 => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
/* Test Compression with limited output size */
// Test compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
// Test compression with output size being exactly what's necessary and external state (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
// Test compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
/* Dictionary tests */
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
LZ4_free (LZ4continue);
// Decompress with dictionary as prefix
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
// Compress using External dictionary
FUZ_DISPLAYTEST;
dict -= 9; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
if (blockSize > 10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
}
// Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 0;
_output_error:
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 1;
}
int FUZ_usage()
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -v : verbose\n");
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, char** argv) {
U32 timestamp = FUZ_GetMilliStart();
U32 seed=0;
int seedset=0;
int argNb;
int nbTests = NB_ATTEMPTS;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
// Check command line
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
// Decode command (note : aggregated commands are allowed)
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }
while (argument[1]!=0)
{
argument++;
switch(*argument)
{
case 'h':
return FUZ_usage();
case 'v':
argument++;
displayLevel=4;
break;
case 'i':
argument++;
nbTests=0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
nbTests += *argument - '0';
argument++;
}
break;
case 's':
argument++;
seed=0; seedset=1;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9'))
{
testNb *= 10;
testNb += *argument - '0';
argument++;
}
break;
case 'p':
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
default: ;
}
}
}
}
// Get Seed
printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
if (!seedset)
{
char userInput[50] = {0};
printf("Select an Initialisation number (default : random) : "); printf("Select an Initialisation number (default : random) : ");
fflush(stdout); fflush(stdout);
if ( no_prompt || fgets(userInput, sizeof userInput, stdin) ) if ( no_prompt || fgets(userInput, sizeof userInput, stdin) )
@ -149,165 +659,13 @@ int main(int argc, char** argv) {
if ( sscanf(userInput, "%u", &seed) == 1 ) {} if ( sscanf(userInput, "%u", &seed) == 1 ) {}
else seed = FUZ_GetMilliSpan(timestamp); else seed = FUZ_GetMilliSpan(timestamp);
} }
}
printf("Seed = %u\n", seed); printf("Seed = %u\n", seed);
randState = seed; if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
//FUZ_SecurityTest(); FUZ_SecurityTest();
for (i = 0; i < 2048; i++) if (nbTests<=0) nbTests=1;
cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16;
for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++) return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
{
int testNb = 0;
// note : promptThrottle is throtting stdout to prevent
// Travis-CI's output limit (10MB) and false hangup detection.
const int promptThrottle = (attemptNb % (NB_ATTEMPTS / 100) == 0);
if (!no_prompt || attemptNb == 0 || promptThrottle)
{
printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);
if (no_prompt)
{
fflush(stdout);
}
}
for (j = 0; j < NUM_SEQ; j++) {
seeds[j] = FUZ_rand(&randState) << 8;
seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535;
}
for (j = 0; j < LEN; j++) {
k = FUZ_rand(&randState);
if (j == 0 || NEW_SEQ(k))
cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK];
if (MOD_SEQ(k)) {
k = (FUZ_rand(&randState) >> 16) & SEQ_MSK;
seeds[k] = FUZ_rand(&randState) << 8;
seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535;
}
buf[j] = FUZ_rand(&cur_seq) >> 16;
}
// Test compression HC
FUZ_DISPLAYTEST; // 1
ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
lenHC = ret;
// Test compression HC using external state
FUZ_DISPLAYTEST; // 1
ret = LZ4_compressHC_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[off_full], LEN);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
FUZ_DISPLAYTEST; // 2
ret = LZ4_compress_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
FUZ_DISPLAYTEST; // 2
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
len = ret;
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST; // 3
ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
// Test decoding with one byte missing => must fail
FUZ_DISPLAYTEST; // 4
ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
// Test decoding with one byte too much => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with enough output size => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
// Test decoding with input size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short");
// Test decoding with input size being one byte too large => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
//if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }
// Test partial decoding with target output size being max/2 => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
// Test compression with output size being exactly what's necessary and external state (should work)
FUZ_DISPLAYTEST; // 2
ret = LZ4_compress_limitedOutput_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN, len);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
// Test compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);
FUZ_CHECKTEST(ret, "compression overran output buffer");
FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1);
FUZ_CHECKTEST(ret, "HC compression overran output buffer");
bytes += LEN;
cbytes += len;
hcbytes += lenHC;
FUZ_rand(&randState);
}
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
if(!no_prompt) getchar();
return 0;
_output_error:
if(!no_prompt) getchar();
return 1;
} }

1
programs/lz4.1
View File

@ -64,6 +64,7 @@ following options
.TP .TP
.B \-B# .B \-B#
block size [4-7](default : 7) block size [4-7](default : 7)
B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB
.TP .TP
.B \-BD .B \-BD
block dependency (improve compression ratio) block dependency (improve compression ratio)

16
programs/lz4cat.1
View File

@ -7,24 +7,22 @@
.hy 0 .hy 0
.nr HY 0 .nr HY 0
.TH lz4cat "1" "2014-04-15" "lz4cat" "User Commands" .TH lz4cat "1" "2014-06-20" "lz4cat" "User Commands"
.SH NAME .SH NAME
\fBlz4cat\fR - Extremely fast compression algorithm \fBlz4cat\fR - Utility based on LZ4
.SH SYNOPSIS .SH SYNOPSIS
.TP 5 .TP 5
\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> \fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
.SH DESCRIPTION .SH DESCRIPTION
.PP .PP
\fBlz4\fR is an extremely fast lossless compression algorithm. \fBlz4cat\fR is an utility based on \fBlz4\fR, an extremely fast lossless compression algorithm.
\fBlz4cat\fR is an utility based on \fBlz4\fR. \fBlz4cat\fR decompress input file or stream, redirecting its output to the console.
It is equivalent to \fBlz4 -cd\fR,
\fBlz4cat\fR is equivalent to \fBlz4 -cd\fR, Available options are the same as \fBlz4\fR ones (man lz4).
which forces decompression and redirect its output to the console.
All other options are the same as \fBlz4\fR ones (man lz4).
.SH BUGS .SH BUGS

32
programs/lz4cli.c
View File

@ -109,7 +109,7 @@
//**************************** //****************************
#define COMPRESSOR_NAME "LZ4 Compression CLI" #define COMPRESSOR_NAME "LZ4 Compression CLI"
#ifndef LZ4_VERSION #ifndef LZ4_VERSION
# define LZ4_VERSION "v1.1.5" # define LZ4_VERSION "v1.2.0"
#endif #endif
#define AUTHOR "Yann Collet" #define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
@ -344,6 +344,19 @@ int main(int argc, char** argv)
if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode) if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)
#endif // DISABLE_LZ4C_LEGACY_OPTIONS #endif // DISABLE_LZ4C_LEGACY_OPTIONS
if ((*argument>='0') && (*argument<='9'))
{
cLevel = 0;
while ((*argument >= '0') && (*argument <= '9'))
{
cLevel *= 10;
cLevel += *argument - '0';
argument++;
}
argument--;
continue;
}
switch(argument[0]) switch(argument[0])
{ {
// Display help // Display help
@ -354,20 +367,6 @@ int main(int argc, char** argv)
// Compression (default) // Compression (default)
case 'z': forceCompress = 1; break; case 'z': forceCompress = 1; break;
// Compression level
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'A': /* non documented (hidden) */
cLevel=*argument -'0'; break;
// Use Legacy format (for Linux kernel compression) // Use Legacy format (for Linux kernel compression)
case 'l': legacy_format=1; break; case 'l': legacy_format=1; break;
@ -405,9 +404,8 @@ int main(int argc, char** argv)
case '7': case '7':
{ {
int B = argument[1] - '0'; int B = argument[1] - '0';
int S = 1 << (8 + 2*B);
BMK_SetBlocksize(S);
blockSize = LZ4IO_setBlockSizeID(B); blockSize = LZ4IO_setBlockSizeID(B);
BMK_SetBlocksize(blockSize);
argument++; argument++;
break; break;
} }

132
programs/lz4io.c
View File

@ -121,7 +121,7 @@
#define CACHELINE 64 #define CACHELINE 64
#define LEGACY_BLOCKSIZE (8 MB) #define LEGACY_BLOCKSIZE (8 MB)
#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) #define MIN_STREAM_BUFSIZE (192 KB)
#define LZ4S_BLOCKSIZEID_DEFAULT 7 #define LZ4S_BLOCKSIZEID_DEFAULT 7
#define LZ4S_CHECKSUM_SEED 0 #define LZ4S_CHECKSUM_SEED 0
#define LZ4S_EOS 0 #define LZ4S_EOS 0
@ -365,17 +365,37 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i
} }
static void* LZ4IO_LZ4_createStream (const char* inputBuffer)
{
(void)inputBuffer;
return LZ4_createStream();
}
static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
(void)compressionLevel;
return LZ4_compress_limitedOutput_continue(ctx, source, dest, inputSize, maxOutputSize);
}
static int LZ4IO_LZ4_slideInputBufferHC (void* ctx, char* buffer, int size)
{
(void)size; (void)buffer;
LZ4_slideInputBufferHC (ctx);
return 1;
}
static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
{ {
void* (*initFunction) (const char*); void* (*initFunction) (const char*);
int (*compressionFunction)(void*, const char*, char*, int, int); int (*compressionFunction)(void*, const char*, char*, int, int, int);
char* (*translateFunction) (void*); int (*nextBlockFunction) (void*, char*, int);
int (*freeFunction) (void*); int (*freeFunction) (void*);
void* ctx; void* ctx;
unsigned long long filesize = 0; unsigned long long filesize = 0;
unsigned long long compressedfilesize = 0; unsigned long long compressedfilesize = 0;
unsigned int checkbits; unsigned int checkbits;
char* in_buff, *in_start, *in_end; char* in_buff, *in_blockStart;
char* out_buff; char* out_buff;
FILE* finput; FILE* finput;
FILE* foutput; FILE* foutput;
@ -384,34 +404,35 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
size_t sizeCheck, header_size; size_t sizeCheck, header_size;
void* streamChecksumState=NULL; void* streamChecksumState=NULL;
// Init // Init
start = clock(); start = clock();
if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
if (compressionlevel>=3)
if (compressionlevel<3)
{ {
initFunction = LZ4_createHC; initFunction = LZ4IO_LZ4_createStream;
compressionFunction = LZ4_compressHC_limitedOutput_continue; compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue;
translateFunction = LZ4_slideInputBufferHC; nextBlockFunction = LZ4_saveDict;
freeFunction = LZ4_freeHC; freeFunction = LZ4_free;
} }
else else
{ {
initFunction = LZ4_create; initFunction = LZ4_createHC;
compressionFunction = LZ4_compress_limitedOutput_continue; compressionFunction = LZ4_compressHC2_limitedOutput_continue;
translateFunction = LZ4_slideInputBuffer; nextBlockFunction = LZ4IO_LZ4_slideInputBufferHC;
freeFunction = LZ4_free; freeFunction = LZ4_free;
} }
get_fileHandle(input_filename, output_filename, &finput, &foutput); get_fileHandle(input_filename, output_filename, &finput, &foutput);
blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
// Allocate Memory // Allocate Memory
inputBufferSize = blockSize + 64 KB; inputBufferSize = 64 KB + blockSize;
if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
in_buff = (char*)malloc(inputBufferSize); in_buff = (char*)malloc(inputBufferSize);
out_buff = (char*)malloc(blockSize+CACHELINE); out_buff = (char*)malloc(blockSize+CACHELINE);
if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
in_start = in_buff; in_end = in_buff + inputBufferSize; in_blockStart = in_buff + 64 KB;
if (compressionlevel>=3) in_blockStart = in_buff;
if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
ctx = initFunction(in_buff); ctx = initFunction(in_buff);
@ -435,19 +456,19 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
{ {
unsigned int outSize; unsigned int outSize;
unsigned int inSize; unsigned int inSize;
// Read Block // Read Block
if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx); inSize = (unsigned int) fread(in_blockStart, (size_t)1, (size_t)blockSize, finput);
inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
if( inSize==0 ) break; // No more input : end of compression if( inSize==0 ) break; // No more input : end of compression
filesize += inSize; filesize += inSize;
DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); if (streamChecksum) XXH32_update(streamChecksumState, in_blockStart, inSize);
// Compress Block // Compress Block
outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel);
if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
if (blockChecksum) compressedfilesize+=4; if (blockChecksum) compressedfilesize+=4;
DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100);
// Write Block // Write Block
if (outSize > 0) if (outSize > 0)
@ -462,24 +483,28 @@ static int compress_file_blockDependency(char* input_filename, char* output_file
sizeToWrite = 4 + outSize + (4*blockChecksum); sizeToWrite = 4 + outSize + (4*blockChecksum);
sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
} }
else // Copy Original else // Copy Original
{ {
* (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
sizeCheck = fwrite(out_buff, 1, 4, foutput); sizeCheck = fwrite(out_buff, 1, 4, foutput);
if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
sizeCheck = fwrite(in_start, 1, inSize, foutput); sizeCheck = fwrite(in_blockStart, 1, inSize, foutput);
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
if (blockChecksum) if (blockChecksum)
{ {
unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED); unsigned int checksum = XXH32(in_blockStart, inSize, LZ4S_CHECKSUM_SEED);
* (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
sizeCheck = fwrite(out_buff, 1, 4, foutput); sizeCheck = fwrite(out_buff, 1, 4, foutput);
if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
} }
} }
in_start += inSize; {
size_t sizeToMove = 64 KB;
if (inSize < 64 KB) sizeToMove = inSize;
nextBlockFunction(ctx, in_blockStart - sizeToMove, (int)sizeToMove);
if (compressionlevel>=3) in_blockStart = in_buff + 64 KB;
}
} }
// End of Stream mark // End of Stream mark
@ -587,7 +612,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel); outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel);
if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
if (blockChecksum) compressedfilesize+=4; if (blockChecksum) compressedfilesize+=4;
DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100);
// Write Block // Write Block
if (outSize > 0) if (outSize > 0)
@ -724,8 +749,11 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
size_t sizeCheck; size_t sizeCheck;
int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
void* streamChecksumState=NULL; void* streamChecksumState=NULL;
int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; int (*decompressionFunction)(void* ctx, const char* src, char* dst, int cSize, int maxOSize) = LZ4_decompress_safe_continue;
unsigned int prefix64k = 0; LZ4_streamDecode_t ctx;
// init
memset(&ctx, 0, sizeof(ctx));
// Decode stream descriptor // Decode stream descriptor
nbReadBytes = fread(descriptor, 1, 3, finput); nbReadBytes = fread(descriptor, 1, 3, finput);
@ -761,23 +789,17 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
} }
if (!blockIndependenceFlag)
{
decompressionFunction = LZ4_decompress_safe_withPrefix64k;
prefix64k = 64 KB;
}
// Allocate Memory // Allocate Memory
{ {
unsigned int outbuffSize = prefix64k+maxBlockSize; size_t outBuffSize = maxBlockSize + 64 KB;
if (outBuffSize < MIN_STREAM_BUFSIZE) outBuffSize = MIN_STREAM_BUFSIZE;
in_buff = (char*)malloc(maxBlockSize); in_buff = (char*)malloc(maxBlockSize);
if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; out_buff = (char*)malloc(outBuffSize);
out_buff = (char*)malloc(outbuffSize); out_start = out_buff;
out_end = out_buff + outbuffSize; out_end = out_start + outBuffSize;
out_start = out_buff + prefix64k;
if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
}
if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
}
// Main Loop // Main Loop
while (1) while (1)
@ -817,23 +839,17 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
if (!blockIndependenceFlag) if (!blockIndependenceFlag)
{ {
if (blockSize >= prefix64k) // handle dictionary for streaming
{ memcpy(in_buff + blockSize - 64 KB, out_buff, 64 KB);
memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks LZ4_setDictDecode(&ctx, out_buff, 64 KB);
out_start = out_buff + prefix64k; out_start = out_buff + 64 KB;
continue;
}
else
{
memcpy(out_start, in_buff, blockSize);
decodedBytes = blockSize;
}
} }
} }
else else
{ {
// Decode Block // Decode Block
decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); if (out_start + maxBlockSize > out_end) out_start = out_buff;
decodedBytes = decompressionFunction(&ctx, in_buff, out_start, blockSize, maxBlockSize);
if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
filesize += decodedBytes; filesize += decodedBytes;
if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
@ -841,17 +857,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
// Write Block // Write Block
sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
out_start += decodedBytes;
} }
if (!blockIndependenceFlag)
{
out_start += decodedBytes;
if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
{
memcpy(out_buff, out_start - prefix64k, prefix64k);
out_start = out_buff + prefix64k;
}
}
} }
// Stream Checksum // Stream Checksum
@ -862,7 +870,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
sizeCheck = fread(&readChecksum, 1, 4, finput); sizeCheck = fread(&readChecksum, 1, 4, finput);
if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); if (checksum != readChecksum) EXM_THROW(79, "Error : invalid stream checksum detected");
} }
// Free // Free